CFD/hypre
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
1c200372e9
hypre/struct_mv/communication.c
falgout 0432470ab0 - Added new StructCoarsen routine to improve setup phase scalability. 
- Modified Neighbor and Grid classes, and in particular, the routines
  BoxNeighborsAssemble and StructGridAssemble.
- Grid boxes now have associated unique IDs to aid in implementation.
2000-03-21 16:52:39 +00:00

1570 lines
49 KiB
C
Raw Blame History

/*BHEADER**********************************************************************
* (c) 1997 The Regents of the University of California
*
* See the file COPYRIGHT_and_DISCLAIMER for a complete copyright
* notice, contact person, and disclaimer.
*
* $Revision$
*********************************************************************EHEADER*/
#include "headers.h"
/*==========================================================================*/
/*==========================================================================*/
/** Create a communication package. A grid-based description of a
communication exchange is passed in. This description is then
compiled into an intermediate processor-based description of the
communication. It may further be compiled into a form based on the
message-passing layer in the routine hypre\_CommitCommPkg. This
proceeds as follows based on several compiler flags:
\begin{itemize}
\item If HYPRE\_COMM\_SIMPLE is defined, the intermediate
processor-based description is not compiled into a form based on
the message-passing layer. This intermediate description is used
directly to pack and unpack buffers during the communications.
No MPI derived datatypes are used.
\item Else if HYPRE\_COMM\_VOLATILE is defined, the communication
package is not committed, and the intermediate processor-based
description is retained. The package is committed at communication
time.
\item Else the communication package is committed, and the intermediate
processor-based description is freed up.
\end{itemize}
{\bf Note:}
The input boxes and processes are destroyed.
{\bf Input files:}
headers.h
@return Communication package.
@param send_boxes [IN]
description of the grid data to be sent to other processors.
@param recv_boxes [IN]
description of the grid data to be received from other processors.
@param send_data_space [IN]
description of the stored grid data associated with the sends.
@param recv_data_space [IN]
description of the stored grid data associated with the receives.
@param send_processes [IN]
processors that data is to be sent to.
@param recv_processes [IN]
processors that data is to be received from.
@param num_values [IN]
number of data values to be sent for each grid index.
@param comm [IN]
communicator.
@see hypre_CommPkgCreateInfo, hypre_CommPkgCommit, hypre_CommPkgDestroy */
/*--------------------------------------------------------------------------*/
hypre_CommPkg *
hypre_CommPkgCreate( hypre_BoxArrayArray *send_boxes,
hypre_BoxArrayArray *recv_boxes,
hypre_Index send_stride,
hypre_Index recv_stride,
hypre_BoxArray *send_data_space,
hypre_BoxArray *recv_data_space,
int **send_processes,
int **recv_processes,
int num_values,
MPI_Comm comm,
hypre_Index periodic )
{
hypre_CommPkg *comm_pkg;
int num_sends;
int *send_procs;
hypre_CommType **send_types;
int num_recvs;
int *recv_procs;
hypre_CommType **recv_types;
hypre_CommType *copy_from_type;
hypre_CommType *copy_to_type;
int i;
/*------------------------------------------------------
* Put arguments into hypre_CommPkg
*------------------------------------------------------*/
comm_pkg = hypre_CTAlloc(hypre_CommPkg, 1);
hypre_CommPkgNumValues(comm_pkg) = num_values;
hypre_CommPkgComm(comm_pkg) = comm;
/*------------------------------------------------------
* Set up communication information
*------------------------------------------------------*/
hypre_CommPkgCreateInfo(send_boxes, send_stride,
send_data_space, send_processes,
num_values, comm, periodic,
&num_sends, &send_procs,
&send_types, &copy_from_type);
hypre_CommPkgNumSends(comm_pkg) = num_sends;
hypre_CommPkgSendProcs(comm_pkg) = send_procs;
hypre_CommPkgSendTypes(comm_pkg) = send_types;
hypre_CommPkgCopyFromType(comm_pkg) = copy_from_type;
hypre_CommPkgCreateInfo(recv_boxes, recv_stride,
recv_data_space, recv_processes,
num_values, comm, periodic,
&num_recvs, &recv_procs,
&recv_types, &copy_to_type);
hypre_CommPkgNumRecvs(comm_pkg) = num_recvs;
hypre_CommPkgRecvProcs(comm_pkg) = recv_procs;
hypre_CommPkgRecvTypes(comm_pkg) = recv_types;
hypre_CommPkgCopyToType(comm_pkg) = copy_to_type;
/*------------------------------------------------------
* Destroy the input boxes and processes
*------------------------------------------------------*/
hypre_ForBoxArrayI(i, send_boxes)
hypre_TFree(send_processes[i]);
hypre_BoxArrayArrayDestroy(send_boxes);
hypre_TFree(send_processes);
hypre_ForBoxArrayI(i, recv_boxes)
hypre_TFree(recv_processes[i]);
hypre_BoxArrayArrayDestroy(recv_boxes);
hypre_TFree(recv_processes);
#if defined(HYPRE_COMM_SIMPLE) || defined(HYPRE_COMM_VOLATILE)
#else
hypre_CommPkgCommit(comm_pkg);
/* free up comm types */
for (i = 0; i < hypre_CommPkgNumSends(comm_pkg); i++)
hypre_CommTypeDestroy(hypre_CommPkgSendType(comm_pkg, i));
hypre_TFree(hypre_CommPkgSendTypes(comm_pkg));
for (i = 0; i < hypre_CommPkgNumRecvs(comm_pkg); i++)
hypre_CommTypeDestroy(hypre_CommPkgRecvType(comm_pkg, i));
hypre_TFree(hypre_CommPkgRecvTypes(comm_pkg));
#endif
return comm_pkg;
}
/*==========================================================================*/
/*==========================================================================*/
/** Destroy a communication package.
{\bf Input files:}
headers.h
@return Error code.
@param comm_pkg [IN/OUT]
communication package.
@see hypre_CommPkgCreate */
/*--------------------------------------------------------------------------*/
int
hypre_CommPkgDestroy( hypre_CommPkg *comm_pkg )
{
int ierr = 0;
#if defined(HYPRE_COMM_SIMPLE) || defined(HYPRE_COMM_VOLATILE)
int i;
#else
#endif
if (comm_pkg)
{
#if defined(HYPRE_COMM_SIMPLE) || defined(HYPRE_COMM_VOLATILE)
/* free up comm types */
for (i = 0; i < hypre_CommPkgNumSends(comm_pkg); i++)
hypre_CommTypeDestroy(hypre_CommPkgSendType(comm_pkg, i));
hypre_TFree(hypre_CommPkgSendTypes(comm_pkg));
for (i = 0; i < hypre_CommPkgNumRecvs(comm_pkg); i++)
hypre_CommTypeDestroy(hypre_CommPkgRecvType(comm_pkg, i));
hypre_TFree(hypre_CommPkgRecvTypes(comm_pkg));
#else
hypre_CommPkgUnCommit(comm_pkg);
#endif
hypre_TFree(hypre_CommPkgSendProcs(comm_pkg));
hypre_TFree(hypre_CommPkgRecvProcs(comm_pkg));
hypre_CommTypeDestroy(hypre_CommPkgCopyFromType(comm_pkg));
hypre_CommTypeDestroy(hypre_CommPkgCopyToType(comm_pkg));
hypre_TFree(comm_pkg);
}
return ierr;
}
/*==========================================================================*/
/*==========================================================================*/
/** Initialize a non-blocking communication exchange.
\begin{itemize}
\item If HYPRE\_COMM\_SIMPLE is defined, the communication buffers are
created, the send buffer is manually packed, and the communication
requests are posted. No MPI derived datatypes are used.
\item Else if HYPRE\_COMM\_VOLATILE is defined, the communication
package is committed, the communication requests are posted, then
the communication package is un-committed.
\item Else the communication requests are posted.
\end{itemize}
{\bf Input files:}
headers.h
@return Error code.
@param comm_pkg [IN]
communication package.
@param send_data [IN]
reference pointer for the send data.
@param recv_data [IN]
reference pointer for the recv data.
@param comm_handle [OUT]
communication handle.
@see hypre_FinalizeCommunication, hypre_CommPkgCreate */
/*--------------------------------------------------------------------------*/
#if defined(HYPRE_COMM_SIMPLE)
int
hypre_InitializeCommunication( hypre_CommPkg *comm_pkg,
double *send_data,
double *recv_data,
hypre_CommHandle **comm_handle_ptr )
{
int ierr = 0;
hypre_CommHandle *comm_handle;
int num_sends = hypre_CommPkgNumSends(comm_pkg);
int num_recvs = hypre_CommPkgNumRecvs(comm_pkg);
MPI_Comm comm = hypre_CommPkgComm(comm_pkg);
int num_requests;
MPI_Request *requests;
MPI_Status *status;
double **send_buffers;
double **recv_buffers;
int *send_sizes;
int *recv_sizes;
hypre_CommType *send_type;
hypre_CommTypeEntry *send_entry;
hypre_CommType *recv_type;
hypre_CommTypeEntry *recv_entry;
int *length_array;
int *stride_array;
double *iptr, *jptr, *kptr, *lptr, *bptr;
int i, j, k, ii, jj, kk, ll;
int entry_size, total_size;
/*--------------------------------------------------------------------
* allocate requests and status
*--------------------------------------------------------------------*/
num_requests = num_sends + num_recvs;
requests = hypre_CTAlloc(MPI_Request, num_requests);
status = hypre_CTAlloc(MPI_Status, num_requests);
/*--------------------------------------------------------------------
* allocate buffers
*--------------------------------------------------------------------*/
/* allocate send buffers */
send_buffers = hypre_TAlloc(double *, num_sends);
send_sizes = hypre_TAlloc(int, num_sends);
total_size = 0;
for (i = 0; i < num_sends; i++)
{
send_type = hypre_CommPkgSendType(comm_pkg, i);
send_sizes[i] = 0;
for (j = 0; j < hypre_CommTypeNumEntries(send_type); j++)
{
send_entry = hypre_CommTypeCommEntry(send_type, j);
length_array = hypre_CommTypeEntryLengthArray(send_entry);
entry_size = 1;
for (k = 0; k < 4; k++)
{
entry_size *= length_array[k];
}
send_sizes[i] += entry_size;
}
total_size += send_sizes[i];
}
if (num_sends > 0)
{
send_buffers[0] = hypre_SharedTAlloc(double, total_size);
for (i = 1; i < num_sends; i++)
{
send_buffers[i] = send_buffers[i-1] + send_sizes[i-1];
}
}
/* allocate recv buffers */
recv_buffers = hypre_TAlloc(double *, num_recvs);
recv_sizes = hypre_TAlloc(int, num_recvs);
total_size = 0;
for (i = 0; i < num_recvs; i++)
{
recv_type = hypre_CommPkgRecvType(comm_pkg, i);
recv_sizes[i] = 0;
for (j = 0; j < hypre_CommTypeNumEntries(recv_type); j++)
{
recv_entry = hypre_CommTypeCommEntry(recv_type, j);
length_array = hypre_CommTypeEntryLengthArray(recv_entry);
entry_size = 1;
for (k = 0; k < 4; k++)
{
entry_size *= length_array[k];
}
recv_sizes[i] += entry_size;
}
total_size += recv_sizes[i];
}
if (num_recvs > 0)
{
recv_buffers[0] = hypre_SharedTAlloc(double, total_size);
for (i = 1; i < num_recvs; i++)
{
recv_buffers[i] = recv_buffers[i-1] + recv_sizes[i-1];
}
}
/*--------------------------------------------------------------------
* pack send buffers
*--------------------------------------------------------------------*/
for (i = 0; i < num_sends; i++)
{
send_type = hypre_CommPkgSendType(comm_pkg, i);
bptr = (double *) send_buffers[i];
for (j = 0; j < hypre_CommTypeNumEntries(send_type); j++)
{
send_entry = hypre_CommTypeCommEntry(send_type, j);
length_array = hypre_CommTypeEntryLengthArray(send_entry);
stride_array = hypre_CommTypeEntryStrideArray(send_entry);
lptr = send_data + hypre_CommTypeEntryOffset(send_entry);
for (ll = 0; ll < length_array[3]; ll++)
{
kptr = lptr;
for (kk = 0; kk < length_array[2]; kk++)
{
jptr = kptr;
for (jj = 0; jj < length_array[1]; jj++)
{
if (stride_array[0] == 1)
{
memcpy(bptr, jptr, length_array[0]*sizeof(double));
}
else
{
iptr = jptr;
for (ii = 0; ii < length_array[0]; ii++)
{
bptr[ii] = *iptr;
iptr += stride_array[0];
}
}
bptr += length_array[0];
jptr += stride_array[1];
}
kptr += stride_array[2];
}
lptr += stride_array[3];
}
}
}
/*--------------------------------------------------------------------
* post receives and initiate sends
*--------------------------------------------------------------------*/
j = 0;
for(i = 0; i < num_recvs; i++)
{
MPI_Irecv(recv_buffers[i], recv_sizes[i], MPI_DOUBLE,
hypre_CommPkgRecvProc(comm_pkg, i),
0, comm, &requests[j++]);
}
for(i = 0; i < num_sends; i++)
{
MPI_Isend(send_buffers[i], send_sizes[i], MPI_DOUBLE,
hypre_CommPkgSendProc(comm_pkg, i),
0, comm, &requests[j++]);
}
hypre_ExchangeLocalData(comm_pkg, send_data, recv_data);
/*--------------------------------------------------------------------
* set up comm_handle and return
*--------------------------------------------------------------------*/
comm_handle = hypre_TAlloc(hypre_CommHandle, 1);
hypre_CommHandleCommPkg(comm_handle) = comm_pkg;
hypre_CommHandleSendData(comm_handle) = send_data;
hypre_CommHandleRecvData(comm_handle) = recv_data;
hypre_CommHandleNumRequests(comm_handle) = num_requests;
hypre_CommHandleRequests(comm_handle) = requests;
hypre_CommHandleStatus(comm_handle) = status;
hypre_CommHandleSendBuffers(comm_handle) = send_buffers;
hypre_CommHandleRecvBuffers(comm_handle) = recv_buffers;
hypre_CommHandleSendSizes(comm_handle) = send_sizes;
hypre_CommHandleRecvSizes(comm_handle) = recv_sizes;
*comm_handle_ptr = comm_handle;
return ierr;
}
/*--------------------------------------------------------------------------*/
#else
int
hypre_InitializeCommunication( hypre_CommPkg *comm_pkg,
double *send_data,
double *recv_data,
hypre_CommHandle **comm_handle_ptr )
{
int ierr = 0;
hypre_CommHandle *comm_handle;
int num_sends = hypre_CommPkgNumSends(comm_pkg);
int num_recvs = hypre_CommPkgNumRecvs(comm_pkg);
MPI_Comm comm = hypre_CommPkgComm(comm_pkg);
int num_requests;
MPI_Request *requests;
MPI_Status *status;
int i, j;
/*--------------------------------------------------------------------
* allocate requests and status
*--------------------------------------------------------------------*/
num_requests = num_sends + num_recvs;
requests = hypre_CTAlloc(MPI_Request, num_requests);
status = hypre_CTAlloc(MPI_Status, num_requests);
/*--------------------------------------------------------------------
* post receives and initiate sends
*--------------------------------------------------------------------*/
#if defined(HYPRE_COMM_VOLATILE)
/* commit the communication package */
hypre_CommPkgCommit(comm_pkg);
#else
#endif
j = 0;
for(i = 0; i < num_recvs; i++)
{
MPI_Irecv((void *)recv_data, 1,
hypre_CommPkgRecvMPIType(comm_pkg, i),
hypre_CommPkgRecvProc(comm_pkg, i),
0, comm, &requests[j++]);
}
for(i = 0; i < num_sends; i++)
{
MPI_Isend((void *)send_data, 1,
hypre_CommPkgSendMPIType(comm_pkg, i),
hypre_CommPkgSendProc(comm_pkg, i),
0, comm, &requests[j++]);
}
#if defined(HYPRE_COMM_VOLATILE)
/* un-commit the communication package */
hypre_CommPkgUnCommit(comm_pkg);
#else
#endif
hypre_ExchangeLocalData(comm_pkg, send_data, recv_data);
/*--------------------------------------------------------------------
* set up comm_handle and return
*--------------------------------------------------------------------*/
comm_handle = hypre_TAlloc(hypre_CommHandle, 1);
hypre_CommHandleCommPkg(comm_handle) = comm_pkg;
hypre_CommHandleSendData(comm_handle) = send_data;
hypre_CommHandleRecvData(comm_handle) = recv_data;
hypre_CommHandleNumRequests(comm_handle) = num_requests;
hypre_CommHandleRequests(comm_handle) = requests;
hypre_CommHandleStatus(comm_handle) = status;
*comm_handle_ptr = comm_handle;
return ierr;
}
#endif
/*==========================================================================*/
/*==========================================================================*/
/** Finalize a communication exchange. This routine blocks until all
of the communication requests are completed.
\begin{itemize}
\item If HYPRE\_COMM\_SIMPLE is defined, the communication requests
are completed, and the receive buffer is manually unpacked.
\item Else if HYPRE\_COMM\_VOLATILE is defined, the communication requests
are completed and the communication package is un-committed.
\item Else the communication requests are completed.
\end{itemize}
{\bf Input files:}
headers.h
@return Error code.
@param comm_handle [IN/OUT]
communication handle.
@see hypre_InitializeCommunication, hypre_CommPkgCreate */
/*--------------------------------------------------------------------------*/
#if defined(HYPRE_COMM_SIMPLE)
int
hypre_FinalizeCommunication( hypre_CommHandle *comm_handle )
{
int ierr = 0;
hypre_CommPkg *comm_pkg = hypre_CommHandleCommPkg(comm_handle);
double **send_buffers = hypre_CommHandleSendBuffers(comm_handle);
double **recv_buffers = hypre_CommHandleRecvBuffers(comm_handle);
int *send_sizes = hypre_CommHandleSendSizes(comm_handle);
int *recv_sizes = hypre_CommHandleRecvSizes(comm_handle);
int num_sends = hypre_CommPkgNumSends(comm_pkg);
int num_recvs = hypre_CommPkgNumRecvs(comm_pkg);
hypre_CommType *recv_type;
hypre_CommTypeEntry *recv_entry;
int *length_array;
int *stride_array;
double *iptr, *jptr, *kptr, *lptr, *bptr;
int i, j, ii, jj, kk, ll;
/*--------------------------------------------------------------------
* finish communications
*--------------------------------------------------------------------*/
if (hypre_CommHandleNumRequests(comm_handle))
{
MPI_Waitall(hypre_CommHandleNumRequests(comm_handle),
hypre_CommHandleRequests(comm_handle),
hypre_CommHandleStatus(comm_handle));
}
/*--------------------------------------------------------------------
* unpack recv buffers
*--------------------------------------------------------------------*/
for (i = 0; i < num_recvs; i++)
{
recv_type = hypre_CommPkgRecvType(comm_pkg, i);
bptr = (double *) recv_buffers[i];
for (j = 0; j < hypre_CommTypeNumEntries(recv_type); j++)
{
recv_entry = hypre_CommTypeCommEntry(recv_type, j);
length_array = hypre_CommTypeEntryLengthArray(recv_entry);
stride_array = hypre_CommTypeEntryStrideArray(recv_entry);
lptr = hypre_CommHandleRecvData(comm_handle) +
hypre_CommTypeEntryOffset(recv_entry);
for (ll = 0; ll < length_array[3]; ll++)
{
kptr = lptr;
for (kk = 0; kk < length_array[2]; kk++)
{
jptr = kptr;
for (jj = 0; jj < length_array[1]; jj++)
{
if (stride_array[0] == 1)
{
memcpy(jptr, bptr, length_array[0]*sizeof(double));
}
else
{
iptr = jptr;
for (ii = 0; ii < length_array[0]; ii++)
{
*iptr = bptr[ii];
iptr += stride_array[0];
}
}
bptr += length_array[0];
jptr += stride_array[1];
}
kptr += stride_array[2];
}
lptr += stride_array[3];
}
}
}
/*--------------------------------------------------------------------
* Free up communication handle
*--------------------------------------------------------------------*/
hypre_TFree(hypre_CommHandleRequests(comm_handle));
hypre_TFree(hypre_CommHandleStatus(comm_handle));
if (num_sends > 0)
{
hypre_SharedTFree(send_buffers[0]);
}
if (num_recvs > 0)
{
hypre_SharedTFree(recv_buffers[0]);
}
hypre_TFree(send_buffers);
hypre_TFree(recv_buffers);
hypre_TFree(send_sizes);
hypre_TFree(recv_sizes);
hypre_TFree(comm_handle);
return ierr;
}
#else
int
hypre_FinalizeCommunication( hypre_CommHandle *comm_handle )
{
int ierr = 0;
if (hypre_CommHandleNumRequests(comm_handle))
{
MPI_Waitall(hypre_CommHandleNumRequests(comm_handle),
hypre_CommHandleRequests(comm_handle),
hypre_CommHandleStatus(comm_handle));
}
/*--------------------------------------------------------------------
* Free up communication handle
*--------------------------------------------------------------------*/
hypre_TFree(hypre_CommHandleRequests(comm_handle));
hypre_TFree(hypre_CommHandleStatus(comm_handle));
hypre_TFree(comm_handle);
return ierr;
}
#endif
/*==========================================================================*/
/*==========================================================================*/
/** Execute local data exchanges.
{\bf Input files:}
headers.h
@return Error code.
@param comm_pkg [IN]
communication package.
@param send_data [IN]
reference pointer for the send data.
@param recv_data [IN]
reference pointer for the recv data.
@see hypre_InitializeCommunication */
/*--------------------------------------------------------------------------*/
int
hypre_ExchangeLocalData( hypre_CommPkg *comm_pkg,
double *send_data,
double *recv_data )
{
hypre_CommType *copy_from_type;
hypre_CommType *copy_to_type;
hypre_CommTypeEntry *copy_from_entry;
hypre_CommTypeEntry *copy_to_entry;
double *from_dp;
int *from_stride_array;
int from_i;
double *to_dp;
int *to_stride_array;
int to_i;
int *length_array;
int i0, i1, i2, i3;
int i;
int ierr = 0;
/*--------------------------------------------------------------------
* copy local data
*--------------------------------------------------------------------*/
copy_from_type = hypre_CommPkgCopyFromType(comm_pkg);
copy_to_type = hypre_CommPkgCopyToType(comm_pkg);
for (i = 0; i < hypre_CommTypeNumEntries(copy_from_type); i++)
{
copy_from_entry = hypre_CommTypeCommEntry(copy_from_type, i);
copy_to_entry = hypre_CommTypeCommEntry(copy_to_type, i);
from_dp = send_data + hypre_CommTypeEntryOffset(copy_from_entry);
to_dp = recv_data + hypre_CommTypeEntryOffset(copy_to_entry);
/* copy data only when necessary */
if (to_dp != from_dp)
{
length_array = hypre_CommTypeEntryLengthArray(copy_from_entry);
from_stride_array = hypre_CommTypeEntryStrideArray(copy_from_entry);
to_stride_array = hypre_CommTypeEntryStrideArray(copy_to_entry);
for (i3 = 0; i3 < length_array[3]; i3++)
{
for (i2 = 0; i2 < length_array[2]; i2++)
{
for (i1 = 0; i1 < length_array[1]; i1++)
{
from_i = (i3*from_stride_array[3] +
i2*from_stride_array[2] +
i1*from_stride_array[1] );
to_i = (i3*to_stride_array[3] +
i2*to_stride_array[2] +
i1*to_stride_array[1] );
for (i0 = 0; i0 < length_array[0]; i0++)
{
to_dp[to_i] = from_dp[from_i];
from_i += from_stride_array[0];
to_i += to_stride_array[0];
}
}
}
}
}
}
return ( ierr );
}
/*==========================================================================*/
/*==========================================================================*/
/** Create a communication type.
{\bf Input files:}
headers.h
@return Communication type.
@param comm_entries [IN]
array of pointers to communication type entries.
@param num_entries [IN]
number of elements in comm\_entries array.
@see hypre_CommTypeDestroy */
/*--------------------------------------------------------------------------*/
hypre_CommType *
hypre_CommTypeCreate( hypre_CommTypeEntry **comm_entries,
int num_entries )
{
hypre_CommType *comm_type;
comm_type = hypre_TAlloc(hypre_CommType, 1);
hypre_CommTypeCommEntries(comm_type) = comm_entries;
hypre_CommTypeNumEntries(comm_type) = num_entries;
return comm_type;
}
/*==========================================================================*/
/*==========================================================================*/
/** Destroy a communication type.
{\bf Input files:}
headers.h
@return Error code.
@param comm_type [IN]
communication type.
@see hypre_CommTypeCreate */
/*--------------------------------------------------------------------------*/
int
hypre_CommTypeDestroy( hypre_CommType *comm_type )
{
int ierr = 0;
hypre_CommTypeEntry *comm_entry;
int i;
if (comm_type)
{
if ( hypre_CommTypeCommEntries(comm_type) != NULL )
{
for (i = 0; i < hypre_CommTypeNumEntries(comm_type); i++)
{
comm_entry = hypre_CommTypeCommEntry(comm_type, i);
hypre_CommTypeEntryDestroy(comm_entry);
}
}
hypre_TFree(hypre_CommTypeCommEntries(comm_type));
hypre_TFree(comm_type);
}
return ierr;
}
/*==========================================================================*/
/*==========================================================================*/
/** Create a communication type entry.
{\bf Input files:}
headers.h
@return Communication type entry.
@param box [IN]
description of the grid data to be communicated.
@param data_box [IN]
description of the stored grid data.
@param num_values [IN]
number of data values to be communicated for each grid index.
@param data_box_offset [IN]
offset from some location in memory of the data associated with the
imin index of data_box.
@see hypre_CommTypeEntryDestroy */
/*--------------------------------------------------------------------------*/
hypre_CommTypeEntry *
hypre_CommTypeEntryCreate( hypre_Box *box,
hypre_Index stride,
hypre_Box *data_box,
int num_values,
int data_box_offset )
{
hypre_CommTypeEntry *comm_entry;
int *length_array;
int *stride_array;
hypre_Index size;
int i, j, dim;
comm_entry = hypre_TAlloc(hypre_CommTypeEntry, 1);
/*------------------------------------------------------
* Set imin, imax, and offset
*------------------------------------------------------*/
hypre_CopyIndex(hypre_BoxIMin(box),
hypre_CommTypeEntryIMin(comm_entry));
hypre_CopyIndex(hypre_BoxIMax(box),
hypre_CommTypeEntryIMax(comm_entry));
hypre_CommTypeEntryOffset(comm_entry) =
data_box_offset + hypre_BoxIndexRank(data_box, hypre_BoxIMin(box));
/*------------------------------------------------------
* Set length_array, stride_array, and dim
*------------------------------------------------------*/
length_array = hypre_CommTypeEntryLengthArray(comm_entry);
stride_array = hypre_CommTypeEntryStrideArray(comm_entry);
/* initialize length_array */
hypre_BoxGetStrideSize(box, stride, size);
for (i = 0; i < 3; i++)
length_array[i] = hypre_IndexD(size, i);
length_array[3] = num_values;
/* initialize stride_array */
for (i = 0; i < 3; i++)
{
stride_array[i] = hypre_IndexD(stride, i);
for (j = 0; j < i; j++)
stride_array[i] *= hypre_BoxSizeD(data_box, j);
}
stride_array[3] = hypre_BoxVolume(data_box);
/* eliminate dimensions with length_array = 1 */
dim = 4;
i = 0;
while (i < dim)
{
if(length_array[i] == 1)
{
for(j = i; j < (dim - 1); j++)
{
length_array[j] = length_array[j+1];
stride_array[j] = stride_array[j+1];
}
length_array[dim - 1] = 1;
stride_array[dim - 1] = 1;
dim--;
}
else
{
i++;
}
}
#if 0
/* sort the array according to length_array (largest to smallest) */
for (i = (dim-1); i > 0; i--)
for (j = 0; j < i; j++)
if (length_array[j] < length_array[j+1])
{
i_tmp = length_array[j];
length_array[j] = length_array[j+1];
length_array[j+1] = i_tmp;
i_tmp = stride_array[j];
stride_array[j] = stride_array[j+1];
stride_array[j+1] = i_tmp;
}
#endif
/* if every len was 1 we need to fix to communicate at least one */
if(!dim)
dim = 1;
hypre_CommTypeEntryDim(comm_entry) = dim;
return comm_entry;
}
/*==========================================================================*/
/*==========================================================================*/
/** Destroy a communication type entry.
{\bf Input files:}
headers.h
@return Error code.
@param comm_entry [IN/OUT]
communication type entry.
@see hypre_CommTypeEntryCreate */
/*--------------------------------------------------------------------------*/
int
hypre_CommTypeEntryDestroy( hypre_CommTypeEntry *comm_entry )
{
int ierr = 0;
if (comm_entry)
{
hypre_TFree(comm_entry);
}
return ierr;
}
/*==========================================================================*/
/*==========================================================================*/
/** Compute a processor-based description of a communication from a
grid-based one. Used to construct a communication package.
{\bf Input files:}
headers.h
@return Error code.
@param boxes [IN]
description of the grid data to be communicated to other processors.
@param data_space [IN]
description of the stored grid data associated with the communications.
@param processes [IN]
processors that data is to be communicated with.
@param num_values [IN]
number of data values to be communicated for each grid index.
@param comm [IN]
communicator.
@param num_comms_ptr [OUT]
number of communications. The number of communications is defined
by the number of processors involved in the communications, not
counting ``my processor''.
@param comm_processes_ptr [OUT]
processor
ranks involved in the communications.
@param comm_types_ptr [OUT]
inter-processor communication types.
@param copy_type_ptr [OUT]
intra-processor communication type (copies).
@see hypre_CommPkgCreate, hypre_CommTypeSort */
/*--------------------------------------------------------------------------*/
int
hypre_CommPkgCreateInfo( hypre_BoxArrayArray *boxes,
hypre_Index stride,
hypre_BoxArray *data_space,
int **processes,
int num_values,
MPI_Comm comm,
hypre_Index periodic,
int *num_comms_ptr,
int **comm_processes_ptr,
hypre_CommType ***comm_types_ptr,
hypre_CommType **copy_type_ptr)
{
int num_comms;
int *comm_processes;
hypre_CommType **comm_types;
hypre_CommType *copy_type;
hypre_CommTypeEntry ***comm_entries;
int *num_entries;
hypre_BoxArray *box_array;
hypre_Box *box;
hypre_Box *data_box;
int data_box_offset;
int i, j, p, m;
int num_procs, my_proc;
int ierr = 0;
/*---------------------------------------------------------
* Misc stuff
*---------------------------------------------------------*/
MPI_Comm_size(comm, &num_procs );
MPI_Comm_rank(comm, &my_proc );
/*------------------------------------------------------
* Loop over boxes and compute num_entries.
*------------------------------------------------------*/
num_entries = hypre_CTAlloc(int, num_procs);
num_comms = 0;
hypre_ForBoxArrayI(i, boxes)
{
box_array = hypre_BoxArrayArrayBoxArray(boxes, i);
hypre_ForBoxI(j, box_array)
{
box = hypre_BoxArrayBox(box_array, j);
p = processes[i][j];
if (hypre_BoxVolume(box) != 0)
{
num_entries[p]++;
if ((num_entries[p] == 1) && (p != my_proc))
{
num_comms++;
}
}
}
}
/*------------------------------------------------------
* Loop over boxes and compute comm_entries
* and comm_processes.
*------------------------------------------------------*/
comm_entries = hypre_CTAlloc(hypre_CommTypeEntry **, num_procs);
comm_processes = hypre_TAlloc(int, num_comms);
m = 0;
data_box_offset = 0;
hypre_ForBoxArrayI(i, boxes)
{
box_array = hypre_BoxArrayArrayBoxArray(boxes, i);
data_box = hypre_BoxArrayBox(data_space, i);
hypre_ForBoxI(j, box_array)
{
box = hypre_BoxArrayBox(box_array, j);
p = processes[i][j];
if (hypre_BoxVolume(box) != 0)
{
/* allocate comm_entries pointer */
if (comm_entries[p] == NULL)
{
comm_entries[p] =
hypre_CTAlloc(hypre_CommTypeEntry *, num_entries[p]);
num_entries[p] = 0;
if (p != my_proc)
{
comm_processes[m] = p;
m++;
}
}
comm_entries[p][num_entries[p]] =
hypre_CommTypeEntryCreate(box, stride, data_box,
num_values, data_box_offset);
num_entries[p]++;
}
}
data_box_offset += hypre_BoxVolume(data_box) * num_values;
}
/*------------------------------------------------------
* Loop over comm_entries and build comm_types
*------------------------------------------------------*/
comm_types = hypre_TAlloc(hypre_CommType *, num_comms);
for (m = 0; m < num_comms; m++)
{
p = comm_processes[m];
comm_types[m] = hypre_CommTypeCreate(comm_entries[p], num_entries[p]);
hypre_CommTypeSort(comm_types[m], periodic);
}
/*------------------------------------------------------
* Build copy_type
*------------------------------------------------------*/
if (comm_entries[my_proc] != NULL)
{
p = my_proc;
copy_type = hypre_CommTypeCreate(comm_entries[p], num_entries[p]);
hypre_CommTypeSort(copy_type, periodic);
}
else
{
copy_type = hypre_CommTypeCreate(NULL, 0);
}
/*------------------------------------------------------
* Return
*------------------------------------------------------*/
hypre_TFree(comm_entries);
hypre_TFree(num_entries);
*num_comms_ptr = num_comms;
*comm_processes_ptr = comm_processes;
*comm_types_ptr = comm_types;
*copy_type_ptr = copy_type;
return ierr;
}
/*==========================================================================*/
/*==========================================================================*/
/** Sort the entries of a communication type. This routine is used to
maintain consistency in communications.
{\bf Input files:}
headers.h
{\bf Note:}
The entries are sorted by imin first. Entries with common imin are
then sorted by imax. This assumes that imin and imax define a unique
communication type.
@return Error code.
@param comm_type [IN/OUT]
communication type to be sorted.
@see hypre_CommPkgCreateInfo */
/*--------------------------------------------------------------------------*/
int
hypre_CommTypeSort( hypre_CommType *comm_type,
hypre_Index periodic )
{
hypre_CommTypeEntry **comm_entries = hypre_CommTypeCommEntries(comm_type);
int num_entries = hypre_CommTypeNumEntries(comm_type);
hypre_CommTypeEntry *comm_entry;
hypre_IndexRef imin0, imin1;
int *imax0, *imax1;
int swap;
int i, j, ii, jj;
int ierr = 0;
#if 1
/*------------------------------------------------
* Sort by imin:
*------------------------------------------------*/
for (i = (num_entries - 1); i > 0; i--)
{
for (j = 0; j < i; j++)
{
swap = 0;
imin0 = hypre_CommTypeEntryIMin(comm_entries[j]);
imin1 = hypre_CommTypeEntryIMin(comm_entries[j+1]);
if ( hypre_IModPeriodZ(imin0, periodic) >
hypre_IModPeriodZ(imin1, periodic) )
{
swap = 1;
}
else if ( hypre_IModPeriodZ(imin0, periodic) ==
hypre_IModPeriodZ(imin1, periodic) )
{
if ( hypre_IModPeriodY(imin0, periodic) >
hypre_IModPeriodY(imin1, periodic) )
{
swap = 1;
}
else if ( hypre_IModPeriodY(imin0, periodic) ==
hypre_IModPeriodY(imin1, periodic) )
{
if ( hypre_IModPeriodX(imin0, periodic) >
hypre_IModPeriodX(imin1, periodic) )
{
swap = 1;
}
}
}
if (swap)
{
comm_entry = comm_entries[j];
comm_entries[j] = comm_entries[j+1];
comm_entries[j+1] = comm_entry;
}
}
}
/*------------------------------------------------
* Sort entries with common imin by imax:
*------------------------------------------------*/
for (ii = 0; ii < (num_entries - 1); ii = jj)
{
/* want jj where entries ii through jj-1 have common imin */
imin0 = hypre_CommTypeEntryIMin(comm_entries[ii]);
for (jj = (ii + 1); jj < num_entries; jj++)
{
imin1 = hypre_CommTypeEntryIMin(comm_entries[jj]);
if ( ( hypre_IModPeriodX(imin0, periodic) !=
hypre_IModPeriodX(imin1, periodic) ) ||
( hypre_IModPeriodY(imin0, periodic) !=
hypre_IModPeriodY(imin1, periodic) ) ||
( hypre_IModPeriodZ(imin0, periodic) !=
hypre_IModPeriodZ(imin1, periodic) ) )
{
break;
}
}
/* sort entries ii through jj-1 by imax */
for (i = (jj - 1); i > ii; i--)
{
for (j = ii; j < i; j++)
{
swap = 0;
imax0 = hypre_CommTypeEntryIMax(comm_entries[j]);
imax1 = hypre_CommTypeEntryIMax(comm_entries[j+1]);
if ( hypre_IModPeriodZ(imax0, periodic) >
hypre_IModPeriodZ(imax1, periodic) )
{
swap = 1;
}
else if ( hypre_IModPeriodZ(imax0, periodic) ==
hypre_IModPeriodZ(imax1, periodic) )
{
if ( hypre_IModPeriodY(imax0, periodic) >
hypre_IModPeriodY(imax1, periodic) )
{
swap = 1;
}
else if ( hypre_IModPeriodY(imax0, periodic) ==
hypre_IModPeriodY(imax1, periodic) )
{
if ( hypre_IModPeriodX(imax0, periodic) >
hypre_IModPeriodX(imax1, periodic) )
{
swap = 1;
}
}
}
if (swap)
{
comm_entry = comm_entries[j];
comm_entries[j] = comm_entries[j+1];
comm_entries[j+1] = comm_entry;
}
}
}
}
#endif
return ierr;
}
/*==========================================================================*/
/*==========================================================================*/
/** Compile a communication package into a form based on the
message-passing layer.
{\bf Input files:}
headers.h
@return Error code.
@param comm_pkg [IN/OUT]
communication package.
@see hypre_CommPkgCreate, hypre_InitializeCommunication,
hypre_CommTypeBuildMPI, hypre_CommPkgUnCommit */
/*--------------------------------------------------------------------------*/
int
hypre_CommPkgCommit( hypre_CommPkg *comm_pkg )
{
int ierr = 0;
/* create send MPI_Datatypes */
hypre_CommPkgSendMPITypes(comm_pkg) =
hypre_TAlloc(MPI_Datatype, hypre_CommPkgNumSends(comm_pkg));
hypre_CommTypeBuildMPI(hypre_CommPkgNumSends(comm_pkg),
hypre_CommPkgSendProcs(comm_pkg),
hypre_CommPkgSendTypes(comm_pkg),
hypre_CommPkgSendMPITypes(comm_pkg));
/* create recv MPI_Datatypes */
hypre_CommPkgRecvMPITypes(comm_pkg) =
hypre_TAlloc(MPI_Datatype, hypre_CommPkgNumRecvs(comm_pkg));
hypre_CommTypeBuildMPI(hypre_CommPkgNumRecvs(comm_pkg),
hypre_CommPkgRecvProcs(comm_pkg),
hypre_CommPkgRecvTypes(comm_pkg),
hypre_CommPkgRecvMPITypes(comm_pkg));
return ierr;
}
/*==========================================================================*/
/*==========================================================================*/
/** Destroy the message-passing-layer component of the communication
package.
{\bf Input files:}
headers.h
@return Error code.
@param comm_pkg [IN/OUT]
communication package.
@see hypre_CommPkgCommit */
/*--------------------------------------------------------------------------*/
int
hypre_CommPkgUnCommit( hypre_CommPkg *comm_pkg )
{
MPI_Datatype *types;
int i;
int ierr = 0;
if (comm_pkg)
{
types = hypre_CommPkgSendMPITypes(comm_pkg);
if (types)
{
for (i = 0; i < hypre_CommPkgNumSends(comm_pkg); i++)
MPI_Type_free(&types[i]);
hypre_TFree(types);
}
types = hypre_CommPkgRecvMPITypes(comm_pkg);
if (types)
{
for (i = 0; i < hypre_CommPkgNumRecvs(comm_pkg); i++)
MPI_Type_free(&types[i]);
hypre_TFree(types);
}
}
return ierr;
}
/*==========================================================================*/
/*==========================================================================*/
/** Create an MPI-based description of a communication from a
processor-based one.
{\bf Input files:}
headers.h
@return Error code.
@param num_comms [IN]
number of communications.
@param comm_procs [IN]
processor ranks involved in the communications.
@param comm_types [IN]
processor-based communication types.
@param comm_mpi_types [OUT]
MPI derived data-types.
@see hypre_CommPkgCommit, hypre_CommTypeEntryBuildMPI */
/*--------------------------------------------------------------------------*/
int
hypre_CommTypeBuildMPI( int num_comms,
int *comm_procs,
hypre_CommType **comm_types,
MPI_Datatype *comm_mpi_types )
{
hypre_CommType *comm_type;
hypre_CommTypeEntry *comm_entry;
int num_entries;
int *comm_entry_blocklengths;
MPI_Aint *comm_entry_displacements;
MPI_Datatype *comm_entry_mpi_types;
int m, i;
int ierr = 0;
for (m = 0; m < num_comms; m++)
{
comm_type = comm_types[m];
num_entries = hypre_CommTypeNumEntries(comm_type);
comm_entry_blocklengths = hypre_TAlloc(int, num_entries);
comm_entry_displacements = hypre_TAlloc(MPI_Aint, num_entries);
comm_entry_mpi_types = hypre_TAlloc(MPI_Datatype, num_entries);
for (i = 0; i < num_entries; i++)
{
comm_entry = hypre_CommTypeCommEntry(comm_type, i);
/* set blocklengths */
comm_entry_blocklengths[i] = 1;
/* compute displacements */
comm_entry_displacements[i] =
hypre_CommTypeEntryOffset(comm_entry) * sizeof(double);
/* compute types */
hypre_CommTypeEntryBuildMPI(comm_entry, &comm_entry_mpi_types[i]);
}
/* create `comm_mpi_types' */
MPI_Type_struct(num_entries, comm_entry_blocklengths,
comm_entry_displacements, comm_entry_mpi_types,
&comm_mpi_types[m]);
MPI_Type_commit(&comm_mpi_types[m]);
/* free up memory */
for (i = 0; i < num_entries; i++)
MPI_Type_free(&comm_entry_mpi_types[i]);
hypre_TFree(comm_entry_blocklengths);
hypre_TFree(comm_entry_displacements);
hypre_TFree(comm_entry_mpi_types);
}
return ierr;
}
/*==========================================================================*/
/*==========================================================================*/
/** Create an MPI-based description of a communication entry.
{\bf Input files:}
headers.h
@return Error code.
@param comm_entry [IN]
communication entry.
@param comm_entry_mpi_type [OUT]
MPI derived data-type.
@see hypre_CommTypeBuildMPI */
/*--------------------------------------------------------------------------*/
int
hypre_CommTypeEntryBuildMPI( hypre_CommTypeEntry *comm_entry,
MPI_Datatype *comm_entry_mpi_type )
{
int dim = hypre_CommTypeEntryDim(comm_entry);
int *length_array = hypre_CommTypeEntryLengthArray(comm_entry);
int *stride_array = hypre_CommTypeEntryStrideArray(comm_entry);
MPI_Datatype *old_type;
MPI_Datatype *new_type;
MPI_Datatype *tmp_type;
int i;
int ierr = 0;
if (dim == 1)
{
MPI_Type_hvector(length_array[0], 1,
(MPI_Aint)(stride_array[0]*sizeof(double)),
MPI_DOUBLE, comm_entry_mpi_type);
}
else
{
old_type = hypre_CTAlloc(MPI_Datatype, 1);
new_type = hypre_CTAlloc(MPI_Datatype, 1);
MPI_Type_hvector(length_array[0], 1,
(MPI_Aint)(stride_array[0]*sizeof(double)),
MPI_DOUBLE, old_type);
for (i = 1; i < (dim - 1); i++)
{
MPI_Type_hvector(length_array[i], 1,
(MPI_Aint)(stride_array[i]*sizeof(double)),
*old_type, new_type);
MPI_Type_free(old_type);
tmp_type = old_type;
old_type = new_type;
new_type = tmp_type;
}
MPI_Type_hvector(length_array[i], 1,
(MPI_Aint)(stride_array[i]*sizeof(double)),
*old_type, comm_entry_mpi_type);
MPI_Type_free(old_type);
hypre_TFree(old_type);
hypre_TFree(new_type);
}
return ierr;
}
Reference in New Issue View Git Blame Copy Permalink