[mpich2-commits] r4001 - in mpich2/trunk/src: include mpi/comm mpi/init mpid/ch3/src
goodell at mcs.anl.gov
goodell at mcs.anl.gov
Tue Mar 10 17:20:00 CDT 2009
Author: goodell
Date: 2009-03-10 17:20:00 -0500 (Tue, 10 Mar 2009)
New Revision: 4001
Modified:
mpich2/trunk/src/include/mpiimpl.h
mpich2/trunk/src/mpi/comm/commutil.c
mpich2/trunk/src/mpi/init/initthread.c
mpich2/trunk/src/mpid/ch3/src/ch3u_port.c
Log:
Compute sub-communicator context IDs from the parent's context ID.
This change removes the MPIR_Allreduces that were added to MPI_Init time as part
of the SMP collectives work. It also brings the number of MPI_Allreduces on
each communicator creation back down to 1 instead of 3. It does this by shaving
2 bits from the context ID to indicate parent, intranode, or internode.
This change also removes one of the context suffix (also called offset) bits
from context IDs, for a final number of 1 bit.
The net change in the context ID space after this change is a reduction by a
factor of 2 (1 bit).
Reviewed by buntinas at .
Modified: mpich2/trunk/src/include/mpiimpl.h
===================================================================
--- mpich2/trunk/src/include/mpiimpl.h 2009-03-10 21:54:41 UTC (rev 4000)
+++ mpich2/trunk/src/include/mpiimpl.h 2009-03-10 22:20:00 UTC (rev 4001)
@@ -1351,42 +1351,47 @@
of the handle is 3-1 (e.g., the index in the builtin array) */
#define MPIR_ICOMM_WORLD ((MPI_Comm)0x44000002)
-/*
- * The order of the context offsets is important. The collective routines
- * in the case of intercommunicator operations use offsets 2 and 3 for
- * the local intracommunicator; thus it is vital that the offsets used
- * for communication between processes in the intercommunicator in a
- * collective operation (MPID_CONTEXT_INTER_COLL) be distinct from the
- * offsets uses for communication on the local intracommunicator (2+
- * MPID_CONTEXT_INTRA_COLL)
- */
+/* Context suffixes for separating pt2pt and collective communication */
+#define MPID_CONTEXT_NUM_SUFFIX_BITS 1
+#define MPID_CONTEXT_SUFFIX_MASK ((1 << MPID_CONTEXT_NUM_SUFFIX_BITS) - 1)
#define MPID_CONTEXT_INTRA_PT2PT 0
#define MPID_CONTEXT_INTRA_COLL 1
-#define MPID_CONTEXT_INTRA_FILE 2
-#define MPID_CONTEXT_INTRA_WIN 3
#define MPID_CONTEXT_INTER_PT2PT 0
#define MPID_CONTEXT_INTER_COLL 1
-#define MPID_CONTEXT_INTER_COLLA 2
-#define MPID_CONTEXT_INTER_COLLB 3
-/*
+/* Used to derive context IDs for sub-communicators from a parent communicator's
+ context ID value. This field comes after the one bit suffix.
+ values are shifted left by 1. */
+#define MPID_CONTEXT_NUM_SUBCOMM_BITS 2
+#define MPID_CONTEXT_SUBCOMM_MASK (((1 << MPID_CONTEXT_NUM_SUBCOMM_BITS) - 1) << MPID_CONTEXT_NUM_SUFFIX_BITS)
+#define MPID_CONTEXT_PARENT_OFFSET (0 << MPID_CONTEXT_NUM_SUFFIX_BITS)
+#define MPID_CONTEXT_INTRANODE_OFFSET (1 << MPID_CONTEXT_NUM_SUFFIX_BITS)
+#define MPID_CONTEXT_INTERNODE_OFFSET (2 << MPID_CONTEXT_NUM_SUFFIX_BITS)
+
+/* MPIR_MAX_CONTEXT_MASK is the number of ints that make up the bit vector that
+ * describes the context ID prefix space.
+ *
* The following must hold:
- * MPIR_MAX_CONTEXT_MASK*32 <= 2**(sizeof(MPIR_Context_id_t) * 8 - 2)
- * For a 16-bit context id field, this implies MPIR_MAX_CONTEXT_MASK <= 512
+ * (num_bits_in_vector) <= (maximum_context_id_prefix)
+ * which is the following in concrete terms:
+ * MPIR_MAX_CONTEXT_MASK*MPIR_CONTEXT_INT_BITS <= 2**(MPIR_CONTEXT_ID_BITS - (MPID_CONTEXT_PREFIX_SHIFT + MPID_CONTEXT_DYNAMIC_PROC_BITS))
*
- * MPIR_Context_id_t can hold 2**(sizeof(MPIR_Context_id_t) * 8)
- * context IDs. Of these, the last two bits are used to determine the
- * class of the context ID, so we only have
- * 2**(sizeof(MPIR_Context_id_t) * 8 - 2) context IDs. We can use
- * (2**(sizeof(MPIR_Context_id_t) * 8 - 2) / 32) integers to hold
- * these many bits assuming each integer is 32 bits, i.e.,
- * 2**(sizeof(MPIR_Context_id_t) * 8 - 7) integers. Finally, we want
- * to reserve the second half of this set for temporary VCs in the
- * dynamic process case. That leaves us with
- * 2**(sizeof(MPIR_Context_id_t) * 8 - 8) integers.
+ * We currently always assume MPIR_CONTEXT_INT_BITS is 32, regardless of the
+ * value of sizeof(int)*CHAR_BITS. We also make the assumption that CHAR_BITS==8.
+ *
+ * For a 16-bit context id field and CHAR_BITS==8, this implies MPIR_MAX_CONTEXT_MASK <= 256
*/
-#define MPIR_MAX_CONTEXT_MASK (1 << ((sizeof(MPIR_Context_id_t) * 8) - 8))
+/* number of bits to shift right by in order to obtain the context ID prefix */
+#define MPID_CONTEXT_PREFIX_SHIFT (MPID_CONTEXT_NUM_SUFFIX_BITS + MPID_CONTEXT_NUM_SUBCOMM_BITS)
+
+/* should probably be (sizeof(int)*CHAR_BITS) once we make the code CHAR_BITS-clean */
+#define MPIR_CONTEXT_INT_BITS (32)
+#define MPIR_CONTEXT_ID_BITS (sizeof(MPIR_Context_id_t)*8) /* 8 --> CHAR_BITS eventually */
+#define MPID_CONTEXT_DYNAMIC_PROC_BITS (1) /* the upper half is reserved for dynamic procs */
+#define MPIR_MAX_CONTEXT_MASK \
+ ((1 << (MPIR_CONTEXT_ID_BITS - (MPID_CONTEXT_PREFIX_SHIFT + MPID_CONTEXT_DYNAMIC_PROC_BITS))) / MPIR_CONTEXT_INT_BITS)
+
/* Utility routines. Where possible, these are kept in the source directory
with the other comm routines (src/mpi/comm, in mpicomm.h). However,
to create a new communicator after a spawn or connect-accept operation,
Modified: mpich2/trunk/src/mpi/comm/commutil.c
===================================================================
--- mpich2/trunk/src/mpi/comm/commutil.c 2009-03-10 21:54:41 UTC (rev 4000)
+++ mpich2/trunk/src/mpi/comm/commutil.c 2009-03-10 22:20:00 UTC (rev 4001)
@@ -242,7 +242,6 @@
int num_local = -1, num_external = -1;
int local_rank = -1, external_rank = -1;
int *local_procs = NULL, *external_procs = NULL;
- MPIR_Context_id_t context_id;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_COMM_COMMIT);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_COMM_COMMIT);
@@ -286,22 +285,12 @@
goto fn_exit;
}
- /* FIXME could just use the same ctxid and a different tag like
- MPIR_BCAST_TAG. Perhaps MPIR_BCAST_INTRANODE_TAG, etc. Or maybe
- just use 3 bits for the ctx suffix and use some of those values. */
- /* MPIR_Get_contextid is collective over the whole communicator, so
- everyone has to allocate one and then processes that don't need one
- can free it as a local operation. */
- mpi_errno = MPIR_Get_contextid(comm, &context_id);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
-
/* we don't need a local comm if this process is the only one on this node */
if (num_local > 1) {
mpi_errno = MPIR_Comm_create(&comm->node_comm);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- comm->node_comm->context_id = context_id;
-
+ comm->node_comm->context_id = comm->context_id + MPID_CONTEXT_INTRANODE_OFFSET;
comm->node_comm->recvcontext_id = comm->node_comm->context_id;
comm->node_comm->rank = local_rank;
comm->node_comm->comm_kind = MPID_INTRACOMM;
@@ -322,23 +311,14 @@
MPID_Dev_comm_create_hook( comm->node_comm );
/* don't call MPIR_Comm_commit here */
}
- else {
- MPIR_Free_contextid(context_id);
- }
- /* FIXME could just use the same ctxid and a different tag like
- MPIR_BCAST_TAG. Perhaps MPIR_BCAST_INTRANODE_TAG, etc. Or maybe
- just use 3 bits for the ctx suffix and use some of those values. */
- mpi_errno = MPIR_Get_contextid(comm, &context_id);
- if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* this process may not be a member of the node_roots_comm */
if (local_rank == 0) {
mpi_errno = MPIR_Comm_create(&comm->node_roots_comm);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
- comm->node_roots_comm->context_id = context_id;
-
+ comm->node_roots_comm->context_id = comm->context_id + MPID_CONTEXT_INTERNODE_OFFSET;
comm->node_roots_comm->recvcontext_id = comm->node_roots_comm->context_id;
comm->node_roots_comm->rank = external_rank;
comm->node_roots_comm->comm_kind = MPID_INTRACOMM;
@@ -359,12 +339,8 @@
MPID_Dev_comm_create_hook( comm->node_roots_comm );
/* don't call MPIR_Comm_commit here */
}
- else {
- MPIR_Free_contextid(context_id);
- }
comm->is_node_aware = 1;
-
}
fn_exit:
@@ -501,7 +477,7 @@
j += 1;
}
context_mask[i] &= ~(1<<j);
- context_id = 4 * (32 * i + j);
+ context_id = (32 * i + j) << MPID_CONTEXT_PREFIX_SHIFT;
MPIU_DBG_MSG_FMT(COMM,VERBOSE,(MPIU_DBG_FDEST,
"allocating contextid = %d, (mask[%d], bit %d)",
context_id, i, j ) );
@@ -812,15 +788,8 @@
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_FREE_CONTEXTID);
/* Convert the context id to the bit position */
- /* FIXME why do we shift right by 2? What exactly are those bottom two bits
- used for? */
- /* Note: The use of the context_id is covered in the design document. They
- are used to provide private context ids for different communication
- operations, such as collective communication, without requiring a
- separate communicator. 4 ids were originally allocated to ensure
- separation between pt-2-pt, collective, RMA, and intercomm pt-2-pt */
- idx = (context_id >> 2) / 32;
- bitpos = (context_id >> 2) % 32;
+ idx = (context_id >> MPID_CONTEXT_PREFIX_SHIFT) / 32;
+ bitpos = (context_id >> MPID_CONTEXT_PREFIX_SHIFT) % 32;
/* --BEGIN ERROR HANDLING-- */
if (idx < 0 || idx >= MPIR_MAX_CONTEXT_MASK) {
@@ -834,11 +803,11 @@
MPIU_DBG_MSG_S(COMM,VERBOSE,"Context mask = %s",MPIR_ContextMaskToStr());
/* FIXME This abort cannot be enabled at this time. The local and
remote communicators in an intercommunicator (always?) share a
- context_id prefix (bits 15..2) and free will be called on both of
- them by the higher level code. This should probably be fixed but we
- can't until we understand the context_id code better. One possible
- solution is to only free when (context_id&0x3)!=0.
- [goodell@ 2008-08-18]
+ context_id prefix (bits 15..MPID_CONTEXT_NUM_SUFFIX_BITS) and free
+ will be called on both of them by the higher level code. This should
+ probably be fixed but we can't until we understand the context_id
+ code better. One possible solution is to only free when
+ (context_id&MPID_CONTEXT_SUFFIX_MASK)!=0. [goodell@ 2008-08-18]
MPID_Abort( 0, MPI_ERR_INTERN, 1,
"In MPIR_Free_contextid, the context id is not in use" );
Modified: mpich2/trunk/src/mpi/init/initthread.c
===================================================================
--- mpich2/trunk/src/mpi/init/initthread.c 2009-03-10 21:54:41 UTC (rev 4000)
+++ mpich2/trunk/src/mpi/init/initthread.c 2009-03-10 22:20:00 UTC (rev 4001)
@@ -285,8 +285,8 @@
MPIR_Process.comm_world = MPID_Comm_builtin + 0;
MPIR_Process.comm_world->handle = MPI_COMM_WORLD;
MPIU_Object_set_ref( MPIR_Process.comm_world, 1 );
- MPIR_Process.comm_world->context_id = 0; /* XXX */
- MPIR_Process.comm_world->recvcontext_id = 0;
+ MPIR_Process.comm_world->context_id = 0 << MPID_CONTEXT_PREFIX_SHIFT;
+ MPIR_Process.comm_world->recvcontext_id = 0 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_world->attributes = NULL;
MPIR_Process.comm_world->local_group = NULL;
MPIR_Process.comm_world->remote_group = NULL;
@@ -302,8 +302,8 @@
MPIR_Process.comm_self = MPID_Comm_builtin + 1;
MPIR_Process.comm_self->handle = MPI_COMM_SELF;
MPIU_Object_set_ref( MPIR_Process.comm_self, 1 );
- MPIR_Process.comm_self->context_id = 4; /* XXX */
- MPIR_Process.comm_self->recvcontext_id = 4; /* XXX */
+ MPIR_Process.comm_self->context_id = 1 << MPID_CONTEXT_PREFIX_SHIFT;
+ MPIR_Process.comm_self->recvcontext_id = 1 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.comm_self->attributes = NULL;
MPIR_Process.comm_self->local_group = NULL;
MPIR_Process.comm_self->remote_group = NULL;
@@ -318,8 +318,8 @@
MPIR_Process.icomm_world = MPID_Comm_builtin + 2;
MPIR_Process.icomm_world->handle = MPIR_ICOMM_WORLD;
MPIU_Object_set_ref( MPIR_Process.icomm_world, 1 );
- MPIR_Process.icomm_world->context_id = 8;
- MPIR_Process.icomm_world->recvcontext_id= 8;
+ MPIR_Process.icomm_world->context_id = 2 << MPID_CONTEXT_PREFIX_SHIFT;
+ MPIR_Process.icomm_world->recvcontext_id= 2 << MPID_CONTEXT_PREFIX_SHIFT;
MPIR_Process.icomm_world->attributes = NULL;
MPIR_Process.icomm_world->local_group = NULL;
MPIR_Process.icomm_world->remote_group = NULL;
Modified: mpich2/trunk/src/mpid/ch3/src/ch3u_port.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/src/ch3u_port.c 2009-03-10 21:54:41 UTC (rev 4000)
+++ mpich2/trunk/src/mpid/ch3/src/ch3u_port.c 2009-03-10 22:20:00 UTC (rev 4001)
@@ -259,7 +259,7 @@
* If the same process opens connections to the multiple
* processes, this context ID might get out of sync.
*/
- tmp_comm->context_id = (MPIR_MAX_CONTEXT_MASK * sizeof(unsigned int) * 8) + context_id_offset;
+ tmp_comm->context_id = (MPIR_MAX_CONTEXT_MASK * MPIR_CONTEXT_INT_BITS) + context_id_offset;
tmp_comm->recvcontext_id = tmp_comm->context_id;
/* FIXME - we probably need a unique context_id. */
More information about the mpich2-commits
mailing list