[mpich2-commits] r4225 - mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad
mercierg at mcs.anl.gov
mercierg at mcs.anl.gov
Mon Mar 30 10:37:41 CDT 2009
Author: mercierg
Date: 2009-03-30 10:37:41 -0500 (Mon, 30 Mar 2009)
New Revision: 4225
Modified:
mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_finalize.c
mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_impl.h
mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_init.c
mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_poll.c
mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_send.c
Log:
switch to events-based iface for recv + bug fixes
Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_finalize.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_finalize.c 2009-03-30 07:00:07 UTC (rev 4224)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_finalize.c 2009-03-30 15:37:41 UTC (rev 4225)
@@ -16,6 +16,8 @@
{
int mpi_errno = MPI_SUCCESS;
+ common_exit(NULL);
+
fn_exit:
return mpi_errno;
fn_fail: ATTRIBUTE((unused))
Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_impl.h
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_impl.h 2009-03-30 07:00:07 UTC (rev 4224)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_impl.h 2009-03-30 15:37:41 UTC (rev 4225)
@@ -54,8 +54,10 @@
/* Callbacks for events */
void MPID_nem_newmad_get_adi_msg(nm_sr_event_t event, const nm_sr_event_info_t*info);
+void MPID_nem_newmad_get_rreq(nm_sr_event_t event, const nm_sr_event_info_t*info);
void MPID_nem_newmad_handle_sreq(nm_sr_event_t event, const nm_sr_event_info_t*info);
+
/* Dtype management */
int MPID_nem_newmad_process_sdtype(MPID_Request **sreq_p, MPI_Datatype datatype, MPID_Datatype * dt_ptr, const void *buf,
int count, MPIDI_msg_sz_t data_sz, struct iovec **newmad_iov, int *num_iov, int first_taken);
@@ -65,14 +67,14 @@
/* Connection management*/
int MPID_nem_newmad_send_conn_info (MPIDI_VC_t *vc);
-#define MPID_NEM_NMAD_MAX_NETS 3
-#define MY_SIZE (MPID_NEM_MAX_NETMOD_STRING_LEN/2)
+#define MPID_NEM_NMAD_MAX_NETS 4
+#define MPID_NEM_NMAD_MAX_SIZE MPID_NEM_MAX_NETMOD_STRING_LEN
typedef nm_gate_t mpid_nem_newmad_p_gate_t;
typedef struct MPID_nem_newmad_vc_area_internal
{
- char hostname[MY_SIZE];
- char url[MPID_NEM_NMAD_MAX_NETS][MY_SIZE];
+ char hostname[MPID_NEM_NMAD_MAX_SIZE];
+ char url[MPID_NEM_NMAD_MAX_NETS][MPID_NEM_NMAD_MAX_SIZE];
uint8_t drv_id[MPID_NEM_NMAD_MAX_NETS];
mpid_nem_newmad_p_gate_t p_gate;
} MPID_nem_newmad_vc_area_internal_t;
@@ -91,6 +93,7 @@
typedef struct
{
nm_sr_request_t newmad_req;
+ void *iov;
} MPID_nem_newmad_req_area;
/* accessor macro to private fields in REQ */
#define REQ_FIELD(reqp, field) (((MPID_nem_newmad_req_area *)((reqp)->ch.netmod_area.padding))->field)
@@ -168,6 +171,7 @@
extern int mpid_nem_newmad_pending_send_req;
#define NMAD_IOV_MAX_DEPTH 15 /* NM_SO_PREALLOC_IOV_LEN */
+//#define DEBUG
#endif //NEWMAD_MODULE_IMPL_H
Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_init.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_init.c 2009-03-30 07:00:07 UTC (rev 4224)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_init.c 2009-03-30 15:37:41 UTC (rev 4225)
@@ -49,7 +49,7 @@
static int mpid_nem_newmad_myrank;
static nm_drv_id_t drv_id[MPID_NEM_NMAD_MAX_NETS];
static char *url[MPID_NEM_NMAD_MAX_NETS];
-static char url_keys[MPID_NEM_NMAD_MAX_NETS][MPID_NEM_MAX_NETMOD_STRING_LEN] = {"url_id0","url_id1","url_id2","url_id3"};
+static char url_keys[MPID_NEM_NMAD_MAX_NETS][MPID_NEM_NMAD_MAX_SIZE] = {"url_id0","url_id1","url_id2","url_id3"};
static int mpid_nem_newmad_num_rails = 1 ;
nm_core_t mpid_nem_newmad_pcore;
int mpid_nem_newmad_pending_send_req = 0;
@@ -170,7 +170,7 @@
strcpy(url[0],hostname);
}
#endif /* !MULTIRAIL */
-#endif /* TCP */
+#endif /* TCP */
#endif
nm_ns_init(mpid_nem_newmad_pcore);
@@ -221,7 +221,7 @@
fprintf(stdout,"Size of MPID_nem_mad_module_vc_area_internal_t : %i | size of nm_sr_request_t :%i | Size of req_area : %i\n",
sizeof(MPID_nem_newmad_vc_area_internal_t),sizeof(nm_sr_request_t), sizeof(MPID_nem_newmad_req_area));
- MPIU_Assert( sizeof(MPID_nem_newmad_vc_area_internal_t) <= MPID_NEM_VC_NETMOD_AREA_LEN);
+ //MPIU_Assert( sizeof(MPID_nem_newmad_vc_area_internal_t) <= MPID_NEM_VC_NETMOD_AREA_LEN);
MPIU_Assert( sizeof(MPID_nem_newmad_req_area) <= MPID_NEM_REQ_NETMOD_AREA_LEN);
mpid_nem_newmad_myrank = pg_rank;
@@ -236,7 +236,8 @@
mpi_errno = MPID_nem_newmad_get_business_card(pg_rank,bc_val_p, val_max_sz_p);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
- nm_sr_monitor(mpid_nem_newmad_pcore, NM_SR_EVENT_RECV_UNEXPECTED, &MPID_nem_newmad_get_adi_msg);
+ nm_sr_monitor(mpid_nem_newmad_pcore, NM_SR_EVENT_RECV_UNEXPECTED,&MPID_nem_newmad_get_adi_msg);
+ nm_sr_monitor(mpid_nem_newmad_pcore, NM_SR_EVENT_RECV_COMPLETED, &MPID_nem_newmad_get_rreq);
nm_sr_monitor(mpid_nem_newmad_pcore, NM_SR_EVENT_SEND_COMPLETED, &MPID_nem_newmad_handle_sreq);
mpi_errno = MPIDI_CH3I_Register_anysource_notification(MPID_nem_newmad_anysource_posted,
@@ -257,10 +258,10 @@
MPID_nem_newmad_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
- char name[MPID_NEM_MAX_NETMOD_STRING_LEN];
+ char name[MPID_NEM_NMAD_MAX_SIZE];
int index;
- gethostname(name,MPID_NEM_MAX_NETMOD_STRING_LEN);
+ gethostname(name,MPID_NEM_NMAD_MAX_SIZE);
mpi_errno = MPIU_Str_add_binary_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_HOSTNAME_KEY, name, strlen(name));
if (mpi_errno != MPIU_STR_SUCCESS){
@@ -288,7 +289,7 @@
fn_exit:
return mpi_errno;
- fn_fail:
+ fn_fail: ATTRIBUTE((unused))
goto fn_exit;
}
@@ -303,13 +304,13 @@
int len;
mpi_errno = MPIU_Str_get_binary_arg (business_card, MPIDI_CH3I_HOSTNAME_KEY, hostname,
- MPID_NEM_MAX_NETMOD_STRING_LEN, &len);
+ MPID_NEM_NMAD_MAX_SIZE, &len);
if ((mpi_errno != MPIU_STR_SUCCESS)){
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**argstr_hostd");
}
mpi_errno = MPIU_Str_get_binary_arg (business_card, url_keys[index], url,
- MPID_NEM_MAX_NETMOD_STRING_LEN, &len);
+ MPID_NEM_NMAD_MAX_SIZE, &len);
if ((mpi_errno != MPIU_STR_SUCCESS)){
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**argstr_hostd");
}
@@ -328,10 +329,10 @@
MPID_nem_newmad_connect_to_root (const char *business_card, MPIDI_VC_t *new_vc)
{
int mpi_errno = MPI_SUCCESS;
- fn_exit:
- return mpi_errno;
- fn_fail:
- goto fn_exit;
+ fn_exit:
+ return mpi_errno;
+ fn_fail: ATTRIBUTE((unused))
+ goto fn_exit;
}
#undef FUNCNAME
@@ -342,12 +343,12 @@
MPID_nem_newmad_vc_init (MPIDI_VC_t *vc)
{
MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
- char business_card[MPID_NEM_MAX_NETMOD_STRING_LEN];
+ char business_card[MPID_NEM_NMAD_MAX_SIZE];
int mpi_errno = MPI_SUCCESS;
int ret;
int index;
- mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, MPID_NEM_MAX_NETMOD_STRING_LEN, vc->pg);
+ mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, MPID_NEM_NMAD_MAX_SIZE, vc->pg);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
(((MPID_nem_newmad_vc_area *)((MPIDI_CH3I_VC *)(vc)->channel_private)->netmod_area.padding)->area) =
@@ -400,7 +401,7 @@
fn_exit:
return mpi_errno;
- fn_fail:
+ fn_fail: ATTRIBUTE((unused))
goto fn_exit;
}
Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_poll.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_poll.c 2009-03-30 07:00:07 UTC (rev 4224)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_poll.c 2009-03-30 15:37:41 UTC (rev 4225)
@@ -37,8 +37,13 @@
void
MPID_nem_newmad_get_adi_msg(nm_sr_event_t event, const nm_sr_event_info_t*info)
{
- nm_tag_t match_info = info->recv_unexpected.tag;
+ nm_tag_t match_info = info->recv_unexpected.tag;
MPIR_Context_id_t ctxt;
+
+#ifdef DEBUG
+ fprintf(stdout,"===> Getting ADI MSG match is %lx \n",match_info);
+#endif
+
NEM_NMAD_MATCH_GET_CTXT(match_info, ctxt);
if(ctxt == NEM_NMAD_INTRA_CTXT)
{
@@ -69,6 +74,7 @@
return;
}
+
#undef FUNCNAME
#define FUNCNAME MPID_nem_newmad_directRecv
#undef FCNAME
@@ -86,12 +92,10 @@
MPIR_Context_id_t context = rreq->dev.match.parts.context_id;
Nmad_Nem_tag_t tag = rreq->dev.match.parts.tag;
int ret;
- int num_seg = 1;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype *dt_ptr;
- struct iovec newmad_iov[NMAD_IOV_MAX_DEPTH];
NEM_NMAD_DIRECT_MATCH(match_info,0,source,context);
if (tag != MPI_ANY_TAG)
@@ -102,25 +106,41 @@
{
MPIU_Assert(0);
}
-
+#ifdef DEBUG
+ fprintf(stdout,"========> Posting Recv req %p (match is %lx) \n",rreq,match_info);
+#endif
MPIDI_Datatype_get_info(rreq->dev.user_count,rreq->dev.datatype, dt_contig, data_sz, dt_ptr,dt_true_lb);
rreq->dev.OnDataAvail = NULL;
-
+
if (dt_contig)
{
- newmad_iov[0].iov_base = (char*)(rreq->dev.user_buf) + dt_true_lb;
- newmad_iov[0].iov_len = data_sz;
+ ret = nm_sr_irecv_with_ref(mpid_nem_newmad_pcore,VC_FIELD(vc,p_gate),match_info,
+ (char*)(rreq->dev.user_buf) + dt_true_lb,data_sz,
+ &(REQ_FIELD(rreq,newmad_req)),(void*)rreq);
}
else
{
- struct iovec *newmad_iov_ptr = &(newmad_iov[0]);
+ int num_seg = 0;
+ struct iovec *newmad_iov_ptr = NULL;
+ struct iovec *newmad_iov = (struct iovec *)MPIU_Malloc(NMAD_IOV_MAX_DEPTH*sizeof(struct iovec));
+ newmad_iov_ptr = &(newmad_iov[0]);
MPID_nem_newmad_process_rdtype(&rreq,dt_ptr,data_sz,&newmad_iov_ptr,&num_seg);
+ MPIU_Assert(num_seg <= NMAD_IOV_MAX_DEPTH);
+#ifdef DEBUG
+ {
+ int index;
+ for(index = 0; index < num_seg ; index++)
+ {
+ fprintf(stdout,"======================\n");
+ fprintf(stdout,"RECV nmad_iov[%i]: [base %p][len %i]\n",index,
+ newmad_iov[index].iov_base,newmad_iov[index].iov_len);
+ }
+ }
+#endif
+ ret = nm_sr_irecv_iov_with_ref(mpid_nem_newmad_pcore,VC_FIELD(vc,p_gate),match_info,
+ newmad_iov,num_seg,&(REQ_FIELD(rreq,newmad_req)),(void*)rreq);
+ REQ_FIELD(rreq,iov) = newmad_iov;
}
-
- MPIU_Assert(num_seg <= NMAD_IOV_MAX_DEPTH);
-
- ret = nm_sr_irecv_iov_with_ref(mpid_nem_newmad_pcore,VC_FIELD(vc,p_gate),match_info,
- newmad_iov,num_seg,&(REQ_FIELD(rreq,newmad_req)),(void*)rreq);
}
fn_exit:
@@ -130,83 +150,74 @@
goto fn_exit;
}
+
#undef FUNCNAME
-#define FUNCNAME MPID_nem_newmad_poll_recv
+#define FUNCNAME MPID_nem_newmad_get_rreq
#undef FCNAME
#define FCNAME MPIDI_QUOTE(FUNCNAME)
-static int
-MPID_nem_newmad_poll_recv(void)
+void MPID_nem_newmad_get_rreq(nm_sr_event_t event, const nm_sr_event_info_t*info)
{
- int mpi_errno = MPI_SUCCESS;
- nm_sr_request_t *p_out_req = NULL;
- MPID_Request *req = NULL;
- int ret;
+ nm_sr_request_t *p_request = info->recv_completed.p_request;
+ MPID_Request *req = NULL;
+ void *ref;
+ nm_tag_t match_info = 0;
+ MPIR_Context_id_t ctxt;
+ size_t size;
- ret = nm_sr_recv_success(mpid_nem_newmad_pcore, &p_out_req);
- if (p_out_req != NULL)
+ nm_sr_get_size(mpid_nem_newmad_pcore, p_request, &size);
+ nm_sr_get_tag(mpid_nem_newmad_pcore, p_request, &match_info);
+ nm_sr_get_ref(mpid_nem_newmad_pcore, p_request, &ref);
+ req = (MPID_Request *)ref;
+ MPIU_Assert(req != NULL);
+#ifdef DEBUG
+ fprintf(stdout,"========> Completing Recv req %p (match is %lx) \n",req,match_info);
+#endif
+ NEM_NMAD_MATCH_GET_CTXT(match_info, ctxt);
+ if(ctxt == NEM_NMAD_INTRA_CTXT)
{
- nm_tag_t match_info = 0;
- MPIR_Context_id_t ctxt;
- size_t size;
- MPID_Request *req;
- void *ref;
- nm_sr_get_size(mpid_nem_newmad_pcore, p_out_req, &size);
- nm_sr_get_tag(mpid_nem_newmad_pcore, p_out_req, &match_info);
- nm_sr_get_ref(mpid_nem_newmad_pcore,p_out_req,&ref);
- req = (MPID_Request *)ref;
- MPIU_Assert(req != NULL);
-
- NEM_NMAD_MATCH_GET_CTXT(match_info, ctxt);
- if(ctxt == NEM_NMAD_INTRA_CTXT)
+ if (req->kind == MPID_REQUEST_RECV)
{
- if (req->kind == MPID_REQUEST_RECV)
+ if (size <= sizeof(MPIDI_CH3_PktGeneric_t))
{
- if (size <= sizeof(MPIDI_CH3_PktGeneric_t))
- {
- MPID_nem_handle_pkt(req->ch.vc,(char *)&(req->dev.pending_pkt),(MPIDI_msg_sz_t)(size));
- }
- else
- {
- MPID_nem_handle_pkt(req->ch.vc,(char *)(req->dev.tmpbuf),(MPIDI_msg_sz_t)(req->dev.tmpbuf_sz));
- MPIU_Free(req->dev.tmpbuf);
- }
- MPIDI_CH3_Request_destroy(req);
+ MPID_nem_handle_pkt(req->ch.vc,(char *)&(req->dev.pending_pkt),(MPIDI_msg_sz_t)(size));
}
else
{
- MPIU_Assert(0);
+ MPID_nem_handle_pkt(req->ch.vc,(char *)(req->dev.tmpbuf),(MPIDI_msg_sz_t)(req->dev.tmpbuf_sz));
+ MPIU_Free(req->dev.tmpbuf);
}
+ MPIDI_CH3_Request_destroy(req);
}
else
{
- if ((req->kind == MPID_REQUEST_RECV) || (req->kind == MPID_PREQUEST_RECV))
- {
- int found = FALSE;
- nm_sr_request_t *nmad_request = NULL;
- MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
- MPIU_THREAD_CS_ENTER(MSGQUEUE,req);
- MPID_NEM_NMAD_GET_REQ_FROM_HASH(req,nmad_request);
- found = MPIDI_CH3U_Recvq_DP(req);
- if(found){
- MPID_nem_newmad_handle_rreq(req,nmad_request,match_info,size);
- }
- if(nmad_request != NULL)
- {
- MPIU_Assert(req->dev.match.parts.rank == MPI_ANY_SOURCE);
- MPIU_Free(nmad_request);
- }
- MPIU_THREAD_CS_EXIT(MSGQUEUE,req);
+ MPIU_Assert(0);
+ }
+ }
+ else
+ {
+ if ((req->kind == MPID_REQUEST_RECV) || (req->kind == MPID_PREQUEST_RECV))
+ {
+ int found = FALSE;
+ nm_sr_request_t *nmad_request = NULL;
+ MPIU_Assert(MPIDI_Request_get_type(req) != MPIDI_REQUEST_TYPE_GET_RESP);
+ MPIU_THREAD_CS_ENTER(MSGQUEUE,req);
+ MPID_NEM_NMAD_GET_REQ_FROM_HASH(req,nmad_request);
+ found = MPIDI_CH3U_Recvq_DP(req);
+ if(found){
+ MPID_nem_newmad_handle_rreq(req,nmad_request,match_info,size);
}
- else
+ if(nmad_request != NULL)
{
- MPIU_Assert(0);
+ MPIU_Assert(req->dev.match.parts.rank == MPI_ANY_SOURCE);
+ MPIU_Free(nmad_request);
}
+ MPIU_THREAD_CS_EXIT(MSGQUEUE,req);
}
+ else
+ {
+ MPIU_Assert(0);
+ }
}
- fn_exit:
- return mpi_errno;
- fn_fail: ATTRIBUTE((unused))
- goto fn_exit;
}
#undef FUNCNAME
@@ -218,21 +229,8 @@
{
int mpi_errno = MPI_SUCCESS;
- MPID_nem_newmad_poll_recv();
+ nm_schedule(mpid_nem_newmad_pcore);
- /*
- if (in_or_out)
- {
- MPID_nem_newmad_poll_recv();
- MPID_nem_newmad_poll_send();
- }
- else
- {
- MPID_nem_newmad_poll_send();
- MPID_nem_newmad_poll_recv();
- }
- */
-
fn_exit:
return mpi_errno;
fn_fail: ATTRIBUTE((unused))
@@ -257,7 +255,9 @@
nm_sr_get_ref(mpid_nem_newmad_pcore,p_request,&ref);
req = (MPID_Request *)ref;
MPIU_Assert(req != NULL);
-
+#ifdef DEBUG
+ fprintf(stdout,"========> Completing Send req %p (match is %lx) \n",req,match_info);
+#endif
NEM_NMAD_MATCH_GET_CTXT(match_info, ctxt);
if(ctxt != NEM_NMAD_INTRA_CTXT)
{
@@ -345,6 +345,7 @@
MPIU_ERR_SETSIMPLE(req->status.MPI_ERROR,MPI_ERR_TYPE,"**dtypemismatch");
}
}
+ MPIU_Free(REQ_FIELD(req,iov));
}
MPIDI_Comm_get_vc(req->comm, req->status.MPI_SOURCE, &vc);
@@ -478,6 +479,14 @@
iov = MPIU_Malloc(iov_num_ub*sizeof(MPID_IOV));
MPID_Segment_unpack_vector(rreq->dev.segment_ptr, rreq->dev.segment_first, &last,iov, &n_iov);
MPIU_Assert(last == rreq->dev.segment_size);
+#ifdef DEBUG
+ for(index = 0; index < n_iov ; index++)
+ {
+ fprintf(stdout,"======================\n");
+ fprintf(stdout,"RECV iov[%i]: [base %p][len %i]\n",index,
+ iov[index].MPID_IOV_BUF,iov[index].MPID_IOV_LEN);
+ }
+#endif
if(n_iov <= NMAD_IOV_MAX_DEPTH)
{
Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_send.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_send.c 2009-03-30 07:00:07 UTC (rev 4224)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/nemesis/netmod/newmad/newmad_send.c 2009-03-30 15:37:41 UTC (rev 4225)
@@ -26,6 +26,9 @@
MPIDI_DBG_Print_packet((MPIDI_CH3_Pkt_t *)hdr);
NEM_NMAD_ADI_MATCH(match_info);
+
+ fprintf(stdout,"iSendContig ========> Sending ADI msg for req %p (match is %lx) \n",sreq,match_info);
+
memcpy(&(sreq->dev.pending_pkt),(char *)hdr,sizeof(MPIDI_CH3_PktGeneric_t));
mad_iov[0].iov_base = (char *)&(sreq->dev.pending_pkt);
mad_iov[0].iov_len = sizeof(MPIDI_CH3_PktGeneric_t);
@@ -74,6 +77,10 @@
sreq->dev.OnDataAvail = 0;
NEM_NMAD_ADI_MATCH(match_info);
+#ifdef DEBUG
+ fprintf(stdout,"StartcontigMsg ========> Sending ADI msg for req %p (match is %lx) \n",sreq,match_info);
+#endif
+
memcpy(&(sreq->dev.pending_pkt),(char *)hdr,sizeof(MPIDI_CH3_PktGeneric_t));
mad_iov[0].iov_base = (char *)&(sreq->dev.pending_pkt);
mad_iov[0].iov_len = sizeof(MPIDI_CH3_PktGeneric_t);
@@ -128,6 +135,9 @@
sreq->dev.user_count,data_sz, &mad_iov_ptr,&num_iov,1);
}
NEM_NMAD_ADI_MATCH(match_info);
+#ifdef DEBUG
+ fprintf(stdout,"SendNonContig ========> Sending ADI msg for req %p (match is %lx) \n",sreq,match_info);
+#endif
memcpy(&(sreq->dev.pending_pkt),(char *)header,sizeof(MPIDI_CH3_PktGeneric_t));
mad_iov[0].iov_base = (char *)&(sreq->dev.pending_pkt);
mad_iov[0].iov_len = sizeof(MPIDI_CH3_PktGeneric_t);
@@ -160,6 +170,7 @@
int dt_contig;
MPIDI_msg_sz_t data_sz;
MPI_Aint dt_true_lb;
+ int index;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_NEWMAD_DIRECTSEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_NEWMAD_DIRECTSEND);
@@ -174,7 +185,9 @@
sreq->ch.vc = vc;
sreq->dev.OnDataAvail = NULL;
NEM_NMAD_DIRECT_MATCH(match_info,tag,comm->rank,comm->context_id + context_offset);
-
+#ifdef DEBUG
+ fprintf(stdout,"Direct Send ========> Sending NON ADI msg for req %p (match is %lx) \n",sreq,match_info);
+#endif
if(data_sz)
{
if (dt_contig)
@@ -189,6 +202,14 @@
MPID_nem_newmad_process_sdtype(&sreq,datatype,dt_ptr,buf,count,data_sz,&mad_iov_ptr,&num_iov,0);
}
MPIU_Assert(num_iov <= NMAD_IOV_MAX_DEPTH);
+#ifdef DEBUG
+ for(index = 0; index < num_iov ; index++)
+ {
+ fprintf(stdout,"======================\n");
+ fprintf(stdout,"SEND nmad_iov[%i]: [base %p][len %i]\n",index,
+ mad_iov[index].iov_base,mad_iov[index].iov_len);
+ }
+#endif
nm_sr_isend_iov_with_ref(mpid_nem_newmad_pcore, VC_FIELD(vc, p_gate), match_info,
mad_iov, num_iov, &(REQ_FIELD(sreq,newmad_req)),(void*)sreq);
}
@@ -302,6 +323,15 @@
MPID_Segment_pack_vector(sreq->dev.segment_ptr, sreq->dev.segment_first, &last,iov, &n_iov);
MPIU_Assert(last == sreq->dev.segment_size);
+#ifdef DEBUG
+ for(index = 0; index < n_iov ; index++)
+ {
+ fprintf(stdout,"======================\n");
+ fprintf(stdout,"SEND: iov[%i]: [base %p][len %i]\n",index,
+ iov[index].MPID_IOV_BUF,iov[index].MPID_IOV_LEN);
+ }
+#endif
+
if(n_iov <= num_entries)
{
for(index = 0; index < n_iov ; index++)
More information about the mpich2-commits
mailing list