[mpich2-commits] r5506 - in mpich2/trunk/src/mpid/ch3/channels/nemesis: include src

[goodell at mcs.anl.gov]

Author: goodell
Date: 2009-10-20 15:35:41 -0500 (Tue, 20 Oct 2009)
New Revision: 5506

Modified:
   mpich2/trunk/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h
   mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
Log:
Nemesis SendQ fixes for multithreading.

This change adds debug logging to the SendQ enqueue/dequeue macros.  It
also adds reference counting in those macros.  The reference counting is
needed to avoid a race between deallocation and dequeuing in the case
where an OnDataAvail function completes and frees the request before we
know if we should dequeue it from the queue.

No reviewer.

Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h	2009-10-20 20:35:39 UTC (rev 5505)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/include/mpidi_ch3_impl.h	2009-10-20 20:35:41 UTC (rev 5506)
@@ -28,9 +28,19 @@
 extern struct MPID_Request *MPIDI_CH3I_active_send[CH3_NUM_QUEUES];
 
 #define MPIDI_CH3I_SendQ_enqueue(req, queue)					\
-{										\
+do {										\
+    MPIU_Assert(req != NULL);                                                   \
     /* MT - not thread safe! */							\
     MPIDI_DBG_PRINTF((50, FCNAME, "SendQ_enqueue req=0x%08x", req->handle));	\
+    MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST,                     \
+                      "MPIDI_CH3I_SendQ_enqueue(req=0x%x (handle=0x%x), queue=%s (%d))", \
+                      (req),                                                    \
+                      (req)->handle,                                            \
+                      #queue, queue));                                          \
+    /* because an OnDataAvail function might complete this request and cause */ \
+    /* it to be freed before it is dequeued, we have to add a reference */      \
+    /* whenever a req is added to a queue */                                    \
+    MPIR_Request_add_ref(req);                                                  \
     req->dev.next = NULL;							\
     if (MPIDI_CH3I_sendq_tail[queue] != NULL)					\
     {										\
@@ -41,19 +51,30 @@
 	MPIDI_CH3I_sendq_head[queue] = req;					\
     }										\
     MPIDI_CH3I_sendq_tail[queue] = req;						\
-}
+} while (0)
 
+/* NOTE: this macro may result in the dequeued request being freed (via
+ * MPID_Request_release) */
 #define MPIDI_CH3I_SendQ_dequeue(queue)						\
-{										\
+do {										\
+    MPID_Request *req_;                                                         \
     /* MT - not thread safe! */							\
     MPIDI_DBG_PRINTF((50, FCNAME, "SendQ_dequeue req=0x%08x",			\
                       MPIDI_CH3I_sendq_head[queue]->handle));			\
+    MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE, (MPIU_DBG_FDEST,                     \
+                      "MPIDI_CH3I_SendQ_dequeue(queue=%s (%d)), head req=0x%x (handle=0x%x)", \
+                      #queue, queue,                                            \
+                      MPIDI_CH3I_sendq_head[queue],                             \
+                      ((MPIDI_CH3I_sendq_head[queue]) ? MPIDI_CH3I_sendq_head[queue]->handle : -1))); \
+    /* see the comment in _enqueue above about refcounts */                     \
+    req_ = MPIDI_CH3I_sendq_head[queue];                                        \
     MPIDI_CH3I_sendq_head[queue] = MPIDI_CH3I_sendq_head[queue]->dev.next;	\
+    MPID_Request_release(req_);                                                 \
     if (MPIDI_CH3I_sendq_head[queue] == NULL)					\
     {										\
 	MPIDI_CH3I_sendq_tail[queue] = NULL;					\
     }										\
-}
+} while (0)
 
 #define MPIDI_CH3I_SendQ_head(queue) (MPIDI_CH3I_sendq_head[queue])
 

Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c	2009-10-20 20:35:39 UTC (rev 5505)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c	2009-10-20 20:35:41 UTC (rev 5506)
@@ -340,8 +340,9 @@
                 MPIU_Assert(MPIDI_Request_get_type(sreq) != MPIDI_REQUEST_TYPE_GET_RESP);
                 MPIDI_CH3U_Request_complete(sreq);
 
+                /* MT - clear the current active send before dequeuing/destroying the current request */
+                MPIDI_CH3I_active_send[CH3_NORMAL_QUEUE] = NULL;
                 MPIDI_CH3I_SendQ_dequeue(CH3_NORMAL_QUEUE);
-                MPIDI_CH3I_active_send[CH3_NORMAL_QUEUE] = NULL;
                 MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
             }
             else