[mpich2-commits] r7683 - in mpich2/trunk/src: mpi/errhan mpid/ch3/channels/nemesis/src mpid/ch3/include mpid/ch3/src
buntinas at mcs.anl.gov
buntinas at mcs.anl.gov
Tue Jan 11 11:35:46 CST 2011
Author: buntinas
Date: 2011-01-11 11:35:45 -0600 (Tue, 11 Jan 2011)
New Revision: 7683
Modified:
mpich2/trunk/src/mpi/errhan/errnames.txt
mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
mpich2/trunk/src/mpid/ch3/include/mpidimpl.h
mpich2/trunk/src/mpid/ch3/src/ch3u_handle_connection.c
Log:
Handle process failure notifications from hydra.
Modified: mpich2/trunk/src/mpi/errhan/errnames.txt
===================================================================
--- mpich2/trunk/src/mpi/errhan/errnames.txt 2011-01-11 17:34:45 UTC (rev 7682)
+++ mpich2/trunk/src/mpi/errhan/errnames.txt 2011-01-11 17:35:45 UTC (rev 7683)
@@ -883,6 +883,7 @@
**signal:signal() failed
**signal %s:signal() failed: %s
+**sigusr1:This version of MPICH requires the SIGUSR1 signal, but the application has already installed a handler
#
# mpi functions
Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c 2011-01-11 17:34:45 UTC (rev 7682)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c 2011-01-11 17:35:45 UTC (rev 7683)
@@ -11,6 +11,9 @@
#if defined (MPID_NEM_INLINE) && MPID_NEM_INLINE
#include "mpid_nem_inline.h"
#endif
+#ifdef HAVE_SIGNAL_H
+#include <signal.h>
+#endif
#define PKTARRAY_SIZE (MPIDI_NEM_PKT_END+1)
@@ -46,6 +49,9 @@
#endif /* MPICH_IS_THREADED */
/* NEMESIS MULTITHREADING - End block*/
+volatile static int sigusr1_count = 0;
+static int my_sigusr1_count = 0;
+
struct MPID_Request *MPIDI_CH3I_sendq_head[CH3_NUM_QUEUES] = {0};
struct MPID_Request *MPIDI_CH3I_sendq_tail[CH3_NUM_QUEUES] = {0};
struct MPID_Request *MPIDI_CH3I_active_send[CH3_NUM_QUEUES] = {0};
@@ -66,6 +72,11 @@
static qn_ent_t *qn_head = NULL;
+static void sigusr1_handler(int sig)
+{
+ ++sigusr1_count;
+}
+
/* MPIDI_CH3I_Shm_send_progress() this function makes progress sending
queued messages on the shared memory queues. This function is
nonblocking and does not call netmod functions..*/
@@ -429,6 +440,12 @@
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
+ if (MPIDI_Sigusr1_count > my_sigusr1_count) {
+ my_sigusr1_count = MPIDI_Sigusr1_count;
+ mpi_errno = MPIDI_CH3U_Check_for_failed_procs();
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ }
+
/* in the case of progress_wait, bail out if anything completed (CC-1) */
if (is_blocking) {
int made_progress = FALSE;
@@ -787,6 +804,18 @@
/* other pkt handlers */
pktArray[MPIDI_NEM_PKT_NETMOD] = pkt_NETMOD_handler;
+#ifdef HAVE_SIGNAL
+ {
+ /* install signal handler for process failure notifications from hydra */
+ void *ret;
+
+ ret = signal(SIGUSR1, &sigusr1_handler);
+ MPIU_ERR_CHKANDJUMP1(ret == SIG_ERR, mpi_errno, MPI_ERR_OTHER, "**signal", "**signal %s", MPIU_Strerror(errno));
+ /* Error if the app set its own SIGUSR1 handler. */
+ MPIU_ERR_CHKANDJUMP(ret != SIG_DFL, mpi_errno, MPI_ERR_OTHER, "**sigusr1");
+ }
+#endif
+
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PROGRESS_INIT);
return mpi_errno;
@@ -826,6 +855,14 @@
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_CONNECTION_TERMINATE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_CONNECTION_TERMINATE);
+
+ MPIU_DBG_MSG_D(CH3_DISCONNECT, TYPICAL, "Terminating VC %d", vc->pg_rank);
+
+ /* if this is already closed, exit */
+ if (vc->state == MPIDI_VC_STATE_MORIBUND ||
+ vc->state == MPIDI_VC_STATE_INACTIVE_CLOSED)
+ goto fn_exit;
+
if (((MPIDI_CH3I_VC *)vc->channel_private)->is_local)
mpi_errno = MPID_nem_vc_terminate(vc);
else
Modified: mpich2/trunk/src/mpid/ch3/include/mpidimpl.h
===================================================================
--- mpich2/trunk/src/mpid/ch3/include/mpidimpl.h 2011-01-11 17:34:45 UTC (rev 7682)
+++ mpich2/trunk/src/mpid/ch3/include/mpidimpl.h 2011-01-11 17:35:45 UTC (rev 7683)
@@ -1586,6 +1586,9 @@
#else
#define MPIDI_CH3_Channel_close( ) MPI_SUCCESS
#endif
+/* MPIDI_CH3U_Check_for_failed_procs() reads PMI_dead_processes key
+ and marks VCs to those processes as failed */
+int MPIDI_CH3U_Check_for_failed_procs(void);
/*@
MPIDI_CH3_Pre_init - Allows the channel to initialize before PMI_init is
Modified: mpich2/trunk/src/mpid/ch3/src/ch3u_handle_connection.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/src/ch3u_handle_connection.c 2011-01-11 17:34:45 UTC (rev 7682)
+++ mpich2/trunk/src/mpid/ch3/src/ch3u_handle_connection.c 2011-01-11 17:35:45 UTC (rev 7683)
@@ -5,6 +5,7 @@
*/
#include "mpidimpl.h"
+#include "pmi.h"
/* Count the number of outstanding close requests */
static volatile int MPIDI_Outstanding_close_ops = 0;
@@ -65,6 +66,18 @@
break;
+ case MPIDI_VC_STATE_INACTIVE:
+ /* VC was terminated before it was activated.
+ This can happen if a failed process was
+ detected before the process used the VC. */
+ MPIU_DBG_MSG(CH3_DISCONNECT,TYPICAL, "VC terminated before it was activated. We probably got a failed"
+ "process notification.");
+ ++MPIDI_Failed_vc_count;
+ MPIDI_CHANGE_VC_STATE(vc, MORIBUND);
+
+ break;
+
+
case MPIDI_VC_STATE_ACTIVE:
case MPIDI_VC_STATE_REMOTE_CLOSE:
/* This is a premature termination. This process
@@ -125,7 +138,7 @@
mpi_errno = MPIR_Err_create_code(
MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__,
MPI_ERR_INTERN, "**ch3|unhandled_connection_state",
- "**ch3|unhandled_connection_state %p %d", vc, event);
+ "**ch3|unhandled_connection_state %p %d", vc, vc->state);
goto fn_fail;
break;
}
@@ -373,3 +386,76 @@
return mpi_errno;
}
+#define parse_rank(r_p) do { \
+ while (isspace(*c)) /* skip spaces */ \
+ ++c; \
+ MPIU_ERR_CHKINTERNAL(!isdigit(*c), mpi_errno, "error parsing failed process list"); \
+ *(r_p) = strtol(c, &c, 0); \
+ while (isspace(*c)) /* skip spaces */ \
+ ++c; \
+ } while (0)
+
+#undef FUNCNAME
+#define FUNCNAME MPIDI_CH3U_Check_for_failed_procs
+#undef FCNAME
+#define FCNAME MPIU_QUOTE(FUNCNAME)
+int MPIDI_CH3U_Check_for_failed_procs(void)
+{
+ int mpi_errno = MPI_SUCCESS;
+ int pmi_errno;
+ char *val;
+ char *c;
+ int len;
+ char *kvsname;
+ int rank, rank_hi;
+ MPIU_CHKLMEM_DECL(1);
+ MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_CHECK_FOR_FAILED_PROCS);
+
+ MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_CHECK_FOR_FAILED_PROCS);
+ mpi_errno = MPIDI_PG_GetConnKVSname(&kvsname);
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ pmi_errno = PMI_KVS_Get_value_length_max(&len);
+ MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get_value_length_max");
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ MPIU_CHKLMEM_MALLOC(val, char *, len, mpi_errno, "val");
+ pmi_errno = PMI_KVS_Get(kvsname, "PMI_dead_processes", val, len);
+ MPIU_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get");
+
+ MPIU_DBG_MSG_S(CH3_DISCONNECT, TYPICAL, "Received proc fail notification: %s", val);
+
+ if (*val == '\0')
+ /* there are no failed processes */
+ goto fn_exit;
+
+ /* parse list of failed processes. This is a comma separated list
+ of ranks or ranges of ranks (e.g., "1, 3-5, 11") */
+ c = val;
+ while(1) {
+ parse_rank(&rank);
+ if (*c == '-') {
+ ++c; /* skip '-' */
+ parse_rank(&rank_hi);
+ } else
+ rank_hi = rank;
+ while (rank <= rank_hi) {
+ MPIDI_VC_t *vc;
+ MPIDI_PG_Get_vc(MPIDI_Process.my_pg, rank, &vc);
+ mpi_errno = MPIU_CALL(MPIDI_CH3,Connection_terminate(vc));
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
+ ++rank;
+ }
+ MPIU_ERR_CHKINTERNAL(*c != ',' && *c != '\0', mpi_errno, "error parsing failed process list");
+ if (*c == '\0')
+ break;
+ ++c; /* skip ',' */
+ }
+
+ fn_exit:
+ MPIU_CHKLMEM_FREEALL();
+ MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_CHECK_FOR_FAILED_PROCS);
+ return mpi_errno;
+ fn_fail:
+ goto fn_exit;
+}
+
+
More information about the mpich2-commits
mailing list