[mpich2-commits] r6664 - in mpich2/trunk: . confdb src/include src/include/thread src/mpi/errhan src/mpi/init src/mpid/ch3/channels/nemesis/src src/mpid/ch3/channels/sctp/src src/mpid/ch3/channels/sock/src src/mpid/ch3/include src/mpid/common/sock/iocp src/mpid/common/sock/poll src/mpid/common/thread src/mpid/dcmfd/include src/util/dbg
goodell at mcs.anl.gov
goodell at mcs.anl.gov
Mon May 17 17:47:53 CDT 2010
Author: goodell
Date: 2010-05-17 17:47:53 -0500 (Mon, 17 May 2010)
New Revision: 6664
Added:
mpich2/trunk/confdb/ax_tls.m4
Modified:
mpich2/trunk/confdb/aclocal.m4
mpich2/trunk/configure.in
mpich2/trunk/src/include/mpiimpl.h
mpich2/trunk/src/include/mpiimplthread.h
mpich2/trunk/src/include/thread/mpiu_thread_posix_funcs.h
mpich2/trunk/src/include/thread/mpiu_thread_solaris_funcs.h
mpich2/trunk/src/mpi/errhan/errutil.c
mpich2/trunk/src/mpi/init/async.c
mpich2/trunk/src/mpi/init/finalize.c
mpich2/trunk/src/mpi/init/init.c
mpich2/trunk/src/mpi/init/initthread.c
mpich2/trunk/src/mpi/init/mpi_init.h
mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
mpich2/trunk/src/mpid/ch3/channels/sctp/src/ch3_progress.c
mpich2/trunk/src/mpid/ch3/channels/sock/src/ch3_progress.c
mpich2/trunk/src/mpid/ch3/include/mpidimpl.h
mpich2/trunk/src/mpid/common/sock/iocp/sock.c
mpich2/trunk/src/mpid/common/sock/poll/sock_wait.i
mpich2/trunk/src/mpid/common/thread/mpe_funcs.i
mpich2/trunk/src/mpid/dcmfd/include/mpidthread.h
mpich2/trunk/src/util/dbg/dbg_printf.c
Log:
overhaul MPICH2's threading macros
This is actually several logical changes, but they are very difficult to
separate into individual commits because of how tangled the old macros
were. Some of the changes are:
- general macro cleanup: do{}while(0) and whitespace issues
- add support for compiler-assisted thread-local storage (TLS)
- make MPID_Thread (e.g. pthread) TLS macros easier to understand and use
- drop non-portable parentheses from MPIU_ISTHREADED macro
- eliminate the MPID_CS_* family of macros which were simultaneously
complementary to and in conflict with the newer MPIU_THREAD_CS_*
macros
No reviewer.
Modified: mpich2/trunk/confdb/aclocal.m4
===================================================================
--- mpich2/trunk/confdb/aclocal.m4 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/confdb/aclocal.m4 2010-05-17 22:47:53 UTC (rev 6664)
@@ -16,6 +16,7 @@
builtin(include,fortran90.m4)
builtin(include,aclocal_libs.m4)
builtin(include,aclocal_attr_alias.m4)
+builtin(include,ax_tls.m4)
dnl Add the libtool files that libtoolize wants
dnl Comment these out until libtool support is enabled.
dnl May need to change this anyway, since libtoolize
Added: mpich2/trunk/confdb/ax_tls.m4
===================================================================
--- mpich2/trunk/confdb/ax_tls.m4 (rev 0)
+++ mpich2/trunk/confdb/ax_tls.m4 2010-05-17 22:47:53 UTC (rev 6664)
@@ -0,0 +1,74 @@
+# ===========================================================================
+# http://www.nongnu.org/autoconf-archive/ax_tls.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+# AX_TLS
+#
+# DESCRIPTION
+#
+# Provides a test for the compiler support of thread local storage (TLS)
+# extensions. Defines TLS if it is found. Currently only knows about GCC
+# and MSVC. I think SunPro uses the same as GCC, and Borland apparently
+# supports either.
+#
+# LICENSE
+#
+# Copyright (c) 2008 Alan Woodland <ajw05 at aber.ac.uk>
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# As a special exception, the respective Autoconf Macro's copyright owner
+# gives unlimited permission to copy, distribute and modify the configure
+# scripts that are the output of Autoconf when processing the Macro. You
+# need not follow the terms of the GNU General Public License when using
+# or distributing such scripts, even though portions of the text of the
+# Macro appear in them. The GNU General Public License (GPL) does govern
+# all other use of the material that constitutes the Autoconf Macro.
+#
+# This special exception to the GPL applies to versions of the Autoconf
+# Macro released by the Autoconf Archive. When you make and distribute a
+# modified version of the Autoconf Macro, you may extend this special
+# exception to the GPL to apply to your modified version as well.
+
+AC_DEFUN([AX_TLS], [
+ AC_MSG_CHECKING(for thread local storage specifier)
+ AC_CACHE_VAL(ac_cv_tls, [
+ ax_tls_keywords="__thread __declspec(thread) none"
+ for ax_tls_keyword in $ax_tls_keywords; do
+ case $ax_tls_keyword in
+ none) ac_cv_tls=none ; break ;;
+ *)
+ # MPICH2 modification: This was an AC_TRY_COMPILE before, but
+ # Darwin with non-standard compilers will accept __thread at
+ # compile time but fail to link due to an undefined
+ # "__emutls_get_address" symbol unless -lgcc_eh is added to the
+ # link line.
+ AC_LINK_IFELSE(
+ [AC_LANG_PROGRAM([$ax_tls_keyword int bar = 5;],[++bar;])],
+ [ac_cv_tls=$ax_tls_keyword ; break],
+ [ac_cv_tls=none])
+ esac
+ done
+])
+
+ if test "$ac_cv_tls" != "none"; then
+ # MPICH2 modification: this was "TLS" before instead of
+ # "MPIU_TLS_SPECIFIER", but TLS had a reasonably high chance of conflicting
+ # with a system library.
+ AC_DEFINE_UNQUOTED([MPIU_TLS_SPECIFIER], $ac_cv_tls, [If the compiler supports a TLS storage class define it to that here])
+ fi
+ AC_MSG_RESULT($ac_cv_tls)
+])
Modified: mpich2/trunk/configure.in
===================================================================
--- mpich2/trunk/configure.in 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/configure.in 2010-05-17 22:47:53 UTC (rev 6664)
@@ -473,14 +473,12 @@
single - No threads (MPI_THREAD_SINGLE)
funneled - Only the main thread calls MPI (MPI_THREAD_FUNNELED)
serialized - User serializes calls to MPI (MPI_THREAD_SERIALIZED)
- multiple(:impl) - Fully multi-threaded (MPI_THREAD_MULTIPLE). DO NOT
+ multiple - Fully multi-threaded (MPI_THREAD_MULTIPLE). DO NOT
select this option. The default option is more
efficient and also supports thread_multiple.
- The following implementations are supported.
- global_mutex - a single global lock guards access to all
- MPI functions.
- The default implementation is global_mutex.
+ See also the --enable-thread-cs option for controlling the granularity of
+ the concurrency inside of the library.
For the ch3:sock and ch3:nemesis channels, a separate build is no
longer needed for thread-multiple. It is compiled by default and
@@ -1041,46 +1039,27 @@
if test "$enable_threads" = "runtime" ; then
AC_DEFINE(HAVE_RUNTIME_THREADCHECK,1,[Define if MPI supports MPI_THREAD_MULTIPLE with a runtime check for thread level])
enable_threads=multiple
- # FIXME: This doesn't support runtime:thread-impl (as in multiple:thread-impl)
fi
MPICH_THREAD_LEVEL=MPI_THREAD_FUNNELED
case "$enable_threads" in
single)
thread_pkg_required=no
- thread_impl=none
MPICH_THREAD_LEVEL=MPI_THREAD_SINGLE
;;
funneled)
thread_pkg_required=no
- thread_impl=none
MPICH_THREAD_LEVEL=MPI_THREAD_FUNNELED
;;
serialized)
# FIXME: Why does serialized require a thread package?
thread_pkg_required=yes
- thread_impl=none
MPICH_THREAD_LEVEL=MPI_THREAD_SERIALIZED
;;
multiple)
thread_pkg_required=yes
- thread_impl=global_mutex
MPICH_THREAD_LEVEL=MPI_THREAD_MULTIPLE
;;
- multiple:*)
- thread_pkg_required=yes
- thread_impl="`echo $enable_threads | sed -e 's/.*://'`"
- MPICH_THREAD_LEVEL=MPI_THREAD_MULTIPLE
- case "$thread_impl" in
- global_mutex)
- ;;
- # global_monitor)
- # ;;
- *)
- AC_MSG_ERROR(["$thread_impl" is not a valid multi-thread implementation for MPICH2])
- ;;
- esac
- ;;
*)
AC_MSG_ERROR(["$enable_threads" is not a valid value for --enable-threads])
;;
@@ -1122,10 +1101,6 @@
export MPICH_THREAD_LEVEL
AC_DEFINE_UNQUOTED(MPICH_THREAD_LEVEL,$MPICH_THREAD_LEVEL,[Level of thread support selected at compile time])
-MPICH_THREAD_IMPL="MPICH_THREAD_IMPL_`echo $thread_impl | tr 'abcdefghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'`"
-export MPICH_THREAD_IMPL
-AC_DEFINE_UNQUOTED(USE_THREAD_IMPL,$MPICH_THREAD_IMPL,[Multi-threaded implementation selected at compile time])
-
if test "$thread_pkg_required" = "no" ; then
MPIU_THREAD_DEFAULT=${MPIU_THREAD_DEFAULT:-none}
fi
@@ -5881,6 +5856,9 @@
export MPICH2_THREAD_PACKAGE
AC_DEFINE_UNQUOTED([MPIU_THREAD_PACKAGE_NAME],[$MPIU_THREAD_PACKAGE_NAME],[set to the name of the thread package])
+# check for compiler-support for thread-local storage (MPIU_TLS_SPECIFIER)
+AX_TLS
+
AC_CHECK_HEADERS(sched.h)
AC_CHECK_FUNCS(sched_yield yield usleep sleep select getpid)
Modified: mpich2/trunk/src/include/mpiimpl.h
===================================================================
--- mpich2/trunk/src/include/mpiimpl.h 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/include/mpiimpl.h 2010-05-17 22:47:53 UTC (rev 6664)
@@ -1767,131 +1767,7 @@
/* end of mpitopo.h (in src/mpi/topo? */
/* ------------------------------------------------------------------------- */
-/* Time stamps */
-/* Get the timer definitions. The source file for this include is
- src/mpi/timer/mpichtimer.h.in */
-#include "mpichtimer.h"
-typedef struct MPID_Stateinfo_t {
- MPID_Time_t stamp;
- int count;
-} MPID_Stateinfo_t;
-#define MPICH_MAX_STATES 512
-/* Timer state routines (src/util/instrm/states.c) */
-void MPID_TimerStateBegin( int, MPID_Time_t * );
-void MPID_TimerStateEnd( int, MPID_Time_t * );
-
-/* ------------------------------------------------------------------------- */
-/* Thread types */
-/* Temporary; this will include "mpichthread.h" eventually */
-
-#ifdef MPICH_DEBUG_NESTING
-#define MPICH_MAX_NESTFILENAME 256
-typedef struct MPICH_Nestinfo {
- char file[MPICH_MAX_NESTFILENAME];
- int line;
-} MPICH_Nestinfo_t;
-#define MPICH_MAX_NESTINFO 16
-#endif /* MPICH_DEBUG_NESTING */
-
-/* arbitrary, just needed to avoid cleaning up heap allocated memory at thread
- * destruction time */
-#define MPIU_STRERROR_BUF_SIZE (1024)
-
-typedef struct MPICH_PerThread_t {
- int nest_count; /* For layered MPI implementation */
- int op_errno; /* For errors in predefined MPI_Ops */
-
- /* error string storage for MPIU_Strerror */
- char strerrbuf[MPIU_STRERROR_BUF_SIZE];
-
-#ifdef MPICH_DEBUG_NESTING
- MPICH_Nestinfo_t nestinfo[MPICH_MAX_NESTINFO];
-#endif
- /* FIXME: Is this used anywhere? */
-#ifdef HAVE_TIMING
- MPID_Stateinfo_t timestamps[MPICH_MAX_STATES]; /* per thread state info */
-#endif
-#if defined(MPID_DEV_PERTHREAD_DECL)
- MPID_DEV_PERTHREAD_DECL
-#endif
-} MPICH_PerThread_t;
-
-#if !defined(MPICH_IS_THREADED)
-/* If single threaded, make this point at a pre-allocated segment.
- This structure is allocated in src/mpi/init/initthread.c */
-extern MPICH_PerThread_t MPIR_Thread;
-
-/* The following three macros define a way to portably access thread-private
- storage in MPICH2, and avoid extra overhead when MPICH2 is single
- threaded
- INITKEY - Create the key. Must happen *before* the other threads
- are created
- INIT - Create the thread-private storage. Must happen once per thread
- DECL - Declare local variables
- GET - Access the thread-private storage
- FIELD - Access the thread-private field (by name)
-
- The "DECL" is the extern so that there is always a statement for
- the declaration.
-*/
-#define MPIU_THREADPRIV_INITKEY
-#define MPIU_THREADPRIV_INIT
-/* Empty declarations are not allowed in C. However multiple decls are allowed */
-#define MPIU_THREADPRIV_DECL extern MPICH_PerThread_t MPIR_Thread
-#define MPIU_THREADPRIV_GET
-#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread._a)
-
-#elif defined(HAVE_RUNTIME_THREADCHECK)
-/* In the case where the thread level is set in MPI_Init_thread, we
- need a blended version of the non-threaded and the thread-multiple
- definitions.
-
- The approach is to have TWO MPICH_PerThread_t pointers. One is local
- (The MPIU_THREADPRIV_DECL is used in the routines local definitions),
- as in the threaded version of these macros. This is set by using a routine
- to get thread-private storage. The second is a preallocated, extern
- MPICH_PerThread_t struct, as in the single threaded case. Based on
- MPIR_Process.isThreaded, one or the other is used.
-
- */
-/* For the single threaded case, we use a preallocated structure
- This structure is allocated in src/mpi/init/initthread.c */
-extern MPICH_PerThread_t MPIR_ThreadSingle;
-
-/* We need to provide a function that will cleanup the storage attached
- to the key. */
-#define MPIU_THREADPRIV_INITKEY \
- {if (MPIR_Process.isThreaded) {\
- MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_Process.thread_storage,NULL);}}
-#define MPIU_THREADPRIV_INIT {if (MPIR_Process.isThreaded) {\
- MPICH_PerThread_t *(pt_) = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t)); \
- MPID_Thread_tls_set(&MPIR_Process.thread_storage, (void *) (pt_)); \
- }}
-#define MPIU_THREADPRIV_DECL \
- MPICH_PerThread_t *MPIR_Thread=0
-#define MPIU_THREADPRIV_GET \
- {if (!MPIR_Thread){MPIR_GetPerThread( &MPIR_Thread );}}
-#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread->_a)
-
-#else /* Thread multiple */
-/* The following three macros define a way to portably access thread-private
- storage in MPICH2, and avoid extra overhead when MPICH2 is single
- threaded. We initialize the MPIR_Thread pointer to null so that
- we need call the routine to get the thread-private storage only once
- in an invocation of a routine. */
-
-#define MPIU_THREADPRIV_INITKEY \
- MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_Process.thread_storage,NULL)
-#define MPIU_THREADPRIV_INIT {\
- MPICH_PerThread_t *(pt_) = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t)); \
- MPID_Thread_tls_set(&MPIR_Process.thread_storage, (void *) (pt_)); \
- }
-#define MPIU_THREADPRIV_DECL MPICH_PerThread_t *MPIR_Thread=0
-#define MPIU_THREADPRIV_GET {if (!MPIR_Thread)MPIR_GetPerThread( &MPIR_Thread );}
-#define MPIU_THREADPRIV_FIELD(_a) (MPIR_Thread->_a)
-#endif
-
/* Per process data */
typedef enum MPIR_MPI_State_t { MPICH_PRE_INIT=0, MPICH_WITHIN_MPI=1,
MPICH_POST_FINALIZED=2 } MPIR_MPI_State_t;
@@ -1947,40 +1823,6 @@
} MPICH_PerProcess_t;
extern MPICH_PerProcess_t MPIR_Process;
-/*D
- MPICH_THREAD_LEVEL - Indicates the maximum level of thread
- support provided at compile time.
-
- Values:
- Any of the 'MPI_THREAD_xxx' values (these are preprocessor-time constants)
-
- Notes:
- The macro 'MPICH_THREAD_LEVEL' defines the maximum level of
- thread support provided, and may be used at compile time to remove
- thread locks and other code needed only in a multithreaded environment.
-
- A typical use is
-.vb
- #ifdef MPICH_IS_THREADED
- lock((r)->lock_ptr);
- (r)->ref_count++;
- unlock((r)->lock_ptr);
- #else
- (r)->ref_count ++;
- #fi
-.ve
-
- Note that 'MPICH_IS_THREADED' is defined as 1 if
-.vb
- MPICH_THREAD_LEVEL >= MPI_THREAD_MULTIPLE
-.ve
- is true. The test should be used only for special cases (such as
- handling 'SERIALIZED').
-
- Module:
- Environment-DS
- D*/
-
/* ------------------------------------------------------------------------- */
/* In MPICH2, each function has an "enter" and "exit" macro. These can be
* used to add various features to each function at compile time, or they
@@ -2195,6 +2037,9 @@
void MPIR_Nest_decr_export(void);
#ifdef MPICH_DEBUG_NESTING
+/* this nesting is different than the MPIU_THREAD_*DEPTH macros, this is
+ * MPI/NMPI nesting, the other one is critical section nesting */
+
/* These two routines export the versions of the nest macros that
provide the file/line where the nest value changes, also for use in ROMIO */
void MPIR_Nest_incr_export_dbg(const char *, int);
@@ -2202,40 +2047,59 @@
/* FIXME: We should move the initialization and error reporting into
routines that can be called when necessary */
-#define MPIR_Nest_init() {\
- int _i;\
- for (_i=0;_i<MPICH_MAX_NESTINFO;_i++) {\
- MPIU_THREADPRIV_FIELD(nestinfo)[_i].file[0] = 0;\
- MPIU_THREADPRIV_FIELD(nestinfo)[_i].line = 0;}}
-#define MPIR_Nest_incr() {\
- if (MPIU_THREADPRIV_FIELD(nest_count) >= MPICH_MAX_NESTINFO) {\
- MPIU_Internal_error_printf("nest stack exceeded at %s:%d\n",\
- __FILE__,__LINE__);\
- }else{\
- MPIU_Strncpy(MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,__FILE__,\
- MPICH_MAX_NESTFILENAME);\
- MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line=__LINE__;}\
- MPIU_THREADPRIV_FIELD(nest_count)++; }
+#define MPIR_Nest_init() \
+ do { \
+ int i_; \
+ MPICH_Nestinfo_t *nestinfo_ = NULL; \
+ MPIU_THREADPRIV_GET; \
+ nestinfo_ = MPIU_THREADPRIV_FIELD(nestinfo); \
+ for (i_ = 0; i_ <MPICH_MAX_NESTINFO; i_++) { \
+ nestinfo_[i_].file[0] = 0; \
+ nestinfo_[i_].line = 0; \
+ } \
+ } while (0)
+#define MPIR_Nest_incr() \
+ do { \
+ MPICH_Nestinfo_t *nestinfo_ = NULL; \
+ MPIU_THREADPRIV_GET; \
+ nestinfo_ = MPIU_THREADPRIV_FIELD(nestinfo); \
+ if (MPIU_THREADPRIV_FIELD(nest_count) >= MPICH_MAX_NESTINFO) { \
+ MPIU_Internal_error_printf("nest stack exceeded at %s:%d\n", \
+ __FILE__,__LINE__); \
+ } \
+ else { \
+ MPIU_Strncpy(nestinfo_[MPIU_THREADPRIV_FIELD(nest_count)].file, \
+ __FILE__, MPICH_MAX_NESTFILENAME); \
+ nestinfo_[MPIU_THREADPRIV_FIELD(nest_count)].line=__LINE__;} \
+ MPIU_THREADPRIV_FIELD(nest_count)++; \
+ } while (0)
/* Set the line for the current entry to - the old line - this can help
identify increments that did not set the fields */
-#define MPIR_Nest_decr() {\
- if (MPIU_THREADPRIV_FIELD(nest_count) >= 0) {\
- MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line=-__LINE__;} \
- MPIU_THREADPRIV_FIELD(nest_count)--; \
- if (MPIU_THREADPRIV_FIELD(nest_count) < MPICH_MAX_NESTINFO && \
- strcmp(MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,__FILE__) != 0) {\
- MPIU_Msg_printf( "Decremented nest count in file %s:%d but incremented in different file (%s:%d)\n",\
- __FILE__,__LINE__,\
- MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].file,\
- MPIU_THREADPRIV_FIELD(nestinfo)[MPIU_THREADPRIV_FIELD(nest_count)].line);\
- }else if (MPIU_THREADPRIV_FIELD(nest_count) < 0){\
- MPIU_Msg_printf("Decremented nest count in file %s:%d is negative\n",\
- __FILE__,__LINE__);}\
-}
+#define MPIR_Nest_decr() \
+ do { \
+ MPICH_Nestinfo_t *nestinfo_ = NULL; \
+ MPIU_THREADPRIV_GET; \
+ nestinfo_ = MPIU_THREADPRIV_FIELD(nestinfo); \
+ if (MPIU_THREADPRIV_FIELD(nest_count) >= 0) { \
+ nestinfo_[MPIU_THREADPRIV_FIELD(nest_count)].line=-__LINE__; \
+ } \
+ MPIU_THREADPRIV_FIELD(nest_count)--; \
+ if (MPIU_THREADPRIV_FIELD(nest_count) < MPICH_MAX_NESTINFO && \
+ strcmp(nestinfo_[MPIU_THREADPRIV_FIELD(nest_count)].file,__FILE__) != 0) { \
+ MPIU_Msg_printf( "Decremented nest count in file %s:%d but incremented in different file (%s:%d)\n", \
+ __FILE__,__LINE__, \
+ nestinfo_[MPIU_THREADPRIV_FIELD(nest_count)].file, \
+ nestinfo_[MPIU_THREADPRIV_FIELD(nest_count)].line); \
+ } \
+ else if (MPIU_THREADPRIV_FIELD(nest_count) < 0){ \
+ MPIU_Msg_printf("Decremented nest count in file %s:%d is negative\n", \
+ __FILE__,__LINE__); \
+ } \
+ } while (0)
#else
-#define MPIR_Nest_init()
-#define MPIR_Nest_incr() {MPIU_THREADPRIV_FIELD(nest_count)++;}
-#define MPIR_Nest_decr() {MPIU_THREADPRIV_FIELD(nest_count)--;}
+#define MPIR_Nest_init() do { MPIU_THREADPRIV_GET; MPIU_THREADPRIV_FIELD(nest_count) = 0; } while (0)
+#define MPIR_Nest_incr() do { MPIU_THREADPRIV_GET; MPIU_THREADPRIV_FIELD(nest_count)++; } while (0)
+#define MPIR_Nest_decr() do { MPIU_THREADPRIV_GET; MPIU_THREADPRIV_FIELD(nest_count)--; } while (0)
#endif /* MPICH_DEBUG_NESTING */
#define MPIR_Nest_value() (MPIU_THREADPRIV_FIELD(nest_count))
Modified: mpich2/trunk/src/include/mpiimplthread.h
===================================================================
--- mpich2/trunk/src/include/mpiimplthread.h 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/include/mpiimplthread.h 2010-05-17 22:47:53 UTC (rev 6664)
@@ -11,17 +11,26 @@
#error 'This file requires mpichconf.h'
#endif
-#if !defined(MPID_THREAD_DEBUG) && defined(MPICH_DEBUG_MUTEXNESTING)
+#if !defined(MPID_THREAD_DEBUG) && defined(MPICH_DEBUG_MUTEXNESTING)
#define MPID_THREAD_DEBUG 1
#endif
-/* Rather than embed a conditional test in the MPICH2 code, we define a
- single value on which we can test */
+/* Make the CPP definition that will be used to control whether threaded
+ code is supported. Test ONLY on whether MPICH_IS_THREADED is defined.
+
+ Rather than embed a conditional test in the MPICH2 code, we define a
+ single value on which we can test
+ */
+#if !defined(MPICH_THREAD_LEVEL) || !defined(MPI_THREAD_MULTIPLE)
+#error Internal error in macro definitions in include/mpiimplthread.h
+#endif
#if (MPICH_THREAD_LEVEL == MPI_THREAD_MULTIPLE)
#define MPICH_IS_THREADED 1
#endif
-#if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED)
+/* mpid_thread.h is needed at serialized and above because we might need to
+ * store some thread-local data */
+#if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED)
#include "mpid_thread.h"
#endif
@@ -29,33 +38,19 @@
# error Invalid thread granularity option specified (possibly none)
#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_BRIEF_GLOBAL
# error The BRIEF_GLOBAL thread granularity option is no longer supported
+#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_LOCK_FREE
+# error MPIU_THREAD_GRANULARITY_LOCK_FREE not implemented yet
#endif
-/*
- * Define possible thread implementations that could be selected at
- * configure time.
- *
- * These mean what ?
- *
- * not_implemented - used in src/mpid/ch3/channels/{sock,sctp}/src/ch3_progress.c
- * only, and basically commented out
- * none - never used
- * global_mutex -
- *
- */
-/* FIXME: These are old and deprecated */
-/* FIXME XXX DJG: delete these soon */
-#define MPICH_THREAD_IMPL_NOT_IMPLEMENTED -1
-#define MPICH_THREAD_IMPL_NONE 1
-#define MPICH_THREAD_IMPL_GLOBAL_MUTEX 2
-
typedef struct MPICH_ThreadInfo_t {
int thread_provided; /* Provided level of thread support */
/* This is a special case for is_thread_main, which must be
implemented even if MPICH2 itself is single threaded. */
-#if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED)
+#if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED)
+# if !defined(MPIU_TLS_SPECIFIER)
MPID_Thread_tls_t thread_storage; /* Id for perthread data */
+# endif /* !TLS */
MPID_Thread_id_t master_thread; /* Thread that started MPI */
#endif
#ifdef HAVE_RUNTIME_THREADCHECK
@@ -66,243 +61,292 @@
/* Define the mutex values used for each kind of implementation */
#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL || \
MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT
- /* The global mutex goes here when we eliminate USE_THREAD_IMPL */
+ MPID_Thread_mutex_t global_mutex;
/* We need the handle mutex to avoid problems with lock nesting */
MPID_Thread_mutex_t handle_mutex;
-#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_LOCK_FREE
-#error MPIU_THREAD_GRANULARITY_LOCK_FREE not implemented yet
#endif
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT
- MPID_Thread_tls_t nest_storage; /* Id for perthread data */
-#endif
-
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
- MPID_Thread_mutex_t global_mutex;
-# endif
-
-#if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED)
+#if (MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED)
MPID_Thread_mutex_t memalloc_mutex; /* for MPIU_{Malloc,Free,Calloc} */
#endif
} MPICH_ThreadInfo_t;
extern MPICH_ThreadInfo_t MPIR_ThreadInfo;
-/*
- * Get a pointer to the thread's private data
- * Also define a macro to release any storage that may be allocated
- * by Malloc to ensure that memory leak tools don't report this when
- * there is no real leak.
- */
#ifndef MPICH_IS_THREADED
-#define MPIR_GetPerThread(pt_) \
-{ \
- *(pt_) = &MPIR_Thread; \
-}
-#define MPIR_ReleasePerThread
-#else
-/* Define a macro to acquire or create the thread private storage */
-#define MPIR_GetOrInitThreadPriv( pt_ ) \
-{ \
- MPID_Thread_tls_get(&MPIR_ThreadInfo.thread_storage, (pt_)); \
- if (*(pt_) == NULL) \
- { \
- *(pt_) = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t)); \
- MPID_Thread_tls_set(&MPIR_ThreadInfo.thread_storage, (void *) *(pt_));\
- } \
- MPIU_DBG_MSG_FMT(THREAD,VERBOSE,(MPIU_DBG_FDEST,\
- "perthread storage (key = %x) is %p", (unsigned int)MPIR_ThreadInfo.thread_storage,*pt_));\
-}
+
+#define MPIU_THREAD_CHECK_BEGIN
+#define MPIU_THREAD_CHECK_END
+#define MPIU_ISTHREADED (0)
+
+#else /* defined(MPICH_IS_THREADED) */
+
/* We want to avoid the overhead of the thread call if we're in the
- runtime state and threads are not in use. In that case, MPIR_Thread
+ runtime state and threads are not in use. In that case, MPIR_Thread
is still a pointer but it was already allocated in InitThread */
#ifdef HAVE_RUNTIME_THREADCHECK
-#define MPIR_GetPerThread(pt_) {\
- if (MPIU_ISTHREADED()) { MPIR_GetOrInitThreadPriv( pt_ ); } \
- else { *(pt_) = &MPIR_ThreadSingle; } \
- }
-/* Note that we set the value on the thread_storage key to zero. This
- is because we have set an exit handler on this thread key when it
- was created; that handler will try to delete the storage associated
- with that value. */
-#define MPIR_ReleasePerThread { \
- if (MPIU_ISTHREADED()) { \
- MPICH_PerThread_t *pt_; \
- MPIR_GetOrInitThreadPriv( &pt_ ); MPIU_Free( pt_ ); \
- MPID_Thread_tls_set(&MPIR_ThreadInfo.thread_storage,(void *)0);} }
-#else
-#define MPIR_GetPerThread(pt_) MPIR_GetOrInitThreadPriv( pt_ )
-#define MPIR_ReleasePerThread { \
-MPICH_PerThread_t *pt_; MPIR_GetOrInitThreadPriv( &pt_ ); MPIU_Free( pt_ ); }
+
+#define MPIU_THREAD_CHECK_BEGIN if (MPIR_ThreadInfo.isThreaded) {
+#define MPIU_THREAD_CHECK_END }
+/* This macro used to take an argument of one or more statements that
+ * would be wrapped in a conditional. However it was not used anywhere
+ * in part because it looked unnatural. This version takes no arguments
+ * and evaluates to a boolean expression that is true if MPICH2 is
+ * threaded (including any runtime check if needed) and false otherwise.
+ * MPIU_THREAD_CHECK_{BEGIN,END} should probably still be used if you
+ * don't have any other conditions that you need to check in order to
+ * completely eliminate unnecessary if's. */
+#define MPIU_ISTHREADED (MPIR_ThreadInfo.isThreaded)
+
+#else /* !defined(HAVE_RUNTIME_THREADCHECK) */
+
+#define MPIU_THREAD_CHECK_BEGIN
+#define MPIU_THREAD_CHECK_END
+#define MPIU_ISTHREADED (1)
+
#endif /* HAVE_RUNTIME_THREADCHECK */
#endif /* MPICH_IS_THREADED */
+/* ------------------------------------------------------------------------- */
+/* thread-local storage macros */
+/* moved here from mpiimpl.h because they logically belong here */
-/*
- * Define MPID Critical Section macros, unless the device will be defining them
- */
-#if !defined(MPID_DEFINES_MPID_CS)
-#ifndef MPICH_IS_THREADED
-#define MPID_CS_INITIALIZE()
-#define MPID_CS_FINALIZE()
-#define MPID_CS_ENTER()
-#define MPID_CS_EXIT()
-#elif (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
-/* Function prototype (needed when no weak symbols available) */
-void MPIR_CleanupThreadStorage( void *a );
+/* Time stamps */
+/* Get the timer definitions. The source file for this include is
+ src/mpi/timer/mpichtimer.h.in */
+#include "mpichtimer.h"
-int MPIR_Thread_CS_Init( void );
-int MPIR_Thread_CS_Finalize( void );
-#define MPID_CS_INITIALIZE() MPIR_Thread_CS_Init()
-#define MPID_CS_FINALIZE() MPIR_Thread_CS_Finalize()
+typedef struct MPID_Stateinfo_t {
+ MPID_Time_t stamp;
+ int count;
+} MPID_Stateinfo_t;
+#define MPICH_MAX_STATES 512
+/* Timer state routines (src/util/instrm/states.c) */
+void MPID_TimerStateBegin( int, MPID_Time_t * );
+void MPID_TimerStateEnd( int, MPID_Time_t * );
-/* FIXME: Figure out what we want to do for the nest count on
- these routines, so as to avoid extra function calls */
-/*
- * The Nest value must *not* be changed by MPID_CS_ENTER or MPID_CS_EXIT.
- * It is the obligation of the implementation of the MPI routines to
- * use MPIR_Nest_incr and MPIR_Nest_decr before invoking other MPI routines
- */
-#define MPID_CS_ENTER() \
-{ \
- MPIU_THREADPRIV_GET; \
- if (MPIR_Nest_value() == 0) \
- { \
- MPIU_DBG_MSG(THREAD,TYPICAL,"Enter global critical section");\
- MPID_Thread_mutex_lock(&MPIR_ThreadInfo.global_mutex); \
- MPIU_THREAD_UPDATEDEPTH(global_mutex,1); \
- } \
-}
-#define MPID_CS_EXIT() \
-{ \
- MPIU_THREADPRIV_GET; \
- if (MPIR_Nest_value() == 0) \
- { \
- MPIU_DBG_MSG(THREAD,TYPICAL,"Exit global critical section");\
- MPID_Thread_mutex_unlock(&MPIR_ThreadInfo.global_mutex); \
- MPIU_THREAD_UPDATEDEPTH(global_mutex,-1); \
- } \
-}
-#else
-#error "Critical section macros not defined"
-#endif
-#endif /* !defined(MPID_DEFINES_MPID_CS) */
+#ifdef MPICH_DEBUG_NESTING
+#define MPICH_MAX_NESTFILENAME 256
+typedef struct MPICH_Nestinfo {
+ char file[MPICH_MAX_NESTFILENAME];
+ int line;
+} MPICH_Nestinfo_t;
+#define MPICH_MAX_NESTINFO 16
+#endif /* MPICH_DEBUG_NESTING */
+/* arbitrary, just needed to avoid cleaning up heap allocated memory at thread
+ * destruction time */
+#define MPIU_STRERROR_BUF_SIZE (1024)
-#if defined(HAVE_THR_YIELD)
-#undef MPID_Thread_yield
-#define MPID_Thread_yield() thr_yield()
-#elif defined(HAVE_SCHED_YIELD)
-#undef MPID_Thread_yield
-#define MPID_Thread_yield() sched_yield()
-#elif defined(HAVE_YIELD)
-#undef MPID_Thread_yield
-#define MPID_Thread_yield() yield()
+/* this structure contains all thread-local variables, will be zeroed at
+ * allocation time */
+typedef struct MPICH_PerThread_t {
+ int nest_count; /* For layered MPI implementation */
+ int op_errno; /* For errors in predefined MPI_Ops */
+
+ /* error string storage for MPIU_Strerror */
+ char strerrbuf[MPIU_STRERROR_BUF_SIZE];
+
+#ifdef MPICH_DEBUG_NESTING
+ MPICH_Nestinfo_t nestinfo[MPICH_MAX_NESTINFO];
+ struct MPIU_ThreadDebug *nest_debug;
#endif
+ /* FIXME: Is this used anywhere? */
+#ifdef HAVE_TIMING
+ MPID_Stateinfo_t timestamps[MPICH_MAX_STATES]; /* per thread state info */
+#endif
+#if defined(MPID_DEV_PERTHREAD_DECL)
+ MPID_DEV_PERTHREAD_DECL
+#endif
+} MPICH_PerThread_t;
-/*
- * New Thread Interface and Macros
- * See http://www-unix.mcs.anl.gov/mpi/mpich2/developer/design/threads.htm
- * for a detailed discussion of this interface and the rationale.
- *
- * This code implements the single CS approach, and is similar to
- * the MPID_CS_ENTER and MPID_CS_EXIT macros above.
- *
- */
+#if !defined(MPICH_IS_THREADED)
+/* If single threaded, make this point at a pre-allocated segment.
+ This structure is allocated in src/mpi/init/initthread.c */
+extern MPICH_PerThread_t MPIR_Thread;
-/* Make the CPP definition that will be used to control whether threaded
- code is supported. Test ONLY on whether MPICH_IS_THREADED is defined.
+/* The following three macros define a way to portably access thread-private
+ storage in MPICH2, and avoid extra overhead when MPICH2 is single
+ threaded
+ INITKEY - Create the key. Must happen *before* the other threads
+ are created
+ INIT - Create the thread-private storage. Must happen once per thread
+ DECL - Declare local variables
+ GET - Access the thread-private storage
+ FIELD - Access the thread-private field (by name)
+ FINALIZE - to be invoked when all threads no longer need access to the thread
+ local storage, such as at MPI_Finalize time
+
+ The "DECL" is the extern so that there is always a statement for
+ the declaration.
*/
-#if !defined(MPICH_THREAD_LEVEL) || !defined(MPI_THREAD_MULTIPLE)
-#error Internal error in macro definitions in include/mpiimplthread.h
-#endif
+#define MPIU_THREADPRIV_INITKEY
+#define MPIU_THREADPRIV_INIT
+/* Empty declarations are not allowed in C. However multiple decls are allowed */
+/* FIXME it might be better to just use a dummy variable here like we do when
+ * real TLS is available to reduce the amount of compiler warnings. */
+#define MPIU_THREADPRIV_DECL extern MPICH_PerThread_t MPIR_Thread
+#define MPIU_THREADPRIV_GET
+#define MPIU_THREADPRIV_FIELD(a_) (MPIR_Thread.a_)
+#else /* defined(MPICH_IS_THREADED) */
-#if !defined(MPID_DEFINES_MPID_CS)
-#ifdef MPICH_IS_THREADED
+/* We need to provide a function that will cleanup the storage attached
+ to the key. */
+void MPIR_CleanupThreadStorage(void *a);
-#ifdef HAVE_RUNTIME_THREADCHECK
-#define MPIU_THREAD_CHECK_BEGIN if (MPIR_ThreadInfo.isThreaded) {
-#define MPIU_THREAD_CHECK_END }
-/* This macro used to take an argument of one or more statements that
- * would be wrapped in a conditional. However it was not used anywhere
- * in part because it looked unnatural. This version takes no arguments
- * and evaluates to a boolean expression that is true if MPICH2 is
- * threaded (including any runtime check if needed) and false otherwise.
- * MPIU_THREAD_CHECK_{BEGIN,END} should probably still be used if you
- * don't have any other conditions that you need to check in order to
- * completely eliminate unnecessary if's. */
-#define MPIU_ISTHREADED() (MPIR_ThreadInfo.isThreaded)
-#else
-#define MPIU_THREAD_CHECK_BEGIN
-#define MPIU_THREAD_CHECK_END
-#define MPIU_ISTHREADED() (1)
-#endif /* HAVE_RUNTIME_THREADCHECK */
+#if !defined(MPIU_TLS_SPECIFIER)
+# if defined(HAVE_RUNTIME_THREADCHECK)
+/* In the case where the thread level is set in MPI_Init_thread, we
+ need a blended version of the non-threaded and the thread-multiple
+ definitions.
-/* SINGLE_CS_DECL needs to take over the decl used by MPID_CS_xxx when that
- is removed */
-#define MPIU_THREAD_SINGLE_CS_DECL
-#define MPIU_THREAD_SINGLE_CS_INITIALIZE MPID_CS_INITIALIZE()
-/* #define MPIU_THREAD_SINGLE_CS_FINALIZE MPIU_THREAD_CHECK_BEGIN MPID_CS_FINALIZE(); MPIU_THREAD_CHECK_END */
-/* Because we unconditionally invoke the initialize, we need to do the
- same with the finalize */
-#define MPIU_THREAD_SINGLE_CS_FINALIZE MPID_CS_FINALIZE()
-#define MPIU_THREAD_SINGLE_CS_ASSERT_WITHIN(_msg)
-/*
-#define MPIU_THREAD_SINGLE_CS_ENTER(_msg) MPIU_THREAD_CHECK_BEGIN MPID_CS_ENTER() MPIU_THREAD_CHECK_END
-#define MPIU_THREAD_SINGLE_CS_EXIT(_msg) MPIU_THREAD_CHECK_BEGIN MPID_CS_EXIT() MPIU_THREAD_CHECK_END
-*/
-#define MPIU_THREAD_SINGLE_CS_ENTER(_msg) ''' remove me '''
-#define MPIU_THREAD_SINGLE_CS_EXIT(_msg) ''' remove me '''
+ The approach is to have TWO MPICH_PerThread_t pointers. One is local
+ (The MPIU_THREADPRIV_DECL is used in the routines local definitions),
+ as in the threaded version of these macros. This is set by using a routine
+ to get thread-private storage. The second is a preallocated, extern
+ MPICH_PerThread_t struct, as in the single threaded case. Based on
+ MPIR_Process.isThreaded, one or the other is used.
-/* These provide a uniform way to perform a first-use initialization
- in a thread-safe way. See the web page or wtime.c */
-#define MPIU_THREADSAFE_INIT_DECL(_var) static volatile int _var=1
-#define MPIU_THREADSAFE_INIT_STMT(_var,_stmt) \
- if (_var) { \
- MPIU_THREAD_SINGLE_CS_ENTER(""); \
- _stmt; _var=0; \
- MPIU_THREAD_SINGLE_CS_EXIT(""); \
- }
-#define MPIU_THREADSAFE_INIT_BLOCK_BEGIN(_var) \
- MPIU_THREAD_SINGLE_CS_ENTER(""); \
- if (_var) {
-#define MPIU_THREADSAFE_INIT_CLEAR(_var) _var=0
-#define MPIU_THREADSAFE_INIT_BLOCK_END(_var) \
- } \
- MPIU_THREAD_SINGLE_CS_EXIT("")
+ */
+/* For the single threaded case, we use a preallocated structure
+ This structure is allocated in src/mpi/init/initthread.c */
+extern MPICH_PerThread_t MPIR_ThreadSingle;
-#else
-/* MPICH user routines are single-threaded */
-#define MPIU_ISTHREADED() (0)
-#define MPIU_THREAD_SINGLE_CS_DECL
-#define MPIU_THREAD_SINGLE_CS_INITIALIZE
-#define MPIU_THREAD_SINGLE_CS_FINALIZE
-#define MPIU_THREAD_SINGLE_CS_ASSERT_WITHIN(_msg)
-#define MPIU_THREAD_SINGLE_CS_ENTER(_msg)
-#define MPIU_THREAD_SINGLE_CS_EXIT(_msg)
-#define MPIU_THREAD_CHECK_BEGIN
-#define MPIU_THREAD_CHECK_END
+#define MPIU_THREADPRIV_INITKEY \
+ do { \
+ if (MPIU_ISTHREADED) { \
+ MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_ThreadInfo.thread_storage,NULL); \
+ } \
+ } while (0)
+#define MPIU_THREADPRIV_INIT \
+ do { \
+ if (MPIU_ISTHREADED) { \
+ MPIR_Thread = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t)); \
+ MPIU_Assert(MPIR_Thread); \
+ MPID_Thread_tls_set(&MPIR_ThreadInfo.thread_storage, (void *)MPIR_Thread); \
+ } \
+ } while (0)
+/* MPIU_THREADPRIV_INIT must *always* be called before MPIU_THREADPRIV_GET */
+#define MPIU_THREADPRIV_GET \
+ do { \
+ if (!MPIR_Thread) { \
+ if (MPIU_ISTHREADED) { \
+ MPID_Thread_tls_get(&MPIR_ThreadInfo.thread_storage, &MPIR_Thread); \
+ if (!MPIR_Thread) { \
+ MPIU_THREADPRIV_INIT; /* subtle, sets MPIR_Thread */ \
+ } \
+ } \
+ else { \
+ MPIR_Thread = &MPIR_ThreadSingle; \
+ } \
+ MPIU_Assert(MPIR_Thread); \
+ } \
+ } while (0)
-/* These provide a uniform way to perform a first-use initialization
- in a thread-safe way. See the web page or mpidtime.c for the generic
- wtick */
-#define MPIU_THREADSAFE_INIT_DECL(_var) static int _var=1
-#define MPIU_THREADSAFE_INIT_STMT(_var,_stmt) if (_var) { _stmt; _var = 0; }
-#define MPIU_THREADSAFE_INIT_BLOCK_BEGIN(_var)
-#define MPIU_THREADSAFE_INIT_CLEAR(_var) _var=0
-#define MPIU_THREADSAFE_INIT_BLOCK_END(_var)
-#endif /* MPICH_IS_THREADED */
-#endif /* !defined(MPID_DEFINES_MPID_CS) */
+# else /* unconditional thread multiple */
+/* We initialize the MPIR_Thread pointer to null so that we need call the
+ * routine to get the thread-private storage only once in an invocation of a
+ * routine. */
+#define MPIU_THREADPRIV_INITKEY \
+ MPID_Thread_tls_create(MPIR_CleanupThreadStorage,&MPIR_ThreadInfo.thread_storage,NULL)
+#define MPIU_THREADPRIV_INIT \
+ do { \
+ MPIR_Thread = (MPICH_PerThread_t *) MPIU_Calloc(1, sizeof(MPICH_PerThread_t)); \
+ MPIU_Assert(MPIR_Thread); \
+ MPID_Thread_tls_set(&MPIR_ThreadInfo.thread_storage, (void *)MPIR_Thread); \
+ } while (0)
+#define MPIU_THREADPRIV_GET \
+ do { \
+ if (!MPIR_Thread) { \
+ MPID_Thread_tls_get(&MPIR_ThreadInfo.thread_storage, &MPIR_Thread); \
+ if (!MPIR_Thread) { \
+ MPIU_THREADPRIV_INIT; /* subtle, sets MPIR_Thread */ \
+ } \
+ MPIU_Assert(MPIR_Thread); \
+ } \
+ } while (0)
+
+# endif /* runtime vs. unconditional */
+
+/* common definitions when using MPID_Thread-based TLS */
+#define MPIU_THREADPRIV_DECL MPICH_PerThread_t *MPIR_Thread=NULL
+#define MPIU_THREADPRIV_FIELD(a_) (MPIR_Thread->a_)
+#define MPIU_THREADPRIV_FINALIZE \
+ do { \
+ MPIU_THREADPRIV_DECL; \
+ MPIU_THREADPRIV_GET; \
+ MPIU_Free(MPIR_Thread); \
+ MPID_Thread_tls_set(&MPIR_ThreadInfo.thread_storage,NULL); \
+ MPID_Thread_tls_destroy(&MPIR_ThreadInfo.thread_storage,NULL); \
+ } while (0)
+
+#else /* defined(MPIU_TLS_SPECIFIER) */
+
+/* We have proper thread-local storage (TLS) support from the compiler, which
+ * should yield the best performance and simplest code, so we'll use that. */
+extern MPIU_TLS_SPECIFIER MPICH_PerThread_t MPIR_Thread;
+
+#define MPIU_THREADPRIV_INITKEY
+#define MPIU_THREADPRIV_INIT
+/* Empty declarations are not allowed in C, so we just define a dummy
+ * variable and tell the compiler it is unused. */
+#define MPIU_THREADPRIV_DECL int MPIU_THREADPRIV_LOCAL_DUMMY_VAR ATTRIBUTE((unused))
+#define MPIU_THREADPRIV_GET
+#define MPIU_THREADPRIV_FIELD(a_) (MPIR_Thread.a_)
+#define MPIU_THREADPRIV_FINALIZE do{}while(0)
+
+#endif /* defined(MPIU_TLS_SPECIFIER) */
+#endif /* defined(MPICH_IS_THREADED) */
+
+
/* ------------------------------------------------------------------------- */
+/* yield macro */
+/* FIXME update mpiatomic.h accordingly if this is ever improved */
+#undef MPID_Thread_yield
+/* FIXME this should probably include sleep/usleep/nanosleep as an option in here */
+#if defined(HAVE_THR_YIELD)
+#define MPID_Thread_yield() thr_yield()
+#elif defined(HAVE_SCHED_YIELD)
+#define MPID_Thread_yield() sched_yield()
+#elif defined(HAVE_YIELD)
+#define MPID_Thread_yield() yield()
+#endif
+
+
+/* ------------------------------------------------------------------------- */
/*
* New definitions for controling the granularity of thread atomicity
*
+ * If MPICH supports threads and the device doesn't wish to override our
+ * definitions, we supply default definitions.
+ *
+ * If the device does choose to override our definitions, it needs to do the
+ * following at a minimum:
+ * 1) #define MPID_DEVICE_DEFINES_THREAD_CS in mpidpre.h
+ * 2) provide #defines for MPIU_THREAD_CS_{INIT,FINALIZE}
+ * 3) provide #defines for MPIU_THREAD_CS_{ENTER,EXIT,YIELD}(name_,context_)
+ * 4) provide #defines for MPIU_THREADSAFE_INIT_{DECL,STMT,BLOCK_BEGIN,BLOCK_END,CLEAR}
*/
-#if !defined(MPID_DEFINES_MPID_CS)
-#ifdef MPICH_IS_THREADED
+#if defined(MPICH_IS_THREADED) && !defined(MPID_DEVICE_DEFINES_THREAD_CS)
+/* MPIU_THREAD_CS_INIT will be invoked early in the top level
+ * MPI_Init/MPI_Init_thread process. It is responsible for performing any
+ * initialization for the MPIU_THREAD_CS_ macros, as well as invoking
+ * MPIU_THREADPRIV_INITKEY and MPIU_THREADPRIV_INIT.
+ *
+ * MPIR_ThreadInfo.isThreaded will be set prior to this macros invocation, based
+ * on the "provided" value to MPI_Init_thread
+ */
+#define MPIU_THREAD_CS_INIT MPIR_Thread_CS_Init() /* in src/mpi/init/initthread.c */
+#define MPIU_THREAD_CS_FINALIZE MPIR_Thread_CS_Finalize() /* in src/mpi/init/initthread.c */
+
+/* some important critical section names:
+ * ALLFUNC - entered/exited at beginning/end of (nearly) every MPI_ function
+ * INIT - entered before MPID_Init and exited near the end of MPI_Init(_thread)
+ */
+
/*M MPIU_THREAD_CS_ENTER - Enter a named critical section
Input Parameters:
@@ -333,57 +377,80 @@
/*M
... move the threadsafe init block
-
+
These use a private critical section called INITFLAG
M*/
-#undef MPIU_THREADSAFE_INIT_DECL
-#undef MPIU_THREADSAFE_INIT_STMT
-#undef MPIU_THREADSAFE_INIT_BLOCK_BEGIN
-#undef MPIU_THREADSAFE_INIT_BLOCK_END
+/* FIXME this is a broken version of the Double-Checked Locking Pattern (DCLP).
+ * In practice it may work OK, but it certainly isn't guaranteed depending on
+ * what the processor, compiler, and pthread implementation choose to do.
+ *
+ * These might be fixable in practice via OPA memory barriers. */
#define MPIU_THREADSAFE_INIT_DECL(_var) static volatile int _var=1
-#define MPIU_THREADSAFE_INIT_STMT(_var,_stmt) \
- if (_var) { \
- MPIU_THREAD_CS_ENTER(INITFLAG,); \
- _stmt; _var=0; \
- MPIU_THREAD_CS_EXIT(INITFLAG,); \
- }
-#define MPIU_THREADSAFE_INIT_BLOCK_BEGIN(_var) \
- MPIU_THREAD_CS_ENTER(INITFLAG,); \
- if (_var) {
+#define MPIU_THREADSAFE_INIT_STMT(_var,_stmt) \
+ do { \
+ if (_var) { \
+ MPIU_THREAD_CS_ENTER(INITFLAG,); \
+ _stmt; \
+ _var=0; \
+ MPIU_THREAD_CS_EXIT(INITFLAG,); \
+ } \
+ while (0)
+#define MPIU_THREADSAFE_INIT_BLOCK_BEGIN(_var) \
+ MPIU_THREAD_CS_ENTER(INITFLAG,); \
+ if (_var) {
#define MPIU_THREADSAFE_INIT_CLEAR(_var) _var=0
-#define MPIU_THREADSAFE_INIT_BLOCK_END(_var) \
- } \
- MPIU_THREAD_CS_EXIT(INITFLAG,)
-#else
+#define MPIU_THREADSAFE_INIT_BLOCK_END(_var) \
+ } \
+ MPIU_THREAD_CS_EXIT(INITFLAG,)
+
+#else /* !defined(MPICH_IS_THREADED) */
+
/* These provide a uniform way to perform a first-use initialization
in a thread-safe way. See the web page or mpidtime.c for the generic
wtick */
#define MPIU_THREADSAFE_INIT_DECL(_var) static int _var=1
-#define MPIU_THREADSAFE_INIT_STMT(_var,_stmt) if (_var) { _stmt; _var = 0; }
-#define MPIU_THREADSAFE_INIT_BLOCK_BEGIN(_var)
+#define MPIU_THREADSAFE_INIT_STMT(_var,_stmt) \
+ do { \
+ if (_var) { \
+ _stmt; \
+ _var = 0; \
+ } \
+ } while (0)
+#define MPIU_THREADSAFE_INIT_BLOCK_BEGIN(_var)
#define MPIU_THREADSAFE_INIT_CLEAR(_var) _var=0
-#define MPIU_THREADSAFE_INIT_BLOCK_END(_var)
+#define MPIU_THREADSAFE_INIT_BLOCK_END(_var)
#endif /* MPICH_IS_THREADED */
-#endif /* !defined(MPID_DEFINES_MPID_CS) */
-#if !defined(MPID_DEFINES_MPID_CS)
-#ifdef MPICH_IS_THREADED
-
-/* Helper definitions */
+/* Helper definitions for the default macro definitions */
+#if defined(MPICH_IS_THREADED) && !defined(MPID_DEVICE_DEFINES_THREAD_CS)
#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
+/* FIXME what is this really supposed to do? Is it supposed to check against
+ * the "MPI nesting" or the critical section nesting? */
#define MPIU_THREAD_CHECKNEST(_name) \
MPIU_THREADPRIV_GET; \
+ MPIU_DBG_MSG_D(THREAD,VERBOSE,"CHECKNEST, MPIR_Nest_value()=%d",MPIR_Nest_value());\
if (MPIR_Nest_value() == 0)
#define MPIU_THREAD_CHECKDEPTH(_name,_value)
#define MPIU_THREAD_UPDATEDEPTH(_name,_value)
#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT
+
+/* this nesting is different than the MPIR_Nest_* macros, this is critical
+ * section nesting, the other is MPI/NMPI nesting */
+
/* This structure is used to keep track of where the last change was made
to the thread cs depth */
#ifdef MPID_THREAD_DEBUG
+
+enum MPIU_Nest_mutexes {
+ MPIU_Nest_global_mutex = 0,
+ MPIU_Nest_handle_mutex,
+ MPIU_Nest_NUM_MUTEXES
+};
+
#define MPIU_THREAD_LOC_LEN 127
#define MPIU_THREAD_FNAME_LEN 31
typedef struct MPIU_ThreadDebug {
@@ -392,45 +459,56 @@
char file[MPIU_THREAD_LOC_LEN+1];
char fname[MPIU_THREAD_FNAME_LEN+1];
} MPIU_ThreadDebug_t;
-#define MPIU_THREAD_CHECKDEPTH(_name,_value) \
- {if (1){ MPIU_ThreadDebug_t *_nest_ptr=0; \
- MPID_Thread_tls_get( &MPIR_ThreadInfo.nest_storage, &_nest_ptr );\
- if (!_nest_ptr) { _nest_ptr = (MPIU_ThreadDebug_t*)MPIU_Calloc(2,sizeof(MPIU_ThreadDebug_t));\
- MPID_Thread_tls_set( &MPIR_ThreadInfo.nest_storage,_nest_ptr);}\
- if (_nest_ptr[MPIUNest_##_name].count != _value) {\
- fprintf(stderr, "%s:%d %s = %d, required %d; previously set in %s:%d(%s)\n",\
- __FILE__, __LINE__, #_name, \
- _nest_ptr[MPIUNest_##_name].count, _value, \
- _nest_ptr[MPIUNest_##_name].file, \
- _nest_ptr[MPIUNest_##_name].line,\
- _nest_ptr[MPIUNest_##_name].fname ); \
- fflush(stderr); \
- }}}
-#define MPIU_THREAD_UPDATEDEPTH(_name,_value) {if (1){ \
- MPIU_ThreadDebug_t *_nest_ptr=0;\
- MPID_Thread_tls_get( &MPIR_ThreadInfo.nest_storage, &_nest_ptr );\
- if (!_nest_ptr) { _nest_ptr = (MPIU_ThreadDebug_t*)MPIU_Calloc(2,sizeof(MPIU_ThreadDebug_t));\
- MPID_Thread_tls_set( &MPIR_ThreadInfo.nest_storage,_nest_ptr);}\
- if (_nest_ptr[MPIUNest_##_name].count +_value< 0) {\
- fprintf(stderr, "%s:%d %s = %d (<0); previously set in %s:%d(%s)\n",\
- __FILE__, __LINE__, #_name, \
- _nest_ptr[MPIUNest_##_name].count, \
- _nest_ptr[MPIUNest_##_name].file, \
- _nest_ptr[MPIUNest_##_name].line,\
- _nest_ptr[MPIUNest_##_name].fname ); \
- fflush(stderr); \
- }\
- _nest_ptr[MPIUNest_##_name].count += _value;\
- _nest_ptr[MPIUNest_##_name].line = __LINE__; \
- MPIU_Strncpy( _nest_ptr[MPIUNest_##_name].file, __FILE__, MPIU_THREAD_LOC_LEN ); \
- MPIU_Strncpy( _nest_ptr[MPIUNest_##_name].fname, FCNAME, MPIU_THREAD_FNAME_LEN );\
-}}
+
+/* helper macro to allocate and initialize storage for the CS nest checks, not
+ * part of the public API */
+/* NOTE: assumes that MPIU_THREADPRIV_DECL has been done already */
+#define MPIU_THREAD_INIT_NEST_CHECK_ \
+ do { \
+ MPIU_THREADPRIV_GET; \
+ if (!MPIU_THREADPRIV_FIELD(nest_debug)) { \
+ MPIU_THREADPRIV_FIELD(nest_debug) = \
+ MPIU_Calloc(MPIU_Nest_NUM_MUTEXES, sizeof(MPIU_ThreadDebug_t)); \
+ } \
+ } while (0)
+#define MPIU_THREAD_CHECKDEPTH(name_,value_) \
+ do { \
+ MPIU_ThreadDebug_t *nest_ptr_ = NULL; \
+ MPIU_THREADPRIV_GET; \
+ MPIU_THREAD_INIT_NEST_CHECK_; \
+ nest_ptr_ = MPIU_THREADPRIV_FIELD(nest_debug); \
+ if (nest_ptr_[MPIU_Nest_##name_].count != value_) { \
+ fprintf(stderr, "%s:%d %s = %d, required %d; previously set in %s:%d(%s)\n", \
+ __FILE__, __LINE__, #name_, \
+ nest_ptr_[MPIU_Nest_##name_].count, value_, \
+ nest_ptr_[MPIU_Nest_##name_].file, \
+ nest_ptr_[MPIU_Nest_##name_].line, \
+ nest_ptr_[MPIU_Nest_##name_].fname ); \
+ fflush(stderr); \
+ } \
+ } while (0)
+#define MPIU_THREAD_UPDATEDEPTH(name_,value_) \
+ do { \
+ MPIU_ThreadDebug_t *nest_ptr_ = NULL; \
+ MPIU_THREADPRIV_GET; \
+ MPIU_THREAD_INIT_NEST_CHECK_; \
+ nest_ptr_ = MPIU_THREADPRIV_FIELD(nest_debug); \
+ if (nest_ptr_[MPIU_Nest_##name_].count + (value_) < 0) { \
+ fprintf(stderr, "%s:%d %s = %d (<0); previously set in %s:%d(%s)\n", \
+ __FILE__, __LINE__, #name_, \
+ nest_ptr_[MPIU_Nest_##name_].count, \
+ nest_ptr_[MPIU_Nest_##name_].file, \
+ nest_ptr_[MPIU_Nest_##name_].line, \
+ nest_ptr_[MPIU_Nest_##name_].fname ); \
+ fflush(stderr); \
+ } \
+ nest_ptr_[MPIU_Nest_##name_].count += value_; \
+ nest_ptr_[MPIU_Nest_##name_].line = __LINE__; \
+ MPIU_Strncpy( nest_ptr_[MPIU_Nest_##name_].file, __FILE__, MPIU_THREAD_LOC_LEN ); \
+ MPIU_Strncpy( nest_ptr_[MPIU_Nest_##name_].fname, FCNAME, MPIU_THREAD_FNAME_LEN ); \
+ } while (0)
#define MPIU_THREAD_CHECKNEST(_name)
-/* __thread would be nice here, but it is not portable (not even available
- on Macs) */
-/* defined in mpi/init/initthread.c */
-#define MPIUNest_global_mutex 0
-#define MPIUNest_handle_mutex 1
+
#else
#define MPIU_THREAD_CHECKDEPTH(_name,_value)
#define MPIU_THREAD_UPDATEDEPTH(_name,_value)
@@ -442,6 +520,7 @@
#define MPIU_THREAD_CS_ENTER_LOCKNAME(_name) \
{ \
+ MPIU_DBG_MSG(THREAD,VERBOSE,"attempting to ENTER " #_name); \
MPIU_THREAD_CHECKDEPTH(_name,0) \
MPIU_THREAD_CHECKNEST(_name) \
{ \
@@ -452,6 +531,7 @@
}
#define MPIU_THREAD_CS_EXIT_LOCKNAME(_name) \
{ \
+ MPIU_DBG_MSG(THREAD,VERBOSE,"attempting to EXIT " #_name); \
MPIU_THREAD_CHECKDEPTH(_name,1) \
MPIU_THREAD_CHECKNEST(_name) \
{ \
@@ -460,6 +540,15 @@
MPIU_THREAD_UPDATEDEPTH(_name,-1) \
} \
}
+#define MPIU_THREAD_CS_YIELD_LOCKNAME(_name) \
+do { \
+ MPIU_DBG_MSG(THREAD,VERBOSE,"attempting to YIELD " #_name); \
+ MPIU_THREAD_CHECKDEPTH(_name,1) \
+ /* don't CHECKNEST here, we want nesting to be >0 */ \
+ MPIU_DBG_MSG(THREAD,TYPICAL,"Yield critical section "#_name); \
+ MPID_Thread_mutex_unlock(&MPIR_ThreadInfo._name); \
+ MPID_Thread_mutex_lock(&MPIR_ThreadInfo._name); \
+} while (0)
/* Some locks (such as the memory allocation locks) are needed to
* bootstrap the lock checking code. These macros provide a similar
@@ -498,14 +587,32 @@
# define MPIU_THREAD_CS_EXIT_MEMALLOC(_context) do {} while (0)
#endif
+/* helper macros to insert thread checks around LOCKNAME actions */
+#define MPIU_THREAD_CS_ENTER_LOCKNAME_CHECKED(name_) \
+ MPIU_THREAD_CHECK_BEGIN \
+ MPIU_THREAD_CS_ENTER_LOCKNAME(name_); \
+ MPIU_THREAD_CHECK_END
+#define MPIU_THREAD_CS_EXIT_LOCKNAME_CHECKED(name_) \
+ MPIU_THREAD_CHECK_BEGIN \
+ MPIU_THREAD_CS_EXIT_LOCKNAME(name_); \
+ MPIU_THREAD_CHECK_END
+#define MPIU_THREAD_CS_YIELD_LOCKNAME_CHECKED(name_) \
+ MPIU_THREAD_CHECK_BEGIN \
+ MPIU_THREAD_CS_YIELD_LOCKNAME(name_); \
+ MPIU_THREAD_CHECK_END
+
/* Definitions of the thread support for various levels of thread granularity */
#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
/* There is a single, global lock, held for the duration of an MPI call */
-#define MPIU_THREAD_CS_ENTER_ALLFUNC(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-#define MPIU_THREAD_CS_EXIT_ALLFUNC(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
+#define MPIU_THREAD_CS_ENTER_ALLFUNC(_context) MPIU_THREAD_CS_ENTER_LOCKNAME_CHECKED(global_mutex)
+#define MPIU_THREAD_CS_EXIT_ALLFUNC(_context) MPIU_THREAD_CS_EXIT_LOCKNAME_CHECKED(global_mutex)
+#define MPIU_THREAD_CS_YIELD_ALLFUNC(_context) MPIU_THREAD_CS_YIELD_LOCKNAME_CHECKED(global_mutex)
+
+/* not _CHECKED, invoked before MPIU_ISTHREADED will work */
+#define MPIU_THREAD_CS_ENTER_INIT(_context) MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex)
+#define MPIU_THREAD_CS_EXIT_INIT(_context) MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex)
+
#define MPIU_THREAD_CS_ENTER_HANDLE(_context)
#define MPIU_THREAD_CS_EXIT_HANDLE(_context)
#define MPIU_THREAD_CS_ENTER_HANDLEALLOC(_context)
@@ -521,88 +628,50 @@
#define MPIU_THREAD_CS_ENTER_CONTEXTID(_context)
#define MPIU_THREAD_CS_EXIT_CONTEXTID(_context)
-/* FIXME this YIELD macro probably needs to be revisited */
-#define MPIU_THREAD_CS_YIELD_CONTEXTID(_context) \
- MPID_Thread_mutex_unlock(&MPIR_ThreadInfo.global_mutex);\
- MPID_Thread_yield();\
- MPID_Thread_mutex_lock(&MPIR_ThreadInfo.global_mutex);
+/* XXX DJG at the global level should CONTEXTID yield even though it
+ * doesn't do anything at ENTER/EXIT? */
+#define MPIU_THREAD_CS_YIELD_CONTEXTID(_context) MPIU_THREAD_CS_YIELD_LOCKNAME_CHECKED(global_mutex)
#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT
/* There are multiple locks, one for each (major) object */
-/* FIXME: That's the ugliest hack I've ever written. Use the same lock
- * macro as LOCKNAME */
-#define MPIU_THREAD_CS_ENTER_POBJ_LOCKNAME(_name) \
-{ \
- MPIU_THREAD_CHECKDEPTH(_name,0) \
- MPIU_THREAD_CHECKNEST(_name) \
- { \
- MPIU_DBG_MSG(THREAD,TYPICAL,"Enter critical section "#_name);\
- MPID_Thread_mutex_lock(&_name); \
- MPIU_THREAD_UPDATEDEPTH(_name,1) \
- } \
-}
-#define MPIU_THREAD_CS_EXIT_POBJ_LOCKNAME(_name) \
-{ \
- MPIU_THREAD_CHECKDEPTH(_name,1) \
- MPIU_THREAD_CHECKNEST(_name) \
- { \
- MPIU_DBG_MSG(THREAD,TYPICAL,"Exit critical section "#_name);\
- MPID_Thread_mutex_unlock(&_name); \
- MPIU_THREAD_UPDATEDEPTH(_name,-1) \
- } \
-}
+/* FIXME needs testing and work */
#define MPIU_THREAD_CS_ENTER_ALLFUNC(_context)
#define MPIU_THREAD_CS_EXIT_ALLFUNC(_context)
-#define MPIU_THREAD_CS_ENTER_HANDLE(_context) { \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(handle_mutex) MPIU_THREAD_CHECK_END \
-}
-#define MPIU_THREAD_CS_EXIT_HANDLE(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(handle_mutex) MPIU_THREAD_CHECK_END
+#define MPIU_THREAD_CS_ENTER_INIT(_context)
+#define MPIU_THREAD_CS_EXIT_INIT(_context)
-#define MPIU_THREAD_CS_ENTER_HANDLEALLOC(_context) { \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(handle_mutex) MPIU_THREAD_CHECK_END \
-}
-#define MPIU_THREAD_CS_EXIT_HANDLEALLOC(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(handle_mutex) MPIU_THREAD_CHECK_END
+#define MPIU_THREAD_CS_ENTER_HANDLE(_context) MPIU_THREAD_CS_ENTER_LOCKNAME_CHECKED(handle_mutex)
+#define MPIU_THREAD_CS_EXIT_HANDLE(_context) MPIU_THREAD_CS_EXIT_LOCKNAME_CHECKED(handle_mutex)
-#define MPIU_THREAD_CS_ENTER_MPIDCOMM(_context) { \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END \
-}
-#define MPIU_THREAD_CS_EXIT_MPIDCOMM(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
+#define MPIU_THREAD_CS_ENTER_HANDLEALLOC(_context) MPIU_THREAD_CS_ENTER_LOCKNAME_CHECKED(handle_mutex)
+#define MPIU_THREAD_CS_EXIT_HANDLEALLOC(_context) MPIU_THREAD_CS_EXIT_LOCKNAME_CHECKED(handle_mutex)
-#define MPIU_THREAD_CS_ENTER_MSGQUEUE(_context) {\
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END \
-}
-#define MPIU_THREAD_CS_EXIT_MSGQUEUE(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
+#define MPIU_THREAD_CS_ENTER_MPIDCOMM(_context) MPIU_THREAD_CS_ENTER_LOCKNAME_CHECKED(global_mutex)
+#define MPIU_THREAD_CS_EXIT_MPIDCOMM(_context) MPIU_THREAD_CS_EXIT_LOCKNAME_CHECKED(global_mutex)
-#define MPIU_THREAD_CS_ENTER_INITFLAG(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-#define MPIU_THREAD_CS_EXIT_INITFLAG(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-#define MPIU_THREAD_CS_ENTER_PMI(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-#define MPIU_THREAD_CS_EXIT_PMI(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
+#define MPIU_THREAD_CS_ENTER_MSGQUEUE(_context) MPIU_THREAD_CS_ENTER_LOCKNAME_CHECKED(global_mutex)
+#define MPIU_THREAD_CS_EXIT_MSGQUEUE(_context) MPIU_THREAD_CS_EXIT_LOCKNAME_CHECKED(global_mutex)
+#define MPIU_THREAD_CS_ENTER_INITFLAG(_context) MPIU_THREAD_CS_ENTER_LOCKNAME_CHECKED(global_mutex)
+#define MPIU_THREAD_CS_EXIT_INITFLAG(_context) MPIU_THREAD_CS_EXIT_LOCKNAME_CHECKED(global_mutex)
+
+#define MPIU_THREAD_CS_ENTER_PMI(_context) MPIU_THREAD_CS_ENTER_LOCKNAME_CHECKED(global_mutex)
+#define MPIU_THREAD_CS_EXIT_PMI(_context) MPIU_THREAD_CS_EXIT_LOCKNAME_CHECKED(global_mutex)
+
#define MPIU_THREAD_CS_ENTER_CONTEXTID(_context) \
MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
#define MPIU_THREAD_CS_EXIT_CONTEXTID(_context) \
MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-/* FIXME this YIELD macro probably needs to be revisited */
-#define MPIU_THREAD_CS_YIELD_CONTEXTID(_context) \
- MPIU_THREAD_CHECKDEPTH(global_mutex,1);\
- MPID_Thread_mutex_unlock(&MPIR_ThreadInfo.global_mutex);\
- MPID_Thread_yield();\
- MPID_Thread_mutex_lock(&MPIR_ThreadInfo.global_mutex);
+/* XXX DJG at the global level should CONTEXTID yield even though it
+ * doesn't do anything at ENTER/EXIT? */
+#define MPIU_THREAD_CS_YIELD_CONTEXTID(_context) MPIU_THREAD_CS_YIELD_LOCKNAME_CHECKED(global_mutex)
#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_LOCK_FREE
-/* Updates to shared data and access to shared services is handled without
+/* Updates to shared data and access to shared services is handled without
locks where ever possible. */
#error lock-free not yet implemented
@@ -615,26 +684,6 @@
#define MPIU_THREAD_CS_EXIT(_name,_context)
#define MPIU_THREAD_CS_YIELD(_name,_context)
#endif /* MPICH_IS_THREADED */
-#endif /* !defined(MPID_DEFINES_MPID_CS) */
#endif /* !defined(MPIIMPLTHREAD_H_INCLUDED) */
-/* This block of text makes it easier to add local use of the thread macros */
-# if 0
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
-/* There is a single, global lock, held for the duration of an MPI call */
-
-#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT
-/* There are multiple locks, one for each logical class (e.g., each type of
- object) */
-
-#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_LOCK_FREE
-/* Updates to shared data and access to shared services is handled without
- locks where ever possible. */
-#error lock-free not yet implemented
-
-#else
-#error Unrecognized thread granularity
-#endif
-#endif /* 0 */
-
Modified: mpich2/trunk/src/include/thread/mpiu_thread_posix_funcs.h
===================================================================
--- mpich2/trunk/src/include/thread/mpiu_thread_posix_funcs.h 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/include/thread/mpiu_thread_posix_funcs.h 2010-05-17 22:47:53 UTC (rev 6664)
@@ -26,27 +26,27 @@
/* MPIU_Thread_create() defined in mpiu_thread.c */
#define MPIU_Thread_exit() \
-{ \
+do { \
pthread_exit(NULL); \
-}
+} while (0)
#define MPIU_Thread_self(id_) \
-{ \
+do { \
*(id_) = pthread_self(); \
-}
+} while (0)
#define MPIU_Thread_same(id1_, id2_, same_) \
-{ \
+do { \
*(same_) = pthread_equal(*(id1_), *(id2_)) ? TRUE : FALSE; \
-}
+} while (0)
#define MPIU_Thread_yield() \
-{ \
+do { \
/* FIXME: need to check for different types of yield */ \
MPIU_DBG_MSG_P(THREAD,TYPICAL,"enter MPIU_Thread_yield"); \
sched_yield(); \
MPIU_DBG_MSG_P(THREAD,TYPICAL,"exit MPIU_Thread_yield"); \
-}
+} while (0)
/*
@@ -62,7 +62,7 @@
/* FIXME: using constant initializer if available */
#if !defined(MPICH_DEBUG_MUTEX) || !defined(PTHREAD_MUTEX_ERRORCHECK_VALUE)
#define MPIU_Thread_mutex_create(mutex_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
err__ = pthread_mutex_init((mutex_ptr_), NULL); \
@@ -72,10 +72,10 @@
*(int *)(err_ptr_) = err__; \
} \
MPIU_DBG_MSG_P(THREAD,TYPICAL,"Created MPIU_Thread_mutex %p", (mutex_ptr_)); \
-}
+} while (0)
#else /* MPICH_DEBUG_MUTEX */
#define MPIU_Thread_mutex_create(mutex_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
pthread_mutexattr_t attr__; \
\
@@ -95,11 +95,11 @@
*(int *)(err_ptr_) = err__; \
} \
MPIU_DBG_MSG_P(THREAD,TYPICAL,"Created MPIU_Thread_mutex %p", (mutex_ptr_)); \
-}
+} while (0)
#endif
#define MPIU_Thread_mutex_destroy(mutex_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
MPIU_DBG_MSG_P(THREAD,TYPICAL,"About to destroy MPIU_Thread_mutex %p", (mutex_ptr_)); \
@@ -109,11 +109,11 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#ifndef MPICH_DEBUG_MUTEX
#define MPIU_Thread_mutex_lock(mutex_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
MPIU_DBG_MSG_P(THREAD,TYPICAL,"enter MPIU_Thread_mutex_lock %p", (mutex_ptr_)); \
err__ = pthread_mutex_lock(mutex_ptr_); \
@@ -123,10 +123,10 @@
*(int *)(err_ptr_) = err__; \
} \
MPIU_DBG_MSG_P(THREAD,TYPICAL,"exit MPIU_Thread_mutex_lock %p", (mutex_ptr_)); \
-}
+} while (0)
#else /* MPICH_DEBUG_MUTEX */
#define MPIU_Thread_mutex_lock(mutex_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
MPIU_DBG_MSG_P(THREAD,TYPICAL,"enter MPIU_Thread_mutex_lock %p", (mutex_ptr_)); \
err__ = pthread_mutex_lock(mutex_ptr_); \
@@ -142,12 +142,12 @@
*(int *)(err_ptr_) = err__; \
} \
MPIU_DBG_MSG_P(THREAD,TYPICAL,"exit MPIU_Thread_mutex_lock %p", (mutex_ptr_)); \
-}
+} while (0)
#endif
#ifndef MPICH_DEBUG_MUTEX
#define MPIU_Thread_mutex_unlock(mutex_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
MPIU_DBG_MSG_P(THREAD,TYPICAL,"MPIU_Thread_mutex_unlock %p", (mutex_ptr_)); \
@@ -157,10 +157,10 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#else /* MPICH_DEBUG_MUTEX */
#define MPIU_Thread_mutex_unlock(mutex_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
MPIU_DBG_MSG_P(THREAD,TYPICAL,"MPIU_Thread_mutex_unlock %p", (mutex_ptr_)); \
@@ -176,12 +176,12 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#endif
#ifndef MPICH_DEBUG_MUTEX
#define MPIU_Thread_mutex_trylock(mutex_ptr_, flag_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
err__ = pthread_mutex_trylock(mutex_ptr_); \
@@ -192,10 +192,10 @@
*(int *)(err_ptr_) = (err__ == EBUSY) : MPIU_THREAD_SUCCESS ? err__;\
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#else /* MPICH_DEBUG_MUTEX */
#define MPIU_Thread_mutex_trylock(mutex_ptr_, flag_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
err__ = pthread_mutex_trylock(mutex_ptr_); \
@@ -212,7 +212,7 @@
*(int *)(err_ptr_) = (err__ == EBUSY) : MPIU_THREAD_SUCCESS ? err__;\
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#endif
/*
@@ -220,7 +220,7 @@
*/
#define MPIU_Thread_cond_create(cond_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
err__ = pthread_cond_init((cond_ptr_), NULL); \
@@ -230,10 +230,10 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#define MPIU_Thread_cond_destroy(cond_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
MPIU_DBG_MSG_P(THREAD,TYPICAL,"About to destroy MPIU_Thread_cond %p", (cond_ptr_)); \
@@ -243,10 +243,10 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#define MPIU_Thread_cond_wait(cond_ptr_, mutex_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
/* The latest pthread specification says that cond_wait routines \
@@ -268,10 +268,10 @@
} \
} \
MPIU_DBG_MSG_FMT(THREAD,TYPICAL,(MPIU_DBG_FDEST,"Exit cond_wait on cond=%p mutex=%p",(cond_ptr_),(mutex_ptr_))) \
-}
+} while (0)
#define MPIU_Thread_cond_broadcast(cond_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
MPIU_DBG_MSG_P(THREAD,TYPICAL,"About to cond_broadcast on MPIU_Thread_cond %p", (cond_ptr_)); \
@@ -282,10 +282,10 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#define MPIU_Thread_cond_signal(cond_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
MPIU_DBG_MSG_P(THREAD,TYPICAL,"About to cond_signal on MPIU_Thread_cond %p", (cond_ptr_)); \
@@ -296,7 +296,7 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
/*
@@ -304,7 +304,7 @@
*/
#define MPIU_Thread_tls_create(exit_func_ptr_, tls_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
err__ = pthread_key_create((tls_ptr_), (exit_func_ptr_)); \
@@ -314,10 +314,10 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#define MPIU_Thread_tls_destroy(tls_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
err__ = pthread_key_delete(*(tls_ptr_)); \
@@ -327,10 +327,10 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#define MPIU_Thread_tls_set(tls_ptr_, value_, err_ptr_) \
-{ \
+do { \
int err__; \
\
err__ = pthread_setspecific(*(tls_ptr_), (value_)); \
@@ -340,10 +340,10 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = err__; \
} \
-}
+} while (0)
#define MPIU_Thread_tls_get(tls_ptr_, value_ptr_, err_ptr_) \
-{ \
+do { \
*(value_ptr_) = pthread_getspecific(*(tls_ptr_)); \
\
if ((err_ptr_) != NULL) \
@@ -351,4 +351,4 @@
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
*(int *)(err_ptr_) = MPIU_THREAD_SUCCESS; \
} \
-}
+} while (0)
Modified: mpich2/trunk/src/include/thread/mpiu_thread_solaris_funcs.h
===================================================================
--- mpich2/trunk/src/include/thread/mpiu_thread_solaris_funcs.h 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/include/thread/mpiu_thread_solaris_funcs.h 2010-05-17 22:47:53 UTC (rev 6664)
@@ -12,24 +12,24 @@
/* MPIU_Thread_create() defined in mpiu_thread.c */
#define MPIU_Thread_exit() \
-{ \
+do { \
thr_exit(NULL); \
-}
+} while (0)
#define MPIU_Thread_self(id_ptr_) \
-{ \
+do { \
*(id_ptr_) = thr_self(); \
-}
+} while (0)
#define MPIU_Thread_same(id1_ptr_, id2_ptr_, same_ptr_) \
-{ \
+do { \
*(same_ptr_) = (*(id1_ptr_) == *(id2_ptr_)) ? TRUE : FALSE; \
-}
+} while (0)
#define MPIU_Thread_yield() \
-{ \
+do { \
thr_yield(); \
-}
+} while (0)
/*
@@ -37,16 +37,16 @@
*/
#define MPIU_Thread_mutex_create(mutex_ptr_, err_ptr_) \
-{ \
+do { \
*(mutex_ptr_) = DEFAULTMUTEX; \
if ((err_ptr_) == NULL) \
{ \
*(err_ptr_) = MPIU_THREAD_SUCCESS; \
} \
-}
+} while (0)
#define MPIU_Thread_mutex_destroy(mutex_ptr_, err_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
mutex_destroy(mutex_ptr_); \
@@ -56,10 +56,10 @@
*(err_ptr_) = mutex_destroy(mutex_ptr_); \
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#define MPIU_Thread_mutex_lock(mutex_ptr_, err_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
mutex_lock(mutex_ptr_); \
@@ -69,10 +69,10 @@
*(err_ptr_) = mutex_lock(mutex_ptr_); \
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#define MPIU_Thread_mutex_unlock(mutex_ptr_, err_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
mutex_unlock(mutex_ptr_); \
@@ -82,10 +82,10 @@
*(err_ptr_) = mutex_unlock(mutex_ptr_); \
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#define MPIU_Thread_mutex_trylock(mutex_ptr_, flag_ptr_, err_ptr_) \
-{ \
+do { \
int err__; \
\
err__ = mutex_trylock(mutex_ptr_); \
@@ -95,7 +95,7 @@
*(err_ptr_) = (err__ == EBUSY) : MPIU_THREAD_SUCCESS ? err__; \
/* FIXME: convert error to an MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
/*
@@ -103,16 +103,16 @@
*/
#define MPIU_Thread_cond_create(cond_ptr_, err_ptr_) \
-{ \
+do { \
*(cond_ptr_) = DEFAULTCV; \
if ((err_ptr_) == NULL) \
{ \
*(err_ptr_) = MPIU_THREAD_SUCCESS; \
} \
-}
+} while (0)
#define MPIU_Thread_cond_destroy(cond_ptr_, err_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
cond_destroy(cond_ptr_); \
@@ -122,10 +122,10 @@
*(err_ptr_) = cond_destroy(cond_ptr_); \
/* FIXME: convert error to a MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#define MPIU_Thread_cond_wait(cond_ptr_, mutex_ptr_, err_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
cond_wait((cond_ptr_), (mutex_ptr_)); \
@@ -135,10 +135,10 @@
*(err_ptr_) = cond_wait((cond_ptr_), (mutex_ptr_)); \
/* FIXME: convert error to a MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#define MPIU_Thread_cond_broadcast(cond_ptr_, err_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
cond_broadcast(cond_ptr_); \
@@ -148,10 +148,10 @@
*(err_ptr_) = cond_broadcast(cond_ptr_); \
/* FIXME: convert error to a MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#define MPIU_Thread_cond_signal(cond_ptr_, err_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
cond_signal(cond_ptr_); \
@@ -161,7 +161,7 @@
*(err_ptr_) = cond_signal(cond_ptr_); \
/* FIXME: convert error to a MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
/*
@@ -171,7 +171,7 @@
#define MPIU_Thread_tls_create(exit_func_ptr_, tls_ptr_, err_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
thr_keycreate((tls_ptr), (exit_func_ptr)); \
@@ -181,10 +181,10 @@
*(err_ptr_) = thr_keycreate((tls_ptr), (exit_func_ptr)); \
/* FIXME: convert error to a MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#define MPIU_Thread_tls_destroy(tls_ptr_, err_ptr_) \
-{ \
+do { \
/* \
* FIXME: Solaris threads does not have a key destroy. We need to create equivalent functionality to prevent a callback from \
* occuring when a thread exits after the TLS is destroyed. This is the only way to prevent subsystems that have shutdown \
@@ -194,10 +194,10 @@
{ \
*(err_ptr) = MPIU_THREAD_SUCCESS; \
} \
-}
+} while (0)
#define MPIU_Thread_tls_set(tls_ptr, value_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
thr_setspecific(*(tls_ptr), (value_)); \
@@ -207,10 +207,10 @@
*(err_ptr_) = thr_setspecific(*(tls_ptr), (value_)); \
/* FIXME: convert error to a MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
#define MPIU_Thread_tls_get(tls_ptr, value_ptr_) \
-{ \
+do { \
if ((err_ptr_) == NULL) \
{ \
thr_setspecific(*(tls_ptr), (value_ptr_)); \
@@ -220,4 +220,4 @@
*(err_ptr_) = thr_setspecific(*(tls_ptr), (value_ptr_)); \
/* FIXME: convert error to a MPIU_THREAD_ERR value */ \
} \
-}
+} while (0)
Modified: mpich2/trunk/src/mpi/errhan/errutil.c
===================================================================
--- mpich2/trunk/src/mpi/errhan/errutil.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpi/errhan/errutil.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -1308,14 +1308,18 @@
static MPID_Thread_mutex_t error_ring_mutex;
#define error_ring_mutex_create(_mpi_errno_p) MPID_Thread_mutex_create(&error_ring_mutex,_mpi_errno_p)
#define error_ring_mutex_destroy(_mpi_errno_p) MPID_Thread_mutex_destroy(&error_ring_mutex,_mpi_errno_p)
-#define error_ring_mutex_lock() \
- MPIU_THREAD_CHECK_BEGIN \
- MPID_Thread_mutex_lock(&error_ring_mutex) \
- MPIU_THREAD_CHECK_END
-#define error_ring_mutex_unlock() \
- MPIU_THREAD_CHECK_BEGIN \
- MPID_Thread_mutex_unlock(&error_ring_mutex) \
- MPIU_THREAD_CHECK_END
+#define error_ring_mutex_lock() \
+ do { \
+ MPIU_THREAD_CHECK_BEGIN \
+ MPID_Thread_mutex_lock(&error_ring_mutex); \
+ MPIU_THREAD_CHECK_END \
+ } while (0)
+#define error_ring_mutex_unlock() \
+ do { \
+ MPIU_THREAD_CHECK_BEGIN \
+ MPID_Thread_mutex_unlock(&error_ring_mutex); \
+ MPIU_THREAD_CHECK_END \
+ } while (0)
#else
#define error_ring_mutex_create(_a)
#define error_ring_mutex_destroy(_a)
Modified: mpich2/trunk/src/mpi/init/async.c
===================================================================
--- mpich2/trunk/src/mpi/init/async.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpi/init/async.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -27,11 +27,7 @@
/* Explicitly add CS_ENTER/EXIT since this thread is created from
* within an internal function and will call NMPI functions
* directly. */
- /* FIXME: The CS_ENTER/EXIT code should be abstracted out
- * correctly, instead of relying on the #if protection here. */
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
- MPID_CS_ENTER();
-#endif
+ MPIU_THREAD_CS_ENTER(ALLFUNC,);
/* FIXME: We assume that waiting on some request forces progress
* on all requests. With fine-grained threads, will this still
@@ -61,9 +57,7 @@
MPIU_Thread_cond_signal(&progress_cond, &mpi_errno);
MPIU_Assert(!mpi_errno);
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
- MPID_CS_EXIT();
-#endif
+ MPIU_THREAD_CS_EXIT(ALLFUNC,);
#endif /* MPICH_THREAD_LEVEL >= MPI_THREAD_SERIALIZED */
return;
@@ -125,9 +119,8 @@
MPIU_Assert(!mpi_errno);
MPIR_Nest_decr();
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
- MPID_CS_EXIT();
-#endif
+ /* XXX DJG why is this unlock/lock necessary? Should we just YIELD here or later? */
+ MPIU_THREAD_CS_EXIT(ALLFUNC,);
MPIU_Thread_mutex_lock(&progress_mutex, &mpi_errno);
MPIU_Assert(!mpi_errno);
@@ -140,9 +133,7 @@
MPIU_Thread_mutex_unlock(&progress_mutex, &mpi_errno);
MPIU_Assert(!mpi_errno);
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
- MPID_CS_ENTER();
-#endif
+ MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPIU_Thread_cond_destroy(&progress_cond, &mpi_errno);
MPIU_Assert(!mpi_errno);
Modified: mpich2/trunk/src/mpi/init/finalize.c
===================================================================
--- mpich2/trunk/src/mpi/init/finalize.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpi/init/finalize.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -251,7 +251,8 @@
MPIU_THREAD_CS_EXIT(ALLFUNC,);
MPIR_Process.initialized = MPICH_POST_FINALIZED;
- MPID_CS_FINALIZE();
+ /* XXX DJG needs to hide behind a macro */
+ MPIR_Thread_CS_Finalize();
/* We place the memory tracing at the very end because any of the other
steps may have allocated memory that they still need to release*/
Modified: mpich2/trunk/src/mpi/init/init.c
===================================================================
--- mpich2/trunk/src/mpi/init/init.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpi/init/init.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -80,23 +80,6 @@
MPIU_DBG_PreInit( argc, argv, rc );
#endif
- MPID_CS_INITIALIZE();
- /* FIXME: Can we get away without locking every time. Now, we
- need a MPID_CS_ENTER/EXIT around MPI_Init and MPI_Init_thread.
- Progress may be called within MPI_Init, e.g., by a spawned
- child process. Within progress, the lock is released and
- reacquired when blocking. If the lock isn't acquired before
- then, the release in progress is incorrect. Furthermore, if we
- don't release the lock after progress, we'll deadlock the next
- time this process tries to acquire the lock.
- MPID_CS_ENTER/EXIT functions are used here instead of
- MPIU_THREAD_SINGLE_CS_ENTER/EXIT because
- MPIR_ThreadInfo.isThreaded hasn't been initialized yet.
- */
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
- MPID_CS_ENTER();
-#endif
-
MPID_MPI_INIT_FUNC_ENTER(MPID_STATE_MPI_INIT);
# ifdef HAVE_ERROR_CHECKING
{
@@ -161,13 +144,9 @@
}
/* ... end of body of routine ... */
-
MPID_MPI_INIT_FUNC_EXIT(MPID_STATE_MPI_INIT);
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
- MPID_CS_EXIT();
-#endif
return mpi_errno;
-
+
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_REPORTING
@@ -178,8 +157,6 @@
}
# endif
mpi_errno = MPIR_Err_return_comm( 0, FCNAME, mpi_errno );
- MPID_CS_EXIT();
- MPID_CS_FINALIZE();
return mpi_errno;
/* --END ERROR HANDLING-- */
}
Modified: mpich2/trunk/src/mpi/init/initthread.c
===================================================================
--- mpich2/trunk/src/mpi/init/initthread.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpi/init/initthread.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -108,13 +108,15 @@
#if !defined(MPICH_IS_THREADED)
/* If single threaded, we preallocate this. Otherwise, we create it */
MPICH_PerThread_t MPIR_Thread = { 0 };
+#elif defined(MPIU_TLS_SPECIFIER)
+MPIU_TLS_SPECIFIER MPICH_PerThread_t MPIR_Thread = { 0 };
#elif defined(HAVE_RUNTIME_THREADCHECK)
/* If we may be single threaded, we need a preallocated version to use
if we are single threaded case */
MPICH_PerThread_t MPIR_ThreadSingle = { 0 };
#endif
-#if defined(MPICH_IS_THREADED) && !defined(MPID_DEFINES_MPID_CS)
+#if defined(MPICH_IS_THREADED)
/* This routine is called when a thread exits; it is passed the value
associated with the key. In our case, this is simply storage allocated
with MPIU_Calloc */
@@ -126,14 +128,21 @@
}
/* These routine handle any thread initialization that my be required */
-int MPIR_Thread_CS_Init( void )
+#undef FUNCNAME
+#define FUNCNAME MPIR_Thread_CS_Init
+#undef FCNAME
+#define FCNAME MPIU_QUOTE(FUNCNAME)
+static int MPIR_Thread_CS_Init( void )
{
- MPID_Thread_tls_create(MPIR_CleanupThreadStorage,
- &MPIR_ThreadInfo.thread_storage, NULL);
+ MPIU_THREADPRIV_DECL;
/* we create this at all granularities right now */
MPID_Thread_mutex_create(&MPIR_ThreadInfo.memalloc_mutex, NULL);
+ /* must come after memalloc_mutex creation */
+ MPIU_THREADPRIV_INITKEY;
+ MPIU_THREADPRIV_INIT;
+
#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
/* There is a single, global lock, held for the duration of an MPI call */
MPID_Thread_mutex_create(&MPIR_ThreadInfo.global_mutex, NULL);
@@ -172,6 +181,10 @@
return MPI_SUCCESS;
}
+#undef FUNCNAME
+#define FUNCNAME MPIR_Thread_CS_Finalize
+#undef FCNAME
+#define FCNAME MPIU_QUOTE(FUNCNAME)
int MPIR_Thread_CS_Finalize( void )
{
MPIU_DBG_MSG(THREAD,TYPICAL,"Freeing global mutex and private storage");
@@ -208,16 +221,19 @@
#else
#error Unrecognized thread granularity
#endif
- MPIR_ReleasePerThread; \
- MPID_Thread_tls_destroy(&MPIR_ThreadInfo.thread_storage, NULL); \
+ MPIU_THREADPRIV_FINALIZE;
+
return MPI_SUCCESS;
}
#endif /* MPICH_IS_THREADED */
-int MPIR_Init_thread(int * argc, char ***argv, int required,
- int * provided)
+#undef FUNCNAME
+#define FUNCNAME MPIR_Init_thread
+#undef FCNAME
+#define FCNAME MPIU_QUOTE(FUNCNAME)
+int MPIR_Init_thread(int * argc, char ***argv, int required, int * provided)
{
int mpi_errno = MPI_SUCCESS;
int has_args;
@@ -225,6 +241,15 @@
int thread_provided;
MPIU_THREADPRIV_DECL;
+ /* For any code in the device that wants to check for runtime
+ decisions on the value of isThreaded, set a provisional
+ value here. We could let the MPID_Init routine override this */
+#ifdef HAVE_RUNTIME_THREADCHECK
+ MPIR_ThreadInfo.isThreaded = required == MPI_THREAD_MULTIPLE;
+#endif
+
+ MPIU_THREAD_CS_INIT;
+
/* FIXME: Move to os-dependent interface? */
#ifdef HAVE_WINDOWS_H
/* prevent the process from bringing up an error message window if mpich
@@ -370,8 +395,6 @@
MPIR_Err_init();
MPIR_Datatype_init();
- MPIU_THREADPRIV_GET;
-
MPIR_Nest_init();
/* MPIU_Timer_pre_init(); */
@@ -379,28 +402,13 @@
MPID_Init if necessary */
MPIR_Process.initialized = MPICH_WITHIN_MPI;
- /* For any code in the device that wants to check for runtime
- decisions on the value of isThreaded, set a provisional
- value here. We could let the MPID_Init routine override this */
-#ifdef HAVE_RUNTIME_THREADCHECK
- MPIR_ThreadInfo.isThreaded = required == MPI_THREAD_MULTIPLE;
-#endif
+ /* We can't acquire any critical sections until this point. Any
+ * earlier the basic data structures haven't been initialized */
+ MPIU_THREAD_CS_ENTER(INIT,required);
+
mpi_errno = MPID_Init(argc, argv, required, &thread_provided,
&has_args, &has_env);
- /* --BEGIN ERROR HANDLING-- */
- if (mpi_errno != MPI_SUCCESS)
- {
- mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL,
- "MPIR_Init_thread", __LINE__, MPI_ERR_OTHER,
- "**init", 0);
- /* FIXME: the default behavior for all MPI routines is to abort.
- This isn't always convenient, because there's no other way to
- get this routine to simply return. But we should provide some
- sort of control for that and follow the default defined
- by the standard */
- return mpi_errno;
- }
- /* --END ERROR HANDLING-- */
+ if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* Capture the level of thread support provided */
MPIR_ThreadInfo.thread_provided = thread_provided;
@@ -450,18 +458,25 @@
#ifdef HAVE_DEBUGGER_SUPPORT
MPIR_WaitForDebugger();
#endif
-
+
/* Let the device know that the rest of the init process is completed */
if (mpi_errno == MPI_SUCCESS)
mpi_errno = MPID_InitCompleted();
+fn_exit:
+ MPIU_THREAD_CS_EXIT(INIT,required);
+ MPIU_THREAD_CS_FINALIZE;
return mpi_errno;
+
+fn_fail:
+ goto fn_exit;
}
#endif
#undef FUNCNAME
#define FUNCNAME MPI_Init_thread
-
+#undef FCNAME
+#define FCNAME MPIU_QUOTE(FUNCNAME)
/*@
MPI_Init_thread - Initialize the MPI execution environment
@@ -502,7 +517,6 @@
@*/
int MPI_Init_thread( int *argc, char ***argv, int required, int *provided )
{
- static const char FCNAME[] = "MPI_Init_thread";
int mpi_errno = MPI_SUCCESS;
int rc, reqd = required;
MPIU_THREADPRIV_DECL;
@@ -513,33 +527,8 @@
MPIU_DBG_PreInit( argc, argv, rc );
#endif
- MPID_CS_INITIALIZE();
- /* FIXME: Can we get away without locking every time. Now, we
- need a MPID_CS_ENTER/EXIT around MPI_Init and MPI_Init_thread.
- Progress may be called within MPI_Init, e.g., by a spawned
- child process. Within progress, the lock is released and
- reacquired when blocking. If the lock isn't acquired before
- then, the release in progress is incorrect. Furthermore, if we
- don't release the lock after progress, we'll deadlock the next
- time this process tries to acquire the lock.
- MPID_CS_ENTER/EXIT functions are used here instead of
- MPIU_THREAD_CS_ENTER/EXIT because
- MPIR_ThreadInfo.isThreaded hasn't been initialized yet.
- */
- /* */
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
- MPID_CS_ENTER();
-#endif
-
-#if 0
- /* Create the thread-private region if necessary and go ahead
- and initialize it */
- MPIU_THREADPRIV_INITKEY;
- MPIU_THREADPRIV_INIT;
-#endif
-
MPID_MPI_INIT_FUNC_ENTER(MPID_STATE_MPI_INIT_THREAD);
-
+
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
@@ -574,13 +563,10 @@
}
/* ... end of body of routine ... */
-
+
MPID_MPI_INIT_FUNC_EXIT(MPID_STATE_MPI_INIT_THREAD);
-#if MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL
- MPID_CS_EXIT();
-#endif
return mpi_errno;
-
+
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_REPORTING
@@ -593,8 +579,9 @@
# endif
mpi_errno = MPIR_Err_return_comm( 0, FCNAME, mpi_errno );
MPID_MPI_INIT_FUNC_EXIT(MPID_STATE_MPI_INIT_THREAD);
- MPID_CS_EXIT();
- MPID_CS_FINALIZE();
+
+ MPIU_THREAD_CS_EXIT(INIT,*provided);
+
return mpi_errno;
/* --END ERROR HANDLING-- */
}
Modified: mpich2/trunk/src/mpi/init/mpi_init.h
===================================================================
--- mpich2/trunk/src/mpi/init/mpi_init.h 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpi/init/mpi_init.h 2010-05-17 22:47:53 UTC (rev 6664)
@@ -8,5 +8,6 @@
int MPIR_Init_thread(int *, char ***, int, int *);
int MPIR_Init_async_thread(void);
int MPIR_Finalize_async_thread(void);
+int MPIR_Thread_CS_Finalize(void);
extern int MPIR_async_thread_initialized;
Modified: mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpid/ch3/channels/nemesis/src/ch3_progress.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -103,6 +103,7 @@
#ifdef MPICH_IS_THREADED
MPIU_THREAD_CHECK_BEGIN;
{
+ MPIU_THREADPRIV_DECL;
/* In the case of threads, we poll for lesser number of
* iterations than the case with only processes, as
* threads contend for CPU and the lock, while processes
@@ -111,9 +112,7 @@
{
pollcount = 0;
MPIDI_CH3I_progress_blocked = TRUE;
- MPID_Thread_mutex_unlock(&MPIR_ThreadInfo.global_mutex);
- MPID_Thread_yield();
- MPID_Thread_mutex_lock(&MPIR_ThreadInfo.global_mutex);
+ MPIU_THREAD_CS_YIELD(ALLFUNC,);
MPIDI_CH3I_progress_blocked = FALSE;
MPIDI_CH3I_progress_wakeup_signalled = FALSE;
}
@@ -423,7 +422,8 @@
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_DELAY);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_DELAY);
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
while (completion_count == MPIDI_CH3I_progress_completion_count && MPIDI_CH3I_progress_blocked == TRUE)
{
@@ -448,7 +448,8 @@
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_CONTINUE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_CONTINUE);
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
MPID_Thread_cond_broadcast(&MPIDI_CH3I_progress_completion_cond);
}
@@ -685,7 +686,8 @@
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_INIT);
MPIU_THREAD_CHECK_BEGIN
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
MPID_Thread_cond_create(&MPIDI_CH3I_progress_completion_cond, NULL);
}
Modified: mpich2/trunk/src/mpid/ch3/channels/sctp/src/ch3_progress.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/sctp/src/ch3_progress.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpid/ch3/channels/sctp/src/ch3_progress.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -14,13 +14,11 @@
#include "sctp_common.h"
volatile unsigned int MPIDI_CH3I_progress_completion_count = 0;
-#if (MPICH_THREAD_LEVEL == MPI_THREAD_MULTIPLE)
+#if defined(MPICH_IS_THREADED
volatile int MPIDI_CH3I_progress_blocked = FALSE;
volatile int MPIDI_CH3I_progress_wakeup_signalled = FALSE;
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
-MPID_Thread_cond_t MPIDI_CH3I_progress_completion_cond;
-# endif
+ MPID_Thread_cond_t MPIDI_CH3I_progress_completion_cond;
static int MPIDI_CH3I_Progress_delay(unsigned int completion_count);
static int MPIDI_CH3I_Progress_continue(unsigned int completion_count);
#endif
@@ -129,7 +127,7 @@
*
* This is presently not possible, and thus the code is commented out.
*/
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_NOT_IMPLEMENTED)
+# if !defined(MPICH_IS_THREADED)
{
if (progress_state->ch.completion_count != MPIDI_CH3I_progress_completion_count)
{
@@ -260,7 +258,8 @@
MPIDI_DBG_PRINTF((60, FCNAME, "entering"));
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
MPID_Thread_cond_create(&MPIDI_CH3I_progress_completion_cond, NULL);
}
@@ -320,7 +319,8 @@
/* finalize hash table */
hash_free(MPIDI_CH3I_assocID_table);
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
MPID_Thread_cond_destroy(&MPIDI_CH3I_progress_completion_cond, NULL);
}
@@ -866,7 +866,7 @@
/* end MPIDI_CH3I_Progress_handle_sctp_event() */
-#if (MPICH_THREAD_LEVEL == MPI_THREAD_MULTIPLE)
+#if defined(MPICH_IS_THREADED)
#undef FUNCNAME
#define FUNCNAME MPIDI_CH3I_Progress_delay
@@ -876,7 +876,8 @@
{
int mpi_errno = MPI_SUCCESS;
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
while (completion_count == MPIDI_CH3I_progress_completion_count)
{
@@ -898,7 +899,8 @@
{
int mpi_errno = MPI_SUCCESS;
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
MPID_Thread_cond_broadcast(&MPIDI_CH3I_progress_completion_cond);
}
@@ -908,7 +910,7 @@
}
/* end MPIDI_CH3I_Progress_continue() */
-#endif /* (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL) */
+#endif /* defined(MPICH_IS_THREADED) */
#define MPIDI_MAX_KVS_KEY_LEN 256
Modified: mpich2/trunk/src/mpid/ch3/channels/sock/src/ch3_progress.c
===================================================================
--- mpich2/trunk/src/mpid/ch3/channels/sock/src/ch3_progress.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpid/ch3/channels/sock/src/ch3_progress.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -26,11 +26,9 @@
volatile int MPIDI_CH3I_progress_blocked = FALSE;
volatile int MPIDI_CH3I_progress_wakeup_signalled = FALSE;
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
-/* This value must be static so that it isn't an uninitialized
- common symbol */
-static MPID_Thread_cond_t MPIDI_CH3I_progress_completion_cond;
-# endif
+ /* This value must be static so that it isn't an uninitialized
+ common symbol */
+ static MPID_Thread_cond_t MPIDI_CH3I_progress_completion_cond;
static int MPIDI_CH3I_Progress_delay(unsigned int completion_count);
static int MPIDI_CH3I_Progress_continue(unsigned int completion_count);
@@ -286,7 +284,8 @@
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_INIT);
MPIU_THREAD_CHECK_BEGIN
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
MPID_Thread_cond_create(&MPIDI_CH3I_progress_completion_cond, NULL);
}
@@ -351,7 +350,8 @@
MPIDU_Sock_finalize();
MPIU_THREAD_CHECK_BEGIN
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
MPID_Thread_cond_destroy(&MPIDI_CH3I_progress_completion_cond, NULL);
}
@@ -669,7 +669,8 @@
{
int mpi_errno = MPI_SUCCESS;
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
while (completion_count == MPIDI_CH3I_progress_completion_count)
{
@@ -693,7 +694,8 @@
int mpi_errno = MPI_SUCCESS;
MPIU_THREAD_CHECK_BEGIN
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+ /* FIXME should be appropriately abstracted somehow */
+# if defined(MPICH_IS_THREADED) && (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
MPID_Thread_cond_broadcast(&MPIDI_CH3I_progress_completion_cond);
}
Modified: mpich2/trunk/src/mpid/ch3/include/mpidimpl.h
===================================================================
--- mpich2/trunk/src/mpid/ch3/include/mpidimpl.h 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpid/ch3/include/mpidimpl.h 2010-05-17 22:47:53 UTC (rev 6664)
@@ -1806,17 +1806,9 @@
#define MPIU_THREAD_CS_ENTER_CH3COMM(_context)
#define MPIU_THREAD_CS_EXIT_CH3COMM(_context)
-/* FIXME: Currently forcing the PER_OBJECT case to do a global
- * lock. We need a better way of fixing this. */
-#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_BRIEF_GLOBAL
-/* There is a single, global lock, held only when needed */
-#define MPIU_THREAD_CS_ENTER_CH3COMM(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-#define MPIU_THREAD_CS_EXIT_CH3COMM(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-
+/* XXX DJG PER_OBJECT needs lots of love down here at the device and channel levels */
#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_PER_OBJECT
-#if 1
+
/* There is a per object lock */
#define MPIU_THREAD_CS_ENTER_CH3COMM(_context) {\
MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_POBJ_LOCKNAME(_context->pobj_mutex) MPIU_THREAD_CHECK_END \
@@ -1824,8 +1816,9 @@
#define MPIU_THREAD_CS_EXIT_CH3COMM(_context) \
MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_POBJ_LOCKNAME(_context->pobj_mutex) MPIU_THREAD_CHECK_END
-#if 1
-static void foofunc() { }
+/* XXX DJG what is this junk all about? */
+#if 0
+/*static void foofunc() { }*/
#define MPIU_THREAD_CS_TRYLOCK(_context) {\
MPIU_THREAD_CHECK_BEGIN \
int ret; \
@@ -1839,7 +1832,7 @@
} \
ret = pthread_mutex_trylock(&MPIR_ThreadInfo.global_mutex); \
if (ret) { \
- foofunc(); \
+ /*foofunc();*/ \
printf("Trylock not successful for global mutex\n"); \
} \
else if (!ret) { \
@@ -1865,14 +1858,6 @@
#define MPIU_THREAD_CS_TRYLOCK(_context)
#endif
-#else
-/* There is a single, global lock, held only when needed */
-#define MPIU_THREAD_CS_ENTER_CH3COMM(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_ENTER_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-#define MPIU_THREAD_CS_EXIT_CH3COMM(_context) \
- MPIU_THREAD_CHECK_BEGIN MPIU_THREAD_CS_EXIT_LOCKNAME(global_mutex) MPIU_THREAD_CHECK_END
-#endif
-
#elif MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_LOCK_FREE
/* Updates to shared data and access to shared services is handled without
locks where ever possible. */
Modified: mpich2/trunk/src/mpid/common/sock/iocp/sock.c
===================================================================
--- mpich2/trunk/src/mpid/common/sock/iocp/sock.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpid/common/sock/iocp/sock.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -1801,8 +1801,8 @@
for (;;)
{
-#if (MPICH_THREAD_LEVEL == MPI_THREAD_MULTIPLE)
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+#if defined(MPICH_IS_THREADED)
+# if (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
/* Release the lock so that other threads may make progress while this thread waits for something to do */
MPIU_DBG_MSG(THREAD,TYPICAL,"Exit global critical section");
@@ -1825,8 +1825,8 @@
/*t2 = PMPI_Wtime();*/
/*printf("[%d] GetQueuedCompletionStatus took %.3f seconds for sock: %d\n", getpid(), t2-t1, sock->sock);*/
MPIDI_FUNC_EXIT(MPID_STATE_GETQUEUEDCOMPLETIONSTATUS);
-#if (MPICH_THREAD_LEVEL == MPI_THREAD_MULTIPLE)
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+#if defined(MPICH_IS_THREADED)
+# if (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
/* Reaquire the lock before processing any of the information returned from GetQueuedCompletionStatus */
MPIU_DBG_MSG(THREAD,TYPICAL,"Enter global critical section");
@@ -2423,8 +2423,8 @@
}
else
{
-#if (MPICH_THREAD_LEVEL == MPI_THREAD_MULTIPLE)
-# if (USE_THREAD_IMPL == MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+#if defined(MPICH_IS_THREADED)
+# if (MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
{
/* Reaquire the lock before processing any of the information returned from GetQueuedCompletionStatus */
MPIU_DBG_MSG(THREAD,TYPICAL,"Enter global critical section");
Modified: mpich2/trunk/src/mpid/common/sock/poll/sock_wait.i
===================================================================
--- mpich2/trunk/src/mpid/common/sock/poll/sock_wait.i 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpid/common/sock/poll/sock_wait.i 2010-05-17 22:47:53 UTC (rev 6664)
@@ -7,7 +7,7 @@
/* Make sure that we can properly ensure atomic access to the poll routine */
#ifdef MPICH_IS_THREADED
-#if (USE_THREAD_IMPL != MPICH_THREAD_IMPL_GLOBAL_MUTEX)
+#if !(MPIU_THREAD_GRANULARITY == MPIU_THREAD_GRANULARITY_GLOBAL)
#error selected multi-threaded implementation is not supported
#endif
#endif
Modified: mpich2/trunk/src/mpid/common/thread/mpe_funcs.i
===================================================================
--- mpich2/trunk/src/mpid/common/thread/mpe_funcs.i 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpid/common/thread/mpe_funcs.i 2010-05-17 22:47:53 UTC (rev 6664)
@@ -10,29 +10,29 @@
*/
#define MPID_Thread_create(func_, data_, id_, err_) \
-{ \
+do { \
MPIU_Thread_create((func_), (data_), (id_), (err_)); \
-}
+} while (0)
#define MPID_Thread_exit() \
-{ \
+do { \
MPIU_Thread_exit(); \
-}
+} while (0)
#define MPID_Thread_self(id_) \
-{ \
+do { \
MPIU_Thread_self(id_); \
-}
+} while (0)
#define MPID_Thread_same(id1_, id2_, same_) \
-{ \
+do { \
MPIU_Thread_same((id1_), (id2_), (same_)); \
-}
+} while (0)
#define MPID_Thread_yield() \
-{ \
+do { \
MPIU_Thread_yield(); \
-}
+} while (0)
/*
@@ -40,55 +40,55 @@
*/
#define MPID_Thread_mutex_create(mutex_, err_) \
-{ \
+do { \
MPIU_Thread_mutex_create((mutex_), (err_)); \
-}
+} while (0)
#define MPID_Thread_mutex_destroy(mutex_, err_) \
-{ \
+do { \
MPIU_Thread_mutex_destroy((mutex_), (err_)); \
-}
+} while (0)
#if !defined(MPID_THREAD_DEBUG)
#define MPID_Thread_mutex_lock(mutex_) \
-{ \
+do { \
MPIU_Thread_mutex_lock((mutex_), NULL); \
-}
+} while (0)
#else
#define MPID_Thread_mutex_lock(mutex_) \
-{ \
+do { \
int err_; \
MPIU_Thread_mutex_lock((mutex_), &err_); \
MPIU_Assert(err_ == MPIU_THREAD_SUCCESS); \
-}
+} while (0)
#endif
#if !defined(MPID_THREAD_DEBUG)
#define MPID_Thread_mutex_unlock(mutex_) \
-{ \
+do { \
MPIU_Thread_mutex_unlock((mutex_), NULL); \
-}
+} while (0)
#else
#define MPID_Thread_mutex_unlock(mutex_) \
-{ \
+do { \
int err_; \
MPIU_Thread_mutex_unlock((mutex_), &err_); \
MPIU_Assert(err_ == MPIU_THREAD_SUCCESS); \
-}
+} while (0)
#endif
#if !defined(MPID_THREAD_DEBUG)
#define MPID_Thread_mutex_trylock(mutex_, flag_) \
-{ \
+do { \
MPIU_Thread_mutex_trylock((mutex_), (flag_), NULL); \
-}
+} while (0)
#else
#define MPID_Thread_mutex_trylock(mutex_, flag_) \
-{ \
+do { \
int err_; \
MPIU_Thread_mutex_trylock((mutex_), (flag_), &err_); \
MPIU_Assert(err_ == MPIU_THREAD_SUCCESS); \
-}
+} while (0)
#endif
@@ -97,55 +97,55 @@
*/
#define MPID_Thread_cond_create(cond_, err_) \
-{ \
+do { \
MPIU_Thread_cond_create((cond_), (err_)); \
-}
+} while (0)
#define MPID_Thread_cond_destroy(cond_, err_) \
-{ \
+do { \
MPIU_Thread_cond_destroy((cond_), (err_)); \
-}
+} while (0)
#if !defined(MPID_THREAD_DEBUG)
#define MPID_Thread_cond_wait(cond_, mutex_) \
-{ \
+do { \
MPIU_Thread_cond_wait((cond_), (mutex_), NULL); \
-}
+} while (0)
#else
#define MPID_Thread_cond_wait(cond_, mutex_) \
-{ \
+do { \
int err_; \
MPIU_Thread_cond_wait((cond_), (mutex_), &err_); \
MPIU_Assert(err_ == MPIU_THREAD_SUCCESS); \
-}
+} while (0)
#endif
#if !defined(MPID_THREAD_DEBUG)
#define MPID_Thread_cond_broadcast(cond_) \
-{ \
+do { \
MPIU_Thread_cond_broadcast(cond_, NULL); \
-}
+} while (0)
#else
#define MPID_Thread_cond_broadcast(cond_) \
-{ \
+do { \
int err_; \
MPIU_Thread_cond_broadcast((cond_), &err_); \
MPIU_Assert(err_ == MPIU_THREAD_SUCCESS); \
-}
+} while (0)
#endif
#if !defined(MPID_THREAD_DEBUG)
#define MPID_Thread_cond_signal(cond_) \
-{ \
+do { \
MPIU_Thread_cond_signal(cond_, NULL); \
-}
+} while (0)
#else
#define MPID_Thread_cond_signal(cond_) \
-{ \
+do { \
int err_; \
MPIU_Thread_cond_signal((cond_), &err_); \
MPIU_Assert(err_ == MPIU_THREAD_SUCCESS); \
-}
+} while (0)
#endif
@@ -154,42 +154,42 @@
*/
#define MPID_Thread_tls_create(exit_func_, tls_, err_) \
-{ \
+do { \
MPIU_Thread_tls_create((exit_func_), (tls_), (err_)); \
-}
+} while (0)
#define MPID_Thread_tls_destroy(tls_, err_) \
-{ \
+do { \
MPIU_Thread_tls_destroy((tls_), (err_)); \
-}
+} while (0)
#if !defined(MPID_THREAD_DEBUG)
#define MPID_Thread_tls_set(tls_, value_) \
-{ \
+do { \
MPIU_Thread_tls_set((tls_), (value_), NULL); \
-}
+} while (0)
#else
#define MPID_Thread_tls_set(tls_, value_) \
-{ \
+do { \
int err_; \
MPIU_Thread_tls_set((tls_), (value_), &err_); \
MPIU_Assert(err_ == MPIU_THREAD_SUCCESS); \
-}
+} while (0)
#endif
#if !defined(MPID_THREAD_DEBUG)
#define MPID_Thread_tls_get(tls_, value_) \
-{ \
+do { \
MPIU_Thread_tls_get((tls_), (value_), NULL); \
-}
+} while (0)
#else
#define MPID_Thread_tls_get(tls_, value_) \
-{ \
+do { \
int err_; \
\
MPIU_Thread_tls_get((tls_), (value_), &err_); \
MPIU_Assert(err_ == MPIU_THREAD_SUCCESS); \
-}
+} while (0)
#endif
Modified: mpich2/trunk/src/mpid/dcmfd/include/mpidthread.h
===================================================================
--- mpich2/trunk/src/mpid/dcmfd/include/mpidthread.h 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/mpid/dcmfd/include/mpidthread.h 2010-05-17 22:47:53 UTC (rev 6664)
@@ -86,6 +86,8 @@
#endif
+/* NOTE, this is incompatible with the MPIU_ISTHREADED definition expected at
+ * the top level */
#define MPIU_ISTHREADED(_s) { MPIU_THREAD_CHECK_BEGIN _s MPIU_THREAD_CHECK_END }
#endif /* !MPICH_MPIDTHREAD_H_INCLUDED */
Modified: mpich2/trunk/src/util/dbg/dbg_printf.c
===================================================================
--- mpich2/trunk/src/util/dbg/dbg_printf.c 2010-05-17 18:16:27 UTC (rev 6663)
+++ mpich2/trunk/src/util/dbg/dbg_printf.c 2010-05-17 22:47:53 UTC (rev 6664)
@@ -385,7 +385,7 @@
#ifdef MPICH_IS_THREADED
/* if we're not initialized, use the static fp, since there should
* only be one thread in here until then */
- if (mpiu_dbg_initialized == MPIU_DBG_INITIALIZED && MPIU_ISTHREADED()) {
+ if (mpiu_dbg_initialized == MPIU_DBG_INITIALIZED && MPIU_ISTHREADED) {
FILE *fp;
MPID_Thread_tls_get(&dbg_tls_key, &fp);
return fp;
@@ -402,7 +402,7 @@
#ifdef MPICH_IS_THREADED
/* if we're not initialized, use the static fp, since there should
* only be one thread in here until then */
- if (mpiu_dbg_initialized == MPIU_DBG_INITIALIZED && MPIU_ISTHREADED()) {
+ if (mpiu_dbg_initialized == MPIU_DBG_INITIALIZED && MPIU_ISTHREADED) {
MPID_Thread_tls_set(&dbg_tls_key, (void *)fp);
}
else
More information about the mpich2-commits
mailing list