[Darshan-commits] [Git][darshan/darshan][mmap-dev] 7 commits: bug fix in darshan put_header routine
Shane Snyder
xgitlab at cels.anl.gov
Tue Jan 26 13:06:48 CST 2016
Shane Snyder pushed to branch mmap-dev at darshan / darshan
Commits:
ac6c6442 by Shane Snyder at 2016-01-22T10:08:27Z
bug fix in darshan put_header routine
- - - - -
d5abad0f by Shane Snyder at 2016-01-22T10:53:52Z
remove profiling macros from posix module
- - - - -
bb25083b by Shane Snyder at 2016-01-25T13:19:59Z
rename merge log utility
- - - - -
e81a9cae by Shane Snyder at 2016-01-25T15:56:06Z
autconf/runtime changes for configuring mmap code
- - - - -
8070c831 by Shane Snyder at 2016-01-26T12:59:32Z
update command line args for log merging utility
- - - - -
b4be0abd by Shane Snyder at 2016-01-26T13:00:11Z
allreduce job start time in darshan-core init.
- - - - -
5f4e06f6 by Shane Snyder at 2016-01-26T13:02:54Z
autogenerate an example epilog script
- - - - -
11 changed files:
- darshan-runtime/Makefile.in
- darshan-runtime/configure
- darshan-runtime/configure.in
- darshan-runtime/darshan-core.h
- darshan-runtime/darshan-runtime-config.h.in
- darshan-runtime/lib/darshan-core.c
- darshan-runtime/lib/darshan-posix.c
- + darshan-runtime/share/darshan-mmap-epilog.sh.in
- darshan-util/Makefile.in
- darshan-util/darshan-logutils.c
- darshan-util/darshan-stitch-logs.c → darshan-util/darshan-merge.c
Changes:
=====================================
darshan-runtime/Makefile.in
=====================================
--- a/darshan-runtime/Makefile.in
+++ b/darshan-runtime/Makefile.in
@@ -154,6 +154,7 @@ endif
install -m 755 share/mpi-profile/darshan-bg-cc.conf $(datarootdir)/mpi-profile/darshan-bg-cc.conf
install -m 755 share/mpi-profile/darshan-bg-cxx.conf $(datarootdir)/mpi-profile/darshan-bg-cxx.conf
install -m 755 share/mpi-profile/darshan-bg-f.conf $(datarootdir)/mpi-profile/darshan-bg-f.conf
+ install -m 755 share/darshan-mmap-epilog.sh $(datarootdir)/darshan-mmap-epilog.sh
install -m 644 darshan-base-ld-opts $(datarootdir)/darshan-base-ld-opts
install -m 644 $(srcdir)/darshan-posix-ld-opts $(datarootdir)/darshan-posix-ld-opts
install -m 644 $(srcdir)/darshan-hdf5-ld-opts $(datarootdir)/darshan-hdf5-ld-opts
=====================================
darshan-runtime/configure
=====================================
--- a/darshan-runtime/configure
+++ b/darshan-runtime/configure
@@ -684,15 +684,16 @@ ac_subst_files=''
ac_user_opts='
enable_option_checking
with_zlib
-enable_cuserid
enable_ld_preload
+enable_cuserid
enable_group_readable_logs
+enable_bgq_mod
+enable_mmap_logs
with_mem_align
with_log_path_by_env
with_log_hints
with_log_path
with_jobid_env
-enable_bgq_mod
with_mod_mem
'
ac_precious_vars='build_alias
@@ -1313,10 +1314,12 @@ Optional Features:
--disable-option-checking ignore unrecognized --enable/--with options
--disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no)
--enable-FEATURE[=ARG] include FEATURE [ARG=yes]
- --disable-cuserid Disables attempted use of cuserid() at run time
--disable-ld-preload Disables support for LD_PRELOAD library
- --enable-group-readable-logs Set log files to be group readable
+ --disable-cuserid Disables attempted use of cuserid() at run time
+ --enable-group-readable-logs
+ Set log files to be group readable
--disable-bgq-mod Disables compilation and use of BG/Q module (for BG/Q systems)
+ --enable-mmmap-logs Enables ability to mmap I/O data to log file
Optional Packages:
--with-PACKAGE[=ARG] use PACKAGE [ARG=yes]
@@ -1325,13 +1328,16 @@ Optional Packages:
/usr/local or /usr if not found in /usr/local
--without-zlib to disable zlib usage completely
--with-mem-align=<num> Memory alignment in bytes
- --with-log-path-by-env=<env var list> Comma separated list of environment variables to check for log path location before --with-log-path path
- --with-log-hints=<hint1=x;hint2=y> Semicolon-separated list of MPI-IO hints for log file write
+ --with-log-path-by-env=<env var list>
+ Comma separated list of environment variables to check for
+ log path location before --with-log-path path
+ --with-log-hints=<hint1=x;hint2=y>
+ Semicolon-separated list of MPI-IO hints for log file write
--with-log-path=<path> Location to store log files at run time
--with-jobid-env=<name> Name of environment variable that stores the jobid
- (specify "NONE" if no appropriate environment variable is available:
- Darshan will use rank 0's pid instead)
- --with-mod-mem=<num> Maximum amount of memory (in MiB) for each Darshan module
+ (specify "NONE" if no appropriate environment variable
+ is available: Darshan will use rank 0's pid instead)
+ --with-mod-mem=<num> Maximum amount of memory (in MiB) for each Darshan module
Some influential environment variables:
CC C compiler command
@@ -3528,21 +3534,21 @@ fi
-# Check whether --enable-cuserid was given.
-if test "${enable_cuserid+set}" = set; then :
- enableval=$enable_cuserid; if test "x$enableval" = "xno" ; then
-
-$as_echo "#define DARSHAN_DISABLE_CUSERID 1" >>confdefs.h
-,
+# Check whether --enable-ld-preload was given.
+if test "${enable_ld_preload+set}" = set; then :
+ enableval=$enable_ld_preload; if test "x$enableval" = "xno" ; then
+ DISABLE_LDPRELOAD="1"
fi
fi
-# Check whether --enable-ld-preload was given.
-if test "${enable_ld_preload+set}" = set; then :
- enableval=$enable_ld_preload; if test "x$enableval" = "xno" ; then
- DISABLE_LDPRELOAD="1"
+# Check whether --enable-cuserid was given.
+if test "${enable_cuserid+set}" = set; then :
+ enableval=$enable_cuserid; if test "x$enableval" = "xno" ; then
+
+$as_echo "#define __DARSHAN_DISABLE_CUSERID 1" >>confdefs.h
+
fi
fi
@@ -3559,6 +3565,26 @@ fi
fi
+# Check whether --enable-bgq_mod was given.
+if test "${enable_bgq_mod+set}" = set; then :
+ enableval=$enable_bgq_mod;
+else
+ enable_bgq_mod=check
+
+fi
+
+
+# Check whether --enable-mmap-logs was given.
+if test "${enable_mmap_logs+set}" = set; then :
+ enableval=$enable_mmap_logs; if test "x$enableval" = "xyes" ; then
+
+$as_echo "#define __DARSHAN_ENABLE_MMAP_LOGS 1" >>confdefs.h
+
+fi
+
+fi
+
+
# Check whether --with-mem-align was given.
if test "${with_mem_align+set}" = set; then :
@@ -3648,6 +3674,54 @@ _ACEOF
fi
+
+# Check whether --with-mod-mem was given.
+if test "${with_mod_mem+set}" = set; then :
+ withval=$with_mod_mem; if test x$withval = xyes; then
+ as_fn_error $? "--with-mod-mem must be given a number" "$LINENO" 5
+ else
+
+cat >>confdefs.h <<_ACEOF
+#define __DARSHAN_MOD_MEM_MAX ${withval}
+_ACEOF
+
+ fi
+
+fi
+
+
+if test x$enable_bgq_mod != xno; then
+ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for BG/Q environment" >&5
+$as_echo_n "checking for BG/Q environment... " >&6; }
+ cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h. */
+
+ #ifndef __bgq__
+ #error __bgq__ not set
+ #endif
+
+int
+main ()
+{
+
+ ;
+ return 0;
+}
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+ DARSHAN_USE_BGQ=1
+else
+ { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+ if test "x$enable_bgq_mod" = xyes; then :
+ as_fn_error $? "BG/Q module enabled in non-BG/Q environment" "$LINENO" 5
+fi
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
if test x$GOT_ALIGNMENT != x1; then
as_fn_error $? "must provide --with-mem-align=<num> argument to configure." "$LINENO" 5
fi
@@ -4239,63 +4313,6 @@ $as_echo "no" >&6; }
MPICH_LIB_OLD=0
fi
-# check to see whether the bgq instrumentation module should be built
-# Check whether --enable-bgq_mod was given.
-if test "${enable_bgq_mod+set}" = set; then :
- enableval=$enable_bgq_mod;
-else
- enable_bgq_mod=check
-
-fi
-
-if test x$enable_bgq_mod != xno; then
- { $as_echo "$as_me:${as_lineno-$LINENO}: checking for BG/Q environment" >&5
-$as_echo_n "checking for BG/Q environment... " >&6; }
- cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h. */
-
- #ifndef __bgq__
- #error __bgq__ not set
- #endif
-
-int
-main ()
-{
-
- ;
- return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
- DARSHAN_USE_BGQ=1
-else
- { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
- if test "x$enable_bgq_mod" = xyes; then :
- as_fn_error $? "BG/Q module enabled in non-BG/Q environment" "$LINENO" 5
-fi
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-fi
-
-
-# Check whether --with-mod-mem was given.
-if test "${with_mod_mem+set}" = set; then :
- withval=$with_mod_mem; if test x$withval = xyes; then
- as_fn_error $? "--with-mod-mem must be given a number" "$LINENO" 5
- else
-
-cat >>confdefs.h <<_ACEOF
-#define __DARSHAN_MOD_MEM_MAX ${withval}
-_ACEOF
-
- fi
-
-fi
-
-
DARSHAN_VERSION="3.0.0-pre3"
@@ -4305,8 +4322,7 @@ DARSHAN_VERSION="3.0.0-pre3"
-
-ac_config_files="$ac_config_files Makefile darshan-mk-log-dirs.pl darshan-gen-cc.pl darshan-gen-cxx.pl darshan-gen-fortran.pl darshan-base-ld-opts darshan-config share/craype-1.x/darshan-module share/craype-2.x/darshan-module lib/pkgconfig/darshan-runtime.pc share/mpi-profile/darshan-cc.conf share/mpi-profile/darshan-cxx.conf share/mpi-profile/darshan-f.conf share/mpi-profile/darshan-bg-cc.conf share/mpi-profile/darshan-bg-cxx.conf share/mpi-profile/darshan-bg-f.conf"
+ac_config_files="$ac_config_files Makefile darshan-mk-log-dirs.pl darshan-gen-cc.pl darshan-gen-cxx.pl darshan-gen-fortran.pl darshan-base-ld-opts darshan-config share/craype-1.x/darshan-module share/craype-2.x/darshan-module lib/pkgconfig/darshan-runtime.pc share/mpi-profile/darshan-cc.conf share/mpi-profile/darshan-cxx.conf share/mpi-profile/darshan-f.conf share/mpi-profile/darshan-bg-cc.conf share/mpi-profile/darshan-bg-cxx.conf share/mpi-profile/darshan-bg-f.conf share/darshan-mmap-epilog.sh"
cat >confcache <<\_ACEOF
# This file is a shell script that caches the results of configure
@@ -5016,6 +5032,7 @@ do
"share/mpi-profile/darshan-bg-cc.conf") CONFIG_FILES="$CONFIG_FILES share/mpi-profile/darshan-bg-cc.conf" ;;
"share/mpi-profile/darshan-bg-cxx.conf") CONFIG_FILES="$CONFIG_FILES share/mpi-profile/darshan-bg-cxx.conf" ;;
"share/mpi-profile/darshan-bg-f.conf") CONFIG_FILES="$CONFIG_FILES share/mpi-profile/darshan-bg-f.conf" ;;
+ "share/darshan-mmap-epilog.sh") CONFIG_FILES="$CONFIG_FILES share/darshan-mmap-epilog.sh" ;;
*) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;;
esac
=====================================
darshan-runtime/configure.in
=====================================
--- a/darshan-runtime/configure.in
+++ b/darshan-runtime/configure.in
@@ -24,27 +24,42 @@ AC_TRY_COMPILE([#include <mpi.h>], [int ret = MPI_Init(0, (void*)0)],
dnl runtime libraries require zlib
CHECK_ZLIB
-AC_ARG_ENABLE(cuserid,
-[ --disable-cuserid Disables attempted use of cuserid() at run time],
+AC_ARG_ENABLE(ld-preload,
+[ --disable-ld-preload Disables support for LD_PRELOAD library],
[if test "x$enableval" = "xno" ; then
- AC_DEFINE(DARSHAN_DISABLE_CUSERID, 1, Define if cuserid() should be disabled),
+ DISABLE_LDPRELOAD="1"
fi]
,)
-AC_ARG_ENABLE(ld-preload,
-[ --disable-ld-preload Disables support for LD_PRELOAD library],
+AC_ARG_ENABLE(cuserid,
+[ --disable-cuserid Disables attempted use of cuserid() at run time],
[if test "x$enableval" = "xno" ; then
- DISABLE_LDPRELOAD="1"
+ AC_DEFINE(__DARSHAN_DISABLE_CUSERID, 1, Define if cuserid() should be disabled)
fi]
,)
AC_ARG_ENABLE(group-readable-logs,
-[ --enable-group-readable-logs Set log files to be group readable],
+[ --enable-group-readable-logs
+ Set log files to be group readable],
[if test "x$enableval" = "xyes" ; then
AC_DEFINE(__DARSHAN_GROUP_READABLE_LOGS, 1, Define if Darshan should set log files to be group readable)
fi]
,)
+AC_ARG_ENABLE(
+ [bgq_mod],
+ [ --disable-bgq-mod Disables compilation and use of BG/Q module (for BG/Q systems)],
+ [],
+ [enable_bgq_mod=check]
+)
+
+AC_ARG_ENABLE(mmap-logs,
+[ --enable-mmmap-logs Enables ability to mmap I/O data to log file],
+[if test "x$enableval" = "xyes" ; then
+ AC_DEFINE(__DARSHAN_ENABLE_MMAP_LOGS, 1, Define if Darshan should mmap data structures to log file)
+fi]
+,)
+
AC_ARG_WITH(mem-align,
[ --with-mem-align=<num> Memory alignment in bytes],
if test x$withval = xyes; then
@@ -56,7 +71,9 @@ AC_ARG_WITH(mem-align,
)
AC_ARG_WITH(log-path-by-env,
-[ --with-log-path-by-env=<env var list> Comma separated list of environment variables to check for log path location before --with-log-path path],
+[ --with-log-path-by-env=<env var list>
+ Comma separated list of environment variables to check for
+ log path location before --with-log-path path],
if test x$withval = xyes; then
AC_MSG_ERROR(--with-log-path-by-env must be given at least one variable name)
else
@@ -67,7 +84,8 @@ AC_ARG_WITH(log-path-by-env,
)
AC_ARG_WITH(log-hints,
-[ --with-log-hints=<hint1=x;hint2=y> Semicolon-separated list of MPI-IO hints for log file write],
+[ --with-log-hints=<hint1=x;hint2=y>
+ Semicolon-separated list of MPI-IO hints for log file write],
if test x$withval = xyes; then
AC_MSG_ERROR(--with-log-hints must be given an argument)
else
@@ -90,8 +108,8 @@ AC_ARG_WITH(log-path,
AC_ARG_WITH(jobid-env,
[ --with-jobid-env=<name> Name of environment variable that stores the jobid
- (specify "NONE" if no appropriate environment variable is available:
- Darshan will use rank 0's pid instead)],
+ (specify "NONE" if no appropriate environment variable
+ is available: Darshan will use rank 0's pid instead)],
if test x$withval = xyes; then
AC_MSG_ERROR(--with-jobid-env must be given a name)
else
@@ -101,6 +119,30 @@ AC_ARG_WITH(jobid-env,
fi
)
+AC_ARG_WITH(mod-mem,
+[ --with-mod-mem=<num> Maximum amount of memory (in MiB) for each Darshan module],
+ if test x$withval = xyes; then
+ AC_MSG_ERROR(--with-mod-mem must be given a number)
+ else
+ AC_DEFINE_UNQUOTED(__DARSHAN_MOD_MEM_MAX, ${withval}, Maximum memory (in MiB) for each Darshan module)
+ fi
+)
+
+if test x$enable_bgq_mod != xno; then
+ AC_MSG_CHECKING(for BG/Q environment)
+ AC_TRY_COMPILE([
+ #ifndef __bgq__
+ #error __bgq__ not set
+ #endif
+ ], [],
+ AC_MSG_RESULT(yes)
+ DARSHAN_USE_BGQ=1,
+ AC_MSG_RESULT(no)
+ AS_IF([test "x$enable_bgq_mod" = xyes],
+ AC_MSG_ERROR(BG/Q module enabled in non-BG/Q environment),
+ []))
+fi
+
if test x$GOT_ALIGNMENT != x1; then
AC_MSG_ERROR(must provide --with-mem-align=<num> argument to configure.)
fi
@@ -278,39 +320,7 @@ AS_IF([$CC -show foo.c -o foo |grep lmpich >& /dev/null],
AC_MSG_RESULT(no)
MPICH_LIB_OLD=0)
-# check to see whether the bgq instrumentation module should be built
-AC_ARG_ENABLE(
- [bgq_mod],
- [ --disable-bgq-mod Disables compilation and use of BG/Q module (for BG/Q systems)],
- [],
- [enable_bgq_mod=check]
-)
-if test x$enable_bgq_mod != xno; then
- AC_MSG_CHECKING(for BG/Q environment)
- AC_TRY_COMPILE([
- #ifndef __bgq__
- #error __bgq__ not set
- #endif
- ], [],
- AC_MSG_RESULT(yes)
- DARSHAN_USE_BGQ=1,
- AC_MSG_RESULT(no)
- AS_IF([test "x$enable_bgq_mod" = xyes],
- AC_MSG_ERROR(BG/Q module enabled in non-BG/Q environment),
- []))
-fi
-
-AC_ARG_WITH(mod-mem,
-[ --with-mod-mem=<num> Maximum amount of memory (in MiB) for each Darshan module],
- if test x$withval = xyes; then
- AC_MSG_ERROR(--with-mod-mem must be given a number)
- else
- AC_DEFINE_UNQUOTED(__DARSHAN_MOD_MEM_MAX, ${withval}, Maximum memory (in MiB) for each Darshan module)
- fi
-)
-
DARSHAN_VERSION="AC_PACKAGE_VERSION"
-
AC_SUBST(darshan_lib_path)
AC_SUBST(darshan_share_path)
AC_SUBST(LDFLAGS)
@@ -335,4 +345,5 @@ share/mpi-profile/darshan-f.conf
share/mpi-profile/darshan-bg-cc.conf
share/mpi-profile/darshan-bg-cxx.conf
share/mpi-profile/darshan-bg-f.conf
+share/darshan-mmap-epilog.sh
)
=====================================
darshan-runtime/darshan-core.h
=====================================
--- a/darshan-runtime/darshan-core.h
+++ b/darshan-runtime/darshan-core.h
@@ -30,6 +30,9 @@
/* Environment variable to override memory per module */
#define DARSHAN_MOD_MEM_OVERRIDE "DARSHAN_MODMEM"
+/* Environment variable to override default mmap log path */
+#define DARSHAN_MMAP_LOG_PATH_OVERRIDE "DARSHAN_MMAP_LOGPATH"
+
/* Maximum amount of memory per instrumentation module in MiB */
#ifdef __DARSHAN_MOD_MEM_MAX
#define DARSHAN_MOD_MEM_MAX (__DARSHAN_MOD_MEM_MAX * 1024 * 1024)
@@ -37,6 +40,9 @@
#define DARSHAN_MOD_MEM_MAX (2 * 1024 * 1024) /* 2 MiB default */
#endif
+/* default path for storing mmap log files is '/tmp' */
+#define DARSHAN_DEF_MMAP_LOG_PATH "/tmp"
+
#define DARSHAN_RECORD_BUF_SIZE (1024 * 100) /* store 1024 records, each of at most 100 bytes */
/* Default runtime compression buffer size */
=====================================
darshan-runtime/darshan-runtime-config.h.in
=====================================
--- a/darshan-runtime/darshan-runtime-config.h.in
+++ b/darshan-runtime/darshan-runtime-config.h.in
@@ -3,9 +3,6 @@
/* Define if building universal (internal helper macro) */
#undef AC_APPLE_UNIVERSAL_BUILD
-/* Define if cuserid() should be disabled */
-#undef DARSHAN_DISABLE_CUSERID
-
/* Define if struct aiocb64 type is defined */
#undef HAVE_AIOCB64
@@ -87,6 +84,12 @@
# endif
#endif
+/* Define if cuserid() should be disabled */
+#undef __DARSHAN_DISABLE_CUSERID
+
+/* Define if Darshan should mmap data structures to log file */
+#undef __DARSHAN_ENABLE_MMAP_LOGS
+
/* Define if Darshan should set log files to be group readable */
#undef __DARSHAN_GROUP_READABLE_LOGS
=====================================
darshan-runtime/lib/darshan-core.c
=====================================
--- a/darshan-runtime/lib/darshan-core.c
+++ b/darshan-runtime/lib/darshan-core.c
@@ -98,6 +98,8 @@ static void darshan_add_record_hashref(
darshan_record_id id, struct darshan_core_record_ref **ref);
static void darshan_block_size_from_path(
const char *path, int *block_size);
+static void darshan_get_user_name(
+ char *user);
static void darshan_get_logfile_name(
char* logfile_name, int jobid, struct tm* start_tm);
static void darshan_get_shared_records(
@@ -127,6 +129,7 @@ void darshan_core_initialize(int argc, char **argv)
int mmap_fd;
int mmap_size;
int sys_page_size;
+ char cuser[L_cuserid] = {0};
char *envstr;
char *jobid_str;
int jobid;
@@ -197,6 +200,7 @@ void darshan_core_initialize(int argc, char **argv)
memset(init_core, 0, sizeof(*init_core));
init_core->wtime_offset = DARSHAN_MPI_CALL(PMPI_Wtime)();
+#ifdef __DARSHAN_ENABLE_MMAP_LOGS
sys_page_size = sysconf(_SC_PAGESIZE);
assert(sys_page_size > 0);
@@ -205,11 +209,21 @@ void darshan_core_initialize(int argc, char **argv)
if(mmap_size % sys_page_size)
mmap_size = ((mmap_size / sys_page_size) + 1) * sys_page_size;
+ char *mmap_log_dir;
+ envstr = getenv(DARSHAN_MMAP_LOG_PATH_OVERRIDE);
+ if(envstr)
+ mmap_log_dir = envstr;
+ else
+ mmap_log_dir = DARSHAN_DEF_MMAP_LOG_PATH;
+
+ darshan_get_user_name(cuser);
+
/* construct a unique temporary log file name for this process
* to write mmap log data to
*/
- snprintf(init_core->mmap_log_name, PATH_MAX, "/tmp/darshan_job%d.%d",
- jobid, my_rank);
+ snprintf(init_core->mmap_log_name, PATH_MAX,
+ "/%s/%s_%s_id%d_mmap-log-%d.darshan",
+ mmap_log_dir, cuser, __progname, jobid, my_rank);
/* create the temporary mmapped darshan log */
mmap_fd = open(init_core->mmap_log_name, O_CREAT|O_RDWR|O_EXCL , 0644);
@@ -282,6 +296,9 @@ void darshan_core_initialize(int argc, char **argv)
/* collect information about command line and mounted file systems */
darshan_get_exe_and_mounts(init_core, argc, argv);
+#else
+ #error "Error: non-mmap code path not ready."
+#endif
/* bootstrap any modules with static initialization routines */
i = 0;
@@ -295,6 +312,10 @@ void darshan_core_initialize(int argc, char **argv)
}
}
+ /* reduce so all ranks agree on the job start time */
+ DARSHAN_MPI_CALL(PMPI_Allreduce)(MPI_IN_PLACE, &init_core->log_job_p->start_time,
+ 1, MPI_INT64_T, MPI_MIN, MPI_COMM_WORLD);
+
if(internal_timing_flag)
{
init_time = DARSHAN_MPI_CALL(PMPI_Wtime)() - init_start;
@@ -318,7 +339,6 @@ void darshan_core_shutdown()
int internal_timing_flag = 0;
struct tm *start_tm;
time_t start_time_tmp;
- int64_t first_start_time;
int64_t last_end_time;
double start_log_time;
double tm_end;
@@ -360,8 +380,9 @@ void darshan_core_shutdown()
/* XXX just copy mmap files somewhere else to avoid corruption */
DARSHAN_MPI_CALL(PMPI_Barrier)(MPI_COMM_WORLD);
- if(my_rank == 0)
- system("cp /tmp/darshan* ~/Desktop");
+ char cp_cmd[500] = {0};
+ sprintf(cp_cmd, "cp %s ~/Desktop", final_core->mmap_log_name);
+ system(cp_cmd);
DARSHAN_MPI_CALL(PMPI_Barrier)(MPI_COMM_WORLD);
memcpy(&out_job, final_core->log_job_p, sizeof(struct darshan_job));
@@ -427,16 +448,11 @@ void darshan_core_shutdown()
final_core->log_job_p->end_time = time(NULL);
- /* reduce to report first start time and last end time across all ranks
- * at rank 0
- */
- DARSHAN_MPI_CALL(PMPI_Reduce)(&final_core->log_job_p->start_time, &first_start_time,
- 1, MPI_LONG_LONG, MPI_MIN, 0, MPI_COMM_WORLD);
+ /* reduce to report last end time across all ranks at rank 0 */
DARSHAN_MPI_CALL(PMPI_Reduce)(&final_core->log_job_p->end_time, &last_end_time,
- 1, MPI_LONG_LONG, MPI_MAX, 0, MPI_COMM_WORLD);
+ 1, MPI_INT64_T, MPI_MAX, 0, MPI_COMM_WORLD);
if(my_rank == 0)
{
- out_job.start_time = first_start_time;
out_job.end_time = last_end_time;
}
@@ -1003,66 +1019,59 @@ static void darshan_get_exe_and_mounts(struct darshan_core_runtime *core,
return;
}
-static void darshan_add_record_hashref(struct darshan_core_runtime *core,
- char *name, darshan_record_id id, struct darshan_core_record_ref **ref)
+static void darshan_block_size_from_path(const char *path, int *block_size)
{
- int record_size = sizeof(darshan_record_id) + strlen(name) + 1;
-
- if((record_size + core->log_hdr_p->rec_map.len) > DARSHAN_RECORD_BUF_SIZE)
- return;
+ int i;
+ *block_size = -1;
- *ref = malloc(sizeof(**ref));
- if(*ref)
+ for(i=0; i<mnt_data_count; i++)
{
- memset(*ref, 0, sizeof(**ref));
-
-#if 0
- if(!mmap)
- {
- ref->rec.name = malloc(strlen(name) + 1);
- }
- else
-#endif
+ if(!(strncmp(mnt_data_array[i].path, path, strlen(mnt_data_array[i].path))))
{
- /* store the rec id and full file path in record hash buffer */
- void *tmp_p = (char *)core->log_rec_p + core->log_hdr_p->rec_map.len;
- *(darshan_record_id *)tmp_p = id;
-
- /* set the name pointer for this record to point to the
- * appropriate location in the record hash buffer
- */
- tmp_p = (char *)tmp_p + sizeof(darshan_record_id);
- (*ref)->name = (char *)tmp_p;
+ *block_size = mnt_data_array[i].block_size;
+ return;
}
-
- /* set record ref fields */
- (*ref)->id = id;
- if((*ref)->name)
- strcpy((*ref)->name, name);
-
- /* TODO: look at HASH_ADD_KEYPTR, use same strategy (big contig pool) for non-mmap darshan */
- HASH_ADD(hlink, core->rec_hash, id, sizeof(darshan_record_id), (*ref));
- core->rec_hash_cnt++;
- core->log_hdr_p->rec_map.len += record_size;
}
return;
}
-static void darshan_block_size_from_path(const char *path, int *block_size)
+static void darshan_get_user_name(char *cuser)
{
- int i;
- *block_size = -1;
+ char* logname_string;
- for(i=0; i<mnt_data_count; i++)
+ /* get the username for this job. In order we will try each of the
+ * following until one of them succeeds:
+ *
+ * - cuserid()
+ * - getenv("LOGNAME")
+ * - snprintf(..., geteuid());
+ *
+ * Note that we do not use getpwuid() because it generally will not
+ * work in statically compiled binaries.
+ */
+
+#ifndef __DARSHAN_DISABLE_CUSERID
+ cuserid(cuser);
+#endif
+
+ /* if cuserid() didn't work, then check the environment */
+ if(strcmp(cuser, "") == 0)
{
- if(!(strncmp(mnt_data_array[i].path, path, strlen(mnt_data_array[i].path))))
+ logname_string = getenv("LOGNAME");
+ if(logname_string)
{
- *block_size = mnt_data_array[i].block_size;
- return;
+ strncpy(cuser, logname_string, (L_cuserid-1));
}
}
+ /* if cuserid() and environment both fail, then fall back to uid */
+ if(strcmp(cuser, "") == 0)
+ {
+ uid_t uid = geteuid();
+ snprintf(cuser, L_cuserid, "%u", uid);
+ }
+
return;
}
@@ -1071,7 +1080,6 @@ static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* s
{
char* user_logfile_name;
char* logpath;
- char* logname_string;
char* logpath_override = NULL;
#ifdef __DARSHAN_LOG_ENV
char env_check[256];
@@ -1110,37 +1118,7 @@ static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* s
#endif
}
- /* get the username for this job. In order we will try each of the
- * following until one of them succeeds:
- *
- * - cuserid()
- * - getenv("LOGNAME")
- * - snprintf(..., geteuid());
- *
- * Note that we do not use getpwuid() because it generally will not
- * work in statically compiled binaries.
- */
-
-#ifndef DARSHAN_DISABLE_CUSERID
- cuserid(cuser);
-#endif
-
- /* if cuserid() didn't work, then check the environment */
- if(strcmp(cuser, "") == 0)
- {
- logname_string = getenv("LOGNAME");
- if(logname_string)
- {
- strncpy(cuser, logname_string, (L_cuserid-1));
- }
- }
-
- /* if cuserid() and environment both fail, then fall back to uid */
- if(strcmp(cuser, "") == 0)
- {
- uid_t uid = geteuid();
- snprintf(cuser, sizeof(cuser), "%u", uid);
- }
+ darshan_get_user_name(cuser);
/* generate a random number to help differentiate the log */
hlevel=DARSHAN_MPI_CALL(PMPI_Wtime)() * 1000000;
@@ -1221,6 +1199,52 @@ static void darshan_get_logfile_name(char* logfile_name, int jobid, struct tm* s
return;
}
+static void darshan_add_record_hashref(struct darshan_core_runtime *core,
+ char *name, darshan_record_id id, struct darshan_core_record_ref **ref)
+{
+ int record_size = sizeof(darshan_record_id) + strlen(name) + 1;
+
+ if((record_size + core->log_hdr_p->rec_map.len) > DARSHAN_RECORD_BUF_SIZE)
+ return;
+
+ *ref = malloc(sizeof(**ref));
+ if(*ref)
+ {
+ memset(*ref, 0, sizeof(**ref));
+
+#if 0
+ if(!mmap)
+ {
+ ref->rec.name = malloc(strlen(name) + 1);
+ }
+ else
+#endif
+ {
+ /* store the rec id and full file path in record hash buffer */
+ void *tmp_p = (char *)core->log_rec_p + core->log_hdr_p->rec_map.len;
+ *(darshan_record_id *)tmp_p = id;
+
+ /* set the name pointer for this record to point to the
+ * appropriate location in the record hash buffer
+ */
+ tmp_p = (char *)tmp_p + sizeof(darshan_record_id);
+ (*ref)->name = (char *)tmp_p;
+ }
+
+ /* set record ref fields */
+ (*ref)->id = id;
+ if((*ref)->name)
+ strcpy((*ref)->name, name);
+
+ /* TODO: look at HASH_ADD_KEYPTR, use same strategy (big contig pool) for non-mmap darshan */
+ HASH_ADD(hlink, core->rec_hash, id, sizeof(darshan_record_id), (*ref));
+ core->rec_hash_cnt++;
+ core->log_hdr_p->rec_map.len += record_size;
+ }
+
+ return;
+}
+
static void darshan_get_shared_records(struct darshan_core_runtime *core,
darshan_record_id **shared_recs, int *shared_rec_cnt)
{
=====================================
darshan-runtime/lib/darshan-posix.c
=====================================
--- a/darshan-runtime/lib/darshan-posix.c
+++ b/darshan-runtime/lib/darshan-posix.c
@@ -340,15 +340,6 @@ static void posix_shutdown(void);
(__file)->file_record->counters[POSIX_STATS] += 1; \
} while(0)
-static double psx_wrap_time = 0.0;
-
-#define PSX_PRE_WRAP() \
- double pre_wrap = darshan_core_wtime();
-
-#define PSX_POST_WRAP() \
- double post_wrap = darshan_core_wtime(); \
- psx_wrap_time += (post_wrap-pre_wrap);
-
/**********************************************************
* Wrappers for POSIX I/O functions of interest *
@@ -360,8 +351,6 @@ int DARSHAN_DECL(open)(const char *path, int flags, ...)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(open);
if(flags & O_CREAT)
@@ -387,8 +376,6 @@ int DARSHAN_DECL(open)(const char *path, int flags, ...)
POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -398,8 +385,6 @@ int DARSHAN_DECL(open64)(const char *path, int flags, ...)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(open64);
if(flags & O_CREAT)
@@ -425,8 +410,6 @@ int DARSHAN_DECL(open64)(const char *path, int flags, ...)
POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -435,8 +418,6 @@ int DARSHAN_DECL(creat)(const char* path, mode_t mode)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(creat);
tm1 = darshan_core_wtime();
@@ -448,8 +429,6 @@ int DARSHAN_DECL(creat)(const char* path, mode_t mode)
POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -458,8 +437,6 @@ int DARSHAN_DECL(creat64)(const char* path, mode_t mode)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(creat64);
tm1 = darshan_core_wtime();
@@ -471,8 +448,6 @@ int DARSHAN_DECL(creat64)(const char* path, mode_t mode)
POSIX_RECORD_OPEN(ret, path, mode, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -482,8 +457,6 @@ FILE* DARSHAN_DECL(fopen)(const char *path, const char *mode)
int fd;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(fopen);
tm1 = darshan_core_wtime();
@@ -500,8 +473,6 @@ FILE* DARSHAN_DECL(fopen)(const char *path, const char *mode)
POSIX_RECORD_OPEN(fd, path, 0, 1, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -511,8 +482,6 @@ FILE* DARSHAN_DECL(fopen64)(const char *path, const char *mode)
int fd;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(fopen64);
tm1 = darshan_core_wtime();
@@ -529,8 +498,6 @@ FILE* DARSHAN_DECL(fopen64)(const char *path, const char *mode)
POSIX_RECORD_OPEN(fd, path, 0, 1, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -539,8 +506,6 @@ int DARSHAN_DECL(mkstemp)(char* template)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(mkstemp);
tm1 = darshan_core_wtime();
@@ -552,8 +517,6 @@ int DARSHAN_DECL(mkstemp)(char* template)
POSIX_RECORD_OPEN(ret, template, 0, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -562,8 +525,6 @@ int DARSHAN_DECL(mkostemp)(char* template, int flags)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(mkostemp);
tm1 = darshan_core_wtime();
@@ -575,8 +536,6 @@ int DARSHAN_DECL(mkostemp)(char* template, int flags)
POSIX_RECORD_OPEN(ret, template, 0, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -585,8 +544,6 @@ int DARSHAN_DECL(mkstemps)(char* template, int suffixlen)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(mkstemps);
tm1 = darshan_core_wtime();
@@ -598,8 +555,6 @@ int DARSHAN_DECL(mkstemps)(char* template, int suffixlen)
POSIX_RECORD_OPEN(ret, template, 0, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -608,8 +563,6 @@ int DARSHAN_DECL(mkostemps)(char* template, int suffixlen, int flags)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(mkostemps);
tm1 = darshan_core_wtime();
@@ -621,8 +574,6 @@ int DARSHAN_DECL(mkostemps)(char* template, int suffixlen, int flags)
POSIX_RECORD_OPEN(ret, template, 0, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -632,8 +583,6 @@ ssize_t DARSHAN_DECL(read)(int fd, void *buf, size_t count)
int aligned_flag = 0;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(read);
if((unsigned long)buf % darshan_mem_alignment == 0) aligned_flag = 1;
@@ -647,8 +596,6 @@ ssize_t DARSHAN_DECL(read)(int fd, void *buf, size_t count)
POSIX_RECORD_READ(ret, fd, 0, 0, aligned_flag, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -658,8 +605,6 @@ ssize_t DARSHAN_DECL(write)(int fd, const void *buf, size_t count)
int aligned_flag = 0;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(write);
if((unsigned long)buf % darshan_mem_alignment == 0) aligned_flag = 1;
@@ -673,8 +618,6 @@ ssize_t DARSHAN_DECL(write)(int fd, const void *buf, size_t count)
POSIX_RECORD_WRITE(ret, fd, 0, 0, aligned_flag, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -684,8 +627,6 @@ ssize_t DARSHAN_DECL(pread)(int fd, void *buf, size_t count, off_t offset)
int aligned_flag = 0;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(pread);
if((unsigned long)buf % darshan_mem_alignment == 0) aligned_flag = 1;
@@ -699,8 +640,6 @@ ssize_t DARSHAN_DECL(pread)(int fd, void *buf, size_t count, off_t offset)
POSIX_RECORD_READ(ret, fd, 1, offset, aligned_flag, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -710,8 +649,6 @@ ssize_t DARSHAN_DECL(pwrite)(int fd, const void *buf, size_t count, off_t offset
int aligned_flag = 0;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(pwrite);
if((unsigned long)buf % darshan_mem_alignment == 0) aligned_flag = 1;
@@ -725,8 +662,6 @@ ssize_t DARSHAN_DECL(pwrite)(int fd, const void *buf, size_t count, off_t offset
POSIX_RECORD_WRITE(ret, fd, 1, offset, aligned_flag, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -736,8 +671,6 @@ ssize_t DARSHAN_DECL(pread64)(int fd, void *buf, size_t count, off64_t offset)
int aligned_flag = 0;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(pread64);
if((unsigned long)buf % darshan_mem_alignment == 0) aligned_flag = 1;
@@ -751,8 +684,6 @@ ssize_t DARSHAN_DECL(pread64)(int fd, void *buf, size_t count, off64_t offset)
POSIX_RECORD_READ(ret, fd, 1, offset, aligned_flag, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -762,8 +693,6 @@ ssize_t DARSHAN_DECL(pwrite64)(int fd, const void *buf, size_t count, off64_t of
int aligned_flag = 0;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(pwrite64);
if((unsigned long)buf % darshan_mem_alignment == 0) aligned_flag = 1;
@@ -777,8 +706,6 @@ ssize_t DARSHAN_DECL(pwrite64)(int fd, const void *buf, size_t count, off64_t of
POSIX_RECORD_WRITE(ret, fd, 1, offset, aligned_flag, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -789,8 +716,6 @@ ssize_t DARSHAN_DECL(readv)(int fd, const struct iovec *iov, int iovcnt)
int i;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(readv);
for(i=0; i<iovcnt; i++)
@@ -808,8 +733,6 @@ ssize_t DARSHAN_DECL(readv)(int fd, const struct iovec *iov, int iovcnt)
POSIX_RECORD_READ(ret, fd, 0, 0, aligned_flag, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -820,8 +743,6 @@ ssize_t DARSHAN_DECL(writev)(int fd, const struct iovec *iov, int iovcnt)
int i;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(writev);
for(i=0; i<iovcnt; i++)
@@ -839,8 +760,6 @@ ssize_t DARSHAN_DECL(writev)(int fd, const struct iovec *iov, int iovcnt)
POSIX_RECORD_WRITE(ret, fd, 0, 0, aligned_flag, 0, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -850,8 +769,6 @@ size_t DARSHAN_DECL(fread)(void *ptr, size_t size, size_t nmemb, FILE *stream)
int aligned_flag = 0;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(fread);
if((unsigned long)ptr % darshan_mem_alignment == 0) aligned_flag = 1;
@@ -874,8 +791,6 @@ size_t DARSHAN_DECL(fread)(void *ptr, size_t size, size_t nmemb, FILE *stream)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -885,8 +800,6 @@ size_t DARSHAN_DECL(fwrite)(const void *ptr, size_t size, size_t nmemb, FILE *st
int aligned_flag = 0;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(fwrite);
if((unsigned long)ptr % darshan_mem_alignment == 0) aligned_flag = 1;
@@ -909,8 +822,6 @@ size_t DARSHAN_DECL(fwrite)(const void *ptr, size_t size, size_t nmemb, FILE *st
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -920,8 +831,6 @@ off_t DARSHAN_DECL(lseek)(int fd, off_t offset, int whence)
struct posix_file_runtime* file;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(lseek);
tm1 = darshan_core_wtime();
@@ -944,8 +853,6 @@ off_t DARSHAN_DECL(lseek)(int fd, off_t offset, int whence)
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -955,8 +862,6 @@ off_t DARSHAN_DECL(lseek64)(int fd, off_t offset, int whence)
struct posix_file_runtime* file;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(lseek64);
tm1 = darshan_core_wtime();
@@ -979,8 +884,6 @@ off_t DARSHAN_DECL(lseek64)(int fd, off_t offset, int whence)
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -990,8 +893,6 @@ int DARSHAN_DECL(fseek)(FILE *stream, long offset, int whence)
struct posix_file_runtime* file;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(fseek);
tm1 = darshan_core_wtime();
@@ -1014,8 +915,6 @@ int DARSHAN_DECL(fseek)(FILE *stream, long offset, int whence)
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1024,8 +923,6 @@ int DARSHAN_DECL(__xstat)(int vers, const char *path, struct stat *buf)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(__xstat);
tm1 = darshan_core_wtime();
@@ -1040,8 +937,6 @@ int DARSHAN_DECL(__xstat)(int vers, const char *path, struct stat *buf)
POSIX_LOOKUP_RECORD_STAT(path, buf, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1050,8 +945,6 @@ int DARSHAN_DECL(__xstat64)(int vers, const char *path, struct stat64 *buf)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(__xstat64);
tm1 = darshan_core_wtime();
@@ -1066,8 +959,6 @@ int DARSHAN_DECL(__xstat64)(int vers, const char *path, struct stat64 *buf)
POSIX_LOOKUP_RECORD_STAT(path, buf, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1076,8 +967,6 @@ int DARSHAN_DECL(__lxstat)(int vers, const char *path, struct stat *buf)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(__lxstat);
tm1 = darshan_core_wtime();
@@ -1092,8 +981,6 @@ int DARSHAN_DECL(__lxstat)(int vers, const char *path, struct stat *buf)
POSIX_LOOKUP_RECORD_STAT(path, buf, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1102,8 +989,6 @@ int DARSHAN_DECL(__lxstat64)(int vers, const char *path, struct stat64 *buf)
int ret;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(__lxstat64);
tm1 = darshan_core_wtime();
@@ -1118,8 +1003,6 @@ int DARSHAN_DECL(__lxstat64)(int vers, const char *path, struct stat64 *buf)
POSIX_LOOKUP_RECORD_STAT(path, buf, tm1, tm2);
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1129,8 +1012,6 @@ int DARSHAN_DECL(__fxstat)(int vers, int fd, struct stat *buf)
struct posix_file_runtime* file;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(__fxstat);
tm1 = darshan_core_wtime();
@@ -1149,8 +1030,6 @@ int DARSHAN_DECL(__fxstat)(int vers, int fd, struct stat *buf)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1160,8 +1039,6 @@ int DARSHAN_DECL(__fxstat64)(int vers, int fd, struct stat64 *buf)
struct posix_file_runtime* file;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(__fxstat64);
tm1 = darshan_core_wtime();
@@ -1180,8 +1057,6 @@ int DARSHAN_DECL(__fxstat64)(int vers, int fd, struct stat64 *buf)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1191,8 +1066,6 @@ void* DARSHAN_DECL(mmap)(void *addr, size_t length, int prot, int flags,
void* ret;
struct posix_file_runtime* file;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(mmap);
ret = __real_mmap(addr, length, prot, flags, fd, offset);
@@ -1208,8 +1081,6 @@ void* DARSHAN_DECL(mmap)(void *addr, size_t length, int prot, int flags,
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1219,8 +1090,6 @@ void* DARSHAN_DECL(mmap64)(void *addr, size_t length, int prot, int flags,
void* ret;
struct posix_file_runtime* file;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(mmap64);
ret = __real_mmap64(addr, length, prot, flags, fd, offset);
@@ -1236,8 +1105,6 @@ void* DARSHAN_DECL(mmap64)(void *addr, size_t length, int prot, int flags,
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1247,8 +1114,6 @@ int DARSHAN_DECL(fsync)(int fd)
struct posix_file_runtime* file;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(fsync);
tm1 = darshan_core_wtime();
@@ -1270,8 +1135,6 @@ int DARSHAN_DECL(fsync)(int fd)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1281,8 +1144,6 @@ int DARSHAN_DECL(fdatasync)(int fd)
struct posix_file_runtime* file;
double tm1, tm2;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(fdatasync);
tm1 = darshan_core_wtime();
@@ -1304,8 +1165,6 @@ int DARSHAN_DECL(fdatasync)(int fd)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1315,8 +1174,6 @@ int DARSHAN_DECL(close)(int fd)
double tm1, tm2;
int ret;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(close);
tm1 = darshan_core_wtime();
@@ -1339,8 +1196,6 @@ int DARSHAN_DECL(close)(int fd)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1351,8 +1206,6 @@ int DARSHAN_DECL(fclose)(FILE *fp)
double tm1, tm2;
int ret;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(fclose);
tm1 = darshan_core_wtime();
@@ -1375,8 +1228,6 @@ int DARSHAN_DECL(fclose)(FILE *fp)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1384,8 +1235,6 @@ int DARSHAN_DECL(aio_read)(struct aiocb *aiocbp)
{
int ret;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(aio_read);
ret = __real_aio_read(aiocbp);
@@ -1397,8 +1246,6 @@ int DARSHAN_DECL(aio_read)(struct aiocb *aiocbp)
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1406,8 +1253,6 @@ int DARSHAN_DECL(aio_write)(struct aiocb *aiocbp)
{
int ret;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(aio_write);
ret = __real_aio_write(aiocbp);
@@ -1419,8 +1264,6 @@ int DARSHAN_DECL(aio_write)(struct aiocb *aiocbp)
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1428,8 +1271,6 @@ int DARSHAN_DECL(aio_read64)(struct aiocb64 *aiocbp)
{
int ret;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(aio_read64);
ret = __real_aio_read64(aiocbp);
@@ -1441,8 +1282,6 @@ int DARSHAN_DECL(aio_read64)(struct aiocb64 *aiocbp)
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1450,8 +1289,6 @@ int DARSHAN_DECL(aio_write64)(struct aiocb64 *aiocbp)
{
int ret;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(aio_write64);
ret = __real_aio_write64(aiocbp);
@@ -1463,8 +1300,6 @@ int DARSHAN_DECL(aio_write64)(struct aiocb64 *aiocbp)
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1475,8 +1310,6 @@ ssize_t DARSHAN_DECL(aio_return)(struct aiocb *aiocbp)
struct posix_aio_tracker *tmp;
int aligned_flag = 0;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(aio_return);
ret = __real_aio_return(aiocbp);
@@ -1505,8 +1338,6 @@ ssize_t DARSHAN_DECL(aio_return)(struct aiocb *aiocbp)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1517,8 +1348,6 @@ ssize_t DARSHAN_DECL(aio_return64)(struct aiocb64 *aiocbp)
struct posix_aio_tracker *tmp;
int aligned_flag = 0;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(aio_return64);
ret = __real_aio_return64(aiocbp);
@@ -1547,8 +1376,6 @@ ssize_t DARSHAN_DECL(aio_return64)(struct aiocb64 *aiocbp)
}
POSIX_UNLOCK();
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1558,8 +1385,6 @@ int DARSHAN_DECL(lio_listio)(int mode, struct aiocb *const aiocb_list[],
int ret;
int i;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(lio_listio);
ret = __real_lio_listio(mode, aiocb_list, nitems, sevp);
@@ -1574,8 +1399,6 @@ int DARSHAN_DECL(lio_listio)(int mode, struct aiocb *const aiocb_list[],
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -1585,8 +1408,6 @@ int DARSHAN_DECL(lio_listio64)(int mode, struct aiocb64 *const aiocb_list[],
int ret;
int i;
- PSX_PRE_WRAP();
-
MAP_OR_FAIL(lio_listio64);
ret = __real_lio_listio64(mode, aiocb_list, nitems, sevp);
@@ -1601,8 +1422,6 @@ int DARSHAN_DECL(lio_listio64)(int mode, struct aiocb64 *const aiocb_list[],
POSIX_UNLOCK();
}
- PSX_POST_WRAP();
-
return(ret);
}
@@ -2199,8 +2018,6 @@ static void posix_begin_shutdown()
instrumentation_disabled = 1;
POSIX_UNLOCK();
- fprintf(stderr, "POSIX:wrappers\t%f\n", psx_wrap_time);
-
return;
}
=====================================
darshan-runtime/share/darshan-mmap-epilog.sh.in
=====================================
--- /dev/null
+++ b/darshan-runtime/share/darshan-mmap-epilog.sh.in
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+#
+# Copyright (C) 2015 University of Chicago.
+# See COPYRIGHT notice in top-level directory.
+#
+
+# grab the prefix of the darshan install directory
+DARSHAN_INSTALL_DIR=@prefix@
+
+# use the log dir specified at configure time
+DARSHAN_LOG_DIR=@__DARSHAN_LOG_PATH@
+
+NODE_NAME=$(uname -n)
+
+# use the default mmap log directory (/tmp), unless the
+# env variable is set to something
+if [ -z "$DARSHAN_MMAP_LOGPATH" ]; then
+ DARSHAN_MMAP_LOG_DIR=/tmp
+else
+ DARSHAN_MMAP_LOG_DIR=$DARSHAN_MMAP_LOGPATH
+fi
+
+DARSHAN_MMAP_LOG_GLOB=${DARSHAN_MMAP_LOG_DIR}/*id${SLURM_JOB_ID}*.darshan
+
+# if no mmap logs found for this job, we have nothing to do
+DARSHAN_TEST_LOG=$(ls $DARSHAN_MMAP_LOG_GLOB 2>/dev/null | head -n 1)
+if [ -z $DARSHAN_TEST_LOG ]; then
+ exit 0
+fi
+
+# get the job start time from the first log file
+JOB_START_DATE=$(${DARSHAN_INSTALL_DIR}/bin/darshan-parser $DARSHAN_TEST_LOG |
+ grep "# start_time_asci" | cut -d':' -f 2- | cut -d' ' -f 2-)
+OUTLOG_YEAR=$(date --date="$(printf "$JOB_START_DATE")" +"%Y")
+OUTLOG_MON=$(date --date="$(printf "$JOB_START_DATE")" +"%-m")
+OUTLOG_DAY=$(date --date="$(printf "$JOB_START_DATE")" +"%-d")
+OUTLOG_SECS=$((
+ ($(date --date="$(printf "$JOB_START_DATE")" +"%-H") * 60 * 60) +
+ ($(date --date="$(printf "$JOB_START_DATE")" +"%-M") * 60) +
+ ($(date --date="$(printf "$JOB_START_DATE")" +"%-S"))
+))
+
+MMAP_LOG_PRE=$(basename $DARSHAN_TEST_LOG | cut -d'_' -f 1-3)
+
+# construct full output log directory name
+OUTLOG_DIR=${DARSHAN_LOG_DIR}/${OUTLOG_YEAR}/${OUTLOG_MON}/${OUTLOG_DAY}/
+OUTLOG_DIR=$OUTLOG_DIR/${MMAP_LOG_PRE}_${OUTLOG_MON}-${OUTLOG_DAY}-${OUTLOG_SECS}
+
+# create the output directory for this job
+mkdir -p $OUTLOG_DIR
+
+# construct the per-node log files and store in the output directory
+$DARSHAN_INSTALL_DIR/bin/darshan-merge --output ${OUTLOG_DIR}/${MMAP_LOG_PRE}_${NODE_NAME}.darshan \
+ $DARSHAN_MMAP_LOG_GLOB
+
+exit 0
=====================================
darshan-util/Makefile.in
=====================================
--- a/darshan-util/Makefile.in
+++ b/darshan-util/Makefile.in
@@ -1,4 +1,4 @@
-all: libdarshan-util.a darshan-null-logutils.o darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-stitch-logs jenkins-hash-gen
+all: libdarshan-util.a darshan-null-logutils.o darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
DESTDIR =
srcdir = @srcdir@
@@ -107,7 +107,7 @@ darshan-diff: darshan-diff.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_
darshan-parser: darshan-parser.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_MOD_LOGUTIL_HEADERS) $(DARSHAN_MOD_LOG_FORMATS) libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $< libdarshan-util.a -o $@ $(LIBS)
-darshan-stitch-logs: darshan-stitch-logs.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_MOD_LOGUTIL_HEADERS) $(DARSHAN_MOD_LOG_FORMATS) libdarshan-util.a | uthash-1.9.2
+darshan-merge: darshan-merge.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_MOD_LOGUTIL_HEADERS) $(DARSHAN_MOD_LOG_FORMATS) libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $< libdarshan-util.a -o $@ $(LIBS)
#test/gztest: test/gztest.c mktestdir
@@ -125,7 +125,7 @@ install:: all
install -m 755 darshan-convert $(bindir)
install -m 755 darshan-diff $(bindir)
install -m 755 darshan-parser $(bindir)
- install -m 755 darshan-stitch-logs $(bindir)
+ install -m 755 darshan-merge $(bindir)
install -m 755 $(srcdir)/darshan-summary-per-file.sh $(bindir)
install -m 755 libdarshan-util.a $(libdir)
ifeq ($(DARSHAN_ENABLE_SHARED),1)
@@ -159,7 +159,7 @@ endif
clean::
- rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-parser darshan-stitch-logs jenkins-hash-gen
+ rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-parser darshan-merge jenkins-hash-gen
distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
=====================================
darshan-util/darshan-logutils.c
=====================================
--- a/darshan-util/darshan-logutils.c
+++ b/darshan-util/darshan-logutils.c
@@ -967,10 +967,9 @@ static int darshan_log_putheader(darshan_fd fd)
header.magic_nr = DARSHAN_MAGIC_NR;
header.comp_type = fd->comp_type;
header.partial_flag = fd->partial_flag;
-
- /* copy the mapping information to the header */
memcpy(&header.rec_map, &fd->rec_map, sizeof(struct darshan_log_map));
- memcpy(&header.mod_map, &fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
+ memcpy(header.mod_map, fd->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
+ memcpy(header.mod_ver, fd->mod_ver, DARSHAN_MAX_MODS * sizeof(uint32_t));
/* write header to file */
ret = darshan_log_write(fd, &header, sizeof(header));
=====================================
darshan-util/darshan-stitch-logs.c → darshan-util/darshan-merge.c
=====================================
--- a/darshan-util/darshan-stitch-logs.c
+++ b/darshan-util/darshan-merge.c
@@ -11,11 +11,6 @@
#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
-/* TODO: are there any checks we should do to ensure tmp logs belong to the same job */
-/* we can't specifically check the job id, since the pid is used if no job scheduler */
-
-/* TODO: how do we set the output logfile name to be unique, and have necessary semantic info contained */
-
/* TODO: set job end timestamp? */
struct darshan_shared_record_ref
@@ -28,24 +23,28 @@ struct darshan_shared_record_ref
void usage(char *exename)
{
- fprintf(stderr, "Usage: %s [options] <tmp_dir> <job_id>\n", exename);
- fprintf(stderr, " TODO: description.\n");
- fprintf(stderr, " --shared-redux Reduce globally shared records into a single record.\n");
+ fprintf(stderr, "Usage: %s --output <output_path> [options] <input-logs>\n", exename);
+ fprintf(stderr, "This utility merges multiple Darshan log files into a single output log file.\n");
+ fprintf(stderr, "Options:\n");
+ fprintf(stderr, "\t--output\t(REQUIRED) Full path of the output darshan log file.\n");
+ fprintf(stderr, "\t--shared-redux\tReduce globally shared records into a single record.\n");
exit(1);
}
-void parse_args(int argc, char **argv, char **tmplog_dir, int *tmplog_jobid,
- int *shared_redux)
+void parse_args(int argc, char **argv, char ***infile_list, int *n_files,
+ char **outlog_path, int *shared_redux)
{
int index;
static struct option long_opts[] =
{
{"shared-redux", no_argument, NULL, 's'},
+ {"output", required_argument, NULL, 'o'},
{0, 0, 0, 0}
};
*shared_redux = 0;
+ *outlog_path = NULL;
while(1)
{
@@ -58,6 +57,9 @@ void parse_args(int argc, char **argv, char **tmplog_dir, int *tmplog_jobid,
case 's':
*shared_redux = 1;
break;
+ case 'o':
+ *outlog_path = optarg;
+ break;
case '?':
default:
usage(argv[0]);
@@ -65,42 +67,20 @@ void parse_args(int argc, char **argv, char **tmplog_dir, int *tmplog_jobid,
}
}
- if(optind + 2 == argc)
- {
- *tmplog_dir = argv[optind];
- *tmplog_jobid = atoi(argv[optind+1]);
- }
- else
+ if(*outlog_path == NULL)
{
usage(argv[0]);
}
- return;
-}
+ *infile_list = &argv[optind];
+ *n_files = argc - optind;
-int logfile_path_comp(const void *a, const void *b)
-{
- char *pathA = *(char **)a;
- char *pathB = *(char **)b;
- char *pathA_rank_s, *pathB_rank_s;
- int pathA_rank, pathB_rank;
-
- /* extract the process rank number from end of each log file path */
- pathA_rank_s = strrchr(pathA, '.') + 1;
- pathA_rank = atoi(pathA_rank_s);
- pathB_rank_s = strrchr(pathB, '.') + 1;
- pathB_rank = atoi(pathB_rank_s);
-
- if(pathA_rank < pathB_rank)
- return(-1);
- else if(pathA_rank > pathB_rank)
- return(1);
- else
- return(0);
+ return;
}
-int build_mod_shared_rec_hash(glob_t *globbuf, darshan_module_id mod_id,
- int nprocs, char *mod_buf, struct darshan_shared_record_ref **shared_rec_hash)
+int build_mod_shared_rec_hash(char **infile_list, int n_infiles,
+ darshan_module_id mod_id, int nprocs, char *mod_buf,
+ struct darshan_shared_record_ref **shared_rec_hash)
{
darshan_fd in_fd;
struct darshan_base_record *base_rec;
@@ -110,14 +90,14 @@ int build_mod_shared_rec_hash(glob_t *globbuf, darshan_module_id mod_id,
int i;
/* loop over each input log file */
- for(i = 0; i < globbuf->gl_pathc; i++)
+ for(i = 0; i < n_infiles; i++)
{
- in_fd = darshan_log_open(globbuf->gl_pathv[i]);
+ in_fd = darshan_log_open(infile_list[i]);
if(in_fd == NULL)
{
fprintf(stderr,
"Error: unable to open input Darshan log file %s.\n",
- globbuf->gl_pathv[i]);
+ infile_list[i]);
return(-1);
}
@@ -167,7 +147,7 @@ int build_mod_shared_rec_hash(glob_t *globbuf, darshan_module_id mod_id,
{
fprintf(stderr,
"Error: unable to read %s module record from input log file %s.\n",
- darshan_module_names[mod_id], globbuf->gl_pathv[i]);
+ darshan_module_names[mod_id], infile_list[i]);
darshan_log_close(in_fd);
return(-1);
}
@@ -190,20 +170,18 @@ int build_mod_shared_rec_hash(glob_t *globbuf, darshan_module_id mod_id,
int main(int argc, char *argv[])
{
+ char **infile_list;
+ int n_infiles;
int shared_redux;
- char *tmplog_dir;
- int job_id;
- glob_t globbuf;
- char glob_pstr[512];
- char *stitch_logname = "/tmp/test123.darshan"; /* XXX default + configurable? */
- darshan_fd in_fd, stitch_fd;
- struct darshan_job in_job, stitch_job;
- char stitch_exe[DARSHAN_EXE_LEN+1];
- char **stitch_mnt_pts;
- char **stitch_fs_types;
- int stitch_mnt_count = 0;
+ char *outlog_path;
+ darshan_fd in_fd, merge_fd;
+ struct darshan_job in_job, merge_job;
+ char merge_exe[DARSHAN_EXE_LEN+1] = {0};
+ char **merge_mnt_pts;
+ char **merge_fs_types;
+ int merge_mnt_count = 0;
struct darshan_record_ref *in_hash = NULL;
- struct darshan_record_ref *stitch_hash = NULL;
+ struct darshan_record_ref *merge_hash = NULL;
struct darshan_record_ref *ref, *tmp, *found;
struct darshan_shared_record_ref *shared_rec_hash = NULL;
struct darshan_shared_record_ref *sref, *stmp;
@@ -213,42 +191,24 @@ int main(int argc, char *argv[])
int ret;
/* grab command line arguments */
- parse_args(argc, argv, &tmplog_dir, &job_id, &shared_redux);
+ parse_args(argc, argv, &infile_list, &n_infiles, &outlog_path, &shared_redux);
- /* construct the list of input log files to stitch together */
- snprintf(glob_pstr, 512, "%s/darshan_job%d*", tmplog_dir, job_id);
+ memset(&merge_job, 0, sizeof(struct darshan_job));
- ret = glob(glob_pstr, GLOB_NOSORT, NULL, &globbuf);
- if(ret != 0)
- {
- fprintf(stderr,
- "Error: unable to construct list of input Darshan log files.\n");
- return(-1);
- }
-
- /* sort the file list according to the rank id appended to each logfile name */
- /* NOTE: we don't rely on glob's default alphabetic sorting, because it won't
- * sort by ascending ranks if pid's are used for job ids, for instance
- */
- qsort(globbuf.gl_pathv, globbuf.gl_pathc, sizeof(char *), logfile_path_comp);
-
- memset(&stitch_job, 0, sizeof(struct darshan_job));
-
- /* first pass at stitching together logs:
+ /* first pass at merging together logs:
* - compose output job-level metadata structure (including exe & mount data)
* - compose output record_id->file_name mapping
*/
- for(i = 0; i < globbuf.gl_pathc; i++)
+ for(i = 0; i < n_infiles; i++)
{
memset(&in_job, 0, sizeof(struct darshan_job));
- in_fd = darshan_log_open(globbuf.gl_pathv[i]);
+ in_fd = darshan_log_open(infile_list[i]);
if(in_fd == NULL)
{
fprintf(stderr,
"Error: unable to open input Darshan log file %s.\n",
- globbuf.gl_pathv[i]);
- globfree(&globbuf);
+ infile_list[i]);
return(-1);
}
@@ -258,9 +218,8 @@ int main(int argc, char *argv[])
{
fprintf(stderr,
"Error: unable to read job data from input Darshan log file %s.\n",
- globbuf.gl_pathv[i]);
+ infile_list[i]);
darshan_log_close(in_fd);
- globfree(&globbuf);
return(-1);
}
@@ -273,47 +232,44 @@ int main(int argc, char *argv[])
{
fprintf(stderr,
"Error: potentially corrupt data found in input log file %s.\n",
- globbuf.gl_pathv[i]);
+ infile_list[i]);
darshan_log_close(in_fd);
- globfree(&globbuf);
return(-1);
}
if(i == 0)
{
/* get job data, exe, & mounts directly from the first input log */
- memcpy(&stitch_job, &in_job, sizeof(struct darshan_job));
+ memcpy(&merge_job, &in_job, sizeof(struct darshan_job));
- ret = darshan_log_getexe(in_fd, stitch_exe);
+ ret = darshan_log_getexe(in_fd, merge_exe);
if(ret < 0)
{
fprintf(stderr,
- "Error: unable to read exe string from Darshan log file %s.\n",
- globbuf.gl_pathv[i]);
+ "Error: unable to read exe string from input Darshan log file %s.\n",
+ infile_list[i]);
darshan_log_close(in_fd);
- globfree(&globbuf);
return(-1);
}
- ret = darshan_log_getmounts(in_fd, &stitch_mnt_pts,
- &stitch_fs_types, &stitch_mnt_count);
+ ret = darshan_log_getmounts(in_fd, &merge_mnt_pts,
+ &merge_fs_types, &merge_mnt_count);
if(ret < 0)
{
fprintf(stderr,
- "Error: unable to read mount info from Darshan log file %s.\n",
- globbuf.gl_pathv[i]);
+ "Error: unable to read mount info from input Darshan log file %s.\n",
+ infile_list[i]);
darshan_log_close(in_fd);
- globfree(&globbuf);
return(-1);
}
}
else
{
/* potentially update job timestamps using remaining logs */
- if(in_job.start_time < stitch_job.start_time)
- stitch_job.start_time = in_job.start_time;
- if(in_job.end_time > stitch_job.end_time)
- stitch_job.end_time = in_job.end_time;
+ if(in_job.start_time < merge_job.start_time)
+ merge_job.start_time = in_job.start_time;
+ if(in_job.end_time > merge_job.end_time)
+ merge_job.end_time = in_job.end_time;
}
/* read the hash of ids->names for the input log */
@@ -322,9 +278,8 @@ int main(int argc, char *argv[])
{
fprintf(stderr,
"Error: unable to read job data from input Darshan log file %s.\n",
- globbuf.gl_pathv[i]);
+ infile_list[i]);
darshan_log_close(in_fd);
- globfree(&globbuf);
return(-1);
}
@@ -333,17 +288,16 @@ int main(int argc, char *argv[])
*/
HASH_ITER(hlink, in_hash, ref, tmp)
{
- HASH_FIND(hlink, stitch_hash, &(ref->id), sizeof(darshan_record_id), found);
+ HASH_FIND(hlink, merge_hash, &(ref->id), sizeof(darshan_record_id), found);
if(!found)
{
- HASH_ADD(hlink, stitch_hash, id, sizeof(darshan_record_id), ref);
+ HASH_ADD(hlink, merge_hash, id, sizeof(darshan_record_id), ref);
}
else if(strcmp(ref->name, found->name))
{
fprintf(stderr,
"Error: invalid Darshan record table entry.\n");
darshan_log_close(in_fd);
- globfree(&globbuf);
return(-1);
}
}
@@ -351,59 +305,54 @@ int main(int argc, char *argv[])
darshan_log_close(in_fd);
}
- /* create the output "stitched together" log */
- stitch_fd = darshan_log_create(stitch_logname, DARSHAN_ZLIB_COMP, 1);
- if(stitch_fd == NULL)
+ /* create the output "merged" log */
+ merge_fd = darshan_log_create(outlog_path, DARSHAN_ZLIB_COMP, 1);
+ if(merge_fd == NULL)
{
fprintf(stderr, "Error: unable to create output darshan log.\n");
- globfree(&globbuf);
return(-1);
}
/* write the darshan job info, exe string, and mount data to output file */
- ret = darshan_log_putjob(stitch_fd, &stitch_job);
+ ret = darshan_log_putjob(merge_fd, &merge_job);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write job data to output darshan log.\n");
- globfree(&globbuf);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
- ret = darshan_log_putexe(stitch_fd, stitch_exe);
+ ret = darshan_log_putexe(merge_fd, merge_exe);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write exe string to output darshan log.\n");
- globfree(&globbuf);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
- ret = darshan_log_putmounts(stitch_fd, stitch_mnt_pts, stitch_fs_types, stitch_mnt_count);
+ ret = darshan_log_putmounts(merge_fd, merge_mnt_pts, merge_fs_types, merge_mnt_count);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write mount data to output darshan log.\n");
- globfree(&globbuf);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
- /* write the stitched together table of records to output file */
- ret = darshan_log_puthash(stitch_fd, stitch_hash);
+ /* write the merged table of records to output file */
+ ret = darshan_log_puthash(merge_fd, merge_hash);
if(ret < 0)
{
fprintf(stderr, "Error: unable to write record table to output darshan log.\n");
- globfree(&globbuf);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
/* iterate over active darshan modules and gather module data to write
- * to the stitched together output log
+ * to the merged output log
*/
for(i = 0; i < DARSHAN_MAX_MODS; i++)
{
@@ -412,33 +361,30 @@ int main(int argc, char *argv[])
if(shared_redux)
{
/* build the hash of records shared globally by this module */
- ret = build_mod_shared_rec_hash(&globbuf, i, stitch_job.nprocs,
- mod_buf, &shared_rec_hash);
+ ret = build_mod_shared_rec_hash(infile_list, n_infiles, i,
+ merge_job.nprocs, mod_buf, &shared_rec_hash);
if(ret < 0)
{
fprintf(stderr,
"Error: unable to build list of %s module's shared records.\n",
darshan_module_names[i]);
- globfree(&globbuf);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
}
- for(j = 0; j < globbuf.gl_pathc; j++)
+ for(j = 0; j < n_infiles; j++)
{
- in_fd = darshan_log_open(globbuf.gl_pathv[j]);
+ in_fd = darshan_log_open(infile_list[j]);
if(in_fd == NULL)
{
fprintf(stderr,
"Error: unable to open input Darshan log file %s.\n",
- globbuf.gl_pathv[j]);
- globfree(&globbuf);
- darshan_log_close(in_fd);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ infile_list[j]);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
@@ -447,15 +393,15 @@ int main(int argc, char *argv[])
/* write out the shared records first */
HASH_ITER(hlink, shared_rec_hash, sref, stmp)
{
- ret = mod_logutils[i]->log_put_record(stitch_fd, sref->agg_rec, in_fd->mod_ver[i]);
+ ret = mod_logutils[i]->log_put_record(merge_fd, sref->agg_rec, in_fd->mod_ver[i]);
if(ret < 0)
{
fprintf(stderr,
"Error: unable to write %s module record to output darshan log.\n",
darshan_module_names[i]);
- globfree(&globbuf);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ darshan_log_close(in_fd);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
}
@@ -470,16 +416,15 @@ int main(int argc, char *argv[])
if(sref)
continue; /* skip shared records */
- ret = mod_logutils[i]->log_put_record(stitch_fd, mod_buf, in_fd->mod_ver[i]);
+ ret = mod_logutils[i]->log_put_record(merge_fd, mod_buf, in_fd->mod_ver[i]);
if(ret < 0)
{
fprintf(stderr,
"Error: unable to write %s module record to output log file %s.\n",
- darshan_module_names[i], globbuf.gl_pathv[j]);
- globfree(&globbuf);
+ darshan_module_names[i], infile_list[j]);
darshan_log_close(in_fd);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
}
@@ -487,11 +432,10 @@ int main(int argc, char *argv[])
{
fprintf(stderr,
"Error: unable to read %s module record from input log file %s.\n",
- darshan_module_names[i], globbuf.gl_pathv[j]);
- globfree(&globbuf);
+ darshan_module_names[i], infile_list[j]);
darshan_log_close(in_fd);
- darshan_log_close(stitch_fd);
- unlink(stitch_logname);
+ darshan_log_close(merge_fd);
+ unlink(outlog_path);
return(-1);
}
@@ -509,8 +453,16 @@ int main(int argc, char *argv[])
}
}
- darshan_log_close(stitch_fd);
- globfree(&globbuf);
+ darshan_log_close(merge_fd);
return(0);
}
+
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * End:
+ *
+ * vim: ts=8 sts=4 sw=4 expandtab
+ */
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/d3ebeb50a7b5adbadd02ef6da5a70bb5360da3ac...5f4e06f67c6195738e7f098fc697461d07bd41ff
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20160126/f19d67b0/attachment-0001.html>
More information about the Darshan-commits
mailing list