[Darshan-commits] [Git][darshan/darshan][master] 7 commits: modular instrumentation for MDHIM keyval
Shane Snyder
xgitlab at cels.anl.gov
Fri Jun 29 16:42:36 CDT 2018
Shane Snyder pushed to branch master at darshan / darshan
Commits:
91fa70eb by Rob Latham at 2018-06-22T11:48:58-05:00
modular instrumentation for MDHIM keyval
- - - - -
f6588980 by Rob Latham at 2018-06-22T11:48:58-05:00
record distribution of keys to servers
this requires a hacked up version of mdhim until there's a real api for
getting this information.
- - - - -
7055c66b by Rob Latham at 2018-06-22T11:48:58-05:00
update for API changes
- - - - -
6129e36f by Rob Latham at 2018-06-22T11:48:58-05:00
get mdhim module working with more than one process
- - - - -
00997bbb by Rob Latham at 2018-06-25T17:18:59-05:00
fix up shared record handling
- - - - -
e8b628a9 by Rob Latham at 2018-06-27T09:03:00-05:00
promote server counters to 64 bits
the struct alignment is important here, as we are going to read this
struct once to get the fixed values and a second time to fill in the
extra histogram buckets.
- - - - -
1a362ac0 by Shane Snyder at 2018-06-29T16:42:28-05:00
Merge branch 'hxhim-module' into 'master'
Hxhim module
See merge request darshan/darshan!23
- - - - -
13 changed files:
- darshan-log-format.h
- + darshan-mdhim-log-format.h
- darshan-runtime/Makefile.in
- darshan-runtime/configure
- darshan-runtime/configure.in
- darshan-runtime/darshan-runtime-config.h.in
- + darshan-runtime/lib/darshan-mdhim.c
- darshan-runtime/share/ld-opts/darshan-base-ld-opts.in
- + darshan-runtime/share/ld-opts/darshan-mdhim-ld-opts
- darshan-util/Makefile.in
- darshan-util/darshan-logutils.h
- + darshan-util/darshan-mdhim-logutils.c
- + darshan-util/darshan-mdhim-logutils.h
Changes:
=====================================
darshan-log-format.h
=====================================
--- a/darshan-log-format.h
+++ b/darshan-log-format.h
@@ -119,6 +119,7 @@ struct darshan_base_record
#include "darshan-stdio-log-format.h"
/* DXT */
#include "darshan-dxt-log-format.h"
+#include "darshan-mdhim-log-format.h"
/* X-macro for keeping module ordering consistent */
/* NOTE: first val used to define module enum values,
@@ -141,7 +142,9 @@ struct darshan_base_record
X(DARSHAN_STDIO_MOD, "STDIO", DARSHAN_STDIO_VER, &stdio_logutils) \
/* DXT */ \
X(DXT_POSIX_MOD, "DXT_POSIX", DXT_POSIX_VER, &dxt_posix_logutils) \
- X(DXT_MPIIO_MOD, "DXT_MPIIO", DXT_MPIIO_VER, &dxt_mpiio_logutils)
+ X(DXT_MPIIO_MOD, "DXT_MPIIO", DXT_MPIIO_VER, &dxt_mpiio_logutils) \
+ X(DARSHAN_MDHIM_MOD, "MDHIM", DARSHAN_MDHIM_VER, &mdhim_logutils)
+
/* unique identifiers to distinguish between available darshan modules */
/* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]
=====================================
darshan-mdhim-log-format.h
=====================================
--- /dev/null
+++ b/darshan-mdhim-log-format.h
@@ -0,0 +1,74 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_MDHIM_LOG_FORMAT_H
+#define __DARSHAN_MDHIM_LOG_FORMAT_H
+
+/* current log format version, to support backwards compatibility */
+#define DARSHAN_MDHIM_VER 1
+
+#define MDHIM_COUNTERS \
+ /* number of 'put' function calls */\
+ X(MDHIM_PUTS) \
+ /* larget payload for a 'put' */ \
+ X(MDHIM_GETS) \
+ /* largest get */ \
+ X(MDHIM_PUT_MAX_SIZE)\
+ /* number of 'get' function calls */\
+ X(MDHIM_GET_MAX_SIZE) \
+ /* how many servers? */ \
+ X(MDHIM_SERVERS) \
+ /* end of counters */ \
+ X(MDHIM_NUM_INDICES)
+
+#define MDHIM_F_COUNTERS \
+ /* timestamp of the first call to a 'put' function */\
+ X(MDHIM_F_PUT_TIMESTAMP) \
+ X(MDHIM_F_GET_TIMESTAMP) \
+ /* timer indicating longest (slowest) call to put/get */\
+ X(MDHIM_F_PUT_MAX_DURATION) \
+ X(MDHIM_F_GET_MAX_DURATION) \
+ /* end of counters */\
+ X(MDHIM_F_NUM_INDICES)
+
+#define X(a) a,
+/* integer counters for the "NULL" example module */
+enum darshan_mdhim_indices
+{
+ MDHIM_COUNTERS
+};
+
+/* floating point counters for the "NULL" example module */
+enum darshan_mdhim_f_indices
+{
+ MDHIM_F_COUNTERS
+};
+#undef X
+
+/* the darshan_mdhim_record structure encompasses the high-level data/counters
+ * which would actually be logged to file by Darshan for the "MDHIM"
+ * module. This implementation logs the following data for each
+ * record:
+ * - a darshan_base_record structure, which contains the record id & rank
+ * - integer I/O counters (operation counts, I/O sizes, etc.)
+ * - floating point I/O counters (timestamps, cumulative timers, etc.)
+ */
+struct darshan_mdhim_record
+{
+ struct darshan_base_record base_rec;
+ int64_t counters[MDHIM_NUM_INDICES];
+ double fcounters[MDHIM_F_NUM_INDICES];
+ /* when we allocate this struct, we'll do so with enough extra memory to
+ * hold N servers. Compare to approach taken with darshan_lustre_record */
+ /* be mindful of struct alignment here: If one reads "sizeof(struct
+ * darshan_mdhim_record)", one might end up reading more than expected.
+ * Second read will then end up reading less than needed */
+ int64_t server_histogram[1];
+};
+
+/* '-1' because d_m_r already allocated with space for one */
+#define MDHIM_RECORD_SIZE(servers) (sizeof(struct darshan_mdhim_record) + sizeof(int64_t) * ((servers) - 1) )
+#endif /* __DARSHAN_MDHIM_LOG_FORMAT_H */
=====================================
darshan-runtime/Makefile.in
=====================================
--- a/darshan-runtime/Makefile.in
+++ b/darshan-runtime/Makefile.in
@@ -36,6 +36,7 @@ CFLAGS_SHARED = -DDARSHAN_CONFIG_H=\"darshan-runtime-config.h\" -I . -I$(srcdir)
LIBS = -lz @LIBBZ2@
BUILD_HDF5_MODULE = @BUILD_HDF5_MODULE@
+BUILD_MDHIM_MODULE = @BUILD_MDHIM_MODULE@
DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-pnetcdf.o lib/darshan-stdio.o lib/darshan-dxt.o
DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-pnetcdf.po lib/darshan-stdio.po lib/darshan-dxt.po
@@ -59,6 +60,13 @@ CFLAGS += -DDARSHAN_LUSTRE
CFLAGS_SHARED += -DDARSHAN_LUSTRE
endif
+ifdef BUILD_MDHIM_MODULE
+DARSHAN_STATIC_MOD_OBJS += lib/darshan-mdhim.o
+DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-mdhim.po
+CFLAGS += -DDARSHAN_MDHIM
+CFLAGS_SHARED += -DDARSHAN_MDHIM
+endif
+
lib::
@mkdir -p $@
@@ -140,6 +148,13 @@ lib/darshan-dxt.o: lib/darshan-dxt.c darshan.h darshan-common.h $(DARSHAN_LOG_FO
lib/darshan-dxt.po: lib/darshan-dxt.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxt-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
+lib/darshan-mdhim.o: lib/darshan-mdhim.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mdhim-log-format.h | lib
+ $(CC) $(CFLAGS) -c $< -o $@
+
+lib/darshan-mdhim.po: lib/darshan-mdhim.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mdhim-log-format.h | lib
+ $(CC) $(CFLAGS_SHARED) -c $< -o $@
+
+
lib/lookup3.o: lib/lookup3.c
$(CC) $(CFLAGS) -c $< -o $@
@@ -197,6 +212,9 @@ endif
ifdef BUILD_HDF5_MODULE
install -m 644 $(srcdir)/share/ld-opts/darshan-hdf5-ld-opts $(datarootdir)/ld-opts/darshan-hdf5-ld-opts
endif
+ifdef BUILD_MDHIM_MODULE
+ install -m 644 $(srcdir)/share/ld-opts/darshan-mdhim-ld-opts $(datarootdir)/ld-opts/darshan-mdhim-ld-opts
+endif
install -m 644 $(srcdir)/share/ld-opts/darshan-pnetcdf-ld-opts $(datarootdir)/ld-opts/darshan-pnetcdf-ld-opts
install -m 644 $(srcdir)/share/ld-opts/darshan-stdio-ld-opts $(datarootdir)/ld-opts/darshan-stdio-ld-opts
install -m 644 $(srcdir)/share/ld-opts/darshan-mpiio-ld-opts $(datarootdir)/ld-opts/darshan-mpiio-ld-opts
=====================================
darshan-runtime/configure
=====================================
--- a/darshan-runtime/configure
+++ b/darshan-runtime/configure
@@ -621,6 +621,8 @@ ac_includes_default="\
ac_subst_vars='LTLIBOBJS
LIBOBJS
+DARSHAN_MDHIM_LD_OPTS
+BUILD_MDHIM_MODULE
DARSHAN_HDF5_LD_OPTS
BUILD_HDF5_MODULE
DARSHAN_USE_LUSTRE
@@ -702,6 +704,7 @@ with_jobid_env
with_mod_mem
enable_HDF5_post_1_10
enable_HDF5_pre_1_10
+enable_mdhim
'
ac_precious_vars='build_alias
host_alias
@@ -1332,6 +1335,7 @@ Optional Features:
Enable HDF5 module for HDF5 version 1.10 or later
--enable-HDF5-pre-1.10
Enable HDF5 module for HDF5 versions earlier than 1.10
+ --enable-mdhim Enable mdhim module
Optional Packages:
--with-PACKAGE[=ARG] use PACKAGE [ARG=yes]
@@ -3761,6 +3765,7 @@ fi
fi
+
if test x$GOT_ALIGNMENT != x1; then
as_fn_error $? "must provide --with-mem-align=<num> argument to configure." "$LINENO" 5
fi
@@ -4223,6 +4228,33 @@ fi
fi
+BUILD_MDHIM_MODULE=
+DARSHAN_MDHIM_LD_OPTS=
+# Check whether --enable-mdhim was given.
+if test "${enable_mdhim+set}" = set; then :
+ enableval=$enable_mdhim;
+fi
+
+if test "x$enable_mdhim" = "xyes"; then :
+ BUILD_MDHIM_MODULE=1
+ DARSHAN_MDHIM_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-mdhim-ld-opts"
+ for ac_header in mdhim.h
+do :
+ ac_fn_c_check_header_mongrel "$LINENO" "mdhim.h" "ac_cv_header_mdhim_h" "$ac_includes_default"
+if test "x$ac_cv_header_mdhim_h" = xyes; then :
+ cat >>confdefs.h <<_ACEOF
+#define HAVE_MDHIM_H 1
+_ACEOF
+
+else
+ as_fn_error $? "mdhim requested but headers cannot be found" "$LINENO" 5
+fi
+
+done
+
+fi
+
+
# determine if the MPI library includes MPI-IO functions or not
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for MPI-IO support in MPI" >&5
$as_echo_n "checking for MPI-IO support in MPI... " >&6; }
@@ -4401,6 +4433,8 @@ DARSHAN_VERSION="3.1.6"
+
+
ac_config_files="$ac_config_files Makefile darshan-mk-log-dirs.pl darshan-gen-cc.pl darshan-gen-cxx.pl darshan-gen-fortran.pl darshan-config share/craype-1.x/darshan-module share/craype-2.x/darshan-module lib/pkgconfig/darshan-runtime.pc share/mpi-profile/darshan-cc.conf share/mpi-profile/darshan-cxx.conf share/mpi-profile/darshan-f.conf share/mpi-profile/darshan-bg-cc.conf share/mpi-profile/darshan-bg-cxx.conf share/mpi-profile/darshan-bg-f.conf share/ld-opts/darshan-base-ld-opts"
cat >confcache <<\_ACEOF
=====================================
darshan-runtime/configure.in
=====================================
--- a/darshan-runtime/configure.in
+++ b/darshan-runtime/configure.in
@@ -162,6 +162,7 @@ if test x$enable_lustre_mod != xno; then
[]))
fi
+
if test x$GOT_ALIGNMENT != x1; then
AC_MSG_ERROR(must provide --with-mem-align=<num> argument to configure.)
fi
@@ -283,6 +284,22 @@ AC_ARG_ENABLE(HDF5-pre-1.10,
fi]
,)
+BUILD_MDHIM_MODULE=
+DARSHAN_MDHIM_LD_OPTS=
+AC_ARG_ENABLE([mdhim],
+ AS_HELP_STRING([--enable-mdhim], [Enable mdhim module]),
+ [],[] )
+AS_IF([test "x$enable_mdhim" = "xyes"],
+ BUILD_MDHIM_MODULE=1
+ DARSHAN_MDHIM_LD_OPTS="@${darshan_share_path}/ld-opts/darshan-mdhim-ld-opts"
+ AC_CHECK_HEADERS([mdhim.h],
+ [],
+ AC_MSG_ERROR([mdhim requested but headers cannot be found]) ),
+ []
+
+ )
+
+
# determine if the MPI library includes MPI-IO functions or not
AC_MSG_CHECKING(for MPI-IO support in MPI)
AC_TRY_LINK([#include <mpi.h>], [
@@ -382,6 +399,8 @@ AC_SUBST(DARSHAN_USE_BGQ)
AC_SUBST(DARSHAN_USE_LUSTRE)
AC_SUBST(BUILD_HDF5_MODULE)
AC_SUBST(DARSHAN_HDF5_LD_OPTS)
+AC_SUBST(BUILD_MDHIM_MODULE)
+AC_SUBST(DARSHAN_MDHIM_LD_OPTS)
AC_OUTPUT(Makefile
darshan-mk-log-dirs.pl
darshan-gen-cc.pl
=====================================
darshan-runtime/darshan-runtime-config.h.in
=====================================
--- a/darshan-runtime/darshan-runtime-config.h.in
+++ b/darshan-runtime/darshan-runtime-config.h.in
@@ -12,6 +12,9 @@
/* Define to 1 if you have the `z' library (-lz). */
#undef HAVE_LIBZ
+/* Define to 1 if you have the <mdhim.h> header file. */
+#undef HAVE_MDHIM_H
+
/* Define to 1 if you have the <memory.h> header file. */
#undef HAVE_MEMORY_H
=====================================
darshan-runtime/lib/darshan-mdhim.c
=====================================
--- /dev/null
+++ b/darshan-runtime/lib/darshan-mdhim.c
@@ -0,0 +1,607 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#define _XOPEN_SOURCE 500
+#define _GNU_SOURCE
+
+#include "darshan-runtime-config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <string.h>
+#include <assert.h>
+
+#include <mdhim.h>
+
+#include "darshan.h"
+#include "darshan-dynamic.h"
+
+#define RECORD_STRING "total-mdhim-obj-stats"
+
+/* The DARSHAN_FORWARD_DECL macro (defined in darshan.h) is used to provide forward
+ * declarations for wrapped funcions, regardless of whether Darshan is used with
+ * statically or dynamically linked executables.
+ */
+DARSHAN_FORWARD_DECL(mdhimPut, mdhim_rm_t *, (mdhim_t *md,
+ index_t *index, void *primary_key, size_t primary_key_len,
+ void *value, int value_len));
+
+DARSHAN_FORWARD_DECL(mdhimGet, mdhim_grm_t *, (mdhim_t *md,
+ index_t *index, void *key, size_t key_len, int op));
+
+DARSHAN_FORWARD_DECL(mdhimInit, int, (mdhim_t *md, mdhim_options_t *opts));
+
+/* The mdhim_record_ref structure maintains necessary runtime metadata
+ * for the MDHIM module record (darshan_mdhim_record structure, defined in
+ * darshan-mdhim-log-format.h) pointed to by 'record_p'. This metadata
+ * assists with the instrumenting of specific statistics in the record.
+ *
+ * RATIONALE: the MDHIM module needs to track some stateful, volatile
+ * information about each record it has registered (for instance, most
+ * recent access time, amount of bytes transferred) to aid in
+ * instrumentation, but this information can't be stored in the
+ * darshan_mdhim_record struct
+ * because we don't want it to appear in the final darshan log file. We
+ * therefore associate a mdhim_record_ref struct with each
+ * darshan_mdhim_record struct in order to track this information (i.e.,
+ * the mapping between mdhim_record_ref structs to darshan_mdhim_record
+ * structs is one-to-one).
+ *
+ * NOTE: we use the 'darshan_record_ref' interface (in darshan-common)
+ * to associate different types of handles with this mdhim_record_ref
+ * struct. This allows us to index this struct (and the underlying
+ * record) by using either the corresponding Darshan record identifier
+ * or by any other arbitrary handle.
+ */
+struct mdhim_record_ref
+{
+ /* Darshan record for the MDHIM example module */
+ struct darshan_mdhim_record *record_p;
+
+ /* ... other runtime data ... */
+};
+
+/* The mdhim_runtime structure maintains necessary state for storing
+ * MDHIM records and for coordinating with darshan-core at shutdown time.
+ */
+struct mdhim_runtime
+{
+ /* rec_id_hash is a pointer to a hash table of MDHIM module record
+ * references, indexed by Darshan record id
+ */
+ void *rec_id_hash;
+ /* number of records currently tracked */
+ int rec_count;
+ int record_buffer_size;
+ void *record_buffer;
+};
+
+/* internal helper functions for the MDHIM module */
+static void mdhim_runtime_initialize(
+ void);
+static struct mdhim_record_ref *mdhim_track_new_record(
+ darshan_record_id rec_id, int nr_servers, const char *name);
+static void mdhim_cleanup_runtime(
+ void);
+
+static void mdhim_subtract_shared_rec_size(void * rec_ref_p);
+
+/* forward declaration for MDHIM shutdown function needed to interface
+ * with darshan-core
+ */
+static void mdhim_shutdown(MPI_Comm mod_comm, darshan_record_id *shared_recs,
+ int shared_rec_count, void **mdhim_buf, int *mdhim_buf_sz);
+
+/* mdhim_runtime is the global data structure encapsulating "MDHIM"
+ * module state */
+static struct mdhim_runtime *mdhim_runtime = NULL;
+/* The mdhim_runtime_mutex is a lock used when updating the
+ * mdhim_runtime global structure (or any other global data structures).
+ * This is necessary to avoid race conditions as multiple threads may
+ * execute function wrappers and update module state.
+ * NOTE: Recursive mutexes are used in case functions wrapped by this
+ * module call other wrapped functions that would result in deadlock,
+ * otherwise. This mechanism may not be necessary for all
+ * instrumentation modules.
+ */
+static pthread_mutex_t mdhim_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+/* my_rank indicates the MPI rank of this process */
+static int my_rank = -1;
+
+/* macros for obtaining/releasing the "MDHIM" module lock */
+#define MDHIM_LOCK() pthread_mutex_lock(&mdhim_runtime_mutex)
+#define MDHIM_UNLOCK() pthread_mutex_unlock(&mdhim_runtime_mutex)
+
+/* the MDHIM_PRE_RECORD macro is executed before performing MDHIM
+ * module instrumentation of a call. It obtains a lock for updating
+ * module data strucutres, and ensure the MDHIM module has been properly
+ * initialized before instrumenting.
+ */
+#define MDHIM_PRE_RECORD() do { \
+ MDHIM_LOCK(); \
+ if(!darshan_core_disabled_instrumentation()) { \
+ if(!mdhim_runtime) mdhim_runtime_initialize(); \
+ if(mdhim_runtime) break; \
+ } \
+ MDHIM_UNLOCK(); \
+} while(0)
+
+/* the MDHIM_POST_RECORD macro is executed after performing MDHIM
+ * module instrumentation. It simply releases the module lock.
+ */
+#define MDHIM_POST_RECORD() do { \
+ MDHIM_UNLOCK(); \
+} while(0)
+
+/* macro for instrumenting the "MDHIM" module's put function */
+#define MDHIM_RECORD_PUT(__ret, __md, __id, __vallen, __tm1, __tm2) do{ \
+ darshan_record_id rec_id; \
+ struct mdhim_record_ref *rec_ref; \
+ double __elapsed = __tm2 - __tm1; \
+ /* if put returns error (return code < 0), don't instrument anything */ \
+ if(__ret < 0) break; \
+ /* posix uses '__name' to generate a unique Darshan record id */ \
+ /* but mdhim doesn't use string names for its keyval store */ \
+ rec_id = darshan_core_gen_record_id(RECORD_STRING); \
+ /* look up a record reference for this record id using darshan rec_ref interface */ \
+ rec_ref = darshan_lookup_record_ref(mdhim_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
+ /* if no reference was found, that's odd: was init not called? */ \
+ if(!rec_ref) break; \
+ /* increment counter indicating number of calls to 'put' */ \
+ rec_ref->record_p->counters[MDHIM_PUTS] += 1; \
+ /* store max data value for calls to 'put', and corresponding time duration */ \
+ if(rec_ref->record_p->counters[MDHIM_PUT_MAX_SIZE] < __vallen) { \
+ rec_ref->record_p->counters[MDHIM_PUT_MAX_SIZE] = __vallen; \
+ rec_ref->record_p->fcounters[MDHIM_F_PUT_MAX_DURATION] = __elapsed; \
+ } \
+ /* store timestamp of first call to 'put' */ \
+ if(rec_ref->record_p->fcounters[MDHIM_F_PUT_TIMESTAMP] == 0 || \
+ rec_ref->record_p->fcounters[MDHIM_F_PUT_TIMESTAMP] > __tm1) \
+ rec_ref->record_p->fcounters[MDHIM_F_PUT_TIMESTAMP] = __tm1; \
+ /* record which server gets this request */ \
+ rec_ref->record_p->server_histogram[(__id)]++; \
+} while(0)
+
+/* macro for instrumenting the "MDHIM" module's get function */
+#define MDHIM_RECORD_GET(__ret, __md, __id, __keylen, __tm1, __tm2) do{ \
+ darshan_record_id rec_id; \
+ struct mdhim_record_ref *rec_ref; \
+ double __elapsed = __tm2 - __tm1; \
+ /* if get returns error (return code < 0), don't instrument anything */ \
+ if(__ret == NULL) break; \
+ /* posix uses '__name' to generate a unique Darshan record id */ \
+ /* but mdhim doesn't use string names for its keyval store */ \
+ rec_id = darshan_core_gen_record_id(RECORD_STRING); \
+ /* look up a record reference for this record id using darshan rec_ref interface */ \
+ rec_ref = darshan_lookup_record_ref(mdhim_runtime->rec_id_hash, &rec_id, sizeof(darshan_record_id)); \
+ /* if no reference was found, we're in trouble */ \
+ if(!rec_ref) break; \
+ /* increment counter indicating number of calls to 'get' */ \
+ rec_ref->record_p->counters[MDHIM_GETS] += 1; \
+ /* store max data value for calls to 'get', and corresponding time duration */ \
+ if(rec_ref->record_p->counters[MDHIM_GET_MAX_SIZE] < __keylen) { \
+ rec_ref->record_p->counters[MDHIM_GET_MAX_SIZE] = __keylen; \
+ rec_ref->record_p->fcounters[MDHIM_F_GET_MAX_DURATION] = __elapsed; \
+ } \
+ /* store timestamp of first call to 'get' */ \
+ if(rec_ref->record_p->fcounters[MDHIM_F_GET_TIMESTAMP] == 0 || \
+ rec_ref->record_p->fcounters[MDHIM_F_GET_TIMESTAMP] > __tm1) \
+ rec_ref->record_p->fcounters[MDHIM_F_GET_TIMESTAMP] = __tm1; \
+ /* server distribution */ \
+ rec_ref->record_p->server_histogram[(__id)]++; \
+} while(0)
+
+/**********************************************************
+ * Wrappers for "MDHIM" module functions of interest *
+ **********************************************************/
+
+/* The DARSHAN_DECL macro provides the appropriate wrapper function
+ * names, depending on whether the Darshan library is statically or
+ * dynamically linked.
+ */
+
+int DARSHAN_DECL(mdhimInit)(mdhim_t *md, mdhim_options_t *opts)
+{
+ /* not counting or tracking anything in this routine, but grabbing a
+ * bit of information about the mdhim instance */
+
+ int ret;
+ darshan_record_id rec_id;
+ struct mdhim_record_ref *rec_ref;
+ int nr_servers;
+
+ MPI_Comm_size(opts->comm, &nr_servers);
+
+ MDHIM_PRE_RECORD();
+ /* posix uses '__name' to generate a unique Darshan record id
+ but mdhim doesn't use string names for its keyval store. Assumes
+ one MDHIM instance */
+ rec_id = darshan_core_gen_record_id(RECORD_STRING);
+ /* look up a record reference for this record id using darshan
+ * rec_ref interface */
+ rec_ref = darshan_lookup_record_ref(mdhim_runtime->rec_id_hash,
+ &rec_id, sizeof(darshan_record_id));
+ /* if no reference was found, track a new one for this record */
+ if(!rec_ref) rec_ref = mdhim_track_new_record(rec_id,
+ nr_servers, RECORD_STRING);
+ /* if we still don't have a valid reference, well that's too dang bad */
+ if (rec_ref) rec_ref->record_p->counters[MDHIM_SERVERS] = nr_servers;
+
+ MDHIM_POST_RECORD();
+
+ MAP_OR_FAIL(mdhimInit);
+ ret = __real_mdhimInit(md, opts);
+ return ret;
+
+}
+mdhim_rm_t *DARSHAN_DECL(mdhimPut)(mdhim_t *md,
+ index_t *index,
+ void *key, size_t key_len,
+ void *value, size_t value_len)
+{
+ mdhim_rm_t *ret;
+ double tm1, tm2;
+
+ /* The MAP_OR_FAIL macro attempts to obtain the address of the actual
+ * underlying put function call (__real_put), in the case of LD_PRELOADing
+ * the Darshan library. For statically linked executables, this macro is
+ * just a NOP.
+ */
+ MAP_OR_FAIL(mdhimPut);
+
+ /* In general, Darshan wrappers begin by calling the real version of the
+ * given wrapper function. Timers are used to record the duration of this
+ * operation. */
+ tm1 = darshan_core_wtime();
+ ret = __real_mdhimPut(md, index, key, key_len, value, value_len);
+ tm2 = darshan_core_wtime();
+
+ int server_id = mdhimWhichDB(md, key, key_len);
+
+ MDHIM_PRE_RECORD();
+ /* Call macro for instrumenting data for mdhimPut function calls. */
+ /* TODO: call the mdhim hash routines and instrument which servers
+ * get this request */
+ MDHIM_RECORD_PUT(ret, md, server_id, value_len, tm1, tm2);
+
+ MDHIM_POST_RECORD();
+
+ return(ret);
+}
+
+mdhim_grm_t * DARSHAN_DECL(mdhimGet)(mdhim_t *md,
+ index_t *index, void *key, size_t key_len,
+ enum TransportGetMessageOp op)
+{
+ mdhim_grm_t *ret;
+ double tm1, tm2;
+
+ MAP_OR_FAIL(mdhimGet);
+
+ /* In general, Darshan wrappers begin by calling the real version of the
+ * given wrapper function. Timers are used to record the duration of this
+ * operation. */
+ tm1 = darshan_core_wtime();
+ ret = __real_mdhimGet(md, index, key, key_len, op);
+ tm2 = darshan_core_wtime();
+
+ int server_id = mdhimWhichDB(md, key, key_len);
+
+ MDHIM_PRE_RECORD();
+ /* Call macro for instrumenting data for get function calls. */
+ MDHIM_RECORD_GET(ret, md, server_id, key_len, tm1, tm2);
+ MDHIM_POST_RECORD();
+ return ret;
+}
+
+/**********************************************************
+ * Internal functions for manipulating MDHIM module state *
+ **********************************************************/
+
+/* Initialize internal MDHIM module data structures and register with
+ * darshan-core. */
+static void mdhim_runtime_initialize()
+{
+ int mdhim_buf_size;
+
+ /* try and store a default number of records for this module */
+ mdhim_buf_size = DARSHAN_DEF_MOD_REC_COUNT *
+ sizeof(struct darshan_mdhim_record);
+
+ /* register the MDHIM module with the darshan-core component */
+ darshan_core_register_module(
+ DARSHAN_MDHIM_MOD, /* Darshan module identifier, defined in darshan-log-format.h */
+ &mdhim_shutdown,
+ &mdhim_buf_size,
+ &my_rank,
+ NULL);
+
+ /* return if darshan-core does not provide enough module memory for at
+ * least one MDHIM record
+ */
+ if(mdhim_buf_size < sizeof(struct darshan_mdhim_record))
+ {
+ darshan_core_unregister_module(DARSHAN_MDHIM_MOD);
+ return;
+ }
+
+ /* initialize module's global state */
+ mdhim_runtime = calloc(1, sizeof(*mdhim_runtime));
+ if(!mdhim_runtime)
+ {
+ darshan_core_unregister_module(DARSHAN_MDHIM_MOD);
+ return;
+ }
+ return;
+}
+
+/* allocate and track a new MDHIM module record */
+static struct mdhim_record_ref *mdhim_track_new_record(
+ darshan_record_id rec_id, int nr_servers, const char *name)
+{
+ struct darshan_mdhim_record *record_p = NULL;
+ struct mdhim_record_ref *rec_ref = NULL;
+ int ret;
+ size_t rec_size;
+
+ rec_ref = calloc(1, sizeof(*rec_ref));
+ if(!rec_ref)
+ return(NULL);
+
+ /* allocate a new MDHIM record reference and add it to the hash
+ * table, using the Darshan record identifier as the handle
+ */
+ ret = darshan_add_record_ref(&(mdhim_runtime->rec_id_hash), &rec_id,
+ sizeof(darshan_record_id), rec_ref);
+ if(ret == 0)
+ {
+ free(rec_ref);
+ return(NULL);
+ }
+
+ rec_size = MDHIM_RECORD_SIZE(nr_servers);
+ /* register the actual file record with darshan-core so it is persisted
+ * in the log file
+ */
+ record_p = darshan_core_register_record(
+ rec_id,
+ name,
+ DARSHAN_MDHIM_MOD,
+ rec_size,
+ NULL);
+
+ if(!record_p)
+ {
+ /* if registration fails, delete record reference and return */
+ darshan_delete_record_ref(&(mdhim_runtime->rec_id_hash),
+ &rec_id, sizeof(darshan_record_id));
+ free(rec_ref);
+ return(NULL);
+ }
+
+ /* registering this file record was successful, so initialize some fields */
+ record_p->base_rec.id = rec_id;
+ record_p->base_rec.rank = my_rank;
+ rec_ref->record_p = record_p;
+ mdhim_runtime->rec_count++;
+
+
+ /* return pointer to the record reference */
+ return(rec_ref);
+}
+
+/* cleanup MDHIM module internal data structures */
+static void mdhim_cleanup_runtime()
+{
+ /* iterate the hash of record references and free them */
+ darshan_clear_record_refs(&(mdhim_runtime->rec_id_hash), 1);
+
+ free(mdhim_runtime);
+ mdhim_runtime = NULL;
+
+ return;
+}
+
+static void mdhim_record_reduction_op(void *infile_v, void *inoutfile_v,
+ int *len, MPI_Datatype *datatype)
+{
+ struct darshan_mdhim_record *tmp_rec;
+ struct darshan_mdhim_record *inrec = infile_v;
+ struct darshan_mdhim_record *inoutrec = inoutfile_v;
+ int i, j;
+
+ for (i=0; i< *len; i++) {
+ /* can't use 'sizeof': server count historgram */
+ tmp_rec = calloc(1,
+ MDHIM_RECORD_SIZE(inrec->counters[MDHIM_SERVERS]));
+ tmp_rec->base_rec.id = inrec->base_rec.id;
+ tmp_rec->base_rec.rank = -1;
+
+ for (j=MDHIM_PUTS; j<=MDHIM_GETS; j++) {
+ tmp_rec->counters[j] = inrec->counters[j] +
+ inoutrec->counters[j];
+ }
+
+ for (j=MDHIM_PUT_MAX_SIZE; j<=MDHIM_GET_MAX_SIZE; j++) {
+ tmp_rec->counters[j] = (
+ (inrec->counters[j] > inoutrec->counters[j] ) ?
+ inrec->counters[j] :
+ inoutrec->counters[j]);
+ }
+ tmp_rec->counters[MDHIM_SERVERS] = inrec->counters[MDHIM_SERVERS];
+
+ /* min non-zero value */
+ for (j=MDHIM_F_PUT_TIMESTAMP; j<=MDHIM_F_GET_TIMESTAMP; j++)
+ {
+ if (( inrec->fcounters[j] < inoutrec->fcounters[j] &&
+ inrec->fcounters[j] > 0)
+ || inoutrec->fcounters[j] == 0)
+ tmp_rec->fcounters[j] = inrec->fcounters[j];
+ else
+ tmp_rec->fcounters[j] = inoutrec->fcounters[j];
+ }
+ /* max */
+ for (j=MDHIM_F_PUT_MAX_DURATION; j<=MDHIM_F_GET_MAX_DURATION; j++)
+ {
+ tmp_rec->fcounters[j] = (
+ (inrec->fcounters[j] > inoutrec->fcounters[j]) ?
+ inrec->fcounters[j] :
+ inoutrec->fcounters[j]);
+ }
+ /* dealing with server histogram a little odd. Every client kept track
+ * of which servers it sent to, so we'll simply sum them all up. The
+ * data lives at the end of the struct (remember, alocated based on
+ * MDHIM_RECORD_SIZE macro) */
+ for (j=0; j< tmp_rec->counters[MDHIM_SERVERS]; j++) {
+ tmp_rec->server_histogram[j] = inrec->server_histogram[j] +
+ inoutrec->server_histogram[j];
+ }
+ memcpy(inoutrec, tmp_rec,
+ MDHIM_RECORD_SIZE(tmp_rec->counters[MDHIM_SERVERS]));
+ free(tmp_rec);
+
+ /* updating not as simple as incrementing, unfortunately */
+ infile_v = (char *) infile_v +
+ MDHIM_RECORD_SIZE(tmp_rec->counters[MDHIM_SERVERS]);
+ inoutfile_v = (char *)inoutfile_v +
+ MDHIM_RECORD_SIZE(tmp_rec->counters[MDHIM_SERVERS]);
+ }
+ return;
+}
+/***********************************************************************
+ * shutdown function exported by the MDHIM module for coordinating with
+ * darshan-core *
+ ***********************************************************************/
+
+/* Pass output data for the MDHIM module back to darshan-core to log to
+ * file, and shutdown/free internal data structures.
+ */
+static void mdhim_shutdown(
+ MPI_Comm mod_comm,
+ darshan_record_id *shared_recs,
+ int shared_rec_count,
+ void **mdhim_buf,
+ int *mdhim_buf_sz)
+{
+
+ int i, nr_servers=0;
+ /* other modules can declar this temporary record on the stack but I need a
+ * bit more space because of the server histogram */
+ struct mdhim_record_ref *rec_ref;
+ /* walking through these arrays will be awkward if there is more than one
+ * record: the 'server_histogram' field is variable */
+ struct darshan_mdhim_record *mdhim_rec_buf =
+ *(struct darshan_mdhim_record **)mdhim_buf;
+ struct darshan_mdhim_record *red_send_buf = NULL;
+ struct darshan_mdhim_record *red_recv_buf = NULL;
+ MPI_Datatype red_type;
+ MPI_Op red_op;
+ int mdhim_rec_count;
+
+ MDHIM_LOCK();
+ assert(mdhim_runtime);
+
+ mdhim_rec_count = mdhim_runtime->rec_count;
+ mdhim_runtime->record_buffer = *mdhim_buf;
+ mdhim_runtime->record_buffer_size = *mdhim_buf_sz;
+
+ /* taking the approach in darshan-mpiio.c, except MDHIM is always a
+ * "shared file" for now. */
+ assert(mdhim_runtime->rec_count == shared_rec_count);
+
+ /* can the number of mdhim servers change? I suppose if there were
+ * multiple mdhim instances, each instance could have a different
+ * number of servers. If that's the case, I'll have to make some of
+ * the memory allocations variable (and I don't do that yet) */
+ rec_ref = darshan_lookup_record_ref(mdhim_runtime->rec_id_hash,
+ &shared_recs[0], sizeof(darshan_record_id));
+ nr_servers = rec_ref->record_p->counters[MDHIM_SERVERS];
+
+ if (shared_rec_count && !getenv("DARSHAN_DISABLE_SHARED_REDUCTION"))
+ {
+ /* already have the zeroth record because we checked how many
+ * servers there were */
+ rec_ref->record_p->base_rec.rank = -1;
+
+ /* there is probably only one shared record, but go ahead and
+ * check for any other shared records, setting their rank to -1.
+ * We will remove those from the report later */
+ /* starting from '1' since we grabbed the first record above */
+ for (i=1; i< shared_rec_count; i++)
+ {
+ rec_ref = darshan_lookup_record_ref(mdhim_runtime->rec_id_hash,
+ &shared_recs[i], sizeof(darshan_record_id));
+ assert(rec_ref);
+ assert(nr_servers == rec_ref->record_p->counters[MDHIM_SERVERS]);
+ rec_ref->record_p->base_rec.rank = -1;
+ }
+
+ darshan_record_sort(mdhim_rec_buf, mdhim_runtime->rec_count,
+ MDHIM_RECORD_SIZE(nr_servers));
+
+ /* make send_buf point to shared files at end */
+ red_send_buf = &(mdhim_rec_buf[mdhim_rec_count-shared_rec_count]);
+
+ if (my_rank == 0)
+ {
+ red_recv_buf = malloc(shared_rec_count *
+ MDHIM_RECORD_SIZE(nr_servers));
+ if (!red_recv_buf)
+ {
+ MDHIM_UNLOCK();
+ return;
+ }
+ }
+ PMPI_Type_contiguous(MDHIM_RECORD_SIZE(nr_servers),
+ MPI_BYTE, &red_type);
+ PMPI_Type_commit(&red_type);
+ PMPI_Op_create(mdhim_record_reduction_op, 1, &red_op);
+ PMPI_Reduce(red_send_buf, red_recv_buf,
+ shared_rec_count, red_type, red_op, 0, mod_comm);
+
+ if (my_rank == 0)
+ {
+ memcpy(&(mdhim_rec_buf[0]), red_recv_buf,
+ shared_rec_count *
+ MDHIM_RECORD_SIZE(nr_servers));
+ free(red_recv_buf);
+ }
+
+ PMPI_Type_free(&red_type);
+ PMPI_Op_free(&red_op);
+ /* drop shared records from non-root ranks or we'll end up writing too
+ * much */
+ if (my_rank != 0)
+ {
+ darshan_iter_record_refs(mdhim_runtime->rec_id_hash,
+ &mdhim_subtract_shared_rec_size);
+ }
+ }
+ *mdhim_buf_sz = mdhim_rec_count * mdhim_runtime->record_buffer_size;
+
+ /* shutdown internal structures used for instrumenting */
+ mdhim_cleanup_runtime();
+
+ MDHIM_UNLOCK();
+ return;
+}
+
+static void mdhim_subtract_shared_rec_size(void * rec_ref_p)
+{
+ struct mdhim_record_ref *m_rec_ref = (struct mdhim_record_ref *)rec_ref_p;
+ if (m_rec_ref->record_p->base_rec.rank == -1)
+ mdhim_runtime->record_buffer_size -=
+ MDHIM_RECORD_SIZE(m_rec_ref->record_p->counters[MDHIM_SERVERS] );
+}
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * End:
+ *
+ * vim: ts=8 sts=4 sw=4 expandtab
+ */
=====================================
darshan-runtime/share/ld-opts/darshan-base-ld-opts.in
=====================================
--- a/darshan-runtime/share/ld-opts/darshan-base-ld-opts.in
+++ b/darshan-runtime/share/ld-opts/darshan-base-ld-opts.in
@@ -12,3 +12,4 @@
@@darshan_share_path@/ld-opts/darshan-stdio-ld-opts
@@darshan_share_path@/ld-opts/darshan-mpiio-ld-opts
@DARSHAN_HDF5_LD_OPTS@
+ at DARSHAN_MDHIM_LD_OPTS@
=====================================
darshan-runtime/share/ld-opts/darshan-mdhim-ld-opts
=====================================
--- /dev/null
+++ b/darshan-runtime/share/ld-opts/darshan-mdhim-ld-opts
@@ -0,0 +1,2 @@
+--wrap=mdhimPut
+--wrap=mdhimGet
=====================================
darshan-util/Makefile.in
=====================================
--- a/darshan-util/Makefile.in
+++ b/darshan-util/Makefile.in
@@ -13,10 +13,45 @@ libdir = $(DESTDIR)@libdir@
pkgconfigdir = $(DESTDIR)$(libdir)/pkgconfig
DARSHAN_LOG_FORMAT = $(srcdir)/../darshan-log-format.h
-DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../darshan-mpiio-log-format.h $(srcdir)/../darshan-hdf5-log-format.h $(srcdir)/../darshan-pnetcdf-log-format.h $(srcdir)/../darshan-bgq-log-format.h $(srcdir)/../darshan-lustre-log-format.h $(srcdir)/../darshan-stdio-log-format.h $(srcdir)/../darshan-dxt-log-format.h
-DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h darshan-bgq-logutils.h darshan-lustre-logutils.h darshan-stdio-logutils.h darshan-dxt-logutils.h
-DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o darshan-lustre-logutils.o darshan-stdio-logutils.o darshan-dxt-logutils.o
-DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po darshan-lustre-logutils.po darshan-stdio-logutils.po darshan-dxt-logutils.po
+DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h \
+ $(srcdir)/../darshan-mpiio-log-format.h \
+ $(srcdir)/../darshan-hdf5-log-format.h \
+ $(srcdir)/../darshan-pnetcdf-log-format.h \
+ $(srcdir)/../darshan-bgq-log-format.h \
+ $(srcdir)/../darshan-lustre-log-format.h \
+ $(srcdir)/../darshan-stdio-log-format.h \
+ $(srcdir)/../darshan-dxt-log-format.h \
+ $(srcdir)/../darshan-mdhim-log-format.h
+
+DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h \
+ darshan-mpiio-logutils.h \
+ darshan-hdf5-logutils.h \
+ darshan-pnetcdf-logutils.h \
+ darshan-bgq-logutils.h \
+ darshan-lustre-logutils.h \
+ darshan-stdio-logutils.h \
+ darshan-dxt-logutils.h \
+ darshan-mdhim-logutils.h
+
+DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o \
+ darshan-mpiio-logutils.o \
+ darshan-hdf5-logutils.o \
+ darshan-pnetcdf-logutils.o \
+ darshan-bgq-logutils.o \
+ darshan-lustre-logutils.o \
+ darshan-stdio-logutils.o \
+ darshan-dxt-logutils.o \
+ darshan-mdhim-logutils.o
+
+DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po \
+ darshan-mpiio-logutils.po \
+ darshan-hdf5-logutils.po \
+ darshan-pnetcdf-logutils.po \
+ darshan-bgq-logutils.po \
+ darshan-lustre-logutils.po \
+ darshan-stdio-logutils.po \
+ darshan-dxt-logutils.po \
+ darshan-mdhim-logutils.po
DARSHAN_ENABLE_SHARED=@DARSHAN_ENABLE_SHARED@
@@ -96,6 +131,12 @@ darshan-dxt-logutils.o: darshan-dxt-logutils.c darshan-logutils.h darshan-dxt-lo
darshan-dxt-logutils.po: darshan-dxt-logutils.c darshan-logutils.h darshan-dxt-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-dxt-log-format.h | uthash-1.9.2
$(CC) $(CFLAGS_SHARED) -c $< -o $@
+darshan-mdhim-logutils.o: darshan-mdhim-logutils.c darshan-logutils.h darshan-mdhim-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mdhim-log-format.h | uthash-1.9.2
+ $(CC) $(CFLAGS) -c $< -o $@
+darshan-mdhim-logutils.po: darshan-mdhim-logutils.c darshan-logutils.h darshan-mdhim-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mdhim-log-format.h | uthash-1.9.2
+ $(CC) $(CFLAGS_SHARED) -c $< -o $@
+
+
libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS)
ar rcs libdarshan-util.a $^
@@ -159,6 +200,7 @@ endif
install -m 644 $(srcdir)/darshan-lustre-logutils.h $(includedir)
install -m 644 $(srcdir)/darshan-stdio-logutils.h $(includedir)
install -m 644 $(srcdir)/darshan-dxt-logutils.h $(includedir)
+ install -m 644 $(srcdir)/darshan-mdhim-logutils.h $(includedir)
install -m 644 $(srcdir)/../darshan-null-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-posix-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-mpiio-log-format.h $(includedir)
@@ -168,6 +210,7 @@ endif
install -m 644 $(srcdir)/../darshan-lustre-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-stdio-log-format.h $(includedir)
install -m 644 $(srcdir)/../darshan-dxt-log-format.h $(includedir)
+ install -m 644 $(srcdir)/../darshan-mdhim-log-format.h $(includedir)
install -d $(includedir)/uthash-1.9.2
install -d $(includedir)/uthash-1.9.2/src
install -m 644 uthash-1.9.2/src/uthash.h $(includedir)/uthash-1.9.2/src/
=====================================
darshan-util/darshan-logutils.h
=====================================
--- a/darshan-util/darshan-logutils.h
+++ b/darshan-util/darshan-logutils.h
@@ -137,6 +137,7 @@ extern struct darshan_mod_logutil_funcs *mod_logutils[];
/* DXT */
#include "darshan-dxt-logutils.h"
+#include "darshan-mdhim-logutils.h"
darshan_fd darshan_log_open(const char *name);
darshan_fd darshan_log_create(const char *name, enum darshan_comp_type comp_type,
=====================================
darshan-util/darshan-mdhim-logutils.c
=====================================
--- /dev/null
+++ b/darshan-util/darshan-mdhim-logutils.c
@@ -0,0 +1,396 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#define _GNU_SOURCE
+#include "darshan-util-config.h"
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <errno.h>
+
+#include "darshan-logutils.h"
+
+/* integer counter name strings for the MDHIM module */
+#define X(a) #a,
+char *mdhim_counter_names[] = {
+ MDHIM_COUNTERS
+};
+
+/* floating point counter name strings for the MDHIM module */
+char *mdhim_f_counter_names[] = {
+ MDHIM_F_COUNTERS
+};
+#undef X
+
+/* prototypes for each of the MDHIM module's logutil functions */
+static int darshan_log_get_mdhim_record(darshan_fd fd, void** mdhim_buf_p);
+static int darshan_log_put_mdhim_record(darshan_fd fd, void* mdhim_buf);
+static void darshan_log_print_mdhim_record(void *file_rec,
+ char *file_name, char *mnt_pt, char *fs_type);
+static void darshan_log_print_mdhim_description(int ver);
+static void darshan_log_print_mdhim_record_diff(void *file_rec1, char *file_name1,
+ void *file_rec2, char *file_name2);
+static void darshan_log_agg_mdhim_records(void *rec, void *agg_rec, int init_flag);
+
+/* structure storing each function needed for implementing the darshan
+ * logutil interface. these functions are used for reading, writing, and
+ * printing module data in a consistent manner.
+ */
+struct darshan_mod_logutil_funcs mdhim_logutils =
+{
+ .log_get_record = &darshan_log_get_mdhim_record,
+ .log_put_record = &darshan_log_put_mdhim_record,
+ .log_print_record = &darshan_log_print_mdhim_record,
+ .log_print_description = &darshan_log_print_mdhim_description,
+ .log_print_diff = &darshan_log_print_mdhim_record_diff,
+ .log_agg_records = &darshan_log_agg_mdhim_records
+};
+
+/* retrieve a MDHIM record from log file descriptor 'fd', storing the
+ * data in the buffer address pointed to by 'mdhim_buf_p'. Return 1 on
+ * successful record read, 0 on no more data, and -1 on error.
+ */
+static int darshan_log_get_mdhim_record(darshan_fd fd, void** mdhim_buf_p)
+{
+ struct darshan_mdhim_record *rec =
+ *((struct darshan_mdhim_record **)mdhim_buf_p);
+ struct darshan_mdhim_record tmp_rec;
+ int i;
+ int ret;
+
+ if(fd->mod_map[DARSHAN_MDHIM_MOD].len == 0)
+ return(0);
+
+ /* read the fixed-sized portion of the MDHIM module record from the
+ * darshan log file */
+ ret = darshan_log_get_mod(fd, DARSHAN_MDHIM_MOD, &tmp_rec,
+ sizeof(struct darshan_mdhim_record));
+ if (ret < 0)
+ return (-1);
+ else if (ret < sizeof(struct darshan_mdhim_record))
+ return (0);
+ /* swap bytes if necessary */
+ if (fd->swap_flag)
+ {
+ /* reader-makes-right: don't look at a field until it has
+ * been swapped */
+ DARSHAN_BSWAP64(&tmp_rec.base_rec.id);
+ DARSHAN_BSWAP64(&tmp_rec.base_rec.rank);
+ for (i=0; i< MDHIM_NUM_INDICES; i++)
+ DARSHAN_BSWAP64(&tmp_rec.counters[i]);
+ for (i=0; i< MDHIM_F_NUM_INDICES; i++)
+ DARSHAN_BSWAP64(&tmp_rec.fcounters[i]);
+ DARSHAN_BSWAP64(&(tmp_rec.server_histogram[0]) );
+ }
+
+ if(*mdhim_buf_p == NULL)
+ {
+ rec = malloc(MDHIM_RECORD_SIZE(tmp_rec.counters[MDHIM_SERVERS]));
+ if (!rec)
+ return (-1);
+ }
+ memcpy(rec, &tmp_rec, sizeof(struct darshan_mdhim_record));
+
+ if (rec->counters[MDHIM_SERVERS] > 1) {
+ ret = darshan_log_get_mod(fd, DARSHAN_MDHIM_MOD,
+ &(rec->server_histogram[1]),
+ (rec->counters[MDHIM_SERVERS] - 1)*sizeof(int64_t));
+
+ if (ret < (rec->counters[MDHIM_SERVERS] -1)*sizeof(int64_t))
+ ret = -1;
+ else
+ {
+ ret = 1;
+ if (fd->swap_flag)
+ for(i=1; i< rec->counters[MDHIM_SERVERS]; i++)
+ DARSHAN_BSWAP64(&(rec->server_histogram[i]));
+ }
+ }
+ else
+ {
+ ret = 1;
+ }
+ if (*mdhim_buf_p == NULL)
+ {
+ if (ret == 1)
+ *mdhim_buf_p = rec;
+ else
+ free(rec);
+ }
+ return (ret);
+}
+
+/* write the MDHIM record stored in 'mdhim_buf' to log file descriptor 'fd'.
+ * Return 0 on success, -1 on failure
+ */
+static int darshan_log_put_mdhim_record(darshan_fd fd, void* mdhim_buf)
+{
+ struct darshan_mdhim_record *rec = (struct darshan_mdhim_record *)mdhim_buf;
+ int ret;
+
+ /* append MDHIM record to darshan log file */
+ ret = darshan_log_put_mod(fd, DARSHAN_MDHIM_MOD, rec,
+ sizeof(struct darshan_mdhim_record), DARSHAN_MDHIM_VER);
+ if(ret < 0)
+ return(-1);
+
+ return(0);
+}
+
+/* print all I/O data record statistics for the given MDHIM record */
+static void darshan_log_print_mdhim_record(void *file_rec, char *file_name,
+ char *mnt_pt, char *fs_type)
+{
+ int i;
+ struct darshan_mdhim_record *mdhim_rec =
+ (struct darshan_mdhim_record *)file_rec;
+
+ /* print each of the integer and floating point counters for the MDHIM module */
+ for(i=0; i<MDHIM_NUM_INDICES; i++)
+ {
+ /* macro defined in darshan-logutils.h */
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ mdhim_rec->base_rec.rank, mdhim_rec->base_rec.id,
+ mdhim_counter_names[i], mdhim_rec->counters[i],
+ file_name, mnt_pt, fs_type);
+ }
+
+ for(i=0; i<MDHIM_F_NUM_INDICES; i++)
+ {
+ /* macro defined in darshan-logutils.h */
+ DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ mdhim_rec->base_rec.rank, mdhim_rec->base_rec.id,
+ mdhim_f_counter_names[i], mdhim_rec->fcounters[i],
+ file_name, mnt_pt, fs_type);
+ }
+ for (i=0; i< mdhim_rec->counters[MDHIM_SERVERS]; i++)
+ {
+ char strbuf[25];
+ snprintf(strbuf, 25, "MDHIM_SERVER_%d", i);
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ mdhim_rec->base_rec.rank,
+ mdhim_rec->base_rec.id,
+ strbuf,
+ (int64_t)mdhim_rec->server_histogram[i],
+ file_name, mnt_pt, fs_type);
+ }
+ return;
+}
+
+/* print out a description of the MDHIM module record fields */
+static void darshan_log_print_mdhim_description(int ver)
+{
+ printf("\n# description of MDHIM counters:\n");
+ printf("# MDHIM_PUTS: number of 'mdhim_put' function calls.\n");
+ printf("# MDHIM_GETS: number of 'mdhim_get' function calls.\n");
+ printf("# MDHIM_SERVERS: how many mdhim servers \n");
+ printf("# MDHIM_F_PUT_TIMESTAMP: timestamp of the first call to function 'mdhim_put'.\n");
+ printf("# MDHIM_F_GET_TIMESTAMP: timestamp of the first call to function 'mdhim_get'.\n");
+ printf("# MDHIM_SERVER_N: how many operations sent to this server\n");
+
+ return;
+}
+
+/* print a diff of two MDHIM records (with the same record id) */
+static void darshan_log_print_mdhim_record_diff(void *file_rec1, char *file_name1,
+ void *file_rec2, char *file_name2)
+{
+ struct darshan_mdhim_record *file1 = (struct darshan_mdhim_record *)file_rec1;
+ struct darshan_mdhim_record *file2 = (struct darshan_mdhim_record *)file_rec2;
+ int i;
+
+ /* NOTE: we assume that both input records are the same module format version */
+
+ for(i=0; i<MDHIM_NUM_INDICES; i++)
+ {
+ if(!file2)
+ {
+ printf("- ");
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file1->base_rec.rank, file1->base_rec.id, mdhim_counter_names[i],
+ file1->counters[i], file_name1, "", "");
+
+ }
+ else if(!file1)
+ {
+ printf("+ ");
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file2->base_rec.rank, file2->base_rec.id, mdhim_counter_names[i],
+ file2->counters[i], file_name2, "", "");
+ }
+ else if(file1->counters[i] != file2->counters[i])
+ {
+ printf("- ");
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file1->base_rec.rank, file1->base_rec.id, mdhim_counter_names[i],
+ file1->counters[i], file_name1, "", "");
+ printf("+ ");
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file2->base_rec.rank, file2->base_rec.id, mdhim_counter_names[i],
+ file2->counters[i], file_name2, "", "");
+ }
+ }
+
+ for(i=0; i<MDHIM_F_NUM_INDICES; i++)
+ {
+ if(!file2)
+ {
+ printf("- ");
+ DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file1->base_rec.rank, file1->base_rec.id, mdhim_f_counter_names[i],
+ file1->fcounters[i], file_name1, "", "");
+
+ }
+ else if(!file1)
+ {
+ printf("+ ");
+ DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file2->base_rec.rank, file2->base_rec.id, mdhim_f_counter_names[i],
+ file2->fcounters[i], file_name2, "", "");
+ }
+ else if(file1->fcounters[i] != file2->fcounters[i])
+ {
+ printf("- ");
+ DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file1->base_rec.rank, file1->base_rec.id, mdhim_f_counter_names[i],
+ file1->fcounters[i], file_name1, "", "");
+ printf("+ ");
+ DARSHAN_F_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file2->base_rec.rank, file2->base_rec.id, mdhim_f_counter_names[i],
+ file2->fcounters[i], file_name2, "", "");
+ }
+ }
+ i=0;
+ while (1)
+ {
+ char strbuf[25];
+ snprintf(strbuf, 25, "MDHIM_SERVER_%d", i);
+ if (!file2 || (i >= file2->counters[MDHIM_SERVERS]))
+ {
+ printf("- ");
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file1->base_rec.rank,
+ file1->base_rec.id,
+ strbuf,
+ (int64_t)file1->server_histogram[i],
+ file_name1, "", "");
+ }
+ else if (!file1 || (i >= file1->counters[MDHIM_SERVERS]))
+ {
+ printf("+ ");
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file2->base_rec.rank,
+ file2->base_rec.id,
+ strbuf,
+ (int64_t)file2->server_histogram[i],
+ file_name2, "", "");
+ }
+ else if (file1->server_histogram[i] != file2->server_histogram[i])
+ {
+ printf("- ");
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file1->base_rec.rank,
+ file1->base_rec.id,
+ strbuf,
+ (int64_t)file1->server_histogram[i],
+ file_name1, "", "");
+ printf("+ ");
+ DARSHAN_COUNTER_PRINT(darshan_module_names[DARSHAN_MDHIM_MOD],
+ file2->base_rec.rank,
+ file2->base_rec.id,
+ strbuf,
+ (int64_t)file2->server_histogram[i],
+ file_name2, "", "");
+ }
+ i++;
+ if ( (!file1 || i >= file1->counters[MDHIM_SERVERS]) &&
+ (!file2 || i >= file2->counters[MDHIM_SERVERS] ) )
+ break;
+ }
+ return;
+}
+
+/* aggregate the input MDHIM record 'rec' into the output record 'agg_rec' */
+static void darshan_log_agg_mdhim_records(void *rec, void *agg_rec, int init_flag)
+{
+ struct darshan_mdhim_record *mdhim_rec = (struct darshan_mdhim_record *)rec;
+ struct darshan_mdhim_record *agg_mdhim_rec = (struct darshan_mdhim_record *)agg_rec;
+ int i;
+
+ for(i = 0; i < MDHIM_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case MDHIM_PUTS:
+ /* sum */
+ agg_mdhim_rec->counters[i] += mdhim_rec->counters[i];
+ break;
+ case MDHIM_GETS:
+ /* sum */
+ agg_mdhim_rec->counters[i] += mdhim_rec->counters[i];
+ break;
+ case MDHIM_SERVERS:
+ /* all clients should have the same value for this, hence
+ * assignment instead of aggregating */
+ agg_mdhim_rec->counters[i] = mdhim_rec->counters[i];
+ default:
+ /* if we don't know how to aggregate this counter, just set to -1 */
+ agg_mdhim_rec->counters[i] = -1;
+ break;
+ }
+ }
+
+ for(i = 0; i < MDHIM_F_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case MDHIM_F_PUT_TIMESTAMP:
+ /* min non-zero */
+ if((mdhim_rec->fcounters[i] > 0) &&
+ ((agg_mdhim_rec->fcounters[i] == 0) ||
+ (mdhim_rec->fcounters[i] < agg_mdhim_rec->fcounters[i])))
+ {
+ agg_mdhim_rec->fcounters[i] = mdhim_rec->fcounters[i];
+ }
+ break;
+ case MDHIM_F_GET_TIMESTAMP:
+ /* min non-zero */
+ if((mdhim_rec->fcounters[i] > 0) &&
+ ((agg_mdhim_rec->fcounters[i] == 0) ||
+ (mdhim_rec->fcounters[i] < agg_mdhim_rec->fcounters[i])))
+ {
+ agg_mdhim_rec->fcounters[i] = mdhim_rec->fcounters[i];
+ }
+ break;
+
+ default:
+ /* if we don't know how to aggregate this counter, just set to -1 */
+ agg_mdhim_rec->fcounters[i] = -1;
+ break;
+ }
+ }
+ for (i=0; i< mdhim_rec->counters[MDHIM_SERVERS]; i++)
+ {
+ agg_mdhim_rec->server_histogram[i] += mdhim_rec->server_histogram[i];
+ }
+
+ return;
+}
+
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * End:
+ *
+ * vim: ts=8 sts=4 sw=4 expandtab
+ */
=====================================
darshan-util/darshan-mdhim-logutils.h
=====================================
--- /dev/null
+++ b/darshan-util/darshan-mdhim-logutils.h
@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_MDHIM_LOG_UTILS_H
+#define __DARSHAN_MDHIM_LOG_UTILS_H
+
+/* declare MDHIM module counter name strings and logutil definition as
+ * extern variables so they can be used in other utilities
+ */
+extern char *mdhim_counter_names[];
+extern char *mdhim_f_counter_names[];
+
+extern struct darshan_mod_logutil_funcs mdhim_logutils;
+
+#endif
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/2c40940ff465c5d0822cb099e8273102d30e2748...1a362ac057543f3a74a3dff23dcfc2a1cdea2ab3
---
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/2c40940ff465c5d0822cb099e8273102d30e2748...1a362ac057543f3a74a3dff23dcfc2a1cdea2ab3
You're receiving this email because of your account on xgitlab.cels.anl.gov.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20180629/4a098805/attachment-0001.html>
More information about the Darshan-commits
mailing list