[Darshan-commits] [Git][darshan/darshan][master] 2 commits: sync null module with new changes
Shane Snyder
xgitlab at cels.anl.gov
Tue Sep 20 16:47:09 CDT 2016
Shane Snyder pushed to branch master at darshan / darshan
Commits:
107dcfc1 by Shane Snyder at 2016-09-20T14:27:06-05:00
sync null module with new changes
this includes defining an example logutils function for
aggregating null records into a single shared null record (needed
for darshan-merge utility)
- - - - -
c38ee987 by Shane Snyder at 2016-09-20T16:46:26-05:00
add aggregation function to stdio module logutils
- - - - -
4 changed files:
- darshan-null-log-format.h
- darshan-runtime/lib/darshan-null.c
- darshan-util/darshan-null-logutils.c
- darshan-util/darshan-stdio-logutils.c
Changes:
=====================================
darshan-null-log-format.h
=====================================
--- a/darshan-null-log-format.h
+++ b/darshan-null-log-format.h
@@ -11,18 +11,18 @@
#define DARSHAN_NULL_VER 1
#define NULL_COUNTERS \
- /* count of number of 'bar' function calls */\
- X(NULL_BARS) \
- /* arbitrary data value set by last call to 'bar' */\
- X(NULL_BAR_DAT) \
+ /* number of 'foo' function calls */\
+ X(NULL_FOOS) \
+ /* maximum data value set by calls to 'foo' */\
+ X(NULL_FOO_MAX_DAT) \
/* end of counters */\
X(NULL_NUM_INDICES)
#define NULL_F_COUNTERS \
- /* timestamp of the first call to function 'bar' */\
- X(NULL_F_BAR_TIMESTAMP) \
- /* timer indicating duration of last call to 'bar' */\
- X(NULL_F_BAR_DURATION) \
+ /* timestamp of the first call to function 'foo' */\
+ X(NULL_F_FOO_TIMESTAMP) \
+ /* timer indicating duration of call to 'foo' with max NULL_FOO_MAX_DAT value */\
+ X(NULL_F_FOO_MAX_DURATION) \
/* end of counters */\
X(NULL_F_NUM_INDICES)
=====================================
darshan-runtime/lib/darshan-null.c
=====================================
--- a/darshan-runtime/lib/darshan-null.c
+++ b/darshan-runtime/lib/darshan-null.c
@@ -32,7 +32,7 @@
* declarations for wrapped funcions, regardless of whether Darshan is used with
* statically or dynamically linked executables.
*/
-DARSHAN_FORWARD_DECL(foo, int, (const char *name, int arg1, int arg2));
+DARSHAN_FORWARD_DECL(foo, int, (const char *name, int arg1));
/* The null_record_ref structure maintains necessary runtime metadata
* for the NULL module record (darshan_null_record structure, defined in
@@ -132,9 +132,6 @@ static int my_rank = -1;
} while(0)
/* macro for instrumenting the "NULL" module's foo function */
-/* NOTE: this macro makes use of the DARSHAN_COUNTER_* macros defined
- * and documented in darshan.h.
- */
#define NULL_RECORD_FOO(__ret, __name, __dat, __tm1, __tm2) do{ \
darshan_record_id rec_id; \
struct null_record_ref *rec_ref; \
@@ -149,14 +146,17 @@ static int my_rank = -1;
if(!rec_ref) null_track_new_record(rec_id, __name); \
/* if we still don't have a valid reference, back out */ \
if(!rec_ref) break; \
- /* increment counter indicating number of calls to 'bar' */ \
- rec_ref->record_p->counters[NULL_BARS] += 1; \
- /* store data value for most recent call to 'bar' */ \
- rec_ref->record_p->counters[NULL_BAR_DAT] = __dat; \
- /* store timestamp of most recent call to 'bar' */ \
- rec_ref->record_p->fcounters[NULL_F_BAR_TIMESTAMP] = __tm1; \
- /* store duration of most recent call to 'bar' */ \
- rec_ref->record_p->fcounters[NULL_F_BAR_DURATION] = __elapsed; \
+ /* increment counter indicating number of calls to 'foo' */ \
+ rec_ref->record_p->counters[NULL_FOOS] += 1; \
+ /* store max data value for calls to 'foo', and corresponding time duration */ \
+ if(rec_ref->record_p->counters[NULL_FOO_MAX_DAT] < __dat) { \
+ rec_ref->record_p->counters[NULL_FOO_MAX_DAT] = __dat; \
+ rec_ref->record_p->fcounters[NULL_F_FOO_MAX_DURATION] = __elapsed; \
+ } \
+ /* store timestamp of first call to 'foo' */ \
+ if(rec_ref->record_p->fcounters[NULL_F_FOO_TIMESTAMP] == 0 || \
+ rec_ref->record_p->fcounters[NULL_F_FOO_TIMESTAMP] > __tm1) \
+ rec_ref->record_p->fcounters[NULL_F_FOO_TIMESTAMP] = __tm1; \
} while(0)
/**********************************************************
@@ -166,7 +166,7 @@ static int my_rank = -1;
/* The DARSHAN_DECL macro provides the appropriate wrapper function names,
* depending on whether the Darshan library is statically or dynamically linked.
*/
-int DARSHAN_DECL(foo)(const char* name, int arg1, int arg2)
+int DARSHAN_DECL(foo)(const char* name, int arg1)
{
ssize_t ret;
double tm1, tm2;
@@ -182,12 +182,12 @@ int DARSHAN_DECL(foo)(const char* name, int arg1, int arg2)
* given wrapper function. Timers are used to record the duration of this
* operation. */
tm1 = darshan_core_wtime();
- ret = __real_foo(name, arg1, arg2);
+ ret = __real_foo(name, arg1);
tm2 = darshan_core_wtime();
NULL_PRE_RECORD();
/* Call macro for instrumenting data for foo function calls. */
- NULL_RECORD_FOO(ret, name, arg1+arg2, tm1, tm2);
+ NULL_RECORD_FOO(ret, name, arg1, tm1, tm2);
NULL_POST_RECORD();
return(ret);
=====================================
darshan-util/darshan-null-logutils.c
=====================================
--- a/darshan-util/darshan-null-logutils.c
+++ b/darshan-util/darshan-null-logutils.c
@@ -56,9 +56,8 @@ struct darshan_mod_logutil_funcs null_logutils =
};
/* retrieve a NULL record from log file descriptor 'fd', storing the
- * buffer in 'null_buf' and the corresponding Darshan record id in
- * 'rec_id'. Return 1 on successful record read, 0 on no more data,
- * and -1 on error.
+ * data in the buffer address pointed to by 'null_buf_p'. Return 1 on
+ * successful record read, 0 on no more data, and -1 on error.
*/
static int darshan_log_get_null_record(darshan_fd fd, void** null_buf_p)
{
@@ -109,8 +108,8 @@ static int darshan_log_get_null_record(darshan_fd fd, void** null_buf_p)
}
}
-/* write the NULL record stored in 'null_buf' to log file descriptor 'fd'.
- * Return 0 on success, -1 on failure
+/* write the NULL record stored in 'null_buf' (with version number 'ver')
+ * to log file descriptor 'fd'. Return 0 on success, -1 on failure
*/
static int darshan_log_put_null_record(darshan_fd fd, void* null_buf, int ver)
{
@@ -160,14 +159,15 @@ static void darshan_log_print_null_record(void *file_rec, char *file_name,
static void darshan_log_print_null_description()
{
printf("\n# description of NULL counters:\n");
- printf("# NULL_BARS: number of 'bar' function calls.\n");
- printf("# NULL_BAR_DAT: value set by last call to function 'bar'.\n");
- printf("# NULL_F_BAR_TIMESTAMP: timestamp of the first call to function 'bar'.\n");
- printf("# NULL_F_BAR_DURATION: duration of the last call to function 'bar'.\n");
+ printf("# NULL_FOOS: number of 'foo' function calls.\n");
+ printf("# NULL_FOO_MAX_DAT: maximum data value set by calls to 'foo'.\n");
+ printf("# NULL_F_FOO_TIMESTAMP: timestamp of the first call to function 'foo'.\n");
+ printf("# NULL_F_FOO_MAX_DURATION: timer indicating duration of call to 'foo' with max NULL_FOO_MAX_DAT value.\n");
return;
}
+/* print a diff of two NULL records (with the same record id) */
static void darshan_log_print_null_record_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2)
{
@@ -240,8 +240,57 @@ static void darshan_log_print_null_record_diff(void *file_rec1, char *file_name1
return;
}
+/* aggregate the input NULL record 'rec' into the output record 'agg_rec' */
static void darshan_log_agg_null_records(void *rec, void *agg_rec, int init_flag)
{
+ struct darshan_null_record *null_rec = (struct darshan_null_record *)rec;
+ struct darshan_null_record *agg_null_rec = (struct darshan_null_record *)agg_rec;
+ int i;
+
+ for(i = 0; i < NULL_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case NULL_FOOS:
+ /* sum */
+ agg_null_rec->counters[i] += null_rec->counters[i];
+ break;
+ case NULL_FOO_MAX_DAT:
+ /* max */
+ if(null_rec->counters[i] > agg_null_rec->counters[i])
+ {
+ agg_null_rec->counters[i] = null_rec->counters[i];
+ agg_null_rec->fcounters[NULL_F_FOO_MAX_DURATION] =
+ null_rec->fcounters[NULL_F_FOO_MAX_DURATION];
+ }
+ break;
+ default:
+ /* if we don't know how to aggregate this counter, just set to -1 */
+ agg_null_rec->counters[i] = -1;
+ break;
+ }
+ }
+
+ for(i = 0; i < NULL_F_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case NULL_F_FOO_TIMESTAMP:
+ /* min non-zero */
+ if((null_rec->fcounters[i] > 0) &&
+ ((agg_null_rec->fcounters[i] == 0) ||
+ (null_rec->fcounters[i] < agg_null_rec->fcounters[i])))
+ {
+ agg_null_rec->fcounters[i] = null_rec->fcounters[i];
+ }
+ break;
+ default:
+ /* if we don't know how to aggregate this counter, just set to -1 */
+ agg_null_rec->fcounters[i] = -1;
+ break;
+ }
+ }
+
return;
}
=====================================
darshan-util/darshan-stdio-logutils.c
=====================================
--- a/darshan-util/darshan-stdio-logutils.c
+++ b/darshan-util/darshan-stdio-logutils.c
@@ -39,6 +39,7 @@ static void darshan_log_print_stdio_record(void *file_rec,
static void darshan_log_print_stdio_description(void);
static void darshan_log_print_stdio_record_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
+static void darshan_log_agg_stdio_records(void *rec, void *agg_rec, int init_flag);
/* structure storing each function needed for implementing the darshan
* logutil interface. these functions are used for reading, writing, and
@@ -50,12 +51,13 @@ struct darshan_mod_logutil_funcs stdio_logutils =
.log_put_record = &darshan_log_put_stdio_record,
.log_print_record = &darshan_log_print_stdio_record,
.log_print_description = &darshan_log_print_stdio_description,
- .log_print_diff = &darshan_log_print_stdio_record_diff
+ .log_print_diff = &darshan_log_print_stdio_record_diff,
+ .log_agg_records = &darshan_log_agg_stdio_records
};
/* retrieve a STDIO record from log file descriptor 'fd', storing the
- * buffer in 'stdio_buf'. Return 1 on successful record read, 0 on no
- * more data, and -1 on error.
+ * data in the buffer address pointed to by 'stdio_buf_p'. Return 1 on
+ * successful record read, 0 on no more data, and -1 on error.
*/
static int darshan_log_get_stdio_record(darshan_fd fd, void** stdio_buf_p)
{
@@ -243,6 +245,178 @@ static void darshan_log_print_stdio_record_diff(void *file_rec1, char *file_name
return;
}
+/* simple helper struct for determining time & byte variances */
+struct var_t
+{
+ double n;
+ double M;
+ double S;
+};
+
+static void darshan_log_agg_stdio_records(void *rec, void *agg_rec, int init_flag)
+{
+ struct darshan_stdio_file *stdio_rec = (struct darshan_stdio_file *)rec;
+ struct darshan_stdio_file *agg_stdio_rec = (struct darshan_stdio_file *)agg_rec;
+ int i;
+ double old_M;
+ double stdio_time = stdio_rec->fcounters[STDIO_F_READ_TIME] +
+ stdio_rec->fcounters[STDIO_F_WRITE_TIME] +
+ stdio_rec->fcounters[STDIO_F_META_TIME];
+ double stdio_bytes = (double)stdio_rec->counters[STDIO_BYTES_READ] +
+ stdio_rec->counters[STDIO_BYTES_WRITTEN];
+ struct var_t *var_time_p = (struct var_t *)
+ ((char *)rec + sizeof(struct darshan_stdio_file));
+ struct var_t *var_bytes_p = (struct var_t *)
+ ((char *)var_time_p + sizeof(struct var_t));
+
+ for(i = 0; i < STDIO_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case STDIO_OPENS:
+ case STDIO_READS:
+ case STDIO_WRITES:
+ case STDIO_SEEKS:
+ case STDIO_FLUSHES:
+ case STDIO_BYTES_WRITTEN:
+ case STDIO_BYTES_READ:
+ /* sum */
+ agg_stdio_rec->counters[i] += stdio_rec->counters[i];
+ break;
+ case STDIO_MAX_BYTE_READ:
+ case STDIO_MAX_BYTE_WRITTEN:
+ /* max */
+ if(stdio_rec->counters[i] > agg_stdio_rec->counters[i])
+ {
+ agg_stdio_rec->counters[i] = stdio_rec->counters[i];
+ }
+ break;
+ case STDIO_FASTEST_RANK:
+ case STDIO_FASTEST_RANK_BYTES:
+ case STDIO_SLOWEST_RANK:
+ case STDIO_SLOWEST_RANK_BYTES:
+ /* these are set with the FP counters */
+ break;
+ default:
+ agg_stdio_rec->counters[i] = -1;
+ break;
+ }
+ }
+
+ for(i = 0; i < STDIO_F_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case STDIO_F_META_TIME:
+ case STDIO_F_WRITE_TIME:
+ case STDIO_F_READ_TIME:
+ /* sum */
+ agg_stdio_rec->fcounters[i] += stdio_rec->fcounters[i];
+ break;
+ case STDIO_F_OPEN_START_TIMESTAMP:
+ case STDIO_F_CLOSE_START_TIMESTAMP:
+ case STDIO_F_WRITE_START_TIMESTAMP:
+ case STDIO_F_READ_START_TIMESTAMP:
+ /* minimum non-zero */
+ if((stdio_rec->fcounters[i] > 0) &&
+ ((agg_stdio_rec->fcounters[i] == 0) ||
+ (stdio_rec->fcounters[i] < agg_stdio_rec->fcounters[i])))
+ {
+ agg_stdio_rec->fcounters[i] = stdio_rec->fcounters[i];
+ }
+ break;
+ case STDIO_F_OPEN_END_TIMESTAMP:
+ case STDIO_F_CLOSE_END_TIMESTAMP:
+ case STDIO_F_WRITE_END_TIMESTAMP:
+ case STDIO_F_READ_END_TIMESTAMP:
+ /* maximum */
+ if(stdio_rec->fcounters[i] > agg_stdio_rec->fcounters[i])
+ {
+ agg_stdio_rec->fcounters[i] = stdio_rec->fcounters[i];
+ }
+ break;
+ case STDIO_F_FASTEST_RANK_TIME:
+ if(init_flag)
+ {
+ /* set fastest rank counters according to root rank. these counters
+ * will be determined as the aggregation progresses.
+ */
+ agg_stdio_rec->counters[STDIO_FASTEST_RANK] = stdio_rec->base_rec.rank;
+ agg_stdio_rec->counters[STDIO_FASTEST_RANK_BYTES] = stdio_bytes;
+ agg_stdio_rec->fcounters[STDIO_F_FASTEST_RANK_TIME] = stdio_time;
+ }
+
+ if(stdio_time < agg_stdio_rec->fcounters[STDIO_F_FASTEST_RANK_TIME])
+ {
+ agg_stdio_rec->counters[STDIO_FASTEST_RANK] = stdio_rec->base_rec.rank;
+ agg_stdio_rec->counters[STDIO_FASTEST_RANK_BYTES] = stdio_bytes;
+ agg_stdio_rec->fcounters[STDIO_F_FASTEST_RANK_TIME] = stdio_time;
+ }
+ break;
+ case STDIO_F_SLOWEST_RANK_TIME:
+ if(init_flag)
+ {
+ /* set slowest rank counters according to root rank. these counters
+ * will be determined as the aggregation progresses.
+ */
+ agg_stdio_rec->counters[STDIO_SLOWEST_RANK] = stdio_rec->base_rec.rank;
+ agg_stdio_rec->counters[STDIO_SLOWEST_RANK_BYTES] = stdio_bytes;
+ agg_stdio_rec->fcounters[STDIO_F_SLOWEST_RANK_TIME] = stdio_time;
+ }
+
+ if(stdio_time > agg_stdio_rec->fcounters[STDIO_F_SLOWEST_RANK_TIME])
+ {
+ agg_stdio_rec->counters[STDIO_SLOWEST_RANK] = stdio_rec->base_rec.rank;
+ agg_stdio_rec->counters[STDIO_SLOWEST_RANK_BYTES] = stdio_bytes;
+ agg_stdio_rec->fcounters[STDIO_F_SLOWEST_RANK_TIME] = stdio_time;
+ }
+ break;
+ case STDIO_F_VARIANCE_RANK_TIME:
+ if(init_flag)
+ {
+ var_time_p->n = 1;
+ var_time_p->M = stdio_time;
+ var_time_p->S = 0;
+ }
+ else
+ {
+ old_M = var_time_p->M;
+
+ var_time_p->n++;
+ var_time_p->M += (stdio_time - var_time_p->M) / var_time_p->n;
+ var_time_p->S += (stdio_time - var_time_p->M) * (stdio_time - old_M);
+
+ agg_stdio_rec->fcounters[STDIO_F_VARIANCE_RANK_TIME] =
+ var_time_p->S / var_time_p->n;
+ }
+ break;
+ case STDIO_F_VARIANCE_RANK_BYTES:
+ if(init_flag)
+ {
+ var_bytes_p->n = 1;
+ var_bytes_p->M = stdio_bytes;
+ var_bytes_p->S = 0;
+ }
+ else
+ {
+ old_M = var_bytes_p->M;
+
+ var_bytes_p->n++;
+ var_bytes_p->M += (stdio_bytes - var_bytes_p->M) / var_bytes_p->n;
+ var_bytes_p->S += (stdio_bytes - var_bytes_p->M) * (stdio_bytes - old_M);
+
+ agg_stdio_rec->fcounters[STDIO_F_VARIANCE_RANK_BYTES] =
+ var_bytes_p->S / var_bytes_p->n;
+ }
+ break;
+ default:
+ agg_stdio_rec->fcounters[i] = -1;
+ break;
+ }
+ }
+
+ return;
+}
/*
* Local variables:
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/c82668224b7ec433158065ef719d3454e39d0611...c38ee9875e25ac8059a9b2fc0d80cad939e162dd
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20160920/117d4d83/attachment-0001.html>
More information about the Darshan-commits
mailing list