<html lang='en'>
<head>
<meta content='text/html; charset=utf-8' http-equiv='Content-Type'>
<title>
GitLab
</title>
</meta>
</head>
<style>
img {
max-width: 100%;
height: auto;
}
p.details {
font-style:italic;
color:#777
}
.footer p {
font-size:small;
color:#777
}
pre.commit-message {
white-space: pre-wrap;
}
.file-stats a {
text-decoration: none;
}
.file-stats .new-file {
color: #090;
}
.file-stats .deleted-file {
color: #B00;
}
</style>
<body>
<div class='content'>
<h3>
Shane Snyder pushed to branch mmap-dev
at <a href="https://xgitlab.cels.anl.gov/darshan/darshan">darshan / darshan</a>
</h3>
<h4>
Commits:
</h4>
<ul>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/f26baefc40a3506a8bf1cfb5e645d9deddf75db4">f26baefc</a></strong>
<div>
<span>by Shane Snyder</span>
<i>at 2016-02-26T12:34:02-06:00</i>
</div>
<pre class='commit-message'>allow job end time to be set by darshan-merge</pre>
</li>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/d4a3d112be89104319ae949516d4c2e72eace870">d4a3d112</a></strong>
<div>
<span>by Shane Snyder</span>
<i>at 2016-02-29T17:22:57-06:00</i>
</div>
<pre class='commit-message'>add variance counter reduction logic to logutils</pre>
</li>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/04063da70950ee779371188c7937956eb89b4273">04063da7</a></strong>
<div>
<span>by Shane Snyder</span>
<i>at 2016-03-01T17:43:38-06:00</i>
</div>
<pre class='commit-message'>common access counters are now sorted in logs
common value counters are sorted first by decreasing counts,
and then by decreasing size.</pre>
</li>
</ul>
<h4>5 changed files:</h4>
<ul>
<li class='file-stats'>
<a href='#diff-0'>
darshan-runtime/darshan-common.h
</a>
</li>
<li class='file-stats'>
<a href='#diff-1'>
darshan-runtime/share/darshan-mmap-epilog.sh.in
</a>
</li>
<li class='file-stats'>
<a href='#diff-2'>
darshan-util/darshan-merge.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-3'>
darshan-util/darshan-mpiio-logutils.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-4'>
darshan-util/darshan-posix-logutils.c
</a>
</li>
</ul>
<h4>Changes:</h4>
<li id='diff-0'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-0'>
<strong>
darshan-runtime/darshan-common.h
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/darshan-common.h
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/darshan-common.h
</span><span style="color: #aaaaaa">@@ -66,7 +66,10 @@
</span> } while(0)
/* potentially set or increment a common value counter, depending on the __count
<span style="color: #000000;background-color: #ffdddd">- * for the given __value
</span><span style="color: #000000;background-color: #ddffdd">+ * for the given __value. This macro ensures common values are stored first in
+ * decreasing order of their total count, and second by decreasing order of
+ * their value.
+
</span> *
* NOTE: This macro is hardcoded to expect that Darshan will only track the 4
* most common (i.e., frequently occuring) values. __val_p is a pointer to the
<span style="color: #aaaaaa">@@ -78,32 +81,50 @@
</span> */
#define DARSHAN_COMMON_VAL_COUNTER_INC(__val_p, __cnt_p, __value, __count, __online_flag) do {\
int i; \
<span style="color: #000000;background-color: #ffdddd">- int set = 0; \
- int64_t min = *(__cnt_p); \
- int min_index = 0; \
- int inc_count; \
</span><span style="color: #000000;background-color: #ddffdd">+ int inc_count, total_count; \
+ int64_t tmp_val[4] = {0}; \
+ int64_t tmp_cnt[4] = {0}; \
+ int tmp_ndx = 0; \
</span> if(__value == 0) break; \
if(__online_flag) \
inc_count = 1; \
else \
inc_count = __count; \
for(i=0; i<4; i++) { \
<span style="color: #000000;background-color: #ffdddd">- /* increment bucket if already exists */ \
</span> if(*(__val_p + i) == __value) { \
<span style="color: #000000;background-color: #ffdddd">- *(__cnt_p + i) += inc_count; \
- set = 1; \
</span><span style="color: #000000;background-color: #ddffdd">+ total_count = *(__cnt_p + i) + inc_count; \
</span> break; \
} \
<span style="color: #000000;background-color: #ffdddd">- /* otherwise find the least frequently used bucket */ \
- else if(*(__cnt_p + i) < min) { \
- min = *(__cnt_p + i); \
- min_index = i; \
</span><span style="color: #000000;background-color: #ddffdd">+ } \
+ if(i == 4) total_count = __count; \
+ /* first, copy over any counters that should be sorted above this one \
+ * (counters with higher counts or equal counts and larger values) \
+ */ \
+ for(i=0;i < 4; i++) { \
+ if((*(__cnt_p + i) > total_count) || \
+ ((*(__cnt_p + i) == total_count) && (*(__val_p + i) > __value))) { \
+ tmp_val[tmp_ndx] = *(__val_p + i); \
+ tmp_cnt[tmp_ndx] = *(__cnt_p + i); \
+ tmp_ndx++; \
</span> } \
<span style="color: #000000;background-color: #ddffdd">+ else break; \
</span> } \
<span style="color: #000000;background-color: #ffdddd">- if(!set && (__count > min)) { \
- *(__cnt_p + min_index) = __count; \
- *(__val_p + min_index) = __value; \
</span><span style="color: #000000;background-color: #ddffdd">+ if(tmp_ndx == 4) break; /* all done, updated counter is not added */ \
+ /* next, add the updated counter */ \
+ tmp_val[tmp_ndx] = __value; \
+ tmp_cnt[tmp_ndx] = total_count; \
+ tmp_ndx++; \
+ /* last, copy over any remaining counters to make sure we have 4 sets total */ \
+ while(tmp_ndx != 4) { \
+ if(*(__val_p + i) != __value) { \
+ tmp_val[tmp_ndx] = *(__val_p + i); \
+ tmp_cnt[tmp_ndx] = *(__cnt_p + i); \
+ tmp_ndx++; \
+ } \
+ i++; \
</span> } \
<span style="color: #000000;background-color: #ddffdd">+ memcpy(__val_p, tmp_val, 4*sizeof(int64_t)); \
+ memcpy(__cnt_p, tmp_cnt, 4*sizeof(int64_t)); \
</span> } while(0)
/* maximum number of common values that darshan will track per file at
</code></pre>
<br>
</li>
<li id='diff-1'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-1'>
<strong>
darshan-runtime/share/darshan-mmap-epilog.sh.in
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/share/darshan-mmap-epilog.sh.in
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/share/darshan-mmap-epilog.sh.in
</span><span style="color: #aaaaaa">@@ -11,6 +11,8 @@ DARSHAN_INSTALL_DIR=@prefix@
</span> # use the log dir specified at configure time
DARSHAN_LOG_DIR=@__DARSHAN_LOG_PATH@
<span style="color: #000000;background-color: #ddffdd">+JOB_END=$(date +"%s")
+
</span> # use the default mmap log directory (/tmp), unless the
# env variable is set to something
if [ -z "$DARSHAN_MMAP_LOGPATH" ]; then
<span style="color: #aaaaaa">@@ -53,7 +55,7 @@ if [ $SLURM_NNODES -gt 1 ]; then
</span> mkdir -p $NODE_LOG_DIR
# construct the per-node log file and store in the output directory
<span style="color: #000000;background-color: #ffdddd">- $DARSHAN_INSTALL_DIR/bin/darshan-merge \
</span><span style="color: #000000;background-color: #ddffdd">+ $DARSHAN_INSTALL_DIR/bin/darshan-merge --job-end-time $JOB_END \
</span> --output ${NODE_LOG_DIR}/${LOG_NAME_PRE}_${NODE_NAME}.darshan \
$DARSHAN_MMAP_LOG_GLOB
else
<span style="color: #aaaaaa">@@ -61,7 +63,7 @@ else
</span>
# single node, just create the final output darshan log
LOG_WRITE_START=$(date +%s)
<span style="color: #000000;background-color: #ffdddd">- $DARSHAN_INSTALL_DIR/bin/darshan-merge
</span><span style="color: #000000;background-color: #ddffdd">+ $DARSHAN_INSTALL_DIR/bin/darshan-merge --job-end-time $JOB_END \
</span> --shared-redux --output ${OUTPUT_LOG_DIR}/${TMP_LOG} \
$DARSHAN_MMAP_LOG_GLOB
LOG_WRITE_END=$(date +%s)
<span style="color: #aaaaaa">@@ -72,4 +74,6 @@ else
</span> mv ${OUTPUT_LOG_DIR}/${TMP_LOG} ${OUTPUT_LOG_DIR}/${FINAL_LOG}
fi
<span style="color: #000000;background-color: #ddffdd">+rm -f $DARSHAN_MMAP_LOG_GLOB
+
</span> exit 0
</code></pre>
<br>
</li>
<li id='diff-2'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-2'>
<strong>
darshan-util/darshan-merge.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-util/darshan-merge.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-util/darshan-merge.c
</span><span style="color: #aaaaaa">@@ -11,8 +11,6 @@
</span>
#define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
<span style="color: #000000;background-color: #ffdddd">-/* TODO: set job end timestamp? */
-
</span> struct darshan_shared_record_ref
{
darshan_record_id id;
<span style="color: #aaaaaa">@@ -28,23 +26,27 @@ void usage(char *exename)
</span> fprintf(stderr, "Options:\n");
fprintf(stderr, "\t--output\t(REQUIRED) Full path of the output darshan log file.\n");
fprintf(stderr, "\t--shared-redux\tReduce globally shared records into a single record.\n");
<span style="color: #000000;background-color: #ddffdd">+ fprintf(stderr, "\t--job-end-time\tSet the output log's job end time (requires argument of seconds since Epoch).\n");
</span>
exit(1);
}
void parse_args(int argc, char **argv, char ***infile_list, int *n_files,
<span style="color: #000000;background-color: #ffdddd">- char **outlog_path, int *shared_redux)
</span><span style="color: #000000;background-color: #ddffdd">+ char **outlog_path, int *shared_redux, int64_t *job_end_time)
</span> {
int index;
<span style="color: #000000;background-color: #ddffdd">+ char *check;
</span> static struct option long_opts[] =
{
<span style="color: #000000;background-color: #ffdddd">- {"shared-redux", no_argument, NULL, 's'},
</span> {"output", required_argument, NULL, 'o'},
<span style="color: #000000;background-color: #ddffdd">+ {"shared-redux", no_argument, NULL, 's'},
+ {"job-end-time", required_argument, NULL, 'e'},
</span> {0, 0, 0, 0}
};
*shared_redux = 0;
*outlog_path = NULL;
<span style="color: #000000;background-color: #ddffdd">+ *job_end_time = 0;
</span>
while(1)
{
<span style="color: #aaaaaa">@@ -60,6 +62,14 @@ void parse_args(int argc, char **argv, char ***infile_list, int *n_files,
</span> case 'o':
*outlog_path = optarg;
break;
<span style="color: #000000;background-color: #ddffdd">+ case 'e':
+ *job_end_time = strtol(optarg, &check, 10);
+ if(optarg == check)
+ {
+ fprintf(stderr, "Error: unable to parse job end time value.\n");
+ exit(1);
+ }
+ break;
</span> case '?':
default:
usage(argv[0]);
<span style="color: #aaaaaa">@@ -85,7 +95,7 @@ int build_mod_shared_rec_hash(char **infile_list, int n_infiles,
</span> darshan_fd in_fd;
struct darshan_base_record *base_rec;
struct darshan_shared_record_ref *ref, *tmp;
<span style="color: #000000;background-color: #ffdddd">- int init = 0;
</span><span style="color: #000000;background-color: #ddffdd">+ int init_rank = -1;
</span> int ret;
int i;
<span style="color: #aaaaaa">@@ -104,9 +114,11 @@ int build_mod_shared_rec_hash(char **infile_list, int n_infiles,
</span> while((ret = mod_logutils[mod_id]->log_get_record(in_fd, mod_buf)) == 1)
{
base_rec = (struct darshan_base_record *)mod_buf;
<span style="color: #000000;background-color: #ddffdd">+ if(init_rank == -1)
+ init_rank = base_rec->rank;
</span>
/* initialize the hash with the first rank's records */
<span style="color: #000000;background-color: #ffdddd">- if(!init)
</span><span style="color: #000000;background-color: #ddffdd">+ if(base_rec->rank == init_rank)
</span> {
struct darshan_base_record *agg_base;
<span style="color: #aaaaaa">@@ -128,10 +140,9 @@ int build_mod_shared_rec_hash(char **infile_list, int n_infiles,
</span> ref->id = base_rec->id;
ref->ref_cnt = 1;
HASH_ADD(hlink, *shared_rec_hash, id, sizeof(darshan_record_id), ref);
<span style="color: #000000;background-color: #ffdddd">- init = 1;
</span> }
else
<span style="color: #000000;background-color: #ffdddd">- {
</span><span style="color: #000000;background-color: #ddffdd">+ {
</span> /* search for this record in shared record hash */
HASH_FIND(hlink, *shared_rec_hash, &(base_rec->id),
sizeof(darshan_record_id), ref);
<span style="color: #aaaaaa">@@ -173,6 +184,7 @@ int main(int argc, char *argv[])
</span> char **infile_list;
int n_infiles;
int shared_redux;
<span style="color: #000000;background-color: #ddffdd">+ int64_t job_end_time = 0;
</span> char *outlog_path;
darshan_fd in_fd, merge_fd;
struct darshan_job in_job, merge_job;
<span style="color: #aaaaaa">@@ -191,7 +203,7 @@ int main(int argc, char *argv[])
</span> int ret;
/* grab command line arguments */
<span style="color: #000000;background-color: #ffdddd">- parse_args(argc, argv, &infile_list, &n_infiles, &outlog_path, &shared_redux);
</span><span style="color: #000000;background-color: #ddffdd">+ parse_args(argc, argv, &infile_list, &n_infiles, &outlog_path, &shared_redux, &job_end_time);
</span>
memset(&merge_job, 0, sizeof(struct darshan_job));
<span style="color: #aaaaaa">@@ -305,6 +317,10 @@ int main(int argc, char *argv[])
</span> darshan_log_close(in_fd);
}
<span style="color: #000000;background-color: #ddffdd">+ /* if a job end time was passed in, apply it to the output job */
+ if(job_end_time > 0)
+ merge_job.end_time = job_end_time;
+
</span> /* create the output "merged" log */
merge_fd = darshan_log_create(outlog_path, DARSHAN_ZLIB_COMP, 1);
if(merge_fd == NULL)
</code></pre>
<br>
</li>
<li id='diff-3'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-3'>
<strong>
darshan-util/darshan-mpiio-logutils.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-util/darshan-mpiio-logutils.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-util/darshan-mpiio-logutils.c
</span><span style="color: #aaaaaa">@@ -224,39 +224,33 @@ static void darshan_log_print_mpiio_file_diff(void *file_rec1, char *file_name1,
</span> return;
}
<span style="color: #000000;background-color: #ddffdd">+/* simple helper struct for determining time & byte variances */
+struct var_t
+{
+ double n;
+ double M;
+ double S;
+};
+
</span> static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag)
{
struct darshan_mpiio_file *mpi_rec = (struct darshan_mpiio_file *)rec;
struct darshan_mpiio_file *agg_mpi_rec = (struct darshan_mpiio_file *)agg_rec;
int i, j, k;
<span style="color: #000000;background-color: #ffdddd">- int set;
- int min_ndx;
- int64_t min;
</span><span style="color: #000000;background-color: #ddffdd">+ int total_count;
+ int64_t tmp_val[4];
+ int64_t tmp_cnt[4];
+ int tmp_ndx;
+ double old_M;
</span> double mpi_time = mpi_rec->fcounters[MPIIO_F_READ_TIME] +
mpi_rec->fcounters[MPIIO_F_WRITE_TIME] +
mpi_rec->fcounters[MPIIO_F_META_TIME];
<span style="color: #000000;background-color: #ffdddd">-
- /* special case initialization of shared record for
- * first call of this function
- */
- if(init_flag)
- {
- /* set fastest/slowest rank counters according to root rank.
- * these counters will be determined as the aggregation progresses.
- */
- agg_mpi_rec->counters[MPIIO_FASTEST_RANK] = mpi_rec->base_rec.rank;
- agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES] =
- mpi_rec->counters[MPIIO_BYTES_READ] +
- mpi_rec->counters[MPIIO_BYTES_WRITTEN];
- agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME] = mpi_time;
-
- agg_mpi_rec->counters[MPIIO_SLOWEST_RANK] =
- agg_mpi_rec->counters[MPIIO_FASTEST_RANK];
- agg_mpi_rec->counters[MPIIO_SLOWEST_RANK_BYTES] =
- agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES];
- agg_mpi_rec->fcounters[MPIIO_F_SLOWEST_RANK_TIME] =
- agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME];
- }
</span><span style="color: #000000;background-color: #ddffdd">+ double mpi_bytes = (double)mpi_rec->counters[MPIIO_BYTES_READ] +
+ mpi_rec->counters[MPIIO_BYTES_WRITTEN];
+ struct var_t *var_time_p = (struct var_t *)
+ ((char *)rec + sizeof(struct darshan_mpiio_file));
+ struct var_t *var_bytes_p = (struct var_t *)
+ ((char *)var_time_p + sizeof(struct var_t));
</span>
for(i = 0; i < MPIIO_NUM_INDICES; i++)
{
<span style="color: #aaaaaa">@@ -315,30 +309,69 @@ static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag)
</span> break;
case MPIIO_ACCESS1_ACCESS:
/* increment common value counters */
<span style="color: #000000;background-color: #ddffdd">+ if(mpi_rec->counters[i] == 0) break;
+
+ /* first, collapse duplicates */
</span> for(j = i; j < i + 4; j++)
{
<span style="color: #000000;background-color: #ffdddd">- min = agg_mpi_rec->counters[i + 4];
- min_ndx = 0;
- set = 0;
</span> for(k = 0; k < 4; k++)
{
if(agg_mpi_rec->counters[i + k] == mpi_rec->counters[j])
{
agg_mpi_rec->counters[i + k + 4] += mpi_rec->counters[j + 4];
<span style="color: #000000;background-color: #ffdddd">- set = 1;
</span><span style="color: #000000;background-color: #ddffdd">+ mpi_rec->counters[j] = mpi_rec->counters[j + 4] = 0;
+ }
+ }
+ }
+
+ /* second, add new counters */
+ for(j = i; j < i + 4; j++)
+ {
+ tmp_ndx = 0;
+ memset(tmp_val, 0, 4 * sizeof(int64_t));
+ memset(tmp_cnt, 0, 4 * sizeof(int64_t));
+
+ for(k = 0; k < 4; k++)
+ {
+ if(agg_mpi_rec->counters[i + k] == mpi_rec->counters[j])
+ {
+ total_count = agg_mpi_rec->counters[i + k + 4] +
+ mpi_rec->counters[j + 4];
</span> break;
}
<span style="color: #000000;background-color: #ffdddd">- else if(agg_mpi_rec->counters[i + k + 4] < min)
</span><span style="color: #000000;background-color: #ddffdd">+ }
+ if(k == 4) total_count = mpi_rec->counters[j + 4];
+
+ for(k = 0; k < 4; k++)
+ {
+ if((agg_mpi_rec->counters[i + k + 4] > total_count) ||
+ ((agg_mpi_rec->counters[i + k + 4] == total_count) &&
+ (agg_mpi_rec->counters[i + k] > mpi_rec->counters[j])))
</span> {
<span style="color: #000000;background-color: #ffdddd">- min = agg_mpi_rec->counters[i + k + 4];
- min_ndx = k;
</span><span style="color: #000000;background-color: #ddffdd">+ tmp_val[tmp_ndx] = agg_mpi_rec->counters[i + k];
+ tmp_cnt[tmp_ndx] = agg_mpi_rec->counters[i + k + 4];
+ tmp_ndx++;
</span> }
<span style="color: #000000;background-color: #ddffdd">+ else break;
</span> }
<span style="color: #000000;background-color: #ffdddd">- if(!set && (mpi_rec->counters[j + 4] > min))
</span><span style="color: #000000;background-color: #ddffdd">+ if(tmp_ndx == 4) break;
+
+ tmp_val[tmp_ndx] = mpi_rec->counters[j];
+ tmp_cnt[tmp_ndx] = mpi_rec->counters[j + 4];
+ tmp_ndx++;
+
+ while(tmp_ndx != 4)
</span> {
<span style="color: #000000;background-color: #ffdddd">- agg_mpi_rec->counters[i + min_ndx] = mpi_rec->counters[j];
- agg_mpi_rec->counters[i + min_ndx + 4] = mpi_rec->counters[j + 4];
</span><span style="color: #000000;background-color: #ddffdd">+ if(agg_mpi_rec->counters[i + k] != mpi_rec->counters[j])
+ {
+ tmp_val[tmp_ndx] = agg_mpi_rec->counters[i + k];
+ tmp_cnt[tmp_ndx] = agg_mpi_rec->counters[i + k + 4];
+ tmp_ndx++;
+ }
+ k++;
</span> }
<span style="color: #000000;background-color: #ddffdd">+ memcpy(&(agg_mpi_rec->counters[i]), tmp_val, 4 * sizeof(int64_t));
+ memcpy(&(agg_mpi_rec->counters[i + 4]), tmp_cnt, 4 * sizeof(int64_t));
</span> }
break;
case MPIIO_ACCESS2_ACCESS:
<span style="color: #aaaaaa">@@ -403,27 +436,80 @@ static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag)
</span> }
break;
case MPIIO_F_FASTEST_RANK_TIME:
<span style="color: #000000;background-color: #ddffdd">+ if(init_flag)
+ {
+ /* set fastest rank counters according to root rank. these counters
+ * will be determined as the aggregation progresses.
+ */
+ agg_mpi_rec->counters[MPIIO_FASTEST_RANK] = mpi_rec->base_rec.rank;
+ agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES] = mpi_bytes;
+ agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME] = mpi_time;
+ }
+
</span> if(mpi_time < agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME])
{
agg_mpi_rec->counters[MPIIO_FASTEST_RANK] = mpi_rec->base_rec.rank;
<span style="color: #000000;background-color: #ffdddd">- agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES] =
- mpi_rec->counters[MPIIO_BYTES_READ] +
- mpi_rec->counters[MPIIO_BYTES_WRITTEN];
</span><span style="color: #000000;background-color: #ddffdd">+ agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES] = mpi_bytes;
</span> agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME] = mpi_time;
}
break;
case MPIIO_F_SLOWEST_RANK_TIME:
<span style="color: #000000;background-color: #ddffdd">+ if(init_flag)
+ {
+ /* set slowest rank counters according to root rank. these counters
+ * will be determined as the aggregation progresses.
+ */
+ agg_mpi_rec->counters[MPIIO_SLOWEST_RANK] = mpi_rec->base_rec.rank;
+ agg_mpi_rec->counters[MPIIO_SLOWEST_RANK_BYTES] = mpi_bytes;
+ agg_mpi_rec->fcounters[MPIIO_F_SLOWEST_RANK_TIME] = mpi_time;
+ }
+
</span> if(mpi_time > agg_mpi_rec->fcounters[MPIIO_F_SLOWEST_RANK_TIME])
{
agg_mpi_rec->counters[MPIIO_SLOWEST_RANK] = mpi_rec->base_rec.rank;
<span style="color: #000000;background-color: #ffdddd">- agg_mpi_rec->counters[MPIIO_SLOWEST_RANK_BYTES] =
- mpi_rec->counters[MPIIO_BYTES_READ] +
- mpi_rec->counters[MPIIO_BYTES_WRITTEN];
</span><span style="color: #000000;background-color: #ddffdd">+ agg_mpi_rec->counters[MPIIO_SLOWEST_RANK_BYTES] = mpi_bytes;
</span> agg_mpi_rec->fcounters[MPIIO_F_SLOWEST_RANK_TIME] = mpi_time;
}
break;
<span style="color: #000000;background-color: #ddffdd">+ case MPIIO_F_VARIANCE_RANK_TIME:
+ if(init_flag)
+ {
+ var_time_p->n = 1;
+ var_time_p->M = mpi_time;
+ var_time_p->S = 0;
+ }
+ else
+ {
+ old_M = var_time_p->M;
+
+ var_time_p->n++;
+ var_time_p->M += (mpi_time - var_time_p->M) / var_time_p->n;
+ var_time_p->S += (mpi_time - var_time_p->M) * (mpi_time - old_M);
+
+ agg_mpi_rec->fcounters[MPIIO_F_VARIANCE_RANK_TIME] =
+ var_time_p->S / var_time_p->n;
+ }
+ break;
+ case MPIIO_F_VARIANCE_RANK_BYTES:
+ if(init_flag)
+ {
+ var_bytes_p->n = 1;
+ var_bytes_p->M = mpi_bytes;
+ var_bytes_p->S = 0;
+ }
+ else
+ {
+ old_M = var_bytes_p->M;
+
+ var_bytes_p->n++;
+ var_bytes_p->M += (mpi_bytes - var_bytes_p->M) / var_bytes_p->n;
+ var_bytes_p->S += (mpi_bytes - var_bytes_p->M) * (mpi_bytes - old_M);
+
+ agg_mpi_rec->fcounters[MPIIO_F_VARIANCE_RANK_BYTES] =
+ var_bytes_p->S / var_bytes_p->n;
+ }
+ break;
</span> default:
<span style="color: #000000;background-color: #ffdddd">- /* TODO: variance */
</span> agg_mpi_rec->fcounters[i] = -1;
break;
}
</code></pre>
<br>
</li>
<li id='diff-4'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-4'>
<strong>
darshan-util/darshan-posix-logutils.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-util/darshan-posix-logutils.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-util/darshan-posix-logutils.c
</span><span style="color: #aaaaaa">@@ -225,39 +225,33 @@ static void darshan_log_print_posix_file_diff(void *file_rec1, char *file_name1,
</span> return;
}
<span style="color: #000000;background-color: #ddffdd">+/* simple helper struct for determining time & byte variances */
+struct var_t
+{
+ double n;
+ double M;
+ double S;
+};
+
</span> static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
{
struct darshan_posix_file *psx_rec = (struct darshan_posix_file *)rec;
struct darshan_posix_file *agg_psx_rec = (struct darshan_posix_file *)agg_rec;
int i, j, k;
<span style="color: #000000;background-color: #ffdddd">- int set;
- int min_ndx;
- int64_t min;
</span><span style="color: #000000;background-color: #ddffdd">+ int total_count;
+ int64_t tmp_val[4];
+ int64_t tmp_cnt[4];
+ int tmp_ndx;
+ double old_M;
</span> double psx_time = psx_rec->fcounters[POSIX_F_READ_TIME] +
psx_rec->fcounters[POSIX_F_WRITE_TIME] +
psx_rec->fcounters[POSIX_F_META_TIME];
<span style="color: #000000;background-color: #ffdddd">-
- /* special case initialization of shared record for
- * first call of this function
- */
- if(init_flag)
- {
- /* set fastest/slowest rank counters according to root rank.
- * these counters will be determined as the aggregation progresses.
- */
- agg_psx_rec->counters[POSIX_FASTEST_RANK] = psx_rec->base_rec.rank;
- agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] =
- psx_rec->counters[POSIX_BYTES_READ] +
- psx_rec->counters[POSIX_BYTES_WRITTEN];
- agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME] = psx_time;
-
- agg_psx_rec->counters[POSIX_SLOWEST_RANK] =
- agg_psx_rec->counters[POSIX_FASTEST_RANK];
- agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] =
- agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES];
- agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME] =
- agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME];
- }
</span><span style="color: #000000;background-color: #ddffdd">+ double psx_bytes = (double)psx_rec->counters[POSIX_BYTES_READ] +
+ psx_rec->counters[POSIX_BYTES_WRITTEN];
+ struct var_t *var_time_p = (struct var_t *)
+ ((char *)rec + sizeof(struct darshan_posix_file));
+ struct var_t *var_bytes_p = (struct var_t *)
+ ((char *)var_time_p + sizeof(struct var_t));
</span>
for(i = 0; i < POSIX_NUM_INDICES; i++)
{
<span style="color: #aaaaaa">@@ -332,30 +326,69 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
</span> case POSIX_STRIDE1_STRIDE:
case POSIX_ACCESS1_ACCESS:
/* increment common value counters */
<span style="color: #000000;background-color: #ddffdd">+ if(psx_rec->counters[i] == 0) break;
+
+ /* first, collapse duplicates */
</span> for(j = i; j < i + 4; j++)
{
<span style="color: #000000;background-color: #ffdddd">- min = agg_psx_rec->counters[i + 4];
- min_ndx = 0;
- set = 0;
</span> for(k = 0; k < 4; k++)
{
if(agg_psx_rec->counters[i + k] == psx_rec->counters[j])
{
agg_psx_rec->counters[i + k + 4] += psx_rec->counters[j + 4];
<span style="color: #000000;background-color: #ffdddd">- set = 1;
</span><span style="color: #000000;background-color: #ddffdd">+ psx_rec->counters[j] = psx_rec->counters[j + 4] = 0;
+ }
+ }
+ }
+
+ /* second, add new counters */
+ for(j = i; j < i + 4; j++)
+ {
+ tmp_ndx = 0;
+ memset(tmp_val, 0, 4 * sizeof(int64_t));
+ memset(tmp_cnt, 0, 4 * sizeof(int64_t));
+
+ for(k = 0; k < 4; k++)
+ {
+ if(agg_psx_rec->counters[i + k] == psx_rec->counters[j])
+ {
+ total_count = agg_psx_rec->counters[i + k + 4] +
+ psx_rec->counters[j + 4];
</span> break;
}
<span style="color: #000000;background-color: #ffdddd">- else if(agg_psx_rec->counters[i + k + 4] < min)
</span><span style="color: #000000;background-color: #ddffdd">+ }
+ if(k == 4) total_count = psx_rec->counters[j + 4];
+
+ for(k = 0; k < 4; k++)
+ {
+ if((agg_psx_rec->counters[i + k + 4] > total_count) ||
+ ((agg_psx_rec->counters[i + k + 4] == total_count) &&
+ (agg_psx_rec->counters[i + k] > psx_rec->counters[j])))
</span> {
<span style="color: #000000;background-color: #ffdddd">- min = agg_psx_rec->counters[i + k + 4];
- min_ndx = k;
</span><span style="color: #000000;background-color: #ddffdd">+ tmp_val[tmp_ndx] = agg_psx_rec->counters[i + k];
+ tmp_cnt[tmp_ndx] = agg_psx_rec->counters[i + k + 4];
+ tmp_ndx++;
</span> }
<span style="color: #000000;background-color: #ddffdd">+ else break;
</span> }
<span style="color: #000000;background-color: #ffdddd">- if(!set && (psx_rec->counters[j + 4] > min))
</span><span style="color: #000000;background-color: #ddffdd">+ if(tmp_ndx == 4) break;
+
+ tmp_val[tmp_ndx] = psx_rec->counters[j];
+ tmp_cnt[tmp_ndx] = psx_rec->counters[j + 4];
+ tmp_ndx++;
+
+ while(tmp_ndx != 4)
</span> {
<span style="color: #000000;background-color: #ffdddd">- agg_psx_rec->counters[i + min_ndx] = psx_rec->counters[j];
- agg_psx_rec->counters[i + min_ndx + 4] = psx_rec->counters[j + 4];
</span><span style="color: #000000;background-color: #ddffdd">+ if(agg_psx_rec->counters[i + k] != psx_rec->counters[j])
+ {
+ tmp_val[tmp_ndx] = agg_psx_rec->counters[i + k];
+ tmp_cnt[tmp_ndx] = agg_psx_rec->counters[i + k + 4];
+ tmp_ndx++;
+ }
+ k++;
</span> }
<span style="color: #000000;background-color: #ddffdd">+ memcpy(&(agg_psx_rec->counters[i]), tmp_val, 4 * sizeof(int64_t));
+ memcpy(&(agg_psx_rec->counters[i + 4]), tmp_cnt, 4 * sizeof(int64_t));
</span> }
break;
case POSIX_STRIDE2_STRIDE:
<span style="color: #aaaaaa">@@ -427,27 +460,80 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
</span> }
break;
case POSIX_F_FASTEST_RANK_TIME:
<span style="color: #000000;background-color: #ddffdd">+ if(init_flag)
+ {
+ /* set fastest rank counters according to root rank. these counters
+ * will be determined as the aggregation progresses.
+ */
+ agg_psx_rec->counters[POSIX_FASTEST_RANK] = psx_rec->base_rec.rank;
+ agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] = psx_bytes;
+ agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME] = psx_time;
+ }
+
</span> if(psx_time < agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME])
{
agg_psx_rec->counters[POSIX_FASTEST_RANK] = psx_rec->base_rec.rank;
<span style="color: #000000;background-color: #ffdddd">- agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] =
- psx_rec->counters[POSIX_BYTES_READ] +
- psx_rec->counters[POSIX_BYTES_WRITTEN];
</span><span style="color: #000000;background-color: #ddffdd">+ agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] = psx_bytes;
</span> agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME] = psx_time;
}
break;
case POSIX_F_SLOWEST_RANK_TIME:
<span style="color: #000000;background-color: #ddffdd">+ if(init_flag)
+ {
+ /* set slowest rank counters according to root rank. these counters
+ * will be determined as the aggregation progresses.
+ */
+ agg_psx_rec->counters[POSIX_SLOWEST_RANK] = psx_rec->base_rec.rank;
+ agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] = psx_bytes;
+ agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME] = psx_time;
+ }
+
</span> if(psx_time > agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME])
{
agg_psx_rec->counters[POSIX_SLOWEST_RANK] = psx_rec->base_rec.rank;
<span style="color: #000000;background-color: #ffdddd">- agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] =
- psx_rec->counters[POSIX_BYTES_READ] +
- psx_rec->counters[POSIX_BYTES_WRITTEN];
</span><span style="color: #000000;background-color: #ddffdd">+ agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] = psx_bytes;
</span> agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME] = psx_time;
}
break;
<span style="color: #000000;background-color: #ddffdd">+ case POSIX_F_VARIANCE_RANK_TIME:
+ if(init_flag)
+ {
+ var_time_p->n = 1;
+ var_time_p->M = psx_time;
+ var_time_p->S = 0;
+ }
+ else
+ {
+ old_M = var_time_p->M;
+
+ var_time_p->n++;
+ var_time_p->M += (psx_time - var_time_p->M) / var_time_p->n;
+ var_time_p->S += (psx_time - var_time_p->M) * (psx_time - old_M);
+
+ agg_psx_rec->fcounters[POSIX_F_VARIANCE_RANK_TIME] =
+ var_time_p->S / var_time_p->n;
+ }
+ break;
+ case POSIX_F_VARIANCE_RANK_BYTES:
+ if(init_flag)
+ {
+ var_bytes_p->n = 1;
+ var_bytes_p->M = psx_bytes;
+ var_bytes_p->S = 0;
+ }
+ else
+ {
+ old_M = var_bytes_p->M;
+
+ var_bytes_p->n++;
+ var_bytes_p->M += (psx_bytes - var_bytes_p->M) / var_bytes_p->n;
+ var_bytes_p->S += (psx_bytes - var_bytes_p->M) * (psx_bytes - old_M);
+
+ agg_psx_rec->fcounters[POSIX_F_VARIANCE_RANK_BYTES] =
+ var_bytes_p->S / var_bytes_p->n;
+ }
+ break;
</span> default:
<span style="color: #000000;background-color: #ffdddd">- /* TODO: variance */
</span> agg_psx_rec->fcounters[i] = -1;
break;
}
</code></pre>
<br>
</li>
</div>
<div class='footer' style='margin-top: 10px;'>
<p>
—
<br>
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273">View it on GitLab</a>.
<br>
You're receiving this email because of your account on xgitlab.cels.anl.gov.
If you'd like to receive fewer emails, you can
adjust your notification settings.
</p>
</div>
</body>
</html>