<html lang='en'>
<head>
<meta content='text/html; charset=utf-8' http-equiv='Content-Type'>
<title>
GitLab
</title>
</meta>
</head>
<style>
  img {
    max-width: 100%;
    height: auto;
  }
  p.details {
    font-style:italic;
    color:#777
  }
  .footer p {
    font-size:small;
    color:#777
  }
  pre.commit-message {
    white-space: pre-wrap;
  }
  .file-stats a {
    text-decoration: none;
  }
  .file-stats .new-file {
    color: #090;
  }
  .file-stats .deleted-file {
    color: #B00;
  }
</style>
<body>
<div class='content'>
<h3>
Shane Snyder pushed to branch mmap-dev
at <a href="https://xgitlab.cels.anl.gov/darshan/darshan">darshan / darshan</a>
</h3>
<h4>
Commits:
</h4>
<ul>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/f26baefc40a3506a8bf1cfb5e645d9deddf75db4">f26baefc</a></strong>
<div>
<span>by Shane Snyder</span>
<i>at 2016-02-26T12:34:02-06:00</i>
</div>
<pre class='commit-message'>allow job end time to be set by darshan-merge</pre>
</li>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/d4a3d112be89104319ae949516d4c2e72eace870">d4a3d112</a></strong>
<div>
<span>by Shane Snyder</span>
<i>at 2016-02-29T17:22:57-06:00</i>
</div>
<pre class='commit-message'>add variance counter reduction logic to logutils</pre>
</li>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/04063da70950ee779371188c7937956eb89b4273">04063da7</a></strong>
<div>
<span>by Shane Snyder</span>
<i>at 2016-03-01T17:43:38-06:00</i>
</div>
<pre class='commit-message'>common access counters are now sorted in logs

common value counters are sorted first by decreasing counts,
and then by decreasing size.</pre>
</li>
</ul>
<h4>5 changed files:</h4>
<ul>
<li class='file-stats'>
<a href='#diff-0'>
darshan-runtime/darshan-common.h
</a>
</li>
<li class='file-stats'>
<a href='#diff-1'>
darshan-runtime/share/darshan-mmap-epilog.sh.in
</a>
</li>
<li class='file-stats'>
<a href='#diff-2'>
darshan-util/darshan-merge.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-3'>
darshan-util/darshan-mpiio-logutils.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-4'>
darshan-util/darshan-posix-logutils.c
</a>
</li>
</ul>
<h4>Changes:</h4>
<li id='diff-0'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-0'>
<strong>
darshan-runtime/darshan-common.h
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/darshan-common.h
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/darshan-common.h
</span><span style="color: #aaaaaa">@@ -66,7 +66,10 @@
</span> } while(0)
 
 /* potentially set or increment a common value counter, depending on the __count
<span style="color: #000000;background-color: #ffdddd">- * for the given __value
</span><span style="color: #000000;background-color: #ddffdd">+ * for the given __value. This macro ensures common values are stored first in
+ * decreasing order of their total count, and second by decreasing order of
+ * their value.
+
</span>  *
  * NOTE: This macro is hardcoded to expect that Darshan will only track the 4
  * most common (i.e., frequently occuring) values. __val_p is a pointer to the
<span style="color: #aaaaaa">@@ -78,32 +81,50 @@
</span>  */
 #define DARSHAN_COMMON_VAL_COUNTER_INC(__val_p, __cnt_p, __value, __count, __online_flag) do {\
     int i; \
<span style="color: #000000;background-color: #ffdddd">-    int set = 0; \
-    int64_t min = *(__cnt_p); \
-    int min_index = 0; \
-    int inc_count; \
</span><span style="color: #000000;background-color: #ddffdd">+    int inc_count, total_count; \
+    int64_t tmp_val[4] = {0}; \
+    int64_t tmp_cnt[4] = {0}; \
+    int tmp_ndx = 0; \
</span>     if(__value == 0) break; \
     if(__online_flag) \
         inc_count = 1; \
     else \
         inc_count = __count; \
     for(i=0; i<4; i++) { \
<span style="color: #000000;background-color: #ffdddd">-        /* increment bucket if already exists */ \
</span>         if(*(__val_p + i) == __value) { \
<span style="color: #000000;background-color: #ffdddd">-            *(__cnt_p + i) += inc_count; \
-            set = 1; \
</span><span style="color: #000000;background-color: #ddffdd">+            total_count = *(__cnt_p + i) + inc_count; \
</span>             break; \
         } \
<span style="color: #000000;background-color: #ffdddd">-        /* otherwise find the least frequently used bucket */ \
-        else if(*(__cnt_p + i) < min) { \
-            min = *(__cnt_p + i); \
-            min_index = i; \
</span><span style="color: #000000;background-color: #ddffdd">+    } \
+    if(i == 4) total_count = __count; \
+    /* first, copy over any counters that should be sorted above this one \
+     * (counters with higher counts or equal counts and larger values) \
+     */ \
+    for(i=0;i < 4; i++) { \
+        if((*(__cnt_p + i) > total_count) || \
+           ((*(__cnt_p + i) == total_count) && (*(__val_p + i) > __value))) { \
+            tmp_val[tmp_ndx] = *(__val_p + i); \
+            tmp_cnt[tmp_ndx] = *(__cnt_p + i); \
+            tmp_ndx++; \
</span>         } \
<span style="color: #000000;background-color: #ddffdd">+        else break; \
</span>     } \
<span style="color: #000000;background-color: #ffdddd">-    if(!set && (__count > min)) { \
-        *(__cnt_p + min_index) = __count; \
-        *(__val_p + min_index) = __value; \
</span><span style="color: #000000;background-color: #ddffdd">+    if(tmp_ndx == 4) break; /* all done, updated counter is not added */ \
+    /* next, add the updated counter */ \
+    tmp_val[tmp_ndx] = __value; \
+    tmp_cnt[tmp_ndx] = total_count; \
+    tmp_ndx++; \
+    /* last, copy over any remaining counters to make sure we have 4 sets total */ \
+    while(tmp_ndx != 4) { \
+        if(*(__val_p + i) != __value) { \
+            tmp_val[tmp_ndx] = *(__val_p + i); \
+            tmp_cnt[tmp_ndx] = *(__cnt_p + i); \
+            tmp_ndx++; \
+        } \
+        i++; \
</span>     } \
<span style="color: #000000;background-color: #ddffdd">+    memcpy(__val_p, tmp_val, 4*sizeof(int64_t)); \
+    memcpy(__cnt_p, tmp_cnt, 4*sizeof(int64_t)); \
</span> } while(0)
 
 /* maximum number of common values that darshan will track per file at
</code></pre>

<br>
</li>
<li id='diff-1'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-1'>
<strong>
darshan-runtime/share/darshan-mmap-epilog.sh.in
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/share/darshan-mmap-epilog.sh.in
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/share/darshan-mmap-epilog.sh.in
</span><span style="color: #aaaaaa">@@ -11,6 +11,8 @@ DARSHAN_INSTALL_DIR=@prefix@
</span> # use the log dir specified at configure time
 DARSHAN_LOG_DIR=@__DARSHAN_LOG_PATH@
 
<span style="color: #000000;background-color: #ddffdd">+JOB_END=$(date +"%s")
+
</span> # use the default mmap log directory (/tmp), unless the 
 # env variable is set to something
 if [ -z "$DARSHAN_MMAP_LOGPATH" ]; then
<span style="color: #aaaaaa">@@ -53,7 +55,7 @@ if [ $SLURM_NNODES -gt 1 ]; then
</span>     mkdir -p $NODE_LOG_DIR
 
     # construct the per-node log file and store in the output directory
<span style="color: #000000;background-color: #ffdddd">-    $DARSHAN_INSTALL_DIR/bin/darshan-merge \
</span><span style="color: #000000;background-color: #ddffdd">+    $DARSHAN_INSTALL_DIR/bin/darshan-merge --job-end-time $JOB_END \
</span>         --output ${NODE_LOG_DIR}/${LOG_NAME_PRE}_${NODE_NAME}.darshan \
         $DARSHAN_MMAP_LOG_GLOB
 else
<span style="color: #aaaaaa">@@ -61,7 +63,7 @@ else
</span> 
     # single node, just create the final output darshan log
     LOG_WRITE_START=$(date +%s)
<span style="color: #000000;background-color: #ffdddd">-    $DARSHAN_INSTALL_DIR/bin/darshan-merge
</span><span style="color: #000000;background-color: #ddffdd">+    $DARSHAN_INSTALL_DIR/bin/darshan-merge --job-end-time $JOB_END \
</span>         --shared-redux --output ${OUTPUT_LOG_DIR}/${TMP_LOG} \
         $DARSHAN_MMAP_LOG_GLOB
     LOG_WRITE_END=$(date +%s)
<span style="color: #aaaaaa">@@ -72,4 +74,6 @@ else
</span>     mv ${OUTPUT_LOG_DIR}/${TMP_LOG} ${OUTPUT_LOG_DIR}/${FINAL_LOG}
 fi
 
<span style="color: #000000;background-color: #ddffdd">+rm -f $DARSHAN_MMAP_LOG_GLOB
+
</span> exit 0
</code></pre>

<br>
</li>
<li id='diff-2'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-2'>
<strong>
darshan-util/darshan-merge.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-util/darshan-merge.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-util/darshan-merge.c
</span><span style="color: #aaaaaa">@@ -11,8 +11,6 @@
</span> 
 #define DEF_MOD_BUF_SIZE 1024 /* 1 KiB is enough for all current mod records ... */
 
<span style="color: #000000;background-color: #ffdddd">-/* TODO: set job end timestamp? */
-
</span> struct darshan_shared_record_ref
 {
     darshan_record_id id;
<span style="color: #aaaaaa">@@ -28,23 +26,27 @@ void usage(char *exename)
</span>     fprintf(stderr, "Options:\n");
     fprintf(stderr, "\t--output\t(REQUIRED) Full path of the output darshan log file.\n");
     fprintf(stderr, "\t--shared-redux\tReduce globally shared records into a single record.\n");
<span style="color: #000000;background-color: #ddffdd">+    fprintf(stderr, "\t--job-end-time\tSet the output log's job end time (requires argument of seconds since Epoch).\n");
</span> 
     exit(1);
 }
 
 void parse_args(int argc, char **argv, char ***infile_list, int *n_files,
<span style="color: #000000;background-color: #ffdddd">-    char **outlog_path, int *shared_redux)
</span><span style="color: #000000;background-color: #ddffdd">+    char **outlog_path, int *shared_redux, int64_t *job_end_time)
</span> {
     int index;
<span style="color: #000000;background-color: #ddffdd">+    char *check;
</span>     static struct option long_opts[] =
     {
<span style="color: #000000;background-color: #ffdddd">-        {"shared-redux", no_argument, NULL, 's'},
</span>         {"output", required_argument, NULL, 'o'},
<span style="color: #000000;background-color: #ddffdd">+        {"shared-redux", no_argument, NULL, 's'},
+        {"job-end-time", required_argument, NULL, 'e'},
</span>         {0, 0, 0, 0}
     };
 
     *shared_redux = 0;
     *outlog_path = NULL;
<span style="color: #000000;background-color: #ddffdd">+    *job_end_time = 0;
</span> 
     while(1)
     {
<span style="color: #aaaaaa">@@ -60,6 +62,14 @@ void parse_args(int argc, char **argv, char ***infile_list, int *n_files,
</span>             case 'o':
                 *outlog_path = optarg;
                 break;
<span style="color: #000000;background-color: #ddffdd">+            case 'e':
+                *job_end_time = strtol(optarg, &check, 10);
+                if(optarg == check)
+                {
+                    fprintf(stderr, "Error: unable to parse job end time value.\n");
+                    exit(1);
+                }
+                break;
</span>             case '?':
             default:
                 usage(argv[0]);
<span style="color: #aaaaaa">@@ -85,7 +95,7 @@ int build_mod_shared_rec_hash(char **infile_list, int n_infiles,
</span>     darshan_fd in_fd;
     struct darshan_base_record *base_rec;
     struct darshan_shared_record_ref *ref, *tmp;
<span style="color: #000000;background-color: #ffdddd">-    int init = 0;
</span><span style="color: #000000;background-color: #ddffdd">+    int init_rank = -1;
</span>     int ret;
     int i;
 
<span style="color: #aaaaaa">@@ -104,9 +114,11 @@ int build_mod_shared_rec_hash(char **infile_list, int n_infiles,
</span>         while((ret = mod_logutils[mod_id]->log_get_record(in_fd, mod_buf)) == 1)
         {
             base_rec = (struct darshan_base_record *)mod_buf;
<span style="color: #000000;background-color: #ddffdd">+            if(init_rank == -1)
+                init_rank = base_rec->rank;
</span> 
             /* initialize the hash with the first rank's records */
<span style="color: #000000;background-color: #ffdddd">-            if(!init)
</span><span style="color: #000000;background-color: #ddffdd">+            if(base_rec->rank == init_rank)
</span>             {
                 struct darshan_base_record *agg_base;
 
<span style="color: #aaaaaa">@@ -128,10 +140,9 @@ int build_mod_shared_rec_hash(char **infile_list, int n_infiles,
</span>                 ref->id = base_rec->id;
                 ref->ref_cnt = 1;
                 HASH_ADD(hlink, *shared_rec_hash, id, sizeof(darshan_record_id), ref);
<span style="color: #000000;background-color: #ffdddd">-                init = 1;
</span>             }
             else
<span style="color: #000000;background-color: #ffdddd">-            {
</span><span style="color: #000000;background-color: #ddffdd">+           {
</span>                 /* search for this record in shared record hash */
                 HASH_FIND(hlink, *shared_rec_hash, &(base_rec->id),
                     sizeof(darshan_record_id), ref);
<span style="color: #aaaaaa">@@ -173,6 +184,7 @@ int main(int argc, char *argv[])
</span>     char **infile_list;
     int n_infiles;
     int shared_redux;
<span style="color: #000000;background-color: #ddffdd">+    int64_t job_end_time = 0;
</span>     char *outlog_path;
     darshan_fd in_fd, merge_fd;
     struct darshan_job in_job, merge_job;
<span style="color: #aaaaaa">@@ -191,7 +203,7 @@ int main(int argc, char *argv[])
</span>     int ret;
 
     /* grab command line arguments */
<span style="color: #000000;background-color: #ffdddd">-    parse_args(argc, argv, &infile_list, &n_infiles, &outlog_path, &shared_redux);
</span><span style="color: #000000;background-color: #ddffdd">+    parse_args(argc, argv, &infile_list, &n_infiles, &outlog_path, &shared_redux, &job_end_time);
</span> 
     memset(&merge_job, 0, sizeof(struct darshan_job));
 
<span style="color: #aaaaaa">@@ -305,6 +317,10 @@ int main(int argc, char *argv[])
</span>         darshan_log_close(in_fd);
     }
 
<span style="color: #000000;background-color: #ddffdd">+    /* if a job end time was passed in, apply it to the output job */
+    if(job_end_time > 0)
+        merge_job.end_time = job_end_time;
+
</span>     /* create the output "merged" log */
     merge_fd = darshan_log_create(outlog_path, DARSHAN_ZLIB_COMP, 1);
     if(merge_fd == NULL)
</code></pre>

<br>
</li>
<li id='diff-3'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-3'>
<strong>
darshan-util/darshan-mpiio-logutils.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-util/darshan-mpiio-logutils.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-util/darshan-mpiio-logutils.c
</span><span style="color: #aaaaaa">@@ -224,39 +224,33 @@ static void darshan_log_print_mpiio_file_diff(void *file_rec1, char *file_name1,
</span>     return;
 }
 
<span style="color: #000000;background-color: #ddffdd">+/* simple helper struct for determining time & byte variances */
+struct var_t
+{
+    double n;
+    double M;
+    double S;
+};
+
</span> static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag)
 {
     struct darshan_mpiio_file *mpi_rec = (struct darshan_mpiio_file *)rec;
     struct darshan_mpiio_file *agg_mpi_rec = (struct darshan_mpiio_file *)agg_rec;
     int i, j, k;
<span style="color: #000000;background-color: #ffdddd">-    int set;
-    int min_ndx;
-    int64_t min;
</span><span style="color: #000000;background-color: #ddffdd">+    int total_count;
+    int64_t tmp_val[4];
+    int64_t tmp_cnt[4];
+    int tmp_ndx;
+    double old_M;
</span>     double mpi_time = mpi_rec->fcounters[MPIIO_F_READ_TIME] +
         mpi_rec->fcounters[MPIIO_F_WRITE_TIME] +
         mpi_rec->fcounters[MPIIO_F_META_TIME];
<span style="color: #000000;background-color: #ffdddd">-
-    /* special case initialization of shared record for
-     * first call of this function
-     */
-    if(init_flag)
-    {
-        /* set fastest/slowest rank counters according to root rank.
-         * these counters will be determined as the aggregation progresses.
-         */
-        agg_mpi_rec->counters[MPIIO_FASTEST_RANK] = mpi_rec->base_rec.rank;
-        agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES] =
-            mpi_rec->counters[MPIIO_BYTES_READ] +
-            mpi_rec->counters[MPIIO_BYTES_WRITTEN];
-        agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME] = mpi_time;
-
-        agg_mpi_rec->counters[MPIIO_SLOWEST_RANK] =
-            agg_mpi_rec->counters[MPIIO_FASTEST_RANK];
-        agg_mpi_rec->counters[MPIIO_SLOWEST_RANK_BYTES] =
-            agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES];
-        agg_mpi_rec->fcounters[MPIIO_F_SLOWEST_RANK_TIME] =
-            agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME];
-    }
</span><span style="color: #000000;background-color: #ddffdd">+    double mpi_bytes = (double)mpi_rec->counters[MPIIO_BYTES_READ] +
+        mpi_rec->counters[MPIIO_BYTES_WRITTEN];
+    struct var_t *var_time_p = (struct var_t *)
+        ((char *)rec + sizeof(struct darshan_mpiio_file));
+    struct var_t *var_bytes_p = (struct var_t *)
+        ((char *)var_time_p + sizeof(struct var_t));
</span> 
     for(i = 0; i < MPIIO_NUM_INDICES; i++)
     {
<span style="color: #aaaaaa">@@ -315,30 +309,69 @@ static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag)
</span>                 break;
             case MPIIO_ACCESS1_ACCESS:
                 /* increment common value counters */
<span style="color: #000000;background-color: #ddffdd">+                if(mpi_rec->counters[i] == 0) break;
+
+                /* first, collapse duplicates */
</span>                 for(j = i; j < i + 4; j++)
                 {
<span style="color: #000000;background-color: #ffdddd">-                    min = agg_mpi_rec->counters[i + 4];
-                    min_ndx = 0;
-                    set = 0;
</span>                     for(k = 0; k < 4; k++)
                     {
                         if(agg_mpi_rec->counters[i + k] == mpi_rec->counters[j])
                         {
                             agg_mpi_rec->counters[i + k + 4] += mpi_rec->counters[j + 4];
<span style="color: #000000;background-color: #ffdddd">-                            set = 1;
</span><span style="color: #000000;background-color: #ddffdd">+                            mpi_rec->counters[j] = mpi_rec->counters[j + 4] = 0;
+                        }
+                    }
+                }
+
+                /* second, add new counters */
+                for(j = i; j < i + 4; j++)
+                {
+                    tmp_ndx = 0;
+                    memset(tmp_val, 0, 4 * sizeof(int64_t));
+                    memset(tmp_cnt, 0, 4 * sizeof(int64_t));
+
+                    for(k = 0; k < 4; k++)
+                    {
+                        if(agg_mpi_rec->counters[i + k] == mpi_rec->counters[j])
+                        {
+                            total_count = agg_mpi_rec->counters[i + k + 4] +
+                                mpi_rec->counters[j + 4];
</span>                             break;
                         }
<span style="color: #000000;background-color: #ffdddd">-                        else if(agg_mpi_rec->counters[i + k + 4] < min)
</span><span style="color: #000000;background-color: #ddffdd">+                    }
+                    if(k == 4) total_count = mpi_rec->counters[j + 4];
+
+                    for(k = 0; k < 4; k++)
+                    {
+                        if((agg_mpi_rec->counters[i + k + 4] > total_count) ||
+                           ((agg_mpi_rec->counters[i + k + 4] == total_count) &&
+                            (agg_mpi_rec->counters[i + k] > mpi_rec->counters[j])))
</span>                         {
<span style="color: #000000;background-color: #ffdddd">-                            min = agg_mpi_rec->counters[i + k + 4];
-                            min_ndx = k;
</span><span style="color: #000000;background-color: #ddffdd">+                            tmp_val[tmp_ndx] = agg_mpi_rec->counters[i + k];
+                            tmp_cnt[tmp_ndx] = agg_mpi_rec->counters[i + k + 4];
+                            tmp_ndx++;
</span>                         }
<span style="color: #000000;background-color: #ddffdd">+                        else break;
</span>                     }
<span style="color: #000000;background-color: #ffdddd">-                    if(!set && (mpi_rec->counters[j + 4] > min))
</span><span style="color: #000000;background-color: #ddffdd">+                    if(tmp_ndx == 4) break;
+
+                    tmp_val[tmp_ndx] = mpi_rec->counters[j];
+                    tmp_cnt[tmp_ndx] = mpi_rec->counters[j + 4];
+                    tmp_ndx++;
+
+                    while(tmp_ndx != 4)
</span>                     {
<span style="color: #000000;background-color: #ffdddd">-                        agg_mpi_rec->counters[i + min_ndx] = mpi_rec->counters[j];
-                        agg_mpi_rec->counters[i + min_ndx + 4] = mpi_rec->counters[j + 4];
</span><span style="color: #000000;background-color: #ddffdd">+                        if(agg_mpi_rec->counters[i + k] != mpi_rec->counters[j])
+                        {
+                            tmp_val[tmp_ndx] = agg_mpi_rec->counters[i + k];
+                            tmp_cnt[tmp_ndx] = agg_mpi_rec->counters[i + k + 4];
+                            tmp_ndx++;
+                        }
+                        k++;
</span>                     }
<span style="color: #000000;background-color: #ddffdd">+                    memcpy(&(agg_mpi_rec->counters[i]), tmp_val, 4 * sizeof(int64_t));
+                    memcpy(&(agg_mpi_rec->counters[i + 4]), tmp_cnt, 4 * sizeof(int64_t));
</span>                 }
                 break;
             case MPIIO_ACCESS2_ACCESS:
<span style="color: #aaaaaa">@@ -403,27 +436,80 @@ static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag)
</span>                 }
                 break;
             case MPIIO_F_FASTEST_RANK_TIME:
<span style="color: #000000;background-color: #ddffdd">+                if(init_flag)
+                {
+                    /* set fastest rank counters according to root rank. these counters
+                     * will be determined as the aggregation progresses.
+                     */
+                    agg_mpi_rec->counters[MPIIO_FASTEST_RANK] = mpi_rec->base_rec.rank;
+                    agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES] = mpi_bytes;
+                    agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME] = mpi_time;
+                }
+
</span>                 if(mpi_time < agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME])
                 {
                     agg_mpi_rec->counters[MPIIO_FASTEST_RANK] = mpi_rec->base_rec.rank;
<span style="color: #000000;background-color: #ffdddd">-                    agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES] =
-                        mpi_rec->counters[MPIIO_BYTES_READ] +
-                        mpi_rec->counters[MPIIO_BYTES_WRITTEN];
</span><span style="color: #000000;background-color: #ddffdd">+                    agg_mpi_rec->counters[MPIIO_FASTEST_RANK_BYTES] = mpi_bytes;
</span>                     agg_mpi_rec->fcounters[MPIIO_F_FASTEST_RANK_TIME] = mpi_time;
                 }
                 break;
             case MPIIO_F_SLOWEST_RANK_TIME:
<span style="color: #000000;background-color: #ddffdd">+                if(init_flag)
+                {
+                    /* set slowest rank counters according to root rank. these counters
+                     * will be determined as the aggregation progresses.
+                     */
+                    agg_mpi_rec->counters[MPIIO_SLOWEST_RANK] = mpi_rec->base_rec.rank;
+                    agg_mpi_rec->counters[MPIIO_SLOWEST_RANK_BYTES] = mpi_bytes;
+                    agg_mpi_rec->fcounters[MPIIO_F_SLOWEST_RANK_TIME] = mpi_time;
+                }
+
</span>                 if(mpi_time > agg_mpi_rec->fcounters[MPIIO_F_SLOWEST_RANK_TIME])
                 {
                     agg_mpi_rec->counters[MPIIO_SLOWEST_RANK] = mpi_rec->base_rec.rank;
<span style="color: #000000;background-color: #ffdddd">-                    agg_mpi_rec->counters[MPIIO_SLOWEST_RANK_BYTES] =
-                        mpi_rec->counters[MPIIO_BYTES_READ] +
-                        mpi_rec->counters[MPIIO_BYTES_WRITTEN];
</span><span style="color: #000000;background-color: #ddffdd">+                    agg_mpi_rec->counters[MPIIO_SLOWEST_RANK_BYTES] = mpi_bytes;
</span>                     agg_mpi_rec->fcounters[MPIIO_F_SLOWEST_RANK_TIME] = mpi_time;
                 }
                 break;
<span style="color: #000000;background-color: #ddffdd">+            case MPIIO_F_VARIANCE_RANK_TIME:
+                if(init_flag)
+                {
+                    var_time_p->n = 1;
+                    var_time_p->M = mpi_time;
+                    var_time_p->S = 0;
+                }
+                else
+                {
+                    old_M = var_time_p->M;
+
+                    var_time_p->n++;
+                    var_time_p->M += (mpi_time - var_time_p->M) / var_time_p->n;
+                    var_time_p->S += (mpi_time - var_time_p->M) * (mpi_time - old_M);
+
+                    agg_mpi_rec->fcounters[MPIIO_F_VARIANCE_RANK_TIME] =
+                        var_time_p->S / var_time_p->n;
+                }
+                break;
+            case MPIIO_F_VARIANCE_RANK_BYTES:
+                if(init_flag)
+                {
+                    var_bytes_p->n = 1;
+                    var_bytes_p->M = mpi_bytes;
+                    var_bytes_p->S = 0;
+                }
+                else
+                {
+                    old_M = var_bytes_p->M;
+
+                    var_bytes_p->n++;
+                    var_bytes_p->M += (mpi_bytes - var_bytes_p->M) / var_bytes_p->n;
+                    var_bytes_p->S += (mpi_bytes - var_bytes_p->M) * (mpi_bytes - old_M);
+
+                    agg_mpi_rec->fcounters[MPIIO_F_VARIANCE_RANK_BYTES] =
+                        var_bytes_p->S / var_bytes_p->n;
+                }
+                break;
</span>             default:
<span style="color: #000000;background-color: #ffdddd">-                /* TODO: variance */
</span>                 agg_mpi_rec->fcounters[i] = -1;
                 break;
         }
</code></pre>

<br>
</li>
<li id='diff-4'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273#diff-4'>
<strong>
darshan-util/darshan-posix-logutils.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-util/darshan-posix-logutils.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-util/darshan-posix-logutils.c
</span><span style="color: #aaaaaa">@@ -225,39 +225,33 @@ static void darshan_log_print_posix_file_diff(void *file_rec1, char *file_name1,
</span>     return;
 }
 
<span style="color: #000000;background-color: #ddffdd">+/* simple helper struct for determining time & byte variances */
+struct var_t
+{
+    double n;
+    double M;
+    double S;
+};
+
</span> static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
 {
     struct darshan_posix_file *psx_rec = (struct darshan_posix_file *)rec;
     struct darshan_posix_file *agg_psx_rec = (struct darshan_posix_file *)agg_rec;
     int i, j, k;
<span style="color: #000000;background-color: #ffdddd">-    int set;
-    int min_ndx;
-    int64_t min;
</span><span style="color: #000000;background-color: #ddffdd">+    int total_count;
+    int64_t tmp_val[4];
+    int64_t tmp_cnt[4];
+    int tmp_ndx;
+    double old_M;
</span>     double psx_time = psx_rec->fcounters[POSIX_F_READ_TIME] +
         psx_rec->fcounters[POSIX_F_WRITE_TIME] +
         psx_rec->fcounters[POSIX_F_META_TIME];
<span style="color: #000000;background-color: #ffdddd">-
-    /* special case initialization of shared record for
-     * first call of this function
-     */
-    if(init_flag)
-    {
-        /* set fastest/slowest rank counters according to root rank.
-         * these counters will be determined as the aggregation progresses.
-         */
-        agg_psx_rec->counters[POSIX_FASTEST_RANK] = psx_rec->base_rec.rank;
-        agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] =
-            psx_rec->counters[POSIX_BYTES_READ] +
-            psx_rec->counters[POSIX_BYTES_WRITTEN];
-        agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME] = psx_time;
-
-        agg_psx_rec->counters[POSIX_SLOWEST_RANK] =
-            agg_psx_rec->counters[POSIX_FASTEST_RANK];
-        agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] =
-            agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES];
-        agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME] =
-            agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME];
-    }
</span><span style="color: #000000;background-color: #ddffdd">+    double psx_bytes = (double)psx_rec->counters[POSIX_BYTES_READ] +
+        psx_rec->counters[POSIX_BYTES_WRITTEN];
+    struct var_t *var_time_p = (struct var_t *)
+        ((char *)rec + sizeof(struct darshan_posix_file));
+    struct var_t *var_bytes_p = (struct var_t *)
+        ((char *)var_time_p + sizeof(struct var_t));
</span> 
     for(i = 0; i < POSIX_NUM_INDICES; i++)
     {
<span style="color: #aaaaaa">@@ -332,30 +326,69 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
</span>             case POSIX_STRIDE1_STRIDE:
             case POSIX_ACCESS1_ACCESS:
                 /* increment common value counters */
<span style="color: #000000;background-color: #ddffdd">+                if(psx_rec->counters[i] == 0) break;
+
+                /* first, collapse duplicates */
</span>                 for(j = i; j < i + 4; j++)
                 {
<span style="color: #000000;background-color: #ffdddd">-                    min = agg_psx_rec->counters[i + 4];
-                    min_ndx = 0;
-                    set = 0;
</span>                     for(k = 0; k < 4; k++)
                     {
                         if(agg_psx_rec->counters[i + k] == psx_rec->counters[j])
                         {
                             agg_psx_rec->counters[i + k + 4] += psx_rec->counters[j + 4];
<span style="color: #000000;background-color: #ffdddd">-                            set = 1;
</span><span style="color: #000000;background-color: #ddffdd">+                            psx_rec->counters[j] = psx_rec->counters[j + 4] = 0;
+                        }
+                    }
+                }
+
+                /* second, add new counters */
+                for(j = i; j < i + 4; j++)
+                {
+                    tmp_ndx = 0;
+                    memset(tmp_val, 0, 4 * sizeof(int64_t));
+                    memset(tmp_cnt, 0, 4 * sizeof(int64_t));
+
+                    for(k = 0; k < 4; k++)
+                    {
+                        if(agg_psx_rec->counters[i + k] == psx_rec->counters[j])
+                        {
+                            total_count = agg_psx_rec->counters[i + k + 4] +
+                                psx_rec->counters[j + 4];
</span>                             break;
                         }
<span style="color: #000000;background-color: #ffdddd">-                        else if(agg_psx_rec->counters[i + k + 4] < min)
</span><span style="color: #000000;background-color: #ddffdd">+                    }
+                    if(k == 4) total_count = psx_rec->counters[j + 4];
+
+                    for(k = 0; k < 4; k++)
+                    {
+                        if((agg_psx_rec->counters[i + k + 4] > total_count) ||
+                           ((agg_psx_rec->counters[i + k + 4] == total_count) &&
+                            (agg_psx_rec->counters[i + k] > psx_rec->counters[j])))
</span>                         {
<span style="color: #000000;background-color: #ffdddd">-                            min = agg_psx_rec->counters[i + k + 4];
-                            min_ndx = k;
</span><span style="color: #000000;background-color: #ddffdd">+                            tmp_val[tmp_ndx] = agg_psx_rec->counters[i + k];
+                            tmp_cnt[tmp_ndx] = agg_psx_rec->counters[i + k + 4];
+                            tmp_ndx++;
</span>                         }
<span style="color: #000000;background-color: #ddffdd">+                        else break;
</span>                     }
<span style="color: #000000;background-color: #ffdddd">-                    if(!set && (psx_rec->counters[j + 4] > min))
</span><span style="color: #000000;background-color: #ddffdd">+                    if(tmp_ndx == 4) break;
+
+                    tmp_val[tmp_ndx] = psx_rec->counters[j];
+                    tmp_cnt[tmp_ndx] = psx_rec->counters[j + 4];
+                    tmp_ndx++;
+
+                    while(tmp_ndx != 4)
</span>                     {
<span style="color: #000000;background-color: #ffdddd">-                        agg_psx_rec->counters[i + min_ndx] = psx_rec->counters[j];
-                        agg_psx_rec->counters[i + min_ndx + 4] = psx_rec->counters[j + 4];
</span><span style="color: #000000;background-color: #ddffdd">+                        if(agg_psx_rec->counters[i + k] != psx_rec->counters[j])
+                        {
+                            tmp_val[tmp_ndx] = agg_psx_rec->counters[i + k];
+                            tmp_cnt[tmp_ndx] = agg_psx_rec->counters[i + k + 4];
+                            tmp_ndx++;
+                        }
+                        k++;
</span>                     }
<span style="color: #000000;background-color: #ddffdd">+                    memcpy(&(agg_psx_rec->counters[i]), tmp_val, 4 * sizeof(int64_t));
+                    memcpy(&(agg_psx_rec->counters[i + 4]), tmp_cnt, 4 * sizeof(int64_t));
</span>                 }
                 break;
             case POSIX_STRIDE2_STRIDE:
<span style="color: #aaaaaa">@@ -427,27 +460,80 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
</span>                 }
                 break;
             case POSIX_F_FASTEST_RANK_TIME:
<span style="color: #000000;background-color: #ddffdd">+                if(init_flag)
+                {
+                    /* set fastest rank counters according to root rank. these counters
+                     * will be determined as the aggregation progresses.
+                     */
+                    agg_psx_rec->counters[POSIX_FASTEST_RANK] = psx_rec->base_rec.rank;
+                    agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] = psx_bytes;
+                    agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME] = psx_time;
+                }
+
</span>                 if(psx_time < agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME])
                 {
                     agg_psx_rec->counters[POSIX_FASTEST_RANK] = psx_rec->base_rec.rank;
<span style="color: #000000;background-color: #ffdddd">-                    agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] =
-                        psx_rec->counters[POSIX_BYTES_READ] +
-                        psx_rec->counters[POSIX_BYTES_WRITTEN];
</span><span style="color: #000000;background-color: #ddffdd">+                    agg_psx_rec->counters[POSIX_FASTEST_RANK_BYTES] = psx_bytes;
</span>                     agg_psx_rec->fcounters[POSIX_F_FASTEST_RANK_TIME] = psx_time;
                 }
                 break;
             case POSIX_F_SLOWEST_RANK_TIME:
<span style="color: #000000;background-color: #ddffdd">+                if(init_flag)
+                {
+                    /* set slowest rank counters according to root rank. these counters
+                     * will be determined as the aggregation progresses.
+                     */
+                    agg_psx_rec->counters[POSIX_SLOWEST_RANK] = psx_rec->base_rec.rank;
+                    agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] = psx_bytes;
+                    agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME] = psx_time;
+                }
+
</span>                 if(psx_time > agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME])
                 {
                     agg_psx_rec->counters[POSIX_SLOWEST_RANK] = psx_rec->base_rec.rank;
<span style="color: #000000;background-color: #ffdddd">-                    agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] =
-                        psx_rec->counters[POSIX_BYTES_READ] +
-                        psx_rec->counters[POSIX_BYTES_WRITTEN];
</span><span style="color: #000000;background-color: #ddffdd">+                    agg_psx_rec->counters[POSIX_SLOWEST_RANK_BYTES] = psx_bytes;
</span>                     agg_psx_rec->fcounters[POSIX_F_SLOWEST_RANK_TIME] = psx_time;
                 }
                 break;
<span style="color: #000000;background-color: #ddffdd">+            case POSIX_F_VARIANCE_RANK_TIME:
+                if(init_flag)
+                {
+                    var_time_p->n = 1;
+                    var_time_p->M = psx_time;
+                    var_time_p->S = 0;
+                }
+                else
+                {
+                    old_M = var_time_p->M;
+
+                    var_time_p->n++;
+                    var_time_p->M += (psx_time - var_time_p->M) / var_time_p->n;
+                    var_time_p->S += (psx_time - var_time_p->M) * (psx_time - old_M);
+
+                    agg_psx_rec->fcounters[POSIX_F_VARIANCE_RANK_TIME] =
+                        var_time_p->S / var_time_p->n;
+                }
+                break;
+            case POSIX_F_VARIANCE_RANK_BYTES:
+                if(init_flag)
+                {
+                    var_bytes_p->n = 1;
+                    var_bytes_p->M = psx_bytes;
+                    var_bytes_p->S = 0;
+                }
+                else
+                {
+                    old_M = var_bytes_p->M;
+
+                    var_bytes_p->n++;
+                    var_bytes_p->M += (psx_bytes - var_bytes_p->M) / var_bytes_p->n;
+                    var_bytes_p->S += (psx_bytes - var_bytes_p->M) * (psx_bytes - old_M);
+
+                    agg_psx_rec->fcounters[POSIX_F_VARIANCE_RANK_BYTES] =
+                        var_bytes_p->S / var_bytes_p->n;
+                }
+                break;
</span>             default:
<span style="color: #000000;background-color: #ffdddd">-                /* TODO: variance */
</span>                 agg_psx_rec->fcounters[i] = -1;
                 break;
         }
</code></pre>

<br>
</li>

</div>
<div class='footer' style='margin-top: 10px;'>
<p>

<br>
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/37b1bb02d5c0a9bb80b024890d8ebaf6a73d977b...04063da70950ee779371188c7937956eb89b4273">View it on GitLab</a>.
<br>
You're receiving this email because of your account on xgitlab.cels.anl.gov.
If you'd like to receive fewer emails, you can
adjust your notification settings.

</p>
</div>
</body>
</html>