<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
<html lang="en">
<head>
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<title>
GitLab
</title>
</head>
<body>
<style type="text/css">
img {
max-width: 100%; height: auto;
}
</style>
<div class="content">
<h3>
Philip Carns pushed to branch dev-stdio
at <a href="https://xgitlab.cels.anl.gov/darshan/darshan">darshan / darshan</a>
</h3>
<h4>
Commits:
</h4>
<ul>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/3d7cc3c6996bc031c9876ca274cec36ad8db3ceb">3d7cc3c6</a></strong>
<div>
<span>by Phil Carns</span>
<i>at 2016-05-24T16:54:43-04:00</i>
</div>
<pre class="commit-message" style="white-space: pre-wrap">implement shared reduction for stdio records</pre>
</li>
</ul>
<h4>2 changed files:</h4>
<ul>
<li class="file-stats">
<a href="#620f2ecad2bb6f74b2fcd0134963a841" style="text-decoration: none">
darshan-runtime/lib/darshan-stdio.c
</a>
</li>
<li class="file-stats">
<a href="#ad29afc395839758d41094872298bd0d" style="text-decoration: none">
darshan-stdio-log-format.h
</a>
</li>
</ul>
<h4>Changes:</h4>
<li id="620f2ecad2bb6f74b2fcd0134963a841">
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/3d7cc3c6996bc031c9876ca274cec36ad8db3ceb#diff-0">
<strong>
darshan-runtime/lib/darshan-stdio.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="background: #ffdddd; color: #000000">--- a/darshan-runtime/lib/darshan-stdio.c
</span><span style="background: #ddffdd; color: #000000">+++ b/darshan-runtime/lib/darshan-stdio.c
</span><span style="color: #aaaaaa">@@ -5,7 +5,6 @@
</span> */
/* TODO list (general) for this module:
<span style="background: #ffdddd; color: #000000">- * - implement reduction operator
</span> * - add stdio page to darshan-job-summary
* - add regression test cases for all functions captured here
* - especially the scanf and printf variants with variable arguments
<span style="color: #aaaaaa">@@ -220,6 +219,9 @@ static void stdio_begin_shutdown(void);
</span> static void stdio_get_output_data(MPI_Comm mod_comm, darshan_record_id *shared_recs,
int shared_rec_count, void **stdio_buf, int *stdio_buf_sz);
static void stdio_shutdown(void);
<span style="background: #ddffdd; color: #000000">+static void stdio_record_reduction_op(void* infile_v, void* inoutfile_v,
+ int *len, MPI_Datatype *datatype);
+static int stdio_record_compare(const void* a, const void* b);
</span>
#define STDIO_LOCK() pthread_mutex_lock(&stdio_runtime_mutex)
#define STDIO_UNLOCK() pthread_mutex_unlock(&stdio_runtime_mutex)
<span style="color: #aaaaaa">@@ -1152,14 +1154,168 @@ static void stdio_get_output_data(
</span> void **stdio_buf,
int *stdio_buf_sz)
{
<span style="background: #ddffdd; color: #000000">+ struct stdio_file_runtime *file;
+ int i;
+ struct darshan_stdio_record *red_send_buf = NULL;
+ struct darshan_stdio_record *red_recv_buf = NULL;
+ MPI_Datatype red_type;
+ MPI_Op red_op;
+
</span> assert(stdio_runtime);
<span style="background: #ddffdd; color: #000000">+ /* if there are globally shared files, do a shared file reduction */
+ /* NOTE: the shared file reduction is also skipped if the
+ * DARSHAN_DISABLE_SHARED_REDUCTION environment variable is set.
+ */
+ if(shared_rec_count && !getenv("DARSHAN_DISABLE_SHARED_REDUCTION"))
+ {
+ /* necessary initialization of shared records */
+ for(i = 0; i < shared_rec_count; i++)
+ {
+ HASH_FIND(hlink, stdio_runtime->file_hash, &shared_recs[i],
+ sizeof(darshan_record_id), file);
+ assert(file);
+
+ file->file_record->rank = -1;
+ }
+
+ /* sort the array of files descending by rank so that we get all of the
+ * shared files (marked by rank -1) in a contiguous portion at end
+ * of the array
+ */
+ qsort(stdio_runtime->file_record_array, stdio_runtime->file_array_ndx,
+ sizeof(struct darshan_stdio_record), stdio_record_compare);
+
+ /* make *send_buf point to the shared files at the end of sorted array */
+ red_send_buf =
+ &(stdio_runtime->file_record_array[stdio_runtime->file_array_ndx-shared_rec_count]);
+
+ /* allocate memory for the reduction output on rank 0 */
+ if(my_rank == 0)
+ {
+ red_recv_buf = malloc(shared_rec_count * sizeof(struct darshan_stdio_record));
+ if(!red_recv_buf)
+ {
+ return;
+ }
+ }
+
+ /* construct a datatype for a STDIO file record. This is serving no purpose
+ * except to make sure we can do a reduction on proper boundaries
+ */
+ DARSHAN_MPI_CALL(PMPI_Type_contiguous)(sizeof(struct darshan_stdio_record),
+ MPI_BYTE, &red_type);
+ DARSHAN_MPI_CALL(PMPI_Type_commit)(&red_type);
+
+ /* register a STDIO file record reduction operator */
+ DARSHAN_MPI_CALL(PMPI_Op_create)(stdio_record_reduction_op, 1, &red_op);
+
+ /* reduce shared STDIO file records */
+ DARSHAN_MPI_CALL(PMPI_Reduce)(red_send_buf, red_recv_buf,
+ shared_rec_count, red_type, red_op, 0, mod_comm);
+
+ /* clean up reduction state */
+ if(my_rank == 0)
+ {
+ int tmp_ndx = stdio_runtime->file_array_ndx - shared_rec_count;
+ memcpy(&(stdio_runtime->file_record_array[tmp_ndx]), red_recv_buf,
+ shared_rec_count * sizeof(struct darshan_stdio_record));
+ free(red_recv_buf);
+ }
+ else
+ {
+ stdio_runtime->file_array_ndx -= shared_rec_count;
+ }
+
+ DARSHAN_MPI_CALL(PMPI_Type_free)(&red_type);
+ DARSHAN_MPI_CALL(PMPI_Op_free)(&red_op);
+ }
+
</span> *stdio_buf = (void *)(stdio_runtime->file_record_array);
*stdio_buf_sz = stdio_runtime->file_array_ndx * sizeof(struct darshan_stdio_record);
return;
}
<span style="background: #ddffdd; color: #000000">+/* compare function for sorting file records by descending rank */
+static int stdio_record_compare(const void* a_p, const void* b_p)
+{
+ const struct darshan_stdio_record* a = a_p;
+ const struct darshan_stdio_record* b = b_p;
+
+ if(a->rank < b->rank)
+ return 1;
+ if(a->rank > b->rank)
+ return -1;
+
+ return 0;
+}
+
+static void stdio_record_reduction_op(void* infile_v, void* inoutfile_v,
+ int *len, MPI_Datatype *datatype)
+{
+ struct darshan_stdio_record tmp_file;
+ struct darshan_stdio_record *infile = infile_v;
+ struct darshan_stdio_record *inoutfile = inoutfile_v;
+ int i, j;
+
+ assert(stdio_runtime);
+
+ for(i=0; i<*len; i++)
+ {
+ memset(&tmp_file, 0, sizeof(struct darshan_stdio_record));
+ tmp_file.f_id = infile->f_id;
+ tmp_file.rank = -1;
+
+ /* sum */
+ for(j=STDIO_OPENS; j<=STDIO_BYTES_READ; j++)
+ {
+ tmp_file.counters[j] = infile->counters[j] + inoutfile->counters[j];
+ }
+
+ /* max */
+ for(j=STDIO_MAX_BYTE_READ; j<=STDIO_MAX_BYTE_WRITTEN; j++)
+ {
+ if(infile->counters[j] > inoutfile->counters[j])
+ tmp_file.counters[j] = infile->counters[j];
+ else
+ tmp_file.counters[j] = inoutfile->counters[j];
+ }
+
+ /* sum */
+ for(j=STDIO_F_META_TIME; j<=STDIO_F_READ_TIME; j++)
+ {
+ tmp_file.fcounters[j] = infile->fcounters[j] + inoutfile->fcounters[j];
+ }
+
+ /* min non-zero (if available) value */
+ for(j=STDIO_F_OPEN_START_TIMESTAMP; j<=STDIO_F_READ_START_TIMESTAMP; j++)
+ {
+ if((infile->fcounters[j] < inoutfile->fcounters[j] &&
+ infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0)
+ tmp_file.fcounters[j] = infile->fcounters[j];
+ else
+ tmp_file.fcounters[j] = inoutfile->fcounters[j];
+ }
+
+ /* max */
+ for(j=STDIO_F_OPEN_END_TIMESTAMP; j<=STDIO_F_READ_END_TIMESTAMP; j++)
+ {
+ if(infile->fcounters[j] > inoutfile->fcounters[j])
+ tmp_file.fcounters[j] = infile->fcounters[j];
+ else
+ tmp_file.fcounters[j] = inoutfile->fcounters[j];
+ }
+
+ /* update pointers */
+ *inoutfile = tmp_file;
+ inoutfile++;
+ infile++;
+ }
+
+ return;
+}
+
</span> static void stdio_shutdown()
{
struct stdio_file_runtime_ref *ref, *tmp;
</code></pre>
<br>
</li>
<li id="ad29afc395839758d41094872298bd0d">
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/3d7cc3c6996bc031c9876ca274cec36ad8db3ceb#diff-1">
<strong>
darshan-stdio-log-format.h
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="background: #ffdddd; color: #000000">--- a/darshan-stdio-log-format.h
</span><span style="background: #ddffdd; color: #000000">+++ b/darshan-stdio-log-format.h
</span><span style="color: #aaaaaa">@@ -13,48 +13,48 @@
</span> #define STDIO_COUNTERS \
/* count of fopens */\
X(STDIO_OPENS) \
<span style="background: #ffdddd; color: #000000">- /* maximum byte (offset) written */\
- X(STDIO_MAX_BYTE_WRITTEN) \
- /* total bytes written */ \
- X(STDIO_BYTES_WRITTEN) \
- /* number of writes */ \
- X(STDIO_WRITES) \
- /* maximum byte (offset) read */\
- X(STDIO_MAX_BYTE_READ) \
- /* total bytes read */ \
- X(STDIO_BYTES_READ) \
</span> /* number of reads */ \
X(STDIO_READS) \
<span style="background: #ddffdd; color: #000000">+ /* number of writes */ \
+ X(STDIO_WRITES) \
</span> /* count of seeks */\
X(STDIO_SEEKS) \
/* count of flushes */\
X(STDIO_FLUSHES) \
<span style="background: #ddffdd; color: #000000">+ /* total bytes written */ \
+ X(STDIO_BYTES_WRITTEN) \
+ /* total bytes read */ \
+ X(STDIO_BYTES_READ) \
+ /* maximum byte (offset) read */\
+ X(STDIO_MAX_BYTE_READ) \
+ /* maximum byte (offset) written */\
+ X(STDIO_MAX_BYTE_WRITTEN) \
</span> /* end of counters */\
X(STDIO_NUM_INDICES)
#define STDIO_F_COUNTERS \
<span style="background: #ddffdd; color: #000000">+ /* cumulative meta time */\
+ X(STDIO_F_META_TIME) \
+ /* cumulative write time */\
+ X(STDIO_F_WRITE_TIME) \
+ /* cumulative read time */\
+ X(STDIO_F_READ_TIME) \
</span> /* timestamp of first open */\
X(STDIO_F_OPEN_START_TIMESTAMP) \
<span style="background: #ffdddd; color: #000000">- /* timestamp of last open completion */\
- X(STDIO_F_OPEN_END_TIMESTAMP) \
</span> /* timestamp of first close */\
X(STDIO_F_CLOSE_START_TIMESTAMP) \
<span style="background: #ffdddd; color: #000000">- /* timestamp of last close completion */\
- X(STDIO_F_CLOSE_END_TIMESTAMP) \
</span> /* timestamp of first write */\
X(STDIO_F_WRITE_START_TIMESTAMP) \
<span style="background: #ffdddd; color: #000000">- /* timestamp of last write completion */\
- X(STDIO_F_WRITE_END_TIMESTAMP) \
</span> /* timestamp of first read */\
X(STDIO_F_READ_START_TIMESTAMP) \
<span style="background: #ddffdd; color: #000000">+ /* timestamp of last open completion */\
+ X(STDIO_F_OPEN_END_TIMESTAMP) \
+ /* timestamp of last close completion */\
+ X(STDIO_F_CLOSE_END_TIMESTAMP) \
+ /* timestamp of last write completion */\
+ X(STDIO_F_WRITE_END_TIMESTAMP) \
</span> /* timestamp of last read completion */\
X(STDIO_F_READ_END_TIMESTAMP) \
<span style="background: #ffdddd; color: #000000">- /* cumulative meta time */\
- X(STDIO_F_META_TIME) \
- /* cumulative write time */\
- X(STDIO_F_WRITE_TIME) \
- /* cumulative read time */\
- X(STDIO_F_READ_TIME) \
</span> /* end of counters */\
X(STDIO_F_NUM_INDICES)
</code></pre>
<br>
</li>
</div>
<div class="footer" style="margin-top: 10px">
<p style="color: #777; font-size: small">
—
<br>
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/3d7cc3c6996bc031c9876ca274cec36ad8db3ceb">View it on GitLab</a>.
<br>
You're receiving this email because of your account on xgitlab.cels.anl.gov.
If you'd like to receive fewer emails, you can
adjust your notification settings.
</p>
</div>
</body>
</html>