<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
<html lang="en">
<head>
<meta content="text/html; charset=utf-8" http-equiv="Content-Type">
<title>
GitLab
</title>
</head>
<body>
<style type="text/css">
img {
max-width: 100%; height: auto;
}
</style>
<div class="content">
<h3>
Glenn K. Lockwood pushed to branch lustre-mod
at <a href="https://xgitlab.cels.anl.gov/darshan/darshan">darshan / darshan</a>
</h3>
<h4>
Commits:
</h4>
<ul>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/6c828abba59715d18419c21a923114e9bb7d8fc1">6c828abb</a></strong>
<div>
<span>by Glenn K. Lockwood</span>
<i>at 2016-06-03T10:24:19-07:00</i>
</div>
<pre class="commit-message" style="white-space: pre-wrap">forgot to add darshan-lustre.h, which is needed to hook together the module and darshan-core-stub.c</pre>
</li>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/3193b2e96fa635e8842615b7c7ccfe13e81f17fc">3193b2e9</a></strong>
<div>
<span>by Glenn K. Lockwood</span>
<i>at 2016-06-03T10:34:48-07:00</i>
</div>
<pre class="commit-message" style="white-space: pre-wrap">make ioctl return the full ost map in test app</pre>
</li>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/8aa23fa33c9a9bbc437c44bd6fd6c75785d03b00">8aa23fa3</a></strong>
<div>
<span>by Glenn K. Lockwood</span>
<i>at 2016-06-03T10:37:00-07:00</i>
</div>
<pre class="commit-message" style="white-space: pre-wrap">partly working lustre module supporting ost mapping; shared file agg still doesn't work, and log parsing doesn't either</pre>
</li>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/dc1a9eb90c044ecb054cc50e7400c2bc3a9080e7">dc1a9eb9</a></strong>
<div>
<span>by Glenn K. Lockwood</span>
<i>at 2016-06-03T10:39:14-07:00</i>
</div>
<pre class="commit-message" style="white-space: pre-wrap">add rank shuffling to non-MPI tester</pre>
</li>
</ul>
<h4>6 changed files:</h4>
<ul>
<li class="file-stats">
<a href="#620f2ecad2bb6f74b2fcd0134963a841" style="text-decoration: none">
darshan-lustre-log-format.h
</a>
</li>
<li class="file-stats">
<a href="#ad29afc395839758d41094872298bd0d" style="text-decoration: none">
<span class="new-file" style="color: #090">
+
darshan-runtime/darshan-lustre.h
</span>
</a>
</li>
<li class="file-stats">
<a href="#c0b0bf6d71bc5fc7e6d50d69c8aa2413" style="text-decoration: none">
darshan-runtime/lib/darshan-lustre.c
</a>
</li>
<li class="file-stats">
<a href="#2ace16276b68c7d5aef163bb260bd9f7" style="text-decoration: none">
darshan-test/2.x/llapi-perf.c
</a>
</li>
<li class="file-stats">
<a href="#4dfe7e78b540daa005ea4b5f0458c90d" style="text-decoration: none">
darshan-test/lustre/Makefile
</a>
</li>
<li class="file-stats">
<a href="#172f4bf093caeeba24730a1adff8f15a" style="text-decoration: none">
darshan-test/lustre/darshan-core-stub.c
</a>
</li>
</ul>
<h4>Changes:</h4>
<li id="620f2ecad2bb6f74b2fcd0134963a841">
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/a61ebca2580f403acfd8c472d2fbb1356003a6b3...dc1a9eb90c044ecb054cc50e7400c2bc3a9080e7#diff-0">
<strong>
darshan-lustre-log-format.h
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="background: #ffdddd; color: #000000">--- a/darshan-lustre-log-format.h
</span><span style="background: #ddffdd; color: #000000">+++ b/darshan-lustre-log-format.h
</span><span style="color: #aaaaaa">@@ -16,12 +16,12 @@
</span> X(LUSTRE_OSTS) \
/* number of MDTs for file system */\
X(LUSTRE_MDTS) \
<span style="background: #ddffdd; color: #000000">+ /* index of first OST for file */\
+ X(LUSTRE_STRIPE_OFFSET) \
</span> /* bytes per stripe for file */\
X(LUSTRE_STRIPE_SIZE) \
/* number of stripes (OSTs) for file */\
X(LUSTRE_STRIPE_WIDTH) \
<span style="background: #ffdddd; color: #000000">- /* index of first OST for file */\
- X(LUSTRE_STRIPE_OFFSET) \
</span> /* end of counters */\
X(LUSTRE_NUM_INDICES)
<span style="color: #aaaaaa">@@ -44,6 +44,7 @@ struct darshan_lustre_record
</span> darshan_record_id rec_id;
int64_t rank;
int64_t counters[LUSTRE_NUM_INDICES];
<span style="background: #ddffdd; color: #000000">+ int64_t ost_ids[1];
</span> };
#endif /* __DARSHAN_LUSTRE_LOG_FORMAT_H */
</code></pre>
<br>
</li>
<li id="ad29afc395839758d41094872298bd0d">
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/a61ebca2580f403acfd8c472d2fbb1356003a6b3...dc1a9eb90c044ecb054cc50e7400c2bc3a9080e7#diff-1">
<strong>
darshan-runtime/darshan-lustre.h
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="background: #ffdddd; color: #000000">--- /dev/null
</span><span style="background: #ddffdd; color: #000000">+++ b/darshan-runtime/darshan-lustre.h
</span><span style="color: #aaaaaa">@@ -0,0 +1,19 @@
</span><span style="background: #ddffdd; color: #000000">+struct lustre_record_runtime
+{
+ struct darshan_lustre_record *record;
+ size_t record_size;
+ UT_hash_handle hlink;
+};
+
+struct lustre_runtime
+{
+ int record_count; /* number of records defined */
+ size_t record_buffer_max; /* size of the allocated buffer pointed to by record_buffer */
+ size_t record_buffer_used; /* size of the allocated buffer actually used */
+ void *next_free_record; /* pointer to end of record_buffer */
+ void *record_buffer; /* buffer in which records are created */
+ struct lustre_record_runtime *record_runtime_array;
+ struct lustre_record_runtime *record_runtime_hash;
+};
+
+
</span></code></pre>
<br>
</li>
<li id="c0b0bf6d71bc5fc7e6d50d69c8aa2413">
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/a61ebca2580f403acfd8c472d2fbb1356003a6b3...dc1a9eb90c044ecb054cc50e7400c2bc3a9080e7#diff-2">
<strong>
darshan-runtime/lib/darshan-lustre.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="background: #ffdddd; color: #000000">--- a/darshan-runtime/lib/darshan-lustre.c
</span><span style="background: #ddffdd; color: #000000">+++ b/darshan-runtime/lib/darshan-lustre.c
</span><span style="color: #aaaaaa">@@ -25,28 +25,9 @@
</span>
#include "darshan.h"
#include "darshan-dynamic.h"
<span style="background: #ddffdd; color: #000000">+#include "darshan-lustre.h"
</span>
<span style="background: #ffdddd; color: #000000">-struct lustre_record_runtime
-{
- struct darshan_lustre_record *record;
- UT_hash_handle hlink;
-};
-
-/* we just use a simple array for storing records. the POSIX module
- * only calls into the Lustre module for new records, so we will never
- * have to search for an existing Lustre record (assuming the Lustre
- * data remains immutable as it is now).
- */
-struct lustre_runtime
-{
- struct darshan_lustre_record *record_array;
- struct lustre_record_runtime *record_runtime_array;
- int record_array_size;
- int record_array_ndx;
- struct lustre_record_runtime *record_hash;
-};
-
-static struct lustre_runtime *lustre_runtime = NULL;
</span><span style="background: #ddffdd; color: #000000">+struct lustre_runtime *lustre_runtime = NULL;
</span> static pthread_mutex_t lustre_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int instrumentation_disabled = 0;
static int my_rank = -1;
<span style="color: #aaaaaa">@@ -63,20 +44,48 @@ static void lustre_record_reduction_op(void* infile_v, void* inoutfile_v,
</span>
#define LUSTRE_LOCK() pthread_mutex_lock(&lustre_runtime_mutex)
#define LUSTRE_UNLOCK() pthread_mutex_unlock(&lustre_runtime_mutex)
<span style="background: #ddffdd; color: #000000">+#define LUSTRE_RECORD_SIZE( osts ) ( sizeof(struct darshan_lustre_record) + sizeof(int64_t) * (osts - 1) )
</span>
void darshan_instrument_lustre_file(const char* filepath, int fd)
{
struct lustre_record_runtime *rec_rt;
<span style="background: #ddffdd; color: #000000">+ struct darshan_lustre_record *rec;
</span> struct darshan_fs_info fs_info;
darshan_record_id rec_id;
int limit_flag;
<span style="background: #ddffdd; color: #000000">+ int i;
+ struct lov_user_md *lum;
+ size_t lumsize = sizeof(struct lov_user_md) +
+ LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data);
+ size_t rec_size;
</span>
LUSTRE_LOCK();
/* make sure the lustre module is already initialized */
lustre_runtime_initialize();
<span style="background: #ffdddd; color: #000000">- /* if the array is full, we just back out */
- limit_flag = (lustre_runtime->record_array_ndx >= lustre_runtime->record_array_size);
</span><span style="background: #ddffdd; color: #000000">+ /* if we can't issue ioctl, we have no counter data at all */
+ if ( (lum = calloc(1, lumsize)) == NULL )
+ return;
+
+ /* find out the OST count of this file so we can allocate memory */
+ lum->lmm_magic = LOV_USER_MAGIC;
+ lum->lmm_stripe_count = LOV_MAX_STRIPE_COUNT;
+
+ /* -1 means ioctl failed, likely because file isn't on Lustre */
+ if ( ioctl( fd, LL_IOC_LOV_GETSTRIPE, (void *)lum ) == -1 )
+ {
+ free(lum);
+ return;
+ }
+
+ rec_size = LUSTRE_RECORD_SIZE( lum->lmm_stripe_count );
+
+ {
+ /* broken out for clarity */
+ void *end_of_new_record = (char*)lustre_runtime->next_free_record + rec_size;
+ void *end_of_rec_buffer = (char*)lustre_runtime->record_buffer + lustre_runtime->record_buffer_max;
+ limit_flag = ( end_of_new_record > end_of_rec_buffer );
+ }
</span>
/* register a Lustre file record with Darshan */
fs_info.fs_type = -1;
<span style="color: #aaaaaa">@@ -92,21 +101,20 @@ void darshan_instrument_lustre_file(const char* filepath, int fd)
</span> /* if record id is 0, darshan has no more memory for instrumenting */
if(rec_id == 0)
{
<span style="background: #ddffdd; color: #000000">+ free(lum);
</span> LUSTRE_UNLOCK();
return;
}
/* search the hash table for this file record, and initialize if not found */
<span style="background: #ffdddd; color: #000000">- HASH_FIND(hlink, lustre_runtime->record_hash, &rec_id, sizeof(darshan_record_id), rec_rt );
</span><span style="background: #ddffdd; color: #000000">+ HASH_FIND(hlink, lustre_runtime->record_runtime_hash, &rec_id, sizeof(darshan_record_id), rec_rt );
</span> if ( !rec_rt ) {
<span style="background: #ffdddd; color: #000000">- struct darshan_lustre_record *rec;
- struct lov_user_md *lum;
- size_t lumsize = sizeof(struct lov_user_md) +
- LOV_MAX_STRIPE_COUNT * sizeof(struct lov_user_ost_data);
-
</span> /* allocate a new lustre record and append it to the array */
<span style="background: #ffdddd; color: #000000">- rec_rt = &(lustre_runtime->record_runtime_array[lustre_runtime->record_array_ndx]);
- rec_rt->record = &(lustre_runtime->record_array[lustre_runtime->record_array_ndx]);
</span><span style="background: #ddffdd; color: #000000">+ rec_rt = &(lustre_runtime->record_runtime_array[lustre_runtime->record_count]);
+ rec_rt->record = lustre_runtime->next_free_record;
+ rec_rt->record_size = rec_size;
+ lustre_runtime->next_free_record = (char*)(lustre_runtime->next_free_record) + rec_size;
+ lustre_runtime->record_buffer_used += rec_size;
</span> rec = rec_rt->record;
rec->rec_id = rec_id;
rec->rank = my_rank;
<span style="color: #aaaaaa">@@ -125,24 +133,16 @@ void darshan_instrument_lustre_file(const char* filepath, int fd)
</span> rec->counters[LUSTRE_MDTS] = -1;
}
<span style="background: #ffdddd; color: #000000">- if ( (lum = calloc(1, lumsize)) != NULL ) {
- lum->lmm_magic = LOV_USER_MAGIC;
- /* don't care about the return code for ioctl */
- ioctl( fd, LL_IOC_LOV_GETSTRIPE, (void *)lum );
- rec->counters[LUSTRE_STRIPE_SIZE] = lum->lmm_stripe_size;
- rec->counters[LUSTRE_STRIPE_WIDTH] = lum->lmm_stripe_count;
- rec->counters[LUSTRE_STRIPE_OFFSET] = 0; /* this currently doesn't work; lum->lmm_objects[0].l_ost_idx isn't being populated */
- /* TODO: add explicit list of OSTs */
- free(lum);
- }
- else
- {
- rec->counters[LUSTRE_STRIPE_SIZE] = -1;
- rec->counters[LUSTRE_STRIPE_WIDTH] = -1;
- rec->counters[LUSTRE_STRIPE_OFFSET] = -1;
- }
- HASH_ADD(hlink, lustre_runtime->record_hash, record->rec_id, sizeof(darshan_record_id), rec_rt);
- lustre_runtime->record_array_ndx++;
</span><span style="background: #ddffdd; color: #000000">+ rec->counters[LUSTRE_STRIPE_SIZE] = lum->lmm_stripe_size;
+ rec->counters[LUSTRE_STRIPE_WIDTH] = lum->lmm_stripe_count;
+ rec->counters[LUSTRE_STRIPE_OFFSET] = lum->lmm_stripe_offset;
+ for ( i = 0; i < lum->lmm_stripe_count; i++ )
+ rec->ost_ids[i] = lum->lmm_objects[i].l_ost_idx;
+ free(lum);
+
+ HASH_ADD(hlink, lustre_runtime->record_runtime_hash, record->rec_id, sizeof(darshan_record_id), rec_rt);
+
+ lustre_runtime->record_count++;
</span> }
LUSTRE_UNLOCK();
<span style="color: #aaaaaa">@@ -152,6 +152,7 @@ void darshan_instrument_lustre_file(const char* filepath, int fd)
</span> static void lustre_runtime_initialize()
{
int mem_limit;
<span style="background: #ddffdd; color: #000000">+ int max_records;
</span> struct darshan_module_funcs lustre_mod_fns =
{
.begin_shutdown = &lustre_begin_shutdown,
<span style="color: #aaaaaa">@@ -180,30 +181,34 @@ static void lustre_runtime_initialize()
</span> return;
memset(lustre_runtime, 0, sizeof(*lustre_runtime));
<span style="background: #ffdddd; color: #000000">- /* allocate array of Lustre records according to the amount of memory
- * assigned by Darshan
- */
- lustre_runtime->record_array_size = mem_limit / sizeof(struct darshan_lustre_record);
-
- lustre_runtime->record_array = malloc(lustre_runtime->record_array_size *
- sizeof(struct darshan_lustre_record));
- if(!lustre_runtime->record_array)
</span><span style="background: #ddffdd; color: #000000">+ /* allocate the full size of the memory limit we are given */
+ lustre_runtime->record_buffer= malloc(mem_limit);
+ if(!lustre_runtime->record_buffer)
</span> {
<span style="background: #ffdddd; color: #000000">- lustre_runtime->record_array_size = 0;
</span><span style="background: #ddffdd; color: #000000">+ lustre_runtime->record_buffer_max = 0;
</span> return;
}
<span style="background: #ffdddd; color: #000000">- memset(lustre_runtime->record_array, 0, lustre_runtime->record_array_size *
- sizeof(struct darshan_lustre_record));
-
- lustre_runtime->record_runtime_array = malloc(lustre_runtime->record_array_size *
- sizeof(struct lustre_record_runtime));
</span><span style="background: #ddffdd; color: #000000">+ lustre_runtime->record_buffer_max = mem_limit;
+ lustre_runtime->next_free_record = lustre_runtime->record_buffer;
+ memset(lustre_runtime->record_buffer, 0, lustre_runtime->record_buffer_max);
+
+ /* Allocate array of Lustre runtime data. We calculate the maximum possible
+ * number of records that will fit into mem_limit by assuming that each
+ * record has the minimum possible OST count, then allocate that many
+ * runtime records. record_buffer will always run out of memory before
+ * we overflow record_runtime_array.
+ */
+ max_records = mem_limit / sizeof(struct darshan_lustre_record);
+ lustre_runtime->record_runtime_array =
+ malloc( max_records * sizeof(struct lustre_record_runtime));
</span> if(!lustre_runtime->record_runtime_array)
{
<span style="background: #ffdddd; color: #000000">- lustre_runtime->record_array_size = 0;
</span><span style="background: #ddffdd; color: #000000">+ lustre_runtime->record_buffer_max = 0;
+ free( lustre_runtime->record_buffer );
</span> return;
}
<span style="background: #ffdddd; color: #000000">- memset(lustre_runtime->record_runtime_array, 0, lustre_runtime->record_array_size *
- sizeof(struct lustre_record_runtime));
</span><span style="background: #ddffdd; color: #000000">+ memset(lustre_runtime->record_runtime_array, 0,
+ max_records * sizeof(struct lustre_record_runtime));
</span>
return;
}
<span style="color: #aaaaaa">@@ -224,6 +229,7 @@ static void lustre_begin_shutdown(void)
</span> return;
}
<span style="background: #ddffdd; color: #000000">+/* XXX - still need to update 5/26/2016 */
</span> static void lustre_get_output_data(
MPI_Comm mod_comm,
darshan_record_id *shared_recs,
<span style="color: #aaaaaa">@@ -249,7 +255,7 @@ static void lustre_get_output_data(
</span> /* necessary initialization of shared records */
for(i = 0; i < shared_rec_count; i++)
{
<span style="background: #ffdddd; color: #000000">- HASH_FIND(hlink, lustre_runtime->record_hash, &shared_recs[i],
</span><span style="background: #ddffdd; color: #000000">+ HASH_FIND(hlink, lustre_runtime->record_runtime_hash, &shared_recs[i],
</span> sizeof(darshan_record_id), file);
assert(file);
<span style="color: #aaaaaa">@@ -260,14 +266,14 @@ static void lustre_get_output_data(
</span> * shared files (marked by rank -1) in a contiguous portion at end
* of the array
*/
<span style="background: #ffdddd; color: #000000">- qsort(lustre_runtime->record_array, lustre_runtime->record_array_ndx,
- sizeof(struct darshan_lustre_record), lustre_record_compare);
</span><span style="background: #ddffdd; color: #000000">+ sort_lustre_records();
</span>
<span style="background: #ffdddd; color: #000000">- /* make *send_buf point to the shared files at the end of sorted array */
</span><span style="background: #ddffdd; color: #000000">+ /* make *send_buf point to the shared files at the end of sorted array
</span> red_send_buf =
<span style="background: #ffdddd; color: #000000">- &(lustre_runtime->record_array[lustre_runtime->record_array_ndx-shared_rec_count]);
</span><span style="background: #ddffdd; color: #000000">+ &(lustre_runtime->record_runtime_array[lustre_runtime->record_count-shared_rec_count]);
+*******************************************************************************/
</span>
<span style="background: #ffdddd; color: #000000">- /* allocate memory for the reduction output on rank 0 */
</span><span style="background: #ddffdd; color: #000000">+ /* allocate memory for the reduction output on rank 0
</span> if(my_rank == 0)
{
red_recv_buf = malloc(shared_rec_count * sizeof(struct darshan_lustre_record));
<span style="color: #aaaaaa">@@ -283,26 +289,27 @@ static void lustre_get_output_data(
</span> DARSHAN_MPI_CALL(PMPI_Op_create)(lustre_record_reduction_op, 1, &red_op);
DARSHAN_MPI_CALL(PMPI_Reduce)(red_send_buf, red_recv_buf,
shared_rec_count, red_type, red_op, 0, mod_comm);
<span style="background: #ddffdd; color: #000000">+*******************************************************************************/
</span>
<span style="background: #ffdddd; color: #000000">- /* clean up reduction state */
</span><span style="background: #ddffdd; color: #000000">+ /* clean up reduction state
</span> if(my_rank == 0)
{
<span style="background: #ffdddd; color: #000000">- int tmp_ndx = lustre_runtime->record_array_ndx - shared_rec_count;
- memcpy(&(lustre_runtime->record_array[tmp_ndx]), red_recv_buf,
</span><span style="background: #ddffdd; color: #000000">+ memcpy(&(lustre_runtime->record_array[lustre_runtime->record_count - shared_rec_count]), red_recv_buf,
</span> shared_rec_count * sizeof(struct darshan_lustre_record));
free(red_recv_buf);
}
else
{
<span style="background: #ffdddd; color: #000000">- lustre_runtime->record_array_ndx -= shared_rec_count;
</span><span style="background: #ddffdd; color: #000000">+ lustre_runtime->record_count -= shared_rec_count;
</span> }
DARSHAN_MPI_CALL(PMPI_Type_free)(&red_type);
DARSHAN_MPI_CALL(PMPI_Op_free)(&red_op);
<span style="background: #ddffdd; color: #000000">+*******************************************************************************/
</span> }
<span style="background: #ffdddd; color: #000000">- *lustre_buf = (void *)(lustre_runtime->record_array);
- *lustre_buf_sz = lustre_runtime->record_array_ndx * sizeof(struct darshan_lustre_record);
</span><span style="background: #ddffdd; color: #000000">+ *lustre_buf = (void *)(lustre_runtime->record_buffer);
+ *lustre_buf_sz = lustre_runtime->record_buffer_used;
</span>
return;
}
<span style="color: #aaaaaa">@@ -311,9 +318,9 @@ static void lustre_shutdown(void)
</span> {
assert(lustre_runtime);
<span style="background: #ffdddd; color: #000000">- HASH_CLEAR(hlink, lustre_runtime->record_hash);
- free(lustre_runtime->record_array);
</span><span style="background: #ddffdd; color: #000000">+ HASH_CLEAR(hlink, lustre_runtime->record_runtime_hash);
</span> free(lustre_runtime->record_runtime_array);
<span style="background: #ddffdd; color: #000000">+ free(lustre_runtime->record_buffer);
</span> free(lustre_runtime);
lustre_runtime = NULL;
<span style="color: #aaaaaa">@@ -323,17 +330,80 @@ static void lustre_shutdown(void)
</span> /* compare function for sorting file records by descending rank */
static int lustre_record_compare(const void* a_p, const void* b_p)
{
<span style="background: #ffdddd; color: #000000">- const struct darshan_lustre_record* a = a_p;
- const struct darshan_lustre_record* b = b_p;
</span><span style="background: #ddffdd; color: #000000">+ const struct lustre_record_runtime* a = a_p;
+ const struct lustre_record_runtime* b = b_p;
</span>
<span style="background: #ffdddd; color: #000000">- if(a->rank < b->rank)
</span><span style="background: #ddffdd; color: #000000">+ if(a->record->rank < b->record->rank)
</span> return 1;
<span style="background: #ffdddd; color: #000000">- if(a->rank > b->rank)
</span><span style="background: #ddffdd; color: #000000">+ if(a->record->rank > b->record->rank)
</span> return -1;
return 0;
}
<span style="background: #ddffdd; color: #000000">+/*
+ * Sort the record_runtimes and records by MPI rank to facilitate shared redux.
+ * This requires craftiness and additional heap utilization because the records
+ * (but not record_runtimes) have variable size. Currently has to temporarily
+ * duplicate the entire record_buffer; there is room for more memory-efficient
+ * optimization if this becomes a scalability issue.
+ */
+int sort_lustre_records()
+{
+ int i;
+ struct darshan_lustre_record *rec;
+ struct lustre_record_runtime *rec_rt, *tmp_rec_rt;
+ char *new_buf, *p;
+
+ /* Create a new buffer to store an entire replica of record_buffer. Since
+ * we know the exact size of record_buffer's useful data at this point, we
+ * can allocate the exact amount we need instead of record_buffer_max */
+ new_buf = malloc(lustre_runtime->record_buffer_used);
+ p = new_buf;
+ if ( !new_buf )
+ return 1;
+
+ /* qsort breaks the hash table, so delete it now to free its memory buffers
+ * and prevent later confusion */
+ HASH_ITER( hlink, lustre_runtime->record_runtime_hash, rec_rt, tmp_rec_rt )
+ HASH_DELETE( hlink, lustre_runtime->record_runtime_hash, rec_rt );
+
+ /* sort the runtime records, which is has fixed-length elements */
+ qsort(
+ lustre_runtime->record_runtime_array,
+ lustre_runtime->record_count,
+ sizeof(struct lustre_record_runtime),
+ lustre_record_compare
+ );
+
+ /* rebuild the hash and array with the qsorted runtime records */
+ for ( i = 0; i < lustre_runtime->record_count; i++ )
+ {
+ rec_rt = &(lustre_runtime->record_runtime_array[i]);
+ HASH_ADD(hlink, lustre_runtime->record_runtime_hash, record->rec_id, sizeof(darshan_record_id), rec_rt );
+ }
+
+ /* create reordered record buffer, then copy it back in place */
+ for ( i = 0; i < lustre_runtime->record_count; i++ )
+ {
+ rec_rt = &(lustre_runtime->record_runtime_array[i]);
+ memcpy( p, rec_rt->record, rec_rt->record_size );
+ /* fix record pointers within each runtime record too - pre-emptively
+ * point them at where they will live in record_buffer after we memcpy
+ * below */
+ rec_rt->record = (struct darshan_lustre_record *)((char*)(lustre_runtime->record_buffer) + (p - new_buf));
+
+ p += rec_rt->record_size;
+ }
+ memcpy(
+ lustre_runtime->record_buffer,
+ new_buf,
+ lustre_runtime->record_buffer_used );
+
+ free(new_buf);
+ return 0;
+}
+
</span> /* this is just boilerplate reduction code that isn't currently used */
static void lustre_record_reduction_op(void* infile_v, void* inoutfile_v,
int *len, MPI_Datatype *datatype)
<span style="color: #aaaaaa">@@ -374,6 +444,71 @@ static void lustre_record_reduction_op(void* infile_v, void* inoutfile_v,
</span> }
/*
<span style="background: #ddffdd; color: #000000">+ * Dump the memory structure of our records and runtime records
+ */
+void print_lustre_runtime( void )
+{
+ int i, j;
+ struct darshan_lustre_record *rec;
+
+ /* print what we just loaded */
+ for ( i = 0; i < lustre_runtime->record_count; i++ )
+ {
+ rec = (lustre_runtime->record_runtime_array[i]).record;
+ printf( "File %2d\n", i );
+ for ( j = 0; j < LUSTRE_NUM_INDICES; j++ )
+ {
+ printf( " Counter %2d: %10ld, addr %ld\n",
+ j,
+ rec->counters[j],
+ (char*)(&(rec->counters[j])) - (char*)(lustre_runtime->record_buffer) );
+ }
+ for ( j = 0; j < rec->counters[LUSTRE_STRIPE_WIDTH]; j++ )
+ {
+ printf( " Stripe %2d: %10ld, addr %ld\n",
+ j,
+ rec->ost_ids[j],
+ (char*)(&(rec->ost_ids[j])) - (char*)(lustre_runtime->record_buffer) );
+ }
+ }
+ return;
+}
+
+/*
+ * Dump the order in which records appear in memory
+ */
+void print_array( void )
+{
+ int i;
+ struct lustre_record_runtime *rec_rt;
+ printf("*** DUMPING RECORD LIST BY ARRAY SEQUENCE\n");
+ for ( i = 0; i < lustre_runtime->record_count; i++ )
+ {
+ rec_rt = &(lustre_runtime->record_runtime_array[i]);
+ printf( "*** record %d rank %d osts %d\n",
+ rec_rt->record->rec_id,
+ rec_rt->record->rank,
+ rec_rt->record->counters[LUSTRE_STRIPE_WIDTH]);
+ }
+}
+void print_hash( void )
+{
+ struct lustre_record_runtime *rec_rt, *tmp_rec_rt;
+ printf("*** DUMPING RECORD LIST BY HASH SEQUENCE\n");
+ HASH_ITER( hlink, lustre_runtime->record_runtime_hash, rec_rt, tmp_rec_rt )
+ {
+ printf( "*** record %d rank %d osts %d\n",
+ rec_rt->record->rec_id,
+ rec_rt->record->rank,
+ rec_rt->record->counters[LUSTRE_STRIPE_WIDTH]);
+ }
+ return;
+}
+
+
+
+
+/*
</span> * Local variables:
* c-indent-level: 4
* c-basic-offset: 4
</code></pre>
<br>
</li>
<li id="2ace16276b68c7d5aef163bb260bd9f7">
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/a61ebca2580f403acfd8c472d2fbb1356003a6b3...dc1a9eb90c044ecb054cc50e7400c2bc3a9080e7#diff-3">
<strong>
darshan-test/2.x/llapi-perf.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="background: #ffdddd; color: #000000">--- a/darshan-test/2.x/llapi-perf.c
</span><span style="background: #ddffdd; color: #000000">+++ b/darshan-test/2.x/llapi-perf.c
</span><span style="color: #aaaaaa">@@ -158,6 +158,7 @@ int main(int argc, char **argv)
</span> else if ( opt_ioctl )
{
lum->lmm_magic = LOV_USER_MAGIC;
<span style="background: #ddffdd; color: #000000">+ lum->lmm_stripe_count = LOV_MAX_STRIPE_COUNT;
</span> ret = ioctl( fd, LL_IOC_LOV_GETSTRIPE, (void *)lum );
}
#ifdef DEBUG
</code></pre>
<br>
</li>
<li id="4dfe7e78b540daa005ea4b5f0458c90d">
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/a61ebca2580f403acfd8c472d2fbb1356003a6b3...dc1a9eb90c044ecb054cc50e7400c2bc3a9080e7#diff-4">
<strong>
darshan-test/lustre/Makefile
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="background: #ffdddd; color: #000000">--- a/darshan-test/lustre/Makefile
</span><span style="background: #ddffdd; color: #000000">+++ b/darshan-test/lustre/Makefile
</span><span style="color: #aaaaaa">@@ -1,6 +1,6 @@
</span> .PHONY: clean
BINS = darshan-tester darshan-tester-mpi
<span style="background: #ffdddd; color: #000000">-OBJS = darshan-core-stub.o darshan-lustre.o
</span><span style="background: #ddffdd; color: #000000">+OBJS = darshan-lustre.o darshan-core-stub.o
</span> CFLAGS = -O0 -g -I../.. -I../../darshan-runtime
### Include -I. when building non-MPI tests to include the mpi.h stub header
</code></pre>
<br>
</li>
<li id="172f4bf093caeeba24730a1adff8f15a">
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/a61ebca2580f403acfd8c472d2fbb1356003a6b3...dc1a9eb90c044ecb054cc50e7400c2bc3a9080e7#diff-5">
<strong>
darshan-test/lustre/darshan-core-stub.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="background: #ffdddd; color: #000000">--- a/darshan-test/lustre/darshan-core-stub.c
</span><span style="background: #ddffdd; color: #000000">+++ b/darshan-test/lustre/darshan-core-stub.c
</span><span style="color: #aaaaaa">@@ -10,6 +10,8 @@
</span> #include <stdlib.h>
#include <fcntl.h>
<span style="background: #ddffdd; color: #000000">+#include "darshan-lustre.h"
+
</span> /*
* Global variables
*/
<span style="color: #aaaaaa">@@ -20,6 +22,7 @@ static struct darshan_module_funcs mod_funcs;
</span> /*
* Import routines from Lustre module
*/
<span style="background: #ddffdd; color: #000000">+extern struct lustre_runtime *lustre_runtime;
</span>
void darshan_core_register_record(
void *name,
<span style="color: #aaaaaa">@@ -90,6 +93,9 @@ int main( int argc, char **argv )
</span> close(fd);
}
<span style="background: #ddffdd; color: #000000">+ for ( i = 0; i < lustre_runtime->record_count; i++ )
+ (lustre_runtime->record_runtime_array[i]).record->rank = rand() % 10;
+
</span> print_lustre_runtime();
darshan_core_shutdown();
</code></pre>
<br>
</li>
</div>
<div class="footer" style="margin-top: 10px">
<p style="color: #777; font-size: small">
—
<br>
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/a61ebca2580f403acfd8c472d2fbb1356003a6b3...dc1a9eb90c044ecb054cc50e7400c2bc3a9080e7">View it on GitLab</a>.
<br>
You're receiving this email because of your account on xgitlab.cels.anl.gov.
If you'd like to receive fewer emails, you can
adjust your notification settings.
</p>
</div>
</body>
</html>