<html lang='en'>
<head>
<meta content='text/html; charset=utf-8' http-equiv='Content-Type'>
<title>
GitLab
</title>
</meta>
</head>
<style>
img {
max-width: 100%;
height: auto;
}
p.details {
font-style:italic;
color:#777
}
.footer p {
font-size:small;
color:#777
}
pre.commit-message {
white-space: pre-wrap;
}
.file-stats a {
text-decoration: none;
}
.file-stats .new-file {
color: #090;
}
.file-stats .deleted-file {
color: #B00;
}
</style>
<body>
<div class='content'>
<h3>Shane Snyder pushed to branch dev-modular at <a href="https://xgitlab.cels.anl.gov/darshan/darshan">darshan / darshan</a></h3>
<h4>
Commits:
</h4>
<ul>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/98c93e0f45ef35bfde0da5d2b143e0b0de9f6ce5">98c93e0f</a></strong>
<div>
<span>by Shane Snyder</span>
<i>at 2015-11-02T12:34:38Z</i>
</div>
<pre class='commit-message'>darshan-runtime commits for configurable mem
user can now give a config option or use a runtime environment
variable to control darshan's per module memory requirements.</pre>
</li>
<li>
<strong><a href="https://xgitlab.cels.anl.gov/darshan/darshan/commit/40d1dc03b7e336f5127f70a68c3a8069fd92536c">40d1dc03</a></strong>
<div>
<span>by Shane Snyder</span>
<i>at 2015-11-02T13:31:25Z</i>
</div>
<pre class='commit-message'>darshan-util updates to support partial logs</pre>
</li>
</ul>
<h4>15 changed files:</h4>
<ul>
<li class='file-stats'>
<a href='#diff-0'>
darshan-log-format.h
</a>
</li>
<li class='file-stats'>
<a href='#diff-1'>
darshan-runtime/configure
</a>
</li>
<li class='file-stats'>
<a href='#diff-2'>
darshan-runtime/configure.in
</a>
</li>
<li class='file-stats'>
<a href='#diff-3'>
darshan-runtime/darshan-core.h
</a>
</li>
<li class='file-stats'>
<a href='#diff-4'>
darshan-runtime/darshan-runtime-config.h.in
</a>
</li>
<li class='file-stats'>
<a href='#diff-5'>
darshan-runtime/darshan.h
</a>
</li>
<li class='file-stats'>
<a href='#diff-6'>
darshan-runtime/lib/darshan-bgq.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-7'>
darshan-runtime/lib/darshan-core.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-8'>
darshan-runtime/lib/darshan-hdf5.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-9'>
darshan-runtime/lib/darshan-mpiio.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-10'>
darshan-runtime/lib/darshan-null.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-11'>
darshan-runtime/lib/darshan-pnetcdf.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-12'>
darshan-runtime/lib/darshan-posix.c
</a>
</li>
<li class='file-stats'>
<a href='#diff-13'>
darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
</a>
</li>
<li class='file-stats'>
<a href='#diff-14'>
darshan-util/darshan-parser.c
</a>
</li>
</ul>
<h4>Changes:</h4>
<li id='diff-0'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-0'>
<strong>
darshan-log-format.h
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-log-format.h
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-log-format.h
</span><span style="color: #aaaaaa">@@ -69,6 +69,11 @@ static char * const darshan_module_names[] =
</span> };
#undef X
<span style="color: #000000;background-color: #ddffdd">+/* simple macros for accessing module flag bitfields */
+#define DARSHAN_MOD_FLAG_SET(flags, id) flags = (flags | (1 << id))
+#define DARSHAN_MOD_FLAG_UNSET(flags, id) flags = (flags & ~(1 << id))
+#define DARSHAN_MOD_FLAG_ISSET(flags, id) (flags & (1 << id))
+
</span> /* compression method used on darshan log file */
enum darshan_comp_type
{
<span style="color: #aaaaaa">@@ -97,7 +102,7 @@ struct darshan_header
</span> char version_string[8];
int64_t magic_nr;
unsigned char comp_type;
<span style="color: #000000;background-color: #ffdddd">- unsigned char partial_flag;
</span><span style="color: #000000;background-color: #ddffdd">+ uint32_t partial_flag;
</span> struct darshan_log_map rec_map;
struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
};
</code></pre>
<br>
</li>
<li id='diff-1'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-1'>
<strong>
darshan-runtime/configure
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/configure
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/configure
</span><span style="color: #aaaaaa">@@ -693,7 +693,7 @@ with_log_hints
</span> with_log_path
with_jobid_env
enable_bgq_mod
-with_max_records
<span style="color: #000000;background-color: #ddffdd">+with_mod_mem
</span> '
ac_precious_vars='build_alias
host_alias
<span style="color: #aaaaaa">@@ -1331,7 +1331,7 @@ Optional Packages:
</span> --with-jobid-env=<name> Name of environment variable that stores the jobid
(specify "NONE" if no appropriate environment variable is available:
Darshan will use rank 0's pid instead)
<span style="color: #000000;background-color: #ffdddd">- --with-max-records=<num> Maximum records for Darshan to track at runtime
</span><span style="color: #000000;background-color: #ddffdd">+ --with-mod-mem=<num> Maximum amount of memory (in MiB) for each Darshan module
</span>
Some influential environment variables:
CC C compiler command
<span style="color: #aaaaaa">@@ -4281,14 +4281,14 @@ rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
</span> fi
-# Check whether --with-max-records was given.
-if test "${with_max_records+set}" = set; then :
<span style="color: #000000;background-color: #ffdddd">- withval=$with_max_records; if test x$withval = xyes; then
- as_fn_error $? "--with-max-records must be given a number" "$LINENO" 5
</span><span style="color: #000000;background-color: #ddffdd">+# Check whether --with-mod-mem was given.
+if test "${with_mod_mem+set}" = set; then :
+ withval=$with_mod_mem; if test x$withval = xyes; then
+ as_fn_error $? "--with-mod-mem must be given a number" "$LINENO" 5
</span> else
cat >>confdefs.h <<_ACEOF
-#define __DARSHAN_MAX_RECORDS ${withval}
<span style="color: #000000;background-color: #ddffdd">+#define __DARSHAN_MOD_MEM_MAX ${withval}
</span> _ACEOF
fi
</code></pre>
<br>
</li>
<li id='diff-2'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-2'>
<strong>
darshan-runtime/configure.in
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/configure.in
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/configure.in
</span><span style="color: #aaaaaa">@@ -300,12 +300,12 @@ if test x$enable_bgq_mod != xno; then
</span> []))
fi
-AC_ARG_WITH(max-records,
-[ --with-max-records=<num> Maximum records for Darshan to track at runtime],
<span style="color: #000000;background-color: #ddffdd">+AC_ARG_WITH(mod-mem,
+[ --with-mod-mem=<num> Maximum amount of memory (in MiB) for each Darshan module],
</span> if test x$withval = xyes; then
<span style="color: #000000;background-color: #ffdddd">- AC_MSG_ERROR(--with-max-records must be given a number)
</span><span style="color: #000000;background-color: #ddffdd">+ AC_MSG_ERROR(--with-mod-mem must be given a number)
</span> else
<span style="color: #000000;background-color: #ffdddd">- AC_DEFINE_UNQUOTED(__DARSHAN_MAX_RECORDS, ${withval}, Maximum Darshan records to track at runtime)
</span><span style="color: #000000;background-color: #ddffdd">+ AC_DEFINE_UNQUOTED(__DARSHAN_MOD_MEM_MAX, ${withval}, Maximum memory (in MiB) for each Darshan module)
</span> fi
)
</code></pre>
<br>
</li>
<li id='diff-3'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-3'>
<strong>
darshan-runtime/darshan-core.h
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/darshan-core.h
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/darshan-core.h
</span><span style="color: #aaaaaa">@@ -26,18 +26,18 @@
</span> /* Environment variable to override __DARSHAN_MEM_ALIGNMENT */
#define DARSHAN_MEM_ALIGNMENT_OVERRIDE "DARSHAN_MEMALIGN"
-#ifdef __DARSHAN_MAX_RECORDS
-#define DARSHAN_CORE_MAX_RECORDS __DARSHAN_MAX_RECORDS
<span style="color: #000000;background-color: #ddffdd">+/* Environment variable to override memory per module */
+#define DARSHAN_MOD_MEM_OVERRIDE "DARSHAN_MODMEM"
+
+/* Maximum amount of memory per instrumentation module in MiB */
+#ifdef __DARSHAN_MOD_MEM_MAX
+#define DARSHAN_MOD_MEM_MAX (__DARSHAN_MOD_MEM_MAX * 1024 * 1024)
</span> #else
-#define DARSHAN_CORE_MAX_RECORDS 2048
<span style="color: #000000;background-color: #ddffdd">+#define DARSHAN_MOD_MEM_MAX (2 * 1024 * 1024) /* 2 MiB default */
</span> #endif
-/* TODO: revisit this default size if we change memory per module */
-#define DARSHAN_CORE_COMP_BUF_SIZE (2 * 1024 * 1024)
-
-#define DARSHAN_CORE_MOD_SET(flags, id) (flags | (1 << id))
-#define DARSHAN_CORE_MOD_UNSET(flags, id) (flags & ~(1 << id))
-#define DARSHAN_CORE_MOD_ISSET(flags, id) (flags & (1 << id))
<span style="color: #000000;background-color: #ddffdd">+/* Default runtime compression buffer size */
+#define DARSHAN_COMP_BUF_SIZE DARSHAN_MOD_MEM_MAX
</span>
/* in memory structure to keep up with job level data */
struct darshan_core_runtime
<span style="color: #aaaaaa">@@ -48,7 +48,7 @@ struct darshan_core_runtime
</span> struct darshan_core_record_ref *rec_hash;
int rec_count;
struct darshan_core_module* mod_array[DARSHAN_MAX_MODS];
<span style="color: #000000;background-color: #ffdddd">- char comp_buf[DARSHAN_CORE_COMP_BUF_SIZE];
</span><span style="color: #000000;background-color: #ddffdd">+ char comp_buf[DARSHAN_COMP_BUF_SIZE];
</span> double wtime_offset;
char *trailing_data;
};
</code></pre>
<br>
</li>
<li id='diff-4'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-4'>
<strong>
darshan-runtime/darshan-runtime-config.h.in
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/darshan-runtime-config.h.in
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/darshan-runtime-config.h.in
</span><span style="color: #aaaaaa">@@ -102,11 +102,11 @@
</span> /* Location to store log files at run time */
#undef __DARSHAN_LOG_PATH
-/* Maximum Darshan records to track at runtime */
-#undef __DARSHAN_MAX_RECORDS
-
/* Memory alignment in bytes */
#undef __DARSHAN_MEM_ALIGNMENT
<span style="color: #000000;background-color: #ddffdd">+/* Maximum memory (in MiB) for each Darshan module */
+#undef __DARSHAN_MOD_MEM_MAX
+
</span> /* Generalized request type for MPI-IO */
#undef __D_MPI_REQUEST
</code></pre>
<br>
</li>
<li id='diff-5'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-5'>
<strong>
darshan-runtime/darshan.h
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/darshan.h
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/darshan.h
</span><span style="color: #aaaaaa">@@ -118,18 +118,21 @@ void darshan_core_unregister_module(
</span> * Register the Darshan record given by 'name' with the darshan-core
* runtime, allowing it to be properly tracked and (potentially)
* correlated with records from other modules. 'len' is the size of
<span style="color: #000000;background-color: #ffdddd">- * the name pointer (string length for string names), 'printable_flag'
- * indicates whether the name is a string, and 'mod_id' is the identifier
- * of the calling module. 'rec_id' is an output pointer storing the
- * correspoing Darshan record identifier and 'file_alignment' is an
- * output pointer storing the file system alignment value for the given
- * record.
</span><span style="color: #000000;background-color: #ddffdd">+ * the name pointer (string length for string names), and 'printable_flag'
+ * indicates whether the name is a string. 'mod_limit_flag' is set if
+ * the calling module is out of memory (to prevent darshan-core from
+ * creating new records and to just search existing records) and 'mod_id'
+ * is the identifier of the calling module. 'rec_id' is an output pointer
+ * storing the correspoing Darshan record identifier and 'file_alignment'
+ * is an output pointer storing the file system alignment value for the
+ * given record.
</span> */
void darshan_core_register_record(
void *name,
int len,
<span style="color: #000000;background-color: #ffdddd">- int printable_flag,
</span> darshan_module_id mod_id,
<span style="color: #000000;background-color: #ddffdd">+ int printable_flag,
+ int mod_limit_flag,
</span> darshan_record_id *rec_id,
int *file_alignment);
</code></pre>
<br>
</li>
<li id='diff-6'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-6'>
<strong>
darshan-runtime/lib/darshan-bgq.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/lib/darshan-bgq.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/lib/darshan-bgq.c
</span><span style="color: #aaaaaa">@@ -156,8 +156,9 @@ void bgq_runtime_initialize()
</span> darshan_core_register_record(
recname,
strlen(recname),
<span style="color: #000000;background-color: #ffdddd">- 1,
</span> DARSHAN_BGQ_MOD,
<span style="color: #000000;background-color: #ddffdd">+ 1,
+ 0,
</span> &bgq_runtime->record.f_id,
&bgq_runtime->record.alignment);
</code></pre>
<br>
</li>
<li id='diff-7'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-7'>
<strong>
darshan-runtime/lib/darshan-core.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/lib/darshan-core.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/lib/darshan-core.c
</span><span style="color: #aaaaaa">@@ -99,7 +99,8 @@ static char* darshan_get_exe_and_mounts(
</span> static void darshan_block_size_from_path(
const char *path, int *block_size);
static void darshan_get_shared_records(
<span style="color: #000000;background-color: #ffdddd">- struct darshan_core_runtime *core, darshan_record_id *shared_recs);
</span><span style="color: #000000;background-color: #ddffdd">+ struct darshan_core_runtime *core, darshan_record_id **shared_recs,
+ int *shared_rec_cnt);
</span> static int darshan_log_open_all(
char *logfile_name, MPI_File *log_fh);
static int darshan_deflate_buffer(
<span style="color: #aaaaaa">@@ -251,7 +252,9 @@ void darshan_core_shutdown()
</span> int64_t last_end_time;
int local_mod_use[DARSHAN_MAX_MODS] = {0};
int global_mod_use_count[DARSHAN_MAX_MODS] = {0};
<span style="color: #000000;background-color: #ffdddd">- darshan_record_id shared_recs[DARSHAN_CORE_MAX_RECORDS] = {0};
</span><span style="color: #000000;background-color: #ddffdd">+ darshan_record_id *shared_recs;
+ darshan_record_id *mod_shared_recs;
+ int shared_rec_cnt = 0;
</span> double start_log_time;
double open1, open2;
double job1, job2;
<span style="color: #aaaaaa">@@ -261,7 +264,7 @@ void darshan_core_shutdown()
</span> double header1, header2;
double tm_end;
uint64_t gz_fp = 0;
<span style="color: #000000;background-color: #ffdddd">- unsigned char tmp_partial_flag;
</span><span style="color: #000000;background-color: #ddffdd">+ uint32_t tmp_partial_flag;
</span> MPI_File log_fh;
MPI_Status status;
<span style="color: #aaaaaa">@@ -368,7 +371,7 @@ void darshan_core_shutdown()
</span> DARSHAN_MPI_CALL(PMPI_Allreduce)(local_mod_use, global_mod_use_count, DARSHAN_MAX_MODS, MPI_INT, MPI_SUM, MPI_COMM_WORLD);
/* get a list of records which are shared across all processes */
<span style="color: #000000;background-color: #ffdddd">- darshan_get_shared_records(final_core, shared_recs);
</span><span style="color: #000000;background-color: #ddffdd">+ darshan_get_shared_records(final_core, &shared_recs, &shared_rec_cnt);
</span>
if(internal_timing_flag)
open1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
<span style="color: #aaaaaa">@@ -463,6 +466,9 @@ void darshan_core_shutdown()
</span> if(internal_timing_flag)
rec2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
<span style="color: #000000;background-color: #ddffdd">+ mod_shared_recs = malloc(shared_rec_cnt * sizeof(darshan_record_id));
+ assert(mod_shared_recs);
+
</span> /* loop over globally used darshan modules and:
* - perform shared file reductions, if possible
* - get final output buffer
<span style="color: #aaaaaa">@@ -475,7 +481,6 @@ void darshan_core_shutdown()
</span> {
struct darshan_core_module* this_mod = final_core->mod_array[i];
struct darshan_core_record_ref *ref = NULL;
<span style="color: #000000;background-color: #ffdddd">- darshan_record_id mod_shared_recs[DARSHAN_CORE_MAX_RECORDS];
</span> int mod_shared_rec_cnt = 0;
void* mod_buf = NULL;
int mod_buf_sz = 0;
<span style="color: #aaaaaa">@@ -495,13 +500,13 @@ void darshan_core_shutdown()
</span> mod1[i] = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* set the shared file list for this module */
<span style="color: #000000;background-color: #ffdddd">- memset(mod_shared_recs, 0, DARSHAN_CORE_MAX_RECORDS * sizeof(darshan_record_id));
- for(j = 0; j < DARSHAN_CORE_MAX_RECORDS && shared_recs[j] != 0; j++)
</span><span style="color: #000000;background-color: #ddffdd">+ memset(mod_shared_recs, 0, shared_rec_cnt * sizeof(darshan_record_id));
+ for(j = 0; j < shared_rec_cnt; j++)
</span> {
HASH_FIND(hlink, final_core->rec_hash, &shared_recs[j],
sizeof(darshan_record_id), ref);
assert(ref);
<span style="color: #000000;background-color: #ffdddd">- if(DARSHAN_CORE_MOD_ISSET(ref->global_mod_flags, i))
</span><span style="color: #000000;background-color: #ddffdd">+ if(DARSHAN_MOD_FLAG_ISSET(ref->global_mod_flags, i))
</span> {
mod_shared_recs[mod_shared_rec_cnt++] = shared_recs[j];
}
<span style="color: #aaaaaa">@@ -551,11 +556,11 @@ void darshan_core_shutdown()
</span> }
/* run a reduction to determine if any application processes had to set the
<span style="color: #000000;background-color: #ffdddd">- * partial flag. this happens when a process has tracked too many records
- * at once and cannot track new records
</span><span style="color: #000000;background-color: #ddffdd">+ * partial flag for any modules. this happens when a module exhausts its memory
+ * and does not track every possible record
</span> */
DARSHAN_MPI_CALL(PMPI_Reduce)(&(final_core->log_header.partial_flag),
<span style="color: #000000;background-color: #ffdddd">- &tmp_partial_flag, 1, MPI_UNSIGNED_CHAR, MPI_MAX, 0, MPI_COMM_WORLD);
</span><span style="color: #000000;background-color: #ddffdd">+ &tmp_partial_flag, 1, MPI_UINT32_T, MPI_BOR, 0, MPI_COMM_WORLD);
</span>
if(internal_timing_flag)
header1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
<span style="color: #aaaaaa">@@ -1085,32 +1090,45 @@ static void darshan_block_size_from_path(const char *path, int *block_size)
</span> }
static void darshan_get_shared_records(struct darshan_core_runtime *core,
<span style="color: #000000;background-color: #ffdddd">- darshan_record_id *shared_recs)
</span><span style="color: #000000;background-color: #ddffdd">+ darshan_record_id **shared_recs, int *shared_rec_cnt)
</span> {
<span style="color: #000000;background-color: #ffdddd">- int i;
- int ndx;
</span><span style="color: #000000;background-color: #ddffdd">+ int i, j;
+ int tmp_cnt = core->rec_count;
</span> struct darshan_core_record_ref *tmp, *ref;
<span style="color: #000000;background-color: #ffdddd">- darshan_record_id id_array[DARSHAN_CORE_MAX_RECORDS] = {0};
- uint64_t mod_flags[DARSHAN_CORE_MAX_RECORDS] = {0};
- uint64_t global_mod_flags[DARSHAN_CORE_MAX_RECORDS] = {0};
</span><span style="color: #000000;background-color: #ddffdd">+ darshan_record_id *id_array;
+ uint64_t *mod_flags;
+ uint64_t *global_mod_flags;
+
+ /* broadcast root's number of records to all other processes */
+ DARSHAN_MPI_CALL(PMPI_Bcast)(&tmp_cnt, 1, MPI_INT, 0, MPI_COMM_WORLD);
+
+ /* use root record count to allocate data structures */
+ id_array = malloc(tmp_cnt * sizeof(darshan_record_id));
+ mod_flags = malloc(tmp_cnt * sizeof(uint64_t));
+ global_mod_flags = malloc(tmp_cnt * sizeof(uint64_t));
+ *shared_recs = malloc(tmp_cnt * sizeof(darshan_record_id));
+ assert(id_array && mod_flags && global_mod_flags && *shared_recs);
+
+ memset(mod_flags, 0, tmp_cnt * sizeof(uint64_t));
+ memset(global_mod_flags, 0, tmp_cnt * sizeof(uint64_t));
+ memset(*shared_recs, 0, tmp_cnt * sizeof(darshan_record_id));
</span>
/* first, determine list of records root process has opened */
if(my_rank == 0)
{
<span style="color: #000000;background-color: #ffdddd">- ndx = 0;
</span><span style="color: #000000;background-color: #ddffdd">+ i = 0;
</span> HASH_ITER(hlink, core->rec_hash, ref, tmp)
{
<span style="color: #000000;background-color: #ffdddd">- id_array[ndx++] = ref->rec.id;
</span><span style="color: #000000;background-color: #ddffdd">+ id_array[i++] = ref->rec.id;
</span> }
}
/* broadcast root's list of records to all other processes */
<span style="color: #000000;background-color: #ffdddd">- DARSHAN_MPI_CALL(PMPI_Bcast)(id_array,
- (DARSHAN_CORE_MAX_RECORDS * sizeof(darshan_record_id)),
</span><span style="color: #000000;background-color: #ddffdd">+ DARSHAN_MPI_CALL(PMPI_Bcast)(id_array, (tmp_cnt * sizeof(darshan_record_id)),
</span> MPI_BYTE, 0, MPI_COMM_WORLD);
/* everyone looks to see if they opened the same records as root */
<span style="color: #000000;background-color: #ffdddd">- for(i=0; (i<DARSHAN_CORE_MAX_RECORDS && id_array[i] != 0); i++)
</span><span style="color: #000000;background-color: #ddffdd">+ for(i=0; i<tmp_cnt; i++)
</span> {
HASH_FIND(hlink, core->rec_hash, &id_array[i], sizeof(darshan_record_id), ref);
if(ref)
<span style="color: #aaaaaa">@@ -1123,15 +1141,15 @@ static void darshan_get_shared_records(struct darshan_core_runtime *core,
</span> /* now allreduce so everyone agrees which files are shared and
* which modules accessed them collectively
*/
<span style="color: #000000;background-color: #ffdddd">- DARSHAN_MPI_CALL(PMPI_Allreduce)(mod_flags, global_mod_flags,
- DARSHAN_CORE_MAX_RECORDS, MPI_UINT64_T, MPI_BAND, MPI_COMM_WORLD);
</span><span style="color: #000000;background-color: #ddffdd">+ DARSHAN_MPI_CALL(PMPI_Allreduce)(mod_flags, global_mod_flags, tmp_cnt,
+ MPI_UINT64_T, MPI_BAND, MPI_COMM_WORLD);
</span>
<span style="color: #000000;background-color: #ffdddd">- ndx = 0;
- for(i=0; (i<DARSHAN_CORE_MAX_RECORDS && id_array[i] != 0); i++)
</span><span style="color: #000000;background-color: #ddffdd">+ j = 0;
+ for(i=0; i<tmp_cnt; i++)
</span> {
if(global_mod_flags[i] != 0)
{
<span style="color: #000000;background-color: #ffdddd">- shared_recs[ndx++] = id_array[i];
</span><span style="color: #000000;background-color: #ddffdd">+ (*shared_recs)[j++] = id_array[i];
</span>
/* set global_mod_flags so we know which modules collectively
* accessed this module. we need this info to support shared
<span style="color: #aaaaaa">@@ -1142,6 +1160,7 @@ static void darshan_get_shared_records(struct darshan_core_runtime *core,
</span> ref->global_mod_flags = global_mod_flags[i];
}
}
<span style="color: #000000;background-color: #ddffdd">+ *shared_rec_cnt = j;
</span>
return;
}
<span style="color: #aaaaaa">@@ -1246,7 +1265,7 @@ static int darshan_deflate_buffer(void **pointers, int *lengths, int count,
</span> }
tmp_stream.next_out = (unsigned char *)comp_buf;
<span style="color: #000000;background-color: #ffdddd">- tmp_stream.avail_out = DARSHAN_CORE_COMP_BUF_SIZE;
</span><span style="color: #000000;background-color: #ddffdd">+ tmp_stream.avail_out = DARSHAN_COMP_BUF_SIZE;
</span>
/* loop over the input pointers */
for(i = 0; i < count; i++)
<span style="color: #aaaaaa">@@ -1307,9 +1326,9 @@ static int darshan_log_write_record_hash(MPI_File log_fh, struct darshan_core_ru
</span> char *hash_buf;
char *hash_buf_off;
<span style="color: #000000;background-color: #ffdddd">- /* allocate a buffer to store at most 64 bytes for each of a max number of records */
</span><span style="color: #000000;background-color: #ddffdd">+ /* allocate a buffer to store at most 64 bytes for each registered record */
</span> /* NOTE: this buffer may be reallocated if estimate is too small */
<span style="color: #000000;background-color: #ffdddd">- hash_buf_sz = DARSHAN_CORE_MAX_RECORDS * 64;
</span><span style="color: #000000;background-color: #ddffdd">+ hash_buf_sz = core->rec_count * 64;
</span> hash_buf = malloc(hash_buf_sz);
if(!hash_buf)
{
<span style="color: #aaaaaa">@@ -1478,7 +1497,10 @@ void darshan_core_register_module(
</span> int *mod_mem_limit,
int *sys_mem_alignment)
{
<span style="color: #000000;background-color: #ddffdd">+ int ret;
+ int tmpval;
</span> struct darshan_core_module* mod;
<span style="color: #000000;background-color: #ddffdd">+ char *mod_mem_str = NULL;
</span> *mod_mem_limit = 0;
if(!darshan_core || (mod_id >= DARSHAN_MAX_MODS))
<span style="color: #aaaaaa">@@ -1514,9 +1536,21 @@ void darshan_core_register_module(
</span> /* get the calling process's rank */
DARSHAN_MPI_CALL(PMPI_Comm_rank)(MPI_COMM_WORLD, my_rank);
<span style="color: #000000;background-color: #ffdddd">- /* TODO: something smarter than just 2 MiB per module */
- *mod_mem_limit = 2 * 1024 * 1024;
</span>-
<span style="color: #000000;background-color: #ddffdd">+ /* set the maximum amount of memory this module can use */
+ mod_mem_str = getenv(DARSHAN_MOD_MEM_OVERRIDE);
+ if(mod_mem_str)
+ {
+ ret = sscanf(mod_mem_str, "%d", &tmpval);
+ /* silently ignore if the env variable is set poorly */
+ if(ret == 1 && tmpval > 0)
+ *mod_mem_limit = (tmpval * 1024 * 1024); /* convert to MiB */
+ else
+ *mod_mem_limit = DARSHAN_MOD_MEM_MAX;
+ }
+ else
+ {
+ *mod_mem_limit = DARSHAN_MOD_MEM_MAX;
+ }
</span> DARSHAN_CORE_UNLOCK();
return;
<span style="color: #aaaaaa">@@ -1550,8 +1584,9 @@ void darshan_core_unregister_module(
</span> void darshan_core_register_record(
void *name,
int len,
<span style="color: #000000;background-color: #ffdddd">- int printable_flag,
</span> darshan_module_id mod_id,
<span style="color: #000000;background-color: #ddffdd">+ int printable_flag,
+ int mod_limit_flag,
</span> darshan_record_id *rec_id,
int *file_alignment)
{
<span style="color: #aaaaaa">@@ -1573,16 +1608,14 @@ void darshan_core_register_record(
</span> HASH_FIND(hlink, darshan_core->rec_hash, &tmp_rec_id, sizeof(darshan_record_id), ref);
if(!ref)
{
<span style="color: #000000;background-color: #ffdddd">- /* record not found -- add it to the hash if we aren't already tracking the
- * maximum number of records
</span><span style="color: #000000;background-color: #ddffdd">+ /* record not found -- add it to the hash if this module has not already used
+ * all of its memory
</span> */
<span style="color: #000000;background-color: #ffdddd">- if(darshan_core->rec_count >= DARSHAN_CORE_MAX_RECORDS)
</span><span style="color: #000000;background-color: #ddffdd">+ if(mod_limit_flag)
</span> {
<span style="color: #000000;background-color: #ffdddd">- /* if we are already tracking the max records, set a flag to indicate
- * that this log file has partial results
- */
- darshan_core->log_header.partial_flag = 1;
</span><span style="color: #000000;background-color: #ddffdd">+ /* if this module is OOM, set a flag in the header to indicate this */
+ DARSHAN_MOD_FLAG_SET(darshan_core->log_header.partial_flag, mod_id);
</span> DARSHAN_CORE_UNLOCK();
return;
}
<span style="color: #aaaaaa">@@ -1600,7 +1633,7 @@ void darshan_core_register_record(
</span> darshan_core->rec_count++;
}
}
<span style="color: #000000;background-color: #ffdddd">- ref->mod_flags = DARSHAN_CORE_MOD_SET(ref->mod_flags, mod_id);
</span><span style="color: #000000;background-color: #ddffdd">+ DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id);
</span> DARSHAN_CORE_UNLOCK();
if(file_alignment)
<span style="color: #aaaaaa">@@ -1625,7 +1658,7 @@ void darshan_core_unregister_record(
</span> assert(ref);
/* disassociate this module from the given record id */
<span style="color: #000000;background-color: #ffdddd">- ref->mod_flags = DARSHAN_CORE_MOD_UNSET(ref->mod_flags, mod_id);
</span><span style="color: #000000;background-color: #ddffdd">+ DARSHAN_MOD_FLAG_UNSET(ref->mod_flags, mod_id);
</span> if(!(ref->mod_flags))
{
/* if no other modules are associated with this rec, delete it */
</code></pre>
<br>
</li>
<li id='diff-8'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-8'>
<strong>
darshan-runtime/lib/darshan-hdf5.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/lib/darshan-hdf5.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/lib/darshan-hdf5.c
</span><span style="color: #aaaaaa">@@ -256,6 +256,7 @@ static struct hdf5_file_runtime* hdf5_file_by_name(const char *name)
</span> struct hdf5_file_runtime *file = NULL;
char *newname = NULL;
darshan_record_id file_id;
<span style="color: #000000;background-color: #ddffdd">+ int limit_flag;
</span>
if(!hdf5_runtime || instrumentation_disabled)
return(NULL);
<span style="color: #aaaaaa">@@ -264,12 +265,15 @@ static struct hdf5_file_runtime* hdf5_file_by_name(const char *name)
</span> if(!newname)
newname = (char*)name;
<span style="color: #000000;background-color: #ddffdd">+ limit_flag = (hdf5_runtime->file_array_ndx >= hdf5_runtime->file_array_size);
+
</span> /* get a unique id for this file from darshan core */
darshan_core_register_record(
(void*)newname,
strlen(newname),
<span style="color: #000000;background-color: #ffdddd">- 1,
</span> DARSHAN_HDF5_MOD,
<span style="color: #000000;background-color: #ddffdd">+ 1,
+ limit_flag,
</span> &file_id,
NULL);
<span style="color: #aaaaaa">@@ -292,19 +296,15 @@ static struct hdf5_file_runtime* hdf5_file_by_name(const char *name)
</span> return(file);
}
<span style="color: #000000;background-color: #ffdddd">- if(hdf5_runtime->file_array_ndx < hdf5_runtime->file_array_size);
- {
- /* no existing record, assign a new file record from the global array */
- file = &(hdf5_runtime->file_runtime_array[hdf5_runtime->file_array_ndx]);
- file->file_record = &(hdf5_runtime->file_record_array[hdf5_runtime->file_array_ndx]);
- file->file_record->f_id = file_id;
- file->file_record->rank = my_rank;
</span>-
<span style="color: #000000;background-color: #ffdddd">- /* add new record to file hash table */
- HASH_ADD(hlink, hdf5_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
</span><span style="color: #000000;background-color: #ddffdd">+ /* no existing record, assign a new file record from the global array */
+ file = &(hdf5_runtime->file_runtime_array[hdf5_runtime->file_array_ndx]);
+ file->file_record = &(hdf5_runtime->file_record_array[hdf5_runtime->file_array_ndx]);
+ file->file_record->f_id = file_id;
+ file->file_record->rank = my_rank;
</span>
<span style="color: #000000;background-color: #ffdddd">- hdf5_runtime->file_array_ndx++;
- }
</span><span style="color: #000000;background-color: #ddffdd">+ /* add new record to file hash table */
+ HASH_ADD(hlink, hdf5_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
+ hdf5_runtime->file_array_ndx++;
</span>
if(newname != name)
free(newname);
</code></pre>
<br>
</li>
<li id='diff-9'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-9'>
<strong>
darshan-runtime/lib/darshan-mpiio.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/lib/darshan-mpiio.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/lib/darshan-mpiio.c
</span><span style="color: #aaaaaa">@@ -878,6 +878,7 @@ static struct mpiio_file_runtime* mpiio_file_by_name(const char *name)
</span> struct mpiio_file_runtime *file = NULL;
char *newname = NULL;
darshan_record_id file_id;
<span style="color: #000000;background-color: #ddffdd">+ int limit_flag;
</span>
if(!mpiio_runtime || instrumentation_disabled)
return(NULL);
<span style="color: #aaaaaa">@@ -886,17 +887,20 @@ static struct mpiio_file_runtime* mpiio_file_by_name(const char *name)
</span> if(!newname)
newname = (char*)name;
<span style="color: #000000;background-color: #ddffdd">+ limit_flag = (mpiio_runtime->file_array_ndx >= mpiio_runtime->file_array_size);
+
</span> /* get a unique id for this file from darshan core */
darshan_core_register_record(
(void*)newname,
strlen(newname),
<span style="color: #000000;background-color: #ffdddd">- 1,
</span> DARSHAN_MPIIO_MOD,
<span style="color: #000000;background-color: #ddffdd">+ 1,
+ limit_flag,
</span> &file_id,
NULL);
<span style="color: #000000;background-color: #ffdddd">- /* if record is set to 0, darshan-core is out of space and will not
- * track this record, so we should avoid tracking it, too
</span><span style="color: #000000;background-color: #ddffdd">+ /* the file record id is set to 0 if no memory is available for tracking
+ * new records -- just fall through and ignore this record
</span> */
if(file_id == 0)
{
<span style="color: #aaaaaa">@@ -914,19 +918,15 @@ static struct mpiio_file_runtime* mpiio_file_by_name(const char *name)
</span> return(file);
}
<span style="color: #000000;background-color: #ffdddd">- if(mpiio_runtime->file_array_ndx < mpiio_runtime->file_array_size);
- {
- /* no existing record, assign a new file record from the global array */
- file = &(mpiio_runtime->file_runtime_array[mpiio_runtime->file_array_ndx]);
- file->file_record = &(mpiio_runtime->file_record_array[mpiio_runtime->file_array_ndx]);
- file->file_record->f_id = file_id;
- file->file_record->rank = my_rank;
</span>-
<span style="color: #000000;background-color: #ffdddd">- /* add new record to file hash table */
- HASH_ADD(hlink, mpiio_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
</span><span style="color: #000000;background-color: #ddffdd">+ /* no existing record, assign a new file record from the global array */
+ file = &(mpiio_runtime->file_runtime_array[mpiio_runtime->file_array_ndx]);
+ file->file_record = &(mpiio_runtime->file_record_array[mpiio_runtime->file_array_ndx]);
+ file->file_record->f_id = file_id;
+ file->file_record->rank = my_rank;
</span>
<span style="color: #000000;background-color: #ffdddd">- mpiio_runtime->file_array_ndx++;
- }
</span><span style="color: #000000;background-color: #ddffdd">+ /* add new record to file hash table */
+ HASH_ADD(hlink, mpiio_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
+ mpiio_runtime->file_array_ndx++;
</span>
if(newname != name)
free(newname);
</code></pre>
<br>
</li>
<li id='diff-10'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-10'>
<strong>
darshan-runtime/lib/darshan-null.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/lib/darshan-null.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/lib/darshan-null.c
</span><span style="color: #aaaaaa">@@ -264,6 +264,7 @@ static struct null_record_runtime* null_record_by_name(const char *name)
</span> {
struct null_record_runtime *rec = NULL;
darshan_record_id rec_id;
<span style="color: #000000;background-color: #ddffdd">+ int limit_flag;
</span>
/* Don't search for a record if the "NULL" module is not initialized or
* if instrumentation has been toggled off.
<span style="color: #aaaaaa">@@ -271,15 +272,27 @@ static struct null_record_runtime* null_record_by_name(const char *name)
</span> if(!null_runtime || instrumentation_disabled)
return(NULL);
<span style="color: #000000;background-color: #ddffdd">+ /* stop tracking new records if we are tracking our maximum count */
+ limit_flag = (null_runtime->rec_array_ndx >= null_runtime->rec_array_size);
+
</span> /* get a unique record identifier for this record from darshan-core */
darshan_core_register_record(
(void*)name,
strlen(name),
<span style="color: #000000;background-color: #ffdddd">- 1,
</span> DARSHAN_NULL_MOD,
<span style="color: #000000;background-color: #ddffdd">+ 1,
+ limit_flag,
</span> &rec_id,
NULL);
<span style="color: #000000;background-color: #ddffdd">+ /* the file record id is set to 0 if no memory is available for tracking
+ * new records -- just fall through and ignore this record
+ */
+ if(rec_id == 0)
+ {
+ return(NULL);
+ }
+
</span> /* search the hash table for this file record, and return if found */
HASH_FIND(hlink, null_runtime->record_hash, &rec_id, sizeof(darshan_record_id), rec);
if(rec)
<span style="color: #aaaaaa">@@ -287,21 +300,17 @@ static struct null_record_runtime* null_record_by_name(const char *name)
</span> return(rec);
}
<span style="color: #000000;background-color: #ffdddd">- if(null_runtime->rec_array_ndx < null_runtime->rec_array_size);
- {
- /* no existing record, assign a new one from the global array */
- rec = &(null_runtime->runtime_record_array[null_runtime->rec_array_ndx]);
- rec->record_p = &(null_runtime->record_array[null_runtime->rec_array_ndx]);
</span><span style="color: #000000;background-color: #ddffdd">+ /* no existing record, assign a new one from the global array */
+ rec = &(null_runtime->runtime_record_array[null_runtime->rec_array_ndx]);
+ rec->record_p = &(null_runtime->record_array[null_runtime->rec_array_ndx]);
</span>
<span style="color: #000000;background-color: #ffdddd">- /* set the darshan record id and corresponding process rank for this record */
- rec->record_p->f_id = rec_id;
- rec->record_p->rank = my_rank;
</span><span style="color: #000000;background-color: #ddffdd">+ /* set the darshan record id and corresponding process rank for this record */
+ rec->record_p->f_id = rec_id;
+ rec->record_p->rank = my_rank;
</span>
<span style="color: #000000;background-color: #ffdddd">- /* add new record to file hash table */
- HASH_ADD(hlink, null_runtime->record_hash, record_p->f_id, sizeof(darshan_record_id), rec);
</span>-
<span style="color: #000000;background-color: #ffdddd">- null_runtime->rec_array_ndx++;
- }
</span><span style="color: #000000;background-color: #ddffdd">+ /* add new record to file hash table */
+ HASH_ADD(hlink, null_runtime->record_hash, record_p->f_id, sizeof(darshan_record_id), rec);
+ null_runtime->rec_array_ndx++;
</span>
return(rec);
}
</code></pre>
<br>
</li>
<li id='diff-11'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-11'>
<strong>
darshan-runtime/lib/darshan-pnetcdf.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/lib/darshan-pnetcdf.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/lib/darshan-pnetcdf.c
</span><span style="color: #aaaaaa">@@ -268,6 +268,7 @@ static struct pnetcdf_file_runtime* pnetcdf_file_by_name(const char *name)
</span> struct pnetcdf_file_runtime *file = NULL;
char *newname = NULL;
darshan_record_id file_id;
<span style="color: #000000;background-color: #ddffdd">+ int limit_flag;
</span>
if(!pnetcdf_runtime || instrumentation_disabled)
return(NULL);
<span style="color: #aaaaaa">@@ -276,17 +277,20 @@ static struct pnetcdf_file_runtime* pnetcdf_file_by_name(const char *name)
</span> if(!newname)
newname = (char*)name;
<span style="color: #000000;background-color: #ddffdd">+ limit_flag = (pnetcdf_runtime->file_array_ndx >= pnetcdf_runtime->file_array_size);
+
</span> /* get a unique id for this file from darshan core */
darshan_core_register_record(
(void*)newname,
strlen(newname),
<span style="color: #000000;background-color: #ffdddd">- 1,
</span> DARSHAN_PNETCDF_MOD,
<span style="color: #000000;background-color: #ddffdd">+ 1,
+ limit_flag,
</span> &file_id,
NULL);
<span style="color: #000000;background-color: #ffdddd">- /* if record is set to 0, darshan-core is out of space and will not
- * track this record, so we should avoid tracking it, too
</span><span style="color: #000000;background-color: #ddffdd">+ /* the file record id is set to 0 if no memory is available for tracking
+ * new records -- just fall through and ignore this record
</span> */
if(file_id == 0)
{
<span style="color: #aaaaaa">@@ -304,19 +308,15 @@ static struct pnetcdf_file_runtime* pnetcdf_file_by_name(const char *name)
</span> return(file);
}
<span style="color: #000000;background-color: #ffdddd">- if(pnetcdf_runtime->file_array_ndx < pnetcdf_runtime->file_array_size);
- {
- /* no existing record, assign a new file record from the global array */
- file = &(pnetcdf_runtime->file_runtime_array[pnetcdf_runtime->file_array_ndx]);
- file->file_record = &(pnetcdf_runtime->file_record_array[pnetcdf_runtime->file_array_ndx]);
- file->file_record->f_id = file_id;
- file->file_record->rank = my_rank;
</span>-
<span style="color: #000000;background-color: #ffdddd">- /* add new record to file hash table */
- HASH_ADD(hlink, pnetcdf_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
</span><span style="color: #000000;background-color: #ddffdd">+ /* no existing record, assign a new file record from the global array */
+ file = &(pnetcdf_runtime->file_runtime_array[pnetcdf_runtime->file_array_ndx]);
+ file->file_record = &(pnetcdf_runtime->file_record_array[pnetcdf_runtime->file_array_ndx]);
+ file->file_record->f_id = file_id;
+ file->file_record->rank = my_rank;
</span>
<span style="color: #000000;background-color: #ffdddd">- pnetcdf_runtime->file_array_ndx++;
- }
</span><span style="color: #000000;background-color: #ddffdd">+ /* add new record to file hash table */
+ HASH_ADD(hlink, pnetcdf_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
+ pnetcdf_runtime->file_array_ndx++;
</span>
if(newname != name)
free(newname);
</code></pre>
<br>
</li>
<li id='diff-12'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-12'>
<strong>
darshan-runtime/lib/darshan-posix.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-runtime/lib/darshan-posix.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-runtime/lib/darshan-posix.c
</span><span style="color: #aaaaaa">@@ -1509,6 +1509,7 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
</span> char *newname = NULL;
darshan_record_id file_id;
int file_alignment;
<span style="color: #000000;background-color: #ddffdd">+ int limit_flag;
</span>
if(!posix_runtime || instrumentation_disabled)
return(NULL);
<span style="color: #aaaaaa">@@ -1517,17 +1518,20 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
</span> if(!newname)
newname = (char*)name;
<span style="color: #000000;background-color: #ddffdd">+ limit_flag = (posix_runtime->file_array_ndx >= posix_runtime->file_array_size);
+
</span> /* get a unique id for this file from darshan core */
darshan_core_register_record(
(void*)newname,
strlen(newname),
<span style="color: #000000;background-color: #ffdddd">- 1,
</span> DARSHAN_POSIX_MOD,
<span style="color: #000000;background-color: #ddffdd">+ 1,
+ limit_flag,
</span> &file_id,
&file_alignment);
<span style="color: #000000;background-color: #ffdddd">- /* if record is set to 0, darshan-core is out of space and will not
- * track this record, so we should avoid tracking it, too
</span><span style="color: #000000;background-color: #ddffdd">+ /* the file record id is set to 0 if no memory is available for tracking
+ * new records -- just fall through and ignore this record
</span> */
if(file_id == 0)
{
<span style="color: #aaaaaa">@@ -1545,21 +1549,17 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
</span> return(file);
}
<span style="color: #000000;background-color: #ffdddd">- if(posix_runtime->file_array_ndx < posix_runtime->file_array_size);
- {
- /* no existing record, assign a new file record from the global array */
- file = &(posix_runtime->file_runtime_array[posix_runtime->file_array_ndx]);
- file->file_record = &(posix_runtime->file_record_array[posix_runtime->file_array_ndx]);
- file->file_record->f_id = file_id;
- file->file_record->rank = my_rank;
- file->file_record->counters[POSIX_MEM_ALIGNMENT] = darshan_mem_alignment;
- file->file_record->counters[POSIX_FILE_ALIGNMENT] = file_alignment;
</span>-
<span style="color: #000000;background-color: #ffdddd">- /* add new record to file hash table */
- HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
</span>-
<span style="color: #000000;background-color: #ffdddd">- posix_runtime->file_array_ndx++;
- }
</span><span style="color: #000000;background-color: #ddffdd">+ /* no existing record, assign a new file record from the global array */
+ file = &(posix_runtime->file_runtime_array[posix_runtime->file_array_ndx]);
+ file->file_record = &(posix_runtime->file_record_array[posix_runtime->file_array_ndx]);
+ file->file_record->f_id = file_id;
+ file->file_record->rank = my_rank;
+ file->file_record->counters[POSIX_MEM_ALIGNMENT] = darshan_mem_alignment;
+ file->file_record->counters[POSIX_FILE_ALIGNMENT] = file_alignment;
+
+ /* add new record to file hash table */
+ HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
+ posix_runtime->file_array_ndx++;
</span>
if(newname != name)
free(newname);
</code></pre>
<br>
</li>
<li id='diff-13'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-13'>
<strong>
darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
</span><span style="color: #aaaaaa">@@ -97,7 +97,6 @@ while($line = <PARSE_OUT>)
</span> $f_save = "";
($junk, $cmdline) = split(':', $line, 2);
<span style="color: #000000;background-color: #ffdddd">- print("PRE: $cmdline\n");
</span> # add escape characters if needed for special characters in
# command line
if ($cmdline =~ /<unknown args>/)
<span style="color: #aaaaaa">@@ -108,7 +107,6 @@ while($line = <PARSE_OUT>)
</span> $cmdline = substr($cmdline, 0, -14);
}
$cmdline = encode('latex', $cmdline) . $f_save;
<span style="color: #000000;background-color: #ffdddd">- print("POST: $cmdline\n");
</span> }
elsif ($line =~ /^# nprocs: /)
{
<span style="color: #aaaaaa">@@ -135,7 +133,7 @@ while($line = <PARSE_OUT>)
</span> ($junk, $version) = split(':', $line, 2);
$version =~ s/^\s+//;
}
<span style="color: #000000;background-color: #ffdddd">- elsif ($line =~ /^# \*WARNING\*: This Darshan log contains incomplete data!/)
</span><span style="color: #000000;background-color: #ddffdd">+ elsif ($line =~ /^# \*WARNING\*: .* contains incomplete data!/)
</span> {
$partial_flag = 1;
}
</code></pre>
<br>
</li>
<li id='diff-14'>
<a href='https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c#diff-14'>
<strong>
darshan-util/darshan-parser.c
</strong>
</a>
<hr>
<pre class="highlight"><code><span style="color: #000000;background-color: #ffdddd">--- a/darshan-util/darshan-parser.c
</span><span style="color: #000000;background-color: #ddffdd">+++ b/darshan-util/darshan-parser.c
</span><span style="color: #aaaaaa">@@ -322,11 +322,6 @@ int main(int argc, char **argv)
</span> }
/* warn user if this log file is incomplete */
<span style="color: #000000;background-color: #ffdddd">- if(fd->partial_flag)
- printf("\n# *WARNING*: This Darshan log contains incomplete data!\n"
- "# This happens when an application creates\n"
- "# more records than Darshan can track.\n");
</span>-
pdata.rank_cumul_io_time = malloc(sizeof(double)*job.nprocs);
pdata.rank_cumul_md_time = malloc(sizeof(double)*job.nprocs);
if (!pdata.rank_cumul_io_time || !pdata.rank_cumul_md_time)
<span style="color: #aaaaaa">@@ -370,6 +365,13 @@ int main(int argc, char **argv)
</span> printf("# %s module data\n", darshan_module_names[i]);
printf("# *******************************************************\n");
<span style="color: #000000;background-color: #ddffdd">+ /* print warning if this module only stored partial data */
+ if(DARSHAN_MOD_FLAG_ISSET(fd->partial_flag, i))
+ printf("\n# *WARNING*: The %s module contains incomplete data!\n"
+ "# This happens when a module runs out of\n"
+ "# memory to store new record data.\n",
+ darshan_module_names[i]);
+
</span> if(mask & OPTION_BASE)
{
/* TODO: does each module print header of what each counter means??? */
</code></pre>
<br>
</li>
</div>
<div class='footer' style='margin-top: 10px;'>
<p>
—
<br>
<a href="https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c">View it on GitLab</a>
<script type="application/ld+json">{"@context":"http://schema.org","@type":"EmailMessage","action":{"@type":"ViewAction","name":["merge_requests","issues","commit"],"url":"https://xgitlab.cels.anl.gov/darshan/darshan/compare/48352d366d6530b22d3fc4a2821c8b7a7e3ea214...40d1dc03b7e336f5127f70a68c3a8069fd92536c"}}</script>
</p>
</div>
</body>
</html>