[Darshan-commits] [Git][darshan/darshan][autoperf-mod] 3 commits: Update darshan-parser to move module specific code for perf,totals,etc. into the…
Kevin Harms
xgitlab at cels.anl.gov
Thu Mar 8 16:27:37 CST 2018
Kevin Harms pushed to branch autoperf-mod at darshan / darshan
Commits:
3f4e5438 by Kevin Harms at 2018-03-08T22:03:19+00:00
Update darshan-parser to move module specific code for perf,totals,etc. into the module specific parts of the code. Make parser more generic
- - - - -
edbae6e2 by Kevin Harms at 2018-03-08T22:26:54+00:00
Correct __apxc_logutils definition
- - - - -
457a18fc by Kevin Harms at 2018-03-08T22:27:15+00:00
Remove requirement for papi in darshan-util
- - - - -
14 changed files:
- darshan-log-format.h
- darshan-util/configure.in
- darshan-util/darshan-bgq-logutils.c
- darshan-util/darshan-dxt-logutils.c
- darshan-util/darshan-hdf5-logutils.c
- darshan-util/darshan-logutils.c
- darshan-util/darshan-logutils.h
- darshan-util/darshan-lustre-logutils.c
- darshan-util/darshan-mpiio-logutils.c
- darshan-util/darshan-null-logutils.c
- darshan-util/darshan-parser.c
- darshan-util/darshan-pnetcdf-logutils.c
- darshan-util/darshan-posix-logutils.c
- darshan-util/darshan-stdio-logutils.c
Changes:
=====================================
darshan-log-format.h
=====================================
--- a/darshan-log-format.h
+++ b/darshan-log-format.h
@@ -139,7 +139,7 @@ struct darshan_base_record
*/
#ifdef DARSHAN_USE_APXC
#define __DARSHAN_APXC_VER DARSHAN_APXC_VER
-#define __apxc_logutils = &apx_logutils;
+#define __apxc_logutils &apxc_logutils
#else
#define __DARSHAN_APXC_VER 0
#define __apxc_logutils NULL
=====================================
darshan-util/configure.in
=====================================
--- a/darshan-util/configure.in
+++ b/darshan-util/configure.in
@@ -86,10 +86,6 @@ AC_ARG_ENABLE(
)
if test x$enable_autoperf_crayxc = xyes; then
- AC_CHECK_HEADER([papi.h],
- with_papi=-lpapi,
- [AC_MSG_ERROR([Cannot find required header for papi])],
- [])
AC_REQUIRE_AUX_FILE([modules/autoperf/crayxc/darshan-apxc-log-format.h])
dnl AC_CHECK_FILE([modules/autoperf/crayxc/darshan-apxc-log-format.h],
dnl DARSHAN_USE_APXC=1,
=====================================
darshan-util/darshan-bgq-logutils.c
=====================================
--- a/darshan-util/darshan-bgq-logutils.c
+++ b/darshan-util/darshan-bgq-logutils.c
@@ -50,7 +50,13 @@ struct darshan_mod_logutil_funcs bgq_logutils =
.log_print_record = &darshan_log_print_bgq_rec,
.log_print_description = &darshan_log_print_bgq_description,
.log_print_diff = &darshan_log_print_bgq_rec_diff,
- .log_agg_records = &darshan_log_agg_bgq_recs
+ .log_agg_records = &darshan_log_agg_bgq_recs,
+ .log_accum_file = NULL,
+ .log_accum_perf = NULL,
+ .log_calc_file = NULL,
+ .log_print_total_file = NULL,
+ .log_file_list = NULL,
+ .log_calc_perf = NULL
};
static int darshan_log_get_bgq_rec(darshan_fd fd, void** bgq_buf_p)
=====================================
darshan-util/darshan-dxt-logutils.c
=====================================
--- a/darshan-util/darshan-dxt-logutils.c
+++ b/darshan-util/darshan-dxt-logutils.c
@@ -41,6 +41,12 @@ struct darshan_mod_logutil_funcs dxt_posix_logutils =
.log_print_description = NULL,
.log_print_diff = NULL,
.log_agg_records = NULL,
+ .log_accum_file = NULL,
+ .log_accum_perf = NULL,
+ .log_calc_file = NULL,
+ .log_print_total_file = NULL,
+ .log_file_list = NULL,
+ .log_calc_perf = NULL
};
struct darshan_mod_logutil_funcs dxt_mpiio_logutils =
=====================================
darshan-util/darshan-hdf5-logutils.c
=====================================
--- a/darshan-util/darshan-hdf5-logutils.c
+++ b/darshan-util/darshan-hdf5-logutils.c
@@ -46,7 +46,13 @@ struct darshan_mod_logutil_funcs hdf5_logutils =
.log_print_record = &darshan_log_print_hdf5_file,
.log_print_description = &darshan_log_print_hdf5_description,
.log_print_diff = &darshan_log_print_hdf5_file_diff,
- .log_agg_records = &darshan_log_agg_hdf5_files
+ .log_agg_records = &darshan_log_agg_hdf5_files,
+ .log_accum_file = NULL,
+ .log_accum_perf = NULL,
+ .log_calc_file = NULL,
+ .log_print_total_file = NULL,
+ .log_file_list = NULL,
+ .log_calc_perf = NULL
};
static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p)
=====================================
darshan-util/darshan-logutils.c
=====================================
--- a/darshan-util/darshan-logutils.c
+++ b/darshan-util/darshan-logutils.c
@@ -1837,8 +1837,50 @@ int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
return(buf_processed);
}
+void darshan_calc_perf(perf_data_t *pdata, int64_t nprocs)
+{
+ int64_t i;
+
+ pdata->shared_time_by_cumul =
+ pdata->shared_time_by_cumul / (double)nprocs;
+
+ pdata->shared_meta_time = pdata->shared_meta_time / (double)nprocs;
+
+ for (i=0; i<nprocs; i++)
+ {
+ if (pdata->rank_cumul_io_time[i] > pdata->slowest_rank_time)
+ {
+ pdata->slowest_rank_time = pdata->rank_cumul_io_time[i];
+ pdata->slowest_rank_meta_time = pdata->rank_cumul_md_time[i];
+ pdata->slowest_rank_rank = i;
+ }
+ }
+
+ if (pdata->slowest_rank_time + pdata->shared_time_by_cumul)
+ pdata->agg_perf_by_cumul = ((double)pdata->total_bytes / 1048576.0) /
+ (pdata->slowest_rank_time +
+ pdata->shared_time_by_cumul);
+
+ if (pdata->slowest_rank_time + pdata->shared_time_by_open)
+ pdata->agg_perf_by_open = ((double)pdata->total_bytes / 1048576.0) /
+ (pdata->slowest_rank_time +
+ pdata->shared_time_by_open);
+
+ if (pdata->slowest_rank_time + pdata->shared_time_by_open_lastio)
+ pdata->agg_perf_by_open_lastio = ((double)pdata->total_bytes / 1048576.0) /
+ (pdata->slowest_rank_time +
+ pdata->shared_time_by_open_lastio);
+
+ if (pdata->slowest_rank_time + pdata->shared_time_by_slowest)
+ pdata->agg_perf_by_slowest = ((double)pdata->total_bytes / 1048576.0) /
+ (pdata->slowest_rank_time +
+ pdata->shared_time_by_slowest);
+
+ return;
+}
+
/*
- * Support functions for use with other languages
+ * Support functions for use with other language bindings
*/
/*
@@ -1872,6 +1914,11 @@ void darshan_log_get_modules (darshan_fd fd,
return;
}
+/*
+ * darshan_log_get_record
+ *
+ * Wrapper to hide the mod_logutils callback functions.
+ */
int darshan_log_get_record (darshan_fd fd,
int mod_idx,
void **buf)
@@ -1884,6 +1931,86 @@ int darshan_log_get_record (darshan_fd fd,
}
/*
+ * darshan_log_stats
+ *
+ * Loops over the POSIX, MPIIO and STDIO module records to generate
+ * derived stats. This is the same as the parser '--perf' and '--total'
+ * options.
+ */
+int darshan_log_stats (darshan_fd fd)
+{
+ int mod_list[] = { DARSHAN_POSIX_MOD,
+ DARSHAN_MPIIO_MOD,
+ DARSHAN_STDIO_MOD };
+ int i;
+ int idx;
+ int ret;
+ char *buf;
+ struct darshan_base_record *base_rec;
+ hash_entry_t *file_hash = NULL;
+ hash_entry_t total;
+ file_data_t fdata;
+ perf_data_t pdata;
+
+ memset(&total, 0, sizeof(total));
+ memset(&fdata, 0, sizeof(fdata));
+ memset(&pdata, 0, sizeof(pdata));
+
+ buf = malloc(DEF_MOD_BUF_SIZE);
+ if (!buf)
+ {
+ return(-1);
+ }
+
+ for (i = 0; i < sizeof(mod_list)/sizeof(int); i++)
+ {
+ idx = mod_list[i];
+ if (fd->mod_map[idx].len == 0)
+ {
+ continue;
+ }
+ memset(buf, 0, DEF_MOD_BUF_SIZE);
+
+ while(1)
+ {
+ int r;
+ hash_entry_t *hfile = NULL;
+
+ r = mod_logutils[i]->log_get_record(fd, (void**)&buf);
+ if (r < 1)
+ {
+ break;
+ }
+ base_rec = (struct darshan_base_record *) buf;
+
+ HASH_FIND(hlink, file_hash, &(base_rec->id), sizeof(darshan_record_id), hfile);
+ if (!hfile)
+ {
+ hfile = malloc(sizeof(*hfile));
+ if (!hfile)
+ {
+ ret = -1;
+ //goto cleanup;
+ }
+
+ memset(hfile, 0, sizeof(*hfile));
+ hfile->rec_id = base_rec->id;
+ hfile->type = 0;
+ hfile->procs = 0;
+ hfile->rec_dat = NULL;
+ hfile->cumul_time = 0.0;
+ hfile->slowest_time = 0.0;
+
+ HASH_ADD(hlink, file_hash, rec_id, sizeof(darshan_record_id), hfile);
+ }
+ }
+ }
+
+ return;
+}
+
+
+/*
* Local variables:
* c-indent-level: 4
* c-basic-offset: 4
=====================================
darshan-util/darshan-logutils.h
=====================================
--- a/darshan-util/darshan-logutils.h
+++ b/darshan-util/darshan-logutils.h
@@ -22,6 +22,12 @@
*/
#define DEF_MOD_BUF_SIZE 81920
+#define FILETYPE_SHARED (1 << 0)
+#define FILETYPE_UNIQUE (1 << 1)
+#define FILETYPE_PARTSHARED (1 << 2)
+
+#define max(a,b) (((a) > (b)) ? (a) : (b))
+
struct darshan_fd_int_state;
/* darshan file descriptor definition */
@@ -62,6 +68,58 @@ struct lustre_record_ref
UT_hash_handle hlink;
};
+typedef struct hash_entry_s
+{
+ UT_hash_handle hlink;
+ darshan_record_id rec_id;
+ int64_t type;
+ int64_t procs;
+ void *rec_dat;
+ double cumul_time;
+ double slowest_time;
+} hash_entry_t;
+
+typedef struct file_data_s
+{
+ int64_t total;
+ int64_t total_size;
+ int64_t total_max;
+ int64_t read_only;
+ int64_t read_only_size;
+ int64_t read_only_max;
+ int64_t write_only;
+ int64_t write_only_size;
+ int64_t write_only_max;
+ int64_t read_write;
+ int64_t read_write_size;
+ int64_t read_write_max;
+ int64_t unique;
+ int64_t unique_size;
+ int64_t unique_max;
+ int64_t shared;
+ int64_t shared_size;
+ int64_t shared_max;
+} file_data_t;
+
+typedef struct perf_data_s
+{
+ int64_t total_bytes;
+ double slowest_rank_time;
+ double slowest_rank_meta_time;
+ int slowest_rank_rank;
+ double shared_time_by_cumul;
+ double shared_time_by_open;
+ double shared_time_by_open_lastio;
+ double shared_time_by_slowest;
+ double shared_meta_time;
+ double agg_perf_by_cumul;
+ double agg_perf_by_open;
+ double agg_perf_by_open_lastio;
+ double agg_perf_by_slowest;
+ double *rank_cumul_io_time;
+ double *rank_cumul_md_time;
+} perf_data_t;
+
struct darshan_mnt_info
{
char mnt_type[DARSHAN_EXE_LEN];
@@ -131,6 +189,38 @@ struct darshan_mod_logutil_funcs
void *agg_rec,
int init_flag
);
+ /* accumulate file statistics across processes */
+ void (*log_accum_file)(
+ void *rec,
+ hash_entry_t *hfile,
+ int64_t nprocs
+ );
+ /* accumulate data for performance estimate */
+ void (*log_accum_perf)(
+ void *rec,
+ perf_data_t *pdata
+ );
+ /* accumulate statistics about files and file counts */
+ void (*log_calc_file)(
+ hash_entry_t *hfile,
+ file_data_t *fdata
+ );
+ /* print out the aggregate file stats */
+ void (*log_print_total_file)(
+ void *rec,
+ int ver
+ );
+ /* print out statistics on a per file basis */
+ void (*log_file_list)(
+ hash_entry_t *hfile,
+ struct darshan_name_record_ref *name_hash,
+ int detail_flag
+ );
+ /* calculate performance estimate */
+ void (*log_calc_perf)(
+ perf_data_t *pdata,
+ int64_t nprocs
+ );
};
extern struct darshan_mod_logutil_funcs *mod_logutils[];
@@ -171,6 +261,7 @@ void darshan_log_close(darshan_fd file);
void darshan_log_print_version_warnings(const char *version_string);
void darshan_log_get_modules (darshan_fd fd, struct darshan_mod_info **mods, int* count);
int darshan_log_get_record (darshan_fd fd, int mod_idx, void **buf);
+void darshan_calc_perf(perf_data_t *pdata, int64_t nprocs);
/* convenience macros for printing Darshan counters */
#define DARSHAN_PRINT_HEADER() \
=====================================
darshan-util/darshan-lustre-logutils.c
=====================================
--- a/darshan-util/darshan-lustre-logutils.c
+++ b/darshan-util/darshan-lustre-logutils.c
@@ -40,7 +40,13 @@ struct darshan_mod_logutil_funcs lustre_logutils =
.log_print_record = &darshan_log_print_lustre_record,
.log_print_description = &darshan_log_print_lustre_description,
.log_print_diff = &darshan_log_print_lustre_record_diff,
- .log_agg_records = &darshan_log_agg_lustre_records
+ .log_agg_records = &darshan_log_agg_lustre_records,
+ .log_accum_file = NULL,
+ .log_accum_perf = NULL,
+ .log_calc_file = NULL,
+ .log_print_total_file = NULL,
+ .log_file_list = NULL,
+ .log_calc_perf = NULL
};
static int darshan_log_get_lustre_record(darshan_fd fd, void** lustre_buf_p)
=====================================
darshan-util/darshan-mpiio-logutils.c
=====================================
--- a/darshan-util/darshan-mpiio-logutils.c
+++ b/darshan-util/darshan-mpiio-logutils.c
@@ -38,6 +38,11 @@ static void darshan_log_print_mpiio_description(int ver);
static void darshan_log_print_mpiio_file_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag);
+static void darshan_log_mpiio_accum_file(void *mfile, hash_entry_t *hfile, int64_t nprocs);
+static void darshan_log_mpiio_accum_perf(void *mfile, perf_data_t *pdata);
+static void darshan_log_mpiio_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
+static void darshan_log_mpiio_print_total_file(void *mfile, int mpiio_ver);
+static void darshan_log_mpiio_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
struct darshan_mod_logutil_funcs mpiio_logutils =
{
@@ -46,7 +51,13 @@ struct darshan_mod_logutil_funcs mpiio_logutils =
.log_print_record = &darshan_log_print_mpiio_file,
.log_print_description = &darshan_log_print_mpiio_description,
.log_print_diff = &darshan_log_print_mpiio_file_diff,
- .log_agg_records = &darshan_log_agg_mpiio_files
+ .log_agg_records = &darshan_log_agg_mpiio_files,
+ .log_accum_file = &darshan_log_mpiio_accum_file,
+ .log_accum_perf = &darshan_log_mpiio_accum_perf,
+ .log_calc_file = &darshan_log_mpiio_calc_file,
+ .log_print_total_file = &darshan_log_mpiio_print_total_file,
+ .log_file_list = &darshan_log_mpiio_file_list,
+ .log_calc_perf = &darshan_calc_perf
};
static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p)
@@ -541,6 +552,447 @@ static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag)
return;
}
+static void darshan_log_mpiio_accum_file(
+ void *infile,
+ hash_entry_t *hfile,
+ int64_t nprocs)
+{
+ struct darshan_mpiio_file *mfile = infile;
+ int i, j;
+ int set;
+ int min_ndx;
+ int64_t min;
+ struct darshan_mpiio_file* tmp;
+
+ hfile->procs += 1;
+
+ if(mfile->base_rec.rank == -1)
+ {
+ hfile->slowest_time = mfile->fcounters[MPIIO_F_SLOWEST_RANK_TIME];
+ }
+ else
+ {
+ hfile->slowest_time = max(hfile->slowest_time,
+ (mfile->fcounters[MPIIO_F_META_TIME] +
+ mfile->fcounters[MPIIO_F_READ_TIME] +
+ mfile->fcounters[MPIIO_F_WRITE_TIME]));
+ }
+
+ if(mfile->base_rec.rank == -1)
+ {
+ hfile->procs = nprocs;
+ hfile->type |= FILETYPE_SHARED;
+
+ }
+ else if(hfile->procs > 1)
+ {
+ hfile->type &= (~FILETYPE_UNIQUE);
+ hfile->type |= FILETYPE_PARTSHARED;
+ }
+ else
+ {
+ hfile->type |= FILETYPE_UNIQUE;
+ }
+
+ hfile->cumul_time += mfile->fcounters[MPIIO_F_META_TIME] +
+ mfile->fcounters[MPIIO_F_READ_TIME] +
+ mfile->fcounters[MPIIO_F_WRITE_TIME];
+
+ if(hfile->rec_dat == NULL)
+ {
+ hfile->rec_dat = malloc(sizeof(struct darshan_mpiio_file));
+ assert(hfile->rec_dat);
+ memset(hfile->rec_dat, 0, sizeof(struct darshan_mpiio_file));
+ }
+ tmp = (struct darshan_mpiio_file*)hfile->rec_dat;
+
+ for(i = 0; i < MPIIO_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case MPIIO_MODE:
+ tmp->counters[i] = mfile->counters[i];
+ break;
+ case MPIIO_ACCESS1_ACCESS:
+ case MPIIO_ACCESS2_ACCESS:
+ case MPIIO_ACCESS3_ACCESS:
+ case MPIIO_ACCESS4_ACCESS:
+ /*
+ * do nothing here because these will be stored
+ * when the _COUNT is accessed.
+ */
+ break;
+ case MPIIO_ACCESS1_COUNT:
+ case MPIIO_ACCESS2_COUNT:
+ case MPIIO_ACCESS3_COUNT:
+ case MPIIO_ACCESS4_COUNT:
+ set = 0;
+ min_ndx = MPIIO_ACCESS1_COUNT;
+ min = tmp->counters[min_ndx];
+ for(j = MPIIO_ACCESS1_COUNT; j <= MPIIO_ACCESS4_COUNT; j++)
+ {
+ if(tmp->counters[j-4] == mfile->counters[i-4])
+ {
+ tmp->counters[j] += mfile->counters[i];
+ set = 1;
+ break;
+ }
+ if(tmp->counters[j] < min)
+ {
+ min_ndx = j;
+ min = tmp->counters[j];
+ }
+ }
+ if(!set && (mfile->counters[i] > min))
+ {
+ tmp->counters[i] = mfile->counters[i];
+ tmp->counters[i-4] = mfile->counters[i-4];
+ }
+ break;
+ case MPIIO_FASTEST_RANK:
+ case MPIIO_SLOWEST_RANK:
+ case MPIIO_FASTEST_RANK_BYTES:
+ case MPIIO_SLOWEST_RANK_BYTES:
+ tmp->counters[i] = 0;
+ break;
+ case MPIIO_MAX_READ_TIME_SIZE:
+ case MPIIO_MAX_WRITE_TIME_SIZE:
+ break;
+ default:
+ tmp->counters[i] += mfile->counters[i];
+ break;
+ }
+ }
+
+ for(i = 0; i < MPIIO_F_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case MPIIO_F_OPEN_TIMESTAMP:
+ case MPIIO_F_READ_START_TIMESTAMP:
+ case MPIIO_F_WRITE_START_TIMESTAMP:
+ if(tmp->fcounters[i] == 0 ||
+ tmp->fcounters[i] > mfile->fcounters[i])
+ {
+ tmp->fcounters[i] = mfile->fcounters[i];
+ }
+ break;
+ case MPIIO_F_READ_END_TIMESTAMP:
+ case MPIIO_F_WRITE_END_TIMESTAMP:
+ case MPIIO_F_CLOSE_TIMESTAMP:
+ if(tmp->fcounters[i] == 0 ||
+ tmp->fcounters[i] < mfile->fcounters[i])
+ {
+ tmp->fcounters[i] = mfile->fcounters[i];
+ }
+ break;
+ case MPIIO_F_FASTEST_RANK_TIME:
+ case MPIIO_F_SLOWEST_RANK_TIME:
+ case MPIIO_F_VARIANCE_RANK_TIME:
+ case MPIIO_F_VARIANCE_RANK_BYTES:
+ tmp->fcounters[i] = 0;
+ break;
+ case MPIIO_F_MAX_READ_TIME:
+ if(tmp->fcounters[i] < mfile->fcounters[i])
+ {
+ tmp->fcounters[i] = mfile->fcounters[i];
+ tmp->counters[MPIIO_MAX_READ_TIME_SIZE] =
+ mfile->counters[MPIIO_MAX_READ_TIME_SIZE];
+ }
+ break;
+ case MPIIO_F_MAX_WRITE_TIME:
+ if(tmp->fcounters[i] < mfile->fcounters[i])
+ {
+ tmp->fcounters[i] = mfile->fcounters[i];
+ tmp->counters[MPIIO_MAX_WRITE_TIME_SIZE] =
+ mfile->counters[MPIIO_MAX_WRITE_TIME_SIZE];
+ }
+ break;
+ default:
+ tmp->fcounters[i] += mfile->fcounters[i];
+ break;
+ }
+ }
+
+ return;
+}
+
+static void darshan_log_mpiio_accum_perf(
+ void *infile,
+ perf_data_t *pdata)
+{
+ struct darshan_mpiio_file *mfile = infile;
+ pdata->total_bytes += mfile->counters[MPIIO_BYTES_READ] +
+ mfile->counters[MPIIO_BYTES_WRITTEN];
+
+ /*
+ * Calculation of Shared File Time
+ * Four Methods!!!!
+ * by_cumul: sum time counters and divide by nprocs
+ * (inaccurate if lots of variance between procs)
+ * by_open: difference between timestamp of open and close
+ * (inaccurate if file is left open without i/o happening)
+ * by_open_lastio: difference between timestamp of open and the
+ * timestamp of last i/o
+ * (similar to above but fixes case where file is left
+ * open after io is complete)
+ * by_slowest: use slowest rank time from log data
+ * (most accurate but requires newer log version)
+ */
+ if(mfile->base_rec.rank == -1)
+ {
+ /* by_open */
+ if(mfile->fcounters[MPIIO_F_CLOSE_TIMESTAMP] >
+ mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP])
+ {
+ pdata->shared_time_by_open +=
+ mfile->fcounters[MPIIO_F_CLOSE_TIMESTAMP] -
+ mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP];
+ }
+
+ /* by_open_lastio */
+ if(mfile->fcounters[MPIIO_F_READ_END_TIMESTAMP] >
+ mfile->fcounters[MPIIO_F_WRITE_END_TIMESTAMP])
+ {
+ /* be careful: file may have been opened but not read or written */
+ if(mfile->fcounters[MPIIO_F_READ_END_TIMESTAMP] > mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP])
+ {
+ pdata->shared_time_by_open_lastio +=
+ mfile->fcounters[MPIIO_F_READ_END_TIMESTAMP] -
+ mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP];
+ }
+ }
+ else
+ {
+ /* be careful: file may have been opened but not read or written */
+ if(mfile->fcounters[MPIIO_F_WRITE_END_TIMESTAMP] > mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP])
+ {
+ pdata->shared_time_by_open_lastio +=
+ mfile->fcounters[MPIIO_F_WRITE_END_TIMESTAMP] -
+ mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP];
+ }
+ }
+
+ pdata->shared_time_by_cumul +=
+ mfile->fcounters[MPIIO_F_META_TIME] +
+ mfile->fcounters[MPIIO_F_READ_TIME] +
+ mfile->fcounters[MPIIO_F_WRITE_TIME];
+ pdata->shared_meta_time += mfile->fcounters[MPIIO_F_META_TIME];
+
+ /* by_slowest */
+ pdata->shared_time_by_slowest +=
+ mfile->fcounters[MPIIO_F_SLOWEST_RANK_TIME];
+ }
+
+ /*
+ * Calculation of Unique File Time
+ * record the data for each file and sum it
+ */
+ else
+ {
+ pdata->rank_cumul_io_time[mfile->base_rec.rank] +=
+ (mfile->fcounters[MPIIO_F_META_TIME] +
+ mfile->fcounters[MPIIO_F_READ_TIME] +
+ mfile->fcounters[MPIIO_F_WRITE_TIME]);
+ pdata->rank_cumul_md_time[mfile->base_rec.rank] +=
+ mfile->fcounters[MPIIO_F_META_TIME];
+ }
+
+ return;
+}
+
+static void darshan_log_mpiio_calc_file(
+ hash_entry_t *file_hash,
+ file_data_t *fdata)
+{
+ hash_entry_t *curr = NULL;
+ hash_entry_t *tmp = NULL;
+ struct darshan_mpiio_file *file_rec;
+
+ memset(fdata, 0, sizeof(*fdata));
+ HASH_ITER(hlink, file_hash, curr, tmp)
+ {
+ int64_t bytes;
+ int64_t r;
+ int64_t w;
+
+ file_rec = (struct darshan_mpiio_file*)curr->rec_dat;
+ assert(file_rec);
+
+ bytes = file_rec->counters[MPIIO_BYTES_READ] +
+ file_rec->counters[MPIIO_BYTES_WRITTEN];
+
+ r = (file_rec->counters[MPIIO_INDEP_READS]+
+ file_rec->counters[MPIIO_COLL_READS] +
+ file_rec->counters[MPIIO_SPLIT_READS] +
+ file_rec->counters[MPIIO_NB_READS]);
+
+ w = (file_rec->counters[MPIIO_INDEP_WRITES]+
+ file_rec->counters[MPIIO_COLL_WRITES] +
+ file_rec->counters[MPIIO_SPLIT_WRITES] +
+ file_rec->counters[MPIIO_NB_WRITES]);
+
+ fdata->total += 1;
+ fdata->total_size += bytes;
+ fdata->total_max = max(fdata->total_max, bytes);
+
+ if (r && !w)
+ {
+ fdata->read_only += 1;
+ fdata->read_only_size += bytes;
+ fdata->read_only_max = max(fdata->read_only_max, bytes);
+ }
+
+ if (!r && w)
+ {
+ fdata->write_only += 1;
+ fdata->write_only_size += bytes;
+ fdata->write_only_max = max(fdata->write_only_max, bytes);
+ }
+
+ if (r && w)
+ {
+ fdata->read_write += 1;
+ fdata->read_write_size += bytes;
+ fdata->read_write_max = max(fdata->read_write_max, bytes);
+ }
+
+ if ((curr->type & (FILETYPE_SHARED|FILETYPE_PARTSHARED)))
+ {
+ fdata->shared += 1;
+ fdata->shared_size += bytes;
+ fdata->shared_max = max(fdata->shared_max, bytes);
+ }
+ if ((curr->type & (FILETYPE_UNIQUE)))
+ {
+ fdata->unique += 1;
+ fdata->unique_size += bytes;
+ fdata->unique_max = max(fdata->unique_max, bytes);
+ }
+ }
+
+ return;
+}
+
+static void darshan_log_mpiio_print_total_file(
+ void *infile,
+ int mpiio_ver)
+{
+ struct darshan_mpiio_file *mfile = infile;
+ int i;
+
+ mod_logutils[DARSHAN_MPIIO_MOD]->log_print_description(mpiio_ver);
+ printf("\n");
+ for(i = 0; i < MPIIO_NUM_INDICES; i++)
+ {
+ printf("total_%s: %"PRId64"\n",
+ mpiio_counter_names[i], mfile->counters[i]);
+ }
+ for(i = 0; i < MPIIO_F_NUM_INDICES; i++)
+ {
+ printf("total_%s: %lf\n",
+ mpiio_f_counter_names[i], mfile->fcounters[i]);
+ }
+ return;
+}
+
+static void darshan_log_mpiio_file_list(
+ hash_entry_t *file_hash,
+ struct darshan_name_record_ref *name_hash,
+ int detail_flag)
+{
+ hash_entry_t *curr = NULL;
+ hash_entry_t *tmp = NULL;
+ struct darshan_mpiio_file *file_rec = NULL;
+ struct darshan_name_record_ref *ref = NULL;
+ int i;
+
+ /* list of columns:
+ *
+ * normal mode
+ * - file id
+ * - file name
+ * - nprocs
+ * - slowest I/O time
+ * - average cumulative I/O time
+ *
+ * detailed mode
+ * - first open
+ * - first read
+ * - first write
+ * - last read
+ * - last write
+ * - last close
+ * - MPI indep opens
+ * - MPI coll opens
+ * - r histogram
+ * - w histogram
+ */
+
+ if(detail_flag)
+ printf("\n# Per-file summary of I/O activity (detailed).\n");
+ else
+ printf("\n# Per-file summary of I/O activity.\n");
+ printf("# -----\n");
+
+ printf("# <record_id>: darshan record id for this file\n");
+ printf("# <file_name>: full file name\n");
+ printf("# <nprocs>: number of processes that opened the file\n");
+ printf("# <slowest>: (estimated) time in seconds consumed in IO by slowest process\n");
+ printf("# <avg>: average time in seconds consumed in IO per process\n");
+ if(detail_flag)
+ {
+ printf("# <start_{open/read/write}>: start timestamp of first open, read, or write\n");
+ printf("# <end_{read/write/close}>: end timestamp of last read, write, or close\n");
+ printf("# <mpi_indep_opens>: independent MPI_File_open calls\n");
+ printf("# <mpi_coll_opens>: collective MPI_File_open calls\n");
+ printf("# <MPIIO_SIZE_READ_AGG_*>: MPI-IO aggregate read size histogram\n");
+ printf("# <MPIIO_SIZE_WRITE_AGG_*>: MPI-IO aggregate write size histogram\n");
+ }
+
+ printf("\n# <record_id>\t<file_name>\t<nprocs>\t<slowest>\t<avg>");
+ if(detail_flag)
+ {
+ printf("\t<start_open>\t<start_read>\t<start_write>");
+ printf("\t<end_read>\t<end_write>\t<end_close>");
+ printf("\t<mpi_indep_opens>\t<mpi_coll_opens>");
+ for(i=MPIIO_SIZE_READ_AGG_0_100; i<= MPIIO_SIZE_WRITE_AGG_1G_PLUS; i++)
+ printf("\t<%s>", mpiio_counter_names[i]);
+ }
+ printf("\n");
+
+ HASH_ITER(hlink, file_hash, curr, tmp)
+ {
+ file_rec = (struct darshan_mpiio_file*)curr->rec_dat;
+ assert(file_rec);
+
+ HASH_FIND(hlink, name_hash, &(curr->rec_id), sizeof(darshan_record_id), ref);
+ assert(ref);
+
+ printf("%" PRIu64 "\t%s\t%" PRId64 "\t%f\t%f",
+ curr->rec_id,
+ ref->name_record->name,
+ curr->procs,
+ curr->slowest_time,
+ curr->cumul_time/(double)curr->procs);
+
+ if(detail_flag)
+ {
+ for(i=MPIIO_F_OPEN_TIMESTAMP; i<=MPIIO_F_CLOSE_TIMESTAMP; i++)
+ {
+ printf("\t%f", file_rec->fcounters[i]);
+ }
+ printf("\t%" PRId64 "\t%" PRId64, file_rec->counters[MPIIO_INDEP_OPENS],
+ file_rec->counters[MPIIO_COLL_OPENS]);
+ for(i=MPIIO_SIZE_READ_AGG_0_100; i<= MPIIO_SIZE_WRITE_AGG_1G_PLUS; i++)
+ printf("\t%" PRId64, file_rec->counters[i]);
+ }
+ printf("\n");
+ }
+
+ return;
+}
+
/*
* Local variables:
* c-indent-level: 4
=====================================
darshan-util/darshan-null-logutils.c
=====================================
--- a/darshan-util/darshan-null-logutils.c
+++ b/darshan-util/darshan-null-logutils.c
@@ -52,7 +52,13 @@ struct darshan_mod_logutil_funcs null_logutils =
.log_print_record = &darshan_log_print_null_record,
.log_print_description = &darshan_log_print_null_description,
.log_print_diff = &darshan_log_print_null_record_diff,
- .log_agg_records = &darshan_log_agg_null_records
+ .log_agg_records = &darshan_log_agg_null_records,
+ .log_accum_file = NULL,
+ .log_accum_perf = NULL,
+ .log_calc_file = NULL,
+ .log_print_total_file = NULL,
+ .log_file_list = NULL,
+ .log_calc_perf = NULL
};
/* retrieve a NULL record from log file descriptor 'fd', storing the
=====================================
darshan-util/darshan-parser.c
=====================================
--- a/darshan-util/darshan-parser.c
+++ b/darshan-util/darshan-parser.c
@@ -37,88 +37,9 @@
OPTION_FILE_LIST|\
OPTION_FILE_LIST_DETAILED)
-#define FILETYPE_SHARED (1 << 0)
-#define FILETYPE_UNIQUE (1 << 1)
-#define FILETYPE_PARTSHARED (1 << 2)
-
-#define max(a,b) (((a) > (b)) ? (a) : (b))
-
-/*
- * Datatypes
- */
-typedef struct hash_entry_s
-{
- UT_hash_handle hlink;
- darshan_record_id rec_id;
- int64_t type;
- int64_t procs;
- void *rec_dat;
- double cumul_time;
- double slowest_time;
-} hash_entry_t;
-
-typedef struct file_data_s
-{
- int64_t total;
- int64_t total_size;
- int64_t total_max;
- int64_t read_only;
- int64_t read_only_size;
- int64_t read_only_max;
- int64_t write_only;
- int64_t write_only_size;
- int64_t write_only_max;
- int64_t read_write;
- int64_t read_write_size;
- int64_t read_write_max;
- int64_t unique;
- int64_t unique_size;
- int64_t unique_max;
- int64_t shared;
- int64_t shared_size;
- int64_t shared_max;
-} file_data_t;
-
-typedef struct perf_data_s
-{
- int64_t total_bytes;
- double slowest_rank_time;
- double slowest_rank_meta_time;
- int slowest_rank_rank;
- double shared_time_by_cumul;
- double shared_time_by_open;
- double shared_time_by_open_lastio;
- double shared_time_by_slowest;
- double shared_meta_time;
- double agg_perf_by_cumul;
- double agg_perf_by_open;
- double agg_perf_by_open_lastio;
- double agg_perf_by_slowest;
- double *rank_cumul_io_time;
- double *rank_cumul_md_time;
-} perf_data_t;
-
/*
* Prototypes
*/
-void posix_accum_file(struct darshan_posix_file *pfile, hash_entry_t *hfile, int64_t nprocs);
-void posix_accum_perf(struct darshan_posix_file *pfile, perf_data_t *pdata);
-void posix_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
-void posix_print_total_file(struct darshan_posix_file *pfile, int posix_ver);
-void posix_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
-
-void mpiio_accum_file(struct darshan_mpiio_file *mfile, hash_entry_t *hfile, int64_t nprocs);
-void mpiio_accum_perf(struct darshan_mpiio_file *mfile, perf_data_t *pdata);
-void mpiio_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
-void mpiio_print_total_file(struct darshan_mpiio_file *mfile, int mpiio_ver);
-void mpiio_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
-
-void stdio_accum_perf(struct darshan_stdio_file *pfile, perf_data_t *pdata);
-void stdio_accum_file(struct darshan_stdio_file *pfile, hash_entry_t *hfile, int64_t nprocs);
-void stdio_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
-void stdio_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
-void stdio_print_total_file(struct darshan_stdio_file *pfile, int stdio_ver);
-
void calc_perf(perf_data_t *pdata, int64_t nprocs);
int usage (char *exename)
@@ -333,6 +254,25 @@ int main(int argc, char **argv)
}
}
+#if 0
+ printf("\n# defs ---\n");
+ printf("DARSHAN_EXE_LEN = %d\n", DARSHAN_EXE_LEN);
+ printf("DARSHAN_APXC_MAGIC = 0x%" PRIu64 "\n", DARSHAN_APXC_MAGIC);
+ printf("APXC_NUM_INDICES = %d\n", APXC_NUM_INDICES);
+ printf("POSIX_F_NUM_INDICES = %d\n", POSIX_F_NUM_INDICES);
+ printf("POSIX_NUM_INDICES = %d\n", POSIX_NUM_INDICES);
+ printf("STDIO_F_NUM_INDICES = %d\n", STDIO_F_NUM_INDICES);
+ printf("STDIO_NUM_INDICES = %d\n", STDIO_NUM_INDICES);
+ printf("MPIIO_F_NUM_INDICES = %d\n", MPIIO_F_NUM_INDICES);
+ printf("MPIIO_NUM_INDICES = %d\n", MPIIO_NUM_INDICES);
+ printf("PNETCDF_F_NUM_INDICES = %d\n", PNETCDF_F_NUM_INDICES);
+ printf("PNETCDF_NUM_INDICES = %d\n", PNETCDF_NUM_INDICES);
+ printf("HDF5_F_NUM_INDICES = %d\n", HDF5_F_NUM_INDICES);
+ printf("HDF5_NUM_INDICES = %d\n", HDF5_NUM_INDICES);
+ printf("BGQ_F_NUM_INDICES = %d\n", BGQ_F_NUM_INDICES);
+ printf("BGQ_NUM_INDICES = %d\n", BGQ_NUM_INDICES);
+#endif
+
/* print table of mounted file systems */
printf("\n# mounted file systems (mount point and fs type)\n");
printf("# -------------------------------------------------------\n");
@@ -486,12 +426,10 @@ int main(int argc, char **argv)
mnt_pt, fs_type);
}
- /* we calculate more detailed stats for POSIX, MPI-IO, and STDIO modules,
- * if the parser is executed with more than the base option
+ /*
+ * Compute derived statsitics for those modules that support it
+ * and if the user requested it.
*/
- if(i != DARSHAN_POSIX_MOD && i != DARSHAN_MPIIO_MOD && i != DARSHAN_STDIO_MOD)
- continue;
-
HASH_FIND(hlink, file_hash, &(base_rec->id), sizeof(darshan_record_id), hfile);
if(!hfile)
{
@@ -514,23 +452,15 @@ int main(int argc, char **argv)
HASH_ADD(hlink, file_hash,rec_id, sizeof(darshan_record_id), hfile);
}
- if(i == DARSHAN_POSIX_MOD)
+ if (mod_logutils[i]->log_accum_file)
{
- posix_accum_file((struct darshan_posix_file*)mod_buf, &total, job.nprocs);
- posix_accum_file((struct darshan_posix_file*)mod_buf, hfile, job.nprocs);
- posix_accum_perf((struct darshan_posix_file*)mod_buf, &pdata);
+ mod_logutils[i]->log_accum_file(mod_buf, &total, job.nprocs);
+ mod_logutils[i]->log_accum_file(mod_buf, hfile, job.nprocs);
}
- else if(i == DARSHAN_MPIIO_MOD)
- {
- mpiio_accum_file((struct darshan_mpiio_file*)mod_buf, &total, job.nprocs);
- mpiio_accum_file((struct darshan_mpiio_file*)mod_buf, hfile, job.nprocs);
- mpiio_accum_perf((struct darshan_mpiio_file*)mod_buf, &pdata);
- }
- else if(i == DARSHAN_STDIO_MOD)
+
+ if (mod_logutils[i]->log_accum_perf)
{
- stdio_accum_file((struct darshan_stdio_file*)mod_buf, &total, job.nprocs);
- stdio_accum_file((struct darshan_stdio_file*)mod_buf, hfile, job.nprocs);
- stdio_accum_perf((struct darshan_stdio_file*)mod_buf, &pdata);
+ mod_logutils[i]->log_accum_perf(mod_buf, &pdata);
}
memset(mod_buf, 0, DEF_MOD_BUF_SIZE);
@@ -538,44 +468,21 @@ int main(int argc, char **argv)
if(ret == -1)
continue; /* move on to the next module if there was an error with this one */
- /* we calculate more detailed stats for POSIX and MPI-IO modules,
- * if the parser is executed with more than the base option
+ /* Calculate more detailed stats if the module supports it and
+ * the user requested it.
*/
- if(i != DARSHAN_POSIX_MOD && i != DARSHAN_MPIIO_MOD && i != DARSHAN_STDIO_MOD)
- continue;
-
/* Total Calc */
- if(mask & OPTION_TOTAL)
+ if ((mask & OPTION_TOTAL) &&
+ (mod_logutils[i]->log_print_total_file))
{
- if(i == DARSHAN_POSIX_MOD)
- {
- posix_print_total_file((struct darshan_posix_file*)total.rec_dat, fd->mod_ver[i]);
- }
- else if(i == DARSHAN_MPIIO_MOD)
- {
- mpiio_print_total_file((struct darshan_mpiio_file*)total.rec_dat, fd->mod_ver[i]);
- }
- else if(i == DARSHAN_STDIO_MOD)
- {
- stdio_print_total_file((struct darshan_stdio_file*)total.rec_dat, fd->mod_ver[i]);
- }
+ mod_logutils[i]->log_print_total_file(total.rec_dat, fd->mod_ver[i]);
}
/* File Calc */
- if(mask & OPTION_FILE)
+ if ((mask & OPTION_FILE) &&
+ (mod_logutils[i]->log_calc_file))
{
- if(i == DARSHAN_POSIX_MOD)
- {
- posix_calc_file(file_hash, &fdata);
- }
- else if(i == DARSHAN_MPIIO_MOD)
- {
- mpiio_calc_file(file_hash, &fdata);
- }
- else if(i == DARSHAN_STDIO_MOD)
- {
- stdio_calc_file(file_hash, &fdata);
- }
+ mod_logutils[i]->log_calc_file(file_hash, &fdata);
printf("\n# Total file counts\n");
printf("# -----\n");
@@ -616,9 +523,10 @@ int main(int argc, char **argv)
}
/* Perf Calc */
- if(mask & OPTION_PERF)
+ if ((mask & OPTION_PERF) &&
+ (mod_logutils[i]->log_calc_perf))
{
- calc_perf(&pdata, job.nprocs);
+ mod_logutils[i]->log_calc_perf(&pdata, job.nprocs);
printf("\n# performance\n");
printf("# -----------\n");
@@ -648,29 +556,12 @@ int main(int argc, char **argv)
printf("# agg_perf_by_slowest: %lf\n", pdata.agg_perf_by_slowest);
}
- if((mask & OPTION_FILE_LIST) || (mask & OPTION_FILE_LIST_DETAILED))
+ if (((mask & OPTION_FILE_LIST) ||
+ (mask & OPTION_FILE_LIST_DETAILED)) &&
+ (mod_logutils[i]->log_file_list))
{
- if(i == DARSHAN_POSIX_MOD)
- {
- if(mask & OPTION_FILE_LIST_DETAILED)
- posix_file_list(file_hash, name_hash, 1);
- else
- posix_file_list(file_hash, name_hash, 0);
- }
- else if(i == DARSHAN_MPIIO_MOD)
- {
- if(mask & OPTION_FILE_LIST_DETAILED)
- mpiio_file_list(file_hash, name_hash, 1);
- else
- mpiio_file_list(file_hash, name_hash, 0);
- }
- else if(i == DARSHAN_STDIO_MOD)
- {
- if(mask & OPTION_FILE_LIST_DETAILED)
- stdio_file_list(file_hash, name_hash, 1);
- else
- stdio_file_list(file_hash, name_hash, 0);
- }
+ int mode = (mask & OPTION_FILE_LIST_DETAILED) ? 1 : 0;
+ mod_logutils[i]->log_file_list(file_hash, name_hash, mode);
}
/* reset data structures for next module */
@@ -719,1313 +610,6 @@ cleanup:
return(ret);
}
-void stdio_accum_file(struct darshan_stdio_file *pfile,
- hash_entry_t *hfile,
- int64_t nprocs)
-{
- int i;
- struct darshan_stdio_file* tmp;
-
- hfile->procs += 1;
-
- if(pfile->base_rec.rank == -1)
- {
- hfile->slowest_time = pfile->fcounters[STDIO_F_SLOWEST_RANK_TIME];
- }
- else
- {
- hfile->slowest_time = max(hfile->slowest_time,
- (pfile->fcounters[STDIO_F_META_TIME] +
- pfile->fcounters[STDIO_F_READ_TIME] +
- pfile->fcounters[STDIO_F_WRITE_TIME]));
- }
-
- if(pfile->base_rec.rank == -1)
- {
- hfile->procs = nprocs;
- hfile->type |= FILETYPE_SHARED;
-
- }
- else if(hfile->procs > 1)
- {
- hfile->type &= (~FILETYPE_UNIQUE);
- hfile->type |= FILETYPE_PARTSHARED;
- }
- else
- {
- hfile->type |= FILETYPE_UNIQUE;
- }
-
- hfile->cumul_time += pfile->fcounters[STDIO_F_META_TIME] +
- pfile->fcounters[STDIO_F_READ_TIME] +
- pfile->fcounters[STDIO_F_WRITE_TIME];
-
- if(hfile->rec_dat == NULL)
- {
- hfile->rec_dat = malloc(sizeof(struct darshan_stdio_file));
- assert(hfile->rec_dat);
- memset(hfile->rec_dat, 0, sizeof(struct darshan_stdio_file));
- }
- tmp = (struct darshan_stdio_file*)hfile->rec_dat;
-
- for(i = 0; i < STDIO_NUM_INDICES; i++)
- {
- switch(i)
- {
- case STDIO_MAX_BYTE_READ:
- case STDIO_MAX_BYTE_WRITTEN:
- if (tmp->counters[i] < pfile->counters[i])
- {
- tmp->counters[i] = pfile->counters[i];
- }
- break;
- case POSIX_FASTEST_RANK:
- case POSIX_SLOWEST_RANK:
- case POSIX_FASTEST_RANK_BYTES:
- case POSIX_SLOWEST_RANK_BYTES:
- tmp->counters[i] = 0;
- break;
- default:
- tmp->counters[i] += pfile->counters[i];
- break;
- }
- }
-
- for(i = 0; i < STDIO_F_NUM_INDICES; i++)
- {
- switch(i)
- {
- case STDIO_F_OPEN_START_TIMESTAMP:
- case STDIO_F_CLOSE_START_TIMESTAMP:
- case STDIO_F_READ_START_TIMESTAMP:
- case STDIO_F_WRITE_START_TIMESTAMP:
- if(tmp->fcounters[i] == 0 ||
- tmp->fcounters[i] > pfile->fcounters[i])
- {
- tmp->fcounters[i] = pfile->fcounters[i];
- }
- break;
- case STDIO_F_READ_END_TIMESTAMP:
- case STDIO_F_WRITE_END_TIMESTAMP:
- case STDIO_F_OPEN_END_TIMESTAMP:
- case STDIO_F_CLOSE_END_TIMESTAMP:
- if(tmp->fcounters[i] == 0 ||
- tmp->fcounters[i] < pfile->fcounters[i])
- {
- tmp->fcounters[i] = pfile->fcounters[i];
- }
- break;
- case STDIO_F_FASTEST_RANK_TIME:
- case STDIO_F_SLOWEST_RANK_TIME:
- case STDIO_F_VARIANCE_RANK_TIME:
- case STDIO_F_VARIANCE_RANK_BYTES:
- tmp->fcounters[i] = 0;
- break;
- default:
- tmp->fcounters[i] += pfile->fcounters[i];
- break;
- }
- }
-
- return;
-}
-
-void posix_accum_file(struct darshan_posix_file *pfile,
- hash_entry_t *hfile,
- int64_t nprocs)
-{
- int i, j;
- int set;
- int min_ndx;
- int64_t min;
- struct darshan_posix_file* tmp;
-
- hfile->procs += 1;
-
- if(pfile->base_rec.rank == -1)
- {
- hfile->slowest_time = pfile->fcounters[POSIX_F_SLOWEST_RANK_TIME];
- }
- else
- {
- hfile->slowest_time = max(hfile->slowest_time,
- (pfile->fcounters[POSIX_F_META_TIME] +
- pfile->fcounters[POSIX_F_READ_TIME] +
- pfile->fcounters[POSIX_F_WRITE_TIME]));
- }
-
- if(pfile->base_rec.rank == -1)
- {
- hfile->procs = nprocs;
- hfile->type |= FILETYPE_SHARED;
-
- }
- else if(hfile->procs > 1)
- {
- hfile->type &= (~FILETYPE_UNIQUE);
- hfile->type |= FILETYPE_PARTSHARED;
- }
- else
- {
- hfile->type |= FILETYPE_UNIQUE;
- }
-
- hfile->cumul_time += pfile->fcounters[POSIX_F_META_TIME] +
- pfile->fcounters[POSIX_F_READ_TIME] +
- pfile->fcounters[POSIX_F_WRITE_TIME];
-
- if(hfile->rec_dat == NULL)
- {
- hfile->rec_dat = malloc(sizeof(struct darshan_posix_file));
- assert(hfile->rec_dat);
- memset(hfile->rec_dat, 0, sizeof(struct darshan_posix_file));
- }
- tmp = (struct darshan_posix_file*)hfile->rec_dat;
-
- for(i = 0; i < POSIX_NUM_INDICES; i++)
- {
- switch(i)
- {
- case POSIX_MODE:
- case POSIX_MEM_ALIGNMENT:
- case POSIX_FILE_ALIGNMENT:
- tmp->counters[i] = pfile->counters[i];
- break;
- case POSIX_MAX_BYTE_READ:
- case POSIX_MAX_BYTE_WRITTEN:
- if (tmp->counters[i] < pfile->counters[i])
- {
- tmp->counters[i] = pfile->counters[i];
- }
- break;
- case POSIX_STRIDE1_STRIDE:
- case POSIX_STRIDE2_STRIDE:
- case POSIX_STRIDE3_STRIDE:
- case POSIX_STRIDE4_STRIDE:
- case POSIX_ACCESS1_ACCESS:
- case POSIX_ACCESS2_ACCESS:
- case POSIX_ACCESS3_ACCESS:
- case POSIX_ACCESS4_ACCESS:
- /*
- * do nothing here because these will be stored
- * when the _COUNT is accessed.
- */
- break;
- case POSIX_STRIDE1_COUNT:
- case POSIX_STRIDE2_COUNT:
- case POSIX_STRIDE3_COUNT:
- case POSIX_STRIDE4_COUNT:
- set = 0;
- min_ndx = POSIX_STRIDE1_COUNT;
- min = tmp->counters[min_ndx];
- for(j = POSIX_STRIDE1_COUNT; j <= POSIX_STRIDE4_COUNT; j++)
- {
- if(tmp->counters[j-4] == pfile->counters[i-4])
- {
- tmp->counters[j] += pfile->counters[i];
- set = 1;
- break;
- }
- if(tmp->counters[j] < min)
- {
- min_ndx = j;
- min = tmp->counters[j];
- }
- }
- if(!set && (pfile->counters[i] > min))
- {
- tmp->counters[min_ndx] = pfile->counters[i];
- tmp->counters[min_ndx-4] = pfile->counters[i-4];
- }
- break;
- case POSIX_ACCESS1_COUNT:
- case POSIX_ACCESS2_COUNT:
- case POSIX_ACCESS3_COUNT:
- case POSIX_ACCESS4_COUNT:
- set = 0;
- min_ndx = POSIX_ACCESS1_COUNT;
- min = tmp->counters[min_ndx];
- for(j = POSIX_ACCESS1_COUNT; j <= POSIX_ACCESS4_COUNT; j++)
- {
- if(tmp->counters[j-4] == pfile->counters[i-4])
- {
- tmp->counters[j] += pfile->counters[i];
- set = 1;
- break;
- }
- if(tmp->counters[j] < min)
- {
- min_ndx = j;
- min = tmp->counters[j];
- }
- }
- if(!set && (pfile->counters[i] > min))
- {
- tmp->counters[i] = pfile->counters[i];
- tmp->counters[i-4] = pfile->counters[i-4];
- }
- break;
- case POSIX_FASTEST_RANK:
- case POSIX_SLOWEST_RANK:
- case POSIX_FASTEST_RANK_BYTES:
- case POSIX_SLOWEST_RANK_BYTES:
- tmp->counters[i] = 0;
- break;
- case POSIX_MAX_READ_TIME_SIZE:
- case POSIX_MAX_WRITE_TIME_SIZE:
- break;
- default:
- tmp->counters[i] += pfile->counters[i];
- break;
- }
- }
-
- for(i = 0; i < POSIX_F_NUM_INDICES; i++)
- {
- switch(i)
- {
- case POSIX_F_OPEN_START_TIMESTAMP:
- case POSIX_F_READ_START_TIMESTAMP:
- case POSIX_F_WRITE_START_TIMESTAMP:
- case POSIX_F_CLOSE_START_TIMESTAMP:
- if(tmp->fcounters[i] == 0 ||
- tmp->fcounters[i] > pfile->fcounters[i])
- {
- tmp->fcounters[i] = pfile->fcounters[i];
- }
- break;
- case POSIX_F_OPEN_END_TIMESTAMP:
- case POSIX_F_READ_END_TIMESTAMP:
- case POSIX_F_WRITE_END_TIMESTAMP:
- case POSIX_F_CLOSE_END_TIMESTAMP:
- if(tmp->fcounters[i] == 0 ||
- tmp->fcounters[i] < pfile->fcounters[i])
- {
- tmp->fcounters[i] = pfile->fcounters[i];
- }
- break;
- case POSIX_F_FASTEST_RANK_TIME:
- case POSIX_F_SLOWEST_RANK_TIME:
- case POSIX_F_VARIANCE_RANK_TIME:
- case POSIX_F_VARIANCE_RANK_BYTES:
- tmp->fcounters[i] = 0;
- break;
- case POSIX_F_MAX_READ_TIME:
- if(tmp->fcounters[i] < pfile->fcounters[i])
- {
- tmp->fcounters[i] = pfile->fcounters[i];
- tmp->counters[POSIX_MAX_READ_TIME_SIZE] =
- pfile->counters[POSIX_MAX_READ_TIME_SIZE];
- }
- break;
- case POSIX_F_MAX_WRITE_TIME:
- if(tmp->fcounters[i] < pfile->fcounters[i])
- {
- tmp->fcounters[i] = pfile->fcounters[i];
- tmp->counters[POSIX_MAX_WRITE_TIME_SIZE] =
- pfile->counters[POSIX_MAX_WRITE_TIME_SIZE];
- }
- break;
- default:
- tmp->fcounters[i] += pfile->fcounters[i];
- break;
- }
- }
-
- return;
-}
-
-void mpiio_accum_file(struct darshan_mpiio_file *mfile,
- hash_entry_t *hfile,
- int64_t nprocs)
-{
- int i, j;
- int set;
- int min_ndx;
- int64_t min;
- struct darshan_mpiio_file* tmp;
-
- hfile->procs += 1;
-
- if(mfile->base_rec.rank == -1)
- {
- hfile->slowest_time = mfile->fcounters[MPIIO_F_SLOWEST_RANK_TIME];
- }
- else
- {
- hfile->slowest_time = max(hfile->slowest_time,
- (mfile->fcounters[MPIIO_F_META_TIME] +
- mfile->fcounters[MPIIO_F_READ_TIME] +
- mfile->fcounters[MPIIO_F_WRITE_TIME]));
- }
-
- if(mfile->base_rec.rank == -1)
- {
- hfile->procs = nprocs;
- hfile->type |= FILETYPE_SHARED;
-
- }
- else if(hfile->procs > 1)
- {
- hfile->type &= (~FILETYPE_UNIQUE);
- hfile->type |= FILETYPE_PARTSHARED;
- }
- else
- {
- hfile->type |= FILETYPE_UNIQUE;
- }
-
- hfile->cumul_time += mfile->fcounters[MPIIO_F_META_TIME] +
- mfile->fcounters[MPIIO_F_READ_TIME] +
- mfile->fcounters[MPIIO_F_WRITE_TIME];
-
- if(hfile->rec_dat == NULL)
- {
- hfile->rec_dat = malloc(sizeof(struct darshan_mpiio_file));
- assert(hfile->rec_dat);
- memset(hfile->rec_dat, 0, sizeof(struct darshan_mpiio_file));
- }
- tmp = (struct darshan_mpiio_file*)hfile->rec_dat;
-
- for(i = 0; i < MPIIO_NUM_INDICES; i++)
- {
- switch(i)
- {
- case MPIIO_MODE:
- tmp->counters[i] = mfile->counters[i];
- break;
- case MPIIO_ACCESS1_ACCESS:
- case MPIIO_ACCESS2_ACCESS:
- case MPIIO_ACCESS3_ACCESS:
- case MPIIO_ACCESS4_ACCESS:
- /*
- * do nothing here because these will be stored
- * when the _COUNT is accessed.
- */
- break;
- case MPIIO_ACCESS1_COUNT:
- case MPIIO_ACCESS2_COUNT:
- case MPIIO_ACCESS3_COUNT:
- case MPIIO_ACCESS4_COUNT:
- set = 0;
- min_ndx = MPIIO_ACCESS1_COUNT;
- min = tmp->counters[min_ndx];
- for(j = MPIIO_ACCESS1_COUNT; j <= MPIIO_ACCESS4_COUNT; j++)
- {
- if(tmp->counters[j-4] == mfile->counters[i-4])
- {
- tmp->counters[j] += mfile->counters[i];
- set = 1;
- break;
- }
- if(tmp->counters[j] < min)
- {
- min_ndx = j;
- min = tmp->counters[j];
- }
- }
- if(!set && (mfile->counters[i] > min))
- {
- tmp->counters[i] = mfile->counters[i];
- tmp->counters[i-4] = mfile->counters[i-4];
- }
- break;
- case MPIIO_FASTEST_RANK:
- case MPIIO_SLOWEST_RANK:
- case MPIIO_FASTEST_RANK_BYTES:
- case MPIIO_SLOWEST_RANK_BYTES:
- tmp->counters[i] = 0;
- break;
- case MPIIO_MAX_READ_TIME_SIZE:
- case MPIIO_MAX_WRITE_TIME_SIZE:
- break;
- default:
- tmp->counters[i] += mfile->counters[i];
- break;
- }
- }
-
- for(i = 0; i < MPIIO_F_NUM_INDICES; i++)
- {
- switch(i)
- {
- case MPIIO_F_OPEN_TIMESTAMP:
- case MPIIO_F_READ_START_TIMESTAMP:
- case MPIIO_F_WRITE_START_TIMESTAMP:
- if(tmp->fcounters[i] == 0 ||
- tmp->fcounters[i] > mfile->fcounters[i])
- {
- tmp->fcounters[i] = mfile->fcounters[i];
- }
- break;
- case MPIIO_F_READ_END_TIMESTAMP:
- case MPIIO_F_WRITE_END_TIMESTAMP:
- case MPIIO_F_CLOSE_TIMESTAMP:
- if(tmp->fcounters[i] == 0 ||
- tmp->fcounters[i] < mfile->fcounters[i])
- {
- tmp->fcounters[i] = mfile->fcounters[i];
- }
- break;
- case MPIIO_F_FASTEST_RANK_TIME:
- case MPIIO_F_SLOWEST_RANK_TIME:
- case MPIIO_F_VARIANCE_RANK_TIME:
- case MPIIO_F_VARIANCE_RANK_BYTES:
- tmp->fcounters[i] = 0;
- break;
- case MPIIO_F_MAX_READ_TIME:
- if(tmp->fcounters[i] < mfile->fcounters[i])
- {
- tmp->fcounters[i] = mfile->fcounters[i];
- tmp->counters[MPIIO_MAX_READ_TIME_SIZE] =
- mfile->counters[MPIIO_MAX_READ_TIME_SIZE];
- }
- break;
- case MPIIO_F_MAX_WRITE_TIME:
- if(tmp->fcounters[i] < mfile->fcounters[i])
- {
- tmp->fcounters[i] = mfile->fcounters[i];
- tmp->counters[MPIIO_MAX_WRITE_TIME_SIZE] =
- mfile->counters[MPIIO_MAX_WRITE_TIME_SIZE];
- }
- break;
- default:
- tmp->fcounters[i] += mfile->fcounters[i];
- break;
- }
- }
-
- return;
-}
-
-void stdio_accum_perf(struct darshan_stdio_file *pfile,
- perf_data_t *pdata)
-{
- pdata->total_bytes += pfile->counters[STDIO_BYTES_READ] +
- pfile->counters[STDIO_BYTES_WRITTEN];
-
- /*
- * Calculation of Shared File Time
- * Four Methods!!!!
- * by_cumul: sum time counters and divide by nprocs
- * (inaccurate if lots of variance between procs)
- * by_open: difference between timestamp of open and close
- * (inaccurate if file is left open without i/o happening)
- * by_open_lastio: difference between timestamp of open and the
- * timestamp of last i/o
- * (similar to above but fixes case where file is left
- * open after io is complete)
- * by_slowest: use slowest rank time from log data
- * (most accurate but requires newer log version)
- */
- if(pfile->base_rec.rank == -1)
- {
- /* by_open */
- if(pfile->fcounters[STDIO_F_CLOSE_END_TIMESTAMP] >
- pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP])
- {
- pdata->shared_time_by_open +=
- pfile->fcounters[STDIO_F_CLOSE_END_TIMESTAMP] -
- pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP];
- }
-
- /* by_open_lastio */
- if(pfile->fcounters[STDIO_F_READ_END_TIMESTAMP] >
- pfile->fcounters[STDIO_F_WRITE_END_TIMESTAMP])
- {
- /* be careful: file may have been opened but not read or written */
- if(pfile->fcounters[STDIO_F_READ_END_TIMESTAMP] > pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP])
- {
- pdata->shared_time_by_open_lastio +=
- pfile->fcounters[STDIO_F_READ_END_TIMESTAMP] -
- pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP];
- }
- }
- else
- {
- /* be careful: file may have been opened but not read or written */
- if(pfile->fcounters[STDIO_F_WRITE_END_TIMESTAMP] > pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP])
- {
- pdata->shared_time_by_open_lastio +=
- pfile->fcounters[STDIO_F_WRITE_END_TIMESTAMP] -
- pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP];
- }
- }
-
- pdata->shared_time_by_cumul +=
- pfile->fcounters[STDIO_F_META_TIME] +
- pfile->fcounters[STDIO_F_READ_TIME] +
- pfile->fcounters[STDIO_F_WRITE_TIME];
- pdata->shared_meta_time += pfile->fcounters[STDIO_F_META_TIME];
-
- /* by_slowest */
- pdata->shared_time_by_slowest +=
- pfile->fcounters[STDIO_F_SLOWEST_RANK_TIME];
- }
-
- /*
- * Calculation of Unique File Time
- * record the data for each file and sum it
- */
- else
- {
- pdata->rank_cumul_io_time[pfile->base_rec.rank] +=
- (pfile->fcounters[STDIO_F_META_TIME] +
- pfile->fcounters[STDIO_F_READ_TIME] +
- pfile->fcounters[STDIO_F_WRITE_TIME]);
- pdata->rank_cumul_md_time[pfile->base_rec.rank] +=
- pfile->fcounters[STDIO_F_META_TIME];
- }
-
- return;
-}
-
-
-void posix_accum_perf(struct darshan_posix_file *pfile,
- perf_data_t *pdata)
-{
- pdata->total_bytes += pfile->counters[POSIX_BYTES_READ] +
- pfile->counters[POSIX_BYTES_WRITTEN];
-
- /*
- * Calculation of Shared File Time
- * Four Methods!!!!
- * by_cumul: sum time counters and divide by nprocs
- * (inaccurate if lots of variance between procs)
- * by_open: difference between timestamp of open and close
- * (inaccurate if file is left open without i/o happening)
- * by_open_lastio: difference between timestamp of open and the
- * timestamp of last i/o
- * (similar to above but fixes case where file is left
- * open after io is complete)
- * by_slowest: use slowest rank time from log data
- * (most accurate but requires newer log version)
- */
- if(pfile->base_rec.rank == -1)
- {
- /* by_open */
- if(pfile->fcounters[POSIX_F_CLOSE_END_TIMESTAMP] >
- pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
- {
- pdata->shared_time_by_open +=
- pfile->fcounters[POSIX_F_CLOSE_END_TIMESTAMP] -
- pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
- }
-
- /* by_open_lastio */
- if(pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] >
- pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP])
- {
- /* be careful: file may have been opened but not read or written */
- if(pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] > pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
- {
- pdata->shared_time_by_open_lastio +=
- pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] -
- pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
- }
- }
- else
- {
- /* be careful: file may have been opened but not read or written */
- if(pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP] > pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
- {
- pdata->shared_time_by_open_lastio +=
- pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP] -
- pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
- }
- }
-
- pdata->shared_time_by_cumul +=
- pfile->fcounters[POSIX_F_META_TIME] +
- pfile->fcounters[POSIX_F_READ_TIME] +
- pfile->fcounters[POSIX_F_WRITE_TIME];
- pdata->shared_meta_time += pfile->fcounters[POSIX_F_META_TIME];
-
- /* by_slowest */
- pdata->shared_time_by_slowest +=
- pfile->fcounters[POSIX_F_SLOWEST_RANK_TIME];
- }
-
- /*
- * Calculation of Unique File Time
- * record the data for each file and sum it
- */
- else
- {
- pdata->rank_cumul_io_time[pfile->base_rec.rank] +=
- (pfile->fcounters[POSIX_F_META_TIME] +
- pfile->fcounters[POSIX_F_READ_TIME] +
- pfile->fcounters[POSIX_F_WRITE_TIME]);
- pdata->rank_cumul_md_time[pfile->base_rec.rank] +=
- pfile->fcounters[POSIX_F_META_TIME];
- }
-
- return;
-}
-
-void mpiio_accum_perf(struct darshan_mpiio_file *mfile,
- perf_data_t *pdata)
-{
- pdata->total_bytes += mfile->counters[MPIIO_BYTES_READ] +
- mfile->counters[MPIIO_BYTES_WRITTEN];
-
- /*
- * Calculation of Shared File Time
- * Four Methods!!!!
- * by_cumul: sum time counters and divide by nprocs
- * (inaccurate if lots of variance between procs)
- * by_open: difference between timestamp of open and close
- * (inaccurate if file is left open without i/o happening)
- * by_open_lastio: difference between timestamp of open and the
- * timestamp of last i/o
- * (similar to above but fixes case where file is left
- * open after io is complete)
- * by_slowest: use slowest rank time from log data
- * (most accurate but requires newer log version)
- */
- if(mfile->base_rec.rank == -1)
- {
- /* by_open */
- if(mfile->fcounters[MPIIO_F_CLOSE_TIMESTAMP] >
- mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP])
- {
- pdata->shared_time_by_open +=
- mfile->fcounters[MPIIO_F_CLOSE_TIMESTAMP] -
- mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP];
- }
-
- /* by_open_lastio */
- if(mfile->fcounters[MPIIO_F_READ_END_TIMESTAMP] >
- mfile->fcounters[MPIIO_F_WRITE_END_TIMESTAMP])
- {
- /* be careful: file may have been opened but not read or written */
- if(mfile->fcounters[MPIIO_F_READ_END_TIMESTAMP] > mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP])
- {
- pdata->shared_time_by_open_lastio +=
- mfile->fcounters[MPIIO_F_READ_END_TIMESTAMP] -
- mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP];
- }
- }
- else
- {
- /* be careful: file may have been opened but not read or written */
- if(mfile->fcounters[MPIIO_F_WRITE_END_TIMESTAMP] > mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP])
- {
- pdata->shared_time_by_open_lastio +=
- mfile->fcounters[MPIIO_F_WRITE_END_TIMESTAMP] -
- mfile->fcounters[MPIIO_F_OPEN_TIMESTAMP];
- }
- }
-
- pdata->shared_time_by_cumul +=
- mfile->fcounters[MPIIO_F_META_TIME] +
- mfile->fcounters[MPIIO_F_READ_TIME] +
- mfile->fcounters[MPIIO_F_WRITE_TIME];
- pdata->shared_meta_time += mfile->fcounters[MPIIO_F_META_TIME];
-
- /* by_slowest */
- pdata->shared_time_by_slowest +=
- mfile->fcounters[MPIIO_F_SLOWEST_RANK_TIME];
- }
-
- /*
- * Calculation of Unique File Time
- * record the data for each file and sum it
- */
- else
- {
- pdata->rank_cumul_io_time[mfile->base_rec.rank] +=
- (mfile->fcounters[MPIIO_F_META_TIME] +
- mfile->fcounters[MPIIO_F_READ_TIME] +
- mfile->fcounters[MPIIO_F_WRITE_TIME]);
- pdata->rank_cumul_md_time[mfile->base_rec.rank] +=
- mfile->fcounters[MPIIO_F_META_TIME];
- }
-
- return;
-}
-
-void stdio_calc_file(hash_entry_t *file_hash,
- file_data_t *fdata)
-{
- hash_entry_t *curr = NULL;
- hash_entry_t *tmp = NULL;
- struct darshan_stdio_file *file_rec;
-
- memset(fdata, 0, sizeof(*fdata));
- HASH_ITER(hlink, file_hash, curr, tmp)
- {
- int64_t bytes;
- int64_t r;
- int64_t w;
-
- file_rec = (struct darshan_stdio_file*)curr->rec_dat;
- assert(file_rec);
-
- bytes = file_rec->counters[STDIO_BYTES_READ] +
- file_rec->counters[STDIO_BYTES_WRITTEN];
-
- r = file_rec->counters[STDIO_READS];
-
- w = file_rec->counters[STDIO_WRITES];
-
- fdata->total += 1;
- fdata->total_size += bytes;
- fdata->total_max = max(fdata->total_max, bytes);
-
- if (r && !w)
- {
- fdata->read_only += 1;
- fdata->read_only_size += bytes;
- fdata->read_only_max = max(fdata->read_only_max, bytes);
- }
-
- if (!r && w)
- {
- fdata->write_only += 1;
- fdata->write_only_size += bytes;
- fdata->write_only_max = max(fdata->write_only_max, bytes);
- }
-
- if (r && w)
- {
- fdata->read_write += 1;
- fdata->read_write_size += bytes;
- fdata->read_write_max = max(fdata->read_write_max, bytes);
- }
-
- if ((curr->type & (FILETYPE_SHARED|FILETYPE_PARTSHARED)))
- {
- fdata->shared += 1;
- fdata->shared_size += bytes;
- fdata->shared_max = max(fdata->shared_max, bytes);
- }
-
- if ((curr->type & (FILETYPE_UNIQUE)))
- {
- fdata->unique += 1;
- fdata->unique_size += bytes;
- fdata->unique_max = max(fdata->unique_max, bytes);
- }
- }
-
- return;
-}
-
-
-void posix_calc_file(hash_entry_t *file_hash,
- file_data_t *fdata)
-{
- hash_entry_t *curr = NULL;
- hash_entry_t *tmp = NULL;
- struct darshan_posix_file *file_rec;
-
- memset(fdata, 0, sizeof(*fdata));
- HASH_ITER(hlink, file_hash, curr, tmp)
- {
- int64_t bytes;
- int64_t r;
- int64_t w;
-
- file_rec = (struct darshan_posix_file*)curr->rec_dat;
- assert(file_rec);
-
- bytes = file_rec->counters[POSIX_BYTES_READ] +
- file_rec->counters[POSIX_BYTES_WRITTEN];
-
- r = file_rec->counters[POSIX_READS];
-
- w = file_rec->counters[POSIX_WRITES];
-
- fdata->total += 1;
- fdata->total_size += bytes;
- fdata->total_max = max(fdata->total_max, bytes);
-
- if (r && !w)
- {
- fdata->read_only += 1;
- fdata->read_only_size += bytes;
- fdata->read_only_max = max(fdata->read_only_max, bytes);
- }
-
- if (!r && w)
- {
- fdata->write_only += 1;
- fdata->write_only_size += bytes;
- fdata->write_only_max = max(fdata->write_only_max, bytes);
- }
-
- if (r && w)
- {
- fdata->read_write += 1;
- fdata->read_write_size += bytes;
- fdata->read_write_max = max(fdata->read_write_max, bytes);
- }
-
- if ((curr->type & (FILETYPE_SHARED|FILETYPE_PARTSHARED)))
- {
- fdata->shared += 1;
- fdata->shared_size += bytes;
- fdata->shared_max = max(fdata->shared_max, bytes);
- }
-
- if ((curr->type & (FILETYPE_UNIQUE)))
- {
- fdata->unique += 1;
- fdata->unique_size += bytes;
- fdata->unique_max = max(fdata->unique_max, bytes);
- }
- }
-
- return;
-}
-
-void mpiio_calc_file(hash_entry_t *file_hash,
- file_data_t *fdata)
-{
- hash_entry_t *curr = NULL;
- hash_entry_t *tmp = NULL;
- struct darshan_mpiio_file *file_rec;
-
- memset(fdata, 0, sizeof(*fdata));
- HASH_ITER(hlink, file_hash, curr, tmp)
- {
- int64_t bytes;
- int64_t r;
- int64_t w;
-
- file_rec = (struct darshan_mpiio_file*)curr->rec_dat;
- assert(file_rec);
-
- bytes = file_rec->counters[MPIIO_BYTES_READ] +
- file_rec->counters[MPIIO_BYTES_WRITTEN];
-
- r = (file_rec->counters[MPIIO_INDEP_READS]+
- file_rec->counters[MPIIO_COLL_READS] +
- file_rec->counters[MPIIO_SPLIT_READS] +
- file_rec->counters[MPIIO_NB_READS]);
-
- w = (file_rec->counters[MPIIO_INDEP_WRITES]+
- file_rec->counters[MPIIO_COLL_WRITES] +
- file_rec->counters[MPIIO_SPLIT_WRITES] +
- file_rec->counters[MPIIO_NB_WRITES]);
-
- fdata->total += 1;
- fdata->total_size += bytes;
- fdata->total_max = max(fdata->total_max, bytes);
-
- if (r && !w)
- {
- fdata->read_only += 1;
- fdata->read_only_size += bytes;
- fdata->read_only_max = max(fdata->read_only_max, bytes);
- }
-
- if (!r && w)
- {
- fdata->write_only += 1;
- fdata->write_only_size += bytes;
- fdata->write_only_max = max(fdata->write_only_max, bytes);
- }
-
- if (r && w)
- {
- fdata->read_write += 1;
- fdata->read_write_size += bytes;
- fdata->read_write_max = max(fdata->read_write_max, bytes);
- }
-
- if ((curr->type & (FILETYPE_SHARED|FILETYPE_PARTSHARED)))
- {
- fdata->shared += 1;
- fdata->shared_size += bytes;
- fdata->shared_max = max(fdata->shared_max, bytes);
- }
-
- if ((curr->type & (FILETYPE_UNIQUE)))
- {
- fdata->unique += 1;
- fdata->unique_size += bytes;
- fdata->unique_max = max(fdata->unique_max, bytes);
- }
- }
-
- return;
-}
-
-void calc_perf(perf_data_t *pdata,
- int64_t nprocs)
-{
- int64_t i;
-
- pdata->shared_time_by_cumul =
- pdata->shared_time_by_cumul / (double)nprocs;
-
- pdata->shared_meta_time = pdata->shared_meta_time / (double)nprocs;
-
- for (i=0; i<nprocs; i++)
- {
- if (pdata->rank_cumul_io_time[i] > pdata->slowest_rank_time)
- {
- pdata->slowest_rank_time = pdata->rank_cumul_io_time[i];
- pdata->slowest_rank_meta_time = pdata->rank_cumul_md_time[i];
- pdata->slowest_rank_rank = i;
- }
- }
-
- if (pdata->slowest_rank_time + pdata->shared_time_by_cumul)
- pdata->agg_perf_by_cumul = ((double)pdata->total_bytes / 1048576.0) /
- (pdata->slowest_rank_time +
- pdata->shared_time_by_cumul);
-
- if (pdata->slowest_rank_time + pdata->shared_time_by_open)
- pdata->agg_perf_by_open = ((double)pdata->total_bytes / 1048576.0) /
- (pdata->slowest_rank_time +
- pdata->shared_time_by_open);
-
- if (pdata->slowest_rank_time + pdata->shared_time_by_open_lastio)
- pdata->agg_perf_by_open_lastio = ((double)pdata->total_bytes / 1048576.0) /
- (pdata->slowest_rank_time +
- pdata->shared_time_by_open_lastio);
-
- if (pdata->slowest_rank_time + pdata->shared_time_by_slowest)
- pdata->agg_perf_by_slowest = ((double)pdata->total_bytes / 1048576.0) /
- (pdata->slowest_rank_time +
- pdata->shared_time_by_slowest);
-
- return;
-}
-
-void stdio_print_total_file(struct darshan_stdio_file *pfile, int stdio_ver)
-{
- int i;
-
- mod_logutils[DARSHAN_STDIO_MOD]->log_print_description(stdio_ver);
- printf("\n");
- for(i = 0; i < STDIO_NUM_INDICES; i++)
- {
- printf("total_%s: %"PRId64"\n",
- stdio_counter_names[i], pfile->counters[i]);
- }
- for(i = 0; i < STDIO_F_NUM_INDICES; i++)
- {
- printf("total_%s: %lf\n",
- stdio_f_counter_names[i], pfile->fcounters[i]);
- }
- return;
-}
-
-void posix_print_total_file(struct darshan_posix_file *pfile, int posix_ver)
-{
- int i;
-
- mod_logutils[DARSHAN_POSIX_MOD]->log_print_description(posix_ver);
- printf("\n");
- for(i = 0; i < POSIX_NUM_INDICES; i++)
- {
- printf("total_%s: %"PRId64"\n",
- posix_counter_names[i], pfile->counters[i]);
- }
- for(i = 0; i < POSIX_F_NUM_INDICES; i++)
- {
- printf("total_%s: %lf\n",
- posix_f_counter_names[i], pfile->fcounters[i]);
- }
- return;
-}
-
-void mpiio_print_total_file(struct darshan_mpiio_file *mfile, int mpiio_ver)
-{
- int i;
-
- mod_logutils[DARSHAN_MPIIO_MOD]->log_print_description(mpiio_ver);
- printf("\n");
- for(i = 0; i < MPIIO_NUM_INDICES; i++)
- {
- printf("total_%s: %"PRId64"\n",
- mpiio_counter_names[i], mfile->counters[i]);
- }
- for(i = 0; i < MPIIO_F_NUM_INDICES; i++)
- {
- printf("total_%s: %lf\n",
- mpiio_f_counter_names[i], mfile->fcounters[i]);
- }
- return;
-}
-
-void stdio_file_list(hash_entry_t *file_hash,
- struct darshan_name_record_ref *name_hash,
- int detail_flag)
-{
- hash_entry_t *curr = NULL;
- hash_entry_t *tmp = NULL;
- struct darshan_stdio_file *file_rec = NULL;
- struct darshan_name_record_ref *ref = NULL;
- int i;
-
- /* list of columns:
- *
- * normal mode
- * - file id
- * - file name
- * - nprocs
- * - slowest I/O time
- * - average cumulative I/O time
- *
- * detailed mode
- * - first open
- * - first read
- * - first write
- * - last read
- * - last write
- * - last close
- * - STDIO opens
- */
-
- if(detail_flag)
- printf("\n# Per-file summary of I/O activity (detailed).\n");
- else
- printf("\n# Per-file summary of I/O activity.\n");
- printf("# -----\n");
-
- printf("# <record_id>: darshan record id for this file\n");
- printf("# <file_name>: full file name\n");
- printf("# <nprocs>: number of processes that opened the file\n");
- printf("# <slowest>: (estimated) time in seconds consumed in IO by slowest process\n");
- printf("# <avg>: average time in seconds consumed in IO per process\n");
- if(detail_flag)
- {
- printf("# <start_{open/close/write/read}>: start timestamp of first open, close, write, or read\n");
- printf("# <end_{open/close/write/read}>: end timestamp of last open, close, write, or read\n");
- printf("# <stdio_opens>: STDIO open calls\n");
- }
-
- printf("\n# <record_id>\t<file_name>\t<nprocs>\t<slowest>\t<avg>");
- if(detail_flag)
- {
- printf("\t<start_open>\t<start_close>\t<start_write>\t<start_read>");
- printf("\t<end_open>\t<end_close>\t<end_write>\t<end_read>\t<stdio_opens>");
- }
- printf("\n");
-
- HASH_ITER(hlink, file_hash, curr, tmp)
- {
- file_rec = (struct darshan_stdio_file*)curr->rec_dat;
- assert(file_rec);
-
- HASH_FIND(hlink, name_hash, &(curr->rec_id), sizeof(darshan_record_id), ref);
- assert(ref);
-
- printf("%" PRIu64 "\t%s\t%" PRId64 "\t%f\t%f",
- curr->rec_id,
- ref->name_record->name,
- curr->procs,
- curr->slowest_time,
- curr->cumul_time/(double)curr->procs);
-
- if(detail_flag)
- {
- for(i=STDIO_F_OPEN_START_TIMESTAMP; i<=STDIO_F_READ_END_TIMESTAMP; i++)
- {
- printf("\t%f", file_rec->fcounters[i]);
- }
- printf("\t%" PRId64, file_rec->counters[STDIO_OPENS]);
- }
- printf("\n");
- }
-
- return;
-}
-
-
-void posix_file_list(hash_entry_t *file_hash,
- struct darshan_name_record_ref *name_hash,
- int detail_flag)
-{
- hash_entry_t *curr = NULL;
- hash_entry_t *tmp = NULL;
- struct darshan_posix_file *file_rec = NULL;
- struct darshan_name_record_ref *ref = NULL;
- int i;
-
- /* list of columns:
- *
- * normal mode
- * - file id
- * - file name
- * - nprocs
- * - slowest I/O time
- * - average cumulative I/O time
- *
- * detailed mode
- * - first open
- * - first read
- * - first write
- * - last read
- * - last write
- * - last close
- * - POSIX opens
- * - r histogram
- * - w histogram
- */
-
- if(detail_flag)
- printf("\n# Per-file summary of I/O activity (detailed).\n");
- else
- printf("\n# Per-file summary of I/O activity.\n");
- printf("# -----\n");
-
- printf("# <record_id>: darshan record id for this file\n");
- printf("# <file_name>: full file name\n");
- printf("# <nprocs>: number of processes that opened the file\n");
- printf("# <slowest>: (estimated) time in seconds consumed in IO by slowest process\n");
- printf("# <avg>: average time in seconds consumed in IO per process\n");
- if(detail_flag)
- {
- printf("# <start_{open/read/write/close}>: start timestamp of first open, read, write, or close\n");
- printf("# <end_{open/read/write/close}>: end timestamp of last open, read, write, or close\n");
- printf("# <posix_opens>: POSIX open calls\n");
- printf("# <POSIX_SIZE_READ_*>: POSIX read size histogram\n");
- printf("# <POSIX_SIZE_WRITE_*>: POSIX write size histogram\n");
- }
-
- printf("\n# <record_id>\t<file_name>\t<nprocs>\t<slowest>\t<avg>");
- if(detail_flag)
- {
- printf("\t<start_open>\t<start_read>\t<start_write>\t<start_close>");
- printf("\t<end_open>\t<end_read>\t<end_write>\t<end_close>\t<posix_opens>");
- for(i=POSIX_SIZE_READ_0_100; i<= POSIX_SIZE_WRITE_1G_PLUS; i++)
- printf("\t<%s>", posix_counter_names[i]);
- }
- printf("\n");
-
- HASH_ITER(hlink, file_hash, curr, tmp)
- {
- file_rec = (struct darshan_posix_file*)curr->rec_dat;
- assert(file_rec);
-
- HASH_FIND(hlink, name_hash, &(curr->rec_id), sizeof(darshan_record_id), ref);
- assert(ref);
-
- printf("%" PRIu64 "\t%s\t%" PRId64 "\t%f\t%f",
- curr->rec_id,
- ref->name_record->name,
- curr->procs,
- curr->slowest_time,
- curr->cumul_time/(double)curr->procs);
-
- if(detail_flag)
- {
- for(i=POSIX_F_OPEN_START_TIMESTAMP; i<=POSIX_F_CLOSE_END_TIMESTAMP; i++)
- {
- printf("\t%f", file_rec->fcounters[i]);
- }
- printf("\t%" PRId64, file_rec->counters[POSIX_OPENS]);
- for(i=POSIX_SIZE_READ_0_100; i<= POSIX_SIZE_WRITE_1G_PLUS; i++)
- printf("\t%" PRId64, file_rec->counters[i]);
- }
- printf("\n");
- }
-
- return;
-}
-
-void mpiio_file_list(hash_entry_t *file_hash,
- struct darshan_name_record_ref *name_hash,
- int detail_flag)
-{
- hash_entry_t *curr = NULL;
- hash_entry_t *tmp = NULL;
- struct darshan_mpiio_file *file_rec = NULL;
- struct darshan_name_record_ref *ref = NULL;
- int i;
-
- /* list of columns:
- *
- * normal mode
- * - file id
- * - file name
- * - nprocs
- * - slowest I/O time
- * - average cumulative I/O time
- *
- * detailed mode
- * - first open
- * - first read
- * - first write
- * - last read
- * - last write
- * - last close
- * - MPI indep opens
- * - MPI coll opens
- * - r histogram
- * - w histogram
- */
-
- if(detail_flag)
- printf("\n# Per-file summary of I/O activity (detailed).\n");
- else
- printf("\n# Per-file summary of I/O activity.\n");
- printf("# -----\n");
-
- printf("# <record_id>: darshan record id for this file\n");
- printf("# <file_name>: full file name\n");
- printf("# <nprocs>: number of processes that opened the file\n");
- printf("# <slowest>: (estimated) time in seconds consumed in IO by slowest process\n");
- printf("# <avg>: average time in seconds consumed in IO per process\n");
- if(detail_flag)
- {
- printf("# <start_{open/read/write}>: start timestamp of first open, read, or write\n");
- printf("# <end_{read/write/close}>: end timestamp of last read, write, or close\n");
- printf("# <mpi_indep_opens>: independent MPI_File_open calls\n");
- printf("# <mpi_coll_opens>: collective MPI_File_open calls\n");
- printf("# <MPIIO_SIZE_READ_AGG_*>: MPI-IO aggregate read size histogram\n");
- printf("# <MPIIO_SIZE_WRITE_AGG_*>: MPI-IO aggregate write size histogram\n");
- }
-
- printf("\n# <record_id>\t<file_name>\t<nprocs>\t<slowest>\t<avg>");
- if(detail_flag)
- {
- printf("\t<start_open>\t<start_read>\t<start_write>");
- printf("\t<end_read>\t<end_write>\t<end_close>");
- printf("\t<mpi_indep_opens>\t<mpi_coll_opens>");
- for(i=MPIIO_SIZE_READ_AGG_0_100; i<= MPIIO_SIZE_WRITE_AGG_1G_PLUS; i++)
- printf("\t<%s>", mpiio_counter_names[i]);
- }
- printf("\n");
-
- HASH_ITER(hlink, file_hash, curr, tmp)
- {
- file_rec = (struct darshan_mpiio_file*)curr->rec_dat;
- assert(file_rec);
-
- HASH_FIND(hlink, name_hash, &(curr->rec_id), sizeof(darshan_record_id), ref);
- assert(ref);
-
- printf("%" PRIu64 "\t%s\t%" PRId64 "\t%f\t%f",
- curr->rec_id,
- ref->name_record->name,
- curr->procs,
- curr->slowest_time,
- curr->cumul_time/(double)curr->procs);
-
- if(detail_flag)
- {
- for(i=MPIIO_F_OPEN_TIMESTAMP; i<=MPIIO_F_CLOSE_TIMESTAMP; i++)
- {
- printf("\t%f", file_rec->fcounters[i]);
- }
- printf("\t%" PRId64 "\t%" PRId64, file_rec->counters[MPIIO_INDEP_OPENS],
- file_rec->counters[MPIIO_COLL_OPENS]);
- for(i=MPIIO_SIZE_READ_AGG_0_100; i<= MPIIO_SIZE_WRITE_AGG_1G_PLUS; i++)
- printf("\t%" PRId64, file_rec->counters[i]);
- }
- printf("\n");
- }
-
- return;
-}
-
/*
* Local variables:
* c-indent-level: 4
=====================================
darshan-util/darshan-pnetcdf-logutils.c
=====================================
--- a/darshan-util/darshan-pnetcdf-logutils.c
+++ b/darshan-util/darshan-pnetcdf-logutils.c
@@ -46,7 +46,13 @@ struct darshan_mod_logutil_funcs pnetcdf_logutils =
.log_print_record = &darshan_log_print_pnetcdf_file,
.log_print_description = &darshan_log_print_pnetcdf_description,
.log_print_diff = &darshan_log_print_pnetcdf_file_diff,
- .log_agg_records = &darshan_log_agg_pnetcdf_files
+ .log_agg_records = &darshan_log_agg_pnetcdf_files,
+ .log_accum_file = NULL,
+ .log_accum_perf = NULL,
+ .log_calc_file = NULL,
+ .log_print_total_file = NULL,
+ .log_file_list = NULL,
+ .log_calc_perf = NULL
};
static int darshan_log_get_pnetcdf_file(darshan_fd fd, void** pnetcdf_buf_p)
=====================================
darshan-util/darshan-posix-logutils.c
=====================================
--- a/darshan-util/darshan-posix-logutils.c
+++ b/darshan-util/darshan-posix-logutils.c
@@ -41,6 +41,11 @@ static void darshan_log_print_posix_description(int ver);
static void darshan_log_print_posix_file_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag);
+static void darshan_log_posix_accum_file(void *pfile, hash_entry_t *hfile, int64_t nprocs);
+static void darshan_log_posix_accum_perf(void *pfile, perf_data_t *pdata);
+static void darshan_log_posix_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
+static void darshan_log_posix_print_total_file(void *pfile, int posix_ver);
+static void darshan_log_posix_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
struct darshan_mod_logutil_funcs posix_logutils =
{
@@ -50,6 +55,12 @@ struct darshan_mod_logutil_funcs posix_logutils =
.log_print_description = &darshan_log_print_posix_description,
.log_print_diff = &darshan_log_print_posix_file_diff,
.log_agg_records = &darshan_log_agg_posix_files,
+ .log_accum_file = &darshan_log_posix_accum_file,
+ .log_accum_perf = &darshan_log_posix_accum_perf,
+ .log_calc_file = &darshan_log_posix_calc_file,
+ .log_print_total_file = &darshan_log_posix_print_total_file,
+ .log_file_list = &darshan_log_posix_file_list,
+ .log_calc_perf = &darshan_calc_perf
};
static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p)
@@ -631,6 +642,479 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
return;
}
+static void darshan_log_posix_accum_file(
+ void *infile,
+ hash_entry_t *hfile,
+ int64_t nprocs)
+{
+ struct darshan_posix_file *pfile = infile;
+ int i, j;
+ int set;
+ int min_ndx;
+ int64_t min;
+ struct darshan_posix_file* tmp;
+
+ hfile->procs += 1;
+
+ if(pfile->base_rec.rank == -1)
+ {
+ hfile->slowest_time = pfile->fcounters[POSIX_F_SLOWEST_RANK_TIME];
+ }
+ else
+ {
+ hfile->slowest_time = max(hfile->slowest_time,
+ (pfile->fcounters[POSIX_F_META_TIME] +
+ pfile->fcounters[POSIX_F_READ_TIME] +
+ pfile->fcounters[POSIX_F_WRITE_TIME]));
+ }
+
+ if(pfile->base_rec.rank == -1)
+ {
+ hfile->procs = nprocs;
+ hfile->type |= FILETYPE_SHARED;
+
+ }
+ else if(hfile->procs > 1)
+ {
+ hfile->type &= (~FILETYPE_UNIQUE);
+ hfile->type |= FILETYPE_PARTSHARED;
+ }
+ else
+ {
+ hfile->type |= FILETYPE_UNIQUE;
+ }
+
+ hfile->cumul_time += pfile->fcounters[POSIX_F_META_TIME] +
+ pfile->fcounters[POSIX_F_READ_TIME] +
+ pfile->fcounters[POSIX_F_WRITE_TIME];
+
+ if(hfile->rec_dat == NULL)
+ {
+ hfile->rec_dat = malloc(sizeof(struct darshan_posix_file));
+ assert(hfile->rec_dat);
+ memset(hfile->rec_dat, 0, sizeof(struct darshan_posix_file));
+ }
+ tmp = (struct darshan_posix_file*)hfile->rec_dat;
+
+ for(i = 0; i < POSIX_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case POSIX_MODE:
+ case POSIX_MEM_ALIGNMENT:
+ case POSIX_FILE_ALIGNMENT:
+ tmp->counters[i] = pfile->counters[i];
+ break;
+ case POSIX_MAX_BYTE_READ:
+ case POSIX_MAX_BYTE_WRITTEN:
+ if (tmp->counters[i] < pfile->counters[i])
+ {
+ tmp->counters[i] = pfile->counters[i];
+ }
+ break;
+ case POSIX_STRIDE1_STRIDE:
+ case POSIX_STRIDE2_STRIDE:
+ case POSIX_STRIDE3_STRIDE:
+ case POSIX_STRIDE4_STRIDE:
+ case POSIX_ACCESS1_ACCESS:
+ case POSIX_ACCESS2_ACCESS:
+ case POSIX_ACCESS3_ACCESS:
+ case POSIX_ACCESS4_ACCESS:
+ /*
+ * do nothing here because these will be stored
+ * when the _COUNT is accessed.
+ */
+ break;
+ case POSIX_STRIDE1_COUNT:
+ case POSIX_STRIDE2_COUNT:
+ case POSIX_STRIDE3_COUNT:
+ case POSIX_STRIDE4_COUNT:
+ set = 0;
+ min_ndx = POSIX_STRIDE1_COUNT;
+ min = tmp->counters[min_ndx];
+ for(j = POSIX_STRIDE1_COUNT; j <= POSIX_STRIDE4_COUNT; j++)
+ {
+ if(tmp->counters[j-4] == pfile->counters[i-4])
+ {
+ tmp->counters[j] += pfile->counters[i];
+ set = 1;
+ break;
+ }
+ if(tmp->counters[j] < min)
+ {
+ min_ndx = j;
+ min = tmp->counters[j];
+ }
+ }
+ if(!set && (pfile->counters[i] > min))
+ {
+ tmp->counters[min_ndx] = pfile->counters[i];
+ tmp->counters[min_ndx-4] = pfile->counters[i-4];
+ }
+ break;
+ case POSIX_ACCESS1_COUNT:
+ case POSIX_ACCESS2_COUNT:
+ case POSIX_ACCESS3_COUNT:
+ case POSIX_ACCESS4_COUNT:
+ set = 0;
+ min_ndx = POSIX_ACCESS1_COUNT;
+ min = tmp->counters[min_ndx];
+ for(j = POSIX_ACCESS1_COUNT; j <= POSIX_ACCESS4_COUNT; j++)
+ {
+ if(tmp->counters[j-4] == pfile->counters[i-4])
+ {
+ tmp->counters[j] += pfile->counters[i];
+ set = 1;
+ break;
+ }
+ if(tmp->counters[j] < min)
+ {
+ min_ndx = j;
+ min = tmp->counters[j];
+ }
+ }
+ if(!set && (pfile->counters[i] > min))
+ {
+ tmp->counters[i] = pfile->counters[i];
+ tmp->counters[i-4] = pfile->counters[i-4];
+ }
+ break;
+ case POSIX_FASTEST_RANK:
+ case POSIX_SLOWEST_RANK:
+ case POSIX_FASTEST_RANK_BYTES:
+ case POSIX_SLOWEST_RANK_BYTES:
+ tmp->counters[i] = 0;
+ break;
+ case POSIX_MAX_READ_TIME_SIZE:
+ case POSIX_MAX_WRITE_TIME_SIZE:
+ break;
+ default:
+ tmp->counters[i] += pfile->counters[i];
+ break;
+ }
+ }
+
+ for(i = 0; i < POSIX_F_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case POSIX_F_OPEN_START_TIMESTAMP:
+ case POSIX_F_READ_START_TIMESTAMP:
+ case POSIX_F_WRITE_START_TIMESTAMP:
+ case POSIX_F_CLOSE_START_TIMESTAMP:
+ if(tmp->fcounters[i] == 0 ||
+ tmp->fcounters[i] > pfile->fcounters[i])
+ {
+ tmp->fcounters[i] = pfile->fcounters[i];
+ }
+ break;
+ case POSIX_F_OPEN_END_TIMESTAMP:
+ case POSIX_F_READ_END_TIMESTAMP:
+ case POSIX_F_WRITE_END_TIMESTAMP:
+ case POSIX_F_CLOSE_END_TIMESTAMP:
+ if(tmp->fcounters[i] == 0 ||
+ tmp->fcounters[i] < pfile->fcounters[i])
+ {
+ tmp->fcounters[i] = pfile->fcounters[i];
+ }
+ break;
+ case POSIX_F_FASTEST_RANK_TIME:
+ case POSIX_F_SLOWEST_RANK_TIME:
+ case POSIX_F_VARIANCE_RANK_TIME:
+ case POSIX_F_VARIANCE_RANK_BYTES:
+ tmp->fcounters[i] = 0;
+ break;
+ case POSIX_F_MAX_READ_TIME:
+ if(tmp->fcounters[i] < pfile->fcounters[i])
+ {
+ tmp->fcounters[i] = pfile->fcounters[i];
+ tmp->counters[POSIX_MAX_READ_TIME_SIZE] =
+ pfile->counters[POSIX_MAX_READ_TIME_SIZE];
+ }
+ break;
+ case POSIX_F_MAX_WRITE_TIME:
+ if(tmp->fcounters[i] < pfile->fcounters[i])
+ {
+ tmp->fcounters[i] = pfile->fcounters[i];
+ tmp->counters[POSIX_MAX_WRITE_TIME_SIZE] =
+ pfile->counters[POSIX_MAX_WRITE_TIME_SIZE];
+ }
+ break;
+ default:
+ tmp->fcounters[i] += pfile->fcounters[i];
+ break;
+ }
+ }
+
+ return;
+}
+
+static void darshan_log_posix_accum_perf(
+ void *infile,
+ perf_data_t *pdata)
+{
+ struct darshan_posix_file *pfile = infile;
+ pdata->total_bytes += pfile->counters[POSIX_BYTES_READ] +
+ pfile->counters[POSIX_BYTES_WRITTEN];
+
+ /*
+ * Calculation of Shared File Time
+ * Four Methods!!!!
+ * by_cumul: sum time counters and divide by nprocs
+ * (inaccurate if lots of variance between procs)
+ * by_open: difference between timestamp of open and close
+ * (inaccurate if file is left open without i/o happening)
+ * by_open_lastio: difference between timestamp of open and the
+ * timestamp of last i/o
+ * (similar to above but fixes case where file is left
+ * open after io is complete)
+ * by_slowest: use slowest rank time from log data
+ * (most accurate but requires newer log version)
+ */
+ if(pfile->base_rec.rank == -1)
+ {
+ /* by_open */
+ if(pfile->fcounters[POSIX_F_CLOSE_END_TIMESTAMP] >
+ pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
+ {
+ pdata->shared_time_by_open +=
+ pfile->fcounters[POSIX_F_CLOSE_END_TIMESTAMP] -
+ pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
+ }
+
+ /* by_open_lastio */
+ if(pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] >
+ pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP])
+ {
+ /* be careful: file may have been opened but not read or written */
+ if(pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] > pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
+ {
+ pdata->shared_time_by_open_lastio +=
+ pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] -
+ pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
+ }
+ }
+ else
+ {
+ /* be careful: file may have been opened but not read or written */
+ if(pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP] > pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
+ {
+ pdata->shared_time_by_open_lastio +=
+ pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP] -
+ pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
+ }
+ }
+
+ pdata->shared_time_by_cumul +=
+ pfile->fcounters[POSIX_F_META_TIME] +
+ pfile->fcounters[POSIX_F_READ_TIME] +
+ pfile->fcounters[POSIX_F_WRITE_TIME];
+ pdata->shared_meta_time += pfile->fcounters[POSIX_F_META_TIME];
+
+ /* by_slowest */
+ pdata->shared_time_by_slowest +=
+ pfile->fcounters[POSIX_F_SLOWEST_RANK_TIME];
+ }
+
+ /*
+ * Calculation of Unique File Time
+ * record the data for each file and sum it
+ */
+ else
+ {
+ pdata->rank_cumul_io_time[pfile->base_rec.rank] +=
+ (pfile->fcounters[POSIX_F_META_TIME] +
+ pfile->fcounters[POSIX_F_READ_TIME] +
+ pfile->fcounters[POSIX_F_WRITE_TIME]);
+ pdata->rank_cumul_md_time[pfile->base_rec.rank] +=
+ pfile->fcounters[POSIX_F_META_TIME];
+ }
+
+ return;
+}
+
+static void darshan_log_posix_calc_file(
+ hash_entry_t *file_hash,
+ file_data_t *fdata)
+{
+ hash_entry_t *curr = NULL;
+ hash_entry_t *tmp = NULL;
+ struct darshan_posix_file *file_rec;
+
+ memset(fdata, 0, sizeof(*fdata));
+ HASH_ITER(hlink, file_hash, curr, tmp)
+ {
+ int64_t bytes;
+ int64_t r;
+ int64_t w;
+
+ file_rec = (struct darshan_posix_file*)curr->rec_dat;
+ assert(file_rec);
+
+ bytes = file_rec->counters[POSIX_BYTES_READ] +
+ file_rec->counters[POSIX_BYTES_WRITTEN];
+
+ r = file_rec->counters[POSIX_READS];
+
+ w = file_rec->counters[POSIX_WRITES];
+
+ fdata->total += 1;
+ fdata->total_size += bytes;
+ fdata->total_max = max(fdata->total_max, bytes);
+
+ if (r && !w)
+ {
+ fdata->read_only += 1;
+ fdata->read_only_size += bytes;
+ fdata->read_only_max = max(fdata->read_only_max, bytes);
+ }
+
+ if (!r && w)
+ {
+ fdata->write_only += 1;
+ fdata->write_only_size += bytes;
+ fdata->write_only_max = max(fdata->write_only_max, bytes);
+ }
+
+ if (r && w)
+ {
+ fdata->read_write += 1;
+ fdata->read_write_size += bytes;
+ fdata->read_write_max = max(fdata->read_write_max, bytes);
+ }
+
+ if ((curr->type & (FILETYPE_SHARED|FILETYPE_PARTSHARED)))
+ {
+ fdata->shared += 1;
+ fdata->shared_size += bytes;
+ fdata->shared_max = max(fdata->shared_max, bytes);
+ }
+
+ if ((curr->type & (FILETYPE_UNIQUE)))
+ {
+ fdata->unique += 1;
+ fdata->unique_size += bytes;
+ fdata->unique_max = max(fdata->unique_max, bytes);
+ }
+ }
+
+ return;
+}
+
+static void darshan_log_posix_print_total_file(
+ void *infile,
+ int posix_ver)
+{
+ struct darshan_posix_file *pfile = infile;
+ int i;
+
+ mod_logutils[DARSHAN_POSIX_MOD]->log_print_description(posix_ver);
+ printf("\n");
+ for(i = 0; i < POSIX_NUM_INDICES; i++)
+ {
+ printf("total_%s: %"PRId64"\n",
+ posix_counter_names[i], pfile->counters[i]);
+ }
+ for(i = 0; i < POSIX_F_NUM_INDICES; i++)
+ {
+ printf("total_%s: %lf\n",
+ posix_f_counter_names[i], pfile->fcounters[i]);
+ }
+ return;
+}
+
+static void darshan_log_posix_file_list(
+ hash_entry_t *file_hash,
+ struct darshan_name_record_ref *name_hash,
+ int detail_flag)
+{
+ hash_entry_t *curr = NULL;
+ hash_entry_t *tmp = NULL;
+ struct darshan_posix_file *file_rec = NULL;
+ struct darshan_name_record_ref *ref = NULL;
+ int i;
+
+ /* list of columns:
+ *
+ * normal mode
+ * - file id
+ * - file name
+ * - nprocs
+ * - slowest I/O time
+ * - average cumulative I/O time
+ *
+ * detailed mode
+ * - first open
+ * - first read
+ * - first write
+ * - last read
+ * - last write
+ * - last close
+ * - POSIX opens
+ * - r histogram
+ * - w histogram
+ */
+
+ if(detail_flag)
+ printf("\n# Per-file summary of I/O activity (detailed).\n");
+ else
+ printf("\n# Per-file summary of I/O activity.\n");
+ printf("# -----\n");
+
+ printf("# <record_id>: darshan record id for this file\n");
+ printf("# <file_name>: full file name\n");
+ printf("# <nprocs>: number of processes that opened the file\n");
+ printf("# <slowest>: (estimated) time in seconds consumed in IO by slowest process\n");
+ printf("# <avg>: average time in seconds consumed in IO per process\n");
+ if(detail_flag)
+ {
+ printf("# <start_{open/read/write/close}>: start timestamp of first open, read, write, or close\n");
+ printf("# <end_{open/read/write/close}>: end timestamp of last open, read, write, or close\n");
+ printf("# <posix_opens>: POSIX open calls\n");
+ printf("# <POSIX_SIZE_READ_*>: POSIX read size histogram\n");
+ printf("# <POSIX_SIZE_WRITE_*>: POSIX write size histogram\n");
+ }
+
+ printf("\n# <record_id>\t<file_name>\t<nprocs>\t<slowest>\t<avg>");
+ if(detail_flag)
+ {
+ printf("\t<start_open>\t<start_read>\t<start_write>\t<start_close>");
+ printf("\t<end_open>\t<end_read>\t<end_write>\t<end_close>\t<posix_opens>");
+ for(i=POSIX_SIZE_READ_0_100; i<= POSIX_SIZE_WRITE_1G_PLUS; i++)
+ printf("\t<%s>", posix_counter_names[i]);
+ }
+ printf("\n");
+ HASH_ITER(hlink, file_hash, curr, tmp)
+ {
+ file_rec = (struct darshan_posix_file*)curr->rec_dat;
+ assert(file_rec);
+
+ HASH_FIND(hlink, name_hash, &(curr->rec_id), sizeof(darshan_record_id), ref);
+ assert(ref);
+
+ printf("%" PRIu64 "\t%s\t%" PRId64 "\t%f\t%f",
+ curr->rec_id,
+ ref->name_record->name,
+ curr->procs,
+ curr->slowest_time,
+ curr->cumul_time/(double)curr->procs);
+
+ if(detail_flag)
+ {
+ for(i=POSIX_F_OPEN_START_TIMESTAMP; i<=POSIX_F_CLOSE_END_TIMESTAMP; i++)
+ {
+ printf("\t%f", file_rec->fcounters[i]);
+ }
+ printf("\t%" PRId64, file_rec->counters[POSIX_OPENS]);
+ for(i=POSIX_SIZE_READ_0_100; i<= POSIX_SIZE_WRITE_1G_PLUS; i++)
+ printf("\t%" PRId64, file_rec->counters[i]);
+ }
+ printf("\n");
+ }
+
+ return;
+}
+
/*
* Local variables:
* c-indent-level: 4
=====================================
darshan-util/darshan-stdio-logutils.c
=====================================
--- a/darshan-util/darshan-stdio-logutils.c
+++ b/darshan-util/darshan-stdio-logutils.c
@@ -40,6 +40,11 @@ static void darshan_log_print_stdio_description(int ver);
static void darshan_log_print_stdio_record_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_stdio_records(void *rec, void *agg_rec, int init_flag);
+static void darshan_log_stdio_accum_perf(void *pfile, perf_data_t *pdata);
+static void darshan_log_stdio_accum_file(void *pfile, hash_entry_t *hfile, int64_t nprocs);
+static void darshan_log_stdio_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
+static void darshan_log_stdio_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
+static void darshan_log_stdio_print_total_file(void *pfile, int stdio_ver);
/* structure storing each function needed for implementing the darshan
* logutil interface. these functions are used for reading, writing, and
@@ -52,7 +57,13 @@ struct darshan_mod_logutil_funcs stdio_logutils =
.log_print_record = &darshan_log_print_stdio_record,
.log_print_description = &darshan_log_print_stdio_description,
.log_print_diff = &darshan_log_print_stdio_record_diff,
- .log_agg_records = &darshan_log_agg_stdio_records
+ .log_agg_records = &darshan_log_agg_stdio_records,
+ .log_accum_file = &darshan_log_stdio_accum_file,
+ .log_accum_perf = &darshan_log_stdio_accum_perf,
+ .log_calc_file = &darshan_log_stdio_calc_file,
+ .log_print_total_file = &darshan_log_stdio_print_total_file,
+ .log_file_list = &darshan_log_stdio_file_list,
+ .log_calc_perf = &darshan_calc_perf
};
/* retrieve a STDIO record from log file descriptor 'fd', storing the
@@ -416,6 +427,375 @@ static void darshan_log_agg_stdio_records(void *rec, void *agg_rec, int init_fla
return;
}
+static void darshan_log_stdio_accum_file(void *infile,
+ hash_entry_t *hfile,
+ int64_t nprocs)
+{
+ struct darshan_stdio_file *pfile = infile;
+ int i;
+ struct darshan_stdio_file* tmp;
+
+ hfile->procs += 1;
+
+ if(pfile->base_rec.rank == -1)
+ {
+ hfile->slowest_time = pfile->fcounters[STDIO_F_SLOWEST_RANK_TIME];
+ }
+ else
+ {
+ hfile->slowest_time = max(hfile->slowest_time,
+ (pfile->fcounters[STDIO_F_META_TIME] +
+ pfile->fcounters[STDIO_F_READ_TIME] +
+ pfile->fcounters[STDIO_F_WRITE_TIME]));
+ }
+
+ if(pfile->base_rec.rank == -1)
+ {
+ hfile->procs = nprocs;
+ hfile->type |= FILETYPE_SHARED;
+
+ }
+ else if(hfile->procs > 1)
+ {
+ hfile->type &= (~FILETYPE_UNIQUE);
+ hfile->type |= FILETYPE_PARTSHARED;
+ }
+ else
+ {
+ hfile->type |= FILETYPE_UNIQUE;
+ }
+
+ hfile->cumul_time += pfile->fcounters[STDIO_F_META_TIME] +
+ pfile->fcounters[STDIO_F_READ_TIME] +
+ pfile->fcounters[STDIO_F_WRITE_TIME];
+
+ if(hfile->rec_dat == NULL)
+ {
+ hfile->rec_dat = malloc(sizeof(struct darshan_stdio_file));
+ assert(hfile->rec_dat);
+ memset(hfile->rec_dat, 0, sizeof(struct darshan_stdio_file));
+ }
+ tmp = (struct darshan_stdio_file*)hfile->rec_dat;
+
+ for(i = 0; i < STDIO_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case STDIO_MAX_BYTE_READ:
+ case STDIO_MAX_BYTE_WRITTEN:
+ if (tmp->counters[i] < pfile->counters[i])
+ {
+ tmp->counters[i] = pfile->counters[i];
+ }
+ break;
+ case POSIX_FASTEST_RANK:
+ case POSIX_SLOWEST_RANK:
+ case POSIX_FASTEST_RANK_BYTES:
+ case POSIX_SLOWEST_RANK_BYTES:
+ tmp->counters[i] = 0;
+ break;
+ default:
+ tmp->counters[i] += pfile->counters[i];
+ break;
+ }
+ }
+
+ for(i = 0; i < STDIO_F_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case STDIO_F_OPEN_START_TIMESTAMP:
+ case STDIO_F_CLOSE_START_TIMESTAMP:
+ case STDIO_F_READ_START_TIMESTAMP:
+ case STDIO_F_WRITE_START_TIMESTAMP:
+ if(tmp->fcounters[i] == 0 ||
+ tmp->fcounters[i] > pfile->fcounters[i])
+ {
+ tmp->fcounters[i] = pfile->fcounters[i];
+ }
+ break;
+ case STDIO_F_READ_END_TIMESTAMP:
+ case STDIO_F_WRITE_END_TIMESTAMP:
+ case STDIO_F_OPEN_END_TIMESTAMP:
+ case STDIO_F_CLOSE_END_TIMESTAMP:
+ if(tmp->fcounters[i] == 0 ||
+ tmp->fcounters[i] < pfile->fcounters[i])
+ {
+ tmp->fcounters[i] = pfile->fcounters[i];
+ }
+ break;
+ case STDIO_F_FASTEST_RANK_TIME:
+ case STDIO_F_SLOWEST_RANK_TIME:
+ case STDIO_F_VARIANCE_RANK_TIME:
+ case STDIO_F_VARIANCE_RANK_BYTES:
+ tmp->fcounters[i] = 0;
+ break;
+ default:
+ tmp->fcounters[i] += pfile->fcounters[i];
+ break;
+ }
+ }
+
+ return;
+}
+
+static void darshan_log_stdio_accum_perf(void *infile,
+ perf_data_t *pdata)
+{
+ struct darshan_stdio_file *pfile = infile;
+ pdata->total_bytes += pfile->counters[STDIO_BYTES_READ] +
+ pfile->counters[STDIO_BYTES_WRITTEN];
+
+ /*
+ * Calculation of Shared File Time
+ * Four Methods!!!!
+ * by_cumul: sum time counters and divide by nprocs
+ * (inaccurate if lots of variance between procs)
+ * by_open: difference between timestamp of open and close
+ * (inaccurate if file is left open without i/o happening)
+ * by_open_lastio: difference between timestamp of open and the
+ * timestamp of last i/o
+ * (similar to above but fixes case where file is left
+ * open after io is complete)
+ * by_slowest: use slowest rank time from log data
+ * (most accurate but requires newer log version)
+ */
+ if(pfile->base_rec.rank == -1)
+ {
+ /* by_open */
+ if(pfile->fcounters[STDIO_F_CLOSE_END_TIMESTAMP] >
+ pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP])
+ {
+ pdata->shared_time_by_open +=
+ pfile->fcounters[STDIO_F_CLOSE_END_TIMESTAMP] -
+ pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP];
+ }
+
+ /* by_open_lastio */
+ if(pfile->fcounters[STDIO_F_READ_END_TIMESTAMP] >
+ pfile->fcounters[STDIO_F_WRITE_END_TIMESTAMP])
+ {
+ /* be careful: file may have been opened but not read or written */
+ if(pfile->fcounters[STDIO_F_READ_END_TIMESTAMP] > pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP])
+ {
+ pdata->shared_time_by_open_lastio +=
+ pfile->fcounters[STDIO_F_READ_END_TIMESTAMP] -
+ pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP];
+ }
+ }
+ else
+ {
+ /* be careful: file may have been opened but not read or written */
+ if(pfile->fcounters[STDIO_F_WRITE_END_TIMESTAMP] > pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP])
+ {
+ pdata->shared_time_by_open_lastio +=
+ pfile->fcounters[STDIO_F_WRITE_END_TIMESTAMP] -
+ pfile->fcounters[STDIO_F_OPEN_START_TIMESTAMP];
+ }
+ }
+
+ pdata->shared_time_by_cumul +=
+ pfile->fcounters[STDIO_F_META_TIME] +
+ pfile->fcounters[STDIO_F_READ_TIME] +
+ pfile->fcounters[STDIO_F_WRITE_TIME];
+ pdata->shared_meta_time += pfile->fcounters[STDIO_F_META_TIME];
+
+ /* by_slowest */
+ pdata->shared_time_by_slowest +=
+ pfile->fcounters[STDIO_F_SLOWEST_RANK_TIME];
+ }
+
+ /*
+ * Calculation of Unique File Time
+ * record the data for each file and sum it
+ */
+ else
+ {
+ pdata->rank_cumul_io_time[pfile->base_rec.rank] +=
+ (pfile->fcounters[STDIO_F_META_TIME] +
+ pfile->fcounters[STDIO_F_READ_TIME] +
+ pfile->fcounters[STDIO_F_WRITE_TIME]);
+ pdata->rank_cumul_md_time[pfile->base_rec.rank] +=
+ pfile->fcounters[STDIO_F_META_TIME];
+ }
+
+ return;
+}
+
+static void darshan_log_stdio_calc_file(hash_entry_t *file_hash,
+ file_data_t *fdata)
+{
+ hash_entry_t *curr = NULL;
+ hash_entry_t *tmp = NULL;
+ struct darshan_stdio_file *file_rec;
+
+ memset(fdata, 0, sizeof(*fdata));
+ HASH_ITER(hlink, file_hash, curr, tmp)
+ {
+ int64_t bytes;
+ int64_t r;
+ int64_t w;
+
+ file_rec = (struct darshan_stdio_file*)curr->rec_dat;
+ assert(file_rec);
+
+ bytes = file_rec->counters[STDIO_BYTES_READ] +
+ file_rec->counters[STDIO_BYTES_WRITTEN];
+
+ r = file_rec->counters[STDIO_READS];
+
+ w = file_rec->counters[STDIO_WRITES];
+
+ fdata->total += 1;
+ fdata->total_size += bytes;
+ fdata->total_max = max(fdata->total_max, bytes);
+
+ if (r && !w)
+ {
+ fdata->read_only += 1;
+ fdata->read_only_size += bytes;
+ fdata->read_only_max = max(fdata->read_only_max, bytes);
+ }
+
+ if (!r && w)
+ {
+ fdata->write_only += 1;
+ fdata->write_only_size += bytes;
+ fdata->write_only_max = max(fdata->write_only_max, bytes);
+ }
+
+ if (r && w)
+ {
+ fdata->read_write += 1;
+ fdata->read_write_size += bytes;
+ fdata->read_write_max = max(fdata->read_write_max, bytes);
+ }
+
+ if ((curr->type & (FILETYPE_SHARED|FILETYPE_PARTSHARED)))
+ {
+ fdata->shared += 1;
+ fdata->shared_size += bytes;
+ fdata->shared_max = max(fdata->shared_max, bytes);
+ }
+
+ if ((curr->type & (FILETYPE_UNIQUE)))
+ {
+ fdata->unique += 1;
+ fdata->unique_size += bytes;
+ fdata->unique_max = max(fdata->unique_max, bytes);
+ }
+ }
+
+ return;
+}
+
+static void darshan_log_stdio_print_total_file(
+ void *infile,
+ int stdio_ver)
+{
+ struct darshan_stdio_file *pfile = infile;
+ int i;
+
+ mod_logutils[DARSHAN_STDIO_MOD]->log_print_description(stdio_ver);
+ printf("\n");
+ for(i = 0; i < STDIO_NUM_INDICES; i++)
+ {
+ printf("total_%s: %"PRId64"\n",
+ stdio_counter_names[i], pfile->counters[i]);
+ }
+ for(i = 0; i < STDIO_F_NUM_INDICES; i++)
+ {
+ printf("total_%s: %lf\n",
+ stdio_f_counter_names[i], pfile->fcounters[i]);
+ }
+ return;
+}
+
+static void darshan_log_stdio_file_list(
+ hash_entry_t *file_hash,
+ struct darshan_name_record_ref *name_hash,
+ int detail_flag)
+{
+ hash_entry_t *curr = NULL;
+ hash_entry_t *tmp = NULL;
+ struct darshan_stdio_file *file_rec = NULL;
+ struct darshan_name_record_ref *ref = NULL;
+ int i;
+
+ /* list of columns:
+ *
+ * normal mode
+ * - file id
+ * - file name
+ * - nprocs
+ * - slowest I/O time
+ * - average cumulative I/O time
+ *
+ * detailed mode
+ * - first open
+ * - first read
+ * - first write
+ * - last read
+ * - last write
+ * - last close
+ * - STDIO opens
+ */
+
+ if(detail_flag)
+ printf("\n# Per-file summary of I/O activity (detailed).\n");
+ else
+ printf("\n# Per-file summary of I/O activity.\n");
+ printf("# -----\n");
+
+ printf("# <record_id>: darshan record id for this file\n");
+ printf("# <file_name>: full file name\n");
+ printf("# <nprocs>: number of processes that opened the file\n");
+ printf("# <slowest>: (estimated) time in seconds consumed in IO by slowest process\n");
+ printf("# <avg>: average time in seconds consumed in IO per process\n");
+ if(detail_flag)
+ {
+ printf("# <start_{open/close/write/read}>: start timestamp of first open, close, write, or read\n");
+ printf("# <end_{open/close/write/read}>: end timestamp of last open, close, write, or read\n");
+ printf("# <stdio_opens>: STDIO open calls\n");
+ }
+
+ printf("\n# <record_id>\t<file_name>\t<nprocs>\t<slowest>\t<avg>");
+ if(detail_flag)
+ {
+ printf("\t<start_open>\t<start_close>\t<start_write>\t<start_read>");
+ printf("\t<end_open>\t<end_close>\t<end_write>\t<end_read>\t<stdio_opens>");
+ }
+ printf("\n");
+
+ HASH_ITER(hlink, file_hash, curr, tmp)
+ {
+ file_rec = (struct darshan_stdio_file*)curr->rec_dat;
+ assert(file_rec);
+
+ HASH_FIND(hlink, name_hash, &(curr->rec_id), sizeof(darshan_record_id), ref);
+ assert(ref);
+
+ printf("%" PRIu64 "\t%s\t%" PRId64 "\t%f\t%f",
+ curr->rec_id,
+ ref->name_record->name,
+ curr->procs,
+ curr->slowest_time,
+ curr->cumul_time/(double)curr->procs);
+
+ if(detail_flag)
+ {
+ for(i=STDIO_F_OPEN_START_TIMESTAMP; i<=STDIO_F_READ_END_TIMESTAMP; i++)
+ {
+ printf("\t%f", file_rec->fcounters[i]);
+ }
+ printf("\t%" PRId64, file_rec->counters[STDIO_OPENS]);
+ }
+ printf("\n");
+ }
+
+ return;
+}
+
/*
* Local variables:
* c-indent-level: 4
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/cc0d9252be1f7ec9c095fbefbaa02e3404ea7e36...457a18fc78cf814e8b0e1d86efd7aed9fac82355
---
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/cc0d9252be1f7ec9c095fbefbaa02e3404ea7e36...457a18fc78cf814e8b0e1d86efd7aed9fac82355
You're receiving this email because of your account on xgitlab.cels.anl.gov.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20180308/65662c26/attachment-0001.html>
More information about the Darshan-commits
mailing list