[Darshan-commits] [Git][darshan/darshan][master] 2 commits: get file accumulators working for stdio module
Philip Carns
xgitlab at cels.anl.gov
Sun Sep 18 09:17:42 CDT 2016
Philip Carns pushed to branch master at darshan / darshan
Commits:
ad21fea6 by Phil Carns at 2016-09-18T09:05:30-04:00
get file accumulators working for stdio module
- - - - -
384e9b5c by Phil Carns at 2016-09-18T10:15:02-04:00
filter out unused stdio records
- - - - -
2 changed files:
- darshan-runtime/lib/darshan-stdio.c
- darshan-util/darshan-parser.c
Changes:
=====================================
darshan-runtime/lib/darshan-stdio.c
=====================================
--- a/darshan-runtime/lib/darshan-stdio.c
+++ b/darshan-runtime/lib/darshan-stdio.c
@@ -78,6 +78,8 @@
#include <assert.h>
#include <libgen.h>
#include <pthread.h>
+#include <stdint.h>
+#include <limits.h>
#include "darshan.h"
#include "darshan-dynamic.h"
@@ -488,7 +490,6 @@ int DARSHAN_DECL(vfprintf)(FILE *stream, const char *format, va_list ap)
{
int ret;
double tm1, tm2;
- long start_off, end_off;
MAP_OR_FAIL(vfprintf);
@@ -1115,6 +1116,8 @@ static void stdio_shutdown(
MPI_Op red_op;
int stdio_rec_count;
double stdio_time;
+ FILE * fp[3];
+ int trim_records = 0;
STDIO_LOCK();
assert(stdio_runtime);
@@ -1219,6 +1222,30 @@ static void stdio_shutdown(
DARSHAN_MPI_CALL(PMPI_Op_free)(&red_op);
}
+ /* filter out any records that have no activity on them; this is
+ * specifically meant to filter out unused stdin, stdout, or stderr
+ * entries
+ *
+ * NOTE: we can no longer use the darshan_lookup_record_ref()
+ * function at this point to find specific records, because the
+ * logic above has likely broken the mapping to the static array.
+ * We walk it manually here instead.
+ */
+ for(i=0; i<stdio_rec_count; i++)
+ {
+ if(stdio_rec_buf[i].counters[STDIO_WRITES] == 0 &&
+ stdio_rec_buf[i].counters[STDIO_READS] == 0)
+ {
+ if(i != (stdio_rec_count-1))
+ {
+ memmove(&stdio_rec_buf[i], &stdio_rec_buf[i+1],
+ (stdio_rec_count-i-1)*sizeof(stdio_rec_buf[i]));
+ stdio_rec_count--;
+ i--;
+ }
+ }
+ }
+
/* update output buffer size to account for shared file reduction */
*stdio_buf_sz = stdio_rec_count * sizeof(struct darshan_stdio_file);
=====================================
darshan-util/darshan-parser.c
=====================================
--- a/darshan-util/darshan-parser.c
+++ b/darshan-util/darshan-parser.c
@@ -114,6 +114,8 @@ void mpiio_print_total_file(struct darshan_mpiio_file *mfile);
void mpiio_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
void stdio_accum_perf(struct darshan_stdio_file *pfile, perf_data_t *pdata);
+void stdio_accum_file(struct darshan_stdio_file *pfile, hash_entry_t *hfile, int64_t nprocs);
+void stdio_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
void calc_perf(perf_data_t *pdata, int64_t nprocs);
@@ -515,6 +517,8 @@ int main(int argc, char **argv)
}
else if(i == DARSHAN_STDIO_MOD)
{
+ stdio_accum_file((struct darshan_stdio_file*)mod_buf, &total, job.nprocs);
+ stdio_accum_file((struct darshan_stdio_file*)mod_buf, hfile, job.nprocs);
stdio_accum_perf((struct darshan_stdio_file*)mod_buf, &pdata);
}
@@ -557,6 +561,10 @@ int main(int argc, char **argv)
{
mpiio_calc_file(file_hash, &fdata);
}
+ else if(i == DARSHAN_STDIO_MOD)
+ {
+ stdio_calc_file(file_hash, &fdata);
+ }
printf("\n# files\n");
printf("# -----\n");
@@ -683,6 +691,117 @@ cleanup:
return(ret);
}
+void stdio_accum_file(struct darshan_stdio_file *pfile,
+ hash_entry_t *hfile,
+ int64_t nprocs)
+{
+ int i;
+ struct darshan_stdio_file* tmp;
+
+ hfile->procs += 1;
+
+ if(pfile->base_rec.rank == -1)
+ {
+ hfile->slowest_time = pfile->fcounters[STDIO_F_SLOWEST_RANK_TIME];
+ }
+ else
+ {
+ hfile->slowest_time = max(hfile->slowest_time,
+ (pfile->fcounters[STDIO_F_META_TIME] +
+ pfile->fcounters[STDIO_F_READ_TIME] +
+ pfile->fcounters[STDIO_F_WRITE_TIME]));
+ }
+
+ if(pfile->base_rec.rank == -1)
+ {
+ hfile->procs = nprocs;
+ hfile->type |= FILETYPE_SHARED;
+
+ }
+ else if(hfile->procs > 1)
+ {
+ hfile->type &= (~FILETYPE_UNIQUE);
+ hfile->type |= FILETYPE_PARTSHARED;
+ }
+ else
+ {
+ hfile->type |= FILETYPE_UNIQUE;
+ }
+
+ hfile->cumul_time += pfile->fcounters[STDIO_F_META_TIME] +
+ pfile->fcounters[STDIO_F_READ_TIME] +
+ pfile->fcounters[STDIO_F_WRITE_TIME];
+
+ if(hfile->rec_dat == NULL)
+ {
+ hfile->rec_dat = malloc(sizeof(struct darshan_stdio_file));
+ assert(hfile->rec_dat);
+ memset(hfile->rec_dat, 0, sizeof(struct darshan_stdio_file));
+ }
+ tmp = (struct darshan_stdio_file*)hfile->rec_dat;
+
+ for(i = 0; i < STDIO_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case STDIO_MAX_BYTE_READ:
+ case STDIO_MAX_BYTE_WRITTEN:
+ if (tmp->counters[i] < pfile->counters[i])
+ {
+ tmp->counters[i] = pfile->counters[i];
+ }
+ break;
+ case POSIX_FASTEST_RANK:
+ case POSIX_SLOWEST_RANK:
+ case POSIX_FASTEST_RANK_BYTES:
+ case POSIX_SLOWEST_RANK_BYTES:
+ tmp->counters[i] = 0;
+ break;
+ default:
+ tmp->counters[i] += pfile->counters[i];
+ break;
+ }
+ }
+
+ for(i = 0; i < STDIO_F_NUM_INDICES; i++)
+ {
+ switch(i)
+ {
+ case STDIO_F_OPEN_START_TIMESTAMP:
+ case STDIO_F_CLOSE_START_TIMESTAMP:
+ case STDIO_F_READ_START_TIMESTAMP:
+ case STDIO_F_WRITE_START_TIMESTAMP:
+ if(tmp->fcounters[i] == 0 ||
+ tmp->fcounters[i] > pfile->fcounters[i])
+ {
+ tmp->fcounters[i] = pfile->fcounters[i];
+ }
+ break;
+ case STDIO_F_READ_END_TIMESTAMP:
+ case STDIO_F_WRITE_END_TIMESTAMP:
+ case STDIO_F_OPEN_END_TIMESTAMP:
+ case STDIO_F_CLOSE_END_TIMESTAMP:
+ if(tmp->fcounters[i] == 0 ||
+ tmp->fcounters[i] < pfile->fcounters[i])
+ {
+ tmp->fcounters[i] = pfile->fcounters[i];
+ }
+ break;
+ case STDIO_F_FASTEST_RANK_TIME:
+ case STDIO_F_SLOWEST_RANK_TIME:
+ case STDIO_F_VARIANCE_RANK_TIME:
+ case STDIO_F_VARIANCE_RANK_BYTES:
+ tmp->fcounters[i] = 0;
+ break;
+ default:
+ tmp->fcounters[i] += pfile->fcounters[i];
+ break;
+ }
+ }
+
+ return;
+}
+
void posix_accum_file(struct darshan_posix_file *pfile,
hash_entry_t *hfile,
int64_t nprocs)
@@ -1296,6 +1415,74 @@ void mpiio_accum_perf(struct darshan_mpiio_file *mfile,
return;
}
+void stdio_calc_file(hash_entry_t *file_hash,
+ file_data_t *fdata)
+{
+ hash_entry_t *curr = NULL;
+ hash_entry_t *tmp = NULL;
+ struct darshan_stdio_file *file_rec;
+
+ memset(fdata, 0, sizeof(*fdata));
+ HASH_ITER(hlink, file_hash, curr, tmp)
+ {
+ int64_t bytes;
+ int64_t r;
+ int64_t w;
+
+ file_rec = (struct darshan_stdio_file*)curr->rec_dat;
+ assert(file_rec);
+
+ bytes = file_rec->counters[STDIO_BYTES_READ] +
+ file_rec->counters[STDIO_BYTES_WRITTEN];
+
+ r = file_rec->counters[STDIO_READS];
+
+ w = file_rec->counters[STDIO_WRITES];
+
+ fdata->total += 1;
+ fdata->total_size += bytes;
+ fdata->total_max = max(fdata->total_max, bytes);
+
+ if (r && !w)
+ {
+ fdata->read_only += 1;
+ fdata->read_only_size += bytes;
+ fdata->read_only_max = max(fdata->read_only_max, bytes);
+ }
+
+ if (!r && w)
+ {
+ fdata->write_only += 1;
+ fdata->write_only_size += bytes;
+ fdata->write_only_max = max(fdata->write_only_max, bytes);
+ }
+
+ if (r && w)
+ {
+ fdata->read_write += 1;
+ fdata->read_write_size += bytes;
+ fdata->read_write_max = max(fdata->read_write_max, bytes);
+ }
+
+ if ((curr->type & (FILETYPE_SHARED|FILETYPE_PARTSHARED)))
+ {
+ fdata->shared += 1;
+ fdata->shared_size += bytes;
+ fdata->shared_max = max(fdata->shared_max, bytes);
+ }
+
+ if ((curr->type & (FILETYPE_UNIQUE)))
+ {
+ fdata->unique += 1;
+ fdata->unique_size += bytes;
+ fdata->unique_max = max(fdata->unique_max, bytes);
+ }
+ }
+
+ return;
+}
+
+
void posix_calc_file(hash_entry_t *file_hash,
file_data_t *fdata)
{
@@ -1316,7 +1503,6 @@ void posix_calc_file(hash_entry_t *file_hash,
bytes = file_rec->counters[POSIX_BYTES_READ] +
file_rec->counters[POSIX_BYTES_WRITTEN];
- /* XXX: need to update this to account for stdio counters, too */
r = file_rec->counters[POSIX_READS];
w = file_rec->counters[POSIX_WRITES];
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/e7f180dc028cb1ea34cf01d2225feeac99467912...384e9b5cf77c0df8529875838b60ea2c2053ab04
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20160918/c55d4354/attachment-0001.html>
More information about the Darshan-commits
mailing list