[Darshan-commits] [Git][darshan/darshan][master] 3 commits: small bug fix in stdio module
Shane Snyder
xgitlab at cels.anl.gov
Thu Sep 22 15:02:08 CDT 2016
Shane Snyder pushed to branch master at darshan / darshan
Commits:
557e586c by Shane Snyder at 2016-09-22T11:32:34-05:00
small bug fix in stdio module
- - - - -
2276d929 by Shane Snyder at 2016-09-22T11:34:18-05:00
modify posix open and close counters
POSIX_F_OPEN_TIMESTAMP is now POSIX_F_OPEN_START_TIMESTAMP
POSIX_F_CLOSE_TIMESTAMP is now POSIX_F_CLOSE_END_TIMESTAMP
I also added POSIX_F_OPEN_END_TIMESTAMP and
POSIX_F_CLOSE_START_TIMESTAMP, so we can better understand how
long open and close phases are for some applications.
- - - - -
5dbe7dc1 by Shane Snyder at 2016-09-22T14:58:57-05:00
add backwards compatability for new posix version
- - - - -
6 changed files:
- darshan-log-format.h
- darshan-posix-log-format.h
- darshan-runtime/lib/darshan-posix.c
- darshan-runtime/lib/darshan-stdio.c
- darshan-util/darshan-parser.c
- darshan-util/darshan-posix-logutils.c
Changes:
=====================================
darshan-log-format.h
=====================================
--- a/darshan-log-format.h
+++ b/darshan-log-format.h
@@ -20,6 +20,10 @@
#endif
/* update this on file format changes */
+/* NOTE: this should be updated when general changes are made to the
+ * log format version, NOT when a new version of a module record is
+ * introduced -- we have module-specific versions to handle that
+ */
#define DARSHAN_LOG_VERSION "3.10"
/* magic number for validating output files and checking byte order */
=====================================
darshan-posix-log-format.h
=====================================
--- a/darshan-posix-log-format.h
+++ b/darshan-posix-log-format.h
@@ -7,7 +7,7 @@
#define __DARSHAN_POSIX_LOG_FORMAT_H
/* current POSIX log format version */
-#define DARSHAN_POSIX_VER 2
+#define DARSHAN_POSIX_VER 3
#define POSIX_COUNTERS \
/* count of posix opens */\
@@ -108,17 +108,21 @@
#define POSIX_F_COUNTERS \
/* timestamp of first open */\
- X(POSIX_F_OPEN_TIMESTAMP) \
+ X(POSIX_F_OPEN_START_TIMESTAMP) \
/* timestamp of first read */\
X(POSIX_F_READ_START_TIMESTAMP) \
/* timestamp of first write */\
X(POSIX_F_WRITE_START_TIMESTAMP) \
+ /* timestamp of first close */\
+ X(POSIX_F_CLOSE_START_TIMESTAMP) \
+ /* timestamp of last open */\
+ X(POSIX_F_OPEN_END_TIMESTAMP) \
/* timestamp of last read */\
X(POSIX_F_READ_END_TIMESTAMP) \
/* timestamp of last write */\
X(POSIX_F_WRITE_END_TIMESTAMP) \
/* timestamp of last close */\
- X(POSIX_F_CLOSE_TIMESTAMP) \
+ X(POSIX_F_CLOSE_END_TIMESTAMP) \
/* cumulative posix read time */\
X(POSIX_F_READ_TIME) \
/* cumulative posix write time */\
=====================================
darshan-runtime/lib/darshan-posix.c
=====================================
--- a/darshan-runtime/lib/darshan-posix.c
+++ b/darshan-runtime/lib/darshan-posix.c
@@ -204,9 +204,10 @@ static int darshan_mem_alignment = 1;
rec_ref->last_byte_written = 0; \
rec_ref->last_byte_read = 0; \
rec_ref->file_rec->counters[POSIX_OPENS] += 1; \
- if(rec_ref->file_rec->fcounters[POSIX_F_OPEN_TIMESTAMP] == 0 || \
- rec_ref->file_rec->fcounters[POSIX_F_OPEN_TIMESTAMP] > __tm1) \
- rec_ref->file_rec->fcounters[POSIX_F_OPEN_TIMESTAMP] = __tm1; \
+ if(rec_ref->file_rec->fcounters[POSIX_F_OPEN_START_TIMESTAMP] == 0 || \
+ rec_ref->file_rec->fcounters[POSIX_F_OPEN_START_TIMESTAMP] > __tm1) \
+ rec_ref->file_rec->fcounters[POSIX_F_OPEN_START_TIMESTAMP] = __tm1; \
+ rec_ref->file_rec->fcounters[POSIX_F_OPEN_END_TIMESTAMP] = __tm2; \
DARSHAN_TIMER_INC_NO_OVERLAP(rec_ref->file_rec->fcounters[POSIX_F_META_TIME], \
__tm1, __tm2, rec_ref->last_meta_end); \
darshan_add_record_ref(&(posix_runtime->fd_hash), &__ret, sizeof(int), rec_ref); \
@@ -1021,8 +1022,10 @@ int DARSHAN_DECL(close)(int fd)
{
rec_ref->last_byte_written = 0;
rec_ref->last_byte_read = 0;
- rec_ref->file_rec->fcounters[POSIX_F_CLOSE_TIMESTAMP] =
- darshan_core_wtime();
+ if(rec_ref->file_rec->fcounters[POSIX_F_CLOSE_START_TIMESTAMP] == 0 ||
+ rec_ref->file_rec->fcounters[POSIX_F_CLOSE_START_TIMESTAMP] > tm1)
+ rec_ref->file_rec->fcounters[POSIX_F_CLOSE_START_TIMESTAMP] = tm1;
+ rec_ref->file_rec->fcounters[POSIX_F_CLOSE_END_TIMESTAMP] = tm2;
DARSHAN_TIMER_INC_NO_OVERLAP(
rec_ref->file_rec->fcounters[POSIX_F_META_TIME],
tm1, tm2, rec_ref->last_meta_end);
@@ -1488,7 +1491,7 @@ static void posix_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* min non-zero (if available) value */
- for(j=POSIX_F_OPEN_TIMESTAMP; j<=POSIX_F_WRITE_START_TIMESTAMP; j++)
+ for(j=POSIX_F_OPEN_START_TIMESTAMP; j<=POSIX_F_CLOSE_START_TIMESTAMP; j++)
{
if((infile->fcounters[j] < inoutfile->fcounters[j] &&
infile->fcounters[j] > 0) || inoutfile->fcounters[j] == 0)
@@ -1498,7 +1501,7 @@ static void posix_record_reduction_op(void* infile_v, void* inoutfile_v,
}
/* max */
- for(j=POSIX_F_READ_END_TIMESTAMP; j<=POSIX_F_CLOSE_TIMESTAMP; j++)
+ for(j=POSIX_F_OPEN_END_TIMESTAMP; j<=POSIX_F_CLOSE_END_TIMESTAMP; j++)
{
if(infile->fcounters[j] > inoutfile->fcounters[j])
tmp_file.fcounters[j] = infile->fcounters[j];
=====================================
darshan-runtime/lib/darshan-stdio.c
=====================================
--- a/darshan-runtime/lib/darshan-stdio.c
+++ b/darshan-runtime/lib/darshan-stdio.c
@@ -971,7 +971,7 @@ static void stdio_runtime_initialize()
/* return if darshan-core does not provide enough module memory */
if(stdio_buf_size < sizeof(struct darshan_stdio_file))
{
- darshan_core_unregister_module(DARSHAN_POSIX_MOD);
+ darshan_core_unregister_module(DARSHAN_STDIO_MOD);
return;
}
=====================================
darshan-util/darshan-parser.c
=====================================
--- a/darshan-util/darshan-parser.c
+++ b/darshan-util/darshan-parser.c
@@ -969,18 +969,20 @@ void posix_accum_file(struct darshan_posix_file *pfile,
{
switch(i)
{
- case POSIX_F_OPEN_TIMESTAMP:
+ case POSIX_F_OPEN_START_TIMESTAMP:
case POSIX_F_READ_START_TIMESTAMP:
case POSIX_F_WRITE_START_TIMESTAMP:
+ case POSIX_F_CLOSE_START_TIMESTAMP:
if(tmp->fcounters[i] == 0 ||
tmp->fcounters[i] > pfile->fcounters[i])
{
tmp->fcounters[i] = pfile->fcounters[i];
}
break;
+ case POSIX_F_OPEN_END_TIMESTAMP:
case POSIX_F_READ_END_TIMESTAMP:
case POSIX_F_WRITE_END_TIMESTAMP:
- case POSIX_F_CLOSE_TIMESTAMP:
+ case POSIX_F_CLOSE_END_TIMESTAMP:
if(tmp->fcounters[i] == 0 ||
tmp->fcounters[i] < pfile->fcounters[i])
{
@@ -1287,12 +1289,12 @@ void posix_accum_perf(struct darshan_posix_file *pfile,
if(pfile->base_rec.rank == -1)
{
/* by_open */
- if(pfile->fcounters[POSIX_F_CLOSE_TIMESTAMP] >
- pfile->fcounters[POSIX_F_OPEN_TIMESTAMP])
+ if(pfile->fcounters[POSIX_F_CLOSE_END_TIMESTAMP] >
+ pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
{
pdata->shared_time_by_open +=
- pfile->fcounters[POSIX_F_CLOSE_TIMESTAMP] -
- pfile->fcounters[POSIX_F_OPEN_TIMESTAMP];
+ pfile->fcounters[POSIX_F_CLOSE_END_TIMESTAMP] -
+ pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
}
/* by_open_lastio */
@@ -1300,21 +1302,21 @@ void posix_accum_perf(struct darshan_posix_file *pfile,
pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP])
{
/* be careful: file may have been opened but not read or written */
- if(pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] > pfile->fcounters[POSIX_F_OPEN_TIMESTAMP])
+ if(pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] > pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
{
pdata->shared_time_by_open_lastio +=
pfile->fcounters[POSIX_F_READ_END_TIMESTAMP] -
- pfile->fcounters[POSIX_F_OPEN_TIMESTAMP];
+ pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
}
}
else
{
/* be careful: file may have been opened but not read or written */
- if(pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP] > pfile->fcounters[POSIX_F_OPEN_TIMESTAMP])
+ if(pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP] > pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP])
{
pdata->shared_time_by_open_lastio +=
pfile->fcounters[POSIX_F_WRITE_END_TIMESTAMP] -
- pfile->fcounters[POSIX_F_OPEN_TIMESTAMP];
+ pfile->fcounters[POSIX_F_OPEN_START_TIMESTAMP];
}
}
@@ -1867,8 +1869,8 @@ void posix_file_list(hash_entry_t *file_hash,
printf("\n# <record_id>\t<file_name>\t<nprocs>\t<slowest>\t<avg>");
if(detail_flag)
{
- printf("\t<start_open>\t<start_read>\t<start_write>");
- printf("\t<end_read>\t<end_write>\t<end_close>\t<posix_opens>");
+ printf("\t<start_open>\t<start_read>\t<start_write>\t<start_close>");
+ printf("\t<end_open>\t<end_read>\t<end_write>\t<end_close>\t<posix_opens>");
for(i=POSIX_SIZE_READ_0_100; i<= POSIX_SIZE_WRITE_1G_PLUS; i++)
printf("\t<%s>", posix_counter_names[i]);
}
@@ -1891,7 +1893,7 @@ void posix_file_list(hash_entry_t *file_hash,
if(detail_flag)
{
- for(i=POSIX_F_OPEN_TIMESTAMP; i<=POSIX_F_CLOSE_TIMESTAMP; i++)
+ for(i=POSIX_F_OPEN_START_TIMESTAMP; i<=POSIX_F_CLOSE_END_TIMESTAMP; i++)
{
printf("\t%f", file_rec->fcounters[i]);
}
=====================================
darshan-util/darshan-posix-logutils.c
=====================================
--- a/darshan-util/darshan-posix-logutils.c
+++ b/darshan-util/darshan-posix-logutils.c
@@ -31,6 +31,7 @@ char *posix_f_counter_names[] = {
#undef X
#define DARSHAN_POSIX_FILE_SIZE_1 680
+#define DARSHAN_POSIX_FILE_SIZE_2 648
static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p);
static int darshan_log_put_posix_file(darshan_fd fd, void* posix_buf, int ver);
@@ -97,12 +98,43 @@ static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p)
p += (4 * sizeof(int64_t)); /* skip old stdio counters */
memcpy(&(file->counters[6]), p, 58 * sizeof(int64_t));
p += (58 * sizeof(int64_t));
- memcpy(&(file->fcounters[0]), p, 15 * sizeof(double));
+ memcpy(&(file->fcounters[0]), p, 3 * sizeof(double));
+ file->fcounters[POSIX_F_CLOSE_START_TIMESTAMP] = -1;
+ file->fcounters[POSIX_F_OPEN_END_TIMESTAMP] = -1;
+ p += 3 * sizeof(double);
+ memcpy(&(file->fcounters[POSIX_F_READ_END_TIMESTAMP]), p, 12 * sizeof(double));
}
free(buffer);
}
else if(fd->mod_ver[DARSHAN_POSIX_MOD] == 2)
{
+ buffer = malloc(DARSHAN_POSIX_FILE_SIZE_2);
+ if(!buffer)
+ {
+ if(*posix_buf_p == NULL)
+ free(file);
+ return(-1);
+ }
+
+ rec_len = DARSHAN_POSIX_FILE_SIZE_2;
+ ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, buffer, rec_len);
+ if(ret == rec_len)
+ {
+ p = buffer;
+ memcpy(&(file->base_rec), p, sizeof(struct darshan_base_record));
+ p += sizeof(struct darshan_base_record);
+ memcpy(&(file->counters[0]), p, POSIX_NUM_INDICES * sizeof(int64_t));
+ p += POSIX_NUM_INDICES * sizeof(int64_t);
+ memcpy(&(file->fcounters[0]), p, 3 * sizeof(double));
+ file->fcounters[POSIX_F_CLOSE_START_TIMESTAMP] = -1;
+ file->fcounters[POSIX_F_OPEN_END_TIMESTAMP] = -1;
+ p += 3 * sizeof(double);
+ memcpy(&(file->fcounters[POSIX_F_READ_END_TIMESTAMP]), p, 12 * sizeof(double));
+ }
+ free(buffer);
+ }
+ else if(fd->mod_ver[DARSHAN_POSIX_MOD] == 3)
+ {
rec_len = sizeof(struct darshan_posix_file);
ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, file, rec_len);
}
@@ -196,10 +228,8 @@ static void darshan_log_print_posix_description()
printf("# POSIX_ACCESS*_COUNT: count of the four most common access sizes.\n");
printf("# POSIX_*_RANK: rank of the processes that were the fastest and slowest at I/O (for shared files).\n");
printf("# POSIX_*_RANK_BYTES: bytes transferred by the fastest and slowest ranks (for shared files).\n");
- printf("# POSIX_F_OPEN_TIMESTAMP: timestamp of first open.\n");
- printf("# POSIX_F_*_START_TIMESTAMP: timestamp of first read/write.\n");
- printf("# POSIX_F_*_END_TIMESTAMP: timestamp of last read/write.\n");
- printf("# POSIX_F_CLOSE_TIMESTAMP: timestamp of last close.\n");
+ printf("# POSIX_F_*_START_TIMESTAMP: timestamp of first open/read/write/close.\n");
+ printf("# POSIX_F_*_END_TIMESTAMP: timestamp of last open/read/write/close.\n");
printf("# POSIX_F_READ/WRITE/META_TIME: cumulative time spent in read, write, or metadata operations.\n");
printf("# POSIX_F_MAX_*_TIME: duration of the slowest read and write operations.\n");
printf("# POSIX_F_*_RANK_TIME: fastest and slowest I/O time for a single rank (for shared files).\n");
@@ -476,9 +506,10 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
/* sum */
agg_psx_rec->fcounters[i] += psx_rec->fcounters[i];
break;
- case POSIX_F_OPEN_TIMESTAMP:
+ case POSIX_F_OPEN_START_TIMESTAMP:
case POSIX_F_READ_START_TIMESTAMP:
case POSIX_F_WRITE_START_TIMESTAMP:
+ case POSIX_F_CLOSE_START_TIMESTAMP:
/* minimum non-zero */
if((psx_rec->fcounters[i] > 0) &&
((agg_psx_rec->fcounters[i] == 0) ||
@@ -487,9 +518,10 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
agg_psx_rec->fcounters[i] = psx_rec->fcounters[i];
}
break;
+ case POSIX_F_OPEN_END_TIMESTAMP:
case POSIX_F_READ_END_TIMESTAMP:
case POSIX_F_WRITE_END_TIMESTAMP:
- case POSIX_F_CLOSE_TIMESTAMP:
+ case POSIX_F_CLOSE_END_TIMESTAMP:
/* maximum */
if(psx_rec->fcounters[i] > agg_psx_rec->fcounters[i])
{
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/8438ee10a1bcdcd87b0078d64cd41634c6a72bfa...5dbe7dc16144f479bc284da92c2cbaf2a2d61229
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20160922/4795533e/attachment-0001.html>
More information about the Darshan-commits
mailing list