[Darshan-commits] [Git][darshan/darshan][master] backwards compatibility support for stdio/posix
Shane Snyder
xgitlab at cels.anl.gov
Fri Jan 18 12:56:05 CST 2019
Shane Snyder pushed to branch master at darshan / darshan
Commits:
3a5d82c6 by Shane Snyder at 2019-01-18T18:55:55Z
backwards compatibility support for stdio/posix
- - - - -
3 changed files:
- darshan-stdio-log-format.h
- darshan-util/darshan-posix-logutils.c
- darshan-util/darshan-stdio-logutils.c
Changes:
=====================================
darshan-stdio-log-format.h
=====================================
@@ -8,7 +8,7 @@
#define __DARSHAN_STDIO_LOG_FORMAT_H
/* current log format version, to support backwards compatibility */
-#define DARSHAN_STDIO_VER 1
+#define DARSHAN_STDIO_VER 2
#define STDIO_COUNTERS \
/* count of fopens */\
=====================================
darshan-util/darshan-posix-logutils.c
=====================================
@@ -32,6 +32,7 @@ char *posix_f_counter_names[] = {
#define DARSHAN_POSIX_FILE_SIZE_1 680
#define DARSHAN_POSIX_FILE_SIZE_2 648
+#define DARSHAN_POSIX_FILE_SIZE_3 664
static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p);
static int darshan_log_put_posix_file(darshan_fd fd, void* posix_buf);
@@ -131,7 +132,38 @@ static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p)
*((double *)src_p) = -1;
*((double *)(src_p + sizeof(double))) = -1;
}
+ if(fd->mod_ver[DARSHAN_POSIX_MOD] <=3)
+ {
+ if(fd->mod_ver[DARSHAN_POSIX_MOD] == 3)
+ {
+ rec_len = DARSHAN_POSIX_FILE_SIZE_3;
+ ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, scratch, rec_len);
+ if(ret != rec_len)
+ goto exit;
+ }
+ /* upconvert version 3 to version 4 in-place */
+ dest_p = scratch + sizeof(struct darshan_base_record) +
+ (3 * sizeof(int64_t));
+ src_p = dest_p - (2 * sizeof(int64_t));
+ len = rec_len - (src_p - scratch);
+ memmove(dest_p, src_p, len);
+ /* set FILENOS and DUPS to -1 */
+ *((int64_t *)src_p) = -1;
+ *((int64_t *)(src_p + sizeof(int64_t))) = -1;
+
+ dest_p = scratch + sizeof(struct darshan_base_record) +
+ (13 * sizeof(int64_t));
+ src_p = dest_p - (3 * sizeof(int64_t));
+ len = rec_len - (src_p - scratch);
+ memmove(dest_p, src_p, len);
+ /* set RENAME_SOURCES and RENAME_TARGETS to -1 */
+ *((int64_t *)src_p) = -1;
+ *((int64_t *)(src_p + sizeof(int64_t))) = -1;
+ /* set RENAMED_FROM to 0 (-1 not possible since this is a uint) */
+ *((int64_t *)(src_p + (2 * sizeof(int64_t)))) = 0;
+ }
+
memcpy(file, scratch, sizeof(struct darshan_posix_file));
}
@@ -159,13 +191,17 @@ exit:
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<POSIX_F_NUM_INDICES; i++)
{
- /* skip counters we explicitly set to -1 since they don't
+ /* skip counters we explicitly set since they don't
* need to be byte swapped
*/
if((fd->mod_ver[DARSHAN_POSIX_MOD] < 3) &&
((i == POSIX_F_CLOSE_START_TIMESTAMP) ||
(i == POSIX_F_OPEN_END_TIMESTAMP)))
continue;
+ if ((fd->mod_ver[DARSHAN_POSIX_MOD] < 4) &&
+ ((i == POSIX_RENAME_SOURCES) || (i == POSIX_RENAME_TARGETS) ||
+ (i == POSIX_RENAMED_FROM)))
+ continue;
DARSHAN_BSWAP64(&file->fcounters[i]);
}
}
@@ -223,7 +259,9 @@ static void darshan_log_print_posix_description(int ver)
{
printf("\n# description of POSIX counters:\n");
printf("# POSIX_*: posix operation counts.\n");
- printf("# READS,WRITES,OPENS,SEEKS,STATS, and MMAPS are types of operations.\n");
+ printf("# READS,WRITES,OPENS,SEEKS,STATS,MMAPS,SYNCS,FILENOS,DUPS are types of operations.\n");
+ printf("# POSIX_RENAME_SOURCES/TARGETS: total count file was source or target of a rename operation\n");
+ printf("# POSIX_RENAMED_FROM: Darshan record ID of the first rename source, if file was a rename target\n");
printf("# POSIX_MODE: mode that file was opened in.\n");
printf("# POSIX_BYTES_*: total bytes read and written.\n");
printf("# POSIX_MAX_BYTE_*: highest offset byte read and written.\n");
@@ -247,7 +285,7 @@ static void darshan_log_print_posix_description(int ver)
printf("# POSIX_F_*_RANK_TIME: fastest and slowest I/O time for a single rank (for shared files).\n");
printf("# POSIX_F_VARIANCE_RANK_*: variance of total I/O time and bytes moved for all ranks (for shared files).\n");
- if(ver <= 1)
+ if(ver == 1)
{
printf("\n# WARNING: POSIX module log format version 1 has the following limitations:\n");
printf("# - Darshan version 3.1.0 and earlier had only partial instrumentation of stdio stream I/O functions.\n");
@@ -256,9 +294,20 @@ static void darshan_log_print_posix_description(int ver)
}
if(ver <= 2)
{
- printf("\n# WARNING: POSIX module log format version <=2 does not support the following counters:\n");
- printf("# - POSIX_F_CLOSE_START_TIMESTAMP\n");
- printf("# - POSIX_F_OPEN_END_TIMESTAMP\n");
+ printf("\n# WARNING: POSIX module log format version <=2 has the following limitations:\n");
+ printf("# - No support for the following timers:\n");
+ printf("# \t- POSIX_F_CLOSE_START_TIMESTAMP\n");
+ printf("# \t- POSIX_F_OPEN_END_TIMESTAMP\n");
+ }
+ if(ver <=3)
+ {
+ printf("\n# WARNING: POSIX module log format version <=3 has the following limitations:\n");
+ printf("# - No support for the following counters to properly instrument dup, fileno, and rename operations:\n");
+ printf("# \t- POSIX_FILENOS\n");
+ printf("# \t- POSIX_DUPS\n");
+ printf("# \t- POSIX_RENAME_SOURCES\n");
+ printf("# \t- POSIX_RENAME_TARGETS\n");
+ printf("# \t- POSIX_RENAMED_FROM\n");
}
return;
@@ -389,6 +438,8 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
switch(i)
{
case POSIX_OPENS:
+ case POSIX_FILENOS:
+ case POSIX_DUPS:
case POSIX_READS:
case POSIX_WRITES:
case POSIX_SEEKS:
@@ -396,6 +447,8 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
case POSIX_MMAPS:
case POSIX_FSYNCS:
case POSIX_FDSYNCS:
+ case POSIX_RENAME_SOURCES:
+ case POSIX_RENAME_TARGETS:
case POSIX_BYTES_READ:
case POSIX_BYTES_WRITTEN:
case POSIX_CONSEC_READS:
@@ -430,6 +483,7 @@ static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag)
if(agg_psx_rec->counters[i] < 0) /* make sure invalid counters are -1 exactly */
agg_psx_rec->counters[i] = -1;
break;
+ case POSIX_RENAMED_FROM:
case POSIX_MODE:
case POSIX_MEM_ALIGNMENT:
case POSIX_FILE_ALIGNMENT:
=====================================
darshan-util/darshan-stdio-logutils.c
=====================================
@@ -31,6 +31,8 @@ char *stdio_f_counter_names[] = {
};
#undef X
+#define DARSHAN_STDIO_FILE_SIZE_1 240
+
/* prototypes for each of the STDIO module's logutil functions */
static int darshan_log_get_stdio_record(darshan_fd fd, void** stdio_buf_p);
static int darshan_log_put_stdio_record(darshan_fd fd, void* stdio_buf);
@@ -62,6 +64,7 @@ struct darshan_mod_logutil_funcs stdio_logutils =
static int darshan_log_get_stdio_record(darshan_fd fd, void** stdio_buf_p)
{
struct darshan_stdio_file *file = *((struct darshan_stdio_file **)stdio_buf_p);
+ int rec_len;
int i;
int ret;
@@ -75,13 +78,44 @@ static int darshan_log_get_stdio_record(darshan_fd fd, void** stdio_buf_p)
return(-1);
}
- /* read a STDIO module record from the darshan log file */
- ret = darshan_log_get_mod(fd, DARSHAN_STDIO_MOD, file,
- sizeof(struct darshan_stdio_file));
+ if(fd->mod_ver[DARSHAN_STDIO_MOD] == DARSHAN_STDIO_VER)
+ {
+ /* log format is in current version, so we don't need to do any
+ * translation of counters while reading
+ */
+ rec_len = sizeof(struct darshan_stdio_file);
+ ret = darshan_log_get_mod(fd, DARSHAN_STDIO_MOD, file, rec_len);
+ }
+ else
+ {
+ char scratch[1024] = {0};
+ char *src_p, *dest_p;
+ int len;
+
+ if(fd->mod_ver[DARSHAN_STDIO_MOD] == 1)
+ {
+ rec_len = DARSHAN_STDIO_FILE_SIZE_1;
+ ret = darshan_log_get_mod(fd, DARSHAN_STDIO_MOD, scratch, rec_len);
+ if(ret != rec_len)
+ goto exit;
+
+ /* upconvert version 1 to version 2 in-place */
+ dest_p = scratch + sizeof(struct darshan_base_record) +
+ (2 * sizeof(int64_t));
+ src_p = dest_p - sizeof(int64_t);
+ len = rec_len - (src_p - scratch);
+ memmove(dest_p, src_p, len);
+ /* set FDOPENS to -1 */
+ *((int64_t *)src_p) = -1;
+ }
+
+ memcpy(file, scratch, sizeof(struct darshan_posix_file));
+ }
+exit:
if(*stdio_buf_p == NULL)
{
- if(ret == sizeof(struct darshan_stdio_file))
+ if(ret == rec_len)
*stdio_buf_p = file;
else
free(file);
@@ -89,7 +123,7 @@ static int darshan_log_get_stdio_record(darshan_fd fd, void** stdio_buf_p)
if(ret < 0)
return(-1);
- else if(ret < sizeof(struct darshan_stdio_file))
+ else if(ret < rec_len)
return(0);
else
{
@@ -99,7 +133,15 @@ static int darshan_log_get_stdio_record(darshan_fd fd, void** stdio_buf_p)
DARSHAN_BSWAP64(&file->base_rec.id);
DARSHAN_BSWAP64(&file->base_rec.rank);
for(i=0; i<STDIO_NUM_INDICES; i++)
+ {
+ /* skip counters we explicitly set since they don't
+ * need to be byte swapped
+ */
+ if((fd->mod_ver[DARSHAN_STDIO_MOD] == 1) &&
+ (i == STDIO_FDOPENS))
+ continue;
DARSHAN_BSWAP64(&file->counters[i]);
+ }
for(i=0; i<STDIO_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
@@ -157,7 +199,7 @@ static void darshan_log_print_stdio_record(void *file_rec, char *file_name,
static void darshan_log_print_stdio_description(int ver)
{
printf("\n# description of STDIO counters:\n");
- printf("# STDIO_{OPENS|WRITES|READS|SEEKS|FLUSHES} are types of operations.\n");
+ printf("# STDIO_{OPENS|FDOPENS|WRITES|READS|SEEKS|FLUSHES} are types of operations.\n");
printf("# STDIO_BYTES_*: total bytes read and written.\n");
printf("# STDIO_MAX_BYTE_*: highest offset byte read and written.\n");
printf("# STDIO_*_RANK: rank of the processes that were the fastest and slowest at I/O (for shared files).\n");
@@ -168,6 +210,13 @@ static void darshan_log_print_stdio_description(int ver)
printf("# STDIO_F_*_RANK_TIME: fastest and slowest I/O time for a single rank (for shared files).\n");
printf("# STDIO_F_VARIANCE_RANK_*: variance of total I/O time and bytes moved for all ranks (for shared files).\n");
+ if(ver == 1)
+ {
+ printf("\n# WARNING: STDIO module log format version 1 has the following limitations:\n");
+ printf("# - No support for properly instrumenting fdopen operations (STDIO_FDOPENS)\n");
+ }
+
+
return;
}
@@ -272,6 +321,7 @@ static void darshan_log_agg_stdio_records(void *rec, void *agg_rec, int init_fla
switch(i)
{
case STDIO_OPENS:
+ case STDIO_FDOPENS:
case STDIO_READS:
case STDIO_WRITES:
case STDIO_SEEKS:
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/commit/3a5d82c61c524e4856065a401ff146d68b39953c
--
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/commit/3a5d82c61c524e4856065a401ff146d68b39953c
You're receiving this email because of your account on xgitlab.cels.anl.gov.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20190118/8e5bc7c3/attachment-0001.html>
More information about the Darshan-commits
mailing list