[Darshan-commits] [Git][darshan/darshan][master] 5 commits: cleanup posix module upconvert code
Shane Snyder
xgitlab at cels.anl.gov
Fri Sep 23 14:38:10 CDT 2016
Shane Snyder pushed to branch master at darshan / darshan
Commits:
85729752 by Shane Snyder at 2016-09-23T11:57:46-05:00
cleanup posix module upconvert code
- - - - -
a9dbd195 by Shane Snyder at 2016-09-23T13:25:52-05:00
more detailed darshan-parser output
- - - - -
51505ca9 by Shane Snyder at 2016-09-23T13:32:56-05:00
cleanup darshan-merge usage printing
- - - - -
f426db8d by Shane Snyder at 2016-09-23T13:41:33-05:00
cleanup darshan-analyzer output
- - - - -
84b8bb1c by Shane Snyder at 2016-09-23T14:36:43-05:00
print version-specific warnings in module logutils
- - - - -
13 changed files:
- darshan-util/darshan-analyzer.c
- darshan-util/darshan-bgq-logutils.c
- darshan-util/darshan-hdf5-logutils.c
- darshan-util/darshan-logutils.c
- darshan-util/darshan-logutils.h
- darshan-util/darshan-lustre-logutils.c
- darshan-util/darshan-merge.c
- darshan-util/darshan-mpiio-logutils.c
- darshan-util/darshan-null-logutils.c
- darshan-util/darshan-parser.c
- darshan-util/darshan-pnetcdf-logutils.c
- darshan-util/darshan-posix-logutils.c
- darshan-util/darshan-stdio-logutils.c
Changes:
=====================================
darshan-util/darshan-analyzer.c
=====================================
--- a/darshan-util/darshan-analyzer.c
+++ b/darshan-util/darshan-analyzer.c
@@ -169,17 +169,18 @@ int main(int argc, char **argv)
}
printf ("log dir: %s\n", base);
- printf (" total: %d\n", total_count);
- printf (" shared: %lf [%d]\n", (double)total_shared/(double)total_count, total_shared);
- printf (" fpp: %lf [%d]\n", (double)total_fpp/(double)total_count, total_fpp);
- printf (" mpio: %lf [%d]\n", (double)total_mpio/(double)total_count, total_mpio);
- printf (" pnet: %lf [%d]\n", (double)total_pnet/(double)total_count, total_pnet);
- printf (" hdf5: %lf [%d]\n", (double)total_hdf5/(double)total_count, total_hdf5);
+ printf ("total logs: %d\n", total_count);
+ printf (" shared file access: %lf [%d]\n", (double)total_shared/(double)total_count, total_shared);
+ printf ("file-per-proccess access: %lf [%d]\n", (double)total_fpp/(double)total_count, total_fpp);
+ printf (" mpio access: %lf [%d]\n", (double)total_mpio/(double)total_count, total_mpio);
+ printf (" pnetcdf access: %lf [%d]\n", (double)total_pnet/(double)total_count, total_pnet);
+ printf (" hdf5 access: %lf [%d]\n", (double)total_hdf5/(double)total_count, total_hdf5);
+ printf("\nI/O percentage of runtime:\n");
printf ("%.2lf-%.2lf: %d\n", (double)0.0, (double)BUCKET1, bucket1);
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET1, (double)BUCKET2, bucket2);
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET2, (double)BUCKET3, bucket3);
printf ("%.2lf-%.2lf: %d\n", (double)BUCKET3, (double)BUCKET4, bucket4);
- printf ("%.2lf-%.2lf: %d\n", (double)BUCKET4, (double)100.0, bucket5);
+ printf ("%.2lf-%.2lf: %d\n", (double)BUCKET4, (double)1.0, bucket5);
return 0;
}
=====================================
darshan-util/darshan-bgq-logutils.c
=====================================
--- a/darshan-util/darshan-bgq-logutils.c
+++ b/darshan-util/darshan-bgq-logutils.c
@@ -38,7 +38,7 @@ static int darshan_log_get_bgq_rec(darshan_fd fd, void** bgq_buf_p);
static int darshan_log_put_bgq_rec(darshan_fd fd, void* bgq_buf, int ver);
static void darshan_log_print_bgq_rec(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type, int ver);
-static void darshan_log_print_bgq_description(void);
+static void darshan_log_print_bgq_description(int ver);
static void darshan_log_print_bgq_rec_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_bgq_recs(void *rec, void *agg_rec, int init_flag);
@@ -175,7 +175,7 @@ static void darshan_log_print_bgq_rec(void *file_rec, char *file_name,
return;
}
-static void darshan_log_print_bgq_description()
+static void darshan_log_print_bgq_description(int ver)
{
printf("\n# description of BGQ counters:\n");
printf("# BGQ_CSJOBID: BGQ control system job ID.\n");
@@ -187,8 +187,6 @@ static void darshan_log_print_bgq_description()
printf("# BGQ_TORUSENABLED: which dimensions of the torus are enabled.\n");
printf("# BGQ_F_TIMESTAMP: timestamp when the BGQ data was collected.\n");
- DARSHAN_PRINT_HEADER();
-
return;
}
=====================================
darshan-util/darshan-hdf5-logutils.c
=====================================
--- a/darshan-util/darshan-hdf5-logutils.c
+++ b/darshan-util/darshan-hdf5-logutils.c
@@ -34,7 +34,7 @@ static int darshan_log_get_hdf5_file(darshan_fd fd, void** hdf5_buf_p);
static int darshan_log_put_hdf5_file(darshan_fd fd, void* hdf5_buf, int ver);
static void darshan_log_print_hdf5_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type, int ver);
-static void darshan_log_print_hdf5_description(void);
+static void darshan_log_print_hdf5_description(int ver);
static void darshan_log_print_hdf5_file_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_hdf5_files(void *rec, void *agg_rec, int init_flag);
@@ -136,15 +136,13 @@ static void darshan_log_print_hdf5_file(void *file_rec, char *file_name,
return;
}
-static void darshan_log_print_hdf5_description()
+static void darshan_log_print_hdf5_description(int ver)
{
printf("\n# description of HDF5 counters:\n");
printf("# HDF5_OPENS: HDF5 file open operation counts.\n");
printf("# HDF5_F_OPEN_TIMESTAMP: timestamp of first HDF5 file open.\n");
printf("# HDF5_F_CLOSE_TIMESTAMP: timestamp of last HDF5 file close.\n");
- DARSHAN_PRINT_HEADER();
-
return;
}
=====================================
darshan-util/darshan-logutils.c
=====================================
--- a/darshan-util/darshan-logutils.c
+++ b/darshan-util/darshan-logutils.c
@@ -765,14 +765,6 @@ void darshan_log_close(darshan_fd fd)
void darshan_log_print_version_warnings(const char *version_string)
{
- if(strcmp(version_string, "3.00") == 0)
- {
- printf("# WARNING: version 3.00 log format has the following limitations:\n");
- printf("# - Partial instrumentation of stdio stream I/O functions not parsable by Darshan versions >= 3.1.0\n");
- printf("# * Using darshan-logutils versions < 3.1.0, this data can be found in the following POSIX counters:\n");
- printf("# * POSIX_FOPENS, POSIX_FREADS, POSIX_FWRITES, POSIX_FSEEKS\n");
- }
-
return;
}
=====================================
darshan-util/darshan-logutils.h
=====================================
--- a/darshan-util/darshan-logutils.h
+++ b/darshan-util/darshan-logutils.h
@@ -101,8 +101,11 @@ struct darshan_mod_logutil_funcs
char *fs_type,
int ver
);
- /* print module-specific description of I/O characterization data */
- void (*log_print_description)(void);
+ /* print module-specific description of I/O characterization data
+ * - 'ver' is the version of the record
+ */
+ void (*log_print_description)(
+ int ver);
/* print a text diff of 2 module I/O records */
void (*log_print_diff)(
void *rec1,
=====================================
darshan-util/darshan-lustre-logutils.c
=====================================
--- a/darshan-util/darshan-lustre-logutils.c
+++ b/darshan-util/darshan-lustre-logutils.c
@@ -28,7 +28,7 @@ static int darshan_log_get_lustre_record(darshan_fd fd, void** lustre_buf_p);
static int darshan_log_put_lustre_record(darshan_fd fd, void* lustre_buf, int ver);
static void darshan_log_print_lustre_record(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type, int ver);
-static void darshan_log_print_lustre_description(void);
+static void darshan_log_print_lustre_description(int ver);
static void darshan_log_print_lustre_record_diff(void *rec1, char *file_name1,
void *rec2, char *file_name2);
static void darshan_log_agg_lustre_records(void *rec, void *agg_rec, int init_flag);
@@ -158,7 +158,7 @@ static void darshan_log_print_lustre_record(void *rec, char *file_name,
return;
}
-static void darshan_log_print_lustre_description()
+static void darshan_log_print_lustre_description(int ver)
{
printf("\n# description of LUSTRE counters:\n");
printf("# LUSTRE_OSTS: number of OSTs across the entire file system.\n");
@@ -168,8 +168,6 @@ static void darshan_log_print_lustre_description()
printf("# LUSTRE_STRIPE_WIDTH: number of OSTs over which file is striped.\n");
printf("# LUSTRE_OST_ID_*: indices of OSTs over which the file is striped.\n");
- DARSHAN_PRINT_HEADER();
-
return;
}
=====================================
darshan-util/darshan-merge.c
=====================================
--- a/darshan-util/darshan-merge.c
+++ b/darshan-util/darshan-merge.c
@@ -19,8 +19,9 @@ struct darshan_shared_record_ref
void usage(char *exename)
{
- fprintf(stderr, "Usage: %s --output <output_path> [options] <input-logs>\n", exename);
+ fprintf(stderr, "Usage: %s --output <output_path> [options] <input_log_glob>\n", exename);
fprintf(stderr, "This utility merges multiple Darshan log files into a single output log file.\n");
+ fprintf(stderr, "<input_log_glob> is a pattern that matches all input log files (e.g., /log-path/*.darshan).\n");
fprintf(stderr, "Options:\n");
fprintf(stderr, "\t--output\t(REQUIRED) Full path of the output darshan log file.\n");
fprintf(stderr, "\t--shared-redux\tReduce globally shared records into a single record.\n");
=====================================
darshan-util/darshan-mpiio-logutils.c
=====================================
--- a/darshan-util/darshan-mpiio-logutils.c
+++ b/darshan-util/darshan-mpiio-logutils.c
@@ -34,7 +34,7 @@ static int darshan_log_get_mpiio_file(darshan_fd fd, void** mpiio_buf_p);
static int darshan_log_put_mpiio_file(darshan_fd fd, void* mpiio_buf, int ver);
static void darshan_log_print_mpiio_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type, int ver);
-static void darshan_log_print_mpiio_description(void);
+static void darshan_log_print_mpiio_description(int ver);
static void darshan_log_print_mpiio_file_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_mpiio_files(void *rec, void *agg_rec, int init_flag);
@@ -136,7 +136,7 @@ static void darshan_log_print_mpiio_file(void *file_rec, char *file_name,
return;
}
-static void darshan_log_print_mpiio_description()
+static void darshan_log_print_mpiio_description(int ver)
{
printf("\n# description of MPIIO counters:\n");
printf("# MPIIO_INDEP_*: MPI independent operation counts.\n");
@@ -165,8 +165,6 @@ static void darshan_log_print_mpiio_description()
printf("# MPIIO_F_*_RANK_TIME: fastest and slowest I/O time for a single rank (for shared files).\n");
printf("# MPIIO_F_VARIANCE_RANK_*: variance of total I/O time and bytes moved for all ranks (for shared files).\n");
- DARSHAN_PRINT_HEADER();
-
return;
}
=====================================
darshan-util/darshan-null-logutils.c
=====================================
--- a/darshan-util/darshan-null-logutils.c
+++ b/darshan-util/darshan-null-logutils.c
@@ -36,7 +36,7 @@ static int darshan_log_get_null_record(darshan_fd fd, void** null_buf_p);
static int darshan_log_put_null_record(darshan_fd fd, void* null_buf, int ver);
static void darshan_log_print_null_record(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type, int ver);
-static void darshan_log_print_null_description(void);
+static void darshan_log_print_null_description(int ver);
static void darshan_log_print_null_record_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_null_records(void *rec, void *agg_rec, int init_flag);
@@ -156,7 +156,7 @@ static void darshan_log_print_null_record(void *file_rec, char *file_name,
}
/* print out a description of the NULL module record fields */
-static void darshan_log_print_null_description()
+static void darshan_log_print_null_description(int ver)
{
printf("\n# description of NULL counters:\n");
printf("# NULL_FOOS: number of 'foo' function calls.\n");
=====================================
darshan-util/darshan-parser.c
=====================================
--- a/darshan-util/darshan-parser.c
+++ b/darshan-util/darshan-parser.c
@@ -104,20 +104,20 @@ typedef struct perf_data_s
void posix_accum_file(struct darshan_posix_file *pfile, hash_entry_t *hfile, int64_t nprocs);
void posix_accum_perf(struct darshan_posix_file *pfile, perf_data_t *pdata);
void posix_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
-void posix_print_total_file(struct darshan_posix_file *pfile);
+void posix_print_total_file(struct darshan_posix_file *pfile, int posix_ver);
void posix_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
void mpiio_accum_file(struct darshan_mpiio_file *mfile, hash_entry_t *hfile, int64_t nprocs);
void mpiio_accum_perf(struct darshan_mpiio_file *mfile, perf_data_t *pdata);
void mpiio_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
-void mpiio_print_total_file(struct darshan_mpiio_file *mfile);
+void mpiio_print_total_file(struct darshan_mpiio_file *mfile, int mpiio_ver);
void mpiio_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
void stdio_accum_perf(struct darshan_stdio_file *pfile, perf_data_t *pdata);
void stdio_accum_file(struct darshan_stdio_file *pfile, hash_entry_t *hfile, int64_t nprocs);
void stdio_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
void stdio_file_list(hash_entry_t *file_hash, struct darshan_name_record_ref *name_hash, int detail_flag);
-void stdio_print_total_file(struct darshan_stdio_file *pfile);
+void stdio_print_total_file(struct darshan_stdio_file *pfile, int stdio_ver);
void calc_perf(perf_data_t *pdata, int64_t nprocs);
@@ -419,7 +419,10 @@ int main(int argc, char **argv)
{
/* print a header describing the module's I/O characterization data */
if(mod_logutils[i]->log_print_description)
- mod_logutils[i]->log_print_description();
+ {
+ mod_logutils[i]->log_print_description(fd->mod_ver[i]);
+ DARSHAN_PRINT_HEADER();
+ }
}
ret = mod_logutils[i]->log_get_record(fd, (void **)&mod_buf);
@@ -544,15 +547,15 @@ int main(int argc, char **argv)
{
if(i == DARSHAN_POSIX_MOD)
{
- posix_print_total_file((struct darshan_posix_file*)total.rec_dat);
+ posix_print_total_file((struct darshan_posix_file*)total.rec_dat, fd->mod_ver[i]);
}
else if(i == DARSHAN_MPIIO_MOD)
{
- mpiio_print_total_file((struct darshan_mpiio_file*)total.rec_dat);
+ mpiio_print_total_file((struct darshan_mpiio_file*)total.rec_dat, fd->mod_ver[i]);
}
else if(i == DARSHAN_STDIO_MOD)
{
- stdio_print_total_file((struct darshan_stdio_file*)total.rec_dat);
+ stdio_print_total_file((struct darshan_stdio_file*)total.rec_dat, fd->mod_ver[i]);
}
}
@@ -572,8 +575,18 @@ int main(int argc, char **argv)
stdio_calc_file(file_hash, &fdata);
}
- printf("\n# files\n");
+ printf("\n# Total file counts\n");
printf("# -----\n");
+ printf("# <file_type>: type of file access:\n");
+ printf("# *read_only: file was only read\n");
+ printf("# *write_only: file was only written\n");
+ printf("# *read_write: file was read and written\n");
+ printf("# *unique: file was opened by a single process only\n");
+ printf("# *shared: file was accessed by a group of processes (maybe all processes)\n");
+ printf("# <file_count> total number of files of this type\n");
+ printf("# <total_bytes> total number of bytes moved to/from files of this type\n");
+ printf("# <max_byte_offset> maximum byte offset accessed for a file of this type\n");
+ printf("\n# <file_type> <file_count> <total_bytes> <max_byte_offset>\n");
printf("# total: %" PRId64 " %" PRId64 " %" PRId64 "\n",
fdata.total,
fdata.total_size,
@@ -1681,9 +1694,11 @@ void calc_perf(perf_data_t *pdata,
return;
}
-void stdio_print_total_file(struct darshan_stdio_file *pfile)
+void stdio_print_total_file(struct darshan_stdio_file *pfile, int stdio_ver)
{
int i;
+
+ mod_logutils[DARSHAN_STDIO_MOD]->log_print_description(stdio_ver);
printf("\n");
for(i = 0; i < STDIO_NUM_INDICES; i++)
{
@@ -1698,9 +1713,11 @@ void stdio_print_total_file(struct darshan_stdio_file *pfile)
return;
}
-void posix_print_total_file(struct darshan_posix_file *pfile)
+void posix_print_total_file(struct darshan_posix_file *pfile, int posix_ver)
{
int i;
+
+ mod_logutils[DARSHAN_POSIX_MOD]->log_print_description(posix_ver);
printf("\n");
for(i = 0; i < POSIX_NUM_INDICES; i++)
{
@@ -1715,9 +1732,11 @@ void posix_print_total_file(struct darshan_posix_file *pfile)
return;
}
-void mpiio_print_total_file(struct darshan_mpiio_file *mfile)
+void mpiio_print_total_file(struct darshan_mpiio_file *mfile, int mpiio_ver)
{
int i;
+
+ mod_logutils[DARSHAN_MPIIO_MOD]->log_print_description(mpiio_ver);
printf("\n");
for(i = 0; i < MPIIO_NUM_INDICES; i++)
{
@@ -1765,6 +1784,7 @@ void stdio_file_list(hash_entry_t *file_hash,
printf("\n# Per-file summary of I/O activity (detailed).\n");
else
printf("\n# Per-file summary of I/O activity.\n");
+ printf("# -----\n");
printf("# <record_id>: darshan record id for this file\n");
printf("# <file_name>: full file name\n");
@@ -1851,6 +1871,7 @@ void posix_file_list(hash_entry_t *file_hash,
printf("\n# Per-file summary of I/O activity (detailed).\n");
else
printf("\n# Per-file summary of I/O activity.\n");
+ printf("# -----\n");
printf("# <record_id>: darshan record id for this file\n");
printf("# <file_name>: full file name\n");
@@ -1859,8 +1880,8 @@ void posix_file_list(hash_entry_t *file_hash,
printf("# <avg>: average time in seconds consumed in IO per process\n");
if(detail_flag)
{
- printf("# <start_{open/read/write}>: start timestamp of first open, read, or write\n");
- printf("# <end_{read/write/close}>: end timestamp of last read, write, or close\n");
+ printf("# <start_{open/read/write/close}>: start timestamp of first open, read, write, or close\n");
+ printf("# <end_{open/read/write/close}>: end timestamp of last open, read, write, or close\n");
printf("# <posix_opens>: POSIX open calls\n");
printf("# <POSIX_SIZE_READ_*>: POSIX read size histogram\n");
printf("# <POSIX_SIZE_WRITE_*>: POSIX write size histogram\n");
@@ -1943,6 +1964,7 @@ void mpiio_file_list(hash_entry_t *file_hash,
printf("\n# Per-file summary of I/O activity (detailed).\n");
else
printf("\n# Per-file summary of I/O activity.\n");
+ printf("# -----\n");
printf("# <record_id>: darshan record id for this file\n");
printf("# <file_name>: full file name\n");
=====================================
darshan-util/darshan-pnetcdf-logutils.c
=====================================
--- a/darshan-util/darshan-pnetcdf-logutils.c
+++ b/darshan-util/darshan-pnetcdf-logutils.c
@@ -34,7 +34,7 @@ static int darshan_log_get_pnetcdf_file(darshan_fd fd, void** pnetcdf_buf_p);
static int darshan_log_put_pnetcdf_file(darshan_fd fd, void* pnetcdf_buf, int ver);
static void darshan_log_print_pnetcdf_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type, int ver);
-static void darshan_log_print_pnetcdf_description(void);
+static void darshan_log_print_pnetcdf_description(int ver);
static void darshan_log_print_pnetcdf_file_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_pnetcdf_files(void *rec, void *agg_rec, int init_flag);
@@ -136,7 +136,7 @@ static void darshan_log_print_pnetcdf_file(void *file_rec, char *file_name,
return;
}
-static void darshan_log_print_pnetcdf_description()
+static void darshan_log_print_pnetcdf_description(int ver)
{
printf("\n# description of PNETCDF counters:\n");
printf("# PNETCDF_INDEP_OPENS: PNETCDF independent file open operation counts.\n");
@@ -144,8 +144,6 @@ static void darshan_log_print_pnetcdf_description()
printf("# PNETCDF_F_OPEN_TIMESTAMP: timestamp of first PNETCDF file open.\n");
printf("# PNETCDF_F_CLOSE_TIMESTAMP: timestamp of last PNETCDF file close.\n");
- DARSHAN_PRINT_HEADER();
-
return;
}
=====================================
darshan-util/darshan-posix-logutils.c
=====================================
--- a/darshan-util/darshan-posix-logutils.c
+++ b/darshan-util/darshan-posix-logutils.c
@@ -37,7 +37,7 @@ static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p);
static int darshan_log_put_posix_file(darshan_fd fd, void* posix_buf, int ver);
static void darshan_log_print_posix_file(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type, int ver);
-static void darshan_log_print_posix_description(void);
+static void darshan_log_print_posix_description(int ver);
static void darshan_log_print_posix_file_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_posix_files(void *rec, void *agg_rec, int init_flag);
@@ -56,7 +56,6 @@ static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p)
{
struct darshan_posix_file *file = *((struct darshan_posix_file **)posix_buf_p);
int rec_len;
- char *buffer, *p;
int i;
int ret = -1;
@@ -70,75 +69,59 @@ static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p)
return(-1);
}
- /* read the POSIX record from file, checking the version first so we
- * can correctly up-convert to the current darshan version
- */
- if(fd->mod_ver[DARSHAN_POSIX_MOD] == 1)
+ if(fd->mod_ver[DARSHAN_POSIX_MOD] == DARSHAN_POSIX_VER)
{
- buffer = malloc(DARSHAN_POSIX_FILE_SIZE_1);
- if(!buffer)
- {
- if(*posix_buf_p == NULL)
- free(file);
- return(-1);
- }
-
- rec_len = DARSHAN_POSIX_FILE_SIZE_1;
- ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, buffer, rec_len);
- if(ret == rec_len)
- {
- /* copy record data directly from the temporary buffer into the
- * corresponding locations in the output file record
- */
- p = buffer;
- memcpy(&(file->base_rec), p, sizeof(struct darshan_base_record));
- p += sizeof(struct darshan_base_record);
- memcpy(&(file->counters[0]), p, 6 * sizeof(int64_t));
- p += (6 * sizeof(int64_t));
- p += (4 * sizeof(int64_t)); /* skip old stdio counters */
- memcpy(&(file->counters[6]), p, 58 * sizeof(int64_t));
- p += (58 * sizeof(int64_t));
- memcpy(&(file->fcounters[0]), p, 3 * sizeof(double));
- file->fcounters[POSIX_F_CLOSE_START_TIMESTAMP] = -1;
- file->fcounters[POSIX_F_OPEN_END_TIMESTAMP] = -1;
- p += 3 * sizeof(double);
- memcpy(&(file->fcounters[POSIX_F_READ_END_TIMESTAMP]), p, 12 * sizeof(double));
- }
- free(buffer);
+ /* log format is in current version, so we don't need to do any
+ * translation of counters while reading
+ */
+ rec_len = sizeof(struct darshan_posix_file);
+ ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, file, rec_len);
}
- else if(fd->mod_ver[DARSHAN_POSIX_MOD] == 2)
+ else
{
- buffer = malloc(DARSHAN_POSIX_FILE_SIZE_2);
- if(!buffer)
+ char scratch[1024] = {0};
+ char *src_p, *dest_p;
+ int len;
+
+ if(fd->mod_ver[DARSHAN_POSIX_MOD] == 1)
{
- if(*posix_buf_p == NULL)
- free(file);
- return(-1);
+ rec_len = DARSHAN_POSIX_FILE_SIZE_1;
+ ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, scratch, rec_len);
+ if(ret != rec_len)
+ goto exit;
+
+ /* upconvert version 1 to version 2 in-place */
+ dest_p = scratch + (sizeof(struct darshan_base_record) +
+ (6 * sizeof(int64_t)));
+ src_p = dest_p + (4 * sizeof(int64_t));
+ len = rec_len - (src_p - scratch);
+ memmove(dest_p, src_p, len);
}
-
- rec_len = DARSHAN_POSIX_FILE_SIZE_2;
- ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, buffer, rec_len);
- if(ret == rec_len)
+ if(fd->mod_ver[DARSHAN_POSIX_MOD] <= 2)
{
- p = buffer;
- memcpy(&(file->base_rec), p, sizeof(struct darshan_base_record));
- p += sizeof(struct darshan_base_record);
- memcpy(&(file->counters[0]), p, POSIX_NUM_INDICES * sizeof(int64_t));
- p += POSIX_NUM_INDICES * sizeof(int64_t);
- memcpy(&(file->fcounters[0]), p, 3 * sizeof(double));
- file->fcounters[POSIX_F_CLOSE_START_TIMESTAMP] = -1;
- file->fcounters[POSIX_F_OPEN_END_TIMESTAMP] = -1;
- p += 3 * sizeof(double);
- memcpy(&(file->fcounters[POSIX_F_READ_END_TIMESTAMP]), p, 12 * sizeof(double));
+ if(fd->mod_ver[DARSHAN_POSIX_MOD] == 2)
+ {
+ rec_len = DARSHAN_POSIX_FILE_SIZE_2;
+ ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, scratch, rec_len);
+ if(ret != rec_len)
+ goto exit;
+ }
+
+ /* upconvert version 2 to version 3 in-place */
+ dest_p = scratch + (sizeof(struct darshan_base_record) +
+ (64 * sizeof(int64_t)) + (5 * sizeof(double)));
+ src_p = dest_p - (2 * sizeof(double));
+ len = (12 * sizeof(double));
+ memmove(dest_p, src_p, len);
+ /* set F_CLOSE_START and F_OPEN_END to -1 */
+ *((double *)src_p) = -1;
+ *((double *)(src_p + sizeof(double))) = -1;
}
- free(buffer);
- }
- else if(fd->mod_ver[DARSHAN_POSIX_MOD] == 3)
- {
- rec_len = sizeof(struct darshan_posix_file);
- ret = darshan_log_get_mod(fd, DARSHAN_POSIX_MOD, file, rec_len);
+
+ memcpy(file, scratch, sizeof(struct darshan_posix_file));
}
+exit:
if(*posix_buf_p == NULL)
{
if(ret == rec_len)
@@ -161,7 +144,16 @@ static int darshan_log_get_posix_file(darshan_fd fd, void** posix_buf_p)
for(i=0; i<POSIX_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<POSIX_F_NUM_INDICES; i++)
+ {
+ /* skip counters we explicitly set to -1 since they don't
+ * need to be byte swapped
+ */
+ if((fd->mod_ver[DARSHAN_POSIX_MOD] < 3) &&
+ ((i == POSIX_F_CLOSE_START_TIMESTAMP) ||
+ (i == POSIX_F_OPEN_END_TIMESTAMP)))
+ continue;
DARSHAN_BSWAP64(&file->fcounters[i]);
+ }
}
return(1);
@@ -207,7 +199,7 @@ static void darshan_log_print_posix_file(void *file_rec, char *file_name,
return;
}
-static void darshan_log_print_posix_description()
+static void darshan_log_print_posix_description(int ver)
{
printf("\n# description of POSIX counters:\n");
printf("# POSIX_*: posix operation counts.\n");
@@ -235,7 +227,19 @@ static void darshan_log_print_posix_description()
printf("# POSIX_F_*_RANK_TIME: fastest and slowest I/O time for a single rank (for shared files).\n");
printf("# POSIX_F_VARIANCE_RANK_*: variance of total I/O time and bytes moved for all ranks (for shared files).\n");
- DARSHAN_PRINT_HEADER();
+ if(ver <= 1)
+ {
+ printf("\n# WARNING: POSIX module log format version 1 has the following limitations:\n");
+ printf("# - Partial instrumentation of stdio stream I/O functions not parsable by Darshan versions >= 3.1.0\n");
+ printf("# * Using darshan-logutils versions < 3.1.0, this data can be found in the following POSIX counters:\n");
+ printf("# * POSIX_FOPENS, POSIX_FREADS, POSIX_FWRITES, POSIX_FSEEKS\n");
+ }
+ if(ver <= 2)
+ {
+ printf("\n# WARNING: POSIX module log format version <=2 does not support the following counters:\n");
+ printf("# - POSIX_F_CLOSE_START_TIMESTAMP\n");
+ printf("# - POSIX_F_OPEN_END_TIMESTAMP\n");
+ }
return;
}
=====================================
darshan-util/darshan-stdio-logutils.c
=====================================
--- a/darshan-util/darshan-stdio-logutils.c
+++ b/darshan-util/darshan-stdio-logutils.c
@@ -36,7 +36,7 @@ static int darshan_log_get_stdio_record(darshan_fd fd, void** stdio_buf_p);
static int darshan_log_put_stdio_record(darshan_fd fd, void* stdio_buf, int ver);
static void darshan_log_print_stdio_record(void *file_rec,
char *file_name, char *mnt_pt, char *fs_type, int ver);
-static void darshan_log_print_stdio_description(void);
+static void darshan_log_print_stdio_description(int ver);
static void darshan_log_print_stdio_record_diff(void *file_rec1, char *file_name1,
void *file_rec2, char *file_name2);
static void darshan_log_agg_stdio_records(void *rec, void *agg_rec, int init_flag);
@@ -154,7 +154,7 @@ static void darshan_log_print_stdio_record(void *file_rec, char *file_name,
}
/* print out a description of the STDIO module record fields */
-static void darshan_log_print_stdio_description()
+static void darshan_log_print_stdio_description(int ver)
{
printf("\n# description of STDIO counters:\n");
printf("# STDIO_{OPENS|WRITES|READS|SEEKS|FLUSHES} are types of operations.\n");
@@ -168,8 +168,6 @@ static void darshan_log_print_stdio_description()
printf("# STDIO_F_*_RANK_TIME: fastest and slowest I/O time for a single rank (for shared files).\n");
printf("# STDIO_F_VARIANCE_RANK_*: variance of total I/O time and bytes moved for all ranks (for shared files).\n");
- DARSHAN_PRINT_HEADER();
-
return;
}
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/5dbe7dc16144f479bc284da92c2cbaf2a2d61229...84b8bb1cc54ec5bf5e8785244a0929a144dc4e7a
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20160923/edd97f1e/attachment-0001.html>
More information about the Darshan-commits
mailing list