[Darshan-commits] [Darshan] branch, dev-modular, updated. darshan-2.3.1-49-g694c6a9
Service Account
git at mcs.anl.gov
Tue Mar 17 15:09:30 CDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "".
The branch, dev-modular has been updated
via 694c6a9ae8bf695a9a84e1e2c9f35535d1b6c9a7 (commit)
from 35dcc69a6d66df480ad9b7ce81688f5b4297ea9f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit 694c6a9ae8bf695a9a84e1e2c9f35535d1b6c9a7
Author: Shane Snyder <ssnyder at mcs.anl.gov>
Date: Tue Mar 17 15:08:23 2015 -0500
bunch of bug fixes for gzip madness
-----------------------------------------------------------------------
Summary of changes:
darshan-log-format.h | 7 -
darshan-runtime/darshan-core.h | 1 +
darshan-runtime/lib/darshan-core.c | 225 ++++++++++---------
darshan-util/Makefile.in | 58 +++---
darshan-util/darshan-logutils.c | 406 +++++++++--------------------------
darshan-util/darshan-logutils.h | 24 ++-
darshan-util/darshan-posix-parser.c | 89 +++-----
doc/darshan-modularization.txt | 3 +-
8 files changed, 314 insertions(+), 499 deletions(-)
Diff of changes:
diff --git a/darshan-log-format.h b/darshan-log-format.h
index 382ab00..dfc98ca 100644
--- a/darshan-log-format.h
+++ b/darshan-log-format.h
@@ -53,12 +53,6 @@ static char *darshan_module_names[] =
"PNETCDF"
};
-enum darshan_comp_type
-{
- DARSHAN_GZ_COMP,
- DARSHAN_BZ2_COMP, /* TODO: no bz2 support util side, yet */
-};
-
struct darshan_log_map
{
uint64_t off;
@@ -69,7 +63,6 @@ struct darshan_header
{
char version_string[8];
int64_t magic_nr;
- unsigned char comp_type;
struct darshan_log_map rec_map;
struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
};
diff --git a/darshan-runtime/darshan-core.h b/darshan-runtime/darshan-core.h
index 5667e9b..6b69b4c 100644
--- a/darshan-runtime/darshan-core.h
+++ b/darshan-runtime/darshan-core.h
@@ -30,6 +30,7 @@
/* in memory structure to keep up with job level data */
struct darshan_core_runtime
{
+ struct darshan_header log_header;
struct darshan_job log_job;
char exe[DARSHAN_EXE_LEN+1];
struct darshan_core_record_ref *rec_hash;
diff --git a/darshan-runtime/lib/darshan-core.c b/darshan-runtime/lib/darshan-core.c
index 4e3eff2..95d649f 100644
--- a/darshan-runtime/lib/darshan-core.c
+++ b/darshan-runtime/lib/darshan-core.c
@@ -70,15 +70,17 @@ static char* darshan_get_exe_and_mounts(
struct darshan_core_runtime *core);
static void darshan_get_shared_records(
struct darshan_core_runtime *core, darshan_record_id *shared_recs);
-static int darshan_log_coll_open(
+static int darshan_log_open_all(
char *logfile_name, MPI_File *log_fh);
-static int darshan_compress_buffer(void **pointers, int *lengths,
- int count, char *comp_buf, int *comp_length);
+static int darshan_deflate_buffer(
+ void **pointers, int *lengths, int count, int nocomp_flag,
+ char *comp_buf, int *comp_length);
static int darshan_log_write_record_hash(
MPI_File log_fh, struct darshan_core_runtime *core,
- struct darshan_log_map *map);
-static int darshan_log_coll_write(
- MPI_File log_fh, void *buf, int count, struct darshan_log_map *map);
+ uint64_t *inout_off);
+static int darshan_log_append_all(
+ MPI_File log_fh, struct darshan_core_runtime *core, void *buf,
+ int count, uint64_t *inout_off, uint64_t *agg_uncomp_sz);
/* intercept MPI initialize and finalize to manage darshan core runtime */
int MPI_Init(int *argc, char ***argv)
@@ -235,9 +237,9 @@ static void darshan_core_shutdown()
double header1, header2;
double tm_end;
long offset;
- struct darshan_header log_header;
+ uint64_t gz_fp = 0;
+ uint64_t tmp_off = 0;
MPI_File log_fh;
- MPI_Offset tmp_off = 0;
MPI_Status status;
if(getenv("DARSHAN_INTERNAL_TIMING"))
@@ -347,7 +349,7 @@ static void darshan_core_shutdown()
if(internal_timing_flag)
open1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* collectively open the darshan log file */
- ret = darshan_log_coll_open(logfile_name, &log_fh);
+ ret = darshan_log_open_all(logfile_name, &log_fh);
if(internal_timing_flag)
open2 = DARSHAN_MPI_CALL(PMPI_Wtime)();
@@ -377,7 +379,7 @@ static void darshan_core_shutdown()
int comp_buf_sz = 0;
/* compress the job info and the trailing mount/exe data */
- all_ret = darshan_compress_buffer(pointers, lengths, 2,
+ all_ret = darshan_deflate_buffer(pointers, lengths, 2, 0,
final_core->comp_buf, &comp_buf_sz);
if(all_ret)
{
@@ -387,9 +389,9 @@ static void darshan_core_shutdown()
else
{
/* write the job information, preallocing space for the log header */
- all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(
- log_fh, sizeof(struct darshan_header), final_core->comp_buf,
- comp_buf_sz, MPI_BYTE, &status);
+ gz_fp += sizeof(struct darshan_header) + 23; /* gzip headers/trailers ... */
+ all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, gz_fp,
+ final_core->comp_buf, comp_buf_sz, MPI_BYTE, &status);
if(all_ret != MPI_SUCCESS)
{
fprintf(stderr, "darshan library warning: unable to write job data to log file %s\n",
@@ -398,8 +400,8 @@ static void darshan_core_shutdown()
}
- /* set the beginning offset of record hash, which precedes job info just written */
- log_header.rec_map.off = sizeof(struct darshan_header) + comp_buf_sz;
+ /* set the beginning offset of record hash, which follows job info just written */
+ gz_fp += comp_buf_sz;
}
}
@@ -417,7 +419,8 @@ static void darshan_core_shutdown()
if(internal_timing_flag)
rec1 = DARSHAN_MPI_CALL(PMPI_Wtime)();
/* write the record name->id hash to the log file */
- ret = darshan_log_write_record_hash(log_fh, final_core, &log_header.rec_map);
+ ret = darshan_log_write_record_hash(log_fh, final_core, &gz_fp);
+ tmp_off = final_core->log_header.rec_map.off + final_core->log_header.rec_map.len;
/* error out if unable to write record hash */
DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
@@ -459,8 +462,10 @@ static void darshan_core_shutdown()
if(global_mod_use_count[i] == 0)
{
if(my_rank == 0)
- log_header.mod_map[i].off = log_header.mod_map[i].len = 0;
-
+ {
+ final_core->log_header.mod_map[i].off = 0;
+ final_core->log_header.mod_map[i].len = 0;
+ }
continue;
}
@@ -523,30 +528,14 @@ static void darshan_core_shutdown()
this_mod->mod_funcs.get_output_data(&mod_buf, &mod_buf_sz);
}
- /* compress the module buffer */
- ret = darshan_compress_buffer((void**)&mod_buf, &mod_buf_sz, 1,
- final_core->comp_buf, &comp_buf_sz);
- if(ret == 0)
- {
- /* set the starting offset of this module */
- if(tmp_off == 0)
- tmp_off = log_header.rec_map.off + log_header.rec_map.len;
+ final_core->log_header.mod_map[i].off = tmp_off;
- log_header.mod_map[i].off = tmp_off;
+ /* append this module's data to the darshan log */
+ ret = darshan_log_append_all(log_fh, final_core, mod_buf, mod_buf_sz,
+ &gz_fp, &(final_core->log_header.mod_map[i].len));
+ tmp_off += final_core->log_header.mod_map[i].len;
- /* write (compressed) module data buffer to the darshan log file */
- ret = darshan_log_coll_write(log_fh, final_core->comp_buf, comp_buf_sz,
- &log_header.mod_map[i]);
- }
- else
- {
- /* error in compression, participate in collective write to avoid
- * deadlock, but preserve return value to terminate darshan_shutdown.
- */
- (void)darshan_log_coll_write(log_fh, NULL, 0, &log_header.mod_map[i]);
- }
-
- /* error out if the compression or collective write failed */
+ /* error out if the log append failed */
DARSHAN_MPI_CALL(PMPI_Allreduce)(&ret, &all_ret, 1, MPI_INT,
MPI_LOR, MPI_COMM_WORLD);
if(all_ret != 0)
@@ -563,8 +552,6 @@ static void darshan_core_shutdown()
return;
}
- tmp_off += log_header.mod_map[i].len;
-
/* shutdown module if registered locally */
if(this_mod)
{
@@ -579,19 +566,39 @@ static void darshan_core_shutdown()
/* rank 0 is responsible for writing the log header */
if(my_rank == 0)
{
- /* initialize the remaining header fields */
- strcpy(log_header.version_string, DARSHAN_LOG_VERSION);
- log_header.magic_nr = DARSHAN_MAGIC_NR;
- log_header.comp_type = DARSHAN_GZ_COMP;
+ void *header_buf = &(final_core->log_header);
+ int header_buf_sz = sizeof(struct darshan_header);
+ int comp_buf_sz = 0;
- all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, 0, &log_header,
- sizeof(struct darshan_header), MPI_BYTE, &status);
- if(all_ret != MPI_SUCCESS)
+ /* initialize the remaining header fields */
+ strcpy(final_core->log_header.version_string, DARSHAN_LOG_VERSION);
+ final_core->log_header.magic_nr = DARSHAN_MAGIC_NR;
+
+ /* deflate the header */
+ /* NOTE: the header is not actually compressed because space for it must
+ * be preallocated before writing. i.e., the "compressed" header
+ * must be constant sized, sizeof(struct darshan_header) + 23.
+ * it is still necessary to deflate the header or the resulting log
+ * file will not be a valid gzip file.
+ */
+ all_ret = darshan_deflate_buffer((void **)&header_buf, &header_buf_sz, 1, 1,
+ final_core->comp_buf, &comp_buf_sz);
+ if(all_ret)
{
- fprintf(stderr, "darshan library warning: unable to write header to log file %s\n",
- logfile_name);
+ fprintf(stderr, "darshan library warning: unable to compress header\n");
unlink(logfile_name);
}
+ else
+ {
+ all_ret = DARSHAN_MPI_CALL(PMPI_File_write_at)(log_fh, 0, final_core->comp_buf,
+ comp_buf_sz, MPI_BYTE, &status);
+ if(all_ret != MPI_SUCCESS)
+ {
+ fprintf(stderr, "darshan library warning: unable to write header to log file %s\n",
+ logfile_name);
+ unlink(logfile_name);
+ }
+ }
}
/* error out if unable to write log header */
@@ -1172,7 +1179,7 @@ static void darshan_get_shared_records(struct darshan_core_runtime *core,
return;
}
-static int darshan_log_coll_open(char *logfile_name, MPI_File *log_fh)
+static int darshan_log_open_all(char *logfile_name, MPI_File *log_fh)
{
char *hints;
char *tok_str;
@@ -1233,12 +1240,13 @@ static int darshan_log_coll_open(char *logfile_name, MPI_File *log_fh)
return(0);
}
-static int darshan_compress_buffer(void **pointers, int *lengths, int count,
- char *comp_buf, int *comp_length)
+static int darshan_deflate_buffer(void **pointers, int *lengths, int count,
+ int nocomp_flag, char *comp_buf, int *comp_length)
{
int ret = 0;
int i;
int total_target = 0;
+ int z_comp_level;
z_stream tmp_stream;
/* just return if there is no data */
@@ -1263,9 +1271,10 @@ static int darshan_compress_buffer(void **pointers, int *lengths, int count,
/* initialize the zlib compression parameters */
/* TODO: check these parameters? */
-// ret = deflateInit2(&tmp_stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED,
-// 31, 8, Z_DEFAULT_STRATEGY);
- ret = deflateInit(&tmp_stream, Z_DEFAULT_COMPRESSION);
+ z_comp_level = nocomp_flag ? Z_NO_COMPRESSION : Z_DEFAULT_COMPRESSION;
+ ret = deflateInit2(&tmp_stream, z_comp_level, Z_DEFLATED,
+ 15 + 16, 8, Z_DEFAULT_STRATEGY);
+// ret = deflateInit(&tmp_stream, z_comp_level);
if(ret != Z_OK)
{
return(-1);
@@ -1323,9 +1332,8 @@ static int darshan_compress_buffer(void **pointers, int *lengths, int count,
* record is opened by multiple ranks, but not all ranks
*/
static int darshan_log_write_record_hash(MPI_File log_fh, struct darshan_core_runtime *core,
- struct darshan_log_map *map)
+ uint64_t *inout_off)
{
- int i;
int ret;
struct darshan_core_record_ref *ref, *tmp;
uint32_t name_len;
@@ -1388,80 +1396,75 @@ static int darshan_log_write_record_hash(MPI_File log_fh, struct darshan_core_ru
hash_buf_off += name_len;
}
- /* collectively write out the record hash to the darshan log */
- if(hash_buf_off > hash_buf)
- {
- int hash_buf_sz = hash_buf_off - hash_buf;
- int comp_buf_sz = 0;
+ /* store uncompressed offset of the record hash */
+ core->log_header.rec_map.off = sizeof(struct darshan_header) +
+ DARSHAN_JOB_RECORD_SIZE;
- /* compress the record hash buffer */
- ret = darshan_compress_buffer((void **)&hash_buf, &hash_buf_sz, 1,
- core->comp_buf, &comp_buf_sz);
- if(ret < 0)
- {
- /* participate in collective write to avoid deadlock, but preserve
- * the compression error to pass back to the caller.
- */
- (void)darshan_log_coll_write(log_fh, NULL, 0, map);
- }
- else
- {
- /* we have records to contribute to the write of the record hash */
- ret = darshan_log_coll_write(log_fh, core->comp_buf, comp_buf_sz, map);
- }
- }
- else
- {
- /* we have no data to write, but participate in the collective anyway */
- ret = darshan_log_coll_write(log_fh, NULL, 0, map);
- }
+ /* collectively write out the record hash to the darshan log */
+ hash_buf_sz = hash_buf_off - hash_buf;
+ ret = darshan_log_append_all(log_fh, core, hash_buf, hash_buf_sz,
+ inout_off, &(core->log_header.rec_map.len));
free(hash_buf);
- if(ret < 0)
- return(-1);
-
- return(0);
+ return(ret);
}
-/* NOTE: The in/out param 'map' is only valid on rank 0 and is used
- * to provide the starting offset of this collective write and
- * to store the aggregate size of this write upon completion.
- * This implies ONLY rank 0 can specify the starting offset
- * and that only rank 0 knows the ending log file offset upon
- * return from this function (starting off + aggregate size).
+/* NOTE: inout_off contains the starting offset of this append at the beginning
+ * of the call, and contains the ending offset at the end of the call.
+ * total_uncomp_sz will store the collective size of the uncompressed
+ * buffer being written. This data is necessary to properly index data
+ * when reading a darshan log. Also, these variables should only be valid
+ * on the root rank (rank 0).
*/
-static int darshan_log_coll_write(MPI_File log_fh, void *buf, int count,
- struct darshan_log_map *map)
+static int darshan_log_append_all(MPI_File log_fh, struct darshan_core_runtime *core,
+ void *buf, int count, uint64_t *inout_off, uint64_t *agg_uncomp_sz)
{
MPI_Offset send_off, my_off;
MPI_Status status;
+ uint64_t uncomp_buf_sz = count;
+ int comp_buf_sz = 0;
int ret;
+ /* compress the input buffer */
+ ret = darshan_deflate_buffer((void **)&buf, &count, 1, 0,
+ core->comp_buf, &comp_buf_sz);
+ if(ret < 0)
+ comp_buf_sz = 0;
+
/* figure out where everyone is writing using scan */
- send_off = count;
+ send_off = comp_buf_sz;
if(my_rank == 0)
{
- send_off += map->off; /* rank 0 knows the beginning offset */
+ send_off += *inout_off; /* rank 0 knows the beginning offset */
}
DARSHAN_MPI_CALL(PMPI_Scan)(&send_off, &my_off, 1, MPI_OFFSET,
MPI_SUM, MPI_COMM_WORLD);
/* scan in inclusive; subtract local size back out */
- my_off -= count;
+ my_off -= comp_buf_sz;
- /* perform the collective write */
- ret = DARSHAN_MPI_CALL(PMPI_File_write_at_all)(log_fh, my_off, buf,
- count, MPI_BYTE, &status);
- if(ret != MPI_SUCCESS)
- return(-1);
+ if(ret == 0)
+ {
+ /* no compression errors, proceed with the collective write */
+ ret = DARSHAN_MPI_CALL(PMPI_File_write_at_all)(log_fh, my_off,
+ core->comp_buf, comp_buf_sz, MPI_BYTE, &status);
+ }
+ else
+ {
+ /* error during compression. preserve and return error to caller,
+ * but participate in collective write to avoid deadlock.
+ */
+ (void)DARSHAN_MPI_CALL(PMPI_File_write_at_all)(log_fh, my_off,
+ core->comp_buf, comp_buf_sz, MPI_BYTE, &status);
+ }
/* send the ending offset from rank (n-1) to rank 0 */
if(nprocs > 1)
{
if(my_rank == (nprocs-1))
{
- my_off += count;
+ my_off += comp_buf_sz;
DARSHAN_MPI_CALL(PMPI_Send)(&my_off, 1, MPI_OFFSET, 0, 0,
MPI_COMM_WORLD);
}
@@ -1470,14 +1473,20 @@ static int darshan_log_coll_write(MPI_File log_fh, void *buf, int count,
DARSHAN_MPI_CALL(PMPI_Recv)(&my_off, 1, MPI_OFFSET, (nprocs-1), 0,
MPI_COMM_WORLD, &status);
- map->len = my_off - map->off;
+ *inout_off = my_off;
}
}
else
{
- map->len = my_off + count - map->off;
+ *inout_off = my_off + comp_buf_sz;
}
+ /* pass back the aggregate uncompressed size of this blob */
+ DARSHAN_MPI_CALL(PMPI_Reduce)(&uncomp_buf_sz, agg_uncomp_sz, 1,
+ MPI_UINT64_T, MPI_SUM, 0, MPI_COMM_WORLD);
+
+ if(ret != 0)
+ return(-1);
return(0);
}
diff --git a/darshan-util/Makefile.in b/darshan-util/Makefile.in
index 73f6e55..ff6cf92 100644
--- a/darshan-util/Makefile.in
+++ b/darshan-util/Makefile.in
@@ -1,4 +1,4 @@
-all: darshan-posix-parser darshan-util-lib
+all: darshan-util-lib darshan-posix-parser
#all: darshan-parser darshan-convert darshan-diff darshan-analyzer darshan-log-params darshan-util-lib
DESTDIR =
@@ -42,8 +42,27 @@ mktestdir::
uthash-1.9.2:
tar xjvf $(srcdir)/extern/uthash-1.9.2.tar.bz2
-darshan-posix-parser: darshan-posix-parser.c $(DARSHAN_LOG_FORMAT) darshan-logutils.h darshan-logutils.o
- $(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS)
+darshan-logutils.o: darshan-logutils.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
+ $(CC) $(CFLAGS) -c $< -o $@
+darshan-logutils.po: darshan-logutils.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
+ $(CC) $(CFLAGS_SHARED) -c $< -o $@
+
+darshan-posix-logutils.o: darshan-posix-logutils.c darshan-logutils.h darshan-posix-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | uthash-1.9.2
+ $(CC) $(CFLAGS) -c $< -o $@
+darshan-posix-logutils.po: darshan-posix-logutils.c darshan-logutils.h darshan-posix-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | uthash-1.9.2
+ $(CC) $(CFLAGS_SHARED) -c $< -o $@
+
+libdarshan-util.so: darshan-logutils.po darshan-posix-logutils.po
+ $(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ $(LIBS)
+
+darshan-util-lib: darshan-logutils.o darshan-posix-logutils.o
+ ar rcs libdarshan-util.a $^
+
+jenkins: util/bin/jenkins.o lookup3.o
+ $(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ lookup3.o $(LIBS)
+
+darshan-posix-parser: darshan-posix-parser.c darshan-logutils.h darshan-posix-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h darshan-util-lib | uthash-1.9.2
+ $(CC) $(CFLAGS) $(LDFLAGS) darshan-posix-parser.c libdarshan-util.a -o $@ $(LIBS)
#darshan-parser: darshan-parser.c $(DARSHAN_LOG_FORMAT) darshan-logutils.h darshan-logutils.o | uthash-1.9.2
# $(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS)
@@ -57,23 +76,10 @@ darshan-posix-parser: darshan-posix-parser.c $(DARSHAN_LOG_FORMAT) darshan-logut
#darshan-log-params: darshan-log-params.c $(DARSHAN_LOG_FORMAT)
# $(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ $(LIBS)
-jenkins: util/bin/jenkins.o lookup3.o
- $(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ lookup3.o $(LIBS)
-
#darshan-diff: darshan-diff.o $(DARSHAN_LOG_FORMAT) darshan-logutils.o darshan-logutils.h
# $(CC) $(CFLAGS) $(LDFLAGS) $< darshan-logutils.o -o $@ $(LIBS)
#darshan-diff.o: darshan-diff.c
# $(CC) $(CFLAGS) -c $< -o $@
-darshan-logutils.o: darshan-logutils.c | uthash-1.9.2
- $(CC) $(CFLAGS) -c $< -o $@
-darshan-logutils.po: darshan-logutils.c
- $(CC) $(CFLAGS_SHARED) -c $< -o $@
-
-libdarshan-util.so: darshan-logutils.po
- $(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ $(LIBS)
-
-darshan-util-lib: darshan-logutils.o
- ar rcs libdarshan-util.a $<
#test/gztest: test/gztest.c mktestdir
# $(CC) $(CFLAGS) $(LDFLAGS) -lz $< -o $@
@@ -90,7 +96,7 @@ install:: all
install -d $(includedir)
install -d $(pkgconfigdir)
# install -m 755 darshan-parser $(bindir)
- install -m 755 darshan-posix-parser $(bindir)
+# install -m 755 darshan-posix-parser $(bindir)
# install -m 755 darshan-convert $(bindir)
# install -m 755 darshan-diff $(bindir)
# install -m 755 darshan-analyzer $(bindir)
@@ -99,21 +105,21 @@ install:: all
ifeq ($(DARSHAN_ENABLE_SHARED),1)
install -m 755 libdarshan-util.so $(libdir)
endif
- install -m 644 $(srcdir)/darshan-logutils.h $(includedir)
- install -m 644 $(DARSHAN_LOG_FORMAT) $(includedir)
+# install -m 644 $(srcdir)/darshan-logutils.h $(includedir)
+# install -m 644 $(DARSHAN_LOG_FORMAT) $(includedir)
# install -m 755 darshan-job-summary/bin/darshan-job-summary.pl $(bindir)
- install -d $(libdir)/TeX
- install -m 644 $(srcdir)/darshan-job-summary/lib/TeX/Encode.pm $(libdir)/TeX/
- install -d $(libdir)/Number
- install -d $(libdir)/Number/Bytes
- install -m 644 $(srcdir)/darshan-job-summary/lib/Number/Bytes/Human.pm $(libdir)/Number/Bytes
- install -d $(datarootdir)
+# install -d $(libdir)/TeX
+# install -m 644 $(srcdir)/darshan-job-summary/lib/TeX/Encode.pm $(libdir)/TeX/
+# install -d $(libdir)/Number
+# install -d $(libdir)/Number/Bytes
+# install -m 644 $(srcdir)/darshan-job-summary/lib/Number/Bytes/Human.pm $(libdir)/Number/Bytes
+# install -d $(datarootdir)
# install -m 644 $(srcdir)/darshan-job-summary/share/* $(datarootdir)
install -m 644 maint/darshan-util.pc $(pkgconfigdir)
clean::
- rm -f *.o *.a darshan-parser darshan-convert darshan-diff darshan-analyzer darshan-log-params
+ rm -f *.o *.a darshan-posix-parser
distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
diff --git a/darshan-util/darshan-logutils.c b/darshan-util/darshan-logutils.c
index 4118a9d..87250ab 100644
--- a/darshan-util/darshan-logutils.c
+++ b/darshan-util/darshan-logutils.c
@@ -16,28 +16,8 @@
#include <fcntl.h>
#include <errno.h>
-#include <zlib.h>
-#ifdef HAVE_LIBBZ2
-#include <bzlib.h>
-#endif
-
#include "darshan-logutils.h"
-/* default to a compression buffer size of 4 MiB */
-#define DARSHAN_DEF_DECOMP_BUF_SZ (4*1024*1024)
-
-struct darshan_fd_s
-{
- int pf;
- int64_t pos;
- char version[8];
- int swap_flag;
- char *exe_mnt_data;
- struct darshan_log_map job_map;
- struct darshan_log_map rec_map;
- struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
-};
-
static int darshan_log_seek(darshan_fd fd, off_t offset);
static int darshan_log_read(darshan_fd fd, void *buf, int len);
static int darshan_log_write(darshan_fd fd, void *buf, int len);
@@ -52,26 +32,32 @@ static int darshan_decompress_buffer(char *comp_buf, int comp_buf_sz,
*/
darshan_fd darshan_log_open(const char *name, const char *mode)
{
- int o_flags;
+ darshan_fd tmp_fd;
/* we only allows "w" or "r" modes, nothing fancy */
assert(strlen(mode) == 1);
assert(mode[0] == 'r' || mode[0] == 'w');
- if(mode[0] == 'r')
- o_flags = O_RDONLY;
- else
- o_flags = O_WRONLY;
- darshan_fd tmp_fd = malloc(sizeof(*tmp_fd));
+ tmp_fd = malloc(sizeof(*tmp_fd));
if(!tmp_fd)
return(NULL);
memset(tmp_fd, 0, sizeof(*tmp_fd));
- tmp_fd->pf = open(name, o_flags);
- if(tmp_fd->pf < 0)
+ if(strcmp(mode, "r") == 0)
+ {
+ /* TODO: check for bz2 */
+ }
+
+ if(strcmp(mode, "w") == 0)
+ {
+ /* TODO: check for bz2 */
+ }
+
+ tmp_fd->gzf = gzopen(name, mode);
+ if(!tmp_fd->gzf)
{
free(tmp_fd);
- return(NULL);
+ tmp_fd = NULL;
}
return(tmp_fd);
@@ -93,7 +79,7 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
if(ret < 0)
{
fprintf(stderr, "Error: unable to seek in darshan log file.\n");
- return(ret);
+ return(-1);
}
/* read header from log file */
@@ -123,7 +109,7 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
/* swap the log map variables in the header */
DARSHAN_BSWAP64(&header->rec_map.off);
DARSHAN_BSWAP64(&header->rec_map.len);
- for(i=0;i<DARSHAN_MAX_MODS;i++)
+ for(i = 0; i < DARSHAN_MAX_MODS; i++)
{
DARSHAN_BSWAP64(&header->mod_map[i].off);
DARSHAN_BSWAP64(&header->mod_map[i].len);
@@ -138,8 +124,6 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
}
/* save the mapping of data within log file to this file descriptor */
- fd->job_map.off = sizeof(struct darshan_header);
- fd->job_map.len = header->rec_map.off - fd->job_map.off;
memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
@@ -154,47 +138,23 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
*/
int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
{
- char *comp_buf;
- char job_buf[DARSHAN_JOB_RECORD_SIZE] = {0};
- int job_buf_sz = DARSHAN_JOB_RECORD_SIZE;
int ret;
- /* allocate a buffer to store the (compressed) darshan job info */
- comp_buf = malloc(fd->job_map.len);
- if(!comp_buf)
- return(-1);
-
- ret = darshan_log_seek(fd, fd->job_map.off);
+ ret = darshan_log_seek(fd, sizeof(struct darshan_header));
if(ret < 0)
{
fprintf(stderr, "Error: unable to seek in darshan log file.\n");
- free(comp_buf);
- return(ret);
+ return(-1);
}
/* read the job data from the log file */
- ret = darshan_log_read(fd, comp_buf, fd->job_map.len);
- if(ret < fd->job_map.len)
+ ret = darshan_log_read(fd, job, sizeof(*job));
+ if(ret < sizeof(*job))
{
fprintf(stderr, "Error: invalid darshan log file (failed to read job data).\n");
- free(comp_buf);
return(-1);
}
- /* decompress the job data */
- ret = darshan_decompress_buffer(comp_buf, fd->job_map.len,
- job_buf, &job_buf_sz);
- if(ret < 0)
- {
- fprintf(stderr, "Error: unable to decompress darshan job data.\n");
- free(comp_buf);
- return(-1);
- }
- assert(job_buf_sz == DARSHAN_JOB_RECORD_SIZE);
- free(comp_buf);
-
- memcpy(job, job_buf, sizeof(*job));
-
if(fd->swap_flag)
{
/* swap bytes if necessary */
@@ -205,37 +165,34 @@ int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
DARSHAN_BSWAP64(&job->jobid);
}
- /* save trailing job data, so exe and mount information can be retrieved later */
- if(!fd->exe_mnt_data)
- fd->exe_mnt_data = malloc(DARSHAN_EXE_LEN+1);
- if(!fd->exe_mnt_data)
- return(-1);
- memcpy(fd->exe_mnt_data, &job_buf[sizeof(*job)], DARSHAN_EXE_LEN+1);
-
return(0);
}
int darshan_log_getexe(darshan_fd fd, char *buf)
{
+ int tmp_off = sizeof(struct darshan_header) + sizeof(struct darshan_job);
+ int ret;
char *newline;
- /* if the exe/mount info has not been saved yet, read in the job
- * header to get this data.
- */
- if(!fd->exe_mnt_data)
+ ret = darshan_log_seek(fd, tmp_off);
+ if(ret < 0)
{
- struct darshan_job job;
- (void)darshan_log_getjob(fd, &job);
-
- if(!fd->exe_mnt_data)
- return(-1);
+ fprintf(stderr, "Error: unable to seek in darshan log file.\n");
+ return(-1);
}
- newline = strchr(fd->exe_mnt_data, '\n');
+ /* read the trailing exe data from the darshan log */
+ ret = darshan_log_read(fd, buf, DARSHAN_EXE_LEN+1);
+ if(ret < DARSHAN_EXE_LEN+1)
+ {
+ fprintf(stderr, "Error: invalid darshan log file (failed to read exe string).\n");
+ return(-1);
+ }
- /* copy over the exe string */
+ /* mount info is stored after the exe string, so truncate there */
+ newline = strchr(buf, '\n');
if(newline)
- memcpy(buf, fd->exe_mnt_data, (newline - fd->exe_mnt_data));
+ *newline = '\0';
return (0);
}
@@ -249,24 +206,30 @@ int darshan_log_getexe(darshan_fd fd, char *buf)
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
char*** fs_types, int* count)
{
+ int tmp_off = sizeof(struct darshan_header) + sizeof(struct darshan_job);
int ret;
char *pos;
int array_index = 0;
+ char buf[DARSHAN_EXE_LEN+1];
- /* if the exe/mount info has not been saved yet, read in the job
- * header to get this data.
- */
- if(!fd->exe_mnt_data)
+ ret = darshan_log_seek(fd, tmp_off);
+ if(ret < 0)
{
- struct darshan_job job;
- (void)darshan_log_getjob(fd, &job);
+ fprintf(stderr, "Error: unable to seek in darshan log file.\n");
+ return(-1);
+ }
- if(!fd->exe_mnt_data)
- return(-1);
+ /* read the trailing mount data from the darshan log */
+ ret = darshan_log_read(fd, buf, DARSHAN_EXE_LEN+1);
+ if(ret < DARSHAN_EXE_LEN+1)
+ {
+ fprintf(stderr, "Error: invalid darshan log file (failed to read mount info).\n");
+ return(-1);
}
+ /* count entries */
*count = 0;
- pos = fd->exe_mnt_data;
+ pos = buf;
while((pos = strchr(pos, '\n')) != NULL)
{
pos++;
@@ -288,7 +251,7 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
/* work backwards through the table and parse each line (except for
* first, which holds command line information)
*/
- while((pos = strrchr(fd->exe_mnt_data, '\n')) != NULL)
+ while((pos = strrchr(buf, '\n')) != NULL)
{
/* overestimate string lengths */
(*mnt_pts)[array_index] = malloc(DARSHAN_EXE_LEN);
@@ -319,9 +282,8 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
*/
int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
{
- char *comp_buf;
- char hash_buf[DARSHAN_DEF_DECOMP_BUF_SZ] = {0};
- int hash_buf_sz = DARSHAN_DEF_DECOMP_BUF_SZ;
+ char *hash_buf;
+ int hash_buf_sz = fd->rec_map.len;
char *buf_ptr;
darshan_record_id *rec_id_ptr;
uint32_t *path_len_ptr;
@@ -329,38 +291,24 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
struct darshan_record_ref *ref;
int ret;
- /* allocate a buffer to store the (compressed, serialized) darshan record hash */
- comp_buf = malloc(fd->rec_map.len);
- if(!comp_buf)
+ hash_buf = malloc(hash_buf_sz);
+ if(!hash_buf)
return(-1);
ret = darshan_log_seek(fd, fd->rec_map.off);
if(ret < 0)
{
fprintf(stderr, "Error: unable to seek in darshan log file.\n");
- free(comp_buf);
- return(ret);
+ return(-1);
}
/* read the record hash from the log file */
- ret = darshan_log_read(fd, comp_buf, fd->rec_map.len);
+ ret = darshan_log_read(fd, hash_buf, fd->rec_map.len);
if(ret < fd->rec_map.len)
{
fprintf(stderr, "Error: invalid darshan log file (failed to read record hash).\n");
- free(comp_buf);
- return(-1);
- }
-
- /* decompress the record hash buffer */
- ret = darshan_decompress_buffer(comp_buf, fd->rec_map.len,
- hash_buf, &hash_buf_sz);
- if(ret < 0)
- {
- fprintf(stderr, "Error: unable to decompress darshan job data.\n");
- free(comp_buf);
return(-1);
}
- free(comp_buf);
buf_ptr = hash_buf;
while(buf_ptr < (hash_buf + hash_buf_sz))
@@ -411,139 +359,41 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
return(0);
}
-int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
- void **mod_buf, int *mod_buf_sz)
+int darshan_log_get_moddat(darshan_fd fd, darshan_module_id mod_id,
+ void *moddat_buf, int moddat_buf_sz)
{
- char *comp_buf;
- char *tmp_buf;
- int tmp_buf_sz;
+ int mod_buf_end = fd->mod_map[mod_id].off + fd->mod_map[mod_id].len;
int ret;
- *mod_buf = NULL;
- *mod_buf_sz = 0;
-
- if(mod_id < 0 || mod_id >= DARSHAN_MAX_MODS)
- {
- fprintf(stderr, "Error: invalid Darshan module id.\n");
- return(-1);
- }
-
- if(fd->mod_map[mod_id].len == 0)
- {
- /* this module has no data in the log */
- return(0);
- }
-
- comp_buf = malloc(fd->mod_map[mod_id].len);
- if(!comp_buf)
- return(-1);
-
- ret = darshan_log_seek(fd, fd->mod_map[mod_id].off);
- if(ret < 0)
- {
- fprintf(stderr, "Error: unable to seek in darshan log file.\n");
- free(comp_buf);
- return(ret);
- }
-
- /* read the given module's (compressed) data from the log file */
- ret = darshan_log_read(fd, comp_buf, fd->mod_map[mod_id].len);
- if(ret < fd->mod_map[mod_id].len)
- {
- fprintf(stderr, "Error: invalid darshan log file (failed to read module %s data).\n",
- darshan_module_names[mod_id]);
- free(comp_buf);
- return(-1);
- }
-
- tmp_buf_sz = DARSHAN_DEF_DECOMP_BUF_SZ;
- tmp_buf = malloc(DARSHAN_DEF_DECOMP_BUF_SZ);
- if(!tmp_buf)
- {
- free(comp_buf);
- return(-1);
- }
-
- /* decompress this module's data */
- ret = darshan_decompress_buffer(comp_buf, fd->mod_map[mod_id].len, tmp_buf,
- &tmp_buf_sz);
- if(ret < 0)
- {
- fprintf(stderr, "Error: unable to decompress module %s data.\n",
- darshan_module_names[mod_id]);
- free(tmp_buf);
- return(-1);
- }
- free(comp_buf);
-
- /* pass back the final decompressed data pointer */
- *mod_buf = tmp_buf;
- *mod_buf_sz = tmp_buf_sz;
-
- return(0);
-}
-/* TODO: hardcoded for posix -- what can we do generally?
- * different function for each module and a way to map to this function?
- */
-/* TODO: we need bswaps here, too */
-int darshan_log_getfile(darshan_fd fd, struct darshan_posix_file *file)
-{
- char *comp_buf;
- char hash_buf[DARSHAN_DEF_DECOMP_BUF_SZ] = {0};
- int ret;
- const char* err_string;
- int i;
+ if(!fd->mod_map[mod_id].len || fd->pos == mod_buf_end)
+ return(0); /* no (more) data corresponding to this mod_id */
- if(fd->pos < fd->mod_map[0].off)
+ /* only seek to start of module data if current log file position
+ * is not within the given mod_id's range. This allows one to
+ * repeatedly call this function and get chunks of a module's
+ * data piecemeal.
+ */
+ if((fd->pos < fd->mod_map[mod_id].off) || (fd->pos > mod_buf_end))
{
- ret = darshan_log_seek(fd, fd->mod_map[0].off);
+ ret = darshan_log_seek(fd, fd->mod_map[mod_id].off);
if(ret < 0)
{
fprintf(stderr, "Error: unable to seek in darshan log file.\n");
- return(ret);
- }
- }
-
- /* reset file record, so that diff compares against a zero'd out record
- * if file is missing
- */
- memset(file, 0, sizeof(*file));
-
- ret = darshan_log_read(fd, file, sizeof(*file));
- if(ret == sizeof(*file))
- {
- /* got exactly one, correct size record */
- if(fd->swap_flag)
- {
- /* swap bytes if necessary */
- DARSHAN_BSWAP64(&file->f_id);
- DARSHAN_BSWAP64(&file->rank);
- for(i=0; i<CP_NUM_INDICES; i++)
- DARSHAN_BSWAP64(&file->counters[i]);
- for(i=0; i<CP_F_NUM_INDICES; i++)
- DARSHAN_BSWAP64(&file->fcounters[i]);
+ return(-1);
}
- return(1);
}
-
- if(ret > 0)
+ /* read the record hash from the log file */
+ ret = darshan_log_read(fd, moddat_buf, moddat_buf_sz);
+ if(ret != moddat_buf_sz)
{
- /* got a short read */
- fprintf(stderr, "Error: invalid file record (too small)\n");
+ fprintf(stderr,
+ "Error: invalid darshan log file (failed to read module %s data).\n",
+ darshan_module_names[mod_id]);
return(-1);
}
- if(ret == 0)
- {
- /* hit end of file */
- return(0);
- }
-
- /* all other errors */
- err_string = strerror(errno);
- fprintf(stderr, "Error: %s\n", err_string);
- return(-1);
+ return(1);
}
/* darshan_log_close()
@@ -554,33 +404,42 @@ int darshan_log_getfile(darshan_fd fd, struct darshan_posix_file *file)
*/
void darshan_log_close(darshan_fd fd)
{
- if(fd->pf)
- close(fd->pf);
+ if(fd->gzf)
+ gzclose(fd->gzf);
- if(fd->exe_mnt_data)
- free(fd->exe_mnt_data);
+ /* TODO: check bz2 */
free(fd);
+
+ return;
}
-/* ******************************************* */
+/* **************************************************** */
/* return 0 on successful seek to offset, -1 on failure.
*/
static int darshan_log_seek(darshan_fd fd, off_t offset)
{
- off_t ret_off;
+ z_off_t zoff = 0;
+ z_off_t zoff_ret = 0;
if(fd->pos == offset)
return(0);
- ret_off = lseek(fd->pf, offset, SEEK_SET);
- if(ret_off == offset)
+ if(fd->gzf)
{
- fd->pos = offset;
- return(0);
+ zoff += offset;
+ zoff_ret = gzseek(fd->gzf, zoff, SEEK_SET);
+ if(zoff_ret == zoff)
+ {
+ fd->pos = offset;
+ return(0);
+ }
+ return(-1);
}
+ /* TODO: check bz2 */
+
return(-1);
}
@@ -590,14 +449,16 @@ static int darshan_log_write(darshan_fd fd, void* buf, int len)
{
int ret;
- if(fd->pf)
+ if(fd->gzf)
{
- ret = write(fd->pf, buf, len);
+ ret = gzwrite(fd->gzf, buf, len);
if(ret > 0)
fd->pos += ret;
return(ret);
}
+ /* TODO: check bz2 */
+
return(-1);
}
@@ -607,72 +468,17 @@ static int darshan_log_read(darshan_fd fd, void* buf, int len)
{
int ret;
- if(fd->pf)
+ if(fd->gzf)
{
- ret = read(fd->pf, buf, len);
+ ret = gzread(fd->gzf, buf, len);
if(ret > 0)
fd->pos += ret;
return(ret);
}
- return(-1);
-}
-
-/* TODO bz2 compression support */
-static int darshan_decompress_buffer(char *comp_buf, int comp_buf_sz,
- char *decomp_buf, int *inout_decomp_buf_sz)
-{
- int ret;
- int total_out = 0;
- z_stream tmp_stream;
-
- memset(&tmp_stream, 0, sizeof(tmp_stream));
- tmp_stream.zalloc = Z_NULL;
- tmp_stream.zfree = Z_NULL;
- tmp_stream.opaque = Z_NULL;
- tmp_stream.next_in = comp_buf;
- tmp_stream.avail_in = comp_buf_sz;
- tmp_stream.next_out = decomp_buf;
- tmp_stream.avail_out = *inout_decomp_buf_sz;
-
- /* initialize the zlib decompression parameters */
- /* TODO: check these parameters? */
- //ret = inflateInit2(&tmp_stream, 31);
- ret = inflateInit(&tmp_stream);
- if(ret != Z_OK)
- {
- return(-1);
- }
-
- /* while we have not finished consuming all of the compressed input data */
- while(tmp_stream.avail_in)
- {
- if(tmp_stream.avail_out == 0)
- {
- /* We ran out of buffer space for compression. In theory,
- * we could just alloc more space, but probably just easier
- * to bump up the default size of the output buffer.
- */
- inflateEnd(&tmp_stream);
- return(-1);
- }
+ /* TODO: check bz2 */
- /* decompress data */
- ret = inflate(&tmp_stream, Z_FINISH);
- if(ret != Z_STREAM_END)
- {
- inflateEnd(&tmp_stream);
- return(-1);
- }
-
- total_out += tmp_stream.total_out;
- if(tmp_stream.avail_in)
- inflateReset(&tmp_stream);
- }
- inflateEnd(&tmp_stream);
-
- *inout_decomp_buf_sz = total_out;
- return(0);
+ return(-1);
}
/*
diff --git a/darshan-util/darshan-logutils.h b/darshan-util/darshan-logutils.h
index 223f8c0..c0dd82e 100644
--- a/darshan-util/darshan-logutils.h
+++ b/darshan-util/darshan-logutils.h
@@ -6,10 +6,26 @@
#ifndef __DARSHAN_LOG_UTILS_H
#define __DARSHAN_LOG_UTILS_H
+#include <zlib.h>
+#ifdef HAVE_LIBBZ2
+#include <bzlib.h>
+#endif
+
#include "darshan-log-format.h"
-#include "darshan-posix-log-format.h"
#include "uthash-1.9.2/src/uthash.h"
+/* default to a compression buffer size of 4 MiB */
+#define DARSHAN_DEF_DECOMP_BUF_SZ (4*1024*1024)
+
+struct darshan_fd_s
+{
+ gzFile gzf;
+ int64_t pos;
+ char version[8];
+ int swap_flag;
+ struct darshan_log_map rec_map;
+ struct darshan_log_map mod_map[DARSHAN_MAX_MODS];
+};
typedef struct darshan_fd_s* darshan_fd;
struct darshan_record_ref
@@ -21,12 +37,12 @@ struct darshan_record_ref
darshan_fd darshan_log_open(const char *name, const char* mode);
int darshan_log_getheader(darshan_fd file, struct darshan_header *header);
int darshan_log_getjob(darshan_fd file, struct darshan_job *job);
-int darshan_log_gethash(darshan_fd file, struct darshan_record_ref **hash);
int darshan_log_getexe(darshan_fd fd, char *buf);
int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
char*** fs_types, int* count);
-int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
- void **mod_buf, int *mod_buf_sz);
+int darshan_log_gethash(darshan_fd file, struct darshan_record_ref **hash);
+int darshan_log_get_moddat(darshan_fd fd, darshan_module_id mod_id,
+ void *moddat_buf, int moddat_buf_sz);
void darshan_log_close(darshan_fd file);
/* convenience macros for printing out counters */
diff --git a/darshan-util/darshan-posix-parser.c b/darshan-util/darshan-posix-parser.c
index 334f341..597c042 100644
--- a/darshan-util/darshan-posix-parser.c
+++ b/darshan-util/darshan-posix-parser.c
@@ -15,7 +15,7 @@
#include <getopt.h>
#include <assert.h>
-#include "darshan-logutils.h"
+#include "darshan-posix-logutils.h"
#include "uthash-1.9.2/src/uthash.h"
int main(int argc, char **argv)
@@ -24,11 +24,11 @@ int main(int argc, char **argv)
int i;
char *filename;
char tmp_string[4096];
- darshan_fd file;
+ darshan_fd fd;
struct darshan_header header;
struct darshan_job job;
struct darshan_record_ref *rec_hash = NULL;
- struct darshan_record_ref *ref, *tmp;
+ struct darshan_record_ref *ref;
int mount_count;
char** mnt_pts;
char** fs_types;
@@ -36,8 +36,7 @@ int main(int argc, char **argv)
char *token;
char *save;
char buffer[DARSHAN_JOB_METADATA_LEN];
- struct darshan_posix_file *posix_mod_buf;
- int posix_mod_buf_sz = 0;
+ struct darshan_posix_file next_file;
assert(argc == 2);
filename = argv[1];
@@ -45,37 +44,37 @@ int main(int argc, char **argv)
struct stat sbuf;
stat(filename, &sbuf);
- file = darshan_log_open(filename, "r");
- if(!file)
+ fd = darshan_log_open(filename, "r");
+ if(!fd)
{
fprintf(stderr, "darshan_log_open() failed to open %s\n.", filename);
return(-1);
}
/* read darshan log header */
- ret = darshan_log_getheader(file, &header);
+ ret = darshan_log_getheader(fd, &header);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getheader() failed to read log header.\n");
- darshan_log_close(file);
+ darshan_log_close(fd);
return(-1);
}
/* read darshan job info */
- ret = darshan_log_getjob(file, &job);
+ ret = darshan_log_getjob(fd, &job);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getjob() failed to read job data.\n");
- darshan_log_close(file);
+ darshan_log_close(fd);
return(-1);
}
/* get the original command line for this job */
- ret = darshan_log_getexe(file, tmp_string);
+ ret = darshan_log_getexe(fd, tmp_string);
if(ret < 0)
{
fprintf(stderr, "Error: unable to read trailing job information.\n");
- darshan_log_close(file);
+ darshan_log_close(fd);
return(-1);
}
@@ -117,11 +116,11 @@ int main(int argc, char **argv)
}
/* get the mount information for this log */
- ret = darshan_log_getmounts(file, &mnt_pts, &fs_types, &mount_count);
+ ret = darshan_log_getmounts(fd, &mnt_pts, &fs_types, &mount_count);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getmounts() failed to read mount information.\n");
- darshan_log_close(file);
+ darshan_log_close(fd);
return(-1);
}
@@ -133,63 +132,47 @@ int main(int argc, char **argv)
printf("# mount entry:\t%s\t%s\n", mnt_pts[i], fs_types[i]);
}
- uint64_t job_size = header.rec_map.off - sizeof(struct darshan_header);
- printf("\nDarshan log file module sizes:\n");
- printf("\tHEADER size = %"PRIu64"\n\tJOB size = %"PRIu64"\n\tREC_HASH size = %"PRIu64"\n",
- sizeof(struct darshan_header), job_size, header.rec_map.len);
- printf("\t%s MODULE size = %"PRIu64"\n", darshan_module_names[DARSHAN_POSIX_MOD],
- header.mod_map[DARSHAN_POSIX_MOD].len);
-
/* read hash of darshan records */
- ret = darshan_log_gethash(file, &rec_hash);
+ ret = darshan_log_gethash(fd, &rec_hash);
if(ret < 0)
{
fprintf(stderr, "darshan_log_getmap() failed to read record map.\n");
- darshan_log_close(file);
+ darshan_log_close(fd);
return(-1);
}
printf("\n*** FILE RECORD DATA ***\n");
-
- /* get the log data for the POSIX module */
- ret = darshan_log_getmod(file, DARSHAN_POSIX_MOD, (void**)&posix_mod_buf, &posix_mod_buf_sz);
- if(ret < 0 || posix_mod_buf_sz % sizeof(struct darshan_posix_file))
+
+ ret = darshan_log_get_posix_file(fd, &next_file);
+ if(ret < 0)
{
- fprintf(stderr, "darshan_log_getmod() failed to read POSIX module data.\n");
- darshan_log_close(file);
+ fprintf(stderr, "darshan_log_get_posix_file() failed to read next record.\n");
+ darshan_log_close(fd);
return(-1);
}
-
- if(posix_mod_buf_sz == 0)
+ if(ret == 0)
{
- /* it looks like the app didn't open any files */
- printf("# no files opened.\n");
- darshan_log_close(file);
+ printf("# no files opened.");
+ darshan_log_close(fd);
return(0);
}
-
- /* TODO: move this functionality somewhere else so it can be reused */
- /* loop over the POSIX file records and print out counters */
- for(i = 0; i < (posix_mod_buf_sz / sizeof(struct darshan_posix_file)); i++)
+
+ /* loop over each stored POSIX file record and print counters */
+ i = 1;
+ do
{
- struct darshan_posix_file next_rec = posix_mod_buf[i];
-
- DARSHAN_BSWAP64(&(next_rec.f_id));
- DARSHAN_BSWAP64(&(next_rec.rank));
- DARSHAN_BSWAP64(&(next_rec.counters[CP_POSIX_OPENS]));
- DARSHAN_BSWAP64(&(next_rec.fcounters[CP_F_OPEN_TIMESTAMP]));
- DARSHAN_BSWAP64(&(next_rec.fcounters[CP_F_CLOSE_TIMESTAMP]));
-
/* get the pathname for this record */
- HASH_FIND(hlink, rec_hash, &next_rec.f_id, sizeof(darshan_record_id), ref);
+ HASH_FIND(hlink, rec_hash, &next_file.f_id, sizeof(darshan_record_id), ref);
assert(ref);
printf("\tRecord %d: id=%"PRIu64" (path=%s, rank=%"PRId64")\n",
- i, next_rec.f_id, ref->rec.name, next_rec.rank);
+ i, next_file.f_id, ref->rec.name, next_file.rank);
printf("\t\tPOSIX_OPENS:\t%"PRIu64"\n\t\tF_OPEN_TIMESTAMP:\t%lf\n\t\tF_CLOSE_TIMESTAMP:\t%lf\n",
- next_rec.counters[CP_POSIX_OPENS], next_rec.fcounters[CP_F_OPEN_TIMESTAMP],
- next_rec.fcounters[CP_F_CLOSE_TIMESTAMP]);
- }
+ next_file.counters[CP_POSIX_OPENS], next_file.fcounters[CP_F_OPEN_TIMESTAMP],
+ next_file.fcounters[CP_F_CLOSE_TIMESTAMP]);
+
+ i++;
+ } while((ret = darshan_log_get_posix_file(fd, &next_file)) == 1);
/* free mount info */
for(i=0; i<mount_count; i++)
@@ -203,7 +186,7 @@ int main(int argc, char **argv)
free(fs_types);
}
- darshan_log_close(file);
+ darshan_log_close(fd);
return(0);
}
diff --git a/doc/darshan-modularization.txt b/doc/darshan-modularization.txt
index 019b9f4..7c4af24 100644
--- a/doc/darshan-modularization.txt
+++ b/doc/darshan-modularization.txt
@@ -277,6 +277,7 @@ char* darshan_clean_file_path(
The `darshan_clean_file_path` function just cleans up the input path string, converting
relative paths to absolute paths and suppressing any potential noise within the string.
+The address of the new string is returned and should be freed by the user.
* _path_ is the input path string to be cleaned up.
@@ -346,7 +347,7 @@ to store the record structure in a hash table or some other structure.
==== Build modifications
-Text
+Text.
== Other resources
hooks/post-receive
--
More information about the Darshan-commits
mailing list