[Darshan-commits] [Git][darshan/darshan][mmap-dev] logutil backwards compatibility for name records
Shane Snyder
xgitlab at cels.anl.gov
Thu Jun 16 23:05:39 CDT 2016
Shane Snyder pushed to branch mmap-dev at darshan / darshan
Commits:
f6769e56 by Shane Snyder at 2016-06-16T19:21:22-05:00
logutil backwards compatibility for name records
- - - - -
5 changed files:
- darshan-log-format.h
- darshan-util/Makefile.in
- + darshan-util/compat/darshan-logutils-compat.c
- + darshan-util/compat/darshan-logutils-compat.h
- darshan-util/darshan-logutils.c
Changes:
=====================================
darshan-log-format.h
=====================================
--- a/darshan-log-format.h
+++ b/darshan-log-format.h
@@ -20,7 +20,7 @@
#endif
/* update this on file format changes */
-#define DARSHAN_LOG_VERSION "3.00"
+#define DARSHAN_LOG_VERSION "3.01"
/* magic number for validating output files and checking byte order */
#define DARSHAN_MAGIC_NR 6567223
=====================================
darshan-util/Makefile.in
=====================================
--- a/darshan-util/Makefile.in
+++ b/darshan-util/Makefile.in
@@ -17,8 +17,8 @@ DARSHAN_MOD_LOG_FORMATS = $(srcdir)/../darshan-posix-log-format.h $(srcdir)/../d
DARSHAN_MOD_LOGUTIL_HEADERS = darshan-posix-logutils.h darshan-mpiio-logutils.h darshan-hdf5-logutils.h darshan-pnetcdf-logutils.h
DARSHAN_STATIC_MOD_OBJS = darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o darshan-bgq-logutils.o
DARSHAN_DYNAMIC_MOD_OBJS = darshan-posix-logutils.po darshan-mpiio-logutils.po darshan-hdf5-logutils.po darshan-pnetcdf-logutils.po darshan-bgq-logutils.po
-
-#XXX BGQ
+DARSHAN_STATIC_COMPAT_OBJS = compat/darshan-logutils-compat.o
+DARSHAN_DYNAMIC_COMPAT_OBJS = compat/darshan-logutils-compat.po
DARSHAN_ENABLE_SHARED=@DARSHAN_ENABLE_SHARED@
@@ -32,7 +32,7 @@ cp_zlib_include_flags = @__DARSHAN_ZLIB_INCLUDE_FLAGS@
# deliberately avoid large file support for host side utilities to avoid
# potentially buggy libz 64 bit offset support
-CFLAGS = -I . -I $(srcdir) -I $(srcdir)/../ -DDARSHAN_CONFIG_H=\"darshan-util-config.h\" @CFLAGS@ @CPPFLAGS@
+CFLAGS = -I . -I .. -I $(srcdir) -I $(srcdir)/../ -DDARSHAN_CONFIG_H=\"darshan-util-config.h\" @CFLAGS@ @CPPFLAGS@
CFLAGS_SHARED = $(CFLAGS) -shared -fpic -DPIC
LDFLAGS=@LDFLAGS@
@@ -83,10 +83,15 @@ darshan-bgq-logutils.o: darshan-bgq-logutils.c darshan-logutils.h darshan-bgq-lo
darshan-bgq-logutils.po: darshan-bgq-logutils.c darshan-logutils.h darshan-bgq-logutils.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-bgq-log-format.h | uthash-1.9.2
$(CC) $(CFLAGS_SHARED) -c $< -o $@
-libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS)
+compat/darshan-logutils-compat.o: compat/darshan-logutils-compat.c compat/darshan-logutils-compat.h darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
+ $(CC) $(CFLAGS) -c $< -o $@
+#compat/darshan-logutils-compat.po: compat/darshan-logutils-compat.c compat/darshan-logutils-compat.h darshan-logutils.h $(DARSHAN_LOG_FORMAT) | uthash-1.9.2
+# $(CC) $(CFLAGS_SHARED) -c $< -o $@
+
+libdarshan-util.a: darshan-logutils.o $(DARSHAN_STATIC_MOD_OBJS) $(DARSHAN_STATIC_COMPAT_OBJS)
ar rcs libdarshan-util.a $^
-libdarshan-util.so: darshan-logutils.po $(DARSHAN_DYNAMIC_MOD_OBJS)
+libdarshan-util.so: darshan-logutils.po $(DARSHAN_DYNAMIC_MOD_OBJS)
$(CC) $(CFLAGS_SHARED) $(LDFLAGS) -o $@ $^ $(LIBS)
jenkins-hash-gen: jenkins-hash-gen.c lookup3.o
@@ -160,7 +165,7 @@ endif
clean::
- rm -f *.o *.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
+ rm -f *.o *.po compat/*.o compat/*.po *.a *.so darshan-analyzer darshan-convert darshan-diff darshan-parser darshan-merge jenkins-hash-gen
distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
=====================================
darshan-util/compat/darshan-logutils-compat.c
=====================================
--- /dev/null
+++ b/darshan-util/compat/darshan-logutils-compat.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#define _GNU_SOURCE
+#include "darshan-util-config.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <sys/types.h>
+
+#include "darshan-logutils-compat.h"
+
+int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
+ int swap_flag, struct darshan_name_record_ref **hash)
+{
+ struct darshan_name_record_ref *ref;
+ char *buf_ptr;
+ darshan_record_id *rec_id_ptr;
+ uint32_t *path_len_ptr;
+ char *path_ptr;
+ int rec_len;
+ int buf_processed = 0;
+
+ /* work through the name record buffer -- deserialize the mapping data and
+ * add to the output hash table
+ * NOTE: these mapping pairs are variable in length, so we have to be able
+ * to handle incomplete mappings temporarily here
+ */
+ buf_ptr = name_rec_buf;
+ while(buf_len > (sizeof(darshan_record_id) + sizeof(uint32_t)))
+ {
+ /* see if we have enough buf space to read in the next full record */
+ path_len_ptr = (uint32_t *)(buf_ptr + sizeof(darshan_record_id));
+ if(swap_flag)
+ DARSHAN_BSWAP32(path_len_ptr);
+ rec_len = sizeof(darshan_record_id) + sizeof(uint32_t) + *path_len_ptr;
+
+ /* we need to read more before we continue deserializing */
+ if(buf_len < rec_len)
+ break;
+
+ /* get pointers for each field of this darshan record */
+ /* NOTE: darshan record hash serialization method:
+ * ... darshan_record_id | (uint32_t) path_len | path ...
+ */
+ rec_id_ptr = (darshan_record_id *)buf_ptr;
+ path_ptr = (char *)(buf_ptr + sizeof(darshan_record_id) + sizeof(uint32_t));
+
+ if(swap_flag)
+ /* we need to sort out endianness issues before deserializing */
+ DARSHAN_BSWAP64(rec_id_ptr);
+
+ HASH_FIND(hlink, *hash, rec_id_ptr, sizeof(darshan_record_id), ref);
+ if(!ref)
+ {
+ ref = malloc(sizeof(*ref));
+ if(!ref)
+ return(-1);
+
+ ref->name_record = malloc(rec_len - sizeof(uint32_t) + 1);
+ if(!ref->name_record)
+ {
+ free(ref);
+ return(-1);
+ }
+
+ /* transform the serialized name record into the zero-length
+ * array structure darshan uses to track name records
+ */
+ ref->name_record->id = *rec_id_ptr;
+ memcpy(ref->name_record->name, path_ptr, *path_len_ptr);
+ ref->name_record->name[*path_len_ptr] = '\0';
+
+ /* add this record to the hash */
+ HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
+ }
+
+ buf_ptr += rec_len;
+ buf_len -= rec_len;
+ buf_processed += rec_len;
+ }
+
+ return(buf_processed);
+}
+
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * End:
+ *
+ * vim: ts=8 sts=4 sw=4 expandtab
+ */
=====================================
darshan-util/compat/darshan-logutils-compat.h
=====================================
--- /dev/null
+++ b/darshan-util/compat/darshan-logutils-compat.h
@@ -0,0 +1,15 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_LOG_UTILS_COMPAT_H
+#define __DARSHAN_LOG_UTILS_COMPAT_H
+
+#include "darshan-logutils.h"
+
+int darshan_log_get_namerecs_3_00(void *name_rec_buf, int buf_len,
+ int swap_flag, struct darshan_name_record_ref **hash);
+
+#endif
=====================================
darshan-util/darshan-logutils.c
=====================================
--- a/darshan-util/darshan-logutils.c
+++ b/darshan-util/darshan-logutils.c
@@ -18,6 +18,7 @@
#include <errno.h>
#include "darshan-logutils.h"
+#include "compat/darshan-logutils-compat.h"
/* default input buffer size for decompression algorithm */
#define DARSHAN_DEF_COMP_BUF_SZ (1024*1024) /* 1 MiB */
@@ -61,12 +62,18 @@ struct darshan_fd_int_state
char *exe_mnt_data;
/* whether previous file operations have failed */
int err;
+ /* log format version-specific function calls for getting
+ * data from the log file
+ */
+ int (*get_namerecs)(void *, int, int, struct darshan_name_record_ref **);
/* compression/decompression stream read/write state */
struct darshan_dz_state dz;
};
static int darshan_mnt_info_cmp(const void *a, const void *b);
+static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
+ int swap_flag, struct darshan_name_record_ref **hash);
static int darshan_log_get_header(darshan_fd fd);
static int darshan_log_put_header(darshan_fd fd);
static int darshan_log_seek(darshan_fd fd, off_t offset);
@@ -493,7 +500,8 @@ int darshan_log_put_mounts(darshan_fd fd, struct darshan_mnt_info *mnt_data_arra
/* darshan_log_get_namehash()
*
- * read the hash of name records from the darshan log file
+ * read the set of name records from the darshan log file and add to the
+ * given hash table
*
* returns 0 on success, -1 on failure
*/
@@ -501,14 +509,11 @@ int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **has
{
struct darshan_fd_int_state *state = fd->state;
char *name_rec_buf;
- char *tmp_p;
int name_rec_buf_sz;
int read;
int read_req_sz;
- struct darshan_name_record_ref *ref;
- struct darshan_name_record *name_rec;
- int buf_rem = 0;
- int rec_len;
+ int buf_len = 0;
+ int buf_processed;
assert(state);
@@ -531,78 +536,29 @@ int darshan_log_get_namehash(darshan_fd fd, struct darshan_name_record_ref **has
/* read chunks of the darshan record id -> name mapping from log file,
* constructing a hash table in the process
*/
- read_req_sz = name_rec_buf_sz - buf_rem;
+ read_req_sz = name_rec_buf_sz - buf_len;
read = darshan_log_dzread(fd, DARSHAN_NAME_MAP_REGION_ID,
- name_rec_buf + buf_rem, read_req_sz);
+ name_rec_buf + buf_len, read_req_sz);
if(read < 0)
{
fprintf(stderr, "Error: failed to read name hash from darshan log file.\n");
free(name_rec_buf);
return(-1);
}
- buf_rem += read;
-
- /* work through the name record buffer -- deserialize the mapping data and
- * add to the output hash table
- * NOTE: these mapping pairs are variable in length, so we have to be able
- * to handle incomplete mappings temporarily here
- */
- name_rec = (struct darshan_name_record *)name_rec_buf;
- while(buf_rem > sizeof(darshan_record_id) + 1)
- {
- if(strnlen(name_rec->name, buf_rem - sizeof(darshan_record_id)) ==
- (buf_rem - sizeof(darshan_record_id)))
- {
- /* if this record name's terminating null character is not
- * present, we need to read more of the buffer before continuing
- */
- break;
- }
-
- if(fd->swap_flag)
- {
- /* we need to sort out endianness issues before deserializing */
- DARSHAN_BSWAP64(&(name_rec->id));
- }
+ buf_len += read;
- HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
- if(!ref)
- {
- rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;
- ref = malloc(sizeof(*ref));
- if(!ref)
- {
- free(name_rec_buf);
- return(-1);
- }
- ref->name_record = malloc(rec_len);
- if(!ref->name_record)
- {
- free(ref);
- free(name_rec_buf);
- return(-1);
- }
-
- /* copy the name record over from the hash buffer */
- memcpy(ref->name_record, name_rec, rec_len);
-
- /* add this record to the hash */
- HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
- }
-
- tmp_p = (char *)name_rec + rec_len;
- name_rec = (struct darshan_name_record *)tmp_p;
- buf_rem -= rec_len;
- }
+ /* extract any name records in the buffer */
+ buf_processed = state->get_namerecs(name_rec_buf, buf_len, fd->swap_flag, hash);
/* copy any leftover data to beginning of buffer to parse next */
- memcpy(name_rec_buf, name_rec, buf_rem);
+ memcpy(name_rec_buf, name_rec_buf + buf_processed, buf_len - buf_processed);
+ buf_len -= buf_processed;
/* we keep reading until we get a short read informing us we have
* read all of the record hash
*/
} while(read == read_req_sz);
- assert(buf_rem == 0);
+ assert(buf_len == 0);
free(name_rec_buf);
return(0);
@@ -813,6 +769,69 @@ static int darshan_mnt_info_cmp(const void *a, const void *b)
return(0);
}
+static int darshan_log_get_namerecs(void *name_rec_buf, int buf_len,
+ int swap_flag, struct darshan_name_record_ref **hash)
+{
+ struct darshan_name_record_ref *ref;
+ struct darshan_name_record *name_rec;
+ char *tmp_p;
+ int buf_processed = 0;
+ int rec_len;
+
+ /* work through the name record buffer -- deserialize the record data
+ * and add to the output hash table
+ * NOTE: these mapping pairs are variable in length, so we have to be able
+ * to handle incomplete mappings temporarily here
+ */
+ name_rec = (struct darshan_name_record *)name_rec_buf;
+ while(buf_len > sizeof(darshan_record_id) + 1)
+ {
+ if(strnlen(name_rec->name, buf_len - sizeof(darshan_record_id)) ==
+ (buf_len - sizeof(darshan_record_id)))
+ {
+ /* if this record name's terminating null character is not
+ * present, we need to read more of the buffer before continuing
+ */
+ break;
+ }
+ rec_len = sizeof(darshan_record_id) + strlen(name_rec->name) + 1;
+
+ if(swap_flag)
+ {
+ /* we need to sort out endianness issues before deserializing */
+ DARSHAN_BSWAP64(&(name_rec->id));
+ }
+
+ HASH_FIND(hlink, *hash, &(name_rec->id), sizeof(darshan_record_id), ref);
+ if(!ref)
+ {
+ ref = malloc(sizeof(*ref));
+ if(!ref)
+ return(-1);
+
+ ref->name_record = malloc(rec_len);
+ if(!ref->name_record)
+ {
+ free(ref);
+ return(-1);
+ }
+
+ /* copy the name record over from the hash buffer */
+ memcpy(ref->name_record, name_rec, rec_len);
+
+ /* add this record to the hash */
+ HASH_ADD(hlink, *hash, name_record->id, sizeof(darshan_record_id), ref);
+ }
+
+ tmp_p = (char *)name_rec + rec_len;
+ name_rec = (struct darshan_name_record *)tmp_p;
+ buf_len -= rec_len;
+ buf_processed += rec_len;
+ }
+
+ return(buf_processed);
+}
+
/* read the header of the darshan log and set internal fd data structures
* NOTE: this is the only portion of the darshan log that is uncompressed
*
@@ -840,10 +859,19 @@ static int darshan_log_get_header(darshan_fd fd)
}
/* other log file versions can be detected and handled here */
- if(strcmp(fd->version, "3.00"))
+ if(strcmp(fd->version, "3.00") == 0)
+ {
+ fd->state->get_namerecs = darshan_log_get_namerecs_3_00;
+ }
+ else if(strcmp(fd->version, "3.01") == 0)
+ {
+ fd->state->get_namerecs = darshan_log_get_namerecs;
+ }
+ else
{
fprintf(stderr, "Error: incompatible darshan file.\n");
- fprintf(stderr, "Error: expected version %s\n", DARSHAN_LOG_VERSION);
+ fprintf(stderr, "Error: expected version %s, but got %s\n",
+ DARSHAN_LOG_VERSION, fd->version);
return(-1);
}
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/commit/f6769e56f2f60297a64c3e9fb142b16e15f03f12
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20160616/5d1e3554/attachment-0001.html>
More information about the Darshan-commits
mailing list