[Darshan-commits] [Darshan] branch, dev-modular, updated. darshan-2.3.1-156-gf42d933
Service Account
git at mcs.anl.gov
Tue Aug 25 12:37:58 CDT 2015
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "".
The branch, dev-modular has been updated
via f42d933f17d04332649575a1206b2492c602d112 (commit)
via 7f0cf8630fc86db60aaef68928ef9023a156bb36 (commit)
via c39904ba0f3a690585b0d93b9cf902a425e3b5f6 (commit)
from 89a97a0f4de39f7ee149308af537390d5db0a8dc (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
commit f42d933f17d04332649575a1206b2492c602d112
Author: Shane Snyder <ssnyder at mcs.anl.gov>
Date: Tue Aug 25 12:37:15 2015 -0500
add darshan-convert utility, hash gen exe
commit 7f0cf8630fc86db60aaef68928ef9023a156bb36
Author: Shane Snyder <ssnyder at mcs.anl.gov>
Date: Tue Aug 25 12:36:02 2015 -0500
minor bug fixes in modput
commit c39904ba0f3a690585b0d93b9cf902a425e3b5f6
Author: Shane Snyder <ssnyder at mcs.anl.gov>
Date: Mon Aug 24 16:03:23 2015 -0500
fix a couple of memory leaks
-----------------------------------------------------------------------
Summary of changes:
darshan-util/Makefile.in | 10 +-
darshan-util/darshan-convert-logs.pl | 7 +-
darshan-util/darshan-convert.c | 138 +++++++++++++++-----
darshan-util/darshan-logutils.c | 21 ++-
.../bin/jenkins.c => jenkins-hash-gen.c} | 0
5 files changed, 128 insertions(+), 48 deletions(-)
rename darshan-util/{darshan-job-summary/bin/jenkins.c => jenkins-hash-gen.c} (100%)
Diff of changes:
diff --git a/darshan-util/Makefile.in b/darshan-util/Makefile.in
index 11bdbf1..0e1bc41 100644
--- a/darshan-util/Makefile.in
+++ b/darshan-util/Makefile.in
@@ -1,4 +1,4 @@
-all: libdarshan-util.a darshan-analyzer darshan-convert darshan-parser
+all: libdarshan-util.a darshan-analyzer darshan-convert darshan-parser jenkins-hash-gen
DESTDIR =
srcdir = @srcdir@
@@ -75,8 +75,8 @@ libdarshan-util.so: darshan-logutils.po darshan-posix-logutils.po darshan-mpiio-
libdarshan-util.a: darshan-logutils.o darshan-posix-logutils.o darshan-mpiio-logutils.o darshan-hdf5-logutils.o darshan-pnetcdf-logutils.o
ar rcs libdarshan-util.a $^
-jenkins: util/bin/jenkins.o lookup3.o
- $(CC) $(CFLAGS) $(LDFLAGS) $< -o $@ lookup3.o $(LIBS)
+jenkins-hash-gen: jenkins-hash-gen.c lookup3.o
+ $(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
lookup3.o: lookup3.c
$(CC) $(CFLAGS) -c $< -o $@
@@ -84,7 +84,7 @@ lookup3.o: lookup3.c
darshan-analyzer: darshan-analyzer.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) $(DARSHAN_MOD_LOGUTIL_HEADERS) $(DARSHAN_MOD_LOG_FORMATS) libdarshan-util.a | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
-darshan-convert: darshan-convert.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) libdarshan-util.a | uthash-1.9.2
+darshan-convert: darshan-convert.c darshan-logutils.h $(DARSHAN_LOG_FORMAT) libdarshan-util.a lookup3.o | uthash-1.9.2
$(CC) $(CFLAGS) $(LDFLAGS) $^ -o $@ $(LIBS)
#darshan-diff: darshan-diff.o $(DARSHAN_LOG_FORMAT) darshan-logutils.o darshan-logutils.h
@@ -129,7 +129,7 @@ endif
clean::
- rm -f *.o *.a darshan-analyzer darshan-convert darshan-parser
+ rm -f *.o *.a darshan-analyzer darshan-convert darshan-parser jenkins-hash-gen
distclean:: clean
rm -f darshan-runtime-config.h aclocal.m4 autom4te.cache/* config.status config.log Makefile util/bin/darshan-job-summary.pl
diff --git a/darshan-util/darshan-convert-logs.pl b/darshan-util/darshan-convert-logs.pl
index 1132a0f..2987b7c 100755
--- a/darshan-util/darshan-convert-logs.pl
+++ b/darshan-util/darshan-convert-logs.pl
@@ -20,7 +20,7 @@
#
my $darshan_convert = "./darshan-convert";
-my $jenkins = "./jenkins";
+my $jenkins_hash_gen = "./jenkins-hash-gen";
sub load_annotations($$)
{
@@ -80,16 +80,17 @@ sub main()
$day = $3;
$logname = $4;
}
- $hashed_fname = `$jenkins --64 --key $hash_key $logname`;
+ $hashed_fname = `$jenkins_hash_gen --64 --key $hash_key $logname`;
chomp($hashed_fname);
@args = ("$darshan_convert",
+ "--bzip2",
"--obfuscate",
"--key=$hash_key",
"--annotate=$annotation",
"--reset-md",
"$logfile",
- "$output_path/$year/$month/$day/$hashed_fname.bz2");
+ "$output_path/$year/$month/$day/$hashed_fname.darshan");
$rc = system(@args);
if ($rc) {
print("$hashed_fname\t$logfile:failed:$rc\n");
diff --git a/darshan-util/darshan-convert.c b/darshan-util/darshan-convert.c
index b8bc455..df5cd53 100644
--- a/darshan-util/darshan-convert.c
+++ b/darshan-util/darshan-convert.c
@@ -56,7 +56,9 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
};
*bzip2 = 0;
+ *obfuscate = 0;
*reset_md = 0;
+ *key = 0;
*hash = 0;
while(1)
@@ -108,7 +110,6 @@ void parse_args (int argc, char **argv, char **infile, char **outfile,
return;
}
-#if 0
static void reset_md_job(struct darshan_job *job)
{
job->metadata[0] = '\0';
@@ -137,13 +138,21 @@ void obfuscate_exe(int key, char *exe)
return;
}
-void obfuscate_file(int key, struct darshan_file *file)
+void obfuscate_filenames(int key, struct darshan_record_ref *rec_hash)
{
+ struct darshan_record_ref *ref, *tmp;
uint32_t hashed;
+ char tmp_string[32];
- hashed = darshan_hashlittle(file->name_suffix, sizeof(file->name_suffix), key);
- memset(file->name_suffix, 0, sizeof(file->name_suffix));
- sprintf(file->name_suffix, "%u", hashed);
+ HASH_ITER(hlink, rec_hash, ref, tmp)
+ {
+ hashed = darshan_hashlittle(ref->rec.name, strlen(ref->rec.name), key);
+ sprintf(tmp_string, "%u", hashed);
+ free(ref->rec.name);
+ ref->rec.name = malloc(strlen(tmp_string));
+ assert(ref->rec.name);
+ memcpy(ref->rec.name, tmp_string, strlen(tmp_string));
+ }
return;
}
@@ -183,13 +192,29 @@ void add_annotation (char *annotation,
{
fprintf(stderr,
"not enough space left in metadata for: current=%s token=%s (remain=%d:need=%d)\n",
- job->metadata, token, remaining-1, strlen(token)+1);
+ job->metadata, token, remaining-1, (int)strlen(token)+1);
+ }
+ }
+
+ return;
+}
+
+static void remove_hash_recs(struct darshan_record_ref **rec_hash, darshan_record_id hash)
+{
+ struct darshan_record_ref *ref, *tmp;
+
+ HASH_ITER(hlink, *rec_hash, ref, tmp)
+ {
+ if(ref->rec.id != hash)
+ {
+ HASH_DELETE(hlink, *rec_hash, ref);
+ free(ref->rec.name);
+ free(ref);
}
}
return;
}
-#endif
int main(int argc, char **argv)
{
@@ -198,25 +223,24 @@ int main(int argc, char **argv)
char *outfile_name;
struct darshan_header header;
struct darshan_job job;
- char tmp_string[4096];
+ char tmp_string[4096] = {0};
darshan_fd infile;
darshan_fd outfile;
int i;
int mount_count;
char** mnt_pts;
+ char** fs_types;
struct darshan_record_ref *rec_hash = NULL;
struct darshan_record_ref *ref, *tmp;
char *mod_buf;
int mod_buf_sz;
- char** fs_types;
- int bzip2;
enum darshan_comp_type comp_type;
-
- int obfuscate = 0;
- int key = 0;
+ int bzip2;
+ int obfuscate;
+ int key;
char *annotation = NULL;
- uint64_t hash;
- int reset_md = 0;
+ darshan_record_id hash;
+ int reset_md;
parse_args(argc, argv, &infile_name, &outfile_name, &bzip2, &obfuscate,
&reset_md, &key, &annotation, &hash);
@@ -262,11 +286,9 @@ int main(int argc, char **argv)
return(-1);
}
-#if 0
if (reset_md) reset_md_job(&job);
if (obfuscate) obfuscate_job(key, &job);
if (annotation) add_annotation(annotation, &job);
-#endif
ret = darshan_log_putjob(outfile, &job);
if (ret < 0)
@@ -286,9 +308,7 @@ int main(int argc, char **argv)
return(-1);
}
-#if 0
if (obfuscate) obfuscate_exe(key, tmp_string);
-#endif
ret = darshan_log_putexe(outfile, tmp_string);
if(ret < 0)
@@ -326,6 +346,12 @@ int main(int argc, char **argv)
return(-1);
}
+ /* NOTE: obfuscating filepaths breaks the ability to map files
+ * to the corresponding FS & mount info maintained by darshan
+ */
+ if(obfuscate) obfuscate_filenames(key, rec_hash);
+ if(hash) remove_hash_recs(&rec_hash, hash);
+
ret = darshan_log_puthash(outfile, rec_hash);
if(ret < 0)
{
@@ -341,6 +367,12 @@ int main(int argc, char **argv)
for(i=0; i<DARSHAN_MAX_MODS; i++)
{
+ int mod_bytes_left;
+ int mod_bytes_left_save;
+ void *mod_buf_p;
+ void *rec_p = NULL;
+ darshan_record_id rec_id;
+
memset(mod_buf, 0, DARSHAN_DEF_COMP_BUF_SZ);
mod_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
@@ -360,8 +392,48 @@ int main(int argc, char **argv)
continue;
}
+ /* skip modules with no defined logutil handlers */
+ if(!mod_logutils[i])
+ {
+ fprintf(stderr, "Warning: no log utility handlers defined "
+ "for module %s, SKIPPING\n", darshan_module_names[i]);
+ continue;
+ }
+
/* we have module data to convert */
- ret = darshan_log_putmod(outfile, i, mod_buf, mod_buf_sz);
+ /* NOTE: it is necessary to iterate through each module's
+ * records to correct any endianness issues before writing
+ * this data back to file
+ */
+ mod_bytes_left = mod_buf_sz;
+ mod_buf_p = mod_buf;
+ while(mod_bytes_left > 0)
+ {
+ mod_bytes_left_save = mod_bytes_left;
+ ret = mod_logutils[i]->log_get_record(&mod_buf_p, &mod_bytes_left,
+ &rec_p, &rec_id, infile->swap_flag);
+ if(ret < 0)
+ {
+ fprintf(stderr, "Error: failed to parse module %s data record\n",
+ darshan_module_names[i]);
+ darshan_log_close(infile);
+ darshan_log_close(outfile);
+ return(-1);
+ }
+
+ if(hash == rec_id)
+ {
+ mod_buf_p = rec_p;
+ mod_buf_sz = mod_bytes_left_save - mod_bytes_left;
+ break;
+ }
+ else if(mod_bytes_left == 0)
+ {
+ mod_buf_p = mod_buf;
+ }
+ }
+
+ ret = darshan_log_putmod(outfile, i, mod_buf_p, mod_buf_sz);
if(ret < 0)
{
fprintf(stderr, "Error: failed to put module %s data.\n",
@@ -373,6 +445,19 @@ int main(int argc, char **argv)
}
free(mod_buf);
+ /* write header to output file */
+ ret = darshan_log_putheader(outfile);
+ if(ret < 0)
+ {
+ fprintf(stderr, "Error: unable to write header to output log file %s.\n", outfile_name);
+ darshan_log_close(infile);
+ darshan_log_close(outfile);
+ return(-1);
+ }
+
+ darshan_log_close(infile);
+ darshan_log_close(outfile);
+
for(i=0; i<mount_count; i++)
{
free(mnt_pts[i]);
@@ -391,19 +476,6 @@ int main(int argc, char **argv)
free(ref);
}
- /* write header to output file */
- ret = darshan_log_putheader(outfile);
- if(ret < 0)
- {
- fprintf(stderr, "Error: unable to write header to output log file %s.\n", outfile_name);
- darshan_log_close(infile);
- darshan_log_close(outfile);
- return(-1);
- }
-
- darshan_log_close(infile);
- darshan_log_close(outfile);
-
return(ret);
}
diff --git a/darshan-util/darshan-logutils.c b/darshan-util/darshan-logutils.c
index 411f2ad..438257f 100644
--- a/darshan-util/darshan-logutils.c
+++ b/darshan-util/darshan-logutils.c
@@ -355,9 +355,11 @@ int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
if(ret != comp_buf_sz)
{
fprintf(stderr, "Error: failed to write darshan log file job data.\n");
+ free(comp_buf);
return(-1);
}
+ free(comp_buf);
return(0);
}
@@ -849,10 +851,10 @@ int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
return(-1);
}
- comp_buf = malloc(mod_buf_sz);
+ comp_buf = malloc(DARSHAN_DEF_COMP_BUF_SZ);
if(!comp_buf)
return(-1);
- comp_buf_sz = mod_buf_sz;
+ comp_buf_sz = DARSHAN_DEF_COMP_BUF_SZ;
/* compress the module's data */
ret = darshan_compress_buf(mod_buf, mod_buf_sz,
@@ -876,9 +878,11 @@ int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
fprintf(stderr,
"Error: failed to write module %s data to darshan log file.\n",
darshan_module_names[mod_id]);
+ free(comp_buf);
return(-1);
}
+ free(comp_buf);
return(0);
}
@@ -1181,12 +1185,15 @@ static int darshan_bzip2_comp(char* decomp_buf, int decomp_buf_sz,
}
/* compress data */
- ret = BZ2_bzCompress(&tmp_stream, BZ_FINISH);
- if(ret != BZ_STREAM_END)
+ do
{
- BZ2_bzCompressEnd(&tmp_stream);
- return(-1);
- }
+ ret = BZ2_bzCompress(&tmp_stream, BZ_FINISH);
+ if(ret < 0)
+ {
+ BZ2_bzCompressEnd(&tmp_stream);
+ return(-1);
+ }
+ } while (ret != BZ_STREAM_END);
BZ2_bzCompressEnd(&tmp_stream);
assert(tmp_stream.total_out_hi32 == 0);
diff --git a/darshan-util/darshan-job-summary/bin/jenkins.c b/darshan-util/jenkins-hash-gen.c
similarity index 100%
rename from darshan-util/darshan-job-summary/bin/jenkins.c
rename to darshan-util/jenkins-hash-gen.c
hooks/post-receive
--
More information about the Darshan-commits
mailing list