[Darshan-commits] [Git][darshan/darshan][master] 3 commits: clarify contents of common access size table
Philip Carns
xgitlab at cels.anl.gov
Thu Sep 15 11:32:38 CDT 2016
Philip Carns pushed to branch master at darshan / darshan
Commits:
f7e280c3 by Phil Carns at 2016-09-12T16:49:42-04:00
clarify contents of common access size table
- - - - -
715368e5 by Phil Carns at 2016-09-15T12:16:03-04:00
get file count summary working with stdio
- - - - -
e7f180dc by Phil Carns at 2016-09-15T12:31:50-04:00
update rest of darshan-job-summary for STDIO mod
- - - - -
3 changed files:
- darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
- darshan-util/darshan-job-summary/share/file-access-eps.gplt
- darshan-util/darshan-job-summary/share/file-access-table.tex
Changes:
=====================================
darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
=====================================
--- a/darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
+++ b/darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
@@ -211,7 +211,7 @@ while($line = <PARSE_OUT>)
$summary{$fields[3]} += $fields[4];
# accumulate independent and shared data as well as fs data
- if ($fields[3] eq "POSIX_F_READ_TIME")
+ if ($fields[3] eq "POSIX_F_READ_TIME" || $fields[3] eq "STDIO_F_READ_TIME")
{
if ($fields[1] == -1)
{
@@ -222,7 +222,7 @@ while($line = <PARSE_OUT>)
$cumul_read_indep += $fields[4];
}
}
- elsif ($fields[3] eq "POSIX_F_WRITE_TIME")
+ elsif ($fields[3] eq "POSIX_F_WRITE_TIME" || $fields[3] eq "STDIO_F_WRITE_TIME")
{
if ($fields[1] == -1)
{
@@ -233,7 +233,7 @@ while($line = <PARSE_OUT>)
$cumul_write_indep += $fields[4];
}
}
- elsif ($fields[3] eq "POSIX_F_META_TIME")
+ elsif ($fields[3] eq "POSIX_F_META_TIME" || $fields[3] eq "STDIO_F_META_TIME")
{
if ($fields[1] == -1)
{
@@ -244,7 +244,7 @@ while($line = <PARSE_OUT>)
$cumul_meta_indep += $fields[4];
}
}
- elsif ($fields[3] eq "POSIX_BYTES_READ")
+ elsif ($fields[3] eq "POSIX_BYTES_READ" || $fields[3] eq "STDIO_BYTES_READ")
{
if ($fields[1] == -1)
{
@@ -260,7 +260,7 @@ while($line = <PARSE_OUT>)
}
$fs_data{$fields[6]}->[0] += $fields[4];
}
- elsif ($fields[3] eq "POSIX_BYTES_WRITTEN")
+ elsif ($fields[3] eq "POSIX_BYTES_WRITTEN" || $fields[3] eq "STDIO_BYTES_WRITTEN")
{
if ($fields[1] == -1)
{
@@ -278,12 +278,13 @@ while($line = <PARSE_OUT>)
}
# record start and end of reads and writes
- elsif ($fields[3] eq "POSIX_F_READ_START_TIMESTAMP")
+ elsif ($fields[3] eq "POSIX_F_READ_START_TIMESTAMP" ||
+ $fields[3] eq "STDIO_F_READ_START_TIMESTAMP")
{
# store until we find the end
$last_read_start = $fields[4];
}
- elsif ($fields[3] eq "POSIX_F_READ_END_TIMESTAMP" && $fields[4] != 0)
+ elsif (($fields[3] eq "POSIX_F_READ_END_TIMESTAMP" || $fields[3] eq "STDIO_F_READ_END_TIMESTAMP") && $fields[4] != 0)
{
# assume we got the read start already
my $xdelta = $fields[4] - $last_read_start;
@@ -296,12 +297,13 @@ while($line = <PARSE_OUT>)
print FA_READ "$last_read_start\t$fields[1]\t$xdelta\t0\n";
}
}
- elsif ($fields[3] eq "POSIX_F_WRITE_START_TIMESTAMP")
+ elsif ($fields[3] eq "POSIX_F_WRITE_START_TIMESTAMP" ||
+ $fields[3] eq "STDIO_F_WRITE_START_TIMESTAMP")
{
# store until we find the end
$last_write_start = $fields[4];
}
- elsif ($fields[3] eq "POSIX_F_WRITE_END_TIMESTAMP" && $fields[4] != 0)
+ elsif (($fields[3] eq "POSIX_F_WRITE_END_TIMESTAMP" || $fields[3] eq "STDIO_F_WRITE_END_TIMESTAMP") && $fields[4] != 0)
{
# assume we got the write start already
my $xdelta = $fields[4] - $last_write_start;
@@ -592,6 +594,7 @@ print ACCESS_TABLE "
\\begin{tabular}{r|r|r}
\\multicolumn{3}{c}{ } \\\\
\\multicolumn{3}{c}{Most Common Access Sizes} \\\\
+\\multicolumn{3}{c}{(POSIX or MPI-IO)} \\\\
\\hline
\& access size \& count \\\\
\\hline
@@ -876,7 +879,7 @@ open(FS_TABLE, ">$tmp_dir/fs-data-table.tex") || die("error opening output files
print FS_TABLE "
\\begin{tabular}{c|r|r|r|r}
\\multicolumn{5}{c}{ } \\\\
-\\multicolumn{5}{c}{Data Transfer Per Filesystem (POSIX)} \\\\
+\\multicolumn{5}{c}{Data Transfer Per Filesystem (POSIX and STDIO)} \\\\
\\hline
\\multirow{2}{*}{File System} \& \\multicolumn{2}{c}{Write} \\vline \& \\multicolumn{2}{c}{Read} \\\\
\\cline{2-5}
@@ -925,7 +928,7 @@ open(VAR_TABLE, ">$tmp_dir/variance-table.tex") || die("error opening output fil
print VAR_TABLE "
\\begin{tabular}{c|r|r|r|r|r|r|r|r|r}
\\multicolumn{10}{c}{} \\\\
-\\multicolumn{10}{c}{Variance in Shared Files (POSIX)} \\\\
+\\multicolumn{10}{c}{Variance in Shared Files (POSIX and STDIO)} \\\\
\\hline
File \& Processes \& \\multicolumn{3}{c}{Fastest} \\vline \&
\\multicolumn{3}{c}{Slowest} \\vline \& \\multicolumn{2}{c}{\$\\sigma\$} \\\\
@@ -1150,14 +1153,14 @@ sub process_file_record
my(%file_record) = %{$_[1]};
my $rank = $file_record{'RANK'};
- if(!defined $file_record{'POSIX_OPENS'})
+ if(!defined $file_record{'POSIX_OPENS'} && !defined $file_record{'STDIO_OPENS'})
{
# ignore data records that don't have POSIX & MPI data
return;
}
- if($file_record{'POSIX_OPENS'} == 0 &&
- (!defined $file_record{'STDIO_OPENS'}) &&
+ if((!defined $file_record{'POSIX_OPENS'} || $file_record{'POSIX_OPENS'} == 0) &&
+ (!defined $file_record{'STDIO_OPENS'} || $file_record{'STDIO_OPENS'} == 0) &&
(!defined $file_record{'MPIIO_INDEP_OPENS'} ||
($file_record{'MPIIO_INDEP_OPENS'} == 0 && $file_record{'MPIIO_COLL_OPENS'} == 0)))
{
@@ -1170,19 +1173,41 @@ sub process_file_record
$hash_files{$hash}{'min_open_size'} = 0;
# record largest size that the file reached at any rank
- if(!defined($hash_files{$hash}{'max_size'}) ||
- $hash_files{$hash}{'max_size'} <
- ($file_record{'POSIX_MAX_BYTE_READ'} + 1))
+ if(defined $file_record{'POSIX_OPENS'})
{
- $hash_files{$hash}{'max_size'} =
- $file_record{'POSIX_MAX_BYTE_READ'} + 1;
+ if(!defined($hash_files{$hash}{'max_size'}) ||
+ $hash_files{$hash}{'max_size'} <
+ ($file_record{'POSIX_MAX_BYTE_READ'} + 1))
+ {
+ $hash_files{$hash}{'max_size'} =
+ $file_record{'POSIX_MAX_BYTE_READ'} + 1;
+ }
+ if(!defined($hash_files{$hash}{'max_size'}) ||
+ $hash_files{$hash}{'max_size'} <
+ ($file_record{'POSIX_MAX_BYTE_WRITTEN'} + 1))
+ {
+ $hash_files{$hash}{'max_size'} =
+ $file_record{'POSIX_MAX_BYTE_WRITTEN'} + 1;
+ }
}
- if(!defined($hash_files{$hash}{'max_size'}) ||
- $hash_files{$hash}{'max_size'} <
- ($file_record{'POSIX_MAX_BYTE_WRITTEN'} + 1))
+
+ # for stdio as well
+ if(defined $file_record{'STDIO_OPENS'})
{
- $hash_files{$hash}{'max_size'} =
- $file_record{'POSIX_MAX_BYTE_WRITTEN'} + 1;
+ if(!defined($hash_files{$hash}{'max_size'}) ||
+ $hash_files{$hash}{'max_size'} <
+ ($file_record{'STDIO_MAX_BYTE_READ'} + 1))
+ {
+ $hash_files{$hash}{'max_size'} =
+ $file_record{'STDIO_MAX_BYTE_READ'} + 1;
+ }
+ if(!defined($hash_files{$hash}{'max_size'}) ||
+ $hash_files{$hash}{'max_size'} <
+ ($file_record{'STDIO_MAX_BYTE_WRITTEN'} + 1))
+ {
+ $hash_files{$hash}{'max_size'} =
+ $file_record{'STDIO_MAX_BYTE_WRITTEN'} + 1;
+ }
}
# make sure there is an initial value for read and write flags
@@ -1220,12 +1245,12 @@ sub process_file_record
else
{
# posix file
- if($file_record{'POSIX_READS'} > 0)
+ if((defined $file_record{'POSIX_READS'} && $file_record{'POSIX_READS'} > 0) || (defined $file_record{'STDIO_READS'} && $file_record{'STDIO_READS'} > 0))
{
# data was read from the file
$hash_files{$hash}{'was_read'} = 1;
}
- if($file_record{'POSIX_WRITES'} > 0)
+ if((defined $file_record{'POSIX_WRITES'} && $file_record{'POSIX_WRITES'} > 0) || (defined $file_record{'STDIO_WRITES'} && $file_record{'STDIO_WRITES'} > 0))
{
# data was written to the file
$hash_files{$hash}{'was_written'} = 1;
@@ -1236,24 +1261,52 @@ sub process_file_record
if ($rank == -1)
{
- $hash_files{$hash}{'procs'} = $nprocs;
- $hash_files{$hash}{'slowest_rank'} = $file_record{'POSIX_SLOWEST_RANK'};
- $hash_files{$hash}{'slowest_time'} = $file_record{'POSIX_F_SLOWEST_RANK_TIME'};
- $hash_files{$hash}{'slowest_bytes'} = $file_record{'POSIX_SLOWEST_RANK_BYTES'};
- $hash_files{$hash}{'fastest_rank'} = $file_record{'POSIX_FASTEST_RANK'};
- $hash_files{$hash}{'fastest_time'} = $file_record{'POSIX_F_FASTEST_RANK_TIME'};
- $hash_files{$hash}{'fastest_bytes'} = $file_record{'POSIX_FASTEST_RANK_BYTES'};
- $hash_files{$hash}{'variance_time'} = $file_record{'POSIX_F_VARIANCE_RANK_TIME'};
- $hash_files{$hash}{'variance_bytes'} = $file_record{'POSIX_F_VARIANCE_RANK_BYTES'};
+ if(defined $file_record{'POSIX_OPENS'} && $file_record{'POSIX_OPENS'} > 0)
+ {
+ $hash_files{$hash}{'procs'} = $nprocs;
+ $hash_files{$hash}{'slowest_rank'} = $file_record{'POSIX_SLOWEST_RANK'};
+ $hash_files{$hash}{'slowest_time'} = $file_record{'POSIX_F_SLOWEST_RANK_TIME'};
+ $hash_files{$hash}{'slowest_bytes'} = $file_record{'POSIX_SLOWEST_RANK_BYTES'};
+ $hash_files{$hash}{'fastest_rank'} = $file_record{'POSIX_FASTEST_RANK'};
+ $hash_files{$hash}{'fastest_time'} = $file_record{'POSIX_F_FASTEST_RANK_TIME'};
+ $hash_files{$hash}{'fastest_bytes'} = $file_record{'POSIX_FASTEST_RANK_BYTES'};
+ $hash_files{$hash}{'variance_time'} = $file_record{'POSIX_F_VARIANCE_RANK_TIME'};
+ $hash_files{$hash}{'variance_bytes'} = $file_record{'POSIX_F_VARIANCE_RANK_BYTES'};
+ }
+ elsif(defined $file_record{'STDIO_OPENS'} && $file_record{'STDIO_OPENS'} > 0)
+ {
+ $hash_files{$hash}{'procs'} = $nprocs;
+ $hash_files{$hash}{'slowest_rank'} = $file_record{'STDIO_SLOWEST_RANK'};
+ $hash_files{$hash}{'slowest_time'} = $file_record{'STDIO_F_SLOWEST_RANK_TIME'};
+ $hash_files{$hash}{'slowest_bytes'} = $file_record{'STDIO_SLOWEST_RANK_BYTES'};
+ $hash_files{$hash}{'fastest_rank'} = $file_record{'STDIO_FASTEST_RANK'};
+ $hash_files{$hash}{'fastest_time'} = $file_record{'STDIO_F_FASTEST_RANK_TIME'};
+ $hash_files{$hash}{'fastest_bytes'} = $file_record{'STDIO_FASTEST_RANK_BYTES'};
+ $hash_files{$hash}{'variance_time'} = $file_record{'STDIO_F_VARIANCE_RANK_TIME'};
+ $hash_files{$hash}{'variance_bytes'} = $file_record{'STDIO_F_VARIANCE_RANK_BYTES'};
+ }
}
else
{
- my $total_time = $file_record{'POSIX_F_META_TIME'} +
+ my $total_time = 0;
+ my $total_bytes = 0;
+
+ if(defined $file_record{'POSIX_OPENS'})
+ {
+ $total_time += $file_record{'POSIX_F_META_TIME'} +
$file_record{'POSIX_F_READ_TIME'} +
$file_record{'POSIX_F_WRITE_TIME'};
-
- my $total_bytes = $file_record{'POSIX_BYTES_READ'} +
+ $total_bytes += $file_record{'POSIX_BYTES_READ'} +
$file_record{'POSIX_BYTES_WRITTEN'};
+ }
+ if(defined $file_record{'STDIO_OPENS'})
+ {
+ $total_time += $file_record{'STDIO_F_META_TIME'} +
+ $file_record{'STDIO_F_READ_TIME'} +
+ $file_record{'STDIO_F_WRITE_TIME'};
+ $total_bytes += $file_record{'STDIO_BYTES_READ'} +
+ $file_record{'STDIO_BYTES_WRITTEN'};
+ }
if(!defined($hash_files{$hash}{'slowest_time'}) ||
$hash_files{$hash}{'slowest_time'} < $total_time)
@@ -1335,20 +1388,41 @@ sub process_file_record
}
else
{
- # add up posix times
- if(defined($hash_unique_file_time{$rank}))
+ if(defined($file_record{POSIX_OPENS}))
{
- $hash_unique_file_time{$rank} +=
- $file_record{POSIX_F_META_TIME} +
- $file_record{POSIX_F_READ_TIME} +
- $file_record{POSIX_F_WRITE_TIME};
+ # add up posix times
+ if(defined($hash_unique_file_time{$rank}))
+ {
+ $hash_unique_file_time{$rank} +=
+ $file_record{POSIX_F_META_TIME} +
+ $file_record{POSIX_F_READ_TIME} +
+ $file_record{POSIX_F_WRITE_TIME};
+ }
+ else
+ {
+ $hash_unique_file_time{$rank} =
+ $file_record{POSIX_F_META_TIME} +
+ $file_record{POSIX_F_READ_TIME} +
+ $file_record{POSIX_F_WRITE_TIME};
+ }
}
- else
+ if(defined($file_record{STDIO_OPENS}))
{
- $hash_unique_file_time{$rank} =
- $file_record{POSIX_F_META_TIME} +
- $file_record{POSIX_F_READ_TIME} +
- $file_record{POSIX_F_WRITE_TIME};
+ # add up posix times
+ if(defined($hash_unique_file_time{$rank}))
+ {
+ $hash_unique_file_time{$rank} +=
+ $file_record{STDIO_F_META_TIME} +
+ $file_record{STDIO_F_READ_TIME} +
+ $file_record{STDIO_F_WRITE_TIME};
+ }
+ else
+ {
+ $hash_unique_file_time{$rank} =
+ $file_record{STDIO_F_META_TIME} +
+ $file_record{STDIO_F_READ_TIME} +
+ $file_record{STDIO_F_WRITE_TIME};
+ }
}
}
}
@@ -1364,7 +1438,14 @@ sub process_file_record
}
else
{
- $shared_file_time += $file_record{'POSIX_F_SLOWEST_RANK_TIME'};
+ if(defined $file_record{'POSIX_OPENS'} && $file_record{'POSIX_OPENS'} > 0)
+ {
+ $shared_file_time += $file_record{'POSIX_F_SLOWEST_RANK_TIME'};
+ }
+ elsif(defined $file_record{'STDIO_OPENS'} && $file_record{'STDIO_OPENS'} > 0)
+ {
+ $shared_file_time += $file_record{'STDIO_F_SLOWEST_RANK_TIME'};
+ }
}
}
@@ -1390,8 +1471,16 @@ sub process_file_record
else
{
# normal case
- $total_job_bytes += $file_record{'POSIX_BYTES_WRITTEN'} +
- $file_record{'POSIX_BYTES_READ'};
+ if(defined $file_record{'POSIX_OPENS'})
+ {
+ $total_job_bytes += $file_record{'POSIX_BYTES_WRITTEN'} +
+ $file_record{'POSIX_BYTES_READ'};
+ }
+ if(defined $file_record{'STDIO_OPENS'})
+ {
+ $total_job_bytes += $file_record{'STDIO_BYTES_WRITTEN'} +
+ $file_record{'STDIO_BYTES_READ'};
+ }
}
}
=====================================
darshan-util/darshan-job-summary/share/file-access-eps.gplt
=====================================
--- a/darshan-util/darshan-job-summary/share/file-access-eps.gplt
+++ b/darshan-util/darshan-job-summary/share/file-access-eps.gplt
@@ -8,7 +8,7 @@ set xdata time
set timefmt "%s"
set format x "%H:%M:%S"
set yrange [-1:ymax]
-set title "Timespan from first to last read access on independent files (POSIX)"
+set title "Timespan from first to last read access on independent files (POSIX and STDIO)"
set xrange ["0":runtime]
set ytics 0,yinc,ymaxtic
set lmargin 6
@@ -25,7 +25,7 @@ set style increment user
plot "file-access-read.dat" using 1:2:3:4 with vectors nohead filled notitle
set output "file-access-write.eps"
-set title "Timespan from first to last write access on independent files (POSIX)"
+set title "Timespan from first to last write access on independent files (POSIX and STDIO)"
# lw 3 to make lines thicker...
plot "file-access-write.dat" using 1:2:3:4 with vectors nohead filled lt 2 notitle
@@ -34,7 +34,7 @@ set output "file-access-shared.eps"
unset ytics
set ylabel "All processes"
set yrange [-1:1]
-set title "Timespan from first to last access on files shared by all processes (POSIX)"
+set title "Timespan from first to last access on files shared by all processes (POSIX and STDIO)"
plot "file-access-read-sh.dat" using 1:2:3:4 with vectors nohead filled lw 10 title "read", \
"file-access-write-sh.dat" using 1:(($2)-.2):3:4 with vectors nohead filled lw 10 title "write"
=====================================
darshan-util/darshan-job-summary/share/file-access-table.tex
=====================================
--- a/darshan-util/darshan-job-summary/share/file-access-table.tex
+++ b/darshan-util/darshan-job-summary/share/file-access-table.tex
@@ -1,5 +1,5 @@
\begin{tabular}{l|p{1.7in}r}
-\multicolumn{3}{c}{Average I/O per process (POSIX)} \\
+\multicolumn{3}{c}{Average I/O per process (POSIX and STDIO)} \\
\hline
& Cumulative time spent in I/O functions (seconds) & Amount of I/O (MB) \\
\hline
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/4addc19b7b8ce1944b98884dc83cc085f361cf5e...e7f180dc028cb1ea34cf01d2225feeac99467912
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20160915/9c280be7/attachment-0001.html>
More information about the Darshan-commits
mailing list