[Swift-commit] r2281 - log-processing/libexec

[noreply at svn.ci.uchicago.edu]

Author: benc
Date: 2008-10-06 11:44:34 -0500 (Mon, 06 Oct 2008)
New Revision: 2281

Added:
   log-processing/libexec/info-and-karajan-actives
   log-processing/libexec/info-and-karajan-actives.2.plot
   log-processing/libexec/info-and-karajan-actives.plot
Modified:
   log-processing/libexec/info.html.template
   log-processing/libexec/makefile.webpage
Log:
plots of active status vs info log start/end times

Added: log-processing/libexec/info-and-karajan-actives
===================================================================
--- log-processing/libexec/info-and-karajan-actives	                        (rev 0)
+++ log-processing/libexec/info-and-karajan-actives	2008-10-06 16:44:34 UTC (rev 2281)
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# for every job which has an info file,
+# create two columns, one being the start time of the job according
+# to the info file and the other being the Active time according to
+# karajan
+
+
+# for each info file
+
+for infofn in $1/*-info; do
+
+#   get info id
+
+  jobid=$(echo $infofn | sed 's%^.*/\([^/]*\)-info$%\1%' )
+#  echo Processing info file for job $jobid in file $infofn >&2
+# Progress  2007-10-29 10:05:50+0000  RM_JOBDIR
+INFOSTART=$(grep -E '^Progress ' $infofn | sed "s/^Progress *\([^ ]* [^ ]*\) *\([^ ]*\).*/\1 $jobid \2/" | head -n 1 | ./iso-to-secs | cut -d ' ' -f 1)
+INFOEND=$(grep -E '^Progress ' $infofn | sed "s/^Progress *\([^ ]* [^ ]*\) *\([^ ]*\).*/\1 $jobid \2/" | tail -n 1 | ./iso-to-secs | cut -d ' ' -f 1)
+
+#   get karajan ID
+# execute2.event lines look like this:
+# 1223271839.650 0.782999992370605 touch-a7kvof0j JOB_END 0-0-4-1 localhost 97kvof0j
+
+threadid=$(grep -E "^.* .* $jobid .* .* .* .*$" execute2.event | sed 's/^.* .* .* .* \(.*\) .* .*$/\1/')
+
+#   get karajan Active timestamp for this
+#1223271840.424 0-0-4-1-1223271839341 Submitting FILE_OPERATION 
+
+KARASTART=$(grep JOB_SUBMISSION karatasks.transitions | grep Active | grep -E "^.* ${threadid}-.* .* .*" | cut -d ' ' -f 1)
+KARAEND=$(grep JOB_SUBMISSION karatasks.transitions | grep Completed | grep -E "^.* ${threadid}-.* .* .*" | cut -d ' ' -f 1)
+
+#   output
+
+DIFFSTART=$(echo $INFOSTART - $KARASTART | bc -l )
+DIFFEND=$(echo $INFOEND - $KARAEND | bc -l )
+
+echo $INFOSTART $KARASTART $DIFFSTART >> info-and-karajan-actives.data
+echo $INFOEND $KARAEND $DIFFEND >> info-and-karajan-actives.end.data
+
+
+done
+
+gnuplot info-and-karajan-actives.plot
+
+cat info-and-karajan-actives.data | cut -d ' ' -f 3 | sort -n | ./number-sites-list > info-and-karajan-actives.2.data
+cat info-and-karajan-actives.end.data | cut -d ' ' -f 3 | sort -n | ./number-sites-list > info-and-karajan-actives.end.2.data
+
+gnuplot info-and-karajan-actives.2.plot
+
+


Property changes on: log-processing/libexec/info-and-karajan-actives
___________________________________________________________________
Name: svn:executable
   + *

Added: log-processing/libexec/info-and-karajan-actives.2.plot
===================================================================
--- log-processing/libexec/info-and-karajan-actives.2.plot	                        (rev 0)
+++ log-processing/libexec/info-and-karajan-actives.2.plot	2008-10-06 16:44:34 UTC (rev 2281)
@@ -0,0 +1,10 @@
+set terminal png
+set output 'info-and-karajan-actives.2.png'
+
+set xlabel 'offset from Active notification'
+set ylabel 'cumulative events below this offset'
+
+set key outside below
+
+plot 'info-and-karajan-actives.2.data' using 1:2 with lines title 'start time (info - karajan)', 'info-and-karajan-actives.end.2.data' using 1:2 with lines title 'end time (info - karajan)'
+

Added: log-processing/libexec/info-and-karajan-actives.plot
===================================================================
--- log-processing/libexec/info-and-karajan-actives.plot	                        (rev 0)
+++ log-processing/libexec/info-and-karajan-actives.plot	2008-10-06 16:44:34 UTC (rev 2281)
@@ -0,0 +1,10 @@
+set terminal png
+set output 'info-and-karajan-actives.png'
+
+set ylabel 'offset from Active notification'
+set xlabel 'time of start according to info'
+
+set key outside below
+
+plot 'info-and-karajan-actives.data' using 1:3 title 'start times (info - karajan)', 'info-and-karajan-actives.end.data' using 1:3 title 'end times (info - karajan)'
+

Modified: log-processing/libexec/info.html.template
===================================================================
--- log-processing/libexec/info.html.template	2008-10-06 05:27:54 UTC (rev 2280)
+++ log-processing/libexec/info.html.template	2008-10-06 16:44:34 UTC (rev 2281)
@@ -24,6 +24,13 @@
 <img src="info-total.png" />
 
 <hr />
+
+<p>Offsets between job submission Active events and start times reported
+by info.</p>
+<img src="info-and-karajan-actives.png" />
+<br/>
+<img src="info-and-karajan-actives.2.png" />
+
 <p>end</p>
 </body>
 </html>

Modified: log-processing/libexec/makefile.webpage
===================================================================
--- log-processing/libexec/makefile.webpage	2008-10-06 05:27:54 UTC (rev 2280)
+++ log-processing/libexec/makefile.webpage	2008-10-06 16:44:34 UTC (rev 2281)
@@ -41,7 +41,8 @@
 initshareddir.png createdirset.png createdirset-total.png dostagein.png \
 dostagein-total.png dostageout.png dostageout-total.png execstages.png \
 dostageout.sorted-start.png dostagein-duration-histogram.png \
-dostageout-duration-histogram.png execute-trails.png execute2-trails.png
+dostageout-duration-histogram.png execute-trails.png execute2-trails.png \
+info-and-karajan-actives.png info-and-karajan-actives.2.png
 
 htmldeps: index.html.template execution-counts.txt jobs-sites.table site-duration.txt execute2.lastsummary execute.lastsummary jobs.retrycount.summary trname-summary
 
@@ -53,3 +54,5 @@
 
 index.html.kickstart: kickstart.stats
 
+info-and-karajan-actives.png info-and-karajan-actives.2.png: execute2.event karatasks.transitions
+	./info-and-karajan-actives $(IDIR)