[Swift-commit] r3164 - trunk/libexec/log-processing
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Thu Oct 15 14:54:45 CDT 2009
Author: hategan
Date: 2009-10-15 14:54:45 -0500 (Thu, 15 Oct 2009)
New Revision: 3164
Added:
trunk/libexec/log-processing/coaster-block-timeline.plot
trunk/libexec/log-processing/coaster-block-utilization-vs-count.plot
trunk/libexec/log-processing/coaster-block-utilization.plot
trunk/libexec/log-processing/coaster-qwait-count.plot
trunk/libexec/log-processing/coaster-qwait-size.plot
trunk/libexec/log-processing/coaster-qwait-wtime.plot
trunk/libexec/log-processing/coasters.html.template
trunk/libexec/log-processing/extract-block-utilization-vs-count
trunk/libexec/log-processing/extract-coaster-block-utilization
trunk/libexec/log-processing/extract-coaster-qwait
trunk/libexec/log-processing/extract-coaster-timeline
trunk/libexec/log-processing/extract-coaster-utilization
trunk/libexec/log-processing/makefile.coasters
Modified:
trunk/libexec/log-processing/falkon.html.template
trunk/libexec/log-processing/info.html.template
trunk/libexec/log-processing/karajan.html.template
trunk/libexec/log-processing/kickstart.html.template
trunk/libexec/log-processing/makefile
trunk/libexec/log-processing/makefile.webpage
trunk/libexec/log-processing/toolbar.html.template
Log:
added coasters page
Added: trunk/libexec/log-processing/coaster-block-timeline.plot
===================================================================
--- trunk/libexec/log-processing/coaster-block-timeline.plot (rev 0)
+++ trunk/libexec/log-processing/coaster-block-timeline.plot 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,6 @@
+set terminal png
+set output "coaster-block-timeline.png"
+
+set title "Queued/Active Coaster Workers"
+set style data steps
+plot "coaster-blocks.data" u 1:2 w steps title "Queued Workers", "coaster-blocks.data" u 1:3 w steps title "Running Workers"
\ No newline at end of file
Added: trunk/libexec/log-processing/coaster-block-utilization-vs-count.plot
===================================================================
--- trunk/libexec/log-processing/coaster-block-utilization-vs-count.plot (rev 0)
+++ trunk/libexec/log-processing/coaster-block-utilization-vs-count.plot 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,5 @@
+set terminal png
+set output "coaster-block-utilization-vs-count.png"
+
+set title "Block Utilization vs. Worker Count"
+plot "coaster-block-utilization-vs-count.data" u 1:2 title ""
\ No newline at end of file
Added: trunk/libexec/log-processing/coaster-block-utilization.plot
===================================================================
--- trunk/libexec/log-processing/coaster-block-utilization.plot (rev 0)
+++ trunk/libexec/log-processing/coaster-block-utilization.plot 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,5 @@
+set terminal png
+set output "coaster-block-utilization.png"
+
+set title "Block Utilization (%)"
+plot "coaster-block-utilization.data" w linespoints pt 3 title ""
\ No newline at end of file
Added: trunk/libexec/log-processing/coaster-qwait-count.plot
===================================================================
--- trunk/libexec/log-processing/coaster-qwait-count.plot (rev 0)
+++ trunk/libexec/log-processing/coaster-qwait-count.plot 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,5 @@
+set terminal png
+set output "coaster-qwait-count.png"
+
+set title "Wait times vs. Block Worker Count"
+plot "coaster-qwait-count.data" u 1:2 title ""
\ No newline at end of file
Added: trunk/libexec/log-processing/coaster-qwait-size.plot
===================================================================
--- trunk/libexec/log-processing/coaster-qwait-size.plot (rev 0)
+++ trunk/libexec/log-processing/coaster-qwait-size.plot 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,5 @@
+set terminal png
+set output "coaster-qwait-size.png"
+
+set title "Wait times vs. Block Size (walltime x worker count)"
+plot "coaster-qwait-size.data" u 1:2 title ""
\ No newline at end of file
Added: trunk/libexec/log-processing/coaster-qwait-wtime.plot
===================================================================
--- trunk/libexec/log-processing/coaster-qwait-wtime.plot (rev 0)
+++ trunk/libexec/log-processing/coaster-qwait-wtime.plot 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,5 @@
+set terminal png
+set output "coaster-qwait-wtime.png"
+
+set title "Wait times vs. Block Walltime"
+plot "coaster-qwait-wtime.data" u 1:2 title ""
\ No newline at end of file
Added: trunk/libexec/log-processing/coasters.html.template
===================================================================
--- trunk/libexec/log-processing/coasters.html.template (rev 0)
+++ trunk/libexec/log-processing/coasters.html.template 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,30 @@
+<html>
+<head><title>Swift log analysis</title></head>
+
+<body><h1>Swift log analysis</h1>
+
+include(`toolbar.html.template')
+
+<h2>Coasters</h2>
+<h3>The block timeline</h3>
+<p>This shows how many workers were requested and how many workers were running at each time:</p>
+<img src="coaster-block-timeline.png" />
+
+<h3>Queue wait times</h3>
+<p>How much a block waited in the queue based on it's total size (CPU seconds):</p>
+<img src="coaster-qwait-size.png" />
+
+<p>How much a block waited in the queue based on the number of workers:</p>
+<img src="coaster-qwait-count.png" />
+
+<p>How much a block waited in the queue based on it's walltime:</p>
+<img src="coaster-qwait-wtime.png" />
+
+<h3>Worker utilization</h3>
+<p>Sorted block utilization (average (time spent running jobs / total time) for each worker in the block):</p>
+<img src="coaster-block-utilization.png" />
+<p>Utilization vs. Number of workers:</p>
+<img src="coaster-block-utilization-vs-count.png" />
+
+<p>end</p>
+</body>
Added: trunk/libexec/log-processing/extract-block-utilization-vs-count
===================================================================
--- trunk/libexec/log-processing/extract-block-utilization-vs-count (rev 0)
+++ trunk/libexec/log-processing/extract-block-utilization-vs-count 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+LOG=$1
+
+rm -f coaster-block-utilization-vs-count.data
+
+IFS=$'\n'
+for LINE in $(cat coasters.tmp); do
+ ID=`echo $LINE | sed 's/^.*id=\([^ ]*\)\(,.*$\|$\)/\1/'`
+ U=`echo $LINE | sed -n 's/^.*BLOCK_UTILIZATION.*u=\([^ ]*\)$/\1/p'`
+ if [ "$U" != "" ]; then
+ COUNT=`cat block-count-table.tmp | grep "$ID" | cut -d , -f 2`
+ echo "$COUNT $U" >> coaster-block-utilization-vs-count.data
+ fi
+done
Property changes on: trunk/libexec/log-processing/extract-block-utilization-vs-count
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/libexec/log-processing/extract-coaster-block-utilization
===================================================================
--- trunk/libexec/log-processing/extract-coaster-block-utilization (rev 0)
+++ trunk/libexec/log-processing/extract-coaster-block-utilization 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+LOG=$1
+
+rm -f coaster-block-utilization.data
+
+cat coasters.tmp | sed -n 's/^.*BLOCK_UTILIZATION.*u=\([^ ]*\)$/\1/p' |sort > coaster-block-utilization.data
Property changes on: trunk/libexec/log-processing/extract-coaster-block-utilization
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/libexec/log-processing/extract-coaster-qwait
===================================================================
--- trunk/libexec/log-processing/extract-coaster-qwait (rev 0)
+++ trunk/libexec/log-processing/extract-coaster-qwait 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+LOG=$1
+
+rm -f block-walltime-table.tmp
+rm -f block-start-time-table.tmp
+rm -f coaster-qwait-size.data
+rm -f coaster-qwait-wtime.data
+rm -f coaster-qwait-count.data
+
+IFS=$'\n'
+for LINE in $(cat coasters.tmp); do
+ TIME=`echo $LINE | sed 's/^\([^ ]*\) .*$/\1/' `
+ ID=`echo $LINE | sed 's/^.*id=\([^ ]*\)\(,.*$\|$\)/\1/'`
+ WALLTIME=`echo $LINE | sed -n 's/^.*BLOCK_REQUESTED.*h=\([^ ]*\)$/\1/p'`
+ if [ "$WALLTIME" != "" ]; then
+ echo "$ID,$WALLTIME" >> block-walltime-table.tmp
+ echo "$ID,$TIME" >> block-start-time-table.tmp
+ else
+ if echo $LINE | grep "BLOCK_ACTIVE" >/dev/null; then
+ COUNT=`cat block-count-table.tmp | grep "$ID" | cut -d , -f 2`
+ WALLTIME=`cat block-walltime-table.tmp | grep "$ID" | cut -d , -f 2`
+ STARTTIME=`cat block-start-time-table.tmp | grep "$ID" | cut -d , -f 2`
+ QTIME=`perl -e "print $TIME - $STARTTIME"`
+ SIZE=$(($WALLTIME * $COUNT))
+ echo "$SIZE $QTIME" >> coaster-qwait-size.data
+ echo "$WALLTIME $QTIME" >> coaster-qwait-wtime.data
+ echo "$COUNT $QTIME" >> coaster-qwait-count.data
+ fi
+ fi
+done
Property changes on: trunk/libexec/log-processing/extract-coaster-qwait
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/libexec/log-processing/extract-coaster-timeline
===================================================================
--- trunk/libexec/log-processing/extract-coaster-timeline (rev 0)
+++ trunk/libexec/log-processing/extract-coaster-timeline 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+LOG=$1
+
+grep "RemoteLogHandler BLOCK_.*" $LOG | iso-to-secs | normalise-event-start-time >coasters.tmp
+rm -f block-count-table.tmp
+rm -f coaster-block.data
+
+IFS=$'\n'
+REQUESTED=0
+RUNNING=0
+for LINE in $(cat coasters.tmp); do
+ TIME=`echo $LINE | sed 's/^\([^ ]*\) .*$/\1/' `
+ ID=`echo $LINE | sed 's/^.*id=\([^ ]*\)\(,.*$\|$\)/\1/'`
+ ADD=`echo $LINE | sed -n 's/^.*BLOCK_REQUESTED.*w=\([^ ]*\),.*$/\1/p'`
+ if [ "$ADD" != "" ]; then
+ REQUESTED=$(($REQUESTED + $ADD))
+ echo "$ID,$ADD" >> block-count-table.tmp
+ else
+ COUNT=`cat block-count-table.tmp | grep "$ID" | cut -d , -f 2`
+ if echo $LINE | grep "BLOCK_ACTIVE" >/dev/null; then
+ RUNNING=$(($RUNNING + $COUNT))
+ fi
+ if echo $LINE | grep "BLOCK_SHUTDOWN" >/dev/null; then
+ RUNNING=$(($RUNNING - $COUNT))
+ REQUESTED=$(($REQUESTED - $COUNT))
+ fi
+ fi
+ echo $TIME $REQUESTED $RUNNING >> coaster-blocks.data
+done
Property changes on: trunk/libexec/log-processing/extract-coaster-timeline
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/libexec/log-processing/extract-coaster-utilization
===================================================================
--- trunk/libexec/log-processing/extract-coaster-utilization (rev 0)
+++ trunk/libexec/log-processing/extract-coaster-utilization 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+LOG=$1
+
+rm -f block-utilization.data
+
+IFS=$'\n'
+for LINE in $(cat coasters.tmp); do
+ TIME=`echo $LINE | sed 's/^\([^ ]*\) .*$/\1/' `
+ ID=`echo $LINE | sed 's/^.*id=\([^ ]*\)\(,.*$\|$\)/\1/'`
+ WALLTIME=`echo $LINE | sed -n 's/^.*BLOCK_REQUESTED.*h=\([^ ]*\)$/\1/p'`
+ if [ "$WALLTIME" != "" ]; then
+ echo "$ID,$WALLTIME" >> block-walltime-hash.tmp
+ echo "$ID,$TIME" >> block-start-time-hash.tmp
+ else
+ if echo $LINE | grep "BLOCK_ACTIVE" >/dev/null; then
+ COUNT=`cat block-count-table.tmp | grep "$ID" | cut -d , -f 2`
+ WALLTIME=`cat block-walltime-table.tmp | grep "$ID" | cut -d , -f 2`
+ STARTTIME=`cat block-start-time-table.tmp | grep "$ID" | cut -d , -f 2`
+ QTIME=`perl -e "print $TIME - $STARTTIME"`
+ SIZE=$(($WALLTIME * $COUNT))
+ echo "$SIZE $QTIME" >> coaster-qwait-size-dist.data
+ echo "$WALLTIME $QTIME" >> coaster-qwait-wtime-dist.data
+ echo "$COUNT $QTIME" >> coaster-qwait-count-dist.data
+ fi
+ fi
+done
Property changes on: trunk/libexec/log-processing/extract-coaster-utilization
___________________________________________________________________
Name: svn:executable
+ *
Modified: trunk/libexec/log-processing/falkon.html.template
===================================================================
--- trunk/libexec/log-processing/falkon.html.template 2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/falkon.html.template 2009-10-15 19:54:45 UTC (rev 3164)
@@ -8,13 +8,13 @@
<h2>falkon</h2>
<p>Falkon taskperf log:</p>
-<img src="falkon.png"/>
+<img src="falkon.png" alt="-"/>
<p>Total jobs that falkon has queued or running:</p>
-<img src="falkon-total.png"/>
+<img src="falkon-total.png" alt="-"/>
<p>Falkon job status trails:</p>
-<img src="falkon-task-status-trails.png"/>
+<img src="falkon-task-status-trails.png" alt="-"/>
<p>end</p>
</body>
Modified: trunk/libexec/log-processing/info.html.template
===================================================================
--- trunk/libexec/log-processing/info.html.template 2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/info.html.template 2009-10-15 19:54:45 UTC (rev 3164)
@@ -9,27 +9,27 @@
<h2>worker.sh info files</h2>
<p>info file plots:</p>
<p>info files:</p>
-<img src="info.png" />
+<img src="info.png" alt="-"/>
<p>info files sorted by start time:</p>
-<img src="info.sorted-start.png" />
+<img src="info.sorted-start.png" alt="-"/>
<p>info trails:</p>
-<img src="info-trails.png" />
+<img src="info-trails.png" alt="-"/>
<p>info duration histogram:</p>
-<img src="info-duration-histogram.png" />
+<img src="info-duration-histogram.png" alt="-"/>
<p>how wrapper.sh is spending its time:</p>
-<img src="info.zeroed-trailsx.png" />
+<img src="info.zeroed-trailsx.png" alt="-"/>
<p>total concurrent wrapper+application execution on worker nodes</p>
-<img src="info-total.png" />
+<img src="info-total.png" alt="-"/>
<hr />
<p>Offsets between job submission Active events and start times reported
by info.</p>
-<img src="info-and-karajan-actives.png" />
+<img src="info-and-karajan-actives.png" alt="-"/>
<br/>
-<img src="info-and-karajan-actives.2.png" />
+<img src="info-and-karajan-actives.2.png" alt="-"/>
<p>end</p>
</body>
Modified: trunk/libexec/log-processing/karajan.html.template
===================================================================
--- trunk/libexec/log-processing/karajan.html.template 2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/karajan.html.template 2009-10-15 19:54:45 UTC (rev 3164)
@@ -106,15 +106,15 @@
<hr />
<h2>job clusters</h2>
-1. <img src="karatasks.clusters.png" />
+1. <img src="karatasks.clusters.png" alt="-"/>
<br />
-2. <img src="karatasks.clusters-total.png" />
+2. <img src="karatasks.clusters-total.png" alt="-"/>
<br />
-3. <img src="karatasks.clusters.sorted-start.png" />
+3. <img src="karatasks.clusters.sorted-start.png" alt="-"/>
<br />
-4. <img src="karatasks.clusters-duration-histogram.png" />
+4. <img src="karatasks.clusters-duration-histogram.png" alt="-"/>
<br />
-5. <img src="cluster-jobcount-histogram.png" />
+5. <img src="cluster-jobcount-histogram.png" alt="-"/>
<hr />
<p>end</p>
Modified: trunk/libexec/log-processing/kickstart.html.template
===================================================================
--- trunk/libexec/log-processing/kickstart.html.template 2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/kickstart.html.template 2009-10-15 19:54:45 UTC (rev 3164)
@@ -9,14 +9,14 @@
<p>kickstart records.
(<a href="kickstart.event">text file of events</a>)
this shows applications which are actually running on a worker node, executing:</p>
-<img src="kickstart.png" />
+<img src="kickstart.png" alt="-"/>
<p>kickstart records total - this represents how many worker nodes were
actually executing applications at any time:</p>
-<img src="kickstart-total.png" />
+<img src="kickstart-total.png" alt="-"/>
<p>kickstart records sorted by start time:</p>
-<img src="kickstart.sorted-start.png" />
+<img src="kickstart.sorted-start.png" alt="-"/>
<p>kickstart stats:
<pre>
@@ -26,7 +26,7 @@
<p>cumulative histogram(?) of kickstart job durations</p>
-<img src="kickstart-duration-histogram.png" />
+<img src="kickstart-duration-histogram.png" alt="-"/>
<p>end</p>
</body>
Modified: trunk/libexec/log-processing/makefile
===================================================================
--- trunk/libexec/log-processing/makefile 2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/makefile 2009-10-15 19:54:45 UTC (rev 3164)
@@ -8,6 +8,7 @@
include makefile.webpage
include makefile.kickstart
include makefile.errors
+include makefile.coasters
distributable:
rm -f *.tmp log *.transitions tmp-*
Added: trunk/libexec/log-processing/makefile.coasters
===================================================================
--- trunk/libexec/log-processing/makefile.coasters (rev 0)
+++ trunk/libexec/log-processing/makefile.coasters 2009-10-15 19:54:45 UTC (rev 3164)
@@ -0,0 +1,41 @@
+coaster-block-timeline.png: coaster-block-timeline.data
+ gnuplot $(SWIFT_PLOT_HOME)/coaster-block-timeline.plot
+
+extract-timeline:
+ extract-coaster-timeline $(LOG)
+
+coaster-block-timeline.data: extract-timeline
+coasters.tmp: extract-timeline
+block-count-table.tmp: extract-timeline
+
+extract-coaster-qwait: coasters.tmp
+ extract-coaster-qwait $(LOG)
+
+coaster-qwait-count.data: extract-coaster-qwait
+coaster-qwait-size.data: extract-coaster-qwait
+coaster-qwait-wtime.data: extract-coaster-qwait
+coaster-block-utilization-vs-count.data: extract-block-utilization-vs-count
+
+extract-block-utilization: coasters.tmp
+ extract-coaster-block-utilization $(LOG)
+
+extract-block-utilization-vs-count: coasters.tmp
+ extract-block-utilization-vs-count $(LOG)
+
+coaster-block-utilization.data:
+ extract-coaster-block-utilization
+
+coaster-qwait-count.png: coaster-qwait-count.data
+ gnuplot $(SWIFT_PLOT_HOME)/coaster-qwait-count.plot
+
+coaster-qwait-size.png: coaster-qwait-size.data
+ gnuplot $(SWIFT_PLOT_HOME)/coaster-qwait-size.plot
+
+coaster-qwait-wtime.png: coaster-qwait-wtime.data
+ gnuplot $(SWIFT_PLOT_HOME)/coaster-qwait-wtime.plot
+
+coaster-block-utilization.png: coaster-block-utilization.data
+ gnuplot $(SWIFT_PLOT_HOME)/coaster-block-utilization.plot
+
+coaster-block-utilization-vs-count.png: coaster-block-utilization-vs-count.data
+ gnuplot $(SWIFT_PLOT_HOME)/coaster-block-utilization-vs-count.plot
Modified: trunk/libexec/log-processing/makefile.webpage
===================================================================
--- trunk/libexec/log-processing/makefile.webpage 2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/makefile.webpage 2009-10-15 19:54:45 UTC (rev 3164)
@@ -1,7 +1,7 @@
webpage: pngs htmls
htmls: index.html execute2.html execute.html karajan.html info.html \
-assorted.html kickstart.html falkon.html scheduler.html
+assorted.html kickstart.html falkon.html scheduler.html coasters.html
webpage.kara: karatasks.png karatasks-coloured.png karatasks-type-counts.txt \
karatasks.lastsummary karatasks-coloured-sorted.png \
@@ -22,6 +22,9 @@
karatasks.JOB_SUBMISSION.Queue.stats \
karatasks.JOB_SUBMISSION.Queue-duration-histogram.png
+webpage.coasters: coaster-block-timeline.png coaster-qwait-count.png coaster-qwait-size.png \
+ coaster-qwait-wtime.png coaster-block-utilization.png coaster-block-utilization-vs-count.png
+
webpage.clusters: karatasks.clusters.png karatasks.clusters-total.png \
karatasks.clusters.sorted-start.png karatasks.clusters-duration-histogram.png
Modified: trunk/libexec/log-processing/toolbar.html.template
===================================================================
--- trunk/libexec/log-processing/toolbar.html.template 2009-10-15 19:51:11 UTC (rev 3163)
+++ trunk/libexec/log-processing/toolbar.html.template 2009-10-15 19:54:45 UTC (rev 3164)
@@ -7,6 +7,7 @@
<a href="kickstart.html">kickstart</a> |
<a href="info.html">info</a> |
<a href="falkon.html">falkon</a> |
+<a href="coasters.html">coasters</a> |
<a href="assorted.html">assorted swift library routines</a>
| Other:
<a href="scheduler.html">scheduler</a>
More information about the Swift-commit
mailing list