[Swift-commit] r6986 - in SwiftTutorials/swift-cray-tutorial: . bin doc part04 part05 part06
davidk at ci.uchicago.edu
davidk at ci.uchicago.edu
Fri Aug 23 16:17:15 CDT 2013
Author: davidk
Date: 2013-08-23 16:17:15 -0500 (Fri, 23 Aug 2013)
New Revision: 6986
Added:
SwiftTutorials/swift-cray-tutorial/bin/plot.sh
SwiftTutorials/swift-cray-tutorial/part04/plot.sh
SwiftTutorials/swift-cray-tutorial/part05/plot.sh
SwiftTutorials/swift-cray-tutorial/part06/plot.sh
Modified:
SwiftTutorials/swift-cray-tutorial/doc/README
SwiftTutorials/swift-cray-tutorial/doc/build_docs.sh
SwiftTutorials/swift-cray-tutorial/setup.sh
Log:
Info on scaling up
Plotting
Added: SwiftTutorials/swift-cray-tutorial/bin/plot.sh
===================================================================
--- SwiftTutorials/swift-cray-tutorial/bin/plot.sh (rev 0)
+++ SwiftTutorials/swift-cray-tutorial/bin/plot.sh 2013-08-23 21:17:15 UTC (rev 6986)
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+#usage: ./plotswiftlogs.ketan <swift-logfile.log>
+
+SWIFTLOGFILE=$1
+
+#TMPDIR=`mktemp -d plotlog.XXX`
+
+grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
+
+SWIFTOUTFILE=swiftoutfile.out
+
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+
+#extract active runs in a file
+(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt)
+
+#extract successful completions in a file
+(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+if [ $activelines -ne 0 ]
+then
+ activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+fi
+
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+if [ $cumulines -ne 0 ]
+then
+ cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+fi
+
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+cat << EOF1 > plotit.gp
+set terminal png enhanced
+set nokey
+set output "cumulativeplot.png"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+set output "activeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of active jobs"
+set title "Active jobs"
+plot "plot_active.txt" using 1:2 with line
+EOF1
+
+gnuplot plotit.gp 2>/dev/null
+rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt
Property changes on: SwiftTutorials/swift-cray-tutorial/bin/plot.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftTutorials/swift-cray-tutorial/doc/README
===================================================================
--- SwiftTutorials/swift-cray-tutorial/doc/README 2013-08-23 20:45:04 UTC (rev 6985)
+++ SwiftTutorials/swift-cray-tutorial/doc/README 2013-08-23 21:17:15 UTC (rev 6986)
@@ -315,6 +315,36 @@
$ swift p6.swift -steps=3 # each simulation takes 3 seconds
----
+Larger runs
+~~~~~~~~~~~
+To test with larger runs, there are two changes that are required. The first is a
+change to the command line arguments. The example below will run 1000 simulations
+with each simulation taking 5 seconds.
+
+-----
+$ swift p6.swift -steps=5 -nsims=1000
+-----
+
+The other change required is to the sites.xml to control how many nodes you request.
+Change the "maxnodes" values to control the number of nodes to request.
+
+-----
+<profile namespace="globus" key="maxNodes">10</profile>
+-----
+
+Plotting
+--------
+Each part directory contains a file called plot.sh that can be used for plotting.
+It generates two image files: cumulativeplot.png and activeplot.png.
+cumulativeplot.png shows overall progress over time, while activeplot.png shows
+the number of active jobs over time. When the worker agents are trasnferring data
+to and from nodes, this time does not get marked as active.
+
+After each swift run, a log file will be created called partNN-<YYYYmmdd>-<hhmm>-<random>.log.
+Once you have identified the log file name, run the following command to generate the plots:
+
+$ ./plot.sh <logfile>
+
Running Swift scripts on Cloud resources
----------------------------------------
Modified: SwiftTutorials/swift-cray-tutorial/doc/build_docs.sh
===================================================================
--- SwiftTutorials/swift-cray-tutorial/doc/build_docs.sh 2013-08-23 20:45:04 UTC (rev 6985)
+++ SwiftTutorials/swift-cray-tutorial/doc/build_docs.sh 2013-08-23 21:17:15 UTC (rev 6986)
@@ -1,14 +1,4 @@
#!/bin/bash -e
-if false; then
- echo skipping
-pushd part11-swift-py-r/code >& /dev/null
-# Strip comments, blank lines; prepend shell prompt ($)
-grep -A 20 stc run-dets.sh | \
- grep -v -e "^$\|#" | \
- sed 's/^/$ /' > run-dets.sh.txt
-popd >& /dev/null
-fi
+asciidoc -a icons -a toc -a toplevels=2 -a stylesheet=$PWD/asciidoc.css -a max-width=800px -o cray-tutorial.html README
-asciidoc -a toc -a toplevels=2 -a stylesheet=$PWD/asciidoc.css -a max-width=800px -o cic-tutorial.html README
-
Added: SwiftTutorials/swift-cray-tutorial/part04/plot.sh
===================================================================
--- SwiftTutorials/swift-cray-tutorial/part04/plot.sh (rev 0)
+++ SwiftTutorials/swift-cray-tutorial/part04/plot.sh 2013-08-23 21:17:15 UTC (rev 6986)
@@ -0,0 +1 @@
+link ../bin/plot.sh
\ No newline at end of file
Property changes on: SwiftTutorials/swift-cray-tutorial/part04/plot.sh
___________________________________________________________________
Added: svn:special
+ *
Added: SwiftTutorials/swift-cray-tutorial/part05/plot.sh
===================================================================
--- SwiftTutorials/swift-cray-tutorial/part05/plot.sh (rev 0)
+++ SwiftTutorials/swift-cray-tutorial/part05/plot.sh 2013-08-23 21:17:15 UTC (rev 6986)
@@ -0,0 +1 @@
+link ../bin/plot.sh
\ No newline at end of file
Property changes on: SwiftTutorials/swift-cray-tutorial/part05/plot.sh
___________________________________________________________________
Added: svn:special
+ *
Added: SwiftTutorials/swift-cray-tutorial/part06/plot.sh
===================================================================
--- SwiftTutorials/swift-cray-tutorial/part06/plot.sh (rev 0)
+++ SwiftTutorials/swift-cray-tutorial/part06/plot.sh 2013-08-23 21:17:15 UTC (rev 6986)
@@ -0,0 +1 @@
+link ../bin/plot.sh
\ No newline at end of file
Property changes on: SwiftTutorials/swift-cray-tutorial/part06/plot.sh
___________________________________________________________________
Added: svn:special
+ *
Modified: SwiftTutorials/swift-cray-tutorial/setup.sh
===================================================================
--- SwiftTutorials/swift-cray-tutorial/setup.sh 2013-08-23 20:45:04 UTC (rev 6985)
+++ SwiftTutorials/swift-cray-tutorial/setup.sh 2013-08-23 21:17:15 UTC (rev 6986)
@@ -19,7 +19,7 @@
echo Using $PATHSWIFT from PATH
else
echo ERROR: $TUTSWIFT not found and no swift in PATH. Tutorial will not function.
- return
+# return
fi
echo Swift version is $(swift -version)
More information about the Swift-commit
mailing list