[Swift-commit] r7173 - in SwiftTutorials/OSDC-2013-10-17: . app bin doc doc/figs doc/images doc/images/icons part01 part02 part03 part04 part05 part06
davidk at ci.uchicago.edu
davidk at ci.uchicago.edu
Thu Oct 17 13:29:33 CDT 2013
Author: davidk
Date: 2013-10-17 13:29:32 -0500 (Thu, 17 Oct 2013)
New Revision: 7173
Added:
SwiftTutorials/OSDC-2013-10-17/README
SwiftTutorials/OSDC-2013-10-17/app/
SwiftTutorials/OSDC-2013-10-17/app/simulate.sh
SwiftTutorials/OSDC-2013-10-17/app/stats.sh
SwiftTutorials/OSDC-2013-10-17/bin/
SwiftTutorials/OSDC-2013-10-17/bin/cleanup
SwiftTutorials/OSDC-2013-10-17/bin/hosts
SwiftTutorials/OSDC-2013-10-17/bin/package_tutorial.sh
SwiftTutorials/OSDC-2013-10-17/bin/plot.sh
SwiftTutorials/OSDC-2013-10-17/doc/
SwiftTutorials/OSDC-2013-10-17/doc/README
SwiftTutorials/OSDC-2013-10-17/doc/TODO
SwiftTutorials/OSDC-2013-10-17/doc/activeplot.png
SwiftTutorials/OSDC-2013-10-17/doc/asciidoc.css
SwiftTutorials/OSDC-2013-10-17/doc/build_docs.sh
SwiftTutorials/OSDC-2013-10-17/doc/cumulativeplot.png
SwiftTutorials/OSDC-2013-10-17/doc/figs/
SwiftTutorials/OSDC-2013-10-17/doc/figs/modis.dia
SwiftTutorials/OSDC-2013-10-17/doc/figs/modis.png
SwiftTutorials/OSDC-2013-10-17/doc/images/
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/caution.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/example.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/home.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/important.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/next.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/note.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/prev.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/tip.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/up.png
SwiftTutorials/OSDC-2013-10-17/doc/images/icons/warning.png
SwiftTutorials/OSDC-2013-10-17/doc/part01.png
SwiftTutorials/OSDC-2013-10-17/doc/part02.png
SwiftTutorials/OSDC-2013-10-17/doc/part03.png
SwiftTutorials/OSDC-2013-10-17/doc/part04.png
SwiftTutorials/OSDC-2013-10-17/doc/part05.png
SwiftTutorials/OSDC-2013-10-17/doc/part06.png
SwiftTutorials/OSDC-2013-10-17/doc/push.sh
SwiftTutorials/OSDC-2013-10-17/doc/raven.1000.progress.out
SwiftTutorials/OSDC-2013-10-17/doc/raven.10000.progress.out
SwiftTutorials/OSDC-2013-10-17/part01/
SwiftTutorials/OSDC-2013-10-17/part01/apps
SwiftTutorials/OSDC-2013-10-17/part01/p1.swift
SwiftTutorials/OSDC-2013-10-17/part01/sites.xml
SwiftTutorials/OSDC-2013-10-17/part02/
SwiftTutorials/OSDC-2013-10-17/part02/apps
SwiftTutorials/OSDC-2013-10-17/part02/p2.swift
SwiftTutorials/OSDC-2013-10-17/part02/sites.xml
SwiftTutorials/OSDC-2013-10-17/part03/
SwiftTutorials/OSDC-2013-10-17/part03/apps
SwiftTutorials/OSDC-2013-10-17/part03/p3.kml
SwiftTutorials/OSDC-2013-10-17/part03/p3.swift
SwiftTutorials/OSDC-2013-10-17/part03/sites.xml
SwiftTutorials/OSDC-2013-10-17/part04/
SwiftTutorials/OSDC-2013-10-17/part04/apps
SwiftTutorials/OSDC-2013-10-17/part04/p4.swift
SwiftTutorials/OSDC-2013-10-17/part05/
SwiftTutorials/OSDC-2013-10-17/part05/OLD.p5.swift
SwiftTutorials/OSDC-2013-10-17/part05/OLDbias.dat
SwiftTutorials/OSDC-2013-10-17/part05/apps
SwiftTutorials/OSDC-2013-10-17/part05/multipools
SwiftTutorials/OSDC-2013-10-17/part05/p5.swift
SwiftTutorials/OSDC-2013-10-17/part06/
SwiftTutorials/OSDC-2013-10-17/part06/apps
SwiftTutorials/OSDC-2013-10-17/part06/p6.swift
SwiftTutorials/OSDC-2013-10-17/setup.sh
Log:
Start of OSDC tutorial scripts
Added: SwiftTutorials/OSDC-2013-10-17/README
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/README (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/README 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1 @@
+link doc/README
\ No newline at end of file
Property changes on: SwiftTutorials/OSDC-2013-10-17/README
___________________________________________________________________
Added: svn:special
+ *
Added: SwiftTutorials/OSDC-2013-10-17/app/simulate.sh
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/app/simulate.sh (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/app/simulate.sh 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,134 @@
+#! /bin/bash
+
+printparams()
+{
+ printf "\nSimulation parameters:\n\n"
+ echo bias=$bias
+ echo biasfile=$biasfile
+ echo initseed=$initseed
+ echo log=$log
+ echo paramfile=$paramfile
+ echo range=$range
+ echo scale=$scale
+ echo seedfile=$seedfile
+ echo timesteps=$timesteps
+ echo output width=$width
+}
+
+log() {
+ printf "\nCalled as: $0: $cmdargs\n\n"
+ printf "Start time: "; /bin/date
+ printf "Running on node: "; /bin/hostname
+ printf "Running as user: "; /usr/bin/id
+ printparams
+ printf "\nEnvironment:\n\n"
+ printenv | sort
+}
+
+addsims() {
+ while read f1 ; do
+ read -u 3 f2
+ if [ _$f1 = _ ]; then f1=$lastf1; fi
+ if [ _$f2 = _ ]; then f2=$lastf2; fi
+ printf "%${width}d\n" $(($f1+$f2))
+ lastf1=$f1
+ lastf2=$f2
+ done <$1 3<$2
+}
+
+# set defaults
+
+bias=0
+biasfile=none
+initseed=none
+log=yes
+paramfile=none
+range=100
+scale=1
+seedfile=none
+timesteps=0
+nvalues=1
+width=8
+cmdargs="$*"
+
+usage()
+{
+ echo $0: usage:
+ cat <<END
+ -b|--bias offset bias: add this integer to all results
+ -B|--biasfile file of integer biases to add to results
+ -l|--log generate a log in stderr if not null
+ -n|--nvalues print this many values per simulation
+ -r|--range range (limit) of generated results
+ -s|--seed use this integer [0..32767] as a seed
+ -S|--seedfile use this file (containing integer seeds [0..32767]) one per line
+ -t|--timesteps number of simulated "timesteps" in seconds (determines runtime)
+ -x|--scale scale the results by this integer
+ -h|-?|?|--help print this help
+END
+}
+
+# FIXME: NOT YET IMPLEMENTED:
+# -p|--paramfile take these parameters (in form param=value) from this file
+# -p|--paramfile) paramfile=$2 ;;
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ -b|--bias) bias=$2 ;;
+ -B|--biasfile) biasfile=$2 ;;
+ -l|--log) log=$2 ;;
+ -n|--nvalues) nvalues=$2 ;;
+ -s|--seed) initseed=$2 ;;
+ -S|--seedfile) seedfile=$2 ;;
+ -t|--timesteps) timesteps=$2 ;;
+ -r|--range) range=$2 ;;
+ -w|--width) width=$2 ;;
+ -x|--scale) scale=$2 ;;
+ -h|-?|--help|*) usage; exit ;;
+ esac
+ shift 2
+done
+
+# process initial seed
+
+if [ $initseed != none ]; then
+ RANDOM=$initseed
+fi
+
+# process file of seeds
+
+if [ $seedfile != none ]; then
+ seed=0
+ while read $seedfile s; do
+ seed=$(($seed+$s))
+ done <$seedfile
+ RANDOM=$seed
+fi
+
+# run for some number of "timesteps"
+
+sleep $timesteps
+
+# emit N (nvalues) "simulation results" scaled and biased by argument values
+
+simout=$(mktemp simout.XXXX)
+for ((i=0;i<nvalues;i++)); do
+ # value=$(( (($RANDOM)*(2**16))+$RANDOM ))
+ value=$(( (($RANDOM)*(2**48)) + (($RANDOM)*(2**32)) + (($RANDOM)*(2**16)) + $RANDOM ))
+ printf "%${width}d\n" $(( ($value%range)*scale+bias))
+done >$simout
+
+# process file of biases
+
+if [ $biasfile != none ]; then
+ addsims $simout $biasfile
+else
+ cat $simout
+fi
+rm $simout
+
+# log environmental data
+
+if [ $log != off ]; then
+ log 1>&2
+fi
Property changes on: SwiftTutorials/OSDC-2013-10-17/app/simulate.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftTutorials/OSDC-2013-10-17/app/stats.sh
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/app/stats.sh (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/app/stats.sh 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,18 @@
+#! /bin/sh
+
+log() {
+ printf "\nCalled as: $0: $cmdargs\n\n"
+ printf "Start time: "; /bin/date
+ printf "Running on node: "; /bin/hostname
+ printf "Running as user: "; /usr/bin/id
+ printf "\nEnvironment:\n\n"
+ printenv | sort
+}
+
+awk '
+
+{ sum += $1}
+
+END { printf("%d\n",sum/NR) }
+' $*
+log 1>&2
Property changes on: SwiftTutorials/OSDC-2013-10-17/app/stats.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftTutorials/OSDC-2013-10-17/bin/cleanup
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/bin/cleanup (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/bin/cleanup 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,2 @@
+rm -rf *.log *.rlog *.d p?-*-*-* *.kml *.swiftx *.out output outdir logs hi.* _concurrent .swift/tmp
+
Property changes on: SwiftTutorials/OSDC-2013-10-17/bin/cleanup
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftTutorials/OSDC-2013-10-17/bin/hosts
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/bin/hosts (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/bin/hosts 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1 @@
+for h in $(cat hosts.txt); do ssh $h hostname -f; done
Property changes on: SwiftTutorials/OSDC-2013-10-17/bin/hosts
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftTutorials/OSDC-2013-10-17/bin/package_tutorial.sh
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/bin/package_tutorial.sh (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/bin/package_tutorial.sh 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,15 @@
+#! /bin/sh
+
+BIN=$(cd $(dirname $0); pwd)
+BASE=$(cd $BIN/..; pwd)
+LOC=$(cd $BASE/..; pwd)
+
+echo executing $0 from BIN=$BIN
+echo making tutorial package of BASE=$BASE
+echo placing tutorial package in LOC=$LOC
+
+(
+ cd $LOC
+ tar zcf swift-cray-tutorial.tgz --exclude-vcs swift-cray-tutorial
+ scp swift-cray-tutorial.tgz p01532 at raven.cray.com:
+)
Property changes on: SwiftTutorials/OSDC-2013-10-17/bin/package_tutorial.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftTutorials/OSDC-2013-10-17/bin/plot.sh
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/bin/plot.sh (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/bin/plot.sh 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#usage: ./plotswiftlogs.ketan <swift-logfile.log>
+
+SWIFTLOGFILE=$1
+
+#TMPDIR=`mktemp -d plotlog.XXX`
+
+grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
+
+SWIFTOUTFILE=swiftoutfile.out
+
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+
+#extract active runs in a file
+(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt)
+
+#extract successful completions in a file
+(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+if [ $activelines -ne 0 ]
+then
+ activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+fi
+
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+if [ $cumulines -ne 0 ]
+then
+ cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+fi
+
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+cat << EOF1 > plotit.gp
+set terminal png enhanced
+set nokey
+set output "cumulativeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of completed apps"
+set title "Cumulative Apps Completed"
+plot "plot_cumulative.txt" using 1:2 with lines
+set output "activeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of active apps"
+set title "Active Apps"
+plot "plot_active.txt" using 1:2 with line
+EOF1
+
+gnuplot plotit.gp 2>/dev/null
+rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt
Property changes on: SwiftTutorials/OSDC-2013-10-17/bin/plot.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftTutorials/OSDC-2013-10-17/doc/README
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/doc/README (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/doc/README 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,1039 @@
+Swift Cray Tutorial
+===================
+
+////
+
+This is the asciidoc input file.
+Its content is viewable as a plain-text README file.
+
+////
+
+This tutorial is viewable at:
+http://swift-lang.org/tutorials/cray/swift-cray-tutorial.html
+
+////
+
+Tutorial Outline:
+
+Introductory example, running apps locally on login node:
+
+ p1 - Run an application under Swift
+ p2 - Parallel loops with foreach
+ p3 - Merging/reducing the results of a parallel foreach loop
+
+Compute-node exercises, running apps via qsub and aprun:
+
+ p4 - Running apps on Cray compute nodes
+ p5 - Running on multiple pools of compute nodes
+ p6 - Running a more complex workflow pattern
+
+////
+
+Introduction: Why Parallel Scripting?
+------------------------------------
+
+Swift is a simple scripting language for executing many instances of
+ordinary application programs on distributed parallel resources.
+Swift scripts run many copies of ordinary programs concurrently, using
+statements like this:
+-----
+foreach protein in proteinList {
+ runBLAST(protein);
+}
+-----
+Swift acts like a structured "shell" language. It runs programs
+concurrently as soon as their inputs are available, reducing the need
+for complex parallel programming. Swift expresses your workflow in a
+portable fashion: The same script runs on multicore computers,
+clusters, clouds, grids, and supercomputers.
+
+In this tutorial, you'll be able to first try a few Swift examples
+(parts 1-3) on the Cray login host, to get a sense of the
+language. Then in parts 4-6 you'll run similar workflows on Cray
+compute nodes, and see how more complex workflows can be expressed
+with Swift scripts.
+
+Swift tutorial setup
+--------------------
+
+To install the tutorial scripts on Cray "Raven" XE6-XK7 test system,
+do:
+
+-----
+$ cd $HOME
+$ tar xzf /home/users/p01532/swift-cray-tutorial.tgz
+$ cd swift-cray-tutorial
+$ source setup.sh # You must run this with "source" !
+-----
+
+Verify your environment
+~~~~~~~~~~~~~~~~~~~~~~~
+
+To verify that Swift (and the Java environment it requires) are working, do:
+
+-----
+$ java -version # verify that you have Java (ideally Oracle JAVA 1.6 or later)
+$ swift -version # verify that you have Swift 0.94.1
+-----
+
+NOTE: If you re-login or open new ssh sessions, you must re-run `source setup.sh` in each ssh shell/window.
+
+To check out the tutorial scripts from SVN
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you later want to get the most recent version of this tutorial from
+the Swift Subversion repository, do:
+
+-----
+$ svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftTutorials/swift-cray-tutorial Cray-Swift
+-----
+
+This will create a directory called "Cray-Swift" which contains all of the
+files used in this tutorial.
+
+
+Simple "science applications" for the workflow tutorial
+-------------------------------------------------------
+
+This tutorial is based on two intentionally trivial example programs,
+`simulation.sh` and `stats.sh`, (implemented as bash shell scripts)
+that serve as easy-to-understand proxies for real science
+applications. These "programs" behave as follows.
+
+simulate.sh
+~~~~~~~~~~~
+
+The simulation.sh script serves as a trivial proxy for any more
+complex scientific simulation application. It generates and prints a
+set of one or more random integers in the range [0-2^62) as controlled
+by its command line arguments, which are:
+
+-----
+$ ./app/simulate.sh --help
+./app/simulate.sh: usage:
+ -b|--bias offset bias: add this integer to all results [0]
+ -B|--biasfile file of integer biases to add to results [none]
+ -l|--log generate a log in stderr if not null [y]
+ -n|--nvalues print this many values per simulation [1]
+ -r|--range range (limit) of generated results [100]
+ -s|--seed use this integer [0..32767] as a seed [none]
+ -S|--seedfile use this file (containing integer seeds [0..32767]) one per line [none]
+ -t|--timesteps number of simulated "timesteps" in seconds (determines runtime) [1]
+ -x|--scale scale the results by this integer [1]
+ -h|-?|?|--help print this help
+$
+-----
+
+All of thess arguments are optional, with default values indicated above as `[n]`.
+
+////
+.simulation.sh arguments
+[width="80%",cols="^2,10",options="header"]
+
+|=======================
+|Argument|Short|Description
+|1 |runtime: sets run time of simulation.sh in seconds
+|2 |range: limits generated values to the range [0,range-1]
+|3 |biasfile: add the integer contained in this file to each value generated
+|4 |scale: multiplies each generated value by this integer
+|5 |count: number of values to generate in the simulation
+|=======================
+////
+
+With no arguments, simulate.sh prints 1 number in the range of
+1-100. Otherwise it generates n numbers of the form (R*scale)+bias
+where R is a random integer. By default it logs information about its
+execution environment to stderr. Here's some examples of its usage:
+
+-----
+$ simulate.sh 2>log
+ 5
+$ head -4 log
+
+Called as: /home/wilde/swift/tut/CIC_2013-08-09/app/simulate.sh:
+Start time: Thu Aug 22 12:40:24 CDT 2013
+Running on node: login01.osgconnect.net
+
+$ simulate.sh -n 4 -r 1000000 2>log
+ 239454
+ 386702
+ 13849
+ 873526
+
+$ simulate.sh -n 3 -r 1000000 -x 100 2>log
+ 6643700
+62182300
+ 5230600
+
+$ simulate.sh -n 2 -r 1000 -x 1000 2>log
+ 565000
+ 636000
+
+$ time simulate.sh -n 2 -r 1000 -x 1000 -t 3 2>log
+ 336000
+ 320000
+real 0m3.012s
+user 0m0.005s
+sys 0m0.006s
+-----
+
+stats.sh
+~~~~~~
+
+The stats.sh script serves as a trivial model of an "analysis"
+program. It reads N files each containing M integers and simply prints
+the\ average of all those numbers to stdout. Similarly to simulate.sh
+it logs environmental information to the stderr.
+
+-----
+$ ls f*
+f1 f2 f3 f4
+
+$ cat f*
+25
+60
+40
+75
+
+$ stats.sh f* 2>log
+50
+-----
+
+
+Basic of the Swift language with local execution
+------------------------------------------------
+
+A Summary of Swift in a nutshell
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Swift scripts are text files ending in `.swift` The `swift` command
+runs on any host, and executes these scripts. `swift` is a Java
+application, which you can install almost anywhere. On Linux, just
+unpack the distribution `tar` file and add its `bin/` directory to
+your `PATH`.
+
+* Swift scripts run ordinary applications, just like shell scripts
+do. Swift makes it easy to run these applications on parallel and
+remote computers (from laptops to supercomputers). If you can `ssh` to
+the system, Swift can likely run applications there.
+
+* The details of where to run applications and how to get files back
+and forth are described in configuration files separate from your
+program. Swift speaks ssh, PBS, Condor, SLURM, LSF, SGE, Cobalt, and
+Globus to run applications, and scp, http, ftp, and GridFTP to move
+data.
+
+* The Swift language has 5 main data types: `boolean`, `int`,
+`string`, `float`, and `file`. Collections of these are dynamic,
+sparse arrays of arbitrary dimension and structures of scalars and/or
+arrays defined by the `type` declaration.
+
+* Swift file variables are "mapped" to external files. Swift sends
+files to and from remote systems for you automatically.
+
+* Swift variables are "single assignment": once you set them you can't
+change them (in a given block of code). This makes Swift a natural,
+"parallel data flow" language. This programming model keeps your
+workflow scripts simple and easy to write and understand.
+
+* Swift lets you define functions to "wrap" application programs, and
+to cleanly structure more complex scripts. Swift `app` functions take
+files and parameters as inputs and return files as outputs.
+
+* A compact set of built-in functions for string and file
+manipulation, type conversions, high level IO, etc. is provided.
+Swift's equivalent of `printf()` is `tracef()`, with limited and
+slightly different format codes.
+
+* Swift's `foreach {}` statement is the main parallel workhorse of the
+language, and executes all iterations of the loop concurrently. The
+actual number of parallel tasks executed is based on available
+resources and settable "throttles".
+
+* In fact, Swift conceptually executes *all* the statements,
+expressions and function calls in your program in parallel, based on
+data flow. These are similarly throttled based on available resources
+and settings.
+
+* Swift also has `if` and `switch` statements for conditional
+execution. These are seldom needed in simple workflows but they enable
+very dynamic workflow patterns to be specified.
+
+We'll see many of these points in action in the examples below. Lets
+get started!
+
+Part 1: Run a single application under Swift
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The first swift script, p1.swift, runs simulate.sh to generate a
+single random number. It writes the number to a file.
+
+image::part01.png["p1 workflow",align="center"]
+
+.p1.swift
+-----
+sys::[cat ../part01/p1.swift]
+-----
+
+
+To run this script, run the following command:
+-----
+$ cd part01
+$ swift p1.swift
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1413-oa6fdib2
+Progress: time: Tue, 27 Aug 2013 14:13:33 -0500
+Final status: Tue, 27 Aug 2013 14:13:33 -0500 Finished successfully:1
+$ cat sim.out
+ 84
+$ swift p1.swift
+$ cat sim.out
+ 36
+-----
+
+To cleanup the directory and remove all outputs (including the log
+files and directories that Swift generates), run the cleanup script
+which is located in the tutorial PATH:
+
+-----
+$ cleanup
+------
+
+NOTE: You'll also find two Swift configuration files in each `partNN`
+directory of this tutorial. These specify the environment-specific
+details of where to find application programs (file `apps`) and where
+to run them (file `sites.xml`). These files will be explained in more
+detail in parts 4-6, and can be ignored for now.
+
+////
+It defines
+things like the work directory, the scheduler to use, and how to
+control parallelism. The sites.xml file below will tell Swift to run
+on the local machine only, and run just 1 task at a time.
+
+.sites.xml
+-----
+sys::[cat ../part01/sites.xml]
+-----
+
+ In this case, it
+indicates that the app "simulate" (the first token in the command line
+declaration of the function `simulation`, at line NNN) is located in the file
+simulate.sh and (since the path `simulate.sh` is specified with no
+directory components) Swift expects that the `simulate.sh` executable
+will be available in your $PATH.
+
+.apps
+-----
+sys::[cat ../part01/apps]
+-----
+
+////
+
+Part 2: Running an ensemble of many apps in parallel with a "foreach" loop
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `p2.swift` script introduces the `foreach` parallel iteration
+construct to run many concurrent simulations.
+
+image::part02.png[align="center"]
+
+.p2.swift
+-----
+sys::[cat ../part02/p2.swift]
+-----
+
+The script also shows an
+example of naming the output files of an ensemble run. In this case, the output files will be named
+`output/sim_N.out`.
+
+To run the script and view the output:
+-----
+$ cd ../part02
+$ swift p2.swift
+$ ls output
+sim_0.out sim_1.out sim_2.out sim_3.out sim_4.out sim_5.out sim_6.out sim_7.out sim_8.out sim_9.out
+$ more output/*
+::::::::::::::
+output/sim_0.out
+::::::::::::::
+ 44
+::::::::::::::
+output/sim_1.out
+::::::::::::::
+ 55
+...
+::::::::::::::
+output/sim_9.out
+::::::::::::::
+ 82
+-----
+
+Part 3: Analyzing results of a parallel ensemble
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+After all the parallel simulations in an ensemble run have completed,
+its typically necessary to gather and analyze their results with some
+kind of post-processing analysis program or script. p3.swift
+introduces such a postprocessing step. In this case, the files created
+by all of the parallel runs of `simulation.sh` will be averaged by by
+the trivial "analysis application" `stats.sh`:
+
+image::part03.png[align="center"]
+
+.p3.swift
+----
+sys::[cat ../part03/p3.swift]
+----
+
+To run:
+----
+$ cd part03
+$ swift p3.swift
+----
+
+Note that in `p3.swift` we expose more of the capabilities of the
+`simulate.sh` application to the `simulation()` app function:
+
+-----
+app (file o) simulation (int sim_steps, int sim_range, int sim_values)
+{
+ simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@filename(o);
+}
+-----
+
+`p3.swift` also shows how to fetch application-specific values from
+the `swift` command line in a Swift script using `@arg()` which
+accepts a keyword-style argument and its default value:
+
+-----
+int nsim = @toInt(@arg("nsim","10"));
+int steps = @toInt(@arg("steps","1"));
+int range = @toInt(@arg("range","100"));
+int values = @toInt(@arg("values","5"));
+-----
+
+Now we can specify that more runs should be performed and that each should run for more timesteps, and produce more that one value each, within a specified range, using command line arguments placed after the Swift script name in the form `-parameterName=value`:
+
+-----
+$ swift p3.swift -nsim=3 -steps=10 -values=4 -range=1000000
+
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1439-s3vvo809
+Progress: time: Tue, 27 Aug 2013 14:39:42 -0500
+Progress: time: Tue, 27 Aug 2013 14:39:53 -0500 Active:2 Stage out:1
+Final status: Tue, 27 Aug 2013 14:39:53 -0500 Finished successfully:4
+
+$ ls output/
+average.out sim_0.out sim_1.out sim_2.out
+$ more output/*
+::::::::::::::
+output/average.out
+::::::::::::::
+651368
+::::::::::::::
+output/sim_0.out
+::::::::::::::
+ 735700
+ 886206
+ 997391
+ 982970
+::::::::::::::
+output/sim_1.out
+::::::::::::::
+ 260071
+ 264195
+ 869198
+ 933537
+::::::::::::::
+output/sim_2.out
+::::::::::::::
+ 201806
+ 213540
+ 527576
+ 944233
+-----
+
+Now try running (`-nsim=`) 100 simulations of (`-steps=`) 1 second each:
+
+-----
+$ swift p3.swift -nsim=100 -steps=1
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1444-rq809ts6
+Progress: time: Tue, 27 Aug 2013 14:44:55 -0500
+Progress: time: Tue, 27 Aug 2013 14:44:56 -0500 Selecting site:79 Active:20 Stage out:1
+Progress: time: Tue, 27 Aug 2013 14:44:58 -0500 Selecting site:58 Active:20 Stage out:1 Finished successfully:21
+Progress: time: Tue, 27 Aug 2013 14:44:59 -0500 Selecting site:37 Active:20 Stage out:1 Finished successfully:42
+Progress: time: Tue, 27 Aug 2013 14:45:00 -0500 Selecting site:16 Active:20 Stage out:1 Finished successfully:63
+Progress: time: Tue, 27 Aug 2013 14:45:02 -0500 Active:15 Stage out:1 Finished successfully:84
+Progress: time: Tue, 27 Aug 2013 14:45:03 -0500 Finished successfully:101
+Final status: Tue, 27 Aug 2013 14:45:03 -0500 Finished successfully:101
+-----
+
+We can see from Swift's "progress" status that the tutorial's default
+`sites.xml` parameters for local execution allow Swift to run up to 20
+application invocations concurrently on the login node. We'll look at
+this in more detail in the next sections where we execute applications
+on the site's compute nodes.
+
+
+Running applications on Cray compute nodes with Swift
+-----------------------------------------------------
+
+Part 4: Running a parallel ensemble on Cray compute nodes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`p4.swift` will run our mock "simulation"
+applications on Cray compute nodes. The script is similar to as
+`p3.swift`, but specifies that each simulation app invocation should
+additionally return the log file which the application writes to
+`stderr`.
+
+////
+
+FIXME: need to revise this figure: drop prog:
+
+, making the parallel portion of the script behave like this:
+
+image::part04.png[align="center"]
+
+.p4.swift
+----
+sys::[cat ../part04/p4.swift]
+----
+////
+
+Now when you run `swift p4.swift` you'll see that two types output
+files will placed in the `output/` directory: `sim_N.out` and
+`sim_N.log`. The log files provide data on the runtime environment of
+each app invocation. For example:
+
+-----
+$ cat output/sim_0.log
+Called as: /home/users/p01532/swift-cray-tutorial/app/simulate.sh: --timesteps 1 --range 100 --nvalues 5
+
+Start time: Tue Aug 27 12:17:43 CDT 2013
+Running on node: nid00018
+Running as user: uid=61532(p01532) gid=61532 groups=61532
+
+Simulation parameters:
+
+bias=0
+biasfile=none
+initseed=none
+log=yes
+paramfile=none
+range=100
+scale=1
+seedfile=none
+timesteps=1
+output width=8
+
+Environment:
+
+ALPS_APP_DEPTH=32
+ASSEMBLER_X86_64=/opt/cray/cce/8.2.0.173/cray-binutils/x86_64-unknown-linux-gnu/bin/as
+ASYNCPE_DIR=/opt/cray/xt-asyncpe/5.23.02
+ASYNCPE_VERSION=5.23.02
+...
+-----
+
+To tell Swift to run the apps on compute nodes, we specify in the
+`apps` file that the apps should be executed on the `raven` site
+(instead of the `localhost` site). We can specify the location of
+each app in the third field of the `apps` file, with either an
+absolute pathname or the name of an executable to be located in
+`PATH`). Here we use the latter form:
+
+-----
+$ cat apps
+raven simulate simulate.sh
+raven stats stats.sh
+-----
+
+You can experiment, for example, with an alternate version of stats.sh by specfying that app's location explicitly:
+
+-----
+$ cat apps
+raven simulate simulate.sh
+raven stats /home/users/p01532/bin/my-alt-stats.sh
+-----
+
+We can see that when we run many apps requesting a larger set of nodes (6), we are indeed running on the compute nodes:
+-----
+$ swift p4.swift -nsim=1000 -steps=1
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1638-t23ax37a
+Progress: time: Tue, 27 Aug 2013 16:38:11 -0500
+Progress: time: Tue, 27 Aug 2013 16:38:12 -0500 Initializing:966
+Progress: time: Tue, 27 Aug 2013 16:38:13 -0500 Selecting site:499 Submitting:500 Submitted:1
+Progress: time: Tue, 27 Aug 2013 16:38:14 -0500 Selecting site:499 Stage in:1 Submitted:500
+Progress: time: Tue, 27 Aug 2013 16:38:16 -0500 Selecting site:499 Submitted:405 Active:95 Stage out:1
+Progress: time: Tue, 27 Aug 2013 16:38:17 -0500 Selecting site:430 Submitted:434 Active:66 Stage out:1 Finished successfully:69
+Progress: time: Tue, 27 Aug 2013 16:38:18 -0500 Selecting site:388 Submitted:405 Active:95 Stage out:1 Finished successfully:111
+...
+Progress: time: Tue, 27 Aug 2013 16:38:30 -0500 Stage in:1 Submitted:93 Active:94 Finished successfully:812
+Progress: time: Tue, 27 Aug 2013 16:38:31 -0500 Submitted:55 Active:95 Stage out:1 Finished successfully:849
+Progress: time: Tue, 27 Aug 2013 16:38:32 -0500 Active:78 Stage out:1 Finished successfully:921
+Progress: time: Tue, 27 Aug 2013 16:38:34 -0500 Active:70 Stage out:1 Finished successfully:929
+Progress: time: Tue, 27 Aug 2013 16:38:37 -0500 Stage in:1 Finished successfully:1000
+Progress: time: Tue, 27 Aug 2013 16:38:38 -0500 Stage out:1 Finished successfully:1000
+Final status: Tue, 27 Aug 2013 16:38:38 -0500 Finished successfully:1001
+
+$ grep "on node:" output/*log | head
+output/sim_0.log:Running on node: nid00063
+output/sim_100.log:Running on node: nid00060
+output/sim_101.log:Running on node: nid00061
+output/sim_102.log:Running on node: nid00032
+output/sim_103.log:Running on node: nid00060
+output/sim_104.log:Running on node: nid00061
+output/sim_105.log:Running on node: nid00032
+output/sim_106.log:Running on node: nid00060
+output/sim_107.log:Running on node: nid00061
+output/sim_108.log:Running on node: nid00062
+
+$ grep "on node:" output/*log | awk '{print $4}' | sort | uniq -c
+ 158 nid00032
+ 156 nid00033
+ 171 nid00060
+ 178 nid00061
+ 166 nid00062
+ 171 nid00063
+$ hostname
+raven
+$ hostname -f
+nid00008
+-----
+
+Swift's `sites.xml` configuration file allows many parameters to
+specify how jobs should be run on a given cluster. Consider for
+example that Raven has several queues, each with limitiations on the
+size of jobs that can be run in them. All Raven queues will only run
+2 jobs per user at one. The Raven queue "small" will only allow up to
+4 nodes per job and 1 hours of walltime per job. The following
+site.xml parameters will allow us to match this:
+
+-----
+ <profile namespace="globus" key="queue">small</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">4</profile>
+ <profile namespace="globus" key="nodeGranularity">4</profile>
+-----
+
+To run large jobs, we can specify:
+
+-----
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">8</profile>
+ <profile namespace="globus" key="nodeGranularity">8</profile>
+ <profile namespace="karajan" key="jobThrottle">50.0</profile>
+ <profile namespace="globus" key="maxTime">21600</profile>
+ <profile namespace="globus" key="lowOverAllocation">10000</profile>
+ <profile namespace="globus" key="highOverAllocation">10000</profile>
+-----
+
+This will enable 512 Swift apps (2 x 8 x 32) to run concurrently
+within 2 8-node jobs on Raven's 32-core nodes. It results in the
+following two PBS jobs submitted by Swift to "provision" compute nodes
+to run thousands of apps, 512 at a time:
+
+-----
+$ qstat -u $USER
+
+Job ID Username Queue Jobname SessID NDS TSK Memory Time S Time
+--------------- -------- -------- ---------- ------ --- --- ------ ----- - -----
+288637.sdb p01532 medium B0827-2703 -- 8 256 -- 05:59 Q --
+288638.sdb p01532 medium B0827-2703 -- 8 256 -- 05:59 Q --
+-----
+
+The following section is a summary of the important `sites.xml`
+attributes for running apps on Cray systems. Many of these attributes
+can be set the same for all Swift users of a given system; only a few
+of the attributes need be overridden by users. We explain these
+attributes in detail here to show the degree of control afforded by
+Swift over application execution. Most users will use templates for a
+given Cray system, only changing a few parameters to meet any unique
+needs of their application workflows.
+
+////
+.sites.xml
+-----
+sys::[egrep -v '<.xml|<config|</config' ../part04/sites.xml | cat -n ]
+-----
+////
+
+The additional attributes in the `sites.xml` file (described here
+without their XML formatting) specify that Swift should run
+applications on Raven in the following manner:
+
+`execution provider coaster, jobmanager local:pbs` specifies that
+Swift should run apps using its "coaster" provider, which submits
+"pilot jobs" using qsub. These pilot jobs hold on to compute nodes and
+allow Swift to run many app invocations within a single job. This
+mechanism is described in
+http://www.swift-lang.org/papers/UCC-coasters.pdf[this paper from UCC-2011].
+
+`profile` tags specify additional attributes for the execution
+provider. (A "provider" is like a driver which knows how to handle
+site-specific aspects of app execution). The attributes are grouped
+into various "namespaces", but we can ignore this for now).
+
+The `env` key `PATHPREFIX` specifies that our tutorial `app` directory
+(`../app`) will be placed at the front of PATH to locate the app on
+the compute node.
+
+`queue small` specifies that pilot (coaster) jobs to run apps will be
+submitted to Raven's `small` queue.
+
+`providerAttributes pbs.aprun;pbs.mpp;depth=32` specifies some
+Cray-specific attributes: that jobs should use Cray-specific PBS "mpp"
+resource attributes (eg `mppwidth` and `mppnppn`) and an mppdepth of
+32 (because we will be running one coaster process per node, and
+Raven's XE6 dual IL-16 nodes have a depth of 32 processing elements
+(PEs).
+
+`jobsPerNode 32` tells Swift that each coaster should run up to 32
+concurrent apps. This can be reduced to place fewer apps per node, eg
+if each app needs more memory (or, rarely, greater than 32, e.g. if the apps are
+IO-bound or for benchmark experiments, etc).
+
+`slots 2` specifies that Swift will run up to 2 concurrent PBS jobs,
+and `maxNodes 1` specifies that each of these jobs will request only 1
+compute node.
+
+`maxWallTime 00:01:00` specifies that Swift should allow each app to
+run for up to one minute of walltime within the larger pilot job. In
+this example Swift will dynamically determine the total PBS walltime
+needed for the pilot job, but this can be specified manually using
+attributes `maxtime` along with `highOverAllocation` and
+`lowOverAllocation`.
+
+`jobThrottle 3.20` specifies that Swift should allow up to 320 apps to
+run on the `raven` site at once. This is typically set to a number
+greater than or equal to the number of slots x compute nodes x apps
+per node (`jobsPerNode` attribute).
+
+`initialscore 10000` is specified to override Swift's automatic
+throttling, and forces an actual throttle value of approximately
+(specifically 1 over) `jobThrottle` * 100 to be used.
+
+The last two attributes specify where and how Swift should perform
+data management. `workdirectory /lus/scratch/{env.USER}/swiftwork`
+specifies where the Swift "application execution sanbox directory"
+used for each app will be located. In some situations this can be a
+directory local to the compute node (eg, for Cray systems, `/dev/shm`
+or `/tmp`, if those are writable by user jobs and the nodes have
+sufficient space in these RAM-based filesystems).
+
+Finally, `stagingMethod sfs` specifies that Swift will copy data to
+and from the shared file system to the application sandbox
+directories.
+
+////
+
+Performing larger Swift runs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To test with larger runs, there are two changes that are required. The first is a
+change to the command line arguments. The example below will run 1000 simulations
+with each simulation taking 5 seconds.
+
+-----
+$ swift p6.swift -steps=5 -nsim=1000
+-----
+
+The other change required is to the sites.xml to control how many nodes you request.
+Change the "maxnodes" values to control the number of nodes to request.
+
+-----
+<profile namespace="globus" key="maxNodes">2</profile>
+-----
+
+////
+
+Plotting run activity
+~~~~~~~~~~~~~~~~~~~~~
+
+The tutorial `bin` directory in your `PATH` provides a script
+`plot.sh` to plot the progress of a Swift script. It generates two
+image files: `activeplot.png`, which shows the number of active jobs
+over time, and `cumulativeplot.png`, which shows the total number of
+app calls completed as the Swift script progresses.
+
+After each swift run, a log file will be created called
+partNN-<YYYYmmdd>-<hhmm>-<random>.log. Once you have identified the
+log file name, run the command `./plot.sh` <logfile>` (where logfile
+is the most recent Swift run log) to generate the plots for that
+specific run. For example:
+
+-----
+$ ls -lt *.log | head
+-rw-r--r-- 1 p01532 61532 2237693 Aug 26 12:45 p4-20130826-1244-kmos0d87.log
+-rw-r--r-- 1 p01532 61532 1008 Aug 26 12:44 swift.log
+-rw-r--r-- 1 p01532 61532 5345345 Aug 26 12:44 p4-20130826-1243-10u2qdbd.log
+-rw-r--r-- 1 p01532 61532 357687 Aug 26 12:00 p4-20130826-1159-j01p4lu0.log
+...
+$ plot.sh p4-20130826-1244-kmos0d87.log
+-----
+
+This yields plots like:
+
+image::activeplot.png[width=700,align=center]
+image::cumulativeplot.png[width=700,align=center]
+
+NOTE: Because systems like Raven are often firewalled, you may need to
+use scp to pull these image files back to a system on which you can
+view them with a browser or preview tool.
+
+
+Part 5: Controlling the compute-node pools where applications run
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In this section we'll use the script `p5.swift`, very similar to
+`p4.swift` of the prior section, to show how we can route apps to
+specific sites, and also how we can make multiple pools of resources
+(on the same or on different computer systems) available to run a
+single Swift script.
+
+////
+image::part05.png[align="center"]
+
+.p5.swift
+----
+sys::[cat ../part05/p5.swift]
+----
+////
+
+First, lets specify that the analysis app `stats.sh` should be run on
+the local login node instead of on the cluster. This is done simply by
+change the site name field of the analyze app in the apps file:
+
+-----
+$ cat apps
+raven simulate simulate.sh
+localhost stats stats.sh
+-----
+
+Running this with `swift p5.swift` we see:
+-----
+$ grep "on node:" output/*.log
+output/average.log:Running on node: raven
+output/sim_0.log:Running on node: nid00029
+output/sim_1.log:Running on node: nid00029
+output/sim_2.log:Running on node: nid00029
+output/sim_3.log:Running on node: nid00029
+output/sim_4.log:Running on node: nid00029
+output/sim_5.log:Running on node: nid00029
+output/sim_6.log:Running on node: nid00029
+output/sim_7.log:Running on node: nid00029
+output/sim_8.log:Running on node: nid00029
+output/sim_9.log:Running on node: nid00029
+-----
+
+
+Now lets make further use of Swift's ability to route specific apps to
+specific pools of resources. The Cray Raven system has two node types,
+XE6 32-core 2 x IL-16, and XK7 16-core 1 x IL-16 plus one GPU. Each
+"pool" of nodes has different queue characteristics. We can define
+these differences to Swift as two separate pools, and then spread the
+load of executing a large ensemble of simulations across all the
+pools. (And we'll continue to run the analysis script on a third pool,
+comprising the single login host.)
+
+We use the following `apps` file:
+-----
+$ cat multipools
+raven simulate simulate.sh
+ravenGPU simulate simulate.sh
+localhost stats stats.sh
+----
+and we adjust the sites file to specify 4-node jobs in the Raven pool:
+-----
+ <profile namespace="globus" key="maxNodes">4</profile>
+ <profile namespace="globus" key="nodeGranularity">4</profile>
+-----
+This results in these PBS jobs:
+-----
+p01532 at raven:~> qstat -u $USER
+
+Job ID Username Queue Jobname SessID NDS TSK Memory Time S Time
+--------------- -------- -------- ---------- ------ --- --- ------ ----- - -----
+288687.sdb p01532 small B0827-3406 9919 4 128 -- 00:59 R 00:00
+288688.sdb p01532 gpu_node B0827-3406 9931 6 96 -- 00:59 R 00:00
+-----
+... and achieves the following parallelism (of about 224 concurrent app tasks):
+
+-----
+$ swift -tc.file multipools p5.swift -nsim=1000 -steps=3
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1829-o3h6mht5
+Progress: time: Tue, 27 Aug 2013 18:29:31 -0500
+Progress: time: Tue, 27 Aug 2013 18:29:32 -0500 Initializing:997
+Progress: time: Tue, 27 Aug 2013 18:29:34 -0500 Selecting site:774 Submitting:225 Submitted:1
+Progress: time: Tue, 27 Aug 2013 18:29:35 -0500 Selecting site:774 Stage in:1 Submitted:225
+Progress: time: Tue, 27 Aug 2013 18:29:36 -0500 Selecting site:774 Stage in:1 Submitted:37 Active:188
+Progress: time: Tue, 27 Aug 2013 18:29:39 -0500 Selecting site:774 Submitted:2 Active:223 Stage out:1
+Progress: time: Tue, 27 Aug 2013 18:29:40 -0500 Selecting site:750 Submitted:17 Active:208 Stage out:1 Finished successfully:24
+Progress: time: Tue, 27 Aug 2013 18:29:41 -0500 Selecting site:640 Stage in:1 Submitted:51 Active:174 Finished successfully:134
+Progress: time: Tue, 27 Aug 2013 18:29:42 -0500 Selecting site:551 Submitted:11 Active:214 Stage out:1 Finished successfully:223
+Progress: time: Tue, 27 Aug 2013 18:29:43 -0500 Selecting site:542 Submitted:2 Active:223 Stage out:1 Finished successfully:232
+Progress: time: Tue, 27 Aug 2013 18:29:44 -0500 Selecting site:511 Submitting:1 Submitted:19 Active:206 Finished successfully:263
+Progress: time: Tue, 27 Aug 2013 18:29:45 -0500 Selecting site:463 Stage in:1 Submitted:43 Active:182 Finished successfully:311
+Progress: time: Tue, 27 Aug 2013 18:29:46 -0500 Selecting site:367 Submitting:1 Submitted:38 Active:186 Stage out:1 Finished successfully:407
+Progress: time: Tue, 27 Aug 2013 18:29:47 -0500 Selecting site:309 Submitted:2 Active:223 Stage out:1 Finished successfully:465
+Progress: time: Tue, 27 Aug 2013 18:29:48 -0500 Selecting site:300 Submitted:2 Active:223 Stage out:1 Finished successfully:474
+Progress: time: Tue, 27 Aug 2013 18:29:50 -0500 Selecting site:259 Submitted:11 Active:214 Stage out:1 Finished successfully:515
+Progress: time: Tue, 27 Aug 2013 18:29:51 -0500 Selecting site:201 Stage in:1 Submitted:39 Active:186 Finished successfully:573
+Progress: time: Tue, 27 Aug 2013 18:29:52 -0500 Selecting site:80 Submitted:42 Active:184 Finished successfully:694
+Progress: time: Tue, 27 Aug 2013 18:29:53 -0500 Selecting site:54 Submitted:2 Active:223 Stage out:1 Finished successfully:720
+Progress: time: Tue, 27 Aug 2013 18:29:54 -0500 Selecting site:32 Submitted:4 Active:220 Stage out:1 Finished successfully:743
+Progress: time: Tue, 27 Aug 2013 18:29:55 -0500 Submitted:3 Active:216 Stage out:1 Finished successfully:780
+Progress: time: Tue, 27 Aug 2013 18:29:56 -0500 Stage in:1 Active:143 Finished successfully:856
+Progress: time: Tue, 27 Aug 2013 18:29:57 -0500 Active:38 Stage out:1 Finished successfully:961
+Progress: time: Tue, 27 Aug 2013 18:29:58 -0500 Active:8 Stage out:1 Finished successfully:991
+Progress: time: Tue, 27 Aug 2013 18:29:59 -0500 Stage out:1 Finished successfully:999
+Progress: time: Tue, 27 Aug 2013 18:30:01 -0500 Stage in:1 Finished successfully:1000
+Progress: time: Tue, 27 Aug 2013 18:30:02 -0500 Active:1 Finished successfully:1000
+Progress: time: Tue, 27 Aug 2013 18:30:06 -0500 Stage out:1 Finished successfully:1000
+Final status: Tue, 27 Aug 2013 18:30:07 -0500 Finished successfully:1001
+-----
+
+Part 6: Specifying more complex workflow patterns
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+p6.swift expands the workflow pattern of p4.swift to add additional
+stages to the workflow. Here, we generate a dynamic seed value that
+will be used by all of the simulations, and for each simulation, we
+run an pre-processing application to generate a unique "bias
+file". This pattern is shown below, followed by the Swift script.
+
+image::part06.png[align="center"]
+
+.p6.swift
+----
+sys::[cat ../part06/p6.swift]
+----
+
+Note that the workflow is based on data flow dependencies: each simulation depends on the seed value, calculated in these two dependent statements:
+-----
+seedfile = genseed(1);
+int seedval = readData(seedfile);
+-----
+and on the bias file, computed and then consumed in these two dependent statements:
+-----
+ biasfile = genbias(1000, 20);
+ (simout,simlog) = simulation(steps, range, biasfile, 1000000, values);
+-----
+
+To run:
+----
+$ cd ../part06
+$ swift p6.swift
+----
+
+The default parameters result in the following execution log:
+
+-----
+$ swift p6.swift
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1917-jvs4gqm5
+Progress: time: Tue, 27 Aug 2013 19:17:56 -0500
+
+*** Script parameters: nsim=10 range=100 num values=10
+
+Progress: time: Tue, 27 Aug 2013 19:17:57 -0500 Stage in:1 Submitted:10
+Generated seed=382537
+Progress: time: Tue, 27 Aug 2013 19:17:59 -0500 Active:9 Stage out:1 Finished successfully:11
+Final status: Tue, 27 Aug 2013 19:18:00 -0500 Finished successfully:22
+-----
+which produces the following output:
+-----
+$ ls -lrt output
+total 264
+-rw-r--r-- 1 p01532 61532 9 Aug 27 19:17 seed.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_9.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_8.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_7.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_6.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_5.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_4.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_3.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_2.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_1.dat
+-rw-r--r-- 1 p01532 61532 180 Aug 27 19:17 bias_0.dat
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_9.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_9.log
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_8.log
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_7.out
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_6.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_6.log
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_5.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_5.log
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:17 sim_4.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_4.log
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_1.log
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_8.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_7.log
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_3.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_3.log
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_2.out
+-rw-r--r-- 1 p01532 61532 14898 Aug 27 19:18 sim_2.log
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_1.out
+-rw-r--r-- 1 p01532 61532 90 Aug 27 19:18 sim_0.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_0.log
+-rw-r--r-- 1 p01532 61532 9 Aug 27 19:18 average.out
+-rw-r--r-- 1 p01532 61532 14675 Aug 27 19:18 average.log
+-----
+
+Each sim_N.out file is the sum of its bias file plus newly "simulated" random output scaled by 1,000,000:
+
+-----
+$ cat output/bias_0.dat
+ 302
+ 489
+ 81
+ 582
+ 664
+ 290
+ 839
+ 258
+ 506
+ 310
+ 293
+ 508
+ 88
+ 261
+ 453
+ 187
+ 26
+ 198
+ 402
+ 555
+
+$ cat output/sim_0.out
+64000302
+38000489
+32000081
+12000582
+46000664
+36000290
+35000839
+22000258
+49000506
+75000310
+-----
+
+We produce 20 values in each bias file. Simulations of less than that
+number of values ignore the unneeded number, while simualtions of more
+than 20 will use the last bias number for all remoaining values past
+20. As an exercise, adjust the code to produce the same number of
+bias values as is needed for each simulation. As a further exercise,
+modify the script to generate a unique seed value for each simulation,
+which is a common practice in ensemble computations.
+
Added: SwiftTutorials/OSDC-2013-10-17/doc/TODO
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/doc/TODO (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/doc/TODO 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,137 @@
+PLAN
+
+basics
+cloud
+uc3
+multisystems
+(modis)
+
+get hosts.txt for cloud dynamically
+
+Show variety of UC3 activity and what ran where
+
+get hostname and app_resources into provider staging (hostname into all _swiftwrap)
+
+Show gridftp and go connect to process the modis dataset
+
+Show SwiftR for R users, SwiftPy for Enthought PY?
+
+fetching apps vs sending apps vs parrot/cvmfs/oasis etc ????
+
+Demo script-aps, bin apps, building apps, caching apps. etc.
+
+Do a Swift BLAST workflow -- with viz !!! (spice it up with help from Dina)
+
+OSG page from Mats: "Is your job HTC-ready"
+
+Use new Trunk config in tutorial?
+
+
+Add viz and status display; add node probing to tutorial.
+
+Debug failure with pinfiles
+
+
+What "exercises to suggest?"
+
+- run stats app locally
+- installed vs transferred apps
+- transferred programs vs transferred scripts
+- summarize things about the env from the .log files, ala stats.
+
+Y-style dag: 2 sims and average
+
+Show how to find temp files at runtime...
+
+Show how to find hung processes at runtime... (or get other trace data...)
+
+make your owb versions of the programs; verify that you can substitute them
+
+Add a layer to isolate shell version v installed version
+
+Show how to handle a paramfile
+
+Test if OSGCOnnect works and account string (PorjectName) works OK for both!
+
+SHow how to handle long lists of files (eg writeData and other swift.properties)
+
+Expand arith to arb prec using bc
+
+
+ISSUES
+
+error in pin files? debug and turn back on
+PATH issues for various providers
+
+bin dir?
+
+setup.csh and .csh testing
+
+IMPROVEMENTS
+
+1. Cleanup.sh mentioned in the tutorial
+[fixed]
+
+2. cd ../part02
+[fixed]
+
+3. support for zsh, csh
+[pending]
+
+4. What results are created and where did the outputs go ?
+and pointers to what the log files mean
+[pending]
+
+5. Part4,
+- cd ../part04 is missing. Cleanup the README
+[fixed]
+
+6. Part5
+- cd ../part05 is missing. Cleanup
+[fixed]
+
+7. Part7
+-Change from running on Tukey analysis cluster compute nodes
+[fixed]
+
+8. Part6
+command line args for nsim and steps, tell user the different behavior
+that can be expected with different arg options
+[pending]
+
+9. Draw workflow for the modis demos
+[pending]
+
+10. Link to more technical swift-lang references
+[pending]
+
+11. Links to detailed language constructs, when explaining
+apps, link to the section on apps in the userguide.
+[pending]
+
+
+
+---
+
+From Yadu:
+
+I've committed all the changes that I made on Friday and you can get them
+from -> https://svn.ci.uchicago.edu/svn/vdl2/SwiftTutorials/CIC_2013-08-09
+
+I'm trying to make this tutorial easily work on several sites so that, we
+do not have to tweak things for every single tutorial. Once this is done,
+I can also add it to the test-battery as well. I think this has value
+considering that we spend atleast 1-2 days for every demo.
+
+Now, the major pending item is the Modis demo, specifically the issue of
+bringing data to any site and scripts. Slight tweaks to allow for sending
+the scripts as args to apps would also be needed.
+
+So the major changes are:
+1. Source the setup.sh script with the target site name
+eg. source setups.sh <cloud|uc3|midway|beagle>
+2. No separate cloud folder
+3. [pending] fix modis data and scripts
+4. [pending] support for zsh and other shells?
+
+---
Added: SwiftTutorials/OSDC-2013-10-17/doc/activeplot.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/activeplot.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/asciidoc.css
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/doc/asciidoc.css (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/doc/asciidoc.css 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,26 @@
+a:link { color:navy; }
+a:visited { color:navy; }
+
+.monospaced, code, pre {
+ font-family: "Courier New", Courier, monospace;
+ font-size: medium; /* inherit; */
+ color: black;
+ padding: 0;
+ margin: 0;
+}
+
+/*
+ background: #f8f8f8;
+ border: 1px solid #dddddd;
+ border-left: 5px solid #f0f0f0;
+ margin-right: 10%;
+*/
+
+div.listingblock > div.content {
+ padding: 0.5em;
+ background: none;
+ border: none;
+ border-left: none;
+ margin-right: none;
+}
+
Added: SwiftTutorials/OSDC-2013-10-17/doc/build_docs.sh
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/doc/build_docs.sh (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/doc/build_docs.sh 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,4 @@
+#!/bin/bash -e
+
+asciidoc -a icons -a toc -a toplevels=2 -a stylesheet=$PWD/asciidoc.css -a max-width=800px -o swift-cray-tutorial.html README
+
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/build_docs.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftTutorials/OSDC-2013-10-17/doc/cumulativeplot.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/cumulativeplot.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/figs/modis.dia
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/figs/modis.dia
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/figs/modis.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/figs/modis.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/caution.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/caution.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/example.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/example.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/home.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/home.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/important.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/important.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/next.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/next.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/note.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/note.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/prev.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/prev.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/tip.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/tip.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/up.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/up.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/warning.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/images/icons/warning.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/part01.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/part01.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/part02.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/part02.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/part03.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/part03.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/part04.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/part04.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/part05.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/part05.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/part06.png
===================================================================
(Binary files differ)
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/part06.png
___________________________________________________________________
Added: svn:mime-type
+ application/octet-stream
Added: SwiftTutorials/OSDC-2013-10-17/doc/push.sh
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/doc/push.sh (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/doc/push.sh 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+# scp -r swift-cray-tutorial.html images *png login.ci.uchicago.edu:/ci/www/projects/swift/tutorials/cray
+
+tar zcf - --exclude-vcs *html *png images | ssh login.ci.uchicago.edu "cd /ci/www/projects/swift/tutorials/cray; tar zxf -"
Property changes on: SwiftTutorials/OSDC-2013-10-17/doc/push.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftTutorials/OSDC-2013-10-17/doc/raven.1000.progress.out
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/doc/raven.1000.progress.out (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/doc/raven.1000.progress.out 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,34 @@
+p01532 at raven:~/swift-cray-tutorial/part04> swift p4.swift -nsim=1000 -steps=5
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130824-2131-g43l2g51
+Progress: time: Sat, 24 Aug 2013 21:31:42 -0500
+
+*** Script parameters: nsim=1000 steps=5 range=100
+
+Progress: time: Sat, 24 Aug 2013 21:31:43 -0500 Initializing:998 Selecting site:2
+Progress: time: Sat, 24 Aug 2013 21:31:45 -0500 Submitting:999 Submitted:1
+Progress: time: Sat, 24 Aug 2013 21:31:46 -0500 Stage in:1 Submitted:999
+Progress: time: Sat, 24 Aug 2013 21:31:52 -0500 Submitted:712 Active:287 Stage out:1
+Progress: time: Sat, 24 Aug 2013 21:31:53 -0500 Submitted:682 Active:278 Stage out:8 Finished successfully:32
+Progress: time: Sat, 24 Aug 2013 21:31:54 -0500 Submitted:605 Active:252 Stage out:32 Finished successfully:111
+Progress: time: Sat, 24 Aug 2013 21:31:55 -0500 Submitted:480 Active:264 Stage out:23 Finished successfully:233
+Progress: time: Sat, 24 Aug 2013 21:31:57 -0500 Submitted:442 Active:287 Stage out:1 Finished successfully:270
+Progress: time: Sat, 24 Aug 2013 21:31:58 -0500 Submitted:432 Active:287 Stage out:1 Finished successfully:280
+Progress: time: Sat, 24 Aug 2013 21:31:59 -0500 Submitted:411 Active:287 Stage out:1 Finished successfully:301
+Progress: time: Sat, 24 Aug 2013 21:32:00 -0500 Submitted:317 Active:287 Finished successfully:396
+Progress: time: Sat, 24 Aug 2013 21:32:01 -0500 Submitted:244 Active:263 Stage out:25 Finished successfully:468
+Progress: time: Sat, 24 Aug 2013 21:32:02 -0500 Submitted:213 Active:233 Stage out:1 Finished successfully:553
+Progress: time: Sat, 24 Aug 2013 21:32:03 -0500 Submitted:210 Active:221 Stage out:1 Finished successfully:568
+Progress: time: Sat, 24 Aug 2013 21:32:04 -0500 Submitted:191 Active:221 Finished successfully:588
+Progress: time: Sat, 24 Aug 2013 21:32:05 -0500 Submitted:122 Active:213 Stage out:7 Finished successfully:658
+Progress: time: Sat, 24 Aug 2013 21:32:06 -0500 Submitted:117 Active:129 Stage out:5 Finished successfully:749
+Progress: time: Sat, 24 Aug 2013 21:32:09 -0500 Submitted:117 Active:98 Stage out:1 Finished successfully:784
+Progress: time: Sat, 24 Aug 2013 21:32:10 -0500 Submitted:96 Active:95 Stage out:1 Finished successfully:808
+Progress: time: Sat, 24 Aug 2013 21:32:11 -0500 Stage in:1 Submitted:33 Active:91 Stage out:4 Finished successfully:871
+Progress: time: Sat, 24 Aug 2013 21:32:12 -0500 Submitted:21 Active:96 Finished successfully:883
+Progress: time: Sat, 24 Aug 2013 21:32:14 -0500 Submitted:21 Active:95 Stage out:1 Finished successfully:883
+Progress: time: Sat, 24 Aug 2013 21:32:15 -0500 Active:93 Stage out:1 Finished successfully:906
+Progress: time: Sat, 24 Aug 2013 21:32:16 -0500 Active:42 Stage out:2 Finished successfully:956
+Progress: time: Sat, 24 Aug 2013 21:32:20 -0500 Active:20 Stage out:1 Finished successfully:979
+Final status: Sat, 24 Aug 2013 21:32:20 -0500 Finished successfully:1000
Added: SwiftTutorials/OSDC-2013-10-17/doc/raven.10000.progress.out
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/doc/raven.10000.progress.out (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/doc/raven.10000.progress.out 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,34 @@
+p01532 at raven:~/swift-cray-tutorial/part04> swift p4.swift -nsim=10000 -steps=2
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130824-2132-n4z3dxk1
+Progress: time: Sat, 24 Aug 2013 21:32:43 -0500
+
+*** Script parameters: nsim=10000 steps=2 range=100
+
+Progress: time: Sat, 24 Aug 2013 21:32:45 -0500 Initializing:3
+Progress: time: Sat, 24 Aug 2013 21:32:46 -0500 Initializing:33
+Progress: time: Sat, 24 Aug 2013 21:32:47 -0500 Initializing:1613
+Progress: time: Sat, 24 Aug 2013 21:32:50 -0500 Selecting site:9999 Submitting:1
+Progress: time: Sat, 24 Aug 2013 21:32:52 -0500 Selecting site:8998 Stage in:1 Submitted:1001
+Progress: time: Sat, 24 Aug 2013 21:32:55 -0500 Selecting site:8998 Submitted:714 Active:287 Stage out:1
+Progress: time: Sat, 24 Aug 2013 21:32:57 -0500 Selecting site:8950 Submitted:714 Active:287 Stage out:1 Finished successfully:48
+Progress: time: Sat, 24 Aug 2013 21:32:58 -0500 Selecting site:8921 Submitting:1 Submitted:713 Active:257 Stage out:31 Finished successfully:77
+Progress: time: Sat, 24 Aug 2013 21:32:59 -0500 Selecting site:8858 Stage in:1 Submitting:5 Submitted:708 Active:178 Stage out:110 Finished successfully:140
+Progress: time: Sat, 24 Aug 2013 21:33:00 -0500 Selecting site:8658 Stage in:1 Submitting:1 Submitted:713 Active:287 Finished successfully:340
+.....
+Progress: time: Sat, 24 Aug 2013 21:34:39 -0500 Submitted:555 Active:287 Finished successfully:9158
+Progress: time: Sat, 24 Aug 2013 21:34:40 -0500 Submitted:454 Active:283 Stage out:4 Finished successfully:9259
+Progress: time: Sat, 24 Aug 2013 21:34:41 -0500 Submitted:361 Active:285 Stage out:3 Finished successfully:9351
+Progress: time: Sat, 24 Aug 2013 21:34:42 -0500 Submitted:265 Active:287 Stage out:1 Finished successfully:9447
+Progress: time: Sat, 24 Aug 2013 21:34:43 -0500 Submitted:206 Active:224 Stage out:6 Finished successfully:9564
+Progress: time: Sat, 24 Aug 2013 21:34:44 -0500 Submitted:185 Active:155 Stage out:10 Finished successfully:9650
+Progress: time: Sat, 24 Aug 2013 21:34:45 -0500 Submitted:181 Active:124 Stage out:39 Finished successfully:9656
+Progress: time: Sat, 24 Aug 2013 21:34:46 -0500 Submitted:169 Active:56 Stage out:94 Finished successfully:9681
+Progress: time: Sat, 24 Aug 2013 21:34:47 -0500 Submitted:76 Active:95 Stage out:1 Finished successfully:9828
+Progress: time: Sat, 24 Aug 2013 21:34:48 -0500 Submitted:67 Active:95 Stage out:1 Finished successfully:9837
+Progress: time: Sat, 24 Aug 2013 21:34:49 -0500 Active:81 Stage out:1 Finished successfully:9918
+Progress: time: Sat, 24 Aug 2013 21:34:50 -0500 Active:71 Stage out:1 Finished successfully:9928
+Progress: time: Sat, 24 Aug 2013 21:34:51 -0500 Active:6 Stage out:2 Finished successfully:9992
+Final status: Sat, 24 Aug 2013 21:34:51 -0500 Finished successfully:10000
+p01532 at raven:~/swift-cray-tutorial/part04>
Added: SwiftTutorials/OSDC-2013-10-17/part01/apps
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part01/apps (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part01/apps 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1 @@
+localhost simulate simulate.sh
Added: SwiftTutorials/OSDC-2013-10-17/part01/p1.swift
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part01/p1.swift (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part01/p1.swift 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,9 @@
+type file;
+
+app (file o) simulation ()
+{
+ simulate stdout=@filename(o);
+}
+
+file f <"sim.out">;
+f = simulation();
Added: SwiftTutorials/OSDC-2013-10-17/part01/sites.xml
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part01/sites.xml (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part01/sites.xml 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="localhost">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">0</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>.swift/tmp</workdirectory>
+ <profile namespace="swift" key="stagingMethod">local</profile>
+ </pool>
+</config>
Added: SwiftTutorials/OSDC-2013-10-17/part02/apps
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part02/apps (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part02/apps 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1 @@
+localhost simulate simulate.sh
Added: SwiftTutorials/OSDC-2013-10-17/part02/p2.swift
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part02/p2.swift (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part02/p2.swift 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,11 @@
+type file;
+
+app (file o) simulation ()
+{
+ simulate stdout=@filename(o);
+}
+
+foreach i in [0:9] {
+ file f <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ f = simulation();
+}
Added: SwiftTutorials/OSDC-2013-10-17/part02/sites.xml
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part02/sites.xml (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part02/sites.xml 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="localhost">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">0</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>.swift/tmp</workdirectory>
+ <profile namespace="swift" key="stagingMethod">local</profile>
+ </pool>
+</config>
Added: SwiftTutorials/OSDC-2013-10-17/part03/apps
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part03/apps (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part03/apps 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,2 @@
+localhost simulate simulate.sh
+localhost stats stats.sh
Added: SwiftTutorials/OSDC-2013-10-17/part03/p3.kml
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part03/p3.kml (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part03/p3.kml 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,273 @@
+<project><!-- CACHE ID f1622cb4-2179-479b-8c8b-201ddea16c6c-no-provenance -->
+ <import file="sys.xml"/>
+ <import file="scheduler.xml"/>
+ <import file="rlog.xml"/>
+ <import file="vdl.k"/>
+ <types>
+ <xs:schema targetNamespace="http://ci.uchicago.edu/swift/2009/02/swiftscript" xmlns="http://ci.uchicago.edu/swift/2009/02/swiftscript" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+ <xs:simpleType name="file">
+ <xs:restriction base="string"/>
+ </xs:simpleType>
+ </xs:schema>
+ </types>
+ <global name="swift#string#17002">
+ <vdl:new type="string" value="--nvalues" />
+ </global>
+ <global name="swift#string#17007">
+ <vdl:new type="string" value="range" />
+ </global>
+ <global name="swift#string#17003">
+ <vdl:new type="string" value="nsim" />
+ </global>
+ <global name="swift#string#17000">
+ <vdl:new type="string" value="--timesteps" />
+ </global>
+ <global name="swift#string#17013">
+ <vdl:new type="string" value=".out" />
+ </global>
+ <global name="swift#string#17005">
+ <vdl:new type="string" value="steps" />
+ </global>
+ <global name="swift#string#17004">
+ <vdl:new type="string" value="10" />
+ </global>
+ <global name="swift#string#17006">
+ <vdl:new type="string" value="1" />
+ </global>
+ <global name="swift#string#17009">
+ <vdl:new type="string" value="values" />
+ </global>
+ <global name="swift#string#17010">
+ <vdl:new type="string" value="5" />
+ </global>
+ <global name="swift#string#17001">
+ <vdl:new type="string" value="--range" />
+ </global>
+ <global name="swift#string#17008">
+ <vdl:new type="string" value="100" />
+ </global>
+ <global name="swift#string#17012">
+ <vdl:new type="string" value="output/sim_" />
+ </global>
+ <global name="swift#int#0">
+ <vdl:new type="int" value="0" />
+ </global>
+ <global name="swift#int#1">
+ <vdl:new type="int" value="1" />
+ </global>
+ <element name="simulation" arguments="o,sim_steps,sim_range,sim_values" _defline="3">
+
+
+
+
+ <unitStart name="simulation" line="3" type="PROCEDURE" outputs="o"/>
+ <vdl:execute>
+ <vdl:tr>simulate</vdl:tr>
+ <vdl:stagein var="{sim_steps}"/>
+ <vdl:stagein var="{sim_range}"/>
+ <vdl:stagein var="{sim_values}"/>
+ <vdl:stageout var="{o}"/>
+
+ <vdl:arguments>
+ <variable>swift#string#17000</variable>
+ <variable>sim_steps</variable>
+ <variable>swift#string#17001</variable>
+ <variable>sim_range</variable>
+ <variable>swift#string#17002</variable>
+ <variable>sim_values</variable>
+ </vdl:arguments>
+ <vdl:stdout>
+ <swiftscript:filename _traceline="5">
+ <variable>o</variable>
+ </swiftscript:filename>
+ </vdl:stdout>
+ </vdl:execute>
+ <vdl:closedataset var="{o}"/>
+ <unitEnd name="simulation" line="3" type="PROCEDURE"/></element>
+
+ <element name="analyze" arguments="o,s" _defline="8">
+
+
+ <unitStart name="analyze" line="8" type="PROCEDURE" outputs="o"/>
+ <vdl:execute>
+ <vdl:tr>stats</vdl:tr>
+ <vdl:stagein var="{s}"/>
+ <vdl:stageout var="{o}"/>
+
+ <vdl:arguments>
+ <swiftscript:filenames _traceline="10">
+ <variable>s</variable>
+ </swiftscript:filenames>
+ </vdl:arguments>
+ <vdl:stdout>
+ <swiftscript:filename _traceline="10">
+ <variable>o</variable>
+ </swiftscript:filename>
+ </vdl:stdout>
+ </vdl:execute>
+ <vdl:closedataset var="{o}"/>
+ <unitEnd name="analyze" line="8" type="PROCEDURE"/></element>
+
+ <set name="nsim">
+ <vdl:new type="int" dbgname="nsim" waitCount="1" _defline="13"/>
+ </set>
+
+ <set name="steps">
+ <vdl:new type="int" dbgname="steps" waitCount="1" _defline="14"/>
+ </set>
+
+ <set name="range">
+ <vdl:new type="int" dbgname="range" waitCount="1" _defline="15"/>
+ </set>
+
+ <set name="values">
+ <vdl:new type="int" dbgname="values" waitCount="1" _defline="16"/>
+ </set>
+
+ <set name="sims">
+ <vdl:new type="file[]" dbgname="sims" waitCount="1" _defline="18">
+ <vdl:mapping descriptor="concurrent_mapper">
+ <vdl:parameter name="prefix">sims-59494c94-f159-4e5b-8084-2a4706a394e3</vdl:parameter>
+ </vdl:mapping>
+ </vdl:new>
+ </set>
+
+ <set name="stats">
+ <vdl:new type="file" dbgname="stats" _defline="26">
+ <vdl:mapping descriptor="single_file_mapper">
+ <vdl:parameter name="file" value="output/average.out"/>
+ </vdl:mapping>
+ </vdl:new>
+ </set>
+
+ <restartLog>
+ <vdl:mains>
+ <vdl:startprogressticker />
+ <vdl:mainp>
+ <uparallel>
+ <sequential>
+ <vdl:setfieldvalue _traceline="14">
+ <variable>nsim</variable>
+ <swiftscript:to_int _traceline="13">
+ <swiftscript:arg _traceline="13">
+ <variable>swift#string#17003</variable><variable>swift#string#17004</variable>
+ </swiftscript:arg>
+ </swiftscript:to_int>
+ </vdl:setfieldvalue>
+ <partialCloseDataset var="{nsim}"/>
+ </sequential>
+ <sequential>
+ <vdl:setfieldvalue _traceline="15">
+ <variable>steps</variable>
+ <swiftscript:to_int _traceline="14">
+ <swiftscript:arg _traceline="14">
+ <variable>swift#string#17005</variable><variable>swift#string#17006</variable>
+ </swiftscript:arg>
+ </swiftscript:to_int>
+ </vdl:setfieldvalue>
+ <partialCloseDataset var="{steps}"/>
+ </sequential>
+ <sequential>
+ <vdl:setfieldvalue _traceline="16">
+ <variable>range</variable>
+ <swiftscript:to_int _traceline="15">
+ <swiftscript:arg _traceline="15">
+ <variable>swift#string#17007</variable><variable>swift#string#17008</variable>
+ </swiftscript:arg>
+ </swiftscript:to_int>
+ </vdl:setfieldvalue>
+ <partialCloseDataset var="{range}"/>
+ </sequential>
+ <sequential>
+ <vdl:setfieldvalue _traceline="18">
+ <variable>values</variable>
+ <swiftscript:to_int _traceline="16">
+ <swiftscript:arg _traceline="16">
+ <variable>swift#string#17009</variable><variable>swift#string#17010</variable>
+ </swiftscript:arg>
+ </swiftscript:to_int>
+ </vdl:setfieldvalue>
+ <partialCloseDataset var="{values}"/>
+ </sequential>
+ <sequential>
+ <vdl:tparallelFor name="$" _vvar="i" refs="sims 1">
+ <getarrayiterator>
+ <sequential>
+ <set name="swift#rangeout">
+ <vdl:range>
+ <argument name="from"><variable>swift#int#0</variable></argument>
+ <argument name="to"><vdlop:subtraction>
+ <variable>nsim</variable>
+ <variable>swift#int#1</variable>
+ </vdlop:subtraction></argument>
+ </vdl:range>
+ </set>
+
+ <variable>swift#rangeout</variable>
+ </sequential>
+ </getarrayiterator>
+ <set names="$$, i">
+ <each items="{$}"/>
+ </set>
+ <unitStart line="20" type="FOREACH_IT"/>
+ <set name="swift#mapper#17011">
+ <vdl:new type="string" dbgname="swift#mapper#17011" _defline=""/>
+ </set>
+
+ <set name="simout">
+ <vdl:new type="file" dbgname="simout" _defline="21">
+ <vdl:mapping descriptor="single_file_mapper">
+ <vdl:parameter name="file"><variable>swift#mapper#17011</variable></vdl:parameter>
+ </vdl:mapping>
+ </vdl:new>
+ </set>
+
+ <uparallel>
+ <sequential>
+ <vdl:setfieldvalue _traceline="-1">
+ <variable>swift#mapper#17011</variable>
+ <swiftscript:strcat _traceline="21">
+ <variable>swift#string#17012</variable><variable>i</variable><variable>swift#string#17013</variable>
+ </swiftscript:strcat>
+ </vdl:setfieldvalue>
+
+ </sequential>
+ <sequential>
+ <simulation _traceline="22">
+ <variable>simout</variable>
+ <variable>steps</variable>
+ <variable>range</variable>
+ <variable>values</variable>
+ </simulation>
+
+ </sequential>
+ <sequential>
+ <vdl:setfieldvalue _traceline="24">
+ <vdl:getfieldsubscript>
+ <argument name="var"><variable>sims</variable></argument>
+ <argument name="subscript"><variable>i</variable></argument>
+ </vdl:getfieldsubscript>
+ <variable>simout</variable>
+ </vdl:setfieldvalue>
+ <partialCloseDataset var="{sims}"/>
+ </sequential>
+ </uparallel>
+ <vdl:cleandataset var="{simout}"/> <unitEnd line="20" type="FOREACH_IT"/>
+ </vdl:tparallelFor>
+ </sequential>
+ <sequential>
+ <analyze _traceline="27">
+ <variable>stats</variable>
+ <variable>sims</variable>
+ </analyze>
+
+ </sequential>
+ </uparallel>
+ </vdl:mainp>
+ <vdl:stopprogressticker />
+ </vdl:mains>
+ </restartLog>
+ <vdl:cleandataset var="{stats}"/>
+ <vdl:cleandataset var="{sims}"/>
+ <vdl:cleandataset shutdown="true"/>
+</project>
Added: SwiftTutorials/OSDC-2013-10-17/part03/p3.swift
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part03/p3.swift (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part03/p3.swift 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,27 @@
+type file;
+
+app (file o) simulation (int sim_steps, int sim_range, int sim_values)
+{
+ simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@filename(o);
+}
+
+app (file o) analyze (file s[])
+{
+ stats @filenames(s) stdout=@filename(o);
+}
+
+int nsim = @toInt(@arg("nsim","10"));
+int steps = @toInt(@arg("steps","1"));
+int range = @toInt(@arg("range","100"));
+int values = @toInt(@arg("values","5"));
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ simout = simulation(steps,range,values);
+ sims[i] = simout;
+}
+
+file stats<"output/average.out">;
+stats = analyze(sims);
Added: SwiftTutorials/OSDC-2013-10-17/part03/sites.xml
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part03/sites.xml (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part03/sites.xml 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="localhost">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">0.20</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>.swift/tmp</workdirectory>
+ <profile namespace="swift" key="stagingMethod">local</profile>
+ </pool>
+</config>
Added: SwiftTutorials/OSDC-2013-10-17/part04/apps
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part04/apps (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part04/apps 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,2 @@
+raven simulate simulate.sh
+raven stats stats.sh
Added: SwiftTutorials/OSDC-2013-10-17/part04/p4.swift
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part04/p4.swift (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part04/p4.swift 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,29 @@
+type file;
+
+app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values)
+{
+ simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log;
+}
+
+app (file out, file log) analyze (file s[])
+{
+ stats @filenames(s) stdout=@out stderr=@log;
+}
+
+int nsim = @toInt(@arg("nsim", "10"));
+int steps = @toInt(@arg("steps", "1"));
+int range = @toInt(@arg("range", "100"));
+int values = @toInt(@arg("values", "5"));
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
+ (simout,simlog) = simulation(steps,range,values);
+ sims[i] = simout;
+}
+
+file stats_out<"output/average.out">;
+file stats_log<"output/average.log">;
+(stats_out, stats_log) = analyze(sims);
Added: SwiftTutorials/OSDC-2013-10-17/part05/OLD.p5.swift
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part05/OLD.p5.swift (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part05/OLD.p5.swift 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,37 @@
+type file;
+
+app (file out, file log) simulation (int timesteps, int sim_range, file bias_file, int scale, int sim_count)
+{
+ simulate "-t" timesteps "-r" sim_range "-B" @bias_file "-x" scale "-n" sim_count stdout=@out stderr=@log;
+}
+
+app (file out) analyze (file s[])
+{
+ analyze @filenames(s) stdout=@filename(out);
+}
+
+# Command line params to this script
+
+int nsim = @toInt(@arg("nsim", "10")); # number of simulation programs to run
+int steps = @toInt(@arg("steps", "1")); # number of "steps" each simulation (==seconds of runtime)
+int range = @toInt(@arg("range", "100")); # range of the generated random numbers
+int count = @toInt(@arg("count", "10")); # number of random numbers generated per simulation
+
+# Perform nsim "simulations"
+
+tracef("\n*** Script parameters: nsim=%i steps=%i range=%i count=%i\n\n", nsim, steps, range, count);
+
+file sims[]; # Array of files to hold each simulation output
+file bias<"bias.dat">; # Input data file to "bias" the numbers:
+ # 1 line: scale offset ( N = n*scale + offset)
+foreach i in [0:nsim-1] {
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
+ (simout, simlog) = simulation(steps, range, bias, 100000, count);
+ sims[i] = simout;
+}
+
+# Generate "analysis" file containing average of all "simulations"
+
+file stats<"output/stats.out">;
+stats = analyze(sims);
Added: SwiftTutorials/OSDC-2013-10-17/part05/OLDbias.dat
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part05/OLDbias.dat (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part05/OLDbias.dat 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,1000 @@
+ 301
+ 439
+ 591
+ 484
+ 300
+ 459
+ 395
+ 351
+ 375
+ 131
+ 144
+ 12
+ 465
+ 682
+ 245
+ 140
+ 862
+ 772
+ 52
+ 728
+ 696
+ 801
+ 487
+ 501
+ 273
+ 869
+ 976
+ 731
+ 634
+ 594
+ 944
+ 973
+ 521
+ 260
+ 997
+ 165
+ 240
+ 844
+ 30
+ 145
+ 29
+ 185
+ 19
+ 332
+ 932
+ 740
+ 750
+ 774
+ 882
+ 258
+ 64
+ 988
+ 980
+ 627
+ 114
+ 722
+ 522
+ 333
+ 201
+ 737
+ 773
+ 226
+ 99
+ 705
+ 435
+ 560
+ 311
+ 96
+ 538
+ 176
+ 831
+ 993
+ 761
+ 917
+ 311
+ 983
+ 259
+ 494
+ 713
+ 503
+ 580
+ 23
+ 687
+ 394
+ 935
+ 386
+ 420
+ 598
+ 424
+ 954
+ 92
+ 528
+ 43
+ 384
+ 531
+ 494
+ 335
+ 913
+ 140
+ 271
+ 817
+ 101
+ 996
+ 742
+ 36
+ 592
+ 321
+ 499
+ 579
+ 454
+ 333
+ 276
+ 360
+ 159
+ 909
+ 791
+ 913
+ 665
+ 862
+ 663
+ 455
+ 39
+ 415
+ 798
+ 222
+ 758
+ 178
+ 656
+ 144
+ 450
+ 182
+ 507
+ 269
+ 839
+ 911
+ 400
+ 506
+ 602
+ 69
+ 729
+ 697
+ 242
+ 39
+ 557
+ 756
+ 677
+ 403
+ 930
+ 660
+ 730
+ 974
+ 23
+ 227
+ 67
+ 465
+ 950
+ 819
+ 100
+ 858
+ 123
+ 979
+ 610
+ 428
+ 623
+ 12
+ 369
+ 96
+ 86
+ 531
+ 777
+ 809
+ 9
+ 589
+ 267
+ 957
+ 722
+ 352
+ 655
+ 145
+ 848
+ 232
+ 752
+ 958
+ 902
+ 206
+ 165
+ 52
+ 452
+ 185
+ 109
+ 676
+ 736
+ 803
+ 893
+ 163
+ 403
+ 573
+ 121
+ 992
+ 818
+ 571
+ 304
+ 822
+ 390
+ 192
+ 174
+ 571
+ 648
+ 924
+ 817
+ 334
+ 791
+ 561
+ 679
+ 457
+ 549
+ 339
+ 264
+ 316
+ 338
+ 726
+ 471
+ 144
+ 748
+ 778
+ 851
+ 432
+ 260
+ 731
+ 436
+ 376
+ 631
+ 373
+ 680
+ 223
+ 169
+ 916
+ 252
+ 236
+ 2
+ 565
+ 749
+ 206
+ 788
+ 801
+ 460
+ 730
+ 795
+ 17
+ 992
+ 847
+ 229
+ 187
+ 690
+ 111
+ 731
+ 754
+ 677
+ 845
+ 707
+ 815
+ 258
+ 526
+ 475
+ 559
+ 970
+ 440
+ 682
+ 471
+ 378
+ 560
+ 821
+ 851
+ 819
+ 106
+ 624
+ 766
+ 628
+ 362
+ 149
+ 523
+ 215
+ 207
+ 446
+ 783
+ 845
+ 556
+ 273
+ 921
+ 519
+ 81
+ 598
+ 440
+ 899
+ 218
+ 623
+ 861
+ 475
+ 789
+ 643
+ 165
+ 526
+ 705
+ 803
+ 907
+ 725
+ 814
+ 306
+ 39
+ 795
+ 144
+ 986
+ 79
+ 726
+ 664
+ 99
+ 71
+ 516
+ 948
+ 601
+ 67
+ 111
+ 856
+ 915
+ 747
+ 621
+ 360
+ 93
+ 787
+ 880
+ 694
+ 5
+ 982
+ 336
+ 426
+ 346
+ 639
+ 10
+ 427
+ 757
+ 981
+ 102
+ 325
+ 305
+ 691
+ 619
+ 398
+ 686
+ 922
+ 674
+ 349
+ 580
+ 165
+ 321
+ 492
+ 432
+ 329
+ 477
+ 894
+ 444
+ 57
+ 577
+ 323
+ 278
+ 179
+ 421
+ 614
+ 616
+ 436
+ 892
+ 432
+ 558
+ 140
+ 517
+ 975
+ 198
+ 805
+ 422
+ 745
+ 250
+ 482
+ 958
+ 538
+ 177
+ 791
+ 489
+ 932
+ 620
+ 138
+ 978
+ 689
+ 192
+ 742
+ 626
+ 946
+ 106
+ 31
+ 178
+ 205
+ 285
+ 576
+ 105
+ 179
+ 307
+ 830
+ 576
+ 679
+ 231
+ 454
+ 833
+ 116
+ 160
+ 616
+ 250
+ 760
+ 205
+ 74
+ 989
+ 576
+ 843
+ 931
+ 267
+ 849
+ 108
+ 850
+ 937
+ 746
+ 607
+ 35
+ 539
+ 550
+ 553
+ 886
+ 459
+ 786
+ 720
+ 650
+ 735
+ 126
+ 100
+ 297
+ 303
+ 921
+ 105
+ 46
+ 954
+ 663
+ 568
+ 478
+ 886
+ 817
+ 690
+ 261
+ 642
+ 511
+ 967
+ 727
+ 492
+ 4
+ 810
+ 320
+ 338
+ 498
+ 842
+ 454
+ 366
+ 316
+ 802
+ 641
+ 293
+ 544
+ 957
+ 768
+ 553
+ 692
+ 749
+ 944
+ 19
+ 762
+ 991
+ 234
+ 247
+ 556
+ 625
+ 189
+ 725
+ 818
+ 764
+ 636
+ 370
+ 138
+ 161
+ 853
+ 309
+ 613
+ 250
+ 75
+ 728
+ 230
+ 248
+ 562
+ 110
+ 879
+ 993
+ 626
+ 308
+ 124
+ 593
+ 169
+ 125
+ 304
+ 143
+ 312
+ 525
+ 639
+ 492
+ 377
+ 153
+ 97
+ 90
+ 1
+ 483
+ 139
+ 277
+ 7
+ 707
+ 977
+ 513
+ 458
+ 95
+ 208
+ 149
+ 293
+ 690
+ 456
+ 558
+ 245
+ 327
+ 500
+ 222
+ 776
+ 863
+ 322
+ 363
+ 892
+ 414
+ 442
+ 283
+ 627
+ 656
+ 785
+ 468
+ 371
+ 657
+ 840
+ 449
+ 500
+ 522
+ 337
+ 784
+ 475
+ 667
+ 345
+ 151
+ 541
+ 843
+ 285
+ 270
+ 684
+ 381
+ 608
+ 731
+ 492
+ 233
+ 540
+ 185
+ 975
+ 422
+ 199
+ 595
+ 242
+ 995
+ 892
+ 509
+ 916
+ 493
+ 549
+ 588
+ 203
+ 653
+ 276
+ 808
+ 245
+ 277
+ 827
+ 724
+ 118
+ 566
+ 406
+ 927
+ 498
+ 727
+ 309
+ 165
+ 412
+ 28
+ 937
+ 797
+ 556
+ 322
+ 59
+ 741
+ 675
+ 89
+ 877
+ 283
+ 518
+ 509
+ 6
+ 486
+ 421
+ 644
+ 258
+ 920
+ 623
+ 725
+ 859
+ 88
+ 773
+ 499
+ 973
+ 661
+ 947
+ 347
+ 679
+ 935
+ 717
+ 65
+ 265
+ 388
+ 740
+ 252
+ 249
+ 969
+ 26
+ 149
+ 338
+ 913
+ 605
+ 639
+ 649
+ 914
+ 890
+ 522
+ 621
+ 779
+ 470
+ 321
+ 171
+ 304
+ 425
+ 626
+ 378
+ 491
+ 925
+ 900
+ 705
+ 830
+ 631
+ 844
+ 274
+ 539
+ 511
+ 664
+ 777
+ 251
+ 190
+ 967
+ 569
+ 140
+ 327
+ 863
+ 525
+ 368
+ 540
+ 475
+ 716
+ 189
+ 442
+ 729
+ 112
+ 726
+ 505
+ 678
+ 323
+ 128
+ 983
+ 660
+ 853
+ 326
+ 312
+ 378
+ 544
+ 278
+ 127
+ 322
+ 360
+ 645
+ 233
+ 665
+ 3
+ 788
+ 698
+ 342
+ 93
+ 214
+ 976
+ 756
+ 677
+ 375
+ 947
+ 363
+ 999
+ 499
+ 139
+ 245
+ 745
+ 830
+ 798
+ 135
+ 83
+ 902
+ 779
+ 617
+ 636
+ 753
+ 867
+ 724
+ 743
+ 924
+ 480
+ 836
+ 409
+ 100
+ 73
+ 298
+ 711
+ 35
+ 289
+ 381
+ 361
+ 901
+ 640
+ 919
+ 708
+ 562
+ 592
+ 26
+ 471
+ 395
+ 728
+ 433
+ 69
+ 856
+ 550
+ 225
+ 714
+ 999
+ 936
+ 266
+ 81
+ 343
+ 123
+ 37
+ 740
+ 593
+ 905
+ 715
+ 481
+ 265
+ 357
+ 338
+ 150
+ 875
+ 938
+ 603
+ 98
+ 245
+ 721
+ 373
+ 901
+ 795
+ 325
+ 955
+ 448
+ 816
+ 753
+ 850
+ 462
+ 628
+ 220
+ 694
+ 423
+ 241
+ 718
+ 730
+ 682
+ 944
+ 408
+ 151
+ 10
+ 826
+ 26
+ 269
+ 439
+ 973
+ 842
+ 884
+ 103
+ 792
+ 435
+ 675
+ 442
+ 955
+ 992
+ 256
+ 206
+ 666
+ 637
+ 725
+ 52
+ 21
+ 108
+ 622
+ 943
+ 583
+ 919
+ 796
+ 80
+ 52
+ 678
+ 845
+ 155
+ 331
+ 909
+ 888
+ 945
+ 979
+ 967
+ 104
+ 446
+ 807
+ 572
+ 354
+ 639
+ 68
+ 813
+ 11
+ 129
+ 647
+ 743
+ 897
+ 600
+ 992
+ 20
+ 896
+ 398
+ 16
+ 434
+ 662
+ 86
+ 200
+ 211
+ 287
+ 485
+ 670
+ 446
+ 495
+ 407
+ 916
+ 395
+ 920
+ 959
+ 817
+ 524
+ 146
+ 378
+ 849
+ 457
+ 536
+ 582
+ 608
+ 686
+ 127
+ 346
+ 934
+ 950
+ 420
+ 294
+ 439
+ 688
+ 953
+ 66
+ 603
+ 875
+ 372
+ 355
+ 105
+ 405
+ 371
+ 246
+ 829
+ 146
+ 403
+ 811
+ 484
+ 266
+ 181
+ 496
+ 25
+ 189
+ 806
+ 1
+ 171
+ 737
+ 381
+ 720
+ 321
+ 862
+ 7
+ 177
+ 838
+ 502
+ 708
+ 819
+ 205
+ 256
+ 166
+ 712
+ 492
+ 716
+ 130
+ 562
+ 308
+ 365
+ 694
+ 372
+ 917
+ 165
+ 366
+ 412
+ 480
+ 681
+ 824
+ 306
+ 970
+ 160
+ 555
+ 336
+ 61
+ 816
+ 611
+ 432
+ 536
+ 830
+ 171
+ 793
+ 33
+ 364
+ 998
+ 814
+ 952
+ 89
+ 24
+ 653
+ 651
+ 8
+ 789
+ 807
+ 869
+ 332
+ 897
+ 499
+ 701
+ 354
+ 688
+ 339
+ 93
+ 822
+ 801
+ 758
+ 915
+ 546
+ 520
+ 469
+ 512
+ 911
+ 482
+ 387
+ 187
Added: SwiftTutorials/OSDC-2013-10-17/part05/apps
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part05/apps (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part05/apps 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,2 @@
+raven simulate simulate.sh
+raven stats stats.sh
Added: SwiftTutorials/OSDC-2013-10-17/part05/multipools
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part05/multipools (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part05/multipools 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,3 @@
+raven simulate simulate.sh
+ravenGPU simulate simulate.sh
+localhost stats stats.sh
Added: SwiftTutorials/OSDC-2013-10-17/part05/p5.swift
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part05/p5.swift (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part05/p5.swift 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,29 @@
+type file;
+
+app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values)
+{
+ simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log;
+}
+
+app (file out, file log) analyze (file s[])
+{
+ stats @filenames(s) stdout=@out stderr=@log;
+}
+
+int nsim = @toInt(@arg("nsim", "10"));
+int steps = @toInt(@arg("steps", "1"));
+int range = @toInt(@arg("range", "100"));
+int values = @toInt(@arg("values", "5"));
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
+ (simout,simlog) = simulation(steps,range,values);
+ sims[i] = simout;
+}
+
+file stats_out<"output/average.out">;
+file stats_log<"output/average.log">;
+(stats_out, stats_log) = analyze(sims);
Added: SwiftTutorials/OSDC-2013-10-17/part06/apps
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part06/apps (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part06/apps 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,4 @@
+raven simulate simulate.sh
+raven stats stats.sh
+raven genseed simulate.sh
+raven genbias simulate.sh
Added: SwiftTutorials/OSDC-2013-10-17/part06/p6.swift
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/part06/p6.swift (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/part06/p6.swift 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,57 @@
+type file;
+
+# app() functions for application programs to be called:
+
+app (file out) genseed (int nseeds)
+{
+ genseed "-r" 2000000 "-n" nseeds stdout=@out;
+}
+
+app (file out) genbias (int bias_range, int nvalues)
+{
+ genbias "-r" bias_range "-n" nvalues stdout=@out;
+}
+
+app (file out, file log) simulation (int timesteps, int sim_range,
+ file bias_file, int scale, int sim_count)
+{
+ simulate "-t" timesteps "-r" sim_range "-B" @bias_file "-x" scale
+ "-n" sim_count stdout=@out stderr=@log;
+}
+
+app (file out, file log) analyze (file s[])
+{
+ stats @filenames(s) stdout=@out stderr=@log;
+}
+
+# Command line arguments
+
+int nsim = @toInt(@arg("nsim", "10")); # number of simulation programs to run
+int steps = @toInt(@arg("steps", "1")); # number of timesteps (seconds) per simulation
+int range = @toInt(@arg("range", "100")); # range of the generated random numbers
+int values = @toInt(@arg("values", "10")); # number of values generated per simulation
+
+# Main script and data
+
+tracef("\n*** Script parameters: nsim=%i range=%i num values=%i\n\n", nsim, range, values);
+
+file seedfile<"output/seed.dat">; # Dynamically generated bias for simulation ensemble
+seedfile = genseed(1);
+
+int seedval = readData(seedfile);
+tracef("Generated seed=%i\n", seedval);
+
+file sims[]; # Array of files to hold each simulation output
+
+foreach i in [0:nsim-1] {
+ file biasfile <single_file_mapper; file=@strcat("output/bias_",i,".dat")>;
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
+ biasfile = genbias(1000, 20);
+ (simout,simlog) = simulation(steps, range, biasfile, 1000000, values);
+ sims[i] = simout;
+}
+
+file stats_out<"output/average.out">;
+file stats_log<"output/average.log">;
+(stats_out,stats_log) = analyze(sims);
Added: SwiftTutorials/OSDC-2013-10-17/setup.sh
===================================================================
--- SwiftTutorials/OSDC-2013-10-17/setup.sh (rev 0)
+++ SwiftTutorials/OSDC-2013-10-17/setup.sh 2013-10-17 18:29:32 UTC (rev 7173)
@@ -0,0 +1,90 @@
+# ensure that this script is being sourced
+if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" = "${0}" ] ; then
+ echo ERROR: script ${BASH_SOURCE[0]} must be executed as: source ${BASH_SOURCE[0]}
+ exit 1
+fi
+
+# Add swift to PATH
+
+TUTSWIFT=/glusterfs/users/swiftlang/dkelly/swift-0.94.1
+PATHSWIFT=$(which swift 2>/dev/null)
+
+if [ _$PATHSWIFT = _$TUTSWIFT/bin/swift ]; then
+ echo using Swift from $TUTSWIFT,already in PATH
+elif [ -x $TUTSWIFT/bin/swift ]; then
+ echo Using Swift from $TUTSWIFT, and adding to PATH
+ PATH=$TUTSWIFT/bin:$PATH
+elif [ _$PATHSWIFT != _ ]; then
+ echo Using $PATHSWIFT from PATH
+else
+ echo ERROR: $TUTSWIFT not found and no swift in PATH. Tutorial will not function.
+# return
+fi
+
+echo Swift version is $(swift -version)
+rm -f swift.log
+
+# Setting scripts folder to the PATH env var.
+
+TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+if [ _$(which cleanup 2>/dev/null) != _$TUTDIR/bin/cleanup ]; then
+ echo Adding $TUTDIR/bin:$TUTDIR/app: to front of PATH
+ PATH=$TUTDIR/bin:$TUTDIR/app:$PATH
+else
+ echo Assuming $TUTDIR/bin:$TUTDIR/app: is already at front of PATH
+fi
+
+# Setting .swift files
+
+if [ -e $HOME/.swift/swift.properties ]; then
+ saveprop=$(mktemp $HOME/.swift/swift.properties.XXXX)
+ echo Saving $HOME/.swift/swift.properties in $saveprop
+ mv $HOME/.swift/swift.properties $saveprop
+else
+ mkdir -p $HOME/.swift
+fi
+
+cat >>$HOME/.swift/swift.properties <<END
+
+# Properties for Swift Tutorial
+
+sites.file=sites.xml
+tc.file=apps
+
+wrapperlog.always.transfer=false
+sitedir.keep=false
+file.gc.enabled=false
+status.mode=provider
+
+execution.retries=0
+lazy.errors=false
+
+use.provider.staging=true
+provider.staging.pin.swiftfiles=true
+use.wrapper.staging=false
+
+END
+
+cat >gen.sites <<END
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://swift-lang.org/sites">
+
+ <pool handle="localhost">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">0.04</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/lus/scratch/$USER/swiftwork</workdirectory>
+ <profile namespace="swift" key="stagingMethod">local</profile>
+ </pool>
+
+</config>
+END
+
+for p in 01 02 03; do
+ cp gen.sites part${p}/sites.xml
+done
+rm gen.sites
+
+return
+
More information about the Swift-commit
mailing list