[Swift-commit] r6922 - in SwiftTutorials/CIC_2013-08-09: . OLD OLD/part01 app bin cloud cloud/service doc part06 part08 scripts
wilde at ci.uchicago.edu
wilde at ci.uchicago.edu
Wed Aug 21 15:44:20 CDT 2013
Author: wilde
Date: 2013-08-21 15:44:20 -0500 (Wed, 21 Aug 2013)
New Revision: 6922
Added:
SwiftTutorials/CIC_2013-08-09/OLD/cf
SwiftTutorials/CIC_2013-08-09/OLD/part01/
SwiftTutorials/CIC_2013-08-09/app/simulate.sh
SwiftTutorials/CIC_2013-08-09/app/simulatepos.sh
SwiftTutorials/CIC_2013-08-09/bin/sitecount.sh
SwiftTutorials/CIC_2013-08-09/cloud/RL2stage.swift
SwiftTutorials/CIC_2013-08-09/cloud/app
SwiftTutorials/CIC_2013-08-09/cloud/nodecount
SwiftTutorials/CIC_2013-08-09/cloud/service/
SwiftTutorials/CIC_2013-08-09/cloud/service/cleancloud
SwiftTutorials/CIC_2013-08-09/cloud/service/coaster-service.conf
SwiftTutorials/CIC_2013-08-09/cloud/service/hosts.txt
SwiftTutorials/CIC_2013-08-09/cloud/service/pool.txt
SwiftTutorials/CIC_2013-08-09/cloud/service/startcloud
SwiftTutorials/CIC_2013-08-09/cloud/service/stopcloud
SwiftTutorials/CIC_2013-08-09/cloud/service/tutorial.pem
SwiftTutorials/CIC_2013-08-09/part08/rensemble.swift
SwiftTutorials/CIC_2013-08-09/scripts/
SwiftTutorials/CIC_2013-08-09/scripts/rensemble.swift
Removed:
SwiftTutorials/CIC_2013-08-09/app/simulate.sh
SwiftTutorials/CIC_2013-08-09/app/simulate2.sh
SwiftTutorials/CIC_2013-08-09/cloud/cf
SwiftTutorials/CIC_2013-08-09/cloud/coaster-service.conf
SwiftTutorials/CIC_2013-08-09/cloud/data.txt
SwiftTutorials/CIC_2013-08-09/cloud/dynamic_bias.dat
SwiftTutorials/CIC_2013-08-09/cloud/hosts.txt
SwiftTutorials/CIC_2013-08-09/cloud/pool.txt
SwiftTutorials/CIC_2013-08-09/cloud/run.catsn
SwiftTutorials/CIC_2013-08-09/cloud/simulate.sh
SwiftTutorials/CIC_2013-08-09/cloud/sitecount.sh
SwiftTutorials/CIC_2013-08-09/cloud/sites.xml
SwiftTutorials/CIC_2013-08-09/cloud/startcloud
SwiftTutorials/CIC_2013-08-09/cloud/stats.sh
SwiftTutorials/CIC_2013-08-09/cloud/stopcloud
SwiftTutorials/CIC_2013-08-09/cloud/tc.data
SwiftTutorials/CIC_2013-08-09/cloud/tutorial.pem
SwiftTutorials/CIC_2013-08-09/part01/
SwiftTutorials/CIC_2013-08-09/part08/p8.swift
Modified:
SwiftTutorials/CIC_2013-08-09/OLD/part01/apps
SwiftTutorials/CIC_2013-08-09/doc/README
SwiftTutorials/CIC_2013-08-09/doc/TODO
SwiftTutorials/CIC_2013-08-09/part06/sites.xml
SwiftTutorials/CIC_2013-08-09/part08/sites.xml
SwiftTutorials/CIC_2013-08-09/setup.sh
Log:
Changes for CIC workshop.
Copied: SwiftTutorials/CIC_2013-08-09/OLD/cf (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/cf)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/OLD/cf (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/OLD/cf 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,6 @@
+use.provider.staging=true
+wrapperlog.always.transfer=false
+execution.retries=10
+provider.staging.pin.swiftfiles=false
+sitedir.keep=false
+use.wrapper.staging=false
Modified: SwiftTutorials/CIC_2013-08-09/OLD/part01/apps
===================================================================
--- SwiftTutorials/CIC_2013-08-09/part01/apps 2013-08-15 16:31:36 UTC (rev 6868)
+++ SwiftTutorials/CIC_2013-08-09/OLD/part01/apps 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,2 +1,2 @@
-localhost simulate simulate.sh
+localhost simulate simulate2.sh
persistent-coasters simulate simulate.sh
Deleted: SwiftTutorials/CIC_2013-08-09/app/simulate.sh
===================================================================
--- SwiftTutorials/CIC_2013-08-09/app/simulate.sh 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/app/simulate.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,38 +0,0 @@
-#! /bin/bash
-
-log() {
- printf "Start time: "; /bin/date
- printf "Job is running on node: "; /bin/hostname
- printf "Job running as user: "; /usr/bin/id
-
- echo "Environment:"
- /bin/env | /bin/sort
-}
-
-runtime=${1:-0}
-range=${2:-100}
-biasfile=${3:-nobias}
-scale=${4:-1}
-n=${5:-1}
-
-if [ $biasfile = nobias ]; then
- offset=0
-else
- read offset <$biasfile
-fi
-
-# run for some number of "timesteps"
-
-sleep $runtime
-
-# emit n "simulation results", scaled and biased by the specified argument values
-
-for ((i=0;i<n;i++)); do
- value=$(((($RANDOM)*32768)+$RANDOM))
- echo $(( ($value%range)*scale+offset))
-done
-
-# log environmental data
-
-log 1>&2
-
Copied: SwiftTutorials/CIC_2013-08-09/app/simulate.sh (from rev 6892, SwiftTutorials/CIC_2013-08-09/app/simulate2.sh)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/app/simulate.sh (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/app/simulate.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,131 @@
+#! /bin/bash
+
+printparams()
+{
+ printf "\nSimulation parameters:\n\n"
+ echo bias=$bias
+ echo biasfile=$biasfile
+ echo initseed=$initseed
+ echo log=$log
+ echo paramfile=$paramfile
+ echo range=$range
+ echo scale=$scale
+ echo seedfile=$seedfile
+ echo timesteps=$timesteps
+ echo output width=$width
+}
+
+log() {
+ printf "\nCalled as: $0: $cmdargs\n\n"
+ printf "Start time: "; /bin/date
+ printf "Running on node: "; /bin/hostname
+ printf "Running as user: "; /usr/bin/id
+ printparams
+ printf "\nEnvironment:\n\n"
+ /bin/env | /bin/sort
+}
+
+addsims() {
+ while read f1 ; do
+ read -u 3 f2
+ if [ _$f1 = _ ]; then f1=0; fi
+ if [ _$f2 = _ ]; then f2=0; fi
+ printf "%${width}d\n" $(($f1+$f2))
+ done <$1 3<$2
+}
+
+# set defaults
+
+bias=0
+biasfile=none
+initseed=none
+log=yes
+paramfile=none
+range=100
+scale=1
+seedfile=none
+timesteps=0
+nvalues=1
+width=8
+cmdargs="$*"
+
+usage()
+{
+ echo $0: usage:
+ cat <<END
+ -b|--bias offset bias: add this integer to all results
+ -B|--biasfile file of integer biases to add to results
+ -l|--log generate a log in stderr if not null
+ -n|--nvalues print this many values per simulation
+ -p|--paramfile take these parameters (in form param=value) from this file
+ -r|--range range (limit) of generated results
+ -s|--seed use this integer [0..32767] as a seed
+ -S|--seedfile use this file (containing integer seeds [0..32767]) one per line
+ -t|--timesteps number of simulated "timesteps" in seconds (determines runtime)
+ -x|--scale scale the results by this integer
+ -h|-?|?|--help print this help
+END
+}
+
+while [ $# -gt 0 ]; do
+ case $1 in
+ -b|--bias) bias=$2 ;;
+ -B|--biasfile) biasfile=$2 ;;
+ -l|--log) log=$2 ;;
+ -n|--nvalues) nvalues=$2 ;;
+ -p|--paramfile) paramfile=$2 ;;
+ -s|--seed) initseed=$2 ;;
+ -S|--seedfile) seedfile=$2 ;;
+ -t|--timesteps) timesteps=$2 ;;
+ -r|--range) range=$2 ;;
+ -w|--width) width=$2 ;;
+ -x|--scale) scale=$2 ;;
+ -h|-?|--help|*) usage; exit ;;
+ esac
+ shift 2
+done
+
+# process initial seed
+
+if [ $initseed != none ]; then
+ RANDOM=$initseed
+fi
+
+# process file of seeds
+
+if [ $seedfile != none ]; then
+ seed=0
+ while read $seedfile s; do
+ seed=$(($seed+$s))
+ done <$seedfile
+ RANDOM=$seed
+fi
+
+# run for some number of "timesteps"
+
+sleep $timesteps
+
+# emit N (nvalues) "simulation results" scaled and biased by argument values
+
+simout=$(mktemp simout.XXXX)
+for ((i=0;i<nvalues;i++)); do
+ # value=$(( (($RANDOM)*(2**16))+$RANDOM ))
+ value=$(( (($RANDOM)*(2**48)) + (($RANDOM)*(2**32)) + (($RANDOM)*(2**16)) + $RANDOM ))
+ printf "%${width}d\n" $(( ($value%range)*scale+bias))
+done >$simout
+
+# process file of biases
+
+if [ $biasfile != none ]; then
+ printf "\nMerged:\n\n"
+ addsims $simout $biasfile
+else
+ cat $simout
+fi
+rm $simout
+
+# log environmental data
+
+if [ $log != none ]; then
+ log 1>&2
+fi
Deleted: SwiftTutorials/CIC_2013-08-09/app/simulate2.sh
===================================================================
--- SwiftTutorials/CIC_2013-08-09/app/simulate2.sh 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/app/simulate2.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,131 +0,0 @@
-#! /bin/bash
-
-printparams()
-{
- printf "\nSimulation parameters:\n\n"
- echo bias=$bias
- echo biasfile=$biasfile
- echo initseed=$initseed
- echo log=$log
- echo paramfile=$paramfile
- echo range=$range
- echo scale=$scale
- echo seedfile=$seedfile
- echo timesteps=$timesteps
- echo output width=$width
-}
-
-log() {
- printf "\nCalled as: $0: $cmdargs\n\n"
- printf "Start time: "; /bin/date
- printf "Running on node: "; /bin/hostname
- printf "Running as user: "; /usr/bin/id
- printparams
- printf "\nEnvironment:\n\n"
- /bin/env | /bin/sort
-}
-
-addsims() {
- while read f1 ; do
- read -u 3 f2
- if [ _$f1 = _ ]; then f1=0; fi
- if [ _$f2 = _ ]; then f2=0; fi
- printf "%${width}d\n" $(($f1+$f2))
- done <$1 3<$2
-}
-
-# set defaults
-
-bias=0
-biasfile=none
-initseed=none
-log=yes
-paramfile=none
-range=100
-scale=1
-seedfile=none
-timesteps=0
-nvalues=1
-width=8
-cmdargs="$*"
-
-usage()
-{
- echo $0: usage:
- cat <<END
- -t|--timesteps number of simulated "timesteps" in seconds (determines runtime)
- -r|--range range (limit) of generated results
- -B|--biasfile file of integer biases to add to results
- -l|--log generate a log in stderr if not null
- -x|--scale scale the results by this integer
- -n|--nvalues print this many values per simulation
- -S|--seedfile use this file (containing integer seeds [0..32767]) one per line
- -i|--seed use this integer [0..32767] as a seed
- -b|--bias offset bias: add this integer to all results
- -p|--paramfile take these parameters (in form param=value) from this file
- -h|-?|?|--help print this help
-END
-}
-
-while [ $# -gt 0 ]; do
- case $1 in
- -t|--timesteps) timesteps=$2 ;;
- -r|--range) range=$2 ;;
- -B|--biasfile) biasfile=$2 ;;
- -l|--log) log=$2 ;;
- -x|--scale) scale=$2 ;;
- -n|--nvalues) nvalues=$2 ;;
- -S|--seedfile) seedfile=$2 ;;
- -s|--seed) initseed=$2 ;;
- -b|--bias) bias=$2 ;;
- -p|--paramfile) paramfile=$2 ;;
- -w|--width) width=$2 ;;
- -h|-?|--help|*) usage; exit ;;
- esac
- shift 2
-done
-
-# process initial seed
-
-if [ $initseed != none ]; then
- RANDOM=$initseed
-fi
-
-# process file of seeds
-
-if [ $seedfile != none ]; then
- seed=0
- while read $seedfile s; do
- seed=$(($seed+$s))
- done <$seedfile
- RANDOM=$seed
-fi
-
-# run for some number of "timesteps"
-
-sleep $timesteps
-
-# emit n "simulation results" scaled and biased by argument values
-
-simout=$(mktemp simout.XXXX)
-for ((i=0;i<nvalues;i++)); do
- # value=$(( (($RANDOM)*(2**16))+$RANDOM ))
- value=$(( (($RANDOM)*(2**48)) + (($RANDOM)*(2**32)) + (($RANDOM)*(2**16)) + $RANDOM ))
- printf "%${width}d\n" $(( ($value%range)*scale+bias))
-done >$simout
-
-# process file of biases
-
-if [ $biasfile != none ]; then
- printf "\nMerged:\n\n"
- addsims $simout $biasfile
-else
- cat $simout
-fi
-rm $simout
-
-# log environmental data
-
-if [ $log != none ]; then
- log 1>&2
-fi
Copied: SwiftTutorials/CIC_2013-08-09/app/simulatepos.sh (from rev 6892, SwiftTutorials/CIC_2013-08-09/app/simulate.sh)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/app/simulatepos.sh (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/app/simulatepos.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,38 @@
+#! /bin/bash
+
+log() {
+ printf "Start time: "; /bin/date
+ printf "Job is running on node: "; /bin/hostname
+ printf "Job running as user: "; /usr/bin/id
+
+ echo "Environment:"
+ /bin/env | /bin/sort
+}
+
+runtime=${1:-0}
+range=${2:-100}
+biasfile=${3:-nobias}
+scale=${4:-1}
+n=${5:-1}
+
+if [ $biasfile = nobias ]; then
+ offset=0
+else
+ read offset <$biasfile
+fi
+
+# run for some number of "timesteps"
+
+sleep $runtime
+
+# emit n "simulation results", scaled and biased by the specified argument values
+
+for ((i=0;i<n;i++)); do
+ value=$(((($RANDOM)*32768)+$RANDOM))
+ echo $(( ($value%range)*scale+offset))
+done
+
+# log environmental data
+
+log 1>&2
+
Copied: SwiftTutorials/CIC_2013-08-09/bin/sitecount.sh (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/sitecount.sh)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/bin/sitecount.sh (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/bin/sitecount.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1 @@
+grep "submitting" cps*.log | awk '{print $5}' | sed -e 's/:.*//' | sort | uniq -c
Copied: SwiftTutorials/CIC_2013-08-09/cloud/RL2stage.swift (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/p9.swift)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/RL2stage.swift (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/RL2stage.swift 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,64 @@
+type file;
+
+# Application programs to be called by this script
+
+global file simulation_prog <"app/simulate.sh">;
+global file analysis_prog <"app/stats.sh">;
+global file genrand_prog <"app/genrand.sh">;
+
+# app() functions for application programs:
+
+app (file out) genrand (file pgm, int timesteps, int sim_range)
+{
+ local_sh @filename(pgm) timesteps sim_range stdout=@filename(out);
+}
+
+app (file out) simulation (file pgm, int timesteps, int sim_range, file bias_file, int scale, int sim_count)
+{
+ sh @filename(pgm) timesteps sim_range @filename(bias_file) scale sim_count stdout=@filename(out);
+}
+
+app (file out) analyze (file pgm, file s[])
+{
+ local_sh @filename(pgm) @filenames(s) stdout=@filename(out);
+}
+
+# Higher-level functions that call app() functions
+
+(int scale) calcScale(int maxScale)
+{
+ scale = readData(genrand(simulation_prog, 1, maxScale));
+}
+
+(file biasFile) calcBias (int maxBias)
+{
+ biasFile = genrand(simulation_prog, 1, maxBias);
+}
+
+# Command line params to this script:
+
+int nsim = @toInt(@arg("nsim", "10")); # number of simulation programs to run
+int range = @toInt(@arg("range", "100")); # range of the generated random numbers
+int count = @toInt(@arg("count", "10")); # number of random numbers generated per simulation
+int steps = @toInt(@arg("steps", "3")); # number of time steps per simulation
+
+# Main script and data
+
+tracef("\n*** Script parameters: nsim=%i range=%i count=%i\n\n", nsim, range, count);
+
+file bias<"dynamic_bias.dat">; # Dynamically generated bias for simulation ensemble
+bias = calcBias(1000);
+
+file sims[]; # Array of files to hold each simulation output
+
+foreach i in [0:nsim-1] {
+ int scale;
+ scale = 1000 * calcScale(100);
+ tracef(" for simulation[%i] scale=%i\n", i, scale);
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ simout = simulation(simulation_prog, steps+1, range, bias, scale, count);
+ sims[i] = simout;
+}
+
+file stats<"output/stats.out">; # Final output file: average of all "simulations"
+stats = analyze(analysis_prog,sims);
Added: SwiftTutorials/CIC_2013-08-09/cloud/app
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/app (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/app 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1 @@
+link ../app
\ No newline at end of file
Property changes on: SwiftTutorials/CIC_2013-08-09/cloud/app
___________________________________________________________________
Added: svn:special
+ *
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/cf
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/cf 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/cf 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,6 +0,0 @@
-use.provider.staging=true
-wrapperlog.always.transfer=false
-execution.retries=10
-provider.staging.pin.swiftfiles=false
-sitedir.keep=false
-use.wrapper.staging=false
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/coaster-service.conf
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/coaster-service.conf 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/coaster-service.conf 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,47 +0,0 @@
-#Location of SWIFT. If empty, PATH is referenced
- export SWIFT=
-
- # Where to place/launch worker.pl on the remote machine for sites.xml
- export WORKER_LOCATION=/root
-
- # How to launch workers: local, ssh, or cobalt
- export WORKER_MODE=ssh
-
- # Worker logging setting passed to worker.pl for sites.xml
- export WORKER_LOGGING_LEVEL=DEBUG
-
- export SSH_TUNNELING=yes
-
- # User name to use for all systems
- export WORKER_USERNAME=root
-
- # Worker host names for ssh
-# export WORKER_HOSTS="crush thwomp stomp crank grind churn trounce thrash vanquish"
-#export WORKER_HOSTS="ip-10-36-196-96.eu-west-1.compute.internal ip-10-36-195-69.eu-west-1.compute.internal"
-
- # Directory to keep log files, relative to working directory when launching start-coaster-service
- export LOG_DIR=logs
- export WORKER_LOG_DIR=/root
-
- # Manually define ports. If not specified, ports will be automatically generated
- #export LOCAL_PORT=50100
- #export SERVICE_PORT=50200
-
- # Set shared filesystem to no since work will be done in local /sandbox directory
- export SHARED_FILESYSTEM=no
-
- # start-coaster-service tries to automatically detect IP address.
- # Specify here if auto detection is not working correctly
- export IPADDR=
-
- # Below are various settings to give information about how to create sites.xml
- export WORK=/tmp/ketan
- export JOBS_PER_NODE=8
- export JOBSPERNODE=8
- export JOBTHROTTLE=$( echo "scale=5; ($JOBS_PER_NODE * $( echo $WORKER_HOSTS | wc -w ))/100 - 0.00001"|bc )
-
- # Swift applications
- #app cat=/bin/cat
- #app bash=/bin/bash
- #app echo=/bin/echo
-
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/data.txt
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/data.txt 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/data.txt 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,3 +0,0 @@
-===========
-Test Data
-===========
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/dynamic_bias.dat
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/dynamic_bias.dat 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/dynamic_bias.dat 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1 +0,0 @@
-821
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/hosts.txt
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/hosts.txt 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/hosts.txt 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,3 +0,0 @@
-54.212.81.224
-54.244.171.51
-54.212.37.161
Added: SwiftTutorials/CIC_2013-08-09/cloud/nodecount
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/nodecount (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/nodecount 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1 @@
+grep node: output/*.log | sed -e 's/^.*Job is //' | sort | uniq -c
Property changes on: SwiftTutorials/CIC_2013-08-09/cloud/nodecount
___________________________________________________________________
Added: svn:executable
+ *
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/pool.txt
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/pool.txt 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/pool.txt 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,3 +0,0 @@
-54.212.81.224
-54.244.171.51
-54.212.37.161
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/run.catsn
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/run.catsn 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/run.catsn 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1 +0,0 @@
-swift -sites.file sites.xml -tc.file tc -config cf catsn.swift -n=10
Added: SwiftTutorials/CIC_2013-08-09/cloud/service/cleancloud
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/service/cleancloud (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/service/cleancloud 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,10 @@
+#! /bin/bash
+
+if [ -d service ]; then
+ ( cd service
+ rm -rf cf sites.xml hosts.txt logs *.log
+ )
+else
+ echo ERROR: no service directory
+fi
+
Property changes on: SwiftTutorials/CIC_2013-08-09/cloud/service/cleancloud
___________________________________________________________________
Added: svn:executable
+ *
Copied: SwiftTutorials/CIC_2013-08-09/cloud/service/coaster-service.conf (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/coaster-service.conf)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/service/coaster-service.conf (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/service/coaster-service.conf 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,48 @@
+#Location of SWIFT. If empty, PATH is referenced
+ export SWIFT=
+
+ # Where to place/launch worker.pl on the remote machine for sites.xml
+ export WORKER_LOCATION=/root
+
+ # How to launch workers: local, ssh, or cobalt
+ export WORKER_MODE=ssh
+
+ # Worker logging setting passed to worker.pl for sites.xml
+ # export WORKER_LOGGING_LEVEL=DEBUG
+ export WORKER_LOGGING_LEVEL=INFO
+
+ export SSH_TUNNELING=yes
+
+ # User name to use for all systems
+ export WORKER_USERNAME=root
+
+ # Worker host names for ssh
+# export WORKER_HOSTS="crush thwomp stomp crank grind churn trounce thrash vanquish"
+#export WORKER_HOSTS="ip-10-36-196-96.eu-west-1.compute.internal ip-10-36-195-69.eu-west-1.compute.internal"
+
+ # Directory to keep log files, relative to working directory when launching start-coaster-service
+ export LOG_DIR=logs
+ export WORKER_LOG_DIR=/root
+
+ # Manually define ports. If not specified, ports will be automatically generated
+ #export LOCAL_PORT=50100
+ #export SERVICE_PORT=50200
+
+ # Set shared filesystem to no since work will be done in local /sandbox directory
+ export SHARED_FILESYSTEM=no
+
+ # start-coaster-service tries to automatically detect IP address.
+ # Specify here if auto detection is not working correctly
+ export IPADDR=
+
+ # Below are various settings to give information about how to create sites.xml
+ export WORK=/tmp
+ export JOBS_PER_NODE=8
+ export JOBSPERNODE=8
+ export JOBTHROTTLE=$( echo "scale=5; ($JOBS_PER_NODE * $( echo $WORKER_HOSTS | wc -w ))/100"|bc )
+
+ # Swift applications
+ # no app cat=/bin/cat
+ # no app bash=/bin/bash
+ #app echo=/bin/echo
+
Copied: SwiftTutorials/CIC_2013-08-09/cloud/service/hosts.txt (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/hosts.txt)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/service/hosts.txt (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/service/hosts.txt 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,3 @@
+54.212.81.224
+54.244.171.51
+54.212.37.161
Copied: SwiftTutorials/CIC_2013-08-09/cloud/service/pool.txt (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/pool.txt)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/service/pool.txt (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/service/pool.txt 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,3 @@
+54.212.81.224
+54.244.171.51
+54.212.37.161
Copied: SwiftTutorials/CIC_2013-08-09/cloud/service/startcloud (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/startcloud)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/service/startcloud (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/service/startcloud 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+echo 0: $0
+echo BASH_SOURCE: ${BASH_SOURCE[0]}
+
+if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" = "${0}" ] ; then
+ echo ERROR_1: script ${BASH_SOURCE[0]} must be executed as: source service/startcloud
+ exit 1
+fi
+
+if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" != service/startcloud ] ; then
+ echo ERROR_2: script ${BASH_SOURCE[0]} must be executed as: source service/startcloud
+ return 1
+fi
+
+addlocalpool()
+{
+cat <<END
+ <pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local" url="http://localhost"/>
+ <profile key="jobThrottle" namespace="karajan">.23</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>$PWD/swiftwork</workdirectory>
+ </pool>
+END
+}
+
+echo -e "\nSetting up Swift cloud service in $PWD/service\n"
+
+( cd service
+
+# pick 3 random hosts from pool # FIXME: disabled for now because sort -R is not univerally available
+# sort -R pool.txt |head -n 3 > hosts.txt
+# FIXME: Get the hosts from a dynamic file for each tutorial
+
+head -n 3 pool.txt > hosts.txt
+
+echo "Enabling cloud host access in $HOME/.ssh/config"
+touch ~/.ssh/config
+chmod 600 ~/.ssh/config
+
+while read line
+do
+ if grep -q $line ~/.ssh/config; then :
+ else
+ echo -ne "Host $line\n IdentityFile ~/.ssh/tutorial.pem\n User root\n" >> ~/.ssh/config
+ fi
+done < hosts.txt
+
+if grep -q StrictHostKeyChecking ~/.ssh/config; then :
+else
+ echo -ne "\nHost * \n StrictHostKeyChecking=no\n" >> ~/.ssh/config
+fi
+
+export WORKER_HOSTS=`cat hosts.txt`
+chmod 0600 ~/.ssh/tutorial.pem
+
+echo "Starting ssh-agent"
+eval $(ssh-agent)
+ssh-add ~/.ssh/tutorial.pem
+
+echo "Starting Swift coaster service"
+start-coaster-service
+
+mv sites.xml _temp.xml
+grep -v config _temp.xml | (
+ echo "<config>"
+ cat
+ addlocalpool
+ echo "</config>"
+) >sites.xml
+rm _temp.xml
+mv sites.xml ..
+
+echo "Swift cloud setup complete"
+# rm -rf hi.* hi-*
+
+) # Leave user in original directory
Copied: SwiftTutorials/CIC_2013-08-09/cloud/service/stopcloud (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/stopcloud)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/service/stopcloud (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/service/stopcloud 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+echo "stopping coaster service"
+stop-coaster-service
+
+echo "Killing ssh-agent"
+killall -u $USER ssh-agent ssh
+
Copied: SwiftTutorials/CIC_2013-08-09/cloud/service/tutorial.pem (from rev 6868, SwiftTutorials/CIC_2013-08-09/cloud/tutorial.pem)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/service/tutorial.pem (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/cloud/service/tutorial.pem 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,23 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAvPJTneRDhB5wlRsWEr2l6LoFi/PulPm0J0A1Vi2skzWYC9J6I4Ftb2YbSbtp
+3d+BJUSK3wyJqfYgY/ynC4vtbdG4HU7/Fr6R2BBKtfiiVHET1K3OTCp3JzFVqQoJoSYkrkgW7XE9
+9GF7hvpza8JVvV9JPwxOcuhDAVHQFkoIRAq1TK9VaRulZFJapubWnEHM2UoUrmBS10IZmUwGHqR5
+I6soUyBqapAYoQmL5fzqT+dhsnu7rXrUmF37Va5kX309N4Dp0D4/WJY7FeF9Q9to2i+G7HPA+gEY
+/ptYxlTk89l5MBIY1czZvz7Fso5NHCEMW1468mD/NtJIWBDS8x1yHQIDAQABAoIBAHY2ur//b4p6
+rP9s+etxwhqQQCYm0O9eJyujjXtEikNz9ZP39LjAImLfJFWGLx9E1f0mHCifJIY02mEUUsLzofHi
+k2mq4ggt5PukfcnKLG3ebYTT1U1sCC56PFjNUecEAVEVA1s9TYDXeLyjd5FE/fXEZwROQ8q6rClo
+MWOwn+lJII9La8lLFIBxXNQbyt2nwO26iSzw51wBYsWDglV/T6qnB9JOcQ+uyn8/e4kQYuD52iV3
+Vx3QN+F+YPC7ebYOS8kVR/bFo+JDggNU4jZ+Zk31Sbepfb7p7IdYTjtrlQAd48tIxdvKFpFqy0mC
+GDmkVKBAXSc7DNqy3osvq1DdtbUCgYEA+IwsADb2iY6QPt7KNl0edJRSjDmeR8wSV7QIJYm0mxTB
+tVUZn+cpdeDsK5RkJYzDNKy3gjXhsp3B9Ogsu1aO2KVuv2ffk4wUF6nFLASQifMebUvMtqZM3g3N
+/yhMh7jGqfEu3tCmRSztCm0K3AodTtxacyRZKlKM4iAe0tbIO9cCgYEAwpypzFQvX83a2X+SU9rz
+rb7S10U2sZz3YwCHTRr2BCS8EaFA5xuEZBDmGR8OegUMfmQweD57JboQKB95KpuojGUpVBt6XoFv
+SaVLg7Ehskp71XZAgG6D9AHV1mm7hGkGoqj9zdlScc+nEZw57UEPzp/hXTTtc9hapOvOH1l7IysC
+gYB7CRh2+r5MFdB97td21d3Po+TBGXyq2fByjeJCgspuwnWph/jpWQsDGVaPtCNnGyk+mAk3YejZ
+/JIY5B3QluabhDkwcwEQDxWbKSU7xv6t6zWysQUjnWsLuRWyK4bG4McnNQcYsXy9XaHxuUzY5ADi
+Gt6DhQjGlJq9DV10zXHNfQKBgQCsVbUbZxVInm+FmS3IKiUNn+TvkrAbWqfLuOUXx59CDhGo/wYN
+YdW8Eybf6huUvDHTHv+5C4Nb5kd2cGCI9aytHswZWffkBnS5X7HeXZY8Xy8H2Z7G3JbnFlT5eUn/
+uE2S71lJ13/lnwAQXSsCyVGpJPWZ50u7AUVLtDEkm8WqXwKBgQDh0X21ufkWeparJ4GxWjodxnYK
+i1mtzghE3FjhcrrKBKJ3Sm5ZUYPVbOe97071zB+u2S0a83/tHP9Sr5KuS1sHiN++IB2FVo7UMBMC
+1Ktg+yK9MoERr1xzNrqeR59U5uvRBypd8MSzxuk4LjePYFYeyFaz8GOusCw/ZPU4R83zWA==
+-----END RSA PRIVATE KEY-----
\ No newline at end of file
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/simulate.sh
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/simulate.sh 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/simulate.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,23 +0,0 @@
-#! /bin/bash
-
-runtime=${1:-0}
-range=${2:-100}
-biasfile=${3:-nobias}
-scale=${4:-1}
-n=${5:-1}
-
-if [ $biasfile = nobias ]; then
- offset=0
-else
- read offset <$biasfile
-fi
-
-# run for some number of "timesteps"
-
-sleep $runtime
-
-# emit n "simulation results", scaled and biased by the specified argument values
-
-for ((i=0;i<n;i++)); do
- echo $(( ($RANDOM%range)*scale+offset))
-done
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/sitecount.sh
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/sitecount.sh 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/sitecount.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1 +0,0 @@
-grep "submitting" cps*.log | awk '{print $5}' | sed -e 's/:.*//' | sort | uniq -c
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/sites.xml
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/sites.xml 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/sites.xml 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,19 +0,0 @@
-<config>
- <pool handle="persistent-coasters">
- <execution provider="coaster-persistent"
- url="http://localhost:55592"
- jobmanager="local:local"/>
- <profile namespace="globus" key="workerManager">passive</profile>
- <profile namespace="globus" key="jobsPerNode">8</profile>
- <profile key="jobThrottle" namespace="karajan">.23999</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local" url="none" />
- <workdirectory>/tmp/ketan</workdirectory>
- </pool>
- <pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local" url="http://localhost"/>
- <profile key="jobThrottle" namespace="karajan">.23</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/home/wilde/swift/demo/CIC.2013.0815/svn/cloud/swiftwork</workdirectory>
- </pool>
-</config>
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/startcloud
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/startcloud 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/startcloud 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,67 +0,0 @@
-#!/bin/bash
-
-if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" = "${0}" ] ; then
- echo ERROR: script ${BASH_SOURCE[0]} must be executed as: source ${BASH_SOURCE[0]}
- exit 1
-fi
-
-addlocalpool()
-{
-cat <<END
- <pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local" url="http://localhost"/>
- <profile key="jobThrottle" namespace="karajan">.23</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>$PWD/swiftwork</workdirectory>
- </pool>
-END
-}
-
-echo -e "\nSwift cloud setup\n"
-
-#pick 3 random hosts from pool
-# sort -R pool.txt |head -n 3 > hosts.txt
-
-head -n 3 pool.txt > hosts.txt # FIXME: disabled for now because sort -R is not univerally available
-
-echo "Enabling cloud host access in $HOME/.ssh/config"
-touch ~/.ssh/config
-chmod 600 ~/.ssh/config
-
-while read line
-do
- count=$(grep $line ~/.ssh/config | wc -l)
- if [ $count -eq 0 ]
- then
- echo -ne "Host $line \n IdentityFile ~/.ssh/tutorial.pem \n User root \n" >> ~/.ssh/config
- fi
-done < hosts.txt
-
-count=$(grep StrictHostKeyChecking ~/.ssh/config | wc -l)
-if [ $count -eq 0 ]
-then
- echo -ne "\n Host * \n StrictHostKeyChecking=no \n" >> ~/.ssh/config
-fi
-
-export WORKER_HOSTS=`cat hosts.txt`
-chmod 0600 ~/.ssh/tutorial.pem
-
-echo "Starting ssh-agent"
-eval $(ssh-agent)
-ssh-add ~/.ssh/tutorial.pem
-
-echo "Starting Swift coaster service"
-start-coaster-service
-
-mv sites.xml cloud.xml
-grep -v config cloud.xml | (
- echo "<config>"
- cat
- addlocalpool
- echo "</config>"
-) >sites.xml
-
-echo "Swift cloud setup complete"
-rm -rf hi.* hi-*
-
-
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/stats.sh
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/stats.sh 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/stats.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,9 +0,0 @@
-#! /bin/sh
-
-awk '
-
-{ sum += $1}
-
-END { print sum/NR }
-' $*
-
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/stopcloud
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/stopcloud 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/stopcloud 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,8 +0,0 @@
-#!/bin/bash
-
-echo "stopping coaster service"
-stop-coaster-service
-
-echo "Killing ssh-agent"
-killall -u $USER ssh-agent
-
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/tc.data
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/tc.data 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/tc.data 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,111 +0,0 @@
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters bash /bin/bash null null null
-persistent-coasters echo /bin/echo null null null
Deleted: SwiftTutorials/CIC_2013-08-09/cloud/tutorial.pem
===================================================================
--- SwiftTutorials/CIC_2013-08-09/cloud/tutorial.pem 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/cloud/tutorial.pem 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,23 +0,0 @@
------BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAvPJTneRDhB5wlRsWEr2l6LoFi/PulPm0J0A1Vi2skzWYC9J6I4Ftb2YbSbtp
-3d+BJUSK3wyJqfYgY/ynC4vtbdG4HU7/Fr6R2BBKtfiiVHET1K3OTCp3JzFVqQoJoSYkrkgW7XE9
-9GF7hvpza8JVvV9JPwxOcuhDAVHQFkoIRAq1TK9VaRulZFJapubWnEHM2UoUrmBS10IZmUwGHqR5
-I6soUyBqapAYoQmL5fzqT+dhsnu7rXrUmF37Va5kX309N4Dp0D4/WJY7FeF9Q9to2i+G7HPA+gEY
-/ptYxlTk89l5MBIY1czZvz7Fso5NHCEMW1468mD/NtJIWBDS8x1yHQIDAQABAoIBAHY2ur//b4p6
-rP9s+etxwhqQQCYm0O9eJyujjXtEikNz9ZP39LjAImLfJFWGLx9E1f0mHCifJIY02mEUUsLzofHi
-k2mq4ggt5PukfcnKLG3ebYTT1U1sCC56PFjNUecEAVEVA1s9TYDXeLyjd5FE/fXEZwROQ8q6rClo
-MWOwn+lJII9La8lLFIBxXNQbyt2nwO26iSzw51wBYsWDglV/T6qnB9JOcQ+uyn8/e4kQYuD52iV3
-Vx3QN+F+YPC7ebYOS8kVR/bFo+JDggNU4jZ+Zk31Sbepfb7p7IdYTjtrlQAd48tIxdvKFpFqy0mC
-GDmkVKBAXSc7DNqy3osvq1DdtbUCgYEA+IwsADb2iY6QPt7KNl0edJRSjDmeR8wSV7QIJYm0mxTB
-tVUZn+cpdeDsK5RkJYzDNKy3gjXhsp3B9Ogsu1aO2KVuv2ffk4wUF6nFLASQifMebUvMtqZM3g3N
-/yhMh7jGqfEu3tCmRSztCm0K3AodTtxacyRZKlKM4iAe0tbIO9cCgYEAwpypzFQvX83a2X+SU9rz
-rb7S10U2sZz3YwCHTRr2BCS8EaFA5xuEZBDmGR8OegUMfmQweD57JboQKB95KpuojGUpVBt6XoFv
-SaVLg7Ehskp71XZAgG6D9AHV1mm7hGkGoqj9zdlScc+nEZw57UEPzp/hXTTtc9hapOvOH1l7IysC
-gYB7CRh2+r5MFdB97td21d3Po+TBGXyq2fByjeJCgspuwnWph/jpWQsDGVaPtCNnGyk+mAk3YejZ
-/JIY5B3QluabhDkwcwEQDxWbKSU7xv6t6zWysQUjnWsLuRWyK4bG4McnNQcYsXy9XaHxuUzY5ADi
-Gt6DhQjGlJq9DV10zXHNfQKBgQCsVbUbZxVInm+FmS3IKiUNn+TvkrAbWqfLuOUXx59CDhGo/wYN
-YdW8Eybf6huUvDHTHv+5C4Nb5kd2cGCI9aytHswZWffkBnS5X7HeXZY8Xy8H2Z7G3JbnFlT5eUn/
-uE2S71lJ13/lnwAQXSsCyVGpJPWZ50u7AUVLtDEkm8WqXwKBgQDh0X21ufkWeparJ4GxWjodxnYK
-i1mtzghE3FjhcrrKBKJ3Sm5ZUYPVbOe97071zB+u2S0a83/tHP9Sr5KuS1sHiN++IB2FVo7UMBMC
-1Ktg+yK9MoERr1xzNrqeR59U5uvRBypd8MSzxuk4LjePYFYeyFaz8GOusCw/ZPU4R83zWA==
------END RSA PRIVATE KEY-----
\ No newline at end of file
Modified: SwiftTutorials/CIC_2013-08-09/doc/README
===================================================================
--- SwiftTutorials/CIC_2013-08-09/doc/README 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/doc/README 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,6 +1,35 @@
-Swift CIC Tutorial - 2013.0815
+Swift CIC Tutorial - 2013.0827
==============================
+////
+
+Outline
+
+Introductory exercises
+
+p1 - Run an application under Swift
+
+p2 - Mapping (naming) output files
+
+p3 - Parallel loops with foreach
+
+p4 - Mapping arrays to files
+
+p5 - merging/reducing the results of a parallel foreach loop
+
+p6 - Sending arguments to applications
+
+p7 - Running on the remote site nodes
+
+p8 - Running the stats summary step on the remote site
+
+p9 - A more complex workflow pattern: multiple parallel pipelines
+
+////
+
+
+
+
Workflow tutorial setup
-----------------------
Modified: SwiftTutorials/CIC_2013-08-09/doc/TODO
===================================================================
--- SwiftTutorials/CIC_2013-08-09/doc/TODO 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/doc/TODO 2013-08-21 20:44:20 UTC (rev 6922)
@@ -10,8 +10,51 @@
Show variety of UC3 activity and what ran where
+get hostname and app_resources into provider staging (hostname into all _swiftwrap)
+Show gridftp and go connect to process the modis dataset
+Show SwiftR for R users, SwiftPy for Enthought PY?
+
+fetching apps vs sending apps vs parrot/cvmfs/oasis etc ????
+
+Demo script-aps, bin apps, building apps, caching apps. etc.
+
+Do a Swift BLAST workflow -- with viz !!! (spice it up with help from Dina)
+
+OSG page from Mats: "Is your job HTC-ready"
+
+Use new Trunk config in tutorial?
+
+
+Add viz and status display; add node probing to tutorial.
+
+Debug failure with pinfiles
+
+
+What "exercises to suggest?"
+
+- run stats app locally
+- installed vs transferred apps
+- transferred programs vs transferred scripts
+
+Y-style dag: 2 sims and average
+
+Show how to find temp files at runtime...
+
+Show how to find hung processes at runtime... (or get other trace data...)
+
+make your owb versions of the programs; verify that you can substitute them
+
+Add a layer to isolate shell version v installed version
+
+Show how to handle a paramfile
+
+Test if OSGCOnnect works and account string (PorjectName) works OK for both!
+
+
+
+
ISSUES
error in pin files? debug and turn back on
Modified: SwiftTutorials/CIC_2013-08-09/part06/sites.xml
===================================================================
--- SwiftTutorials/CIC_2013-08-09/part06/sites.xml 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/part06/sites.xml 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,7 +1,7 @@
<config>
<pool handle="localhost">
<execution provider="coaster" jobmanager="local:local"/>
- <profile namespace="karajan" key="jobThrottle">0.03</profile>
+ <profile namespace="karajan" key="jobThrottle">0.20</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<filesystem provider="local"/>
<workdirectory>{env.HOME}/work</workdirectory>
Deleted: SwiftTutorials/CIC_2013-08-09/part08/p8.swift
===================================================================
--- SwiftTutorials/CIC_2013-08-09/part08/p8.swift 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/part08/p8.swift 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,41 +0,0 @@
-type file;
-
-# Applications to be called by this script
-
-file simulation_prog <"app/simulate.sh">;
-file analysis_prog <"app/stats.sh">;
-
-# app() functions for application programs to be called:
-
-app (file out) simulation (file prog, int timesteps, int sim_range, file bias_file, int scale, int sim_count)
-{
- sh @filename(prog) timesteps sim_range @filename(bias_file) scale sim_count stdout=@filename(out);
-}
-
-app (file out) analyze (file prog, file s[])
-{
- sh @filename(prog) @filenames(s) stdout=@filename(out);
-}
-
-# Command line params to this script:
-
-int nsim = @toInt(@arg("nsim", "10")); # number of simulation programs to run
-int steps = @toInt(@arg("steps", "1")); # number of "steps" each simulation (==seconds of runtime)
-int range = @toInt(@arg("range", "100")); # range of the generated random numbers
-int count = @toInt(@arg("count", "10")); # number of random numbers generated per simulation
-
-# Main script and data
-
-tracef("\n*** Script parameters: nsim=%i steps=%i range=%i count=%i\n\n", nsim, steps, range, count);
-
-file sims[]; # Array of files to hold each simulation output
-file bias<"bias.dat">; # Input data file to "bias" the numbers:
- # 1 line: scale offset ( N = n*scale + offset)
-foreach i in [0:nsim-1] {
- file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
- simout = simulation(simulation_prog, steps, range, bias, 100000, count);
- sims[i] = simout;
-}
-
-file stats<"output/stats.out">; # Final output file: average of all "simulations"
-stats = analyze(analysis_prog,sims);
Added: SwiftTutorials/CIC_2013-08-09/part08/rensemble.swift
===================================================================
--- SwiftTutorials/CIC_2013-08-09/part08/rensemble.swift (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/part08/rensemble.swift 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1 @@
+link ../scripts/rensemble.swift
\ No newline at end of file
Property changes on: SwiftTutorials/CIC_2013-08-09/part08/rensemble.swift
___________________________________________________________________
Added: svn:special
+ *
Modified: SwiftTutorials/CIC_2013-08-09/part08/sites.xml
===================================================================
--- SwiftTutorials/CIC_2013-08-09/part08/sites.xml 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/part08/sites.xml 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,20 +1,28 @@
<config>
<pool handle="uc3">
<execution provider="coaster" jobmanager="local:condor"/>
- <profile namespace="karajan" key="jobThrottle">2.00</profile>
+ <profile namespace="karajan" key="jobThrottle">5.00</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxtime">300</profile>
+ <profile namespace="globus" key="maxtime">3600</profile>
<profile namespace="globus" key="maxWalltime">00:01:00</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">20</profile>
+ <profile namespace="globus" key="highOverAllocation">10000</profile>
+ <profile namespace="globus" key="lowOverAllocation">10000</profile>
+ <profile namespace="globus" key="internalHostname">128.135.158.243</profile>
+ <profile namespace="globus" key="slots">1</profile>
<profile namespace="globus" key="maxNodes">1</profile>
<profile namespace="globus" key="nodeGranularity">1</profile>
+ <!-- For UC3: -->
<profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <!-- For OSGConnect -->
+ <!-- <profile namespace="globus" key="condor.+ProjectName">"swift"</profile> -->
<profile namespace="globus" key="jobType">nonshared</profile>
- <!-- UC3 --> <profile namespace="globus" key="condor.Requirements">regexp("uc3-c*", Machine)</profile>
+ <!-- UC3 --> <!-- <profile namespace="globus" key="condor.Requirements">regexp("uc3-c*", Machine)</profile> -->
+ <!-- UCIT --> <profile namespace="globus" key="condor.Requirements">regexp("appcloud[0-1][0-9].*", Machine)</profile>
+ <!-- MWUC --> <!-- <profile namespace="globus" key="condor.Requirements">regexp("uct2-c*", Machine)</profile> -->
+ <!-- MWIU --> <!-- <profile namespace="globus" key="condor.Requirements">regexp("iut2-c*", Machine)</profile> -->
+ <!-- MWUI --> <!-- <profile namespace="globus" key="condor.Requirements">regexp("taub*", Machine)</profile> -->
<!-- MWT2 --> <!-- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile> -->
<!-- OSG --> <!-- <profile namespace="globus" key="condor.Requirements">isUndefined(GLIDECLIENT_Name) == FALSE</profile> -->
<!-- MULT --> <!-- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org" && (regexp("iut2-c*", Machine) || regexp("uct2-c*", Machine)) </profile> -->
@@ -23,3 +31,15 @@
</pool>
</config>
+<!--
+
+MWT2 @UC Requirements = regexp("uct2-c*", Machine)
+MWT2 @IU Requirements = regexp("iut2-c*", Machine)
+MWT2 @UIUC Requirements = regexp("taub*", Machine)
+UC ATLAS Tier 3 Requirements = regexp("uct3*", Machine)
+ITS Virtualization Lab Requirements = regexp("appcloud[0-1][0-9].*", Machine)
+Open Science Grid
+Requirements = isUndefined(GLIDECLIENT_Name) == FALSE
+
+ -->
+
Copied: SwiftTutorials/CIC_2013-08-09/scripts/rensemble.swift (from rev 6872, SwiftTutorials/CIC_2013-08-09/part08/p8.swift)
===================================================================
--- SwiftTutorials/CIC_2013-08-09/scripts/rensemble.swift (rev 0)
+++ SwiftTutorials/CIC_2013-08-09/scripts/rensemble.swift 2013-08-21 20:44:20 UTC (rev 6922)
@@ -0,0 +1,42 @@
+type file;
+
+# Applications to be called by this script
+
+file simulation_prog <"app/simulate.sh">;
+file analysis_prog <"app/stats.sh">;
+
+# app() functions for application programs to be called:
+
+app (file out, file err) simulation (file prog, int timesteps, int sim_range, file bias_file, int scale, int sim_count)
+{
+ sh @filename(prog) timesteps sim_range @filename(bias_file) scale sim_count stdout=@filename(out) stderr=@filename(err);
+}
+
+app (file out) analyze (file prog, file s[])
+{
+ sh @filename(prog) @filenames(s) stdout=@filename(out);
+}
+
+# Command line params to this script:
+
+int nsim = @toInt(@arg("nsim", "10")); # number of simulation programs to run
+int steps = @toInt(@arg("steps", "1")); # number of "steps" each simulation (==seconds of runtime)
+int range = @toInt(@arg("range", "100")); # range of the generated random numbers
+int count = @toInt(@arg("count", "10")); # number of random numbers generated per simulation
+
+# Main script and data
+
+tracef("\n*** Script parameters: nsim=%i steps=%i range=%i count=%i\n\n", nsim, steps, range, count);
+
+file sims[]; # Array of files to hold each simulation output
+file bias<"bias.dat">; # Input data file to "bias" the numbers:
+ # 1 line: scale offset ( N = n*scale + offset)
+foreach i in [0:nsim-1] {
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
+ (simout, simlog) = simulation(simulation_prog, steps, range, bias, 100000, count);
+ sims[i] = simout;
+}
+
+file stats<"output/stats.out">; # Final output file: average of all "simulations"
+stats = analyze(analysis_prog,sims);
Modified: SwiftTutorials/CIC_2013-08-09/setup.sh
===================================================================
--- SwiftTutorials/CIC_2013-08-09/setup.sh 2013-08-21 20:30:28 UTC (rev 6921)
+++ SwiftTutorials/CIC_2013-08-09/setup.sh 2013-08-21 20:44:20 UTC (rev 6922)
@@ -1,3 +1,4 @@
+
# ensure that this script is being sourced
if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" = "${0}" ] ; then
@@ -2,16 +3,41 @@
echo ERROR: script ${BASH_SOURCE[0]} must be executed as: source ${BASH_SOURCE[0]}
+ exit 1
fi
+# Add swift to PATH
+
+TUTSWIFT=/usr/local/swift/stable
+PATHSWIFT=$(which swift 2>/dev/null)
+
+if [ _$PATHSWIFT = _$TUTSWIFT/bin/swift ]; then
+ echo using Swift from $TUTSWIFT,already in PATH
+elif [ -x $TUTSWIFT/bin/swift ]; then
+ echo Using Swift from $TUTSWIFT, and adding to PATH
+ PATH=$TUTSWIFT/bin:$PATH
+elif [ _$PATHSWIFT != _ ]; then
+ echo Using $PATHSWIFT from PATH
+else
+ echo ERROR: $TUTSWIFT not found and no swift in PATH. Tutorial will not function.
+ return
+fi
+
+echo Swift version is $(swift -version)
+
# Setting scripts folder to the PATH env var.
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-echo Adding $DIR to PATH:
-PATH=$DIR/bin:$DIR/app:$PATH
+TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+if [ _$(which cleanup 2>/dev/null) != _$TUTDIR/bin/cleanup ]; then
+ echo Adding $TUTDIR/bin:$TUTDIR/app: to front of PATH
+ PATH=$TUTDIR/bin:$TUTDIR/app:$PATH
+else
+ echo Assuming $TUTDIR/bin:$TUTDIR/app: is already at front of PATH
+fi
+
# Setting .swift files
-if [ -e $HOME/.swift ]; then
- savedotswift=$(mktemp -d $HOME/.swift.save.XXXX)
- echo Saving $HOME/.swift in $savedotswift
- mv $HOME/.swift/* $savedotswift
+if [ -e $HOME/.swift/swift.properties ]; then
+ saveprop=$(mktemp $HOME/.swift/swift.properties.XXXX)
+ echo Saving $HOME/.swift/swift.properties in $saveprop
+ mv $HOME/.swift/swift.properties $saveprop
else
@@ -41,7 +67,6 @@
END
-
return
# OLD, to remove:
More information about the Swift-commit
mailing list