[Swift-commit] r7864 - in SwiftApps/swift-galaxy/swift: . old
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Mon May 12 14:42:49 CDT 2014
Author: ketan
Date: 2014-05-12 14:42:49 -0500 (Mon, 12 May 2014)
New Revision: 7864
Added:
SwiftApps/swift-galaxy/swift/old/
SwiftApps/swift-galaxy/swift/old/cf
SwiftApps/swift-galaxy/swift/old/face-it/
SwiftApps/swift-galaxy/swift/old/genswift.sh
SwiftApps/swift-galaxy/swift/old/swift-mysim
Removed:
SwiftApps/swift-galaxy/swift/cf
SwiftApps/swift-galaxy/swift/face-it/
SwiftApps/swift-galaxy/swift/genswift.sh
SwiftApps/swift-galaxy/swift/swift-mysim
Modified:
SwiftApps/swift-galaxy/swift/swiftforeach.py
Log:
move old stuff in old
Deleted: SwiftApps/swift-galaxy/swift/cf
===================================================================
--- SwiftApps/swift-galaxy/swift/cf 2014-05-12 19:10:34 UTC (rev 7863)
+++ SwiftApps/swift-galaxy/swift/cf 2014-05-12 19:42:49 UTC (rev 7864)
@@ -1,7 +0,0 @@
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-use.wrapper.staging=false
Deleted: SwiftApps/swift-galaxy/swift/genswift.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/genswift.sh 2014-05-12 19:10:34 UTC (rev 7863)
+++ SwiftApps/swift-galaxy/swift/genswift.sh 2014-05-12 19:42:49 UTC (rev 7864)
@@ -1,113 +0,0 @@
-#!/bin/bash
-
-echo $@ > /tmp/args.txt
-wdir=$(mktemp -d /tmp/swift-gal.XXXX)
-
-# numeric range
-sites=$1 # comma separated list of sites
-shift
-swiftargs=$1
-shift
-interpreter=$1
-shift
-executable=$1
-shift
-rstart=$1
-shift
-rend=$1
-shift
-stepsize=$1
-shift
-outloc=$1
-shift
-outlistfile=$1
-shift
-logfile=$1
-shift
-
-stringargs=$1
-shift
-
-#workout the array only if user adds args else make it blank
-if [ "${stringargs}" ] ; then
- stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
-else
- stringarrayexpr='string stringargs[];'
-fi
-
-fileargs=()
-while [ $# -gt 0 ] ; do
- fileargs+=("\"$1\"")
- shift
-done
-
-#workout the array only if user adds file args else make it blank
-if [ ${#fileargs[@]} -gt 0 ] ; then
- tmp=$(printf -- '%s,' "${fileargs[@]}")
- filearrayexpr='file fileargs[]=['$(echo $tmp | sed s/,$//)'];';
-else
- filearrayexpr='file fileargs[];'
-fi
-
-#Build Swift source code
-cat << EOF > $wdir/script.swift
-type file;
-
-app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){
- $interpreter @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err;
-}
-
-$stringarrayexpr
-$filearrayexpr
-
-file exec<"$executable">;
-foreach i in [$rstart:$rend:$stepsize]{
- file out <single_file_mapper; file=@strcat("$outloc", "/", i, ".out")>;
- file err <single_file_mapper; file=@strcat("$outloc", "/", i, ".err")>;
- (out,err) = anapp(exec, i, stringargs, fileargs);
-}
-EOF
-
-swift=$(which swift)
-
-#build site
-BASEDIR=$(dirname $0)
-$BASEDIR/buildsite.sh $sites $wdir
-
-#build config
-cat << EOF > $wdir/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-
-#build tc
-IFS=',' read -ra S <<< "$sites"
-for site in "${S[@]}" ; do
-cat << EOF >> $wdir/tc
-$site $interpreter $interpreter
-EOF
-done
-
-$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
-EXITCODE=$?
-
-#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
-#Populate output file
-cat << EOF > $outlistfile
-`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done`
-EOF
-
-#cleanup
-#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat $logfile >&2
-fi
-
-exit $EXITCODE
-
-#dum ditty dum ditty dum dum dum
Copied: SwiftApps/swift-galaxy/swift/old/cf (from rev 7862, SwiftApps/swift-galaxy/swift/cf)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/cf (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/cf 2014-05-12 19:42:49 UTC (rev 7864)
@@ -0,0 +1,7 @@
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+use.wrapper.staging=false
Copied: SwiftApps/swift-galaxy/swift/old/genswift.sh (from rev 7862, SwiftApps/swift-galaxy/swift/genswift.sh)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/genswift.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/genswift.sh 2014-05-12 19:42:49 UTC (rev 7864)
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+echo $@ > /tmp/args.txt
+wdir=$(mktemp -d /tmp/swift-gal.XXXX)
+
+# numeric range
+sites=$1 # comma separated list of sites
+shift
+swiftargs=$1
+shift
+interpreter=$1
+shift
+executable=$1
+shift
+rstart=$1
+shift
+rend=$1
+shift
+stepsize=$1
+shift
+outloc=$1
+shift
+outlistfile=$1
+shift
+logfile=$1
+shift
+
+stringargs=$1
+shift
+
+#workout the array only if user adds args else make it blank
+if [ "${stringargs}" ] ; then
+ stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
+else
+ stringarrayexpr='string stringargs[];'
+fi
+
+fileargs=()
+while [ $# -gt 0 ] ; do
+ fileargs+=("\"$1\"")
+ shift
+done
+
+#workout the array only if user adds file args else make it blank
+if [ ${#fileargs[@]} -gt 0 ] ; then
+ tmp=$(printf -- '%s,' "${fileargs[@]}")
+ filearrayexpr='file fileargs[]=['$(echo $tmp | sed s/,$//)'];';
+else
+ filearrayexpr='file fileargs[];'
+fi
+
+#Build Swift source code
+cat << EOF > $wdir/script.swift
+type file;
+
+app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){
+ $interpreter @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err;
+}
+
+$stringarrayexpr
+$filearrayexpr
+
+file exec<"$executable">;
+foreach i in [$rstart:$rend:$stepsize]{
+ file out <single_file_mapper; file=@strcat("$outloc", "/", i, ".out")>;
+ file err <single_file_mapper; file=@strcat("$outloc", "/", i, ".err")>;
+ (out,err) = anapp(exec, i, stringargs, fileargs);
+}
+EOF
+
+swift=$(which swift)
+
+#build site
+BASEDIR=$(dirname $0)
+$BASEDIR/buildsite.sh $sites $wdir
+
+#build config
+cat << EOF > $wdir/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+#build tc
+IFS=',' read -ra S <<< "$sites"
+for site in "${S[@]}" ; do
+cat << EOF >> $wdir/tc
+$site $interpreter $interpreter
+EOF
+done
+
+$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
+EXITCODE=$?
+
+#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
+#Populate output file
+cat << EOF > $outlistfile
+`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done`
+EOF
+
+#cleanup
+#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+fi
+
+exit $EXITCODE
+
+#dum ditty dum ditty dum dum dum
Copied: SwiftApps/swift-galaxy/swift/old/swift-mysim (from rev 7862, SwiftApps/swift-galaxy/swift/swift-mysim)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-mysim (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-mysim 2014-05-12 19:42:49 UTC (rev 7864)
@@ -0,0 +1,282 @@
+#!/bin/bash
+
+site=$1
+nsim=$2
+steps=$3
+range=$4
+values=$5
+outdir=$6
+logfile=$7
+htmlfile=$8
+
+mkdir -p $outdir
+
+swift=`which swift`
+# use "here" document for configs
+
+cat << EOF > /tmp/mysim.swift
+type file;
+
+app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values) {
+ simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log;
+}
+
+app (file out, file log) analyze (file s[]) {
+ stats @filenames(s) stdout=@out stderr=@log;
+}
+
+int nsim = @toInt(@arg("nsim", "10"));
+int steps = @toInt(@arg("steps", "1"));
+int range = @toInt(@arg("range", "100"));
+int values = @toInt(@arg("values", "5"));
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
+ (simout,simlog) = simulation(steps,range,values);
+ sims[i] = simout;
+}
+
+file stats_out<"output/average.out">;
+file stats_log<"output/average.log">;
+(stats_out, stats_log) = analyze(sims);
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+cat << 'EOF' > /tmp/tc
+swiftgal echo echo
+swiftgal cat cat
+swiftgal simulate simulate.sh
+swiftgal stats stats.sh
+EOF
+
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="swiftgal">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">4</profile>
+ <profile namespace="globus" key="maxNodes">2</profile>
+ <profile namespace="globus" key="nodeGranularity">2</profile>
+ <profile namespace="karajan" key="jobThrottle">1.27</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/scratch/midway/maheshwari/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="swiftgal">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="swiftgal">
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="swiftgal">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="swiftgal">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+;;
+esac
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/mysim.swift -nsim=$nsim -steps=$steps -range=$range -values=$values 2> std.err
+EXITCODE=$?
+
+swiftlog=`\ls -t *.log | head -1`
+
+cat << 'EOF' > /tmp/plotswiftlogs
+#!/bin/bash
+
+#usage: ./plotswiftlogs.ketan <swift-logfile.log>
+
+SWIFTLOGFILE=$1
+
+#TMPDIR=`mktemp -d plotlog.XXX`
+
+grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
+
+SWIFTOUTFILE=swiftoutfile.out
+
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+
+#extract active runs in a file
+(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt)
+
+#extract successful completions in a file
+(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+if [ $activelines -ne 0 ]
+then
+ activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+fi
+
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+if [ $cumulines -ne 0 ]
+then
+ cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+fi
+
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+cat << EOF1 > plotit.gp
+set terminal png enhanced
+set nokey
+set output "cumulativeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of completed apps"
+set title "Cumulative Apps Completed"
+plot "plot_cumulative.txt" using 1:2 with lines
+set output "activeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of active apps"
+set title "Active Apps"
+plot "plot_active.txt" using 1:2 with line
+EOF1
+
+gnuplot plotit.gp 2>/dev/null
+rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt
+
+EOF
+
+sh /tmp/plotswiftlogs $swiftlog
+cp *.png $outdir/
+
+cat << EOF > $htmlfile
+<!DOCTYPE HTML>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>Application output</title>
+</head>
+<body>
+<h2>Swift run trace plots</h2>
+
+<a href="activeplot.png" alt="active plot">active plot</a>
+<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
+
+<h2>Browse output of application</h2>
+`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
+</body>
+</html>
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+cp std.err $logfile
+rm std.err
+exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swift-mysim
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-mysim 2014-05-12 19:10:34 UTC (rev 7863)
+++ SwiftApps/swift-galaxy/swift/swift-mysim 2014-05-12 19:42:49 UTC (rev 7864)
@@ -1,282 +0,0 @@
-#!/bin/bash
-
-site=$1
-nsim=$2
-steps=$3
-range=$4
-values=$5
-outdir=$6
-logfile=$7
-htmlfile=$8
-
-mkdir -p $outdir
-
-swift=`which swift`
-# use "here" document for configs
-
-cat << EOF > /tmp/mysim.swift
-type file;
-
-app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values) {
- simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log;
-}
-
-app (file out, file log) analyze (file s[]) {
- stats @filenames(s) stdout=@out stderr=@log;
-}
-
-int nsim = @toInt(@arg("nsim", "10"));
-int steps = @toInt(@arg("steps", "1"));
-int range = @toInt(@arg("range", "100"));
-int values = @toInt(@arg("values", "5"));
-
-file sims[];
-
-foreach i in [0:nsim-1] {
- file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
- file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
- (simout,simlog) = simulation(steps,range,values);
- sims[i] = simout;
-}
-
-file stats_out<"output/average.out">;
-file stats_log<"output/average.log">;
-(stats_out, stats_log) = analyze(sims);
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-
-cat << 'EOF' > /tmp/tc
-swiftgal echo echo
-swiftgal cat cat
-swiftgal simulate simulate.sh
-swiftgal stats stats.sh
-EOF
-
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="swiftgal">
- <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">4</profile>
- <profile namespace="globus" key="maxNodes">2</profile>
- <profile namespace="globus" key="nodeGranularity">2</profile>
- <profile namespace="karajan" key="jobThrottle">1.27</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/scratch/midway/maheshwari/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="swiftgal">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="swiftgal">
- <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="swiftgal">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="swiftgal">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-;;
-esac
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/mysim.swift -nsim=$nsim -steps=$steps -range=$range -values=$values 2> std.err
-EXITCODE=$?
-
-swiftlog=`\ls -t *.log | head -1`
-
-cat << 'EOF' > /tmp/plotswiftlogs
-#!/bin/bash
-
-#usage: ./plotswiftlogs.ketan <swift-logfile.log>
-
-SWIFTLOGFILE=$1
-
-#TMPDIR=`mktemp -d plotlog.XXX`
-
-grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
-
-SWIFTOUTFILE=swiftoutfile.out
-
-#extract start time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
-START_TIME=`date +%s -d "$TMPDATE"`
-
-#extract end time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
-END_TIME=`date +%s -d "$TMPDATE"`
-
-#duration
-DIFFTIME=$((END_TIME - START_TIME))
-
-#extract active runs in a file
-(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt)
-
-#extract successful completions in a file
-(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
-
-#prepare tics
-activelines=`wc -l active.txt | awk '{print $1}'`
-cumulines=`wc -l cumulative.txt | awk '{print $1}'`
-
-if [ $activelines -ne 0 ]
-then
- activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
-fi
-
-seq 0 $activelinespertic $DIFFTIME > activetics.txt
-
-if [ $cumulines -ne 0 ]
-then
- cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
-fi
-
-seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
-
-#final plot data
-paste activetics.txt active.txt > plot_active.txt
-paste cumultics.txt cumulative.txt > plot_cumulative.txt
-
-cat << EOF1 > plotit.gp
-set terminal png enhanced
-set nokey
-set output "cumulativeplot.png"
-set xlabel "Time in sec"
-set ylabel "number of completed apps"
-set title "Cumulative Apps Completed"
-plot "plot_cumulative.txt" using 1:2 with lines
-set output "activeplot.png"
-set xlabel "Time in sec"
-set ylabel "number of active apps"
-set title "Active Apps"
-plot "plot_active.txt" using 1:2 with line
-EOF1
-
-gnuplot plotit.gp 2>/dev/null
-rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt
-
-EOF
-
-sh /tmp/plotswiftlogs $swiftlog
-cp *.png $outdir/
-
-cat << EOF > $htmlfile
-<!DOCTYPE HTML>
-
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
- <title>Application output</title>
-</head>
-<body>
-<h2>Swift run trace plots</h2>
-
-<a href="activeplot.png" alt="active plot">active plot</a>
-<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
-
-<h2>Browse output of application</h2>
-`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
-</body>
-</html>
-EOF
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-cp std.err $logfile
-rm std.err
-exit $EXITCODE
Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 19:10:34 UTC (rev 7863)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 19:42:49 UTC (rev 7864)
@@ -1,5 +1,12 @@
#!/usr/bin/env python
+"""
+This is the Swift foreach driver for Galaxy. It receives inputs from the Swift
+"foreach" Galaxy tool and depending on the options specified, builds a
+canonical Swift/K or Swift/T script and runs over specified computational
+environment.
+"""
+
import subprocess
import sys
import os
@@ -25,7 +32,6 @@
f.write("lazy.errors=false\n")
f.close()
-# numeric range
def main():
myargs = genargs(sys.argv)
try:
@@ -46,9 +52,11 @@
#print "stdin => " + stdin
if range_or_list == "list":
+ #list of items
listfile = next(myargs)
n = sum(1 for line in open(listfile))
else:
+ # numeric range
rstart = next(myargs)
rend = next(myargs)
stepsize = next(myargs)
@@ -62,7 +70,10 @@
except:
traceback.print_exc()
sys.exit(1)
+
+ #which swift
swift = distutils.spawn.find_executable("swift")
+
BASEDIR = os.path.dirname(os.path.realpath(__file__))
# workout the file args
@@ -93,7 +104,6 @@
#workout standard input file
if stdin != "None":
- print stdin
if k_or_t == "swiftK":
#print "stdinK"
stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n"
More information about the Swift-commit
mailing list