[Swift-commit] r7615 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Sun Feb 23 11:21:45 CST 2014
Author: ketan
Date: 2014-02-23 11:21:45 -0600 (Sun, 23 Feb 2014)
New Revision: 7615
Added:
SwiftApps/swift-galaxy/swift/genswift.sh
SwiftApps/swift-galaxy/swift/genswift.xml
Removed:
SwiftApps/swift-galaxy/swift/swiftforeachlist.xml
SwiftApps/swift-galaxy/swift/swiftforeachrange.xml
Log:
generate swift via Galaxy
Added: SwiftApps/swift-galaxy/swift/genswift.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/genswift.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/genswift.sh 2014-02-23 17:21:45 UTC (rev 7615)
@@ -0,0 +1,113 @@
+#!/bin/bash
+
+echo $@ > /tmp/args.txt
+wdir=$(mktemp -d /tmp/swift-gal.XXXX)
+
+# numeric range
+sites=$1 # comma separated list of sites
+shift
+swiftargs=$1
+shift
+interpreter=$1
+shift
+executable=$1
+shift
+rstart=$1
+shift
+rend=$1
+shift
+stepsize=$1
+shift
+outloc=$1
+shift
+outlistfile=$1
+shift
+logfile=$1
+shift
+
+stringargs=$1
+shift
+
+#workout the array only if user adds args else make it blank
+if [ "${stringargs}" ] ; then
+ stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
+else
+ stringarrayexpr='string stringargs[];'
+fi
+
+fileargs=()
+while [ $# -gt 0 ] ; do
+ fileargs+=("\"$1\"")
+ shift
+done
+
+#workout the array only if user adds file args else make it blank
+if [ ${#fileargs[@]} -gt 0 ] ; then
+ tmp=$(printf -- '%s,' "${fileargs[@]}")
+ filearrayexpr='file fileargs[]=['$(echo $tmp | sed s/,$//)'];';
+else
+ filearrayexpr='file fileargs[];'
+fi
+
+#Build Swift source code
+cat << EOF > $wdir/script.swift
+type file;
+
+app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){
+ $interpreter @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err;
+}
+
+$stringarrayexpr
+$filearrayexpr
+
+file exec<"$executable">;
+foreach i in [$rstart:$rend:$stepsize]{
+ file out <single_file_mapper; file=@strcat("$outloc", "/", i, ".out")>;
+ file err <single_file_mapper; file=@strcat("$outloc", "/", i, ".err")>;
+ (out,err) = anapp(exec, i, stringargs, fileargs);
+}
+EOF
+
+swift=$(which swift)
+
+#build site
+BASEDIR=$(dirname $0)
+$BASEDIR/buildsite.sh $sites $wdir
+
+#build config
+cat << EOF > $wdir/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+#build tc
+IFS=',' read -ra S <<< "$sites"
+for site in "${S[@]}" ; do
+cat << EOF >> $wdir/tc
+$site $interpreter $interpreter
+EOF
+done
+
+$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
+EXITCODE=$?
+
+#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
+#Populate output file
+cat << EOF > $outlistfile
+`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done`
+EOF
+
+#cleanup
+#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+fi
+
+exit $EXITCODE
+
+#dum ditty dum ditty dum dum dum
Property changes on: SwiftApps/swift-galaxy/swift/genswift.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/swift-galaxy/swift/genswift.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/genswift.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/genswift.xml 2014-02-23 17:21:45 UTC (rev 7615)
@@ -0,0 +1,170 @@
+<tool id="swiftforeach" name="foreach">
+ <!--
+ ToDo:
+ 1. Option if the counter should be an argument, first one or last one.
+ 2. Option to add files (in Galaxy dataset format) as arguments. These files then be added to Swift app as arguments.
+ 3. Allow the tool to select another tool and run it.
+ 4. Allow user to add more than one resource to run a job for load balancing.
+ 5. Add ability to run Swift/T.
+ 6. Let the localhost option be selected by default
+ -->
+ <description>A generic tool to run executable via Swift foreach parallel construct</description>
+
+ <command interpreter="bash">
+
+ #if $rangeorlist.rl == "rng"
+ swiftforeachrange.sh
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #elif $rangeorlist.rl == "lst"
+ swiftforeachlist.sh
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #else
+ swiftforeachtest.sh
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #end if
+ </command>
+ <!--
+ #if $swiftargs.mode == "advanced"
+ #for $sa in $swiftarg
+ ${sa.swiftargname}
+ #end for
+ #end if
+ -->
+ <inputs>
+ <param name="site" type="select" multiple="true" label="Execution Location" help="Multi-select list - hold the appropriate key while clicking to select multiple items">
+ <option value="localhost" selected="true">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ <option value="tukey">Tukey</option>
+ </param>
+
+ <param name="swiftargs" size="60" type="text" label="swiftargs" help="Add Swift level commandline arguments" />
+
+ <param name="interpret" type="select" label="Execution interpreter">
+ <option value="sh">sh</option>
+ <option value="python">python</option>
+ <option value="java">java</option>
+ <option value="R">R</option>
+ <option value="matlab">matlab</option>
+ </param>
+
+ <!--<param format="sh,binexec" name="exec" type="data" label="Executable"/>-->
+ <param name="exec" type="data" label="Executable" help="select executable" />
+
+ <conditional name="rangeorlist">
+ <param name="rl" type="select" label="Select range or list">
+ <option value="rng">numeric range</option>
+ <option value="lst">items list</option>
+ <option value="tst">dry test</option>
+ </param>
+
+ <when value="rng">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ </when>
+
+ <when value="lst">
+ <param format="data" name="listfile" type="data" label="List file" help="Select list file"/>
+ </when>
+
+ <when value="tst">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ <param format="data" name="listfile" type="data" label="List file"/>
+ </when>
+
+ </conditional>
+
+ <param name="stringargs" size="60" type="text" label="stringargs" help="Add application call commandline arguments here" />
+
+ <repeat name="filearg" title="file arg">
+ <param name="fileargname" type="data" label="filearg" />
+ </repeat>
+
+ <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc">
+ <sanitizer sanitize="False" />
+ </param>
+ </inputs>
+ <outputs>
+ <data format="txt" name="logfile" type="data" label="Swift output" />
+ <data format="txt" name="outlist" type="data" label="Output list" />
+ </outputs>
+
+ <!--
+ <code file="postprocess.py">
+ <hook postprocess="writeoutlist" />
+ </code>
+ -->
+
+<help>
+.. class:: warningmark
+
+**TIP**. Add args to provide additional arguments to your executable.
+
+-----
+
+**What it does**
+
+This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file.
+
+Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user.
+
+Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces.
+
+-----
+
+**Example**
+
+Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello".
+
+If an executable is echo.sh with following line:
+
+echo $@
+
+The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8:
+
+echo n hello
+
+</help>
+</tool>
Deleted: SwiftApps/swift-galaxy/swift/swiftforeachlist.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.xml 2014-02-23 00:47:07 UTC (rev 7614)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.xml 2014-02-23 17:21:45 UTC (rev 7615)
@@ -1,102 +0,0 @@
-<tool id="swiftforeachlist" name="listforeach">
- <description>A tool to run executable over an itemlist via Swift foreach parallel construct</description>
-
- <command interpreter="bash">
- swiftforeachlist.sh
- $site
- $interpreter
- $executable
- $listfile
- $outloc
- $log_file
- $outlist
- #for $a in $arg
- ${a.argname}
- #end for
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="localhost">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- </param>
-
- <param name="interpreter" type="select" label="Execution interpreter">
- <option value="sh">sh</option>
- <option value="python">python</option>
- <option value="java">java</option>
- <option value="R">R</option>
- <option value="matlab">matlab</option>
- </param>
-
- <param format="sh, binexec" name="executable" type="data" label="Executable"/>
- <param format="data" name="listfile" type="data" label="List file"/>
- <repeat name="arg" title="arg">
- <param name="argname" type="text" label="arg" />
- </repeat>
-
- <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc">
- <sanitizer sanitize="False"/>
- </param>
-
- <conditional name="configuration">
- <param name="mode" type="select" label="Swift configuration">
- <option value="default">default configuration</option>
- <option value="advanced">advanced configuration</option>
- </param>
-
- <when value="default">
- </when>
- <when value="advanced">
- <param name="remoteurl" size="50" type="text" value="swift.rcc.uchicago.edu" label="remote url" help="remote resource to run jobs on"></param>
- <param name="throttle" size="5" type="float" value="0.07" label="job throttle" help="number of parallel jobs to run"></param>
- <param name="project" type="text" value="TG-STA110005S" label="Project allocation" help="name/code of project"></param>
- <param name="slots" size="5" type="integer" value="1" label="slots" help="number of scheduler jobs"></param>
- <param name="queue" size="5" type="text" value="normal" label="queue" help="queue to run jobs on"></param>
- <param name="nodes" size="5" type="integer" value="1" label="nodes" help="number of nodes requested"></param>
- <param name="nodegranularity" size="5" type="integer" value="1" label="node granularity" help="node granularity"></param>
- <param name="jobspernode" size="5" type="integer" value="8" label="jobs per node" help="number of jobs per node requested"></param>
- <param name="ppn" size="5" type="integer" value="8" label="ppn" help="processes per node requested"></param>
- <param name="walltime" size="5" type="text" value="00:10:00" label="Job walltime" help="time in hh:mm:ss to request to scheduler for this job"></param>
- <param name="maxtime" size="5" type="integer" value="700" label="application maxtime" help="Application maxtime in seconds for this job"></param>
- </when>
- </conditional>
- </inputs>
-
- <outputs>
- <data format="txt" name="log_file" type="data" label="Swift output" />
- <data format="txt" name="outlist" type="data" label="Output list" />
- </outputs>
-
-<help>
-.. class:: warningmark
-
-**TIP**. A generic Swift foreach to run arbitrary executables in parallel over a list of items listed in a file.
-
------
-
-**What it does**
-
-This is a Swift tool.
-
------
-
-**Example**
-
-To run an executable in parallel over a list of parameters read from an input file, chose an executable from the Executable dropdown and a list containing items as parameters from List File.
-
-A sample list file is:
-
-/home/ketan/afile.txt
-/home/ketan/bfile.txt
-/home/ketan/cfile.txt
-
-A sample executable upload.sh is:
-
-scp $1 aserver.adomain.edu:~/
-
-The tool will run thrice invoking the above scp command in parallel for three items listed in the list file.
-</help>
-</tool>
Deleted: SwiftApps/swift-galaxy/swift/swiftforeachrange.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.xml 2014-02-23 00:47:07 UTC (rev 7614)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.xml 2014-02-23 17:21:45 UTC (rev 7615)
@@ -1,113 +0,0 @@
-<tool id="swiftforeachrange" name="numrangeforeach">
- <description>A generic tool to run executable via Swift foreach parallel construct</description>
-
- <command interpreter="bash">
- swiftforeachrange.sh
- $site
- $interpreter
- $executable
- $rstart
- $rend
- $stepsize
- $outloc
- $logfile
- $outlist
- #for $a in $arg
- ${a.argname}
- #end for
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="localhost">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- </param>
-
- <param name="interpreter" type="select" label="Execution interpreter">
- <option value="sh">sh</option>
- <option value="python">python</option>
- <option value="java">java</option>
- <option value="R">R</option>
- <option value="matlab">matlab</option>
- </param>
-
- <param format="sh,binexec" name="executable" type="data" label="Executable"/>
-
- <param name="rstart" size="2" type="integer" value="0" label="start"/>
- <param name="rend" size="2" type="integer" value="9" label="end"/>
- <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
- <repeat name="arg" title="arg">
- <param name="argname" type="text" label="arg" />
- </repeat>
-
- <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc">
- <sanitizer sanitize="False" />
- </param>
-
- <conditional name="configuration">
- <param name="mode" type="select" label="Swift configuration">
- <option value="default">default configuration</option>
- <option value="advanced">advanced configuration</option>
- </param>
-
- <when value="default">
- </when>
- <when value="advanced">
- <param name="remoteurl" size="50" type="text" value="midway.swift.rcc.uchicago.edu" label="remote url" help="remote resource to run jobs on"></param>
- <param name="throttle" size="5" type="float" value="0.07" label="job throttle" help="number of parallel jobs to run"></param>
- <param name="project" type="text" value="TG-STA110005S" label="Project allocation" help="name/code of project"></param>
- <param name="slots" size="5" type="integer" value="1" label="slots" help="number of scheduler jobs"></param>
- <param name="queue" size="5" type="text" value="normal" label="queue" help="queue to run jobs on"></param>
- <param name="nodes" size="5" type="integer" value="1" label="nodes" help="number of nodes requested"></param>
- <param name="nodegranularity" size="5" type="integer" value="1" label="node granularity" help="node granularity"></param>
- <param name="jobspernode" size="5" type="integer" value="8" label="jobs per node" help="number of jobs per node requested"></param>
- <param name="ppn" size="5" type="integer" value="8" label="ppn" help="processes per node requested"></param>
- <param name="walltime" size="5" type="text" value="00:10:00" label="Job walltime" help="time in hh:mm:ss to request to scheduler for this job"></param>
- <param name="maxtime" size="5" type="integer" value="700" label="application maxtime" help="Application maxtime in seconds for this job"></param>
- </when>
- </conditional>
- </inputs>
- <outputs>
- <data format="txt" name="logfile" type="data" label="Swift output" />
- <data format="txt" name="outlist" type="data" label="Output list" />
- </outputs>
-
- <!--
- <code file="postprocess.py">
- <hook postprocess="writeoutlist" />
- </code>
- -->
-
-<help>
-.. class:: warningmark
-
-**TIP**. Add args to provide additional arguments to your executable.
-
------
-
-**What it does**
-
-This is a generic Swift tool that runs an executable over a range of numbers with arbitrary stepsize.
-
-Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user.
-
-Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces.
-
------
-
-**Example**
-
-Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello".
-
-If an executable is echo.sh with following line:
-
-echo $@
-
-The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8:
-
-echo n hello
-
-</help>
-</tool>
More information about the Swift-commit
mailing list