[Swift-commit] r7515 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Sat Jan 25 22:46:44 CST 2014
Author: ketan
Date: 2014-01-25 22:46:44 -0600 (Sat, 25 Jan 2014)
New Revision: 7515
Added:
SwiftApps/swift-galaxy/swift/swiftforeach.sh
SwiftApps/swift-galaxy/swift/swiftforeach.xml
Modified:
SwiftApps/swift-galaxy/swift/genericswift.sh
SwiftApps/swift-galaxy/swift/genericswift.xml
Log:
first version of a generic swift foreach for Galaxy
Modified: SwiftApps/swift-galaxy/swift/genericswift.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/genericswift.sh 2014-01-26 02:44:01 UTC (rev 7514)
+++ SwiftApps/swift-galaxy/swift/genericswift.sh 2014-01-26 04:46:44 UTC (rev 7515)
@@ -1,9 +1,34 @@
#!/bin/bash
site=$1
-script=$2
-logfile=$3
+shift
+script=$1
+shift
+logfile=$1
+shift
+remoteurl=$1
+shift
+throttle=$1
+shift
+project=$1
+shift
+slots=$1
+shift
+queue=$1
+shift
+nodes=$1
+shift
+nodegranularity=$1
+shift
+jobspernode=$1
+shift
+ppn=$1
+shift
+walltime=$1
+shift
+maxtime=$1
+
ln -s $script script.swift
swift=$(which swift)
# use "here" document for configs
Modified: SwiftApps/swift-galaxy/swift/genericswift.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/genericswift.xml 2014-01-26 02:44:01 UTC (rev 7514)
+++ SwiftApps/swift-galaxy/swift/genericswift.xml 2014-01-26 04:46:44 UTC (rev 7515)
@@ -28,9 +28,9 @@
<inputs>
<param name="site" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
<option value="midway">Midway</option>
<option value="uc3">UC3</option>
- <option value="localhost">Localhost</option>
<option value="stampede">Stampede</option>
</param>
<param format="swift" name="script" type="data" label="Swift script"/>
Added: SwiftApps/swift-galaxy/swift/swiftforeach.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-01-26 04:46:44 UTC (rev 7515)
@@ -0,0 +1,243 @@
+#!/bin/bash
+
+if [ $# -gt 2 ]
+then
+ # numeric range
+ executable = $1
+ shift
+ rstart = $1
+ shift
+ rend = $1
+ shift
+ stepsize = $1
+
+ cat << EOF > /tmp/script.swift
+ type file;
+
+ (file _out, file _err) app anapp(file _anarg){
+ anapp @_anarg stdout=@_out stderr=@_err;
+ }
+
+ file out[];
+ file err[];
+ file anarg = <"somefile">;
+ foreach i in [$rstart:$rend:$stepsize]{
+ (out[i],err[i]) = anapp(anarg);
+ }
+EOF
+
+else
+ # readData
+ executable = $1
+ shift
+ raft = $1
+
+ cat << EOF > /tmp/script.swift
+ type file;
+
+ (file _out, file _err) app anapp(file _anarg){
+ anapp @_anarg stdout=@_out stderr=@_err;
+ }
+
+ file out[];
+ file err[];
+ file raft<"$raft">;
+ file anarg = <"somefile">;
+ string arglist=readData(raft);
+
+ foreach i in arglist{
+ (out[i],err[i]) = anapp(anarg);
+ }
+EOF
+
+fi
+
+swift=$(which swift)
+# use "here" document for configs
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+midway echo echo
+midway cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="stampede">
+ <!--<execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>-->
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+stampede echo echo
+stampede cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+esac
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/script.swift 2> $logfile
+EXITCODE=$?
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+#cp std.err $logfile
+#rm std.err
+exit $EXITCODE
Property changes on: SwiftApps/swift-galaxy/swift/swiftforeach.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/swift-galaxy/swift/swiftforeach.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-01-26 04:46:44 UTC (rev 7515)
@@ -0,0 +1,80 @@
+<tool id="swift" name="Generic-Swift">
+ <description>A generic Swift tool to run Swift workflows</description>
+
+ <command interpreter="bash">
+ #if $configuration.mode == "default"
+ genericswift.sh
+ $site
+ $script
+ $log_file
+ #else
+ genericswift.sh
+ $site
+ $script
+ $log_file
+ $remoteurl
+ $throttle
+ $project
+ $slots
+ $queue
+ $nodes
+ $nodegranularity
+ $jobspernode
+ $ppn
+ $walltime
+ $maxtime
+ #end if
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param format="swift" name="script" type="data" label="Swift script"/>
+ <conditional name="configuration">
+ <param name="mode" type="select" label="Swift configuration">
+ <option value="default">default configuration</option>
+ <option value="advanced">advanced configuration</option>
+ </param>
+ <when value="default">
+ </when>
+ <when value="advanced">
+ <param name="remoteurl" size="50" type="text" value="midway.swift.rcc.uchicago.edu" label="remote url" help="remote resource to run jobs on"></param>
+ <param name="throttle" size="5" type="float" value="0.07" label="job throttle" help="number of parallel jobs to run"></param>
+ <param name="project" type="text" value="TG-STA110005S" label="Project allocation" help="name/code of project"></param>
+ <param name="slots" size="5" type="integer" value="1" label="slots" help="number of scheduler jobs"></param>
+ <param name="queue" size="5" type="text" value="normal" label="queue" help="queue to run jobs on"></param>
+ <param name="nodes" size="5" type="integer" value="1" label="nodes" help="number of nodes requested"></param>
+ <param name="nodegranularity" size="5" type="integer" value="1" label="node granularity" help="node granularity"></param>
+ <param name="jobspernode" size="5" type="integer" value="8" label="jobs per node" help="number of jobs per node requested"></param>
+ <param name="ppn" size="5" type="integer" value="8" label="ppn" help="processes per node requested"></param>
+ <param name="walltime" size="5" type="text" value="00:10:00" label="Job walltime" help="time in hh:mm:ss to request to scheduler for this job"></param>
+ <param name="maxtime" size="5" type="integer" value="700" label="application maxtime" help="Application maxtime in seconds for this job"></param>
+ </when>
+ </conditional>
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Swift output" />
+ </outputs>
+ <help>
+.. class:: warningmark
+
+**TIP**. The data referenced in your Swift script must be absolute path.
+
+-----
+
+**What it does**
+
+This is a generic Swift tool. The tool allows you to select an arbritrary Swift script and run it on the resource of your choice. Currently supported resources are Midway, UC3, Stampede and localhost.
+
+-----
+
+**Example**
+
+Upload a Swift script via get data tools to bring it in history. The tool then will identify Swift scripts and will populate them. Select the resource from the "Execution Location" dropdown.
+
+</help>
+</tool>
More information about the Swift-commit
mailing list