[Swift-commit] r7517 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Sun Jan 26 17:26:20 CST 2014
Author: ketan
Date: 2014-01-26 17:26:19 -0600 (Sun, 26 Jan 2014)
New Revision: 7517
Added:
SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
SwiftApps/swift-galaxy/swift/swiftforeachrange.xml
Removed:
SwiftApps/swift-galaxy/swift/swiftforeach.sh
SwiftApps/swift-galaxy/swift/swiftforeach.xml
Log:
swift for each for Galaxy, version numrange
Deleted: SwiftApps/swift-galaxy/swift/swiftforeach.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-01-26 05:29:34 UTC (rev 7516)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-01-26 23:26:19 UTC (rev 7517)
@@ -1,243 +0,0 @@
-#!/bin/bash
-
-if [ $# -gt 2 ]
-then
- # numeric range
- executable = $1
- shift
- rstart = $1
- shift
- rend = $1
- shift
- stepsize = $1
-
- cat << EOF > /tmp/script.swift
- type file;
-
- (file _out, file _err) app anapp(file _anarg){
- anapp @_anarg stdout=@_out stderr=@_err;
- }
-
- file out[];
- file err[];
- file anarg = <"somefile">;
- foreach i in [$rstart:$rend:$stepsize]{
- (out[i],err[i]) = anapp(anarg);
- }
-EOF
-
-else
- # readData
- executable = $1
- shift
- raft = $1
-
- cat << EOF > /tmp/script.swift
- type file;
-
- (file _out, file _err) app anapp(file _anarg){
- anapp @_anarg stdout=@_out stderr=@_err;
- }
-
- file out[];
- file err[];
- file raft<"$raft">;
- file anarg = <"somefile">;
- string arglist=readData(raft);
-
- foreach i in arglist{
- (out[i],err[i]) = anapp(anarg);
- }
-EOF
-
-fi
-
-swift=$(which swift)
-# use "here" document for configs
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="midway">
- <execution provider="coaster" jobmanager="local:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-midway echo echo
-midway cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-uc3 echo /bin/echo
-uc3 cat /bin/cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="stampede">
- <!--<execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>-->
- <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
- <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-stampede echo echo
-stampede cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-esac
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/script.swift 2> $logfile
-EXITCODE=$?
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-#cp std.err $logfile
-#rm std.err
-exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swiftforeach.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-01-26 05:29:34 UTC (rev 7516)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-01-26 23:26:19 UTC (rev 7517)
@@ -1,80 +0,0 @@
-<tool id="swift" name="Generic-Swift">
- <description>A generic Swift tool to run Swift workflows</description>
-
- <command interpreter="bash">
- #if $configuration.mode == "default"
- genericswift.sh
- $site
- $script
- $log_file
- #else
- genericswift.sh
- $site
- $script
- $log_file
- $remoteurl
- $throttle
- $project
- $slots
- $queue
- $nodes
- $nodegranularity
- $jobspernode
- $ppn
- $walltime
- $maxtime
- #end if
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="localhost">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- </param>
- <param format="swift" name="script" type="data" label="Swift script"/>
- <conditional name="configuration">
- <param name="mode" type="select" label="Swift configuration">
- <option value="default">default configuration</option>
- <option value="advanced">advanced configuration</option>
- </param>
- <when value="default">
- </when>
- <when value="advanced">
- <param name="remoteurl" size="50" type="text" value="midway.swift.rcc.uchicago.edu" label="remote url" help="remote resource to run jobs on"></param>
- <param name="throttle" size="5" type="float" value="0.07" label="job throttle" help="number of parallel jobs to run"></param>
- <param name="project" type="text" value="TG-STA110005S" label="Project allocation" help="name/code of project"></param>
- <param name="slots" size="5" type="integer" value="1" label="slots" help="number of scheduler jobs"></param>
- <param name="queue" size="5" type="text" value="normal" label="queue" help="queue to run jobs on"></param>
- <param name="nodes" size="5" type="integer" value="1" label="nodes" help="number of nodes requested"></param>
- <param name="nodegranularity" size="5" type="integer" value="1" label="node granularity" help="node granularity"></param>
- <param name="jobspernode" size="5" type="integer" value="8" label="jobs per node" help="number of jobs per node requested"></param>
- <param name="ppn" size="5" type="integer" value="8" label="ppn" help="processes per node requested"></param>
- <param name="walltime" size="5" type="text" value="00:10:00" label="Job walltime" help="time in hh:mm:ss to request to scheduler for this job"></param>
- <param name="maxtime" size="5" type="integer" value="700" label="application maxtime" help="Application maxtime in seconds for this job"></param>
- </when>
- </conditional>
- </inputs>
- <outputs>
- <data format="txt" name="log_file" type="data" label="Swift output" />
- </outputs>
- <help>
-.. class:: warningmark
-
-**TIP**. The data referenced in your Swift script must be absolute path.
-
------
-
-**What it does**
-
-This is a generic Swift tool. The tool allows you to select an arbritrary Swift script and run it on the resource of your choice. Currently supported resources are Midway, UC3, Stampede and localhost.
-
------
-
-**Example**
-
-Upload a Swift script via get data tools to bring it in history. The tool then will identify Swift scripts and will populate them. Select the resource from the "Execution Location" dropdown.
-
-</help>
-</tool>
Copied: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh (from rev 7515, SwiftApps/swift-galaxy/swift/swiftforeach.sh)
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-01-26 23:26:19 UTC (rev 7517)
@@ -0,0 +1,189 @@
+#!/bin/bash
+
+# numeric range
+site = $1
+shift
+executable = $1
+shift
+rstart = $1
+shift
+rend = $1
+shift
+stepsize = $1
+shift
+logfile = $1
+
+cat << EOF > /tmp/script.swift
+type file;
+
+(file _out, file _err) app anapp(){
+ anapp stdout=@_out stderr=@_err;
+}
+
+file out[];
+file err[];
+foreach i in [$rstart:$rend:$stepsize]{
+ (out[i],err[i]) = anapp();
+}
+EOF
+
+swift=$(which swift)
+
+# use "here" document for configs
+case "$site" in
+midway )
+cat << EOF > /tmp/sites.xml
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" />
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+midway echo echo
+midway anapp $executable
+midway cat cat
+EOF
+
+;;
+
+uc3 )
+cat << EOF > /tmp/sites.xml
+<config>
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+uc3 anapp $executable
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+EOF
+
+;;
+
+stampede )
+cat << EOF > /tmp/sites.xml
+<config>
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+stampede anapp $executable
+stampede echo echo
+stampede cat cat
+EOF
+
+;;
+
+localhost )
+cat << EOF > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+localhost anapp $executable
+localhost echo echo
+localhost cat cat
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+cat << EOF > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+localhost anapp $executable
+localhost echo echo
+localhost cat cat
+EOF
+
+;;
+esac
+
+cat << EOF > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/script.swift 2> $logfile
+EXITCODE=$?
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/swiftforeachrange.xml (from rev 7515, SwiftApps/swift-galaxy/swift/swiftforeach.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.xml 2014-01-26 23:26:19 UTC (rev 7517)
@@ -0,0 +1,92 @@
+<tool id="swiftforeachrange" name="Run any executable over a range via Swift foreach parallel construct">
+ <description>A generic tool to run executable via Swift foreach parallel construct</description>
+
+ <command interpreter="bash">
+ #if $configuration.mode == "default"
+ swiftforeachrange.sh
+ $site
+ $executable
+ $rstart
+ $rend
+ $stepsize
+ $log_file
+ #else
+ swiftforeachrange.sh
+ $site
+ $executable
+ $rstart
+ $rend
+ $stepsize
+ $log_file
+ $remoteurl
+ $throttle
+ $project
+ $slots
+ $queue
+ $nodes
+ $nodegranularity
+ $jobspernode
+ $ppn
+ $walltime
+ $maxtime
+ #end if
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ </param>
+
+ <param format="data" name="executable" type="data" label="Executable"/>
+
+ <param name="rstart" size="2" type="integer" value="0" label="Executable"/>
+ <param name="rend" size="2" type="integer" value="9" label="Executable"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="Executable"/>
+
+ <conditional name="configuration">
+ <param name="mode" type="select" label="Swift configuration">
+ <option value="default">default configuration</option>
+ <option value="advanced">advanced configuration</option>
+ </param>
+
+ <when value="default">
+ </when>
+ <when value="advanced">
+ <param name="remoteurl" size="50" type="text" value="midway.swift.rcc.uchicago.edu" label="remote url" help="remote resource to run jobs on"></param>
+ <param name="throttle" size="5" type="float" value="0.07" label="job throttle" help="number of parallel jobs to run"></param>
+ <param name="project" type="text" value="TG-STA110005S" label="Project allocation" help="name/code of project"></param>
+ <param name="slots" size="5" type="integer" value="1" label="slots" help="number of scheduler jobs"></param>
+ <param name="queue" size="5" type="text" value="normal" label="queue" help="queue to run jobs on"></param>
+ <param name="nodes" size="5" type="integer" value="1" label="nodes" help="number of nodes requested"></param>
+ <param name="nodegranularity" size="5" type="integer" value="1" label="node granularity" help="node granularity"></param>
+ <param name="jobspernode" size="5" type="integer" value="8" label="jobs per node" help="number of jobs per node requested"></param>
+ <param name="ppn" size="5" type="integer" value="8" label="ppn" help="processes per node requested"></param>
+ <param name="walltime" size="5" type="text" value="00:10:00" label="Job walltime" help="time in hh:mm:ss to request to scheduler for this job"></param>
+ <param name="maxtime" size="5" type="integer" value="700" label="application maxtime" help="Application maxtime in seconds for this job"></param>
+ </when>
+ </conditional>
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Swift output" />
+ </outputs>
+ <help>
+.. class:: warningmark
+
+**TIP**. A generic Swift foreach to run arbitrary executables in parallel over a range of numbers.
+
+-----
+
+**What it does**
+
+This is a generic Swift tool.
+
+-----
+
+**Example**
+
+ToDo
+</help>
+</tool>
More information about the Swift-commit
mailing list