[Swift-commit] r7519 - in SwiftApps/swift-galaxy/swift: . test_exec
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Mon Jan 27 12:03:44 CST 2014
Author: ketan
Date: 2014-01-27 12:03:43 -0600 (Mon, 27 Jan 2014)
New Revision: 7519
Added:
SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
SwiftApps/swift-galaxy/swift/swiftforeachlist.xml
SwiftApps/swift-galaxy/swift/test_exec/
SwiftApps/swift-galaxy/swift/test_exec/alist.txt
SwiftApps/swift-galaxy/swift/test_exec/echo.sh
SwiftApps/swift-galaxy/swift/test_exec/ls.sh
Modified:
SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
Log:
for each list
Added: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-01-27 18:03:43 UTC (rev 7519)
@@ -0,0 +1,196 @@
+#!/bin/bash
+
+#ToDo:
+#
+#1. Add arguments, variable number such that user can add and remove them.
+#2. Add a way for users to specify expected outputs.
+#3. Add a type such that only binaries and executable scripts are populated in the executable list.
+#
+
+echo $@ > /tmp/args.txt
+# numeric range
+site=$1
+shift
+executable=$1
+shift
+listfile=$1
+shift
+logfile=$1
+
+cat << EOF > /tmp/script.swift
+type file;
+
+app (file _out, file _err) anapp(file _exec, string _arg){
+ sh @_exec _arg stdout=@_out stderr=@_err;
+}
+
+file out[];
+file err[];
+file exec<"$executable">;
+file listfile<"$listfile">;
+string list[] = readData(listfile);
+foreach l, i in list{
+ (out[i],err[i]) = anapp(exec, l);
+}
+EOF
+
+swift=$(which swift)
+
+# use "here" document for configs
+case "$site" in
+midway )
+cat << EOF > /tmp/sites.xml
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" />
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+midway echo echo
+midway sh /bin/sh
+midway cat cat
+EOF
+
+;;
+
+uc3 )
+cat << EOF > /tmp/sites.xml
+<config>
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+uc3 sh /bin/sh
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+EOF
+
+;;
+
+stampede )
+cat << EOF > /tmp/sites.xml
+<config>
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+stampede sh /bin/sh
+stampede echo echo
+stampede cat cat
+EOF
+
+;;
+
+localhost )
+cat << EOF > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+localhost sh /bin/sh
+localhost echo echo
+localhost cat cat
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+cat << EOF > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+localhost sh /bin/sh
+localhost echo echo
+localhost cat cat
+EOF
+
+;;
+esac
+
+cat << EOF > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/script.swift 2>$logfile
+EXITCODE=$?
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+fi
+exit $EXITCODE
Property changes on: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/swift-galaxy/swift/swiftforeachlist.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.xml 2014-01-27 18:03:43 UTC (rev 7519)
@@ -0,0 +1,65 @@
+<tool id="swiftforeachlist" name="Run any executable over an item list via Swift foreach parallel construct">
+ <description>A tool to run executable over an itemlist via Swift foreach parallel construct</description>
+
+ <command interpreter="bash">
+ swiftforeachlist.sh
+ $site
+ $executable
+ $listfile
+ $log_file
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ </param>
+
+ <param format="data" name="executable" type="data" label="Executable"/>
+ <param format="data" name="listfile" type="data" label="List file"/>
+
+ <conditional name="configuration">
+ <param name="mode" type="select" label="Swift configuration">
+ <option value="default">default configuration</option>
+ <option value="advanced">advanced configuration</option>
+ </param>
+
+ <when value="default">
+ </when>
+ <when value="advanced">
+ <param name="remoteurl" size="50" type="text" value="swift.rcc.uchicago.edu" label="remote url" help="remote resource to run jobs on"></param>
+ <param name="throttle" size="5" type="float" value="0.07" label="job throttle" help="number of parallel jobs to run"></param>
+ <param name="project" type="text" value="TG-STA110005S" label="Project allocation" help="name/code of project"></param>
+ <param name="slots" size="5" type="integer" value="1" label="slots" help="number of scheduler jobs"></param>
+ <param name="queue" size="5" type="text" value="normal" label="queue" help="queue to run jobs on"></param>
+ <param name="nodes" size="5" type="integer" value="1" label="nodes" help="number of nodes requested"></param>
+ <param name="nodegranularity" size="5" type="integer" value="1" label="node granularity" help="node granularity"></param>
+ <param name="jobspernode" size="5" type="integer" value="8" label="jobs per node" help="number of jobs per node requested"></param>
+ <param name="ppn" size="5" type="integer" value="8" label="ppn" help="processes per node requested"></param>
+ <param name="walltime" size="5" type="text" value="00:10:00" label="Job walltime" help="time in hh:mm:ss to request to scheduler for this job"></param>
+ <param name="maxtime" size="5" type="integer" value="700" label="application maxtime" help="Application maxtime in seconds for this job"></param>
+ </when>
+ </conditional>
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Swift output" />
+ </outputs>
+ <help>
+.. class:: warningmark
+
+**TIP**. A generic Swift foreach to run arbitrary executables in parallel over a list of items listed in a file.
+
+-----
+
+**What it does**
+
+This is a Swift tool.
+
+-----
+
+**Example**
+ToDo
+</help>
+</tool>
Modified: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-01-27 01:43:25 UTC (rev 7518)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-01-27 18:03:43 UTC (rev 7519)
@@ -1,5 +1,12 @@
#!/bin/bash
+#ToDo:
+#
+#1. Add arguments, variable number such that user can add and remove them.
+#2. Add a way for users to specify expected outputs.
+#3. Add a type such that only binaries and executable scripts are populated in the executable list.
+#
+
echo $@ > /tmp/args.txt
# numeric range
site=$1
@@ -17,15 +24,15 @@
cat << EOF > /tmp/script.swift
type file;
-app (file _out, file _err) anapp(file _exec){
- sh @_exec stdout=@_out stderr=@_err;
+app (file _out, file _err) anapp(file _exec, int _i){
+ sh @_exec _i stdout=@_out stderr=@_err;
}
file out[];
file err[];
file exec<"$executable">;
foreach i in [$rstart:$rend:$stepsize]{
- (out[i],err[i]) = anapp(exec);
+ (out[i],err[i]) = anapp(exec, i);
}
EOF
Added: SwiftApps/swift-galaxy/swift/test_exec/alist.txt
===================================================================
--- SwiftApps/swift-galaxy/swift/test_exec/alist.txt (rev 0)
+++ SwiftApps/swift-galaxy/swift/test_exec/alist.txt 2014-01-27 18:03:43 UTC (rev 7519)
@@ -0,0 +1,5 @@
+hello
+21
+55
+/tmp/args.txt
+
Added: SwiftApps/swift-galaxy/swift/test_exec/echo.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/test_exec/echo.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/test_exec/echo.sh 2014-01-27 18:03:43 UTC (rev 7519)
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+echo $@
+
Property changes on: SwiftApps/swift-galaxy/swift/test_exec/echo.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/swift-galaxy/swift/test_exec/ls.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/test_exec/ls.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/test_exec/ls.sh 2014-01-27 18:03:43 UTC (rev 7519)
@@ -0,0 +1 @@
+ls
Property changes on: SwiftApps/swift-galaxy/swift/test_exec/ls.sh
___________________________________________________________________
Added: svn:executable
+ *
More information about the Swift-commit
mailing list