[Swift-commit] r7541 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Sat Feb 1 15:34:43 CST 2014
Author: ketan
Date: 2014-02-01 15:34:43 -0600 (Sat, 01 Feb 2014)
New Revision: 7541
Added:
SwiftApps/swift-galaxy/swift/buildsite.sh
Modified:
SwiftApps/swift-galaxy/swift/genericswift.sh
SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
Log:
outsource buildsite
Added: SwiftApps/swift-galaxy/swift/buildsite.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/buildsite.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/buildsite.sh 2014-02-01 21:34:43 UTC (rev 7541)
@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# generates sites file in /tmp/sites.xml based on one of the following argument provided
+# localhost
+# stampede
+# midway
+# uc3
+
+# defaults to localhost of none of the above provided
+
+
+echo $@ > /tmp/args.txt
+# numeric range
+site=$1
+
+# use "here" document for configs
+case "$site" in
+midway )
+cat << EOF > /tmp/sites.xml
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" />
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+;;
+
+uc3 )
+cat << EOF > /tmp/sites.xml
+<config>
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+;;
+
+stampede )
+cat << EOF > /tmp/sites.xml
+<config>
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+;;
+
+localhost )
+cat << EOF > /tmp/sites.xml
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">0.07</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="swift" key="stagingMethod">local</profile>
+</pool>
+</config>
+EOF
+
+;;
+
+* )
+echo "defaulting to localhost"
+cat << EOF > /tmp/sites.xml
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">0.07</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="swift" key="stagingMethod">local</profile>
+</pool>
+</config>
+EOF
+
+;;
+esac
+
Property changes on: SwiftApps/swift-galaxy/swift/buildsite.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/swift-galaxy/swift/genericswift.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/genericswift.sh 2014-02-01 17:17:49 UTC (rev 7540)
+++ SwiftApps/swift-galaxy/swift/genericswift.sh 2014-02-01 21:34:43 UTC (rev 7541)
@@ -32,184 +32,34 @@
ln -s $script script.swift
swift=$(which swift)
# use "here" document for configs
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="midway">
- <execution provider="coaster" jobmanager="local:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
+./buildsite.sh $site
-cat << 'EOF' > /tmp/tc
-midway echo echo
-midway cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
+#build config
+cat << EOF > /tmp/cf
+use.provider.staging=false
wrapperlog.always.transfer=true
-execution.retries=1
+execution.retries=0
provider.staging.pin.swiftfiles=false
sitedir.keep=true
lazy.errors=false
EOF
-;;
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
+#build tc
+cat << EOF > /tmp/tc
+localhost sh /bin/sh
+localhost echo echo
+localhost cat cat
+uc3 sh /bin/sh
uc3 echo /bin/echo
uc3 cat /bin/cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="stampede">
- <!--<execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>-->
- <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
- <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
+stampede sh /bin/sh
stampede echo echo
stampede cat cat
+midway echo echo
+midway sh /bin/sh
+midway cat cat
EOF
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-esac
-
$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf script.swift 2> $logfile
EXITCODE=$?
Modified: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-02-01 17:17:49 UTC (rev 7540)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-02-01 21:34:43 UTC (rev 7541)
@@ -42,118 +42,10 @@
swift=$(which swift)
-# use "here" document for configs
-case "$site" in
-midway )
-cat << EOF > /tmp/sites.xml
-<config>
- <pool handle="midway">
- <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local" />
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-;;
+#build site
+./buildsite.sh $site
-uc3 )
-cat << EOF > /tmp/sites.xml
-<config>
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-;;
-
-stampede )
-cat << EOF > /tmp/sites.xml
-<config>
-<pool handle="stampede">
- <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-;;
-
-localhost )
-cat << EOF > /tmp/sites.xml
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="local" />
- <profile namespace="karajan" key="jobThrottle">0.07</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local"/>
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="swift" key="stagingMethod">local</profile>
-</pool>
-</config>
-EOF
-
-;;
-
-* )
-echo "defaulting to localhost"
-cat << EOF > /tmp/sites.xml
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="local" />
- <profile namespace="karajan" key="jobThrottle">0.07</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local"/>
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="swift" key="stagingMethod">local</profile>
-</pool>
-</config>
-EOF
-
-;;
-esac
-
+#build config
cat << EOF > /tmp/cf
use.provider.staging=false
wrapperlog.always.transfer=true
@@ -163,6 +55,7 @@
lazy.errors=false
EOF
+#build tc
cat << EOF > /tmp/tc
localhost sh /bin/sh
localhost echo echo
@@ -186,8 +79,14 @@
`for i in \`find $outloc -type f\`; do echo "$i"; done`
EOF
+#cleanup
+rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift
if [ "$EXITCODE" -ne "0" ]; then
cat $logfile >&2
fi
+
+
exit $EXITCODE
+
+#dum ditty dum ditty dum dum dum
More information about the Swift-commit
mailing list