[Swift-commit] r7579 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Fri Feb 7 11:33:50 CST 2014
Author: ketan
Date: 2014-02-07 11:33:50 -0600 (Fri, 07 Feb 2014)
New Revision: 7579
Modified:
SwiftApps/swift-galaxy/swift/buildsite.sh
SwiftApps/swift-galaxy/swift/sites.xml
SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
Log:
support for multiple sites
Modified: SwiftApps/swift-galaxy/swift/buildsite.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/buildsite.sh 2014-02-07 15:49:49 UTC (rev 7578)
+++ SwiftApps/swift-galaxy/swift/buildsite.sh 2014-02-07 17:33:50 UTC (rev 7579)
@@ -9,69 +9,69 @@
# defaults to localhost of none of the above provided
echo $@ > /tmp/buildsiteargs.txt
# numeric range
-site=$1
+sites=$1
wdir=$2
# use "here" document for configs
-#pre='<?xml version="1.0" encoding="UTF-8"?>
-#<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">'
-
-case "$site" in
-midway )
-cat << EOF > $wdir/sites.xml
+cat << EOF > $wdir/pre.txt
<?xml version="1.0" encoding="UTF-8"?>
<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="midway">
- <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local" />
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
+EOF
+
+cat << EOF > $wdir/post.txt
</config>
EOF
+
+IFS=',' read -ra S <<< "$sites"
+for site in "${S[@]}" ; do
+case "$site" in
+midway )
+cat << EOF >> $wdir/pool.xml
+<pool handle="midway">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" />
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+EOF
;;
uc3 )
-cat << EOF > $wdir/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
+cat << EOF >> $wdir/pool.xml
+<pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
EOF
;;
stampede )
-cat << EOF > $wdir/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+cat << EOF >> $wdir/pool.xml
<pool handle="stampede">
<execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
<filesystem provider="local"/>
@@ -92,42 +92,35 @@
<profile namespace="karajan" key="initialScore">10000</profile>
<workdirectory>/tmp/swift.work</workdirectory>
</pool>
-</config>
EOF
;;
localhost )
-cat << EOF > $wdir/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+cat << EOF >> $wdir/pool.xml
<pool handle="localhost">
- <execution provider="local" />
- <profile namespace="karajan" key="jobThrottle">0.07</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local"/>
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="swift" key="stagingMethod">local</profile>
+ <execution provider="coaster" jobmanager="local:local" />
+ <profile namespace="karajan" key="jobThrottle">0.07</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/tmp/swift.work</workdirectory>
</pool>
-</config>
EOF
-
;;
* )
echo "defaulting to localhost"
-cat << EOF > $wdir/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+cat << EOF >> $wdir/pool.xml
<pool handle="localhost">
- <execution provider="local" />
+ <execution provider="coaster" jobmanager="local:local" />
<profile namespace="karajan" key="jobThrottle">0.07</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<filesystem provider="local"/>
<workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="swift" key="stagingMethod">local</profile>
</pool>
-</config>
EOF
;;
esac
+done
+cat $wdir/pre.txt $wdir/pool.xml $wdir/post.txt > $wdir/sites.xml
+
Modified: SwiftApps/swift-galaxy/swift/sites.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/sites.xml 2014-02-07 15:49:49 UTC (rev 7578)
+++ SwiftApps/swift-galaxy/swift/sites.xml 2014-02-07 17:33:50 UTC (rev 7579)
@@ -17,11 +17,11 @@
-->
- <pool handle="localhost">
- <filesystem provider="local" />
- <execution provider="local" />
- <workdirectory>/tmp/swift.workdir</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0</profile>
+<pool handle="localhost">
+ <filesystem provider="local" />
+ <execution provider="local" />
+ <workdirectory>/tmp/swift.workdir</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0</profile>
</pool>
<!-- The remainder of this file is commented out by default. It contains
Modified: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-02-07 15:49:49 UTC (rev 7578)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-02-07 17:33:50 UTC (rev 7579)
@@ -3,7 +3,7 @@
wdir=$(mktemp -d /tmp/swift-gal.XXXX)
# numeric range
-site=$1 # comma separated list of sites
+sites=$1 # comma separated list of sites
shift
interpreter=$1
shift
@@ -54,20 +54,10 @@
swift=$(which swift)
BASEDIR=$(dirname $0)
-$BASEDIR/buildsite.sh $site $wdir
+$BASEDIR/buildsite.sh $sites $wdir
#build config
-if [ "$site" = "localhost" ] ; then
cat << EOF > $wdir/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-else
-cat << EOF > $wdir/cf
use.provider.staging=true
wrapperlog.always.transfer=true
execution.retries=0
@@ -75,11 +65,14 @@
sitedir.keep=true
lazy.errors=false
EOF
-fi
-cat << EOF > $wdir/tc
+#build tc
+IFS=',' read -ra S <<< "$sites"
+for site in "${S[@]}" ; do
+cat << EOF >> $wdir/tc
$site $interpreter $interpreter
EOF
+done
$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift 2>$logfile
EXITCODE=$?
Modified: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-02-07 15:49:49 UTC (rev 7578)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-02-07 17:33:50 UTC (rev 7579)
@@ -4,7 +4,7 @@
wdir=$(mktemp -d /tmp/swift-gal.XXXX)
# numeric range
-site=$1 # comma separated list of sites
+sites=$1 # comma separated list of sites
shift
interpreter=$1
shift
@@ -58,20 +58,10 @@
#build site
BASEDIR=$(dirname $0)
-$BASEDIR/buildsite.sh $site $wdir
+$BASEDIR/buildsite.sh $sites $wdir
#build config
-if [ "$site" = "localhost" ] ; then
cat << EOF > $wdir/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-else
-cat << EOF > $wdir/cf
use.provider.staging=true
wrapperlog.always.transfer=true
execution.retries=0
@@ -79,12 +69,14 @@
sitedir.keep=true
lazy.errors=false
EOF
-fi
#build tc
-cat << EOF > $wdir/tc
+IFS=',' read -ra S <<< "$sites"
+for site in "${S[@]}" ; do
+cat << EOF >> $wdir/tc
$site $interpreter $interpreter
EOF
+done
$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift 2>$logfile
EXITCODE=$?
More information about the Swift-commit
mailing list