[Swift-commit] r7532 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Thu Jan 30 20:03:11 CST 2014
Author: ketan
Date: 2014-01-30 20:03:11 -0600 (Thu, 30 Jan 2014)
New Revision: 7532
Added:
SwiftApps/swift-galaxy/swift/swift-skeleton.sh
Modified:
SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
SwiftApps/swift-galaxy/swift/swiftforeachlist.xml
SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
SwiftApps/swift-galaxy/swift/swiftforeachrange.xml
Log:
user controlled outdir
Added: SwiftApps/swift-galaxy/swift/swift-skeleton.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-skeleton.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/swift-skeleton.sh 2014-01-31 02:03:11 UTC (rev 7532)
@@ -0,0 +1,187 @@
+#!/bin/bash
+
+#set -x
+set -e
+
+site=$1
+data=$2
+n=$3
+outdir=$4
+logfile=$5
+htmlfile=$6
+mkdir -p $outdir
+
+swift=`which swift`
+# use "here" document for configs
+
+cat << EOF > /tmp/script.swift
+### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+# Place your Swift script here
+# Make sure all data references are full-path
+### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
+EOF
+
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="midway">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+;;
+esac
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+stampede echo echo
+stampede cat cat
+uc3 echo echo
+uc3 cat cat
+midway echo echo
+midway cat cat
+EOF
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> $logfile
+EXITCODE=$?
+
+#for i in `find $outdir`
+#do
+# lastfile=$i
+#done
+#bname=`basename $lastfile`
+
+cat << EOF > $htmlfile
+<!DOCTYPE HTML>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>Catsn output</title>
+</head>
+<body>
+<h2>Swift run trace plots</h2>
+
+<a href="activeplot.png" alt="active plot">active plot</a>
+<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
+
+<h2>Browse output of catsn application</h2>
+`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
+</body>
+</html>
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+fi
+exit $EXITCODE
Property changes on: SwiftApps/swift-galaxy/swift/swift-skeleton.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-01-30 23:35:08 UTC (rev 7531)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-01-31 02:03:11 UTC (rev 7532)
@@ -15,6 +15,8 @@
shift
listfile=$1
shift
+outloc=$1
+shift
logfile=$1
shift
outlistfile=$1
@@ -30,8 +32,8 @@
file listfile<"$listfile">;
string list[] = readData(listfile);
foreach l, i in list{
- file out <single_file_mapper; file=@strcat("$HOME/swift-sandbox/", l, ".", i, ".out")>;
- file err <single_file_mapper; file=@strcat("$HOME/swift-sandbox/", l, ".", i, ".err")>;
+ file out <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".out")>;
+ file err <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".err")>;
(out,err) = anapp(exec, l);
}
EOF
Modified: SwiftApps/swift-galaxy/swift/swiftforeachlist.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.xml 2014-01-30 23:35:08 UTC (rev 7531)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.xml 2014-01-31 02:03:11 UTC (rev 7532)
@@ -6,6 +6,7 @@
$site
$executable
$listfile
+ $outloc
$log_file
$outlist
</command>
@@ -20,6 +21,7 @@
<param format="sh" name="executable" type="data" label="Executable"/>
<param format="data" name="listfile" type="data" label="List file"/>
+ <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc"></param>
<conditional name="configuration">
<param name="mode" type="select" label="Swift configuration">
Modified: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-01-30 23:35:08 UTC (rev 7531)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-01-31 02:03:11 UTC (rev 7532)
@@ -32,8 +32,8 @@
file exec<"$executable">;
foreach i in [$rstart:$rend:$stepsize]{
- file out <single_file_mapper; file=@strcat("$HOME/swift-sandbox/", i, ".out")>;
- file err <single_file_mapper; file=@strcat("$HOME/swift-sandbox/", i, ".err")>;
+ file out <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".out")>;
+ file err <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".err")>;
(out,err) = anapp(exec, i);
}
EOF
Modified: SwiftApps/swift-galaxy/swift/swiftforeachrange.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.xml 2014-01-30 23:35:08 UTC (rev 7531)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.xml 2014-01-31 02:03:11 UTC (rev 7532)
@@ -8,6 +8,7 @@
$rstart
$rend
$stepsize
+ $outloc
$logfile
$outlist
</command>
@@ -26,6 +27,8 @@
<param name="rend" size="2" type="integer" value="9" label="end"/>
<param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc"></param>
+
<conditional name="configuration">
<param name="mode" type="select" label="Swift configuration">
<option value="default">default configuration</option>
More information about the Swift-commit
mailing list