[Swift-commit] r7509 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Sat Jan 25 10:11:31 CST 2014
Author: ketan
Date: 2014-01-25 10:11:30 -0600 (Sat, 25 Jan 2014)
New Revision: 7509
Added:
SwiftApps/swift-galaxy/swift/genericswift.xml
SwiftApps/swift-galaxy/swift/swift-galaxy.sh
Removed:
SwiftApps/swift-galaxy/swift/swift-galaxy
SwiftApps/swift-galaxy/swift/swift.xml
Modified:
SwiftApps/swift-galaxy/swift/catsn.swift
Log:
generic swift
Modified: SwiftApps/swift-galaxy/swift/catsn.swift
===================================================================
--- SwiftApps/swift-galaxy/swift/catsn.swift 2014-01-24 21:53:49 UTC (rev 7508)
+++ SwiftApps/swift-galaxy/swift/catsn.swift 2014-01-25 16:11:30 UTC (rev 7509)
@@ -5,7 +5,7 @@
}
file out[]<simple_mapper; location="outdir", prefix="f.",suffix=".out">;
-foreach j in [1:1000] {
- file data<single_file_mapper; file="data.txt">;
+foreach j in [1:10] {
+ file data<single_file_mapper; file="/Users/ketan/SwiftApps/swift-galaxy/swift/data.txt">;
out[j] = cat(data);
}
Copied: SwiftApps/swift-galaxy/swift/genericswift.xml (from rev 7508, SwiftApps/swift-galaxy/swift/swift.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/genericswift.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/genericswift.xml 2014-01-25 16:11:30 UTC (rev 7509)
@@ -0,0 +1,43 @@
+<tool id="swift" name="Swift-tool">
+ <description>A generic Swift tool to run Swift workflows</description>
+
+ <command interpreter="bash">
+ swift-galaxy.sh
+ $site
+ $script
+ $log_file
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="localhost">Localhost</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param format="swift" name="script" type="data" label="Swift script"/>
+ <conditional name="configuration">
+ <param name="mode" type="select" label="Swift configuration">
+ <option value="default">default configuration</option>
+ <option value="advanced">advanced configuration</option>
+ </param>
+ <when value="default">
+ </when>
+ <when value="advanced">
+ <param name="throttle" size="5" type="float" value="0.07" label="job throttle" help="number of parallel jobs to run"></param>
+ <param name="project" type="text" value="TG-STA110005S" label="Project allocation" help="name/code of project"></param>
+ <param name="slots" size="5" type="integer" value="1" label="slots" help="number of scheduler jobs"></param>
+ <param name="queue" size="5" type="text" value="normal" label="queue" help="queue to run jobs on"></param>
+ <param name="nodes" size="5" type="integer" value="1" label="nodes" help="number of nodes requested"></param>
+ <param name="nodegranularity" size="5" type="integer" value="1" label="node granularity" help="node granularity"></param>
+ <param name="jobspernode" size="5" type="integer" value="8" label="jobs per node" help="number of jobs per node requested"></param>
+ <param name="ppn" size="5" type="integer" value="8" label="ppn" help="processes per node requested"></param>
+ <param name="walltime" size="5" type="text" value="00:10:00" label="Job walltime" help="time in hh:mm:ss to request to scheduler for this job"></param>
+ <param name="maxtime" size="5" type="integer" value="700" label="application maxtime" help="Application maxtime in seconds for this job"></param>
+ </when>
+ </conditional>
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Swift output" />
+ </outputs>
+</tool>
Deleted: SwiftApps/swift-galaxy/swift/swift-galaxy
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-galaxy 2014-01-24 21:53:49 UTC (rev 7508)
+++ SwiftApps/swift-galaxy/swift/swift-galaxy 2014-01-25 16:11:30 UTC (rev 7509)
@@ -1,196 +0,0 @@
-#!/bin/bash
-
-site=$1
-script=$2
-logfile=$3
-
-ln -s $script script.swift
-swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift
-# use "here" document for configs
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="midway">
- <execution provider="coaster" jobmanager="local:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-midway echo echo
-midway cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-uc3 echo /bin/echo
-uc3 cat /bin/cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="stampede">
- <!--<execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>-->
- <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
- <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-stampede echo echo
-stampede cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-esac
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf script.swift 2> std.err
-EXITCODE=$?
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-cp std.err $logfile
-rm std.err
-exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/swift-galaxy.sh (from rev 7508, SwiftApps/swift-galaxy/swift/swift-galaxy)
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-galaxy.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/swift-galaxy.sh 2014-01-25 16:11:30 UTC (rev 7509)
@@ -0,0 +1,196 @@
+#!/bin/bash
+
+site=$1
+script=$2
+logfile=$3
+
+ln -s $script script.swift
+swift=$(which swift)
+# use "here" document for configs
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+midway echo echo
+midway cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="stampede">
+ <!--<execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>-->
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+stampede echo echo
+stampede cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+esac
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf script.swift 2> $logfile
+EXITCODE=$?
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+#cp std.err $logfile
+#rm std.err
+exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swift.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swift.xml 2014-01-24 21:53:49 UTC (rev 7508)
+++ SwiftApps/swift-galaxy/swift/swift.xml 2014-01-25 16:11:30 UTC (rev 7509)
@@ -1,40 +0,0 @@
-<tool id="swift" name="Swift-tool">
- <description>A generic Swift tool to run Swift workflows</description>
-
- <command interpreter="bash">
- swift-galaxy $site $script $log_file
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="localhost">Localhost</option>
- <option value="stampede">Stampede</option>
- </param>
- <param format="swift" name="script" type="data" label="Swift script"/>
- <conditional name="configuration">
- <param name="mode" type="select" label="Swift configuration">
- <option value="default">default configuration</option>
- <option value="advanced">advanced configuration</option>
- </param>
- <when value="default">
- </when>
- <when value="advanced">
- <param name="throttle" size="5" type="float" value="0.07" label="job throttle" help="number of parallel jobs to run"></param>
- <param name="project" type="text" value="TG-STA110005S" label="Project allocation" help="name/code of project"></param>
- <param name="slots" size="5" type="integer" value="1" label="slots" help="number of scheduler jobs"></param>
- <param name="queue" size="5" type="text" value="normal" label="queue" help="queue to run jobs on"></param>
- <param name="nodes" size="5" type="integer" value="1" label="nodes" help="number of nodes requested"></param>
- <param name="nodegranularity" size="5" type="integer" value="1" label="node granularity" help="node granularity"></param>
- <param name="jobspernode" size="5" type="integer" value="8" label="jobs per node" help="number of jobs per node requested"></param>
- <param name="ppn" size="5" type="integer" value="8" label="ppn" help="processes per node requested"></param>
- <param name="walltime" size="5" type="text" value="00:10:00" label="Job walltime" help="time in hh:mm:ss to request to scheduler for this job"></param>
- <param name="maxtime" size="5" type="integer" value="700" label="application maxtime" help="Application maxtime in seconds for this job"></param>
- </when>
- </conditional>
- </inputs>
- <outputs>
- <data format="txt" name="log_file" type="data" label="Swift output" />
- </outputs>
-</tool>
More information about the Swift-commit
mailing list