[Swift-commit] r7869 - in SwiftApps/swift-galaxy/swift: . old
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Tue May 13 09:33:33 CDT 2014
Author: ketan
Date: 2014-05-13 09:33:31 -0500 (Tue, 13 May 2014)
New Revision: 7869
Added:
SwiftApps/swift-galaxy/swift/old/catsn.swift
SwiftApps/swift-galaxy/swift/old/dssat.xml
SwiftApps/swift-galaxy/swift/old/genswift.xml
SwiftApps/swift-galaxy/swift/old/postprocess.py
SwiftApps/swift-galaxy/swift/old/powergrid.xml
SwiftApps/swift-galaxy/swift/old/sites.xml
SwiftApps/swift-galaxy/swift/old/statswift.xml
SwiftApps/swift-galaxy/swift/old/swift-catsn.sh
SwiftApps/swift-galaxy/swift/old/swift-catsn.xml
SwiftApps/swift-galaxy/swift/old/swift-powergrid
SwiftApps/swift-galaxy/swift/old/swift-stats
SwiftApps/swift-galaxy/swift/old/swift-try
SwiftApps/swift-galaxy/swift/old/swiftforeachpy.xml
SwiftApps/swift-galaxy/swift/old/swiftforeachtest.py
SwiftApps/swift-galaxy/swift/old/testrun
SwiftApps/swift-galaxy/swift/old/try.xml
Removed:
SwiftApps/swift-galaxy/swift/catsn.swift
SwiftApps/swift-galaxy/swift/dssat.xml
SwiftApps/swift-galaxy/swift/genswift.xml
SwiftApps/swift-galaxy/swift/postprocess.py
SwiftApps/swift-galaxy/swift/powergrid.xml
SwiftApps/swift-galaxy/swift/sites.xml
SwiftApps/swift-galaxy/swift/statswift.xml
SwiftApps/swift-galaxy/swift/swift-catsn.sh
SwiftApps/swift-galaxy/swift/swift-catsn.xml
SwiftApps/swift-galaxy/swift/swift-powergrid
SwiftApps/swift-galaxy/swift/swift-stats
SwiftApps/swift-galaxy/swift/swift-try
SwiftApps/swift-galaxy/swift/swiftforeachpy.xml
SwiftApps/swift-galaxy/swift/swiftforeachtest.py
SwiftApps/swift-galaxy/swift/testrun
SwiftApps/swift-galaxy/swift/try.xml
Log:
move remaining old stuff
Deleted: SwiftApps/swift-galaxy/swift/catsn.swift
===================================================================
--- SwiftApps/swift-galaxy/swift/catsn.swift 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/catsn.swift 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,11 +0,0 @@
-type file;
-
-app (file o) cat (file i) {
- cat @i stdout=@o;
-}
-file out[]<simple_mapper; location="outdir", prefix="f.",suffix=".out">;
-
-foreach j in [1:10] {
- file data<single_file_mapper; file="/Users/ketan/SwiftApps/swift-galaxy/swift/data.txt">;
- out[j] = cat(data);
-}
Deleted: SwiftApps/swift-galaxy/swift/dssat.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/dssat.xml 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/dssat.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,27 +0,0 @@
-<tool id="dssat" name="Swift - DSSAT">
- <description>Swift - DSSAT</description>
-
- <command interpreter="bash">
- /scratch/midway/maheshwari/DSSAT.cloud/DSSAT/dssat-galaxy.sh $njobs $template
- </command>
-
- <inputs>
- <profile name="dataset" type="select" label="Data set">
- <option value="smalltest">Small test</option>
- <option value="test">Test</option>
- </profile>
- <param name="template" type="select" label="Execution Location">
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="localhost">Localhost</option>
- <option value="stampede">Stampede</option>
- </param>
- <param name="njobs" type="text" value="1" label="Number of jobs to run (Valid ranges 1 - 100000)"/>
- </inputs>
-
- <outputs>
- <data format="txt" type="data" name="log_file"/>
- </outputs>
-
-</tool>
-
Deleted: SwiftApps/swift-galaxy/swift/genswift.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/genswift.xml 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/genswift.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,170 +0,0 @@
-<tool id="swiftforeach" name="foreach">
- <!--
- ToDo:
- 1. Option if the counter should be an argument, first one or last one.
- 2. Option to add files (in Galaxy dataset format) as arguments. These files then be added to Swift app as arguments.
- 3. Allow the tool to select another tool and run it.
- 4. Allow user to add more than one resource to run a job for load balancing.
- 5. Add ability to run Swift/T.
- 6. Let the localhost option be selected by default
- -->
- <description>A generic tool to run executable via Swift foreach parallel construct</description>
-
- <command interpreter="bash">
-
- #if $rangeorlist.rl == "rng"
- swiftforeachrange.sh
- "${site}"
- "${swiftargs}"
- "${interpret}"
- "${exec}"
- "${rangeorlist.rngstart}"
- "${rangeorlist.rend}"
- "${rangeorlist.stepsize}"
- "${outloc}"
- "${logfile}"
- "${outlist}"
- "${stringargs}"
- #for $a in $filearg
- ${a.fileargname}
- #end for
- #elif $rangeorlist.rl == "lst"
- swiftforeachlist.sh
- "${site}"
- "${swiftargs}"
- "${interpret}"
- "${exec}"
- "${rangeorlist.listfile}"
- "${outloc}"
- "${logfile}"
- "${outlist}"
- "${stringargs}"
- #for $a in $filearg
- ${a.fileargname}
- #end for
- #else
- swiftforeachtest.sh
- "${site}"
- "${swiftargs}"
- "${interpret}"
- "${exec}"
- "${rangeorlist.rngstart}"
- "${rangeorlist.rend}"
- "${rangeorlist.stepsize}"
- "${rangeorlist.listfile}"
- "${outloc}"
- "${logfile}"
- "${outlist}"
- "${stringargs}"
- #for $a in $filearg
- ${a.fileargname}
- #end for
- #end if
- </command>
- <!--
- #if $swiftargs.mode == "advanced"
- #for $sa in $swiftarg
- ${sa.swiftargname}
- #end for
- #end if
- -->
- <inputs>
- <param name="site" type="select" multiple="true" label="Execution Location" help="Multi-select list - hold the appropriate key while clicking to select multiple items">
- <option value="localhost" selected="true">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- <option value="tukey">Tukey</option>
- </param>
-
- <param name="swiftargs" size="60" type="text" label="swiftargs" help="Add Swift level commandline arguments" />
-
- <param name="interpret" type="select" label="Execution interpreter">
- <option value="sh">sh</option>
- <option value="python">python</option>
- <option value="java">java</option>
- <option value="R">R</option>
- <option value="matlab">matlab</option>
- </param>
-
- <!--<param format="sh,binexec" name="exec" type="data" label="Executable"/>-->
- <param name="exec" type="data" label="Executable" help="select executable" />
-
- <conditional name="rangeorlist">
- <param name="rl" type="select" label="Select range or list">
- <option value="rng">numeric range</option>
- <option value="lst">items list</option>
- <option value="tst">dry test</option>
- </param>
-
- <when value="rng">
- <param name="rngstart" size="2" type="integer" value="0" label="start"/>
- <param name="rend" size="2" type="integer" value="9" label="end"/>
- <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
- </when>
-
- <when value="lst">
- <param format="data" name="listfile" type="data" label="List file" help="Select list file"/>
- </when>
-
- <when value="tst">
- <param name="rngstart" size="2" type="integer" value="0" label="start"/>
- <param name="rend" size="2" type="integer" value="9" label="end"/>
- <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
- <param format="data" name="listfile" type="data" label="List file"/>
- </when>
-
- </conditional>
-
- <param name="stringargs" size="60" type="text" label="stringargs" help="Add application call commandline arguments here" />
-
- <repeat name="filearg" title="file arg">
- <param name="fileargname" type="data" label="filearg" />
- </repeat>
-
- <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc">
- <sanitizer sanitize="False" />
- </param>
- </inputs>
- <outputs>
- <data format="txt" name="logfile" type="data" label="Swift output" />
- <data format="txt" name="outlist" type="data" label="Output list" />
- </outputs>
-
- <!--
- <code file="postprocess.py">
- <hook postprocess="writeoutlist" />
- </code>
- -->
-
-<help>
-.. class:: warningmark
-
-**TIP**. Add args to provide additional arguments to your executable.
-
------
-
-**What it does**
-
-This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file.
-
-Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user.
-
-Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces.
-
------
-
-**Example**
-
-Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello".
-
-If an executable is echo.sh with following line:
-
-echo $@
-
-The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8:
-
-echo n hello
-
-</help>
-</tool>
Copied: SwiftApps/swift-galaxy/swift/old/catsn.swift (from rev 7868, SwiftApps/swift-galaxy/swift/catsn.swift)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/catsn.swift (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/catsn.swift 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,11 @@
+type file;
+
+app (file o) cat (file i) {
+ cat @i stdout=@o;
+}
+file out[]<simple_mapper; location="outdir", prefix="f.",suffix=".out">;
+
+foreach j in [1:10] {
+ file data<single_file_mapper; file="/Users/ketan/SwiftApps/swift-galaxy/swift/data.txt">;
+ out[j] = cat(data);
+}
Copied: SwiftApps/swift-galaxy/swift/old/dssat.xml (from rev 7868, SwiftApps/swift-galaxy/swift/dssat.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/dssat.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/dssat.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,27 @@
+<tool id="dssat" name="Swift - DSSAT">
+ <description>Swift - DSSAT</description>
+
+ <command interpreter="bash">
+ /scratch/midway/maheshwari/DSSAT.cloud/DSSAT/dssat-galaxy.sh $njobs $template
+ </command>
+
+ <inputs>
+ <profile name="dataset" type="select" label="Data set">
+ <option value="smalltest">Small test</option>
+ <option value="test">Test</option>
+ </profile>
+ <param name="template" type="select" label="Execution Location">
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="localhost">Localhost</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param name="njobs" type="text" value="1" label="Number of jobs to run (Valid ranges 1 - 100000)"/>
+ </inputs>
+
+ <outputs>
+ <data format="txt" type="data" name="log_file"/>
+ </outputs>
+
+</tool>
+
Copied: SwiftApps/swift-galaxy/swift/old/genswift.xml (from rev 7868, SwiftApps/swift-galaxy/swift/genswift.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/genswift.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/genswift.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,170 @@
+<tool id="swiftforeach" name="foreach">
+ <!--
+ ToDo:
+ 1. Option if the counter should be an argument, first one or last one.
+ 2. Option to add files (in Galaxy dataset format) as arguments. These files then be added to Swift app as arguments.
+ 3. Allow the tool to select another tool and run it.
+ 4. Allow user to add more than one resource to run a job for load balancing.
+ 5. Add ability to run Swift/T.
+ 6. Let the localhost option be selected by default
+ -->
+ <description>A generic tool to run executable via Swift foreach parallel construct</description>
+
+ <command interpreter="bash">
+
+ #if $rangeorlist.rl == "rng"
+ swiftforeachrange.sh
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #elif $rangeorlist.rl == "lst"
+ swiftforeachlist.sh
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #else
+ swiftforeachtest.sh
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #end if
+ </command>
+ <!--
+ #if $swiftargs.mode == "advanced"
+ #for $sa in $swiftarg
+ ${sa.swiftargname}
+ #end for
+ #end if
+ -->
+ <inputs>
+ <param name="site" type="select" multiple="true" label="Execution Location" help="Multi-select list - hold the appropriate key while clicking to select multiple items">
+ <option value="localhost" selected="true">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ <option value="tukey">Tukey</option>
+ </param>
+
+ <param name="swiftargs" size="60" type="text" label="swiftargs" help="Add Swift level commandline arguments" />
+
+ <param name="interpret" type="select" label="Execution interpreter">
+ <option value="sh">sh</option>
+ <option value="python">python</option>
+ <option value="java">java</option>
+ <option value="R">R</option>
+ <option value="matlab">matlab</option>
+ </param>
+
+ <!--<param format="sh,binexec" name="exec" type="data" label="Executable"/>-->
+ <param name="exec" type="data" label="Executable" help="select executable" />
+
+ <conditional name="rangeorlist">
+ <param name="rl" type="select" label="Select range or list">
+ <option value="rng">numeric range</option>
+ <option value="lst">items list</option>
+ <option value="tst">dry test</option>
+ </param>
+
+ <when value="rng">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ </when>
+
+ <when value="lst">
+ <param format="data" name="listfile" type="data" label="List file" help="Select list file"/>
+ </when>
+
+ <when value="tst">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ <param format="data" name="listfile" type="data" label="List file"/>
+ </when>
+
+ </conditional>
+
+ <param name="stringargs" size="60" type="text" label="stringargs" help="Add application call commandline arguments here" />
+
+ <repeat name="filearg" title="file arg">
+ <param name="fileargname" type="data" label="filearg" />
+ </repeat>
+
+ <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc">
+ <sanitizer sanitize="False" />
+ </param>
+ </inputs>
+ <outputs>
+ <data format="txt" name="logfile" type="data" label="Swift output" />
+ <data format="txt" name="outlist" type="data" label="Output list" />
+ </outputs>
+
+ <!--
+ <code file="postprocess.py">
+ <hook postprocess="writeoutlist" />
+ </code>
+ -->
+
+<help>
+.. class:: warningmark
+
+**TIP**. Add args to provide additional arguments to your executable.
+
+-----
+
+**What it does**
+
+This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file.
+
+Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user.
+
+Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces.
+
+-----
+
+**Example**
+
+Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello".
+
+If an executable is echo.sh with following line:
+
+echo $@
+
+The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8:
+
+echo n hello
+
+</help>
+</tool>
Copied: SwiftApps/swift-galaxy/swift/old/postprocess.py (from rev 7868, SwiftApps/swift-galaxy/swift/postprocess.py)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/postprocess.py (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/postprocess.py 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,65 @@
+#post processing, set build for data and add additional data to history^M
+from galaxy import datatypes, config, jobs, tools
+from galaxy.model import HistoryDatasetAssociation
+import operator
+import shutil
+import os
+import glob
+from os.path import basename
+
+def createDsRef(outDataItems, param_dict, outputName, outputFormat, outputDsRefName, nameSpace):
+ dsrefXML="<?xml version=\"1.0\" ?>\n"
+ dsrefXML=dsrefXML+"<dataSetRefs xmlns=\"http://ci.uchicago.edu/galaxy-es/datasetrefs/"+nameSpace+"\" >\n"
+
+ dsrefFileName=""
+
+ for outDataName, outDataItem in outDataItems:
+ if outDataName==outputDsRefName:
+ dsrefFileName=outDataItem.get_file_name()
+ else:
+ if outDataName==outputName:
+ dsrefXML=dsrefXML+"<dataSetRef id=\"%s\" name=\"%s\" path=\"%s\" type=\"%s\"/>\n" % (outDataItem.id,outDataItem.name,outDataItem.get_file_name(),outputFormat)
+
+ print "dsrefFileName:"+dsrefFileName
+
+ primaryCollectedDatasets=param_dict["__collected_datasets__"]["primary"]
+
+ for primaryCollectedDataset in primaryCollectedDatasets:
+ primaryCollectedDatasetItems=primaryCollectedDatasets[primaryCollectedDataset]
+ for primaryCollectedDatasetItem in primaryCollectedDatasetItems:
+ outDataItem=primaryCollectedDatasetItems[primaryCollectedDatasetItem]
+ if outputFormat in outDataItem.extension:
+
+ dsrefXML=dsrefXML+"<dataSetRef id=\"%s\" name=\"%s\" path=\"%s\" type=\"%s\"/>\n" % (outDataItem.id,outDataItem.name,outDataItem.get_file_name(),outDataItem.extension)
+
+ dsrefXML=dsrefXML+"</dataSetRefs>\n"
+
+ return dsrefFileName, dsrefXML
+
+def writeDsRef(dsrefFileName, dsrefXML):
+ dsrefFile = open(dsrefFileName, "w", 0)
+ dsrefFile.write(dsrefXML)
+ dsrefFile.flush()
+ dsrefFile.close()
+
+def setSizeAndPeak(outDataItems,outputDsRefName):
+ for outDataName, outDataItem in outDataItems:
+ if outDataName==outputDsRefName:
+ outDataItem.dataset.file_size=None
+ outDataItem.set_size()
+ outDataItem.set_peek()
+
+def writeoutlist(app, inp_data, out_data, param_dict, tool, stdout, stderr):
+
+ outDataItems = out_data.items()
+ outDataItems = sorted(outDataItems, key=operator.itemgetter(0))
+
+ dsrefAcmoFileName, dsrefAcmoXML = createDsRef(outDataItems,param_dict,"acmo","acmo.csv","acmoDsRef","facit/acmo")
+
+ writeDsRef(dsrefAcmoFileName, dsrefAcmoXML)
+
+ setSizeAndPeak(outDataItems,"acmoDsRef")
+
+
+ app.model.context.flush()
+
Copied: SwiftApps/swift-galaxy/swift/old/powergrid.xml (from rev 7868, SwiftApps/swift-galaxy/swift/powergrid.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/powergrid.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/powergrid.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,32 @@
+<tool id="powergrid" name="Swift - Power Grid Inference for Stochastic Optimization">
+ <description>Swift - Power Grid Inference for Stochastic Optimization </description>
+
+ <command interpreter="bash">
+ swift-powergrid $site $batch $log_file
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="localhost">Localhost</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param name="batch" type="text" value="20" label="value for batch size"/>
+
+<!--
+ <param name="stochmodel" type="data" label="stochmodel"/>
+ <param name="edrevised" type="data" label="edrevised"/>
+ <param name="asciidata" type="data" label="ascii data "/>
+ <param name="printres" type="data" label="print res inc file"/>
+ <param name="illinois" type="data" label="Illinois data file"/>
+ <param name="wind_data" type="data" label="wind data file"/>
+ -->
+ </inputs>
+
+ <outputs>
+ <data format="txt" type="data" name="log_file"/>
+ </outputs>
+
+</tool>
+
Copied: SwiftApps/swift-galaxy/swift/old/sites.xml (from rev 7868, SwiftApps/swift-galaxy/swift/sites.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/sites.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/sites.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,139 @@
+<config>
+
+<!-- sites.xml specifies details of the sites that Swift can run on.
+
+The first entry entry, for localhost, should work on most linux-like systems
+without any change.
+
+It may be necessary to change the two occurences of /var/tmp to a different
+working directory.
+
+The jobThrottle profile is set to 0 so that Swift will not try to run more
+than two jobs at once (the minimum) when using the local sites. This is
+appropriate for local execution, but make sure if/when using another
+execution mechanism to submit to an HPC resource that this jobThrottle
+is removed.
+
+-->
+
+
+<pool handle="localhost">
+ <filesystem provider="local" />
+ <execution provider="local" />
+ <workdirectory>/tmp/swift.workdir</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0</profile>
+ </pool>
+
+<!-- The remainder of this file is commented out by default. It contains
+example site definitions for a number of sites on TeraGrid and OSG.
+
+How this list was constructed:
+
+1. Teragrid
+http://www.teragrid.org/userinfo/hardware/resources.php
+fill out the gatekeeper info and the gridftp hostname
+for the storage/workspace, login into the machine, and make yourself a
+temporary directory
+
+2. OSG
+http://osg-cat.grid.iu.edu/index.php?site_name=osgcat
+sort by CPUs
+Add the gatekeeper hostname into the jobmanager name and into the gridftp url
+Add the ($TMP) or ($WNTMP) to the storage element in the gridftp url AND
+into the workspace element
+
+TODO by user:
+customize the storage and workdirectory to use your personal working directories
+
+-->
+
+
+<!--
+ <pool handle="skynet" >
+ <gridftp url="gsiftp://skynet-login.isi.edu" >
+ </gridftp>
+ <jobmanager universe="vanilla" url="skynet-login.isi.edu/jobmanager-fork" major="2" />
+ <workdirectory >/nfs/scratch01/tiberius</workdirectory>
+ </pool>
+-->
+<!--
+ <pool handle="teraport" >
+ <gridftp url="gsiftp://tp-grid1.uchicago.edu" />
+ <jobmanager universe="vanilla" url="tp-grid1.uchicago.edu/jobmanager-pbs" major="2" />
+ <workdirectory >/home/tiberius/scratch/SWIFT-WORK</workdirectory>
+ </pool>
+-->
+<!--
+<pool handle="ncsa_32" >
+ <gridftp url="gsiftp://login-w.ncsa.teragrid.org" />
+ <jobmanager universe="vanilla" url="grid-w.ncsa.teragrid.org/jobmanager" major="2" />
+ <workdirectory >/u/ac/tstef</workdirectory>
+</pool>
+-->
+<!--
+<pool handle="purdue" >
+ <gridftp url="gsiftp://tg-data.purdue.teragrid.org" />
+ <jobmanager universe="vanilla" url="tg-gatekeeper.purdue.teragrid.org" major="2" />
+ <workdirectory >/scratch/teragrid/stef</workdirectory>
+</pool>
+-->
+<!--
+<pool handle="tacc" >
+ <gridftp url="gsiftp://tg-gridftp.lonestar.tacc.teragrid.org" />
+ <jobmanager universe="vanilla" url="tg-login.tacc.teragrid.org/jobmanager-lsf" major="2" />
+ <workdirectory >/work/teragrid/tstef</workdirectory>
+</pool>
+-->
+<!--
+<pool handle="UC" >
+ <gridftp url="gsiftp://tg-gridftp.uc.teragrid.org" />
+ <execution provider="gt4" jobmanager="PBS" url="tg-grid.uc.teragrid.org" />
+ <workdirectory >/disks/scratchgpfs1/tstef</workdirectory>
+</pool>
+ -->
+
+<!-- OSG SITES -->
+<!--
+<pool handle="OSG_LIGO_PSU" >
+ <gridftp url="gsiftp://grid3.aset.psu.edu" />
+ <jobmanager universe="vanilla" url="grid3.aset.psu.edu/jobmanager-pbs" major="2" />
+ <workdirectory >/usr1/grid3/data</workdirectory>
+</pool>
+ -->
+<!--
+<pool handle="fnal" >
+ <gridftp url="gsiftp://fngp-osg.fnal.gov" />
+ <jobmanager universe="vanilla" url="fngp-osg.fnal.gov/jobmanager-pbs" major="2" minor="2" />
+ <workdirectory >/grid/data</workdirectory>
+</pool>
+-->
+<!--
+<pool handle="prod_slac" >
+ <gridftp url="gsiftp://osgserv01.slac.stanford.edu" />
+ <jobmanager universe="vanilla" url="osgserv01.slac.stanford.edu/jobmanager-lsf" major="2" />
+ <workdirectory >/nfs/slac/g/grid/osg/data</workdirectory>
+</pool>
+-->
+<!--
+<pool handle="bnl_atlas_2" >
+ <gridftp url="gsiftp://gridgk02.racf.bnl.gov" />
+ <jobmanager universe="vanilla" url="gridgk02.racf.bnl.gov/jobmanager-condor" major="2" />
+ <workdirectory >/usatlas/prodjob/share/</workdirectory>
+</pool>
+-->
+<!--
+<pool handle="unm_hpc" >
+ <gridftp url="gsiftp://milta.alliance.unm.edu" />
+ <jobmanager universe="vanilla" url="milta.alliance.unm.edu/jobmanager-pbs" major="2" />
+ <workdirectory >/tmp</workdirectory>
+</pool>
+-->
+<!--
+<pool handle="UWM_CMS" >
+ <gridftp url="gsiftp://cmsgrid02.hep.wisc.edu" />
+ <jobmanager universe="vanilla" url="cmsgrid02.hep.wisc.edu/jobmanager-condor" major="2" />
+ <workdirectory >/tmp</workdirectory>
+</pool>
+-->
+
+</config>
Copied: SwiftApps/swift-galaxy/swift/old/statswift.xml (from rev 7868, SwiftApps/swift-galaxy/swift/statswift.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/statswift.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/statswift.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,47 @@
+<tool id="stats" name="Swift stats app">
+ <description>A Swift tool to run stats on compute nodes</description>
+
+ <command interpreter="bash">
+ swift-stats "$site.site_select" "$n" "$html_file.files_path" "$log_file" "$html_file" "$site.walltime" "$site.slots" "$site.queue" "$site.nodes" "$site.throttle" "$site.workdir"
+ </command>
+ <inputs>
+ <param name="n" type="text" value="2" label="Number of tasks to run"/>
+
+ <conditional name="site">
+ <param name="site_select" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
+ <option value="clusterPBS">PBS-Cluster</option>
+ <option value="clusterSLURM">SLURM-Cluster</option>
+ </param>
+
+ <when value="localhost">
+ <param name="walltime" type="hidden" value="00:10:00" label="Walltime in hh:mm:ss" />
+ <param name="slots" type="hidden" value="1" label="Number of scheduler jobs" />
+ <param name="queue" type="hidden" value="normal" label="Queue name" />
+ <param name="nodes" type="hidden" value="1" label="number of nodes per scheduler job" />
+ <param name="throttle" type="hidden" value="30" label="application wide degree of parallelism" />
+ <param name="workdir" type="hidden" value="/tmp/swift.workdir" label="application execution sanbox directory" />
+ </when>
+ <when value="clusterPBS">
+ <param name="walltime" type="text" value="00:10:00" label="Walltime in hh:mm:ss" />
+ <param name="slots" type="text" value="1" label="Number of scheduler jobs" />
+ <param name="queue" type="text" value="normal" label="Queue name" />
+ <param name="nodes" type="text" value="1" label="number of nodes per scheduler job" />
+ <param name="throttle" type="text" value="30" label="application wide degree of parallelism" />
+ <param name="workdir" type="text" value="/tmp/swift.workdir" label="application execution sanbox directory" />
+ </when>
+ <when value="clusterSLURM">
+ <param name="walltime" type="text" value="00:10:00" label="Walltime in hh:mm:ss" />
+ <param name="slots" type="text" value="1" label="Number of scheduler jobs" />
+ <param name="queue" type="text" value="normal" label="Queue name" />
+ <param name="nodes" type="text" value="1" label="number of nodes per scheduler job" />
+ <param name="throttle" type="text" value="30" label="application wide degree of parallelism" />
+ <param name="workdir" type="text" value="/tmp/swift.workdir" label="application execution sanbox directory" />
+ </when>
+ </conditional>
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Swift output" />
+ <data format="html" name="html_file" label="catsnoutput.html" />
+ </outputs>
+</tool>
Copied: SwiftApps/swift-galaxy/swift/old/swift-catsn.sh (from rev 7868, SwiftApps/swift-galaxy/swift/swift-catsn.sh)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-catsn.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-catsn.sh 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,276 @@
+#!/bin/bash
+
+#set -x
+set -e
+
+site=$1
+data=$2
+n=$3
+outdir=$4
+logfile=$5
+htmlfile=$6
+mkdir -p $outdir
+
+swift=`which swift`
+# use "here" document for configs
+
+cat << EOF > /tmp/catsn.swift
+type file;
+
+app (file o) cat (file i) {
+ cat @i stdout=@o;
+}
+#file out[]<simple_mapper; location="outdir", prefix="f.",suffix=".out">;
+file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">;
+
+string d=@arg("data");
+
+foreach j in [1:@toInt(@arg("n","1"))] {
+ file data<single_file_mapper; file=d>;
+ out[j] = cat(data);
+}
+EOF
+
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="midway">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+;;
+esac
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+stampede echo echo
+stampede cat cat
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+midway echo echo
+midway cat cat
+EOF
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> std.err
+EXITCODE=$?
+
+swiftlog=`\ls -t *.log | head -1`
+
+#/tmp/plotswiftlogs $logfile 2>gnuplot.err
+cat << 'EOF' > /tmp/plotswiftlogs
+#!/bin/bash
+
+#usage: ./extract4plots <swift.logfile>
+
+SWIFTLOGFILE=$1
+#TMPDIR=`mktemp -d plotlog.XXX`
+grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
+SWIFTOUTFILE=swiftoutfile.out
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+#echo 0 > active.txt
+#echo 0 > cumulative.txt
+#extract active runs in a file
+(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt)
+
+#extract successful completions in a file
+(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+if [ $activelines -ne 0 ]
+then
+ activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+fi
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+if [ $cumulines -ne 0 ]
+then
+ cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+fi
+
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+cat << EOF1 > plotit.gp
+set terminal png enhanced
+set output "activeplot.png"
+set nokey
+set xlabel "Time in sec"
+set ylabel "number of active jobs"
+set title "Active jobs"
+plot "plot_active.txt" using 1:2 with line
+
+set output "cumulativeplot.png"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+EOF1
+
+gnuplot plotit.gp 2>/dev/null
+
+#cp *.png ../
+echo `pwd`>/tmp/curdir
+echo `ls`>/tmp/dirlist
+#cd ..
+EOF
+
+sh /tmp/plotswiftlogs $swiftlog
+cp *.png $outdir/
+
+#for i in `find $outdir`
+#do
+# lastfile=$i
+#done
+#bname=`basename $lastfile`
+
+cat << EOF > $htmlfile
+<!DOCTYPE HTML>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>Catsn output</title>
+</head>
+<body>
+<h2>Swift run trace plots</h2>
+
+<a href="activeplot.png" alt="active plot">active plot</a>
+<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
+
+<h2>Browse output of catsn application</h2>
+`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
+</body>
+</html>
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+cp std.err $logfile
+rm std.err
+exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/old/swift-catsn.xml (from rev 7868, SwiftApps/swift-galaxy/swift/swift-catsn.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-catsn.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-catsn.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,22 @@
+<tool id="catsn" name="Swift catsn">
+ <description>A Swift tool to run the catsn workflow</description>
+
+ <command interpreter="bash">
+ swift-catsn "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file"
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="localhost">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param name="n" type="text" value="1" label="Number of tasks to run "/>
+ <param name="data" type="data" label="Input data file"/>
+ </inputs>
+ <outputs>
+ <data format="txt" name="log_file" type="data" label="Swift output" />
+ <data format="html" name="html_file" label="catsnoutput.html" />
+ </outputs>
+</tool>
Copied: SwiftApps/swift-galaxy/swift/old/swift-powergrid (from rev 7868, SwiftApps/swift-galaxy/swift/swift-powergrid)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-powergrid (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-powergrid 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,321 @@
+#!/bin/bash
+
+site=$1
+batch=$2
+#stochmodel=$3
+#edrevised=$4
+#asciidata=$5
+#printres=$6
+#illinois=$7
+#wind_data=$8
+logfile=$3
+
+#cp $stochmodel stochmodel.run
+#cp $edrevised edrevised.mod
+#cp $asciidata set_data_ascii.inc
+#cp $printres print_results_revised.inc
+#cp $illinois Illinois.tgz
+#cp $wind_data wind_data.dat
+echo $@>>/tmp/tmp.txt
+#swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift
+swift=/home/maheshwari/swift-devel/cog/modules/swift/dist/swift-svn/bin/swift
+# use "here" document for configs
+
+cat << 'EOF' > /tmp/inference4galaxy.swift
+type file;
+
+app ( file _obj_out, file _xhat_out1, file _xhat_out2, file _ampl_res, file _out)
+ ampl_app ( int _s,
+ int _fix_x,
+ file _stoch_model,
+ file _illinois,
+ file _edrevised_model,
+ file _ascii_data,
+ file _print_results,
+ file _wind_data
+ ) {
+ runampl _s _fix_x @_stoch_model @_xhat_out1 @_xhat_out2 @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_res @_wind_data stdout=@_out;
+}
+
+app (file _obj_out, file _ampl_l_res, file _out) ampl_app_L (
+ int _s,
+ int _fix_x,
+ file _stoch_model,
+ file _illinois,
+ file _edrevised_model,
+ file _ascii_data,
+ file _print_results
+ ) {
+
+ runampl_l _s _fix_x @_stoch_model @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_l_res stdout=@_out;
+}
+
+app (file _obj_out, file _ampl_u_res, file _out) ampl_app_U (
+ int _s,
+ int _fix_x,
+ file _stoch_model,
+ file _xhat,
+ file _illinois,
+ file _edrevised_model,
+ file _ascii_data,
+ file _print_results,
+ file _wind_data
+ ) {
+
+ runampl_u _s _fix_x @_stoch_model @_xhat @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_u_res @_wind_data stdout=@_out;
+}
+file stoch_model<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/stochmodel.run">;
+file edrevised_model<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/edrevised.mod">;
+file ascii_data<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/set_data_ascii.inc">;
+file print_results<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/print_results_revised.inc">;
+file illinois<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/Illinois.tgz">;
+file wind_data<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/wind_data.dat">;
+
+file out[] <simple_mapper; location="outdir", prefix="ampl",suffix=".out">;
+file obj_out[] <simple_mapper; location="obj_outdir", prefix="obj_ampl",suffix=".out">;
+file ampl_res[] <simple_mapper; location="Results", prefix="ampl_res",suffix=".out">;
+
+file xhat_out1[] <simple_mapper; location="outdir", prefix="xhat1",suffix=".out">;
+file xhat_out2[] <simple_mapper; location="outdir", prefix="xhat2",suffix=".out">;
+
+file out_l[][][] <simple_mapper; location="outdir", prefix="out_l", suffix=".out">;
+file obj_out_l[][][] <simple_mapper; location="obj_outdir", prefix="obj_out_l", suffix=".out">;
+file ampl_l_res[][][] <simple_mapper; location="Results", prefix="ampl_res_l", suffix=".out">;
+file xhat_out_l1[][][] <simple_mapper; location="outdir", prefix="xhat_l1",suffix=".out">;
+file xhat_out_l2[][][] <simple_mapper; location="outdir", prefix="xhat_l2",suffix=".out">;
+
+file out_u[][][] <simple_mapper; location="outdir", prefix="out_u", suffix=".out">;
+file obj_out_u[][][] <simple_mapper; location="obj_outdir", prefix="obj_out_u", suffix=".out">;
+file ampl_u_res[][][] <simple_mapper; location="Results", prefix="ampl_res_u", suffix=".out">;
+#file upper_out[][][] <simple_mapper; location="outdir", prefix="upper",suffix=".out">;
+#file lower_out[][][] <simple_mapper; location="outdir", prefix="lower",suffix=".out">;
+
+int nS[] = [10]; //number of scenario sweeps
+int Batch = @toInt(@arg("batch", "20"));
+
+foreach S, idxs in nS{
+(obj_out[idxs], xhat_out1[idxs], xhat_out2[idxs], ampl_res[idxs], out[idxs]) = ampl_app (S, 0, stoch_model, illinois, edrevised_model, ascii_data, print_results, wind_data);
+
+
+foreach B, idxb in [10:Batch:10]{
+ foreach k in [0:B]{
+ (obj_out_l[idxs][idxb][k], xhat_out_l1[idxs][idxb][k], xhat_out_l2[idxs][idxb][k], ampl_l_res[idxs][idxb][k], out_l[idxs][idxb][k]) = ampl_app (S, 0, stoch_model, illinois, edrevised_model, ascii_data, print_results, wind_data);
+
+ (obj_out_u[idxs][idxb][k], ampl_u_res[idxs][idxb][k], out_u[idxs][idxb][k]) = ampl_app_U(S, 1, stoch_model, xhat_out1[idxs], illinois, edrevised_model, ascii_data, print_results, wind_data);
+
+}
+}
+}
+EOF
+
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <filesystem provider="local" />
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.31</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+midway echo echo
+midway cat cat
+midway gensample /home/maheshwari/powergridapps/bin/gensample.py
+midway runampl /home/maheshwari/powergridapps/bin/run_ampl.py
+midway runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py
+midway runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+use.wrapper.staging=false
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+uc3 runampl run_ampl.py
+uc3 runampl_l run_ampl_l.py
+uc3 runampl_u run_ampl_u.py
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-EAR130015</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+stampede echo echo
+stampede cat cat
+stampede gensample gensample.py
+stampede runampl run_ampl.py
+stampede runampl_l run_ampl_l.py
+stampede runampl_u run_ampl_u.py
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+use.wrapper.staging=false
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="local" />
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.2</profile>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+localhost gensample /home/maheshwari/powergridapps/bin/gensample.py
+localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py
+localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py
+localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+use.wrapper.staging=false
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+cat << 'EOF' > /tmp/sites.xml
+<config>
+<pool handle="localhost">
+ <execution provider="local" />
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+localhost gensample /home/maheshwari/powergridapps/bin/gensample.py
+localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py
+localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py
+localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+esac
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/inference4galaxy.swift -batch=$batch 2> std.err
+#$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/inference4galaxy.swift -batch=$batch -stochmodel=stochmodel.run -edrevised=edrevised.mod -asciidata=set_data_ascii.inc -printres=print_results_revised.inc -illinois=Illinois.tgz -wind_data=wind_data.dat 2> std.err
+
+EXITCODE=$?
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+cp std.err $logfile
+rm std.err
+exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/old/swift-stats (from rev 7868, SwiftApps/swift-galaxy/swift/swift-stats)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-stats (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-stats 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,205 @@
+#!/bin/bash
+# swift-stats "$site.site_select" "$n" "$html_file.files_path" "$log_file" "$html_file" "$site.walltime" "$site.slots" "$site.queue" "$site.nodes" "$site.throttle" "$site.workdir"
+# 1 2 3 4 5 6 7 8 9 10 11
+
+site=$1
+n=$2
+outdir=$3
+logfile=$4
+htmlfile=$5
+walltime=$6
+slots=$7
+queue=$8
+nodes=$9
+conc=${10}
+workdir=${11}
+throttle=$(echo "scale=5;($conc-1)/100" |bc)
+
+mkdir -p $outdir
+
+echo $6 > /tmp/args.txt
+
+swift=`\which swift`
+
+# use "here" document for configs
+cat << EOF > /tmp/stats.swift
+type file;
+
+app (file o) statsapp () {
+ stats stdout=@o;
+}
+
+file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">;
+
+foreach j in [1:@toInt(@arg("n","2"))] {
+ out[j] = statsapp ();
+}
+EOF
+
+case "$site" in
+clusterPBS )
+cat << EOF > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="clusterpbs">
+ <execution provider="coaster" jobmanager="local:pbs"/>
+ <profile namespace="globus" key="queue">default</profile>
+ <profile namespace="globus" key="jobsPerNode">24</profile>
+ <profile namespace="globus" key="maxWalltime">${walltime}</profile>
+ <profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">${slots}</profile>
+ <profile namespace="globus" key="maxNodes">${nodes}</profile>
+ <profile namespace="globus" key="nodeGranularity">${nodes}</profile>
+ <profile namespace="karajan" key="jobThrottle">${throttle}</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>${workdir}</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+clusterpbs echo echo
+clusterpbs cat cat
+clusterpbs stats stats.sh
+EOF
+
+cat << EOF > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+clusterSLURM )
+cat << EOF > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="clusterslurm">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="queue">${queue}</profile>
+ <profile namespace="globus" key="jobsPerNode">8</profile>
+ <profile namespace="globus" key="ppn">8</profile>
+ <profile namespace="globus" key="maxWalltime">${walltime}</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">${slots}</profile>
+ <profile namespace="globus" key="maxNodes">${nodes}</profile>
+ <profile namespace="globus" key="nodeGranularity">${nodes}</profile>
+ <profile namespace="karajan" key="jobThrottle">${throttle}</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>${workdir}</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+clusterslurm echo echo
+clusterslurm cat cat
+clusterslurm stats stats.sh
+EOF
+
+cat << EOF > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+localhost )
+cat << EOF > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="localhost">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">0.02</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="swift" key="stagingMethod">local</profile>
+ </pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+localhost echo echo
+localhost cat cat
+localhost stats stats.sh
+EOF
+
+cat << EOF > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+
+cat << EOF > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">0.02</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="swift" key="stagingMethod">local</profile>
+</pool>
+</config>
+EOF
+
+cat << EOF > /tmp/tc
+localhost echo echo
+localhost cat cat
+localhost stats stats.sh
+EOF
+
+cat << EOF > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+esac
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/stats.swift -n=$n -data=$data 2> std.err
+EXITCODE=$?
+
+swiftlog=`\ls -t *.log | head -1`
+
+cat << EOF > $htmlfile
+<!DOCTYPE HTML>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>Hello output</title>
+</head>
+<body>
+<h2>Browse output of hello application</h2>
+`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
+</body>
+</html>
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+cp std.err $logfile
+rm std.err
+exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/old/swift-try (from rev 7868, SwiftApps/swift-galaxy/swift/swift-try)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swift-try (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swift-try 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,341 @@
+#!/bin/bash
+
+site=$1
+nsim=$2
+steps=$3
+range=$4
+values=$5
+outdir=$6
+logfile=$7
+htmlfile=$8
+
+mkdir -p $outdir
+
+swift=`which swift`
+# use "here" document for configs
+
+cat << EOF > /tmp/mysim.swift
+type file;
+
+app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values)
+{
+ simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log;
+}
+
+app (file out, file log) analyze (file s[])
+{
+ stats @filenames(s) stdout=@out stderr=@log;
+}
+
+int nsim = @toInt(@arg("nsim", "10"));
+int steps = @toInt(@arg("steps", "1"));
+int range = @toInt(@arg("range", "100"));
+int values = @toInt(@arg("values", "5"));
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+ file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
+ file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
+ (simout,simlog) = simulation(steps,range,values);
+ sims[i] = simout;
+}
+
+file stats_out<"output/average.out">;
+file stats_log<"output/average.log">;
+(stats_out, stats_log) = analyze(sims);
+
+EOF
+
+case "$site" in
+midway )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="midway">
+ <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">7500</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">4</profile>
+ <profile namespace="globus" key="maxNodes">2</profile>
+ <profile namespace="globus" key="nodeGranularity">2</profile>
+ <profile namespace="karajan" key="jobThrottle">1.27</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/scratch/midway/maheshwari/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+midway echo echo
+midway cat cat
+midway simulate simulate.sh
+midway stats stats.sh
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+uc3 )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+ <pool handle="uc3">
+ <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <!--<execution provider="coaster" jobmanager="local:condor"/>-->
+ <profile namespace="karajan" key="jobThrottle">.09</profile>
+ <profile namespace="karajan" key="initialScore">1000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxWalltime">3600</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <filesystem provider="local" url="none" />
+ <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+uc3 echo /bin/echo
+uc3 cat /bin/cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+stampede )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="stampede">
+ <execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>
+ <!--<execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>-->
+ <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
+ <filesystem provider="local"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="ppn">16</profile>
+ <profile namespace="globus" key="project">TG-STA110005S</profile>
+ <profile namespace="globus" key="queue">normal</profile>
+ <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
+ <profile namespace="globus" key="slurm.mail-type">ALL</profile>
+ <profile namespace="globus" key="maxWalltime">02:00:00</profile>
+ <profile namespace="globus" key="maxTime">8000</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.work</workdirectory>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+stampede echo echo
+stampede cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+localhost )
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+</pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+
+* )
+echo "defaulting to localhost"
+
+cat << 'EOF' > /tmp/sites.xml
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localhost">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <filesystem provider="local"/>
+ <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
+ <workdirectory>/tmp/swift.work</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0.31</profile>
+ </pool>
+</config>
+EOF
+
+cat << 'EOF' > /tmp/tc
+localhost echo echo
+localhost cat cat
+EOF
+
+cat << 'EOF' > /tmp/cf
+use.provider.staging=false
+wrapperlog.always.transfer=true
+execution.retries=1
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+;;
+esac
+
+$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/mysim.swift -nsim=$nsim -steps=$steps -range=$range -values=$values 2> std.err
+EXITCODE=$?
+
+swiftlog=`\ls -t *.log | head -1`
+
+#/tmp/plotswiftlogs $logfile 2>gnuplot.err
+cat << 'EOF' > /tmp/plotswiftlogs
+#!/bin/bash
+
+#usage: ./plotswiftlogs.ketan <swift-logfile.log>
+
+SWIFTLOGFILE=$1
+
+#TMPDIR=`mktemp -d plotlog.XXX`
+
+grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
+
+SWIFTOUTFILE=swiftoutfile.out
+
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+
+#extract active runs in a file
+(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt)
+
+#extract successful completions in a file
+(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+if [ $activelines -ne 0 ]
+then
+ activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+fi
+
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+if [ $cumulines -ne 0 ]
+then
+ cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+fi
+
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+cat << EOF1 > plotit.gp
+set terminal png enhanced
+set nokey
+set output "cumulativeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of completed apps"
+set title "Cumulative Apps Completed"
+plot "plot_cumulative.txt" using 1:2 with lines
+set output "activeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of active apps"
+set title "Active Apps"
+plot "plot_active.txt" using 1:2 with line
+EOF1
+
+gnuplot plotit.gp 2>/dev/null
+rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt
+
+EOF
+
+sh /tmp/plotswiftlogs $swiftlog
+cp *.png $outdir/
+
+cat << EOF > $htmlfile
+<!DOCTYPE HTML>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+ <title>Application output</title>
+</head>
+<body>
+<h2>Swift run trace plots</h2>
+
+<a href="activeplot.png" alt="active plot">active plot</a>
+<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
+
+<h2>Browse output of application</h2>
+`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
+</body>
+</html>
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat std.err >&2
+fi
+cp std.err $logfile
+rm std.err
+exit $EXITCODE
Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachpy.xml (from rev 7868, SwiftApps/swift-galaxy/swift/swiftforeachpy.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swiftforeachpy.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swiftforeachpy.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,170 @@
+<tool id="swiftforeach" name="foreach">
+ <!--
+ ToDo:
+ 1. Option if the counter should be an argument, first one or last one.
+ 2. Option to add files (in Galaxy dataset format) as arguments. These files then be added to Swift app as arguments.
+ 3. Allow the tool to select another tool and run it.
+ 4. Allow user to add more than one resource to run a job for load balancing.
+ 5. Add ability to run Swift/T.
+ 6. Let the localhost option be selected by default
+ -->
+ <description>A generic tool to run executable via Swift foreach parallel construct</description>
+
+ <command interpreter="python">
+
+ #if $rangeorlist.rl == "rng"
+ swiftforeachrange.py
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #elif $rangeorlist.rl == "lst"
+ swiftforeachlist.py
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #else
+ swiftforeachtest.py
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #end if
+ </command>
+ <!--
+ #if $swiftargs.mode == "advanced"
+ #for $sa in $swiftarg
+ ${sa.swiftargname}
+ #end for
+ #end if
+ -->
+ <inputs>
+ <param name="site" type="select" multiple="true" label="Execution Location" help="Multi-select list - hold the appropriate key while clicking to select multiple items">
+ <option value="localhost" selected="true">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ <option value="tukey">Tukey</option>
+ </param>
+
+ <param name="swiftargs" size="60" type="text" label="swiftargs" help="Add Swift level commandline arguments" />
+
+ <param name="interpret" type="select" label="Execution interpreter">
+ <option value="sh">sh</option>
+ <option value="python">python</option>
+ <option value="java">java</option>
+ <option value="R">R</option>
+ <option value="matlab">matlab</option>
+ </param>
+
+ <!--<param format="sh,binexec" name="exec" type="data" label="Executable"/>-->
+ <param name="exec" type="data" label="Executable" help="select executable" />
+
+ <conditional name="rangeorlist">
+ <param name="rl" type="select" label="Select range or list">
+ <option value="rng">numeric range</option>
+ <option value="lst">items list</option>
+ <option value="tst">dry test</option>
+ </param>
+
+ <when value="rng">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ </when>
+
+ <when value="lst">
+ <param format="data" name="listfile" type="data" label="List file" help="Select list file"/>
+ </when>
+
+ <when value="tst">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ <param format="data" name="listfile" type="data" label="List file"/>
+ </when>
+
+ </conditional>
+
+ <param name="stringargs" size="60" type="text" label="stringargs" help="Add application call commandline arguments here" />
+
+ <repeat name="filearg" title="file arg">
+ <param name="fileargname" type="data" label="filearg" />
+ </repeat>
+
+ <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc">
+ <sanitizer sanitize="False" />
+ </param>
+ </inputs>
+ <outputs>
+ <data format="txt" name="logfile" type="data" label="Swift output" />
+ <data format="txt" name="outlist" type="data" label="Output list" />
+ </outputs>
+
+ <!--
+ <code file="postprocess.py">
+ <hook postprocess="writeoutlist" />
+ </code>
+ -->
+
+<help>
+.. class:: warningmark
+
+**TIP**. Add args to provide additional arguments to your executable.
+
+-----
+
+**What it does**
+
+This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file.
+
+Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user.
+
+Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces.
+
+-----
+
+**Example**
+
+Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello".
+
+If an executable is echo.sh with following line:
+
+echo $@
+
+The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8:
+
+echo n hello
+
+</help>
+</tool>
Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachtest.py (from rev 7868, SwiftApps/swift-galaxy/swift/swiftforeachtest.py)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swiftforeachtest.py (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swiftforeachtest.py 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+
+import subprocess
+import sys
+#import rpy
+import os
+
+def setwdir():
+ return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
+
+# numeric range
+def main():
+ print "%d" % (len(sys.argv))
+ print "%s" % (sys.argv)
+ try:
+ sites=sys.argv[1] # comma separated list of sites
+ swiftargs=sys.argv[2]
+ interpreter=sys.argv[3]
+ executable=sys.argv[4]
+ rstart=sys.argv[5]
+ rend=sys.argv[6]
+ stepsize=sys.argv[7]
+ outloc=sys.argv[8]
+ outlistfile=sys.argv[9]
+ logfile=sys.argv[10]
+ stringargs=sys.argv[11]
+ except:
+ #rpy.stop_error('Wrong number or type of args')
+ sys.exit(1)
+
+ print sites
+ print swiftargs
+ print interpreter
+
+ #workout the array only if user adds args else make it blank
+ if stringargs:
+ stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
+ else:
+ stringarrayexpr="string stringargs[];\n"
+
+ fileargs=list()
+ for files in sys.argv[12:]:
+ fileargs.append(files)
+
+ #workout the array only if user adds file args else make it blank
+ if fileargs:
+ filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
+ else:
+ filearrayexpr="file fileargs[];\n"
+
+ wdir=setwdir().rstrip()
+ #Build Swift source code
+ f=open(wdir+"/script.swift", 'w');
+ f.write("type file; \n")
+
+ f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
+ f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
+ f.write("}\n")
+
+ f.write(stringarrayexpr)
+ f.write(filearrayexpr)
+
+ f.write("file exec<'"+executable+"'>;\n")
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
+ f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
+ f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
+ f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
+ f.write("}\n")
+ f.close()
+
+ swift=subprocess.check_output(["which", "swift"]).rstrip()
+
+ #build site
+ if os.path.dirname(sys.argv[0]):
+ basedir=os.path.dirname(sys.argv[0])
+ else:
+ basedir='.'
+
+ print "%s %s %s" %(basedir, sites, wdir)
+ subprocess.call([basedir+"/buildsite.sh", sites, wdir])
+
+ #build config
+ f=open(wdir+"/cf", 'w');
+ f.write('use.provider.staging=true')
+ f.write('wrapperlog.always.transfer=true')
+ f.write('execution.retries=0')
+ f.write('provider.staging.pin.swiftfiles=false')
+ f.write('sitedir.keep=true')
+ f.write('lazy.errors=false')
+ f.close()
+
+
+ #build tc
+ f=open(wdir+"/tc", 'w');
+ for site in sites.split():
+ f.write(site+' '+interpreter+' '+interpreter)
+ f.close()
+
+ #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"])
+
+ #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
+ #Populate output file
+ outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
+ f=open(outlistfile)
+ f.write(outlist);
+ f.close()
+ #dum ditty dum ditty dum dum dum
+
+if __name__=='__main__':
+ main()
Copied: SwiftApps/swift-galaxy/swift/old/testrun (from rev 7868, SwiftApps/swift-galaxy/swift/testrun)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/testrun (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/testrun 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,2 @@
+python swiftforeachrange.py localhost someargs java /Users/ketan/galaxy-dist/database/files/000/dataset_248.dat 0 9 1 /Users/ketan/galaxy-dist/database/files/000/dataset_306.dat /Users/ketan/swift-sandbox /Users/ketan/galaxy-dist/database/files/000/dataset_307.dat /Users/ketan/galaxy-dist/database/files/000/dataset_308.dat hello /Users/ketan/galaxy-dist/database/files/000/dataset_302.dat
+
Copied: SwiftApps/swift-galaxy/swift/old/try.xml (from rev 7868, SwiftApps/swift-galaxy/swift/try.xml)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/try.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/try.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -0,0 +1,31 @@
+<tool id="try" name="Try Swift">
+ <description>Try Swift</description>
+
+ <command interpreter="bash">
+ swift-try $site $swiftscript $log_file
+ </command>
+
+ <inputs>
+ <param name="site" type="select" label="Execution Location">
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="localhost">Localhost</option>
+ <option value="stampede">Stampede</option>
+ </param>
+ <param name="script" type="text" area="True" size="25x25" label="Swift script"/>
+<!--
+ <param name="stochmodel" type="data" label="stochmodel"/>
+ <param name="edrevised" type="data" label="edrevised"/>
+ <param name="asciidata" type="data" label="ascii data "/>
+ <param name="printres" type="data" label="print res inc file"/>
+ <param name="illinois" type="data" label="Illinois data file"/>
+ <param name="wind_data" type="data" label="wind data file"/>
+ -->
+ </inputs>
+
+ <outputs>
+ <data format="txt" type="data" name="log_file"/>
+ </outputs>
+
+</tool>
+
Deleted: SwiftApps/swift-galaxy/swift/postprocess.py
===================================================================
--- SwiftApps/swift-galaxy/swift/postprocess.py 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/postprocess.py 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,65 +0,0 @@
-#post processing, set build for data and add additional data to history^M
-from galaxy import datatypes, config, jobs, tools
-from galaxy.model import HistoryDatasetAssociation
-import operator
-import shutil
-import os
-import glob
-from os.path import basename
-
-def createDsRef(outDataItems, param_dict, outputName, outputFormat, outputDsRefName, nameSpace):
- dsrefXML="<?xml version=\"1.0\" ?>\n"
- dsrefXML=dsrefXML+"<dataSetRefs xmlns=\"http://ci.uchicago.edu/galaxy-es/datasetrefs/"+nameSpace+"\" >\n"
-
- dsrefFileName=""
-
- for outDataName, outDataItem in outDataItems:
- if outDataName==outputDsRefName:
- dsrefFileName=outDataItem.get_file_name()
- else:
- if outDataName==outputName:
- dsrefXML=dsrefXML+"<dataSetRef id=\"%s\" name=\"%s\" path=\"%s\" type=\"%s\"/>\n" % (outDataItem.id,outDataItem.name,outDataItem.get_file_name(),outputFormat)
-
- print "dsrefFileName:"+dsrefFileName
-
- primaryCollectedDatasets=param_dict["__collected_datasets__"]["primary"]
-
- for primaryCollectedDataset in primaryCollectedDatasets:
- primaryCollectedDatasetItems=primaryCollectedDatasets[primaryCollectedDataset]
- for primaryCollectedDatasetItem in primaryCollectedDatasetItems:
- outDataItem=primaryCollectedDatasetItems[primaryCollectedDatasetItem]
- if outputFormat in outDataItem.extension:
-
- dsrefXML=dsrefXML+"<dataSetRef id=\"%s\" name=\"%s\" path=\"%s\" type=\"%s\"/>\n" % (outDataItem.id,outDataItem.name,outDataItem.get_file_name(),outDataItem.extension)
-
- dsrefXML=dsrefXML+"</dataSetRefs>\n"
-
- return dsrefFileName, dsrefXML
-
-def writeDsRef(dsrefFileName, dsrefXML):
- dsrefFile = open(dsrefFileName, "w", 0)
- dsrefFile.write(dsrefXML)
- dsrefFile.flush()
- dsrefFile.close()
-
-def setSizeAndPeak(outDataItems,outputDsRefName):
- for outDataName, outDataItem in outDataItems:
- if outDataName==outputDsRefName:
- outDataItem.dataset.file_size=None
- outDataItem.set_size()
- outDataItem.set_peek()
-
-def writeoutlist(app, inp_data, out_data, param_dict, tool, stdout, stderr):
-
- outDataItems = out_data.items()
- outDataItems = sorted(outDataItems, key=operator.itemgetter(0))
-
- dsrefAcmoFileName, dsrefAcmoXML = createDsRef(outDataItems,param_dict,"acmo","acmo.csv","acmoDsRef","facit/acmo")
-
- writeDsRef(dsrefAcmoFileName, dsrefAcmoXML)
-
- setSizeAndPeak(outDataItems,"acmoDsRef")
-
-
- app.model.context.flush()
-
Deleted: SwiftApps/swift-galaxy/swift/powergrid.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/powergrid.xml 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/powergrid.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,32 +0,0 @@
-<tool id="powergrid" name="Swift - Power Grid Inference for Stochastic Optimization">
- <description>Swift - Power Grid Inference for Stochastic Optimization </description>
-
- <command interpreter="bash">
- swift-powergrid $site $batch $log_file
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="localhost">Localhost</option>
- <option value="stampede">Stampede</option>
- </param>
- <param name="batch" type="text" value="20" label="value for batch size"/>
-
-<!--
- <param name="stochmodel" type="data" label="stochmodel"/>
- <param name="edrevised" type="data" label="edrevised"/>
- <param name="asciidata" type="data" label="ascii data "/>
- <param name="printres" type="data" label="print res inc file"/>
- <param name="illinois" type="data" label="Illinois data file"/>
- <param name="wind_data" type="data" label="wind data file"/>
- -->
- </inputs>
-
- <outputs>
- <data format="txt" type="data" name="log_file"/>
- </outputs>
-
-</tool>
-
Deleted: SwiftApps/swift-galaxy/swift/sites.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/sites.xml 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/sites.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,139 +0,0 @@
-<config>
-
-<!-- sites.xml specifies details of the sites that Swift can run on.
-
-The first entry entry, for localhost, should work on most linux-like systems
-without any change.
-
-It may be necessary to change the two occurences of /var/tmp to a different
-working directory.
-
-The jobThrottle profile is set to 0 so that Swift will not try to run more
-than two jobs at once (the minimum) when using the local sites. This is
-appropriate for local execution, but make sure if/when using another
-execution mechanism to submit to an HPC resource that this jobThrottle
-is removed.
-
--->
-
-
-<pool handle="localhost">
- <filesystem provider="local" />
- <execution provider="local" />
- <workdirectory>/tmp/swift.workdir</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0</profile>
- </pool>
-
-<!-- The remainder of this file is commented out by default. It contains
-example site definitions for a number of sites on TeraGrid and OSG.
-
-How this list was constructed:
-
-1. Teragrid
-http://www.teragrid.org/userinfo/hardware/resources.php
-fill out the gatekeeper info and the gridftp hostname
-for the storage/workspace, login into the machine, and make yourself a
-temporary directory
-
-2. OSG
-http://osg-cat.grid.iu.edu/index.php?site_name=osgcat
-sort by CPUs
-Add the gatekeeper hostname into the jobmanager name and into the gridftp url
-Add the ($TMP) or ($WNTMP) to the storage element in the gridftp url AND
-into the workspace element
-
-TODO by user:
-customize the storage and workdirectory to use your personal working directories
-
--->
-
-
-<!--
- <pool handle="skynet" >
- <gridftp url="gsiftp://skynet-login.isi.edu" >
- </gridftp>
- <jobmanager universe="vanilla" url="skynet-login.isi.edu/jobmanager-fork" major="2" />
- <workdirectory >/nfs/scratch01/tiberius</workdirectory>
- </pool>
--->
-<!--
- <pool handle="teraport" >
- <gridftp url="gsiftp://tp-grid1.uchicago.edu" />
- <jobmanager universe="vanilla" url="tp-grid1.uchicago.edu/jobmanager-pbs" major="2" />
- <workdirectory >/home/tiberius/scratch/SWIFT-WORK</workdirectory>
- </pool>
--->
-<!--
-<pool handle="ncsa_32" >
- <gridftp url="gsiftp://login-w.ncsa.teragrid.org" />
- <jobmanager universe="vanilla" url="grid-w.ncsa.teragrid.org/jobmanager" major="2" />
- <workdirectory >/u/ac/tstef</workdirectory>
-</pool>
--->
-<!--
-<pool handle="purdue" >
- <gridftp url="gsiftp://tg-data.purdue.teragrid.org" />
- <jobmanager universe="vanilla" url="tg-gatekeeper.purdue.teragrid.org" major="2" />
- <workdirectory >/scratch/teragrid/stef</workdirectory>
-</pool>
--->
-<!--
-<pool handle="tacc" >
- <gridftp url="gsiftp://tg-gridftp.lonestar.tacc.teragrid.org" />
- <jobmanager universe="vanilla" url="tg-login.tacc.teragrid.org/jobmanager-lsf" major="2" />
- <workdirectory >/work/teragrid/tstef</workdirectory>
-</pool>
--->
-<!--
-<pool handle="UC" >
- <gridftp url="gsiftp://tg-gridftp.uc.teragrid.org" />
- <execution provider="gt4" jobmanager="PBS" url="tg-grid.uc.teragrid.org" />
- <workdirectory >/disks/scratchgpfs1/tstef</workdirectory>
-</pool>
- -->
-
-<!-- OSG SITES -->
-<!--
-<pool handle="OSG_LIGO_PSU" >
- <gridftp url="gsiftp://grid3.aset.psu.edu" />
- <jobmanager universe="vanilla" url="grid3.aset.psu.edu/jobmanager-pbs" major="2" />
- <workdirectory >/usr1/grid3/data</workdirectory>
-</pool>
- -->
-<!--
-<pool handle="fnal" >
- <gridftp url="gsiftp://fngp-osg.fnal.gov" />
- <jobmanager universe="vanilla" url="fngp-osg.fnal.gov/jobmanager-pbs" major="2" minor="2" />
- <workdirectory >/grid/data</workdirectory>
-</pool>
--->
-<!--
-<pool handle="prod_slac" >
- <gridftp url="gsiftp://osgserv01.slac.stanford.edu" />
- <jobmanager universe="vanilla" url="osgserv01.slac.stanford.edu/jobmanager-lsf" major="2" />
- <workdirectory >/nfs/slac/g/grid/osg/data</workdirectory>
-</pool>
--->
-<!--
-<pool handle="bnl_atlas_2" >
- <gridftp url="gsiftp://gridgk02.racf.bnl.gov" />
- <jobmanager universe="vanilla" url="gridgk02.racf.bnl.gov/jobmanager-condor" major="2" />
- <workdirectory >/usatlas/prodjob/share/</workdirectory>
-</pool>
--->
-<!--
-<pool handle="unm_hpc" >
- <gridftp url="gsiftp://milta.alliance.unm.edu" />
- <jobmanager universe="vanilla" url="milta.alliance.unm.edu/jobmanager-pbs" major="2" />
- <workdirectory >/tmp</workdirectory>
-</pool>
--->
-<!--
-<pool handle="UWM_CMS" >
- <gridftp url="gsiftp://cmsgrid02.hep.wisc.edu" />
- <jobmanager universe="vanilla" url="cmsgrid02.hep.wisc.edu/jobmanager-condor" major="2" />
- <workdirectory >/tmp</workdirectory>
-</pool>
--->
-
-</config>
Deleted: SwiftApps/swift-galaxy/swift/statswift.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/statswift.xml 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/statswift.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,47 +0,0 @@
-<tool id="stats" name="Swift stats app">
- <description>A Swift tool to run stats on compute nodes</description>
-
- <command interpreter="bash">
- swift-stats "$site.site_select" "$n" "$html_file.files_path" "$log_file" "$html_file" "$site.walltime" "$site.slots" "$site.queue" "$site.nodes" "$site.throttle" "$site.workdir"
- </command>
- <inputs>
- <param name="n" type="text" value="2" label="Number of tasks to run"/>
-
- <conditional name="site">
- <param name="site_select" type="select" label="Execution Location">
- <option value="localhost">Localhost</option>
- <option value="clusterPBS">PBS-Cluster</option>
- <option value="clusterSLURM">SLURM-Cluster</option>
- </param>
-
- <when value="localhost">
- <param name="walltime" type="hidden" value="00:10:00" label="Walltime in hh:mm:ss" />
- <param name="slots" type="hidden" value="1" label="Number of scheduler jobs" />
- <param name="queue" type="hidden" value="normal" label="Queue name" />
- <param name="nodes" type="hidden" value="1" label="number of nodes per scheduler job" />
- <param name="throttle" type="hidden" value="30" label="application wide degree of parallelism" />
- <param name="workdir" type="hidden" value="/tmp/swift.workdir" label="application execution sanbox directory" />
- </when>
- <when value="clusterPBS">
- <param name="walltime" type="text" value="00:10:00" label="Walltime in hh:mm:ss" />
- <param name="slots" type="text" value="1" label="Number of scheduler jobs" />
- <param name="queue" type="text" value="normal" label="Queue name" />
- <param name="nodes" type="text" value="1" label="number of nodes per scheduler job" />
- <param name="throttle" type="text" value="30" label="application wide degree of parallelism" />
- <param name="workdir" type="text" value="/tmp/swift.workdir" label="application execution sanbox directory" />
- </when>
- <when value="clusterSLURM">
- <param name="walltime" type="text" value="00:10:00" label="Walltime in hh:mm:ss" />
- <param name="slots" type="text" value="1" label="Number of scheduler jobs" />
- <param name="queue" type="text" value="normal" label="Queue name" />
- <param name="nodes" type="text" value="1" label="number of nodes per scheduler job" />
- <param name="throttle" type="text" value="30" label="application wide degree of parallelism" />
- <param name="workdir" type="text" value="/tmp/swift.workdir" label="application execution sanbox directory" />
- </when>
- </conditional>
- </inputs>
- <outputs>
- <data format="txt" name="log_file" type="data" label="Swift output" />
- <data format="html" name="html_file" label="catsnoutput.html" />
- </outputs>
-</tool>
Deleted: SwiftApps/swift-galaxy/swift/swift-catsn.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-catsn.sh 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/swift-catsn.sh 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,276 +0,0 @@
-#!/bin/bash
-
-#set -x
-set -e
-
-site=$1
-data=$2
-n=$3
-outdir=$4
-logfile=$5
-htmlfile=$6
-mkdir -p $outdir
-
-swift=`which swift`
-# use "here" document for configs
-
-cat << EOF > /tmp/catsn.swift
-type file;
-
-app (file o) cat (file i) {
- cat @i stdout=@o;
-}
-#file out[]<simple_mapper; location="outdir", prefix="f.",suffix=".out">;
-file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">;
-
-string d=@arg("data");
-
-foreach j in [1:@toInt(@arg("n","1"))] {
- file data<single_file_mapper; file=d>;
- out[j] = cat(data);
-}
-EOF
-
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="midway">
- <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="stampede">
- <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="local" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="local" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-;;
-esac
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-stampede echo echo
-stampede cat cat
-uc3 echo /bin/echo
-uc3 cat /bin/cat
-midway echo echo
-midway cat cat
-EOF
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> std.err
-EXITCODE=$?
-
-swiftlog=`\ls -t *.log | head -1`
-
-#/tmp/plotswiftlogs $logfile 2>gnuplot.err
-cat << 'EOF' > /tmp/plotswiftlogs
-#!/bin/bash
-
-#usage: ./extract4plots <swift.logfile>
-
-SWIFTLOGFILE=$1
-#TMPDIR=`mktemp -d plotlog.XXX`
-grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
-SWIFTOUTFILE=swiftoutfile.out
-#extract start time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
-START_TIME=`date +%s -d "$TMPDATE"`
-
-#extract end time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
-END_TIME=`date +%s -d "$TMPDATE"`
-
-#duration
-DIFFTIME=$((END_TIME - START_TIME))
-#echo 0 > active.txt
-#echo 0 > cumulative.txt
-#extract active runs in a file
-(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt)
-
-#extract successful completions in a file
-(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
-
-#prepare tics
-activelines=`wc -l active.txt | awk '{print $1}'`
-cumulines=`wc -l cumulative.txt | awk '{print $1}'`
-
-if [ $activelines -ne 0 ]
-then
- activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
-fi
-seq 0 $activelinespertic $DIFFTIME > activetics.txt
-
-if [ $cumulines -ne 0 ]
-then
- cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
-fi
-
-seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
-
-#final plot data
-paste activetics.txt active.txt > plot_active.txt
-paste cumultics.txt cumulative.txt > plot_cumulative.txt
-
-cat << EOF1 > plotit.gp
-set terminal png enhanced
-set output "activeplot.png"
-set nokey
-set xlabel "Time in sec"
-set ylabel "number of active jobs"
-set title "Active jobs"
-plot "plot_active.txt" using 1:2 with line
-
-set output "cumulativeplot.png"
-set ylabel "number of completed jobs"
-set title "Cumulative jobs"
-plot "plot_cumulative.txt" using 1:2 with lines
-EOF1
-
-gnuplot plotit.gp 2>/dev/null
-
-#cp *.png ../
-echo `pwd`>/tmp/curdir
-echo `ls`>/tmp/dirlist
-#cd ..
-EOF
-
-sh /tmp/plotswiftlogs $swiftlog
-cp *.png $outdir/
-
-#for i in `find $outdir`
-#do
-# lastfile=$i
-#done
-#bname=`basename $lastfile`
-
-cat << EOF > $htmlfile
-<!DOCTYPE HTML>
-
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
- <title>Catsn output</title>
-</head>
-<body>
-<h2>Swift run trace plots</h2>
-
-<a href="activeplot.png" alt="active plot">active plot</a>
-<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
-
-<h2>Browse output of catsn application</h2>
-`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
-</body>
-</html>
-EOF
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-cp std.err $logfile
-rm std.err
-exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swift-catsn.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-catsn.xml 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/swift-catsn.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,22 +0,0 @@
-<tool id="catsn" name="Swift catsn">
- <description>A Swift tool to run the catsn workflow</description>
-
- <command interpreter="bash">
- swift-catsn "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file"
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="localhost">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- </param>
- <param name="n" type="text" value="1" label="Number of tasks to run "/>
- <param name="data" type="data" label="Input data file"/>
- </inputs>
- <outputs>
- <data format="txt" name="log_file" type="data" label="Swift output" />
- <data format="html" name="html_file" label="catsnoutput.html" />
- </outputs>
-</tool>
Deleted: SwiftApps/swift-galaxy/swift/swift-powergrid
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-powergrid 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/swift-powergrid 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,321 +0,0 @@
-#!/bin/bash
-
-site=$1
-batch=$2
-#stochmodel=$3
-#edrevised=$4
-#asciidata=$5
-#printres=$6
-#illinois=$7
-#wind_data=$8
-logfile=$3
-
-#cp $stochmodel stochmodel.run
-#cp $edrevised edrevised.mod
-#cp $asciidata set_data_ascii.inc
-#cp $printres print_results_revised.inc
-#cp $illinois Illinois.tgz
-#cp $wind_data wind_data.dat
-echo $@>>/tmp/tmp.txt
-#swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift
-swift=/home/maheshwari/swift-devel/cog/modules/swift/dist/swift-svn/bin/swift
-# use "here" document for configs
-
-cat << 'EOF' > /tmp/inference4galaxy.swift
-type file;
-
-app ( file _obj_out, file _xhat_out1, file _xhat_out2, file _ampl_res, file _out)
- ampl_app ( int _s,
- int _fix_x,
- file _stoch_model,
- file _illinois,
- file _edrevised_model,
- file _ascii_data,
- file _print_results,
- file _wind_data
- ) {
- runampl _s _fix_x @_stoch_model @_xhat_out1 @_xhat_out2 @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_res @_wind_data stdout=@_out;
-}
-
-app (file _obj_out, file _ampl_l_res, file _out) ampl_app_L (
- int _s,
- int _fix_x,
- file _stoch_model,
- file _illinois,
- file _edrevised_model,
- file _ascii_data,
- file _print_results
- ) {
-
- runampl_l _s _fix_x @_stoch_model @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_l_res stdout=@_out;
-}
-
-app (file _obj_out, file _ampl_u_res, file _out) ampl_app_U (
- int _s,
- int _fix_x,
- file _stoch_model,
- file _xhat,
- file _illinois,
- file _edrevised_model,
- file _ascii_data,
- file _print_results,
- file _wind_data
- ) {
-
- runampl_u _s _fix_x @_stoch_model @_xhat @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_u_res @_wind_data stdout=@_out;
-}
-file stoch_model<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/stochmodel.run">;
-file edrevised_model<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/edrevised.mod">;
-file ascii_data<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/set_data_ascii.inc">;
-file print_results<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/print_results_revised.inc">;
-file illinois<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/Illinois.tgz">;
-file wind_data<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/wind_data.dat">;
-
-file out[] <simple_mapper; location="outdir", prefix="ampl",suffix=".out">;
-file obj_out[] <simple_mapper; location="obj_outdir", prefix="obj_ampl",suffix=".out">;
-file ampl_res[] <simple_mapper; location="Results", prefix="ampl_res",suffix=".out">;
-
-file xhat_out1[] <simple_mapper; location="outdir", prefix="xhat1",suffix=".out">;
-file xhat_out2[] <simple_mapper; location="outdir", prefix="xhat2",suffix=".out">;
-
-file out_l[][][] <simple_mapper; location="outdir", prefix="out_l", suffix=".out">;
-file obj_out_l[][][] <simple_mapper; location="obj_outdir", prefix="obj_out_l", suffix=".out">;
-file ampl_l_res[][][] <simple_mapper; location="Results", prefix="ampl_res_l", suffix=".out">;
-file xhat_out_l1[][][] <simple_mapper; location="outdir", prefix="xhat_l1",suffix=".out">;
-file xhat_out_l2[][][] <simple_mapper; location="outdir", prefix="xhat_l2",suffix=".out">;
-
-file out_u[][][] <simple_mapper; location="outdir", prefix="out_u", suffix=".out">;
-file obj_out_u[][][] <simple_mapper; location="obj_outdir", prefix="obj_out_u", suffix=".out">;
-file ampl_u_res[][][] <simple_mapper; location="Results", prefix="ampl_res_u", suffix=".out">;
-#file upper_out[][][] <simple_mapper; location="outdir", prefix="upper",suffix=".out">;
-#file lower_out[][][] <simple_mapper; location="outdir", prefix="lower",suffix=".out">;
-
-int nS[] = [10]; //number of scenario sweeps
-int Batch = @toInt(@arg("batch", "20"));
-
-foreach S, idxs in nS{
-(obj_out[idxs], xhat_out1[idxs], xhat_out2[idxs], ampl_res[idxs], out[idxs]) = ampl_app (S, 0, stoch_model, illinois, edrevised_model, ascii_data, print_results, wind_data);
-
-
-foreach B, idxb in [10:Batch:10]{
- foreach k in [0:B]{
- (obj_out_l[idxs][idxb][k], xhat_out_l1[idxs][idxb][k], xhat_out_l2[idxs][idxb][k], ampl_l_res[idxs][idxb][k], out_l[idxs][idxb][k]) = ampl_app (S, 0, stoch_model, illinois, edrevised_model, ascii_data, print_results, wind_data);
-
- (obj_out_u[idxs][idxb][k], ampl_u_res[idxs][idxb][k], out_u[idxs][idxb][k]) = ampl_app_U(S, 1, stoch_model, xhat_out1[idxs], illinois, edrevised_model, ascii_data, print_results, wind_data);
-
-}
-}
-}
-EOF
-
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="midway">
- <execution provider="coaster" jobmanager="local:slurm"/>
- <filesystem provider="local" />
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">2</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.31</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-midway echo echo
-midway cat cat
-midway gensample /home/maheshwari/powergridapps/bin/gensample.py
-midway runampl /home/maheshwari/powergridapps/bin/run_ampl.py
-midway runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py
-midway runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-use.wrapper.staging=false
-EOF
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<config>
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-uc3 echo /bin/echo
-uc3 cat /bin/cat
-uc3 runampl run_ampl.py
-uc3 runampl_l run_ampl_l.py
-uc3 runampl_u run_ampl_u.py
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="stampede">
- <execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-EAR130015</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-stampede echo echo
-stampede cat cat
-stampede gensample gensample.py
-stampede runampl run_ampl.py
-stampede runampl_l run_ampl_l.py
-stampede runampl_u run_ampl_u.py
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-use.wrapper.staging=false
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="local" />
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.2</profile>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-localhost gensample /home/maheshwari/powergridapps/bin/gensample.py
-localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py
-localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py
-localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-use.wrapper.staging=false
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-cat << 'EOF' > /tmp/sites.xml
-<config>
-<pool handle="localhost">
- <execution provider="local" />
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-localhost gensample /home/maheshwari/powergridapps/bin/gensample.py
-localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py
-localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py
-localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-esac
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/inference4galaxy.swift -batch=$batch 2> std.err
-#$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/inference4galaxy.swift -batch=$batch -stochmodel=stochmodel.run -edrevised=edrevised.mod -asciidata=set_data_ascii.inc -printres=print_results_revised.inc -illinois=Illinois.tgz -wind_data=wind_data.dat 2> std.err
-
-EXITCODE=$?
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-cp std.err $logfile
-rm std.err
-exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swift-stats
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-stats 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/swift-stats 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,205 +0,0 @@
-#!/bin/bash
-# swift-stats "$site.site_select" "$n" "$html_file.files_path" "$log_file" "$html_file" "$site.walltime" "$site.slots" "$site.queue" "$site.nodes" "$site.throttle" "$site.workdir"
-# 1 2 3 4 5 6 7 8 9 10 11
-
-site=$1
-n=$2
-outdir=$3
-logfile=$4
-htmlfile=$5
-walltime=$6
-slots=$7
-queue=$8
-nodes=$9
-conc=${10}
-workdir=${11}
-throttle=$(echo "scale=5;($conc-1)/100" |bc)
-
-mkdir -p $outdir
-
-echo $6 > /tmp/args.txt
-
-swift=`\which swift`
-
-# use "here" document for configs
-cat << EOF > /tmp/stats.swift
-type file;
-
-app (file o) statsapp () {
- stats stdout=@o;
-}
-
-file out[]<simple_mapper; location="$outdir", prefix="f.",suffix=".txt">;
-
-foreach j in [1:@toInt(@arg("n","2"))] {
- out[j] = statsapp ();
-}
-EOF
-
-case "$site" in
-clusterPBS )
-cat << EOF > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="clusterpbs">
- <execution provider="coaster" jobmanager="local:pbs"/>
- <profile namespace="globus" key="queue">default</profile>
- <profile namespace="globus" key="jobsPerNode">24</profile>
- <profile namespace="globus" key="maxWalltime">${walltime}</profile>
- <profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">${slots}</profile>
- <profile namespace="globus" key="maxNodes">${nodes}</profile>
- <profile namespace="globus" key="nodeGranularity">${nodes}</profile>
- <profile namespace="karajan" key="jobThrottle">${throttle}</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>${workdir}</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << EOF > /tmp/tc
-clusterpbs echo echo
-clusterpbs cat cat
-clusterpbs stats stats.sh
-EOF
-
-cat << EOF > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-clusterSLURM )
-cat << EOF > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="clusterslurm">
- <execution provider="coaster" jobmanager="local:slurm"/>
- <profile namespace="globus" key="queue">${queue}</profile>
- <profile namespace="globus" key="jobsPerNode">8</profile>
- <profile namespace="globus" key="ppn">8</profile>
- <profile namespace="globus" key="maxWalltime">${walltime}</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">${slots}</profile>
- <profile namespace="globus" key="maxNodes">${nodes}</profile>
- <profile namespace="globus" key="nodeGranularity">${nodes}</profile>
- <profile namespace="karajan" key="jobThrottle">${throttle}</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>${workdir}</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << EOF > /tmp/tc
-clusterslurm echo echo
-clusterslurm cat cat
-clusterslurm stats stats.sh
-EOF
-
-cat << EOF > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-localhost )
-cat << EOF > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="localhost">
- <execution provider="local" />
- <profile namespace="karajan" key="jobThrottle">0.02</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local"/>
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="swift" key="stagingMethod">local</profile>
- </pool>
-</config>
-EOF
-
-cat << EOF > /tmp/tc
-localhost echo echo
-localhost cat cat
-localhost stats stats.sh
-EOF
-
-cat << EOF > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-
-cat << EOF > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="local" />
- <profile namespace="karajan" key="jobThrottle">0.02</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local"/>
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="swift" key="stagingMethod">local</profile>
-</pool>
-</config>
-EOF
-
-cat << EOF > /tmp/tc
-localhost echo echo
-localhost cat cat
-localhost stats stats.sh
-EOF
-
-cat << EOF > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-esac
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/stats.swift -n=$n -data=$data 2> std.err
-EXITCODE=$?
-
-swiftlog=`\ls -t *.log | head -1`
-
-cat << EOF > $htmlfile
-<!DOCTYPE HTML>
-
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
- <title>Hello output</title>
-</head>
-<body>
-<h2>Browse output of hello application</h2>
-`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
-</body>
-</html>
-EOF
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-cp std.err $logfile
-rm std.err
-exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swift-try
===================================================================
--- SwiftApps/swift-galaxy/swift/swift-try 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/swift-try 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,341 +0,0 @@
-#!/bin/bash
-
-site=$1
-nsim=$2
-steps=$3
-range=$4
-values=$5
-outdir=$6
-logfile=$7
-htmlfile=$8
-
-mkdir -p $outdir
-
-swift=`which swift`
-# use "here" document for configs
-
-cat << EOF > /tmp/mysim.swift
-type file;
-
-app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values)
-{
- simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log;
-}
-
-app (file out, file log) analyze (file s[])
-{
- stats @filenames(s) stdout=@out stderr=@log;
-}
-
-int nsim = @toInt(@arg("nsim", "10"));
-int steps = @toInt(@arg("steps", "1"));
-int range = @toInt(@arg("range", "100"));
-int values = @toInt(@arg("values", "5"));
-
-file sims[];
-
-foreach i in [0:nsim-1] {
- file simout <single_file_mapper; file=@strcat("output/sim_",i,".out")>;
- file simlog <single_file_mapper; file=@strcat("output/sim_",i,".log")>;
- (simout,simlog) = simulation(steps,range,values);
- sims[i] = simout;
-}
-
-file stats_out<"output/average.out">;
-file stats_log<"output/average.log">;
-(stats_out, stats_log) = analyze(sims);
-
-EOF
-
-case "$site" in
-midway )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="midway">
- <execution provider="coaster" url="swift.rcc.uchicago.edu" jobmanager="ssh-cl:slurm"/>
- <profile namespace="globus" key="queue">sandyb</profile>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">7500</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">4</profile>
- <profile namespace="globus" key="maxNodes">2</profile>
- <profile namespace="globus" key="nodeGranularity">2</profile>
- <profile namespace="karajan" key="jobThrottle">1.27</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/scratch/midway/maheshwari/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-midway echo echo
-midway cat cat
-midway simulate simulate.sh
-midway stats stats.sh
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-uc3 )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
- <pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
- <!--<execution provider="coaster" jobmanager="local:condor"/>-->
- <profile namespace="karajan" key="jobThrottle">.09</profile>
- <profile namespace="karajan" key="initialScore">1000</profile>
- <profile namespace="globus" key="jobsPerNode">1</profile>
- <profile namespace="globus" key="maxWalltime">3600</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="condor.Requirements">UidDomain == "osg-gk.mwt2.org"</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <filesystem provider="local" url="none" />
- <!--<workdirectory>/home/maheshwari/test/catsn/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-uc3 echo /bin/echo
-uc3 cat /bin/cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-stampede )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="stampede">
- <execution provider="coaster" jobmanager="gt2:gt2:slurm" url="login5.stampede.tacc.utexas.edu:2119/jobmanager-slurm"/>
- <!--<execution provider="coaster" jobmanager="ssh-cl:slurm" url="stampede.tacc.utexas.edu"/>-->
- <!--<filesystem provider="gsiftp" url="gsiftp://gridftp.stampede.tacc.utexas.edu:2811"/>-->
- <filesystem provider="local"/>
- <profile namespace="globus" key="jobsPerNode">16</profile>
- <profile namespace="globus" key="ppn">16</profile>
- <profile namespace="globus" key="project">TG-STA110005S</profile>
- <profile namespace="globus" key="queue">normal</profile>
- <profile namespace="globus" key="slurm.mail-user">ketancmaheshwari at gmail.com</profile>
- <profile namespace="globus" key="slurm.mail-type">ALL</profile>
- <profile namespace="globus" key="maxWalltime">02:00:00</profile>
- <profile namespace="globus" key="maxTime">8000</profile>
- <profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">.15</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <workdirectory>/tmp/swift.work</workdirectory>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-stampede echo echo
-stampede cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-localhost )
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
-</pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-
-* )
-echo "defaulting to localhost"
-
-cat << 'EOF' > /tmp/sites.xml
-<?xml version="1.0" encoding="UTF-8"?>
-<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
-<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local"/>
- <filesystem provider="local"/>
- <!--<workdirectory>/home/kcm92/powergridapps/swiftscripts/swift.work</workdirectory>-->
- <workdirectory>/tmp/swift.work</workdirectory>
- <profile namespace="karajan" key="jobThrottle">0.31</profile>
- </pool>
-</config>
-EOF
-
-cat << 'EOF' > /tmp/tc
-localhost echo echo
-localhost cat cat
-EOF
-
-cat << 'EOF' > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=1
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-;;
-esac
-
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/mysim.swift -nsim=$nsim -steps=$steps -range=$range -values=$values 2> std.err
-EXITCODE=$?
-
-swiftlog=`\ls -t *.log | head -1`
-
-#/tmp/plotswiftlogs $logfile 2>gnuplot.err
-cat << 'EOF' > /tmp/plotswiftlogs
-#!/bin/bash
-
-#usage: ./plotswiftlogs.ketan <swift-logfile.log>
-
-SWIFTLOGFILE=$1
-
-#TMPDIR=`mktemp -d plotlog.XXX`
-
-grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
-
-SWIFTOUTFILE=swiftoutfile.out
-
-#extract start time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
-START_TIME=`date +%s -d "$TMPDATE"`
-
-#extract end time
-TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
-END_TIME=`date +%s -d "$TMPDATE"`
-
-#duration
-DIFFTIME=$((END_TIME - START_TIME))
-
-#extract active runs in a file
-(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt)
-
-#extract successful completions in a file
-(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
-
-#prepare tics
-activelines=`wc -l active.txt | awk '{print $1}'`
-cumulines=`wc -l cumulative.txt | awk '{print $1}'`
-
-if [ $activelines -ne 0 ]
-then
- activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
-fi
-
-seq 0 $activelinespertic $DIFFTIME > activetics.txt
-
-if [ $cumulines -ne 0 ]
-then
- cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
-fi
-
-seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
-
-#final plot data
-paste activetics.txt active.txt > plot_active.txt
-paste cumultics.txt cumulative.txt > plot_cumulative.txt
-
-cat << EOF1 > plotit.gp
-set terminal png enhanced
-set nokey
-set output "cumulativeplot.png"
-set xlabel "Time in sec"
-set ylabel "number of completed apps"
-set title "Cumulative Apps Completed"
-plot "plot_cumulative.txt" using 1:2 with lines
-set output "activeplot.png"
-set xlabel "Time in sec"
-set ylabel "number of active apps"
-set title "Active Apps"
-plot "plot_active.txt" using 1:2 with line
-EOF1
-
-gnuplot plotit.gp 2>/dev/null
-rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt
-
-EOF
-
-sh /tmp/plotswiftlogs $swiftlog
-cp *.png $outdir/
-
-cat << EOF > $htmlfile
-<!DOCTYPE HTML>
-
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head>
- <title>Application output</title>
-</head>
-<body>
-<h2>Swift run trace plots</h2>
-
-<a href="activeplot.png" alt="active plot">active plot</a>
-<a href="cumulativeplot.png" alt="cumulative plot">cumulative plot</a>
-
-<h2>Browse output of application</h2>
-`for i in \`find $outdir -type f\`; do echo "<a href=\`basename $i\`>Output: \`basename $i\`</a><br/>"; done`
-</body>
-</html>
-EOF
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat std.err >&2
-fi
-cp std.err $logfile
-rm std.err
-exit $EXITCODE
Deleted: SwiftApps/swift-galaxy/swift/swiftforeachpy.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachpy.xml 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/swiftforeachpy.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,170 +0,0 @@
-<tool id="swiftforeach" name="foreach">
- <!--
- ToDo:
- 1. Option if the counter should be an argument, first one or last one.
- 2. Option to add files (in Galaxy dataset format) as arguments. These files then be added to Swift app as arguments.
- 3. Allow the tool to select another tool and run it.
- 4. Allow user to add more than one resource to run a job for load balancing.
- 5. Add ability to run Swift/T.
- 6. Let the localhost option be selected by default
- -->
- <description>A generic tool to run executable via Swift foreach parallel construct</description>
-
- <command interpreter="python">
-
- #if $rangeorlist.rl == "rng"
- swiftforeachrange.py
- "${site}"
- "${swiftargs}"
- "${interpret}"
- "${exec}"
- "${rangeorlist.rngstart}"
- "${rangeorlist.rend}"
- "${rangeorlist.stepsize}"
- "${outloc}"
- "${logfile}"
- "${outlist}"
- "${stringargs}"
- #for $a in $filearg
- ${a.fileargname}
- #end for
- #elif $rangeorlist.rl == "lst"
- swiftforeachlist.py
- "${site}"
- "${swiftargs}"
- "${interpret}"
- "${exec}"
- "${rangeorlist.listfile}"
- "${outloc}"
- "${logfile}"
- "${outlist}"
- "${stringargs}"
- #for $a in $filearg
- ${a.fileargname}
- #end for
- #else
- swiftforeachtest.py
- "${site}"
- "${swiftargs}"
- "${interpret}"
- "${exec}"
- "${rangeorlist.rngstart}"
- "${rangeorlist.rend}"
- "${rangeorlist.stepsize}"
- "${rangeorlist.listfile}"
- "${outloc}"
- "${logfile}"
- "${outlist}"
- "${stringargs}"
- #for $a in $filearg
- ${a.fileargname}
- #end for
- #end if
- </command>
- <!--
- #if $swiftargs.mode == "advanced"
- #for $sa in $swiftarg
- ${sa.swiftargname}
- #end for
- #end if
- -->
- <inputs>
- <param name="site" type="select" multiple="true" label="Execution Location" help="Multi-select list - hold the appropriate key while clicking to select multiple items">
- <option value="localhost" selected="true">Localhost</option>
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="stampede">Stampede</option>
- <option value="tukey">Tukey</option>
- </param>
-
- <param name="swiftargs" size="60" type="text" label="swiftargs" help="Add Swift level commandline arguments" />
-
- <param name="interpret" type="select" label="Execution interpreter">
- <option value="sh">sh</option>
- <option value="python">python</option>
- <option value="java">java</option>
- <option value="R">R</option>
- <option value="matlab">matlab</option>
- </param>
-
- <!--<param format="sh,binexec" name="exec" type="data" label="Executable"/>-->
- <param name="exec" type="data" label="Executable" help="select executable" />
-
- <conditional name="rangeorlist">
- <param name="rl" type="select" label="Select range or list">
- <option value="rng">numeric range</option>
- <option value="lst">items list</option>
- <option value="tst">dry test</option>
- </param>
-
- <when value="rng">
- <param name="rngstart" size="2" type="integer" value="0" label="start"/>
- <param name="rend" size="2" type="integer" value="9" label="end"/>
- <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
- </when>
-
- <when value="lst">
- <param format="data" name="listfile" type="data" label="List file" help="Select list file"/>
- </when>
-
- <when value="tst">
- <param name="rngstart" size="2" type="integer" value="0" label="start"/>
- <param name="rend" size="2" type="integer" value="9" label="end"/>
- <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
- <param format="data" name="listfile" type="data" label="List file"/>
- </when>
-
- </conditional>
-
- <param name="stringargs" size="60" type="text" label="stringargs" help="Add application call commandline arguments here" />
-
- <repeat name="filearg" title="file arg">
- <param name="fileargname" type="data" label="filearg" />
- </repeat>
-
- <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc">
- <sanitizer sanitize="False" />
- </param>
- </inputs>
- <outputs>
- <data format="txt" name="logfile" type="data" label="Swift output" />
- <data format="txt" name="outlist" type="data" label="Output list" />
- </outputs>
-
- <!--
- <code file="postprocess.py">
- <hook postprocess="writeoutlist" />
- </code>
- -->
-
-<help>
-.. class:: warningmark
-
-**TIP**. Add args to provide additional arguments to your executable.
-
------
-
-**What it does**
-
-This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file.
-
-Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user.
-
-Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces.
-
------
-
-**Example**
-
-Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello".
-
-If an executable is echo.sh with following line:
-
-echo $@
-
-The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8:
-
-echo n hello
-
-</help>
-</tool>
Deleted: SwiftApps/swift-galaxy/swift/swiftforeachtest.py
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachtest.py 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/swiftforeachtest.py 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,110 +0,0 @@
-#!/usr/bin/env python
-
-import subprocess
-import sys
-#import rpy
-import os
-
-def setwdir():
- return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
-
-# numeric range
-def main():
- print "%d" % (len(sys.argv))
- print "%s" % (sys.argv)
- try:
- sites=sys.argv[1] # comma separated list of sites
- swiftargs=sys.argv[2]
- interpreter=sys.argv[3]
- executable=sys.argv[4]
- rstart=sys.argv[5]
- rend=sys.argv[6]
- stepsize=sys.argv[7]
- outloc=sys.argv[8]
- outlistfile=sys.argv[9]
- logfile=sys.argv[10]
- stringargs=sys.argv[11]
- except:
- #rpy.stop_error('Wrong number or type of args')
- sys.exit(1)
-
- print sites
- print swiftargs
- print interpreter
-
- #workout the array only if user adds args else make it blank
- if stringargs:
- stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
- else:
- stringarrayexpr="string stringargs[];\n"
-
- fileargs=list()
- for files in sys.argv[12:]:
- fileargs.append(files)
-
- #workout the array only if user adds file args else make it blank
- if fileargs:
- filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
- else:
- filearrayexpr="file fileargs[];\n"
-
- wdir=setwdir().rstrip()
- #Build Swift source code
- f=open(wdir+"/script.swift", 'w');
- f.write("type file; \n")
-
- f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
- f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
- f.write("}\n")
-
- f.write(stringarrayexpr)
- f.write(filearrayexpr)
-
- f.write("file exec<'"+executable+"'>;\n")
- f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
- f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
- f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
- f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
- f.write("}\n")
- f.close()
-
- swift=subprocess.check_output(["which", "swift"]).rstrip()
-
- #build site
- if os.path.dirname(sys.argv[0]):
- basedir=os.path.dirname(sys.argv[0])
- else:
- basedir='.'
-
- print "%s %s %s" %(basedir, sites, wdir)
- subprocess.call([basedir+"/buildsite.sh", sites, wdir])
-
- #build config
- f=open(wdir+"/cf", 'w');
- f.write('use.provider.staging=true')
- f.write('wrapperlog.always.transfer=true')
- f.write('execution.retries=0')
- f.write('provider.staging.pin.swiftfiles=false')
- f.write('sitedir.keep=true')
- f.write('lazy.errors=false')
- f.close()
-
-
- #build tc
- f=open(wdir+"/tc", 'w');
- for site in sites.split():
- f.write(site+' '+interpreter+' '+interpreter)
- f.close()
-
- #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"])
-
- #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
- #Populate output file
- outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
- f=open(outlistfile)
- f.write(outlist);
- f.close()
- #dum ditty dum ditty dum dum dum
-
-if __name__=='__main__':
- main()
Deleted: SwiftApps/swift-galaxy/swift/testrun
===================================================================
--- SwiftApps/swift-galaxy/swift/testrun 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/testrun 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,2 +0,0 @@
-python swiftforeachrange.py localhost someargs java /Users/ketan/galaxy-dist/database/files/000/dataset_248.dat 0 9 1 /Users/ketan/galaxy-dist/database/files/000/dataset_306.dat /Users/ketan/swift-sandbox /Users/ketan/galaxy-dist/database/files/000/dataset_307.dat /Users/ketan/galaxy-dist/database/files/000/dataset_308.dat hello /Users/ketan/galaxy-dist/database/files/000/dataset_302.dat
-
Deleted: SwiftApps/swift-galaxy/swift/try.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/try.xml 2014-05-12 22:06:49 UTC (rev 7868)
+++ SwiftApps/swift-galaxy/swift/try.xml 2014-05-13 14:33:31 UTC (rev 7869)
@@ -1,31 +0,0 @@
-<tool id="try" name="Try Swift">
- <description>Try Swift</description>
-
- <command interpreter="bash">
- swift-try $site $swiftscript $log_file
- </command>
-
- <inputs>
- <param name="site" type="select" label="Execution Location">
- <option value="midway">Midway</option>
- <option value="uc3">UC3</option>
- <option value="localhost">Localhost</option>
- <option value="stampede">Stampede</option>
- </param>
- <param name="script" type="text" area="True" size="25x25" label="Swift script"/>
-<!--
- <param name="stochmodel" type="data" label="stochmodel"/>
- <param name="edrevised" type="data" label="edrevised"/>
- <param name="asciidata" type="data" label="ascii data "/>
- <param name="printres" type="data" label="print res inc file"/>
- <param name="illinois" type="data" label="Illinois data file"/>
- <param name="wind_data" type="data" label="wind data file"/>
- -->
- </inputs>
-
- <outputs>
- <data format="txt" type="data" name="log_file"/>
- </outputs>
-
-</tool>
-
More information about the Swift-commit
mailing list