[Swift-commit] r8320 - in SwiftApps: . dsp-swift dsp-swift/galaxy
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Wed Dec 10 15:40:08 CST 2014
Author: ketan
Date: 2014-12-10 15:40:07 -0600 (Wed, 10 Dec 2014)
New Revision: 8320
Added:
SwiftApps/dsp-swift/
SwiftApps/dsp-swift/apps
SwiftApps/dsp-swift/cf
SwiftApps/dsp-swift/dsp.blues.swift
SwiftApps/dsp-swift/galaxy/
SwiftApps/dsp-swift/galaxy/README
SwiftApps/dsp-swift/galaxy/setup.py
SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py
SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml
SwiftApps/dsp-swift/runblues
SwiftApps/dsp-swift/runlocal
SwiftApps/dsp-swift/sites.blues.xml
SwiftApps/dsp-swift/sites.local.xml
Log:
adding dsp and galaxy app
Added: SwiftApps/dsp-swift/apps
===================================================================
--- SwiftApps/dsp-swift/apps (rev 0)
+++ SwiftApps/dsp-swift/apps 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,22 @@
+localhost cat /bin/cat null null null
+localhost python /usr/bin/python null null null
+localhost bash /bin/bash null null null
+#
+cluster bgsh /home/ketan/SwiftApps/subjobs/bg.sh null null null
+cluster2 bgsh /home/ketan/SwiftApps/subjobs/bg.sh null null null
+#
+blues dsp /lcrc/project/NEXTGENOPT/DSP/Dsp null null null
+#
+blues lammps lmp_blues null null null
+blues smeagol /home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg null null null
+blues bash /bin/bash null null null
+blues python /soft/python/2.7.6/bin/python null null null
+blues genfdf /home/ketan/ematter/smallrun/swift null null null
+blues mpiexec mpiexec null null null
+#
+localblues lammps lmp_blues null null null
+localblues smeagol /home/heinonen/smeagol-1.2-Feb-2014/smeagol-1.2_blues/Src/smeagol-1.2_csg null null null
+localblues mpiexec mpiexec null null null
+localblues bash /bin/bash null null null
+localblues python /soft/python/2.7.6/bin/python null null null
+localblues dsp /lcrc/project/NEXTGENOPT/DSP/Dsp null null null
Added: SwiftApps/dsp-swift/cf
===================================================================
--- SwiftApps/dsp-swift/cf (rev 0)
+++ SwiftApps/dsp-swift/cf 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,7 @@
+use.provider.staging=false
+wrapperlog.always.transfer=false
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=false
+use.wrapper.staging=false
+
Added: SwiftApps/dsp-swift/dsp.blues.swift
===================================================================
--- SwiftApps/dsp-swift/dsp.blues.swift (rev 0)
+++ SwiftApps/dsp-swift/dsp.blues.swift 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,45 @@
+/* ==
+This is a Swift workflow script for the DSP application. The script contains the following parts:
+
+-- A one-line type declaration
+-- App function definition
+-- Input files and variables definitions
+-- Output files definitions
+-- Invocation of app function
+== */
+
+type file;
+
+/* == App function definitions == */
+
+/* == DSP app definition == */
+app (file _out, file _err) dsp_app (string _filespath, file _sto, file _cor, file _tim, int _a, int _n, int _p, int _I)
+{
+ dsp "-f" _filespath "-a" _a "-n" _n "-p" _p "-I" _I stdout=@_out stderr=@_err;
+}
+
+/* == Input files definitions: Make sure these files are available in the current directory == */
+file farmer_sto <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.sto">;
+file farmer_cor <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.cor">;
+file farmer_tim <"/lcrc/project/NEXTGENOPT/DSP/examples/smps/farmer/farmer.tim">;
+
+file timfiles[]<filesys_mapper; location="./dcap", pattern="*.tim">;
+
+
+int a=1;
+int n=1;
+int p=1;
+int I=10;
+
+foreach in_tim,i in timfiles{
+
+ /* == Output files definitions == */
+ file out <single_file_mapper; file=strcat("outdir/std_", i, ".out")>;
+ file err <single_file_mapper; file=strcat("outdir/std_", i, ".err")>;
+ string filespath[]=strsplit(filename(in_tim),"\\.");
+ file in_sto <single_file_mapper; file=regexp(filename(in_tim), "tim", "sto")>;
+ file in_cor <single_file_mapper; file=regexp(filename(in_tim), "tim", "cor")>;
+
+ /* == App invocation == */
+ (out, err) = dsp_app (filespath[0], in_sto, in_cor, in_tim, a, n, p, I);
+}
Added: SwiftApps/dsp-swift/galaxy/README
===================================================================
--- SwiftApps/dsp-swift/galaxy/README (rev 0)
+++ SwiftApps/dsp-swift/galaxy/README 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,196 @@
+The jump-DSP Swift Galaxy Integration
+======================================
+
+Introduction
+-------------
+
+The Swift-Galaxy integration project aims to build tools which can be used to
+leverage the best features of http://swift-lang.org/main[Swift] and
+http://galaxyproject.org[Galaxy] combined. We start with building a generic
+Galaxy tool, called `foreach` which allows user to compose Galaxy workflows
+such that each computation stage in the workflow is executed by Swift. User can
+compose Galaxy tool and pick up arbitrary executables and dataset to run the
+executable over. User can also run the executable over a list of items such as
+string or file parameters.
+
+NOTE: This document can be found http://swift-lang.org/tutorials/galaxy/tutorial.html[here].
+
+Swift-Galaxy `foreach` Mechanism
+---------------------------------
+
+The basic mechanism is that the tool user-interface will accept parameters and
+will pass them along to a backend Swift "generator". This generator program
+will run the parameters over Swift source and configuration templates. As a
+result, the right source and configuration files are produced in a temporary
+directory (`/tmp/tmpname`). This is then executed and the results collected.
+
+There are two ways the `foreach` Swift tool can be utilized. First, over a range
+of numbers. The tool is used to run any arbitrary executable via Swift `foreach`
+loop over a range of numbers. The executable will run with number in range as
+commandline argument, one at a time. Furthermore, user can add arbitrary string
+arguments in addition to the number argument to the call to executable.
+
+The second way runs the `foreach` loop over a list of items read from an input
+file. These items can be file references or simple strings. The tool will pass
+one item at a time to the executable as argument parameter and run them all in
+parallel. This is similar to the above described number range except that the
+iteration takes place over a list of items provided as a file.
+
+
+Installation
+-------------
+This installation recipe assumes that you have Galaxy preinstalled. If not,
+installation instructions for Galaxy can be found
+https://wiki.galaxyproject.org/Admin/GetGalaxy[here].
+
+Step 0. Checkout the swift-galaxy SVN directory as follows:
+
+----
+svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy
+----
+
+Step 1. Create a symbolic link to the +swift-galaxy/swift+ directory from
+galaxy's +tools+ directory. For example, if Galaxy top level directory is
++galaxy-dist+ and swift directory is $HOME/swift-galaxy/swift:
+
+----
+cd galaxy-dist/tools
+ln -s $HOME/swift-galaxy/swift swift
+----
+
+The advantage of symbolic links is that as users update the tool directory via
+`svn up`, they need not worry about reinstallation. Updates in installation
+gets automatically propagated to Galaxy.
+
+Step 2. Add the tool definitions of the tools contained in this Swift to
+Galaxy's tool_conf.xml. This can be done by copying the text in
+`<section>...</section>` from the tool_conf.xml in swift-galaxy directory and
+paste it to the tool_conf.xml of the Galaxy directory. In the current installation, the text to be copied is:
+
+----
+<section name="swift" id="swift">
+ <tool file="swift/tryswift.xml" />
+ <tool file="swift/genericswift.xml" />
+ <tool file="swift/swiftforeach.xml" />
+</section>
+----
+
+Step 3. (Re)start Galaxy.
+
+----
+sh run.sh
+----
+
+In the Galaxy Web GUI, you should see the tools in a toolset titled Swift.
+
+Screenshots
+-----------
+
+The following screenshot shows overall look-and-feel of the Swift-Galaxy tool
+in the Galaxy web GUI.
+
+image::./screenshots/overall.png["swift-galaxy screenshot", width=600, align="center"]
+
+The rest of this section walks you through each of the components shown in the
+above screen and their utility. The following screenshot shows execution
+location selector. User can select one or more execution locations from the
+list as their target execution site.
+
+image::./screenshots/execloc.png["execution location selector screenshot", width=200, align="center"]
+
+The following screenshot shows a dialog box to add Swift-level commandline
+arguments to be passed to the script.
+
+image::./screenshots/swift-level-args.png["swift-args screenshot", width=200, align="center"]
+
+The following screenshot shows a selection dropdown list to chose the execution
+interpreter from. The interpreters currently supported and tested are shell,
+java and python. Upcoming are R and matlab.
+
+image::./screenshots/execution-interpreter.png["interpreter screenshot", width=150, align="center"]
+
+The following screenshot shows the selection for an executable. User can upload
+an executable and select it from the list.
+
+image::./screenshots/executable.png["executable screenshot", width=200, align="center"]
+
+The following screenshot shows a selection for range of numbers or list of
+items to chose from. User can select one and the corresponding dialog box will
+appear below.
+
+image::./screenshots/rangeorlist.png["select range or list screenshot", width=150, align="center"]
+
+The following screenshot shows a dialog for string or file type arguments. A
+user can add string and filetype arguments to their executable and the
+commandline will be built accordingly.
+
+image::./screenshots/string-or-fileargs.png["string or fileargs screenshot", width=250, align="center"]
+
+The following screenshot shows a dialog box to enter the directory location for
+results of execution. The Swift output files will be placed here:
+
+image::./screenshots/outlocation.png["output location screenshot", width=250, align="center"]
+
+An example
+----------
+
+A simple Hello World Swift tool to run with Galaxy.
+
+To set up with your Galaxy environment follow the steps below:
+Prerequisite: Sun/Oracle java. Should work with IBM java but not tested. To test your java version:
+
+----
+java -version
+----
+
+Typical output is:
+
+----
+java version "1.7.0_17"
+Java(TM) SE Runtime Environment (build 1.7.0_17-b02)
+Java HotSpot(TM) 64-Bit Server VM (build 23.7-b01, mixed mode)
+----
+
+Step 1. Download and install Swift where Galaxy server is running. From a
+command prompt the following wget will pull the latest release from
+http://swift-lang.org/downloads/index.php[here].
+
+Step 2. SVN checkout the swift-galaxy directory and copy the Swift tool directory to Galaxy's tools directory:
+
+----
+svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy
+cp -r swift-galaxy/swift /location/of/galaxy-server/tools/
+----
+
+Step 3. Edit the +tool_conf.xml+ file present at the top-level of galaxy directory tree to add the following lines between the +<toolbox>+ and +</toolbox>+ xml tags:
+
+----
+<section name="swift-galaxy" id="swift-galaxy">
+ <tool file="currdir/nameoftoolfile.xml" />
+</section>
+----
+
+Step 4. Restart the Galaxy server. The swift-galaxy tool should be available in the tool box of Galaxy GUI.
+
+Step 5. Invoke it by entering any number in the dialog box or leaving it to default.
+
+Step 6. For examples of other Galaxy-Swift tools, look into the swift directoty.
+
+Troubleshoot
+-------------
+
+Each tool generates a set of Swift source and configuration files. These files are placed in a unique directory with a random name generated on the fly. The location of the directory is in `/tmp` and it is prefixed with name `swift-gal.` . The suffix is a four letter random number. You can run Swift from this directory to reproduce possible issues as follows:
+
+----
+swift -tc.file tc -sites.file sites.xml -config cf script.swift
+----
+
+Feel free to contact Swift http://swift-lang.org/support/index.php[support] if issues persist.
+
+Upcoming Features
+------------------
+
+* Mixing the order of file and string arguments
+* Optionally allowing user to not use range number as arguments
+* Providing more Swift patterns, eg. `if-else` conditionals
+
Added: SwiftApps/dsp-swift/galaxy/setup.py
===================================================================
--- SwiftApps/dsp-swift/galaxy/setup.py (rev 0)
+++ SwiftApps/dsp-swift/galaxy/setup.py 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+"""
+Setup script to install swift-galaxy framework to your Galaxy installation.
+
+Usage: ./setup.py <path/to/galaxy>
+"""
+
+import xml.etree.ElementTree as ET
+import subprocess
+import sys
+import os
+import distutils.spawn
+import traceback
+import fnmatch
+import tarfile
+import urllib
+
+galaxyloc=sys.argv[1]
+
+#step 0. Download swift-galaxy.tgz and swift-rel.tgz untar and set path
+if not os.path.isfile("/tmp/swift-galaxy.tgz"):
+ urllib.urlretrieve("http://mcs.anl.gov/~ketan/swift-galaxy.tgz", filename="/tmp/swift-galaxy.tgz")
+ swiftgaltar = tarfile.open("/tmp/swift-galaxy.tgz")
+ swiftgaltar.extractall(path="/tmp")
+ swiftgaltar.close()
+
+if not os.path.isfile("/tmp/swift-0.94.1-RC3.tar.gz"):
+ urllib.urlretrieve("http://swift-lang.org/packages/swift-0.94.1-RC3.tar.gz", filename="/tmp/swift-0.94.1-RC3.tar.gz")
+ swiftar = tarfile.open("/tmp/swift-0.94.1-RC3.tar.gz")
+ swiftar.extractall(path="/tmp")
+ swiftar.close()
+ os.environ["PATH"] = os.environ["PATH"] + ":/tmp/swift-0.94.1-RC3/bin"
+
+#step 1. Create symlink to the swift directory from the tools directory of galaxyloc if not already present
+if os.path.lexists(galaxyloc+"/tools/swift"):
+ os.remove(galaxyloc+"/tools/swift")
+
+os.symlink ("/tmp/swift-galaxy/swift", galaxyloc+"/tools/swift");
+
+#step 2. update the tool_conf.xml
+tree = ET.parse(galaxyloc+"/config/tool_conf.xml.main")
+root = tree.getroot()
+
+if 'swift' not in root.attrib:
+ root.append((ET.fromstring('<section name="swift" id="swift">\n <tool file="swift/tryswift.xml" />\n <tool file="swift/genericswift.xml" />\n <tool file="swift/swiftforeach.xml" />\n </section>\n ')))
+
+tree.write (galaxyloc+"/config/tool_conf.xml.main")
+
+#step 3. Print message
+print "Swift/Galaxy Installed."
+
Property changes on: SwiftApps/dsp-swift/galaxy/setup.py
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py
===================================================================
--- SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py (rev 0)
+++ SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,255 @@
+#!/usr/bin/env python
+
+"""
+This is the Swift foreach driver for Galaxy. It receives inputs from the Swift
+"foreach" Galaxy tool and depending on the options specified, builds a
+canonical Swift/K or Swift/T script and runs over specified computational
+environment.
+"""
+
+import subprocess
+import sys
+import os
+import distutils.spawn
+import traceback
+import fnmatch
+import re
+
+def setwdir():
+ return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
+
+def genargs(args):
+ for a in args:
+ yield a
+
+def buildcfg(wdir):
+ f=open(wdir+"/cf", 'w')
+ f.write("use.provider.staging=true\n")
+ f.write("wrapperlog.always.transfer=true\n")
+ f.write("execution.retries=0\n")
+ f.write("provider.staging.pin.swiftfiles=false\n")
+ f.write("sitedir.keep=true\n")
+ f.write("lazy.errors=false\n")
+ f.close()
+
+def main():
+ myargs = genargs(sys.argv)
+ try:
+ this=next(myargs)
+ range_or_list = next(myargs)
+ k_or_t = next(myargs)
+ sites = next(myargs)
+ swiftargs = next(myargs)
+ interpreter = next(myargs)
+ executable = next(myargs)
+ stdin = next(myargs)
+
+ if range_or_list == "list":
+ #list of items
+ listfile = next(myargs)
+ n = sum(1 for line in open(listfile))
+ else:
+ # numeric range
+ rstart = next(myargs)
+ rend = next(myargs)
+ stepsize = next(myargs)
+ n = (int(rend) - int(rstart))/int(stepsize)
+
+ outloc = next(myargs)
+ outlistfile = next(myargs)
+ logfile = next(myargs)
+ stringargs = next(myargs)
+ except:
+ traceback.print_exc()
+ sys.exit(1)
+
+ #which swift
+ swift = distutils.spawn.find_executable("swift")
+
+ BASEDIR = os.path.dirname(os.path.realpath(__file__))
+
+ # workout the file args
+ fileargs=[]
+ for f in myargs:
+ fileargs.append(f)
+
+ #workout the file array only if user adds args else make it blank
+ if fileargs:
+ if k_or_t == "swiftK":
+ filearrayexpr="file fileargs[] <fixed_array_mapper; files=\""+','.join(fileargs)+"\">;\n"
+ else: #T
+ filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
+ else:
+ filearrayexpr="file fileargs[];\n"
+
+ # create outloc dir
+ if not os.path.exists(outloc):
+ os.makedirs(outloc)
+
+ # workout the stringargs
+ if stringargs:
+ stringarrayexpr="string stringargs[]="+re.sub('\'','"',str(stringargs.split()))+";\n"
+ else:
+ stringarrayexpr="string stringargs[];\n"
+
+ wdir = setwdir().rstrip()
+
+ #workout standard input file
+ if stdin != "None":
+ if k_or_t == "swiftK":
+ stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n"
+ else:
+ stdinfilexpr = "file stdinfile = input(\""+stdin+"\");\n"
+ else:
+ #print "no stdin"
+ if k_or_t == "swiftK":
+ stdinfilexpr = "file stdinfile<\"/dev/null\">;\n"
+ else:
+ stdinfilexpr = "file stdinfile = input(\"/dev/null\");\n"
+
+
+ #Build Swift source code
+ f = open(wdir+"/script.swift", "w");
+
+ if k_or_t == "swiftK":
+
+ f.write("type file; \n")
+ f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ \n")
+ f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; \n")
+ f.write("}\n")
+ f.write(stringarrayexpr)
+ f.write(filearrayexpr)
+ f.write(stdinfilexpr)
+ f.write("file exec<\""+executable+"\">;\n")
+
+ if range_or_list == "list":
+
+ f.write("file listfile<\""+listfile+"\">;\n")
+ f.write("string list[] = readData(listfile);\n");
+ f.write("foreach l, i in list{\n")
+ f.write(" file out <single_file_mapper; file=@strcat(\""+outloc+"\", \"/\", l, \".\", i, \".out\")>;\n")
+ f.write(" file err <single_file_mapper; file=@strcat(\""+outloc+"\", \"/\", l, \".\", i, \".err\")>;\n")
+
+ else:
+
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
+ f.write(" file out <single_file_mapper; file=@strcat(\""+outloc+"\", \"/\", i, \".out\")>;\n")
+ f.write(" file err <single_file_mapper; file=@strcat(\""+outloc+"\", \"/\", i, \".err\")>;\n")
+
+ f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n")
+ f.write("}\n")
+
+ #build site
+ if os.path.dirname(sys.argv[0]):
+ basedir=os.path.dirname(sys.argv[0])
+ else:
+ basedir='.'
+ #print "%s %s %s" %(basedir, sites, wdir)
+ subprocess.call([basedir+"/buildsite.sh", sites, wdir])
+
+ #build config file
+ buildcfg(wdir)
+
+ #build tc file
+ with open(wdir+"/tc", 'w') as tc:
+ for site in sites.split():
+ tc.write(site+' '+interpreter+' '+interpreter)
+
+ else: # T
+
+ f.write("import files;\n");
+ f.write("import string;\n");
+ f.write("import io;\n");
+ f.write("\n");
+ f.write(" app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){\n");
+ f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n");
+ f.write("}\n");
+ f.write("main{\n");
+ f.write(stringarrayexpr + " \n");
+ f.write(filearrayexpr + " \n");
+ f.write(stdinfilexpr + " \n");
+ f.write("file exec = input(\""+executable+"\");\n");
+
+ if range_or_list == "list":
+
+ f.write(" \n");
+ f.write("file listfile=input(\""+listfile+"\");\n")
+ f.write("string list[] = split(read(listfile),);\n");
+ f.write("foreach l, i in list{\n")
+ f.write(" file out <sprintf(\""+outloc+"/%s.%i.out\", l, i)>;\n")
+ f.write(" file err <sprintf(\""+outloc+"/%s.%i.err\", l, i)>;\n")
+
+ else:
+
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{ \n");
+ f.write(" file out <sprintf(\""+outloc+"/%i.out\", i)>; \n");
+ f.write(" file err <sprintf(\""+outloc+"/%i.err\", i)>; \n");
+
+ f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n")
+ f.write("}\n");
+ f.write("printf(\"%s\", \"Done!\");\n");
+ f.write("}\n");
+
+ f.close()
+
+ #Invocation
+ if k_or_t == "swiftK":
+
+ with open(logfile,"wb") as errfile:
+ exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile)
+ print exitcode
+
+ else:
+
+ #print "Invoking T"
+ stc=distutils.spawn.find_executable("stc")
+ java=distutils.spawn.find_executable("java")
+ turbine= distutils.spawn.find_executable("turbine")
+
+ if sites == "localhost":
+
+ print "localhost"
+ with open(logfile, "wb") as errfile:
+ exitcode=subprocess.check_call([stc, "-j", java, wdir+"/script.swift"], stderr=errfile)
+
+ if exitcode == 0:
+ with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile:
+ exitcode=subprocess.check_call([turbine, "-n", str(n), wdir+"/script.tcl", swiftargs], stdin=infile, stderr=errfile)
+
+ elif sites == "Vesta":
+
+ print "vesta"
+ pwd=os.path.getcwd()
+ turbine_home="/home/wozniak/Public/sfw/turbine"
+ with open(logfile, "wb") as errfile:
+ exitcode=subprocess.check_call([stc, "-t", "checkpointing", "-r", pwd, "-j", java, wdir+"/script.swift"], stderr=errfile)
+
+ if exitcode == 0:
+ with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile:
+ exitcode=subprocess.check_call([turbine_home+"/scripts/submit/cobalt/turbine-cobalt-run.zsh", "-n", str(n), wdir+"/script.tcl"], stdin=infile, stderr=errfile)
+
+ elif sites == "Mira":
+
+ print "mira"
+ pwd=os.path.getcwd()
+ turbine_home="/home/wozniak/Public/sfw/ppc64/turbine"
+ stc="/home/wozniak/Public/sfw/ppc64/stc/bin/stc"
+ with open(logfile, "wb") as errfile:
+ exitcode=subprocess.check_call([stc, "-t", "checkpointing", "-r", pwd, "-j", java, wdir+"/script.swift"], stderr=errfile)
+
+ if exitcode == 0:
+ with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile:
+ exitcode=subprocess.check_call([turbine_home+"/scripts/submit/cobalt/turbine-cobalt-run.zsh", "-n", str(n), wdir+"/script.tcl"], stdin=infile, stderr=errfile)
+
+ else:
+
+ print "Site not recognized."
+
+ #Populate output file
+ outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
+ f=open(outlistfile, "w")
+ f.write(outlist);
+ f.close()
+
+if __name__=='__main__':
+ main()
Property changes on: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.py
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml
===================================================================
--- SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml (rev 0)
+++ SwiftApps/dsp-swift/galaxy/swiftjumpdsp.xml 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,194 @@
+<tool id="swiftjumpdsp" name="jumpdsp">
+ <!--
+ ToDo:
+ 1. Option if the counter should be an argument, first one or last one.
+ 2. Option to add files (in Galaxy dataset format) as arguments. These files then be added to Swift app as arguments.
+ 3. Allow the tool to select another tool and run it.
+ 4. Allow user to add more than one resource to run a job for load balancing.
+ 5. Add ability to run Swift/T.
+ 6. Let the localhost option be selected by default
+ -->
+ <description>A generic tool to run executable via Swift foreach parallel construct</description>
+
+ <command interpreter="python">
+ #if $rangeorlist.rl == "rng"
+ swiftforeach.py
+ "range"
+ "${k_or_t}"
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${stdin}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #elif $rangeorlist.rl == "lst"
+ swiftforeach.py
+ "list"
+ "${k_or_t}"
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${stdin}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #else
+ swiftforeach.py
+ "drytest"
+ "${k_or_t}"
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${stdin}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #end if
+ </command>
+ <!--
+ #if $swiftargs.mode == "advanced"
+ #for $sa in $swiftarg
+ ${sa.swiftargname}
+ #end for
+ #end if
+ -->
+ <inputs>
+ <param name="k_or_t" type="select" display="radio" label="choose target execution engine">
+ <option value="swiftK" selected="true">Swift/K</option>
+ <option value="swiftT" >Swift/T</option>
+ </param>
+
+ <param name="site" type="select" multiple="true" label="Execution Location" help="Multi-select list - NOTE: Swift/T option currently supports one site">
+ <option value="localhost" selected="true">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="blues">Blues</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ <option value="tukey">Tukey</option>
+ <option value="vesta">Vesta</option>
+ <option value="mira">Mira</option>
+ </param>
+
+ <param name="interpret" type="select" label="Execution interpreter">
+ <option value="sh">sh</option>
+ <option value="python">python</option>
+ <option value="java">java</option>
+ <option value="R">R</option>
+ <option value="matlab">matlab</option>
+ </param>
+
+ <!--<param format="sh,binexec" name="exec" type="data" label="Executable"/>-->
+ <param name="exec" type="data" label="Executable" help="select executable" />
+
+ <param name="stdin" type="data" label="Standard input file" optional="true" help="select standard input" />
+
+ <conditional name="rangeorlist">
+ <param name="rl" type="select" label="Select range or list">
+ <option value="rng">numeric range</option>
+ <option value="lst">items list</option>
+ <option value="tst">dry test</option>
+ </param>
+
+ <when value="rng">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ </when>
+
+ <when value="lst">
+ <param format="data" name="listfile" type="data" label="List file" help="Select list file"/>
+ </when>
+
+ <when value="tst">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ <param format="data" name="listfile" type="data" label="List file"/>
+ </when>
+
+ </conditional>
+
+ <param name="stringargs" size="60" type="text" label="stringargs" help="Add application call commandline arguments here" />
+
+ <repeat name="filearg" title="file arg">
+ <param name="fileargname" type="data" label="filearg" />
+ </repeat>
+
+ <param name="swiftargs" size="60" type="text" label="swiftargs" help="Add Swift level commandline arguments" />
+
+ <param name="outloc" size="50" type="text" value="$HOME/sandbox" label="location for output files" help="Swift will write output files here on disc">
+ <sanitizer sanitize="False" />
+ </param>
+ </inputs>
+ <outputs>
+ <data format="txt" name="logfile" type="data" label="Swift output" />
+ <data format="txt" name="outlist" type="data" label="Output list" />
+ </outputs>
+
+ <!--
+ <code file="postprocess.py">
+ <hook postprocess="writeoutlist" />
+ </code>
+ -->
+
+<help>
+.. class:: warningmark
+
+**TIP**. Add args to provide additional arguments to your executable.
+
+-----
+
+**What it does**
+
+This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items from a file.
+
+Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable uploaded by user.
+
+Start, end, and stepsize are integer values. Note that the stepsize cannot be
+less than 1. Optionally, user can add new arguments to be provided to the tool
+executable after the numeric argument.
+
+-----
+
+**Example**
+
+Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello".
+
+If an executable is echo.sh with following line:
+
+echo $@
+
+The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8:
+
+echo n hello
+
+-----
+
+For more help, see: http://swift-lang.org/tutorials/galaxy/tutorial.html
+
+</help>
+</tool>
Added: SwiftApps/dsp-swift/runblues
===================================================================
--- SwiftApps/dsp-swift/runblues (rev 0)
+++ SwiftApps/dsp-swift/runblues 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,3 @@
+#! /bin/sh
+
+nohup /home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file sites.blues.xml -config cf -tc.file apps dsp.blues.swift </dev/null &>/dev/null &
Property changes on: SwiftApps/dsp-swift/runblues
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/dsp-swift/runlocal
===================================================================
--- SwiftApps/dsp-swift/runlocal (rev 0)
+++ SwiftApps/dsp-swift/runlocal 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,3 @@
+#! /bin/sh
+
+/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file sites.local.xml -config cf -tc.file apps dsp.blues.swift
Property changes on: SwiftApps/dsp-swift/runlocal
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/dsp-swift/sites.blues.xml
===================================================================
--- SwiftApps/dsp-swift/sites.blues.xml (rev 0)
+++ SwiftApps/dsp-swift/sites.blues.xml 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="blues">
+ <execution provider="coaster" jobmanager="local:pbs"/>
+ <filesystem provider="local" url="none" />
+ <!--
+ <profile namespace="globus" key="maxtime">10400</profile>
+ <profile namespace="globus" key="maxWalltime">02:40:00</profile>
+ -->
+ <profile namespace="globus" key="maxtime">370</profile>
+ <profile namespace="globus" key="maxWalltime">00:05:00</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="ppn">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxnodes">1</profile>
+ <profile namespace="karajan" key="jobThrottle">2.20</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/swift.workdir</workdirectory>
+</pool>
+</config>
+
Added: SwiftApps/dsp-swift/sites.local.xml
===================================================================
--- SwiftApps/dsp-swift/sites.local.xml (rev 0)
+++ SwiftApps/dsp-swift/sites.local.xml 2014-12-10 21:40:07 UTC (rev 8320)
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ This is a sites file. This file defines
+ the properties of an execution site used by Swift
+-->
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+<pool handle="localblues">
+<execution provider="coaster" url="none" jobmanager="local:local" />
+<profile namespace="globus" key="maxwalltime">09:55:00</profile>
+<profile namespace="globus" key="walltime">35000</profile>
+<profile namespace="globus" key="maxTime">36000</profile>
+<workdirectory>/home/ketan/swiftwork</workdirectory>
+<profile namespace="karajan" key="jobThrottle">0.09</profile>
+<filesystem provider="local"/>
+</pool>
+</config>
+
More information about the Swift-commit
mailing list