[Swift-commit] r7623 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Thu Feb 27 20:07:23 CST 2014
Author: ketan
Date: 2014-02-27 20:07:23 -0600 (Thu, 27 Feb 2014)
New Revision: 7623
Added:
SwiftApps/swift-galaxy/swift/swiftforeachpy.xml
SwiftApps/swift-galaxy/swift/swiftforeachtest.py
SwiftApps/swift-galaxy/swift/testrun
Modified:
SwiftApps/swift-galaxy/swift/swiftforeachlist.py
Log:
py tool and a test
Modified: SwiftApps/swift-galaxy/swift/swiftforeachlist.py
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.py 2014-02-27 23:23:51 UTC (rev 7622)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.py 2014-02-28 02:07:23 UTC (rev 7623)
@@ -2,15 +2,16 @@
import subprocess
import sys
-import rpy
+#import rpy
import os
-#echo $@ > /tmp/args.txt
def setwdir():
return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
# numeric range
def main():
+ print "%d" % (len(sys.argv))
+ print "%s" % (sys.argv)
try:
sites=sys.argv[1] # comma separated list of sites
swiftargs=sys.argv[2]
@@ -24,13 +25,18 @@
logfile=sys.argv[10]
stringargs=sys.argv[11]
except:
- rpy.stop_error('Wrong number or type of args')
+ #rpy.stop_error('Wrong number or type of args')
+ sys.exit(1)
+
+ print sites
+ print swiftargs
+ print interpreter
#workout the array only if user adds args else make it blank
if stringargs:
- stringarrayexpr='string stringargs[]='+str(stringargs.split())+';'
+ stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
else:
- stringarrayexpr='string stringargs[];'
+ stringarrayexpr="string stringargs[];\n"
fileargs=list()
for files in sys.argv[12:]:
@@ -38,36 +44,40 @@
#workout the array only if user adds file args else make it blank
if fileargs:
- filearrayexpr='file fileargs[]='+str(fileargs)';'
- else
- filearrayexpr='file fileargs[];'
- fi
+ filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
+ else:
+ filearrayexpr="file fileargs[];\n"
- wdir=setwdir()
+ wdir=setwdir().rstrip()
#Build Swift source code
f=open(wdir+"/script.swift", 'w');
- f.write(type file;)
+ f.write("type file; \n")
- f.write('app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){')
- f.write(' '+interpreter+' @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err;')
- f.write('}')
+ f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
+ f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
+ f.write("}\n")
f.write(stringarrayexpr)
f.write(filearrayexpr)
- f.write('file exec<'+executable+'>;')
- f.write('foreach i in ['+rstart+':'+rend+':'+stepsize+']{')
- f.write(' file out <single_file_mapper; file=@strcat("'+outloc+'", "/", i, ".out")>;')
- f.write(' file err <single_file_mapper; file=@strcat("'+outloc+'", "/", i, ".err")>;')
- f.write(' (out,err) = anapp(exec, i, stringargs, fileargs);')
- f.write('}')
+ f.write("file exec<'"+executable+"'>;\n")
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
+ f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
+ f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
+ f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
+ f.write("}\n")
f.close()
- swift=subprocess.check_output(["which", "swift"])
+ swift=subprocess.check_output(["which", "swift"]).rstrip()
#build site
- basedir=os.path.dirname(sys.argv[0])
- subprocess.call([basedir, "/buildsite.sh", "sites", "wdir"])
+ if os.path.dirname(sys.argv[0]):
+ basedir=os.path.dirname(sys.argv[0])
+ else:
+ basedir='.'
+
+ print "%s %s %s" %(basedir, sites, wdir)
+ subprocess.call([basedir+"/buildsite.sh", sites, wdir])
#build config
f=open(wdir+"/cf", 'w');
@@ -81,18 +91,20 @@
#build tc
-
f=open(wdir+"/tc", 'w');
for site in sites.split():
f.write(site+' '+interpreter+' '+interpreter)
f.close()
- exitcode=subprocess.call([swift, '-sites.file', wdir, '/sites.xml', '-tc.file', wdir, '/tc', '-config', wdir, '/cf', wdir, '/script.swift', swiftargs, '2>logfile'])
+ #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"])
#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
#Populate output file
outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
f=open(outlistfile)
- f.write(i);
+ f.write(outlist);
f.close()
#dum ditty dum ditty dum dum dum
+
+if __name__=='__main__':
+ main()
Added: SwiftApps/swift-galaxy/swift/swiftforeachpy.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachpy.xml (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeachpy.xml 2014-02-28 02:07:23 UTC (rev 7623)
@@ -0,0 +1,170 @@
+<tool id="swiftforeach" name="foreach">
+ <!--
+ ToDo:
+ 1. Option if the counter should be an argument, first one or last one.
+ 2. Option to add files (in Galaxy dataset format) as arguments. These files then be added to Swift app as arguments.
+ 3. Allow the tool to select another tool and run it.
+ 4. Allow user to add more than one resource to run a job for load balancing.
+ 5. Add ability to run Swift/T.
+ 6. Let the localhost option be selected by default
+ -->
+ <description>A generic tool to run executable via Swift foreach parallel construct</description>
+
+ <command interpreter="python">
+
+ #if $rangeorlist.rl == "rng"
+ swiftforeachrange.py
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #elif $rangeorlist.rl == "lst"
+ swiftforeachlist.py
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #else
+ swiftforeachtest.py
+ "${site}"
+ "${swiftargs}"
+ "${interpret}"
+ "${exec}"
+ "${rangeorlist.rngstart}"
+ "${rangeorlist.rend}"
+ "${rangeorlist.stepsize}"
+ "${rangeorlist.listfile}"
+ "${outloc}"
+ "${logfile}"
+ "${outlist}"
+ "${stringargs}"
+ #for $a in $filearg
+ ${a.fileargname}
+ #end for
+ #end if
+ </command>
+ <!--
+ #if $swiftargs.mode == "advanced"
+ #for $sa in $swiftarg
+ ${sa.swiftargname}
+ #end for
+ #end if
+ -->
+ <inputs>
+ <param name="site" type="select" multiple="true" label="Execution Location" help="Multi-select list - hold the appropriate key while clicking to select multiple items">
+ <option value="localhost" selected="true">Localhost</option>
+ <option value="midway">Midway</option>
+ <option value="uc3">UC3</option>
+ <option value="stampede">Stampede</option>
+ <option value="tukey">Tukey</option>
+ </param>
+
+ <param name="swiftargs" size="60" type="text" label="swiftargs" help="Add Swift level commandline arguments" />
+
+ <param name="interpret" type="select" label="Execution interpreter">
+ <option value="sh">sh</option>
+ <option value="python">python</option>
+ <option value="java">java</option>
+ <option value="R">R</option>
+ <option value="matlab">matlab</option>
+ </param>
+
+ <!--<param format="sh,binexec" name="exec" type="data" label="Executable"/>-->
+ <param name="exec" type="data" label="Executable" help="select executable" />
+
+ <conditional name="rangeorlist">
+ <param name="rl" type="select" label="Select range or list">
+ <option value="rng">numeric range</option>
+ <option value="lst">items list</option>
+ <option value="tst">dry test</option>
+ </param>
+
+ <when value="rng">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ </when>
+
+ <when value="lst">
+ <param format="data" name="listfile" type="data" label="List file" help="Select list file"/>
+ </when>
+
+ <when value="tst">
+ <param name="rngstart" size="2" type="integer" value="0" label="start"/>
+ <param name="rend" size="2" type="integer" value="9" label="end"/>
+ <param name="stepsize" size="2" type="integer" value="1" label="stepsize"/>
+ <param format="data" name="listfile" type="data" label="List file"/>
+ </when>
+
+ </conditional>
+
+ <param name="stringargs" size="60" type="text" label="stringargs" help="Add application call commandline arguments here" />
+
+ <repeat name="filearg" title="file arg">
+ <param name="fileargname" type="data" label="filearg" />
+ </repeat>
+
+ <param name="outloc" size="50" type="text" value="$HOME/swift-sandbox" label="location for output files" help="Swift will write output files here on disc">
+ <sanitizer sanitize="False" />
+ </param>
+ </inputs>
+ <outputs>
+ <data format="txt" name="logfile" type="data" label="Swift output" />
+ <data format="txt" name="outlist" type="data" label="Output list" />
+ </outputs>
+
+ <!--
+ <code file="postprocess.py">
+ <hook postprocess="writeoutlist" />
+ </code>
+ -->
+
+<help>
+.. class:: warningmark
+
+**TIP**. Add args to provide additional arguments to your executable.
+
+-----
+
+**What it does**
+
+This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file.
+
+Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user.
+
+Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces.
+
+-----
+
+**Example**
+
+Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello".
+
+If an executable is echo.sh with following line:
+
+echo $@
+
+The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8:
+
+echo n hello
+
+</help>
+</tool>
Added: SwiftApps/swift-galaxy/swift/swiftforeachtest.py
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachtest.py (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeachtest.py 2014-02-28 02:07:23 UTC (rev 7623)
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+
+import subprocess
+import sys
+#import rpy
+import os
+
+def setwdir():
+ return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
+
+# numeric range
+def main():
+ print "%d" % (len(sys.argv))
+ print "%s" % (sys.argv)
+ try:
+ sites=sys.argv[1] # comma separated list of sites
+ swiftargs=sys.argv[2]
+ interpreter=sys.argv[3]
+ executable=sys.argv[4]
+ rstart=sys.argv[5]
+ rend=sys.argv[6]
+ stepsize=sys.argv[7]
+ outloc=sys.argv[8]
+ outlistfile=sys.argv[9]
+ logfile=sys.argv[10]
+ stringargs=sys.argv[11]
+ except:
+ #rpy.stop_error('Wrong number or type of args')
+ sys.exit(1)
+
+ print sites
+ print swiftargs
+ print interpreter
+
+ #workout the array only if user adds args else make it blank
+ if stringargs:
+ stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
+ else:
+ stringarrayexpr="string stringargs[];\n"
+
+ fileargs=list()
+ for files in sys.argv[12:]:
+ fileargs.append(files)
+
+ #workout the array only if user adds file args else make it blank
+ if fileargs:
+ filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
+ else:
+ filearrayexpr="file fileargs[];\n"
+
+ wdir=setwdir().rstrip()
+ #Build Swift source code
+ f=open(wdir+"/script.swift", 'w');
+ f.write("type file; \n")
+
+ f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
+ f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
+ f.write("}\n")
+
+ f.write(stringarrayexpr)
+ f.write(filearrayexpr)
+
+ f.write("file exec<'"+executable+"'>;\n")
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
+ f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
+ f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
+ f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
+ f.write("}\n")
+ f.close()
+
+ swift=subprocess.check_output(["which", "swift"]).rstrip()
+
+ #build site
+ if os.path.dirname(sys.argv[0]):
+ basedir=os.path.dirname(sys.argv[0])
+ else:
+ basedir='.'
+
+ print "%s %s %s" %(basedir, sites, wdir)
+ subprocess.call([basedir+"/buildsite.sh", sites, wdir])
+
+ #build config
+ f=open(wdir+"/cf", 'w');
+ f.write('use.provider.staging=true')
+ f.write('wrapperlog.always.transfer=true')
+ f.write('execution.retries=0')
+ f.write('provider.staging.pin.swiftfiles=false')
+ f.write('sitedir.keep=true')
+ f.write('lazy.errors=false')
+ f.close()
+
+
+ #build tc
+ f=open(wdir+"/tc", 'w');
+ for site in sites.split():
+ f.write(site+' '+interpreter+' '+interpreter)
+ f.close()
+
+ #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"])
+
+ #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
+ #Populate output file
+ outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
+ f=open(outlistfile)
+ f.write(outlist);
+ f.close()
+ #dum ditty dum ditty dum dum dum
+
+if __name__=='__main__':
+ main()
Property changes on: SwiftApps/swift-galaxy/swift/swiftforeachtest.py
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/swift-galaxy/swift/testrun
===================================================================
--- SwiftApps/swift-galaxy/swift/testrun (rev 0)
+++ SwiftApps/swift-galaxy/swift/testrun 2014-02-28 02:07:23 UTC (rev 7623)
@@ -0,0 +1,2 @@
+python swiftforeachrange.py localhost someargs java /Users/ketan/galaxy-dist/database/files/000/dataset_248.dat 0 9 1 /Users/ketan/galaxy-dist/database/files/000/dataset_306.dat /Users/ketan/swift-sandbox /Users/ketan/galaxy-dist/database/files/000/dataset_307.dat /Users/ketan/galaxy-dist/database/files/000/dataset_308.dat hello /Users/ketan/galaxy-dist/database/files/000/dataset_302.dat
+
More information about the Swift-commit
mailing list