[Swift-commit] r7852 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Sun May 11 14:32:50 CDT 2014
Author: ketan
Date: 2014-05-11 14:32:48 -0500 (Sun, 11 May 2014)
New Revision: 7852
Added:
SwiftApps/swift-galaxy/swift/swiftforeach.py
Modified:
SwiftApps/swift-galaxy/swift/swiftforeach.sh
Log:
translate python driver script to python
Added: SwiftApps/swift-galaxy/swift/swiftforeach.py
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.py (rev 0)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-11 19:32:48 UTC (rev 7852)
@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+
+import subprocess
+import sys
+import os
+import distutils.
+
+def setwdir():
+ return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
+
+def genargs(args):
+ for a in args:
+ yield a
+
+def buildcfg():
+ f=open(wdir+"/cf", 'w');
+ f.write('use.provider.staging=true')
+ f.write('wrapperlog.always.transfer=true')
+ f.write('execution.retries=0')
+ f.write('provider.staging.pin.swiftfiles=false')
+ f.write('sitedir.keep=true')
+ f.write('lazy.errors=false')
+ f.close()
+
+# numeric range
+def main():
+ myargs = genargs(sys.argv)
+ try:
+ range_or_list = next(myargs)
+ k_or_t = next(myargs)
+ sites = next(myargs)
+ swiftargs = next(myargs)
+ interpreter = next(myargs)
+ executable = next(myargs)
+ stdin = next(myargs)
+
+ if range_or_list == "list":
+ listfile = next(myargs)
+ n = sum(1 for line in open(listfile))
+ else:
+ rstart = next(myargs)
+ rend = next(myargs)
+ stepsize = next(myargs)
+
+ rstart = next(myargs)
+ rend = next(myargs)
+ stepsize = next(myargs)
+ outloc = next(myargs)
+ outlistfile = next(myargs)
+ logfile = next(myargs)
+ stringargs = next(myargs)
+ except:
+ sys.exit(1)
+
+ swift = distutils.spawn.find_executable("swift")
+ BASEDIR = os.path.dirname(os.path.realpath(__file__))
+
+ # workout the file args
+ fileargs=[]
+ for f in myargs:
+ fileargs.append(f)
+
+ #workout the file array only if user adds args else make it blank
+ if fileargs:
+ if k_or_t == "swiftK":
+ filearrayexpr="file fileargs[] <fixed_array_mapper; files=\""+','.join(fileargs)+"\">;"
+ else: #T
+ filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
+ else:
+ filearrayexpr="file fileargs[];"
+
+ # create outloc dir
+ if not os.path.exists(directory):
+ os.makedirs(outloc)
+
+ # workout the stringargs
+ if stringargs:
+ stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
+ else:
+ stringarrayexpr="string stringargs[];\n"
+
+ wdir = setwdir().rstrip()
+
+ #workout standard input file
+ if stdin:
+ if k_or_t == "swiftK":
+ stdinfilexpr = "file stdinfile<"+stdin+">;"
+ else:
+ stdinfilexpr = "file stdinfile = input("+stdin+");"
+ else:
+ if k_or_t == "swiftK":
+ stdinfilexpr = "file stdinfile<\"/dev/null\">;"
+ else:
+ stdinfilexpr = "file stdinfile = input(\"/dev/null\");"
+
+
+ #Build Swift source code
+ f = open(wdir+"/script.swift", 'w');
+
+ if k_or_t == "swiftK":
+
+ f.write("type file; \n")
+ f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
+ f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
+ f.write("}\n")
+ f.write(stringarrayexpr)
+ f.write(filearrayexpr)
+ f.write("file exec<'"+executable+"'>;\n")
+
+ if range_or_list == "list":
+
+ f.write("file listfile<"+listfile+">;\n")
+ f.write("string list[] = readData(listfile);\n");
+ f.write("foreach l, i in list{\n")
+ f.write("file out <single_file_mapper; file=@strcat(\""+outloc+"\", \"/\", l, \".\", i, \".out\")>;\n")
+ f.write("file err <single_file_mapper; file=@strcat(\""+outloc+"\", \"/\", l, \".\", i, \".err\")>;\n")
+ f.write("(out,err) = anapp(exec, i, stringargs, fileargs, stdinfile\n");
+
+ else:
+
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
+ f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
+ f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
+ f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
+
+ f.write("}\n")
+
+ #build site
+ if os.path.dirname(sys.argv[0]):
+ basedir=os.path.dirname(sys.argv[0])
+ else:
+ basedir='.'
+ print "%s %s %s" %(basedir, sites, wdir)
+ subprocess.call([basedir+"/buildsite.sh", sites, wdir])
+
+ #build config
+ buildcfg()
+
+ #build tc
+ tc=open(wdir+"/tc", 'w');
+ for site in sites.split():
+ tc.write(site+' '+interpreter+' '+interpreter)
+
+ tc.close()
+
+ else: # T
+
+ f.write("import files;\n");
+ f.write("import string;\n");
+ f.write("import io;\n");
+ f.write("\n");
+ f.write(" app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){\n");
+ f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n");
+ f.write("}\n");
+ f.write("main{\n");
+ f.write(stringarrayexpr + " \n");
+ f.write(filearrayexpr + " \n");
+ f.write(stdinfilexpr + " \n");
+ f.write("file exec = input("+executable+");\n");
+
+ if range_or_list == "list":
+
+ f.write(" \n");
+ f.write("file listfile=input("$listfile");\n")
+ f.write("string list[] = split(read(listfile),);\n");
+ f.write("foreach l, i in list{"\n)
+ f.write(" file out <sprintf(\""+outloc+"/%s.%i.out\", l, i)>;\n")
+ f.write(" file err <sprintf(\""+outloc+"/%s.%i.err\", l, i)>;\n")
+
+ else:
+
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{ \n");
+ f.write(" file out <sprintf(\""+outloc+"/%i.out\", i)>; \n");
+ f.write(" file err <sprintf(\""+outloc+"/%i.err\", i)>; \n");
+
+ f.write("(out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n")
+ f.write("}\n");
+ f.write("printf("%s", "Done!");\n");
+ f.write("}\n");
+
+ f.close()
+
+ #Populate output file
+ outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
+ f=open(outlistfile)
+ f.write(outlist);
+ f.close()
+ #dum ditty dum ditty dum dum dum
+
+if __name__=='__main__':
+ main()
Property changes on: SwiftApps/swift-galaxy/swift/swiftforeach.py
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-10 18:05:51 UTC (rev 7851)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-11 19:32:48 UTC (rev 7852)
@@ -92,8 +92,9 @@
filearrayexpr='file fileargs[];'
fi
+#build swift source
if [ "$range_or_list" == "list" ] ; then ## list
-
+
cat << EOF > $wdir/script.swift
/*
*
More information about the Swift-commit
mailing list