[Swift-commit] r7854 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Sun May 11 22:08:22 CDT 2014
Author: ketan
Date: 2014-05-11 22:08:21 -0500 (Sun, 11 May 2014)
New Revision: 7854
Modified:
SwiftApps/swift-galaxy/swift/buildsite.sh
SwiftApps/swift-galaxy/swift/swiftforeach.py
Log:
fixes in python and test for T
Modified: SwiftApps/swift-galaxy/swift/buildsite.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/buildsite.sh 2014-05-11 23:09:31 UTC (rev 7853)
+++ SwiftApps/swift-galaxy/swift/buildsite.sh 2014-05-12 03:08:21 UTC (rev 7854)
@@ -116,10 +116,11 @@
localhost )
cat << EOF >> $wdir/pool.xml
<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local" />
+ <execution provider="local" />
<profile namespace="karajan" key="jobThrottle">0.07</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<filesystem provider="local"/>
+ <profile namespace="swift" key="stagingMethod">local</profile>
<workdirectory>/tmp/swift.work</workdirectory>
</pool>
EOF
@@ -129,10 +130,11 @@
echo "defaulting to localhost"
cat << EOF >> $wdir/pool.xml
<pool handle="localhost">
- <execution provider="coaster" jobmanager="local:local" />
+ <execution provider="local" />
<profile namespace="karajan" key="jobThrottle">0.07</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<filesystem provider="local"/>
+ <profile namespace="swift" key="stagingMethod">local</profile>
<workdirectory>/tmp/swift.work</workdirectory>
</pool>
EOF
Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-11 23:09:31 UTC (rev 7853)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 03:08:21 UTC (rev 7854)
@@ -5,6 +5,7 @@
import os
import distutils.spawn
import traceback
+import fnmatch
def setwdir():
return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
@@ -15,12 +16,12 @@
def buildcfg(wdir):
f=open(wdir+"/cf", 'w');
- f.write('use.provider.staging=true')
- f.write('wrapperlog.always.transfer=true')
- f.write('execution.retries=0')
- f.write('provider.staging.pin.swiftfiles=false')
- f.write('sitedir.keep=true')
- f.write('lazy.errors=false')
+ f.write("use.provider.staging=true\n")
+ f.write("wrapperlog.always.transfer=true\n")
+ f.write("execution.retries=0\n")
+ f.write("provider.staging.pin.swiftfiles=false\n")
+ f.write("sitedir.keep=true\n")
+ f.write("lazy.errors=false\n")
f.close()
# numeric range
@@ -50,6 +51,7 @@
rstart = next(myargs)
rend = next(myargs)
stepsize = next(myargs)
+ n = (rend - rstart)/stepsize
outloc = next(myargs)
outlistfile = next(myargs)
@@ -70,11 +72,11 @@
#workout the file array only if user adds args else make it blank
if fileargs:
if k_or_t == "swiftK":
- filearrayexpr="file fileargs[] <fixed_array_mapper; files=\""+','.join(fileargs)+"\">;"
+ filearrayexpr="file fileargs[] <fixed_array_mapper; files=\""+','.join(fileargs)+"\">;\n"
else: #T
filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
else:
- filearrayexpr="file fileargs[];"
+ filearrayexpr="file fileargs[];\n"
# create outloc dir
if not os.path.exists(outloc):
@@ -90,15 +92,19 @@
#workout standard input file
if stdin:
+ #print "stdin"
if k_or_t == "swiftK":
- stdinfilexpr = "file stdinfile<"+stdin+">;"
+ #print "stdinK"
+ stdinfilexpr = "file stdinfile<"+stdin+">;\n"
else:
- stdinfilexpr = "file stdinfile = input("+stdin+");"
+ #print "stdinT"
+ stdinfilexpr = "file stdinfile = input("+stdin+");\n"
else:
+ #print "no stdin"
if k_or_t == "swiftK":
- stdinfilexpr = "file stdinfile<\"/dev/null\">;"
+ stdinfilexpr = "file stdinfile<\"/dev/null\">;\n"
else:
- stdinfilexpr = "file stdinfile = input(\"/dev/null\");"
+ stdinfilexpr = "file stdinfile = input(\"/dev/null\");\n"
#Build Swift source code
@@ -107,29 +113,29 @@
if k_or_t == "swiftK":
f.write("type file; \n")
- f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
- f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
+ f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ \n")
+ f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; \n")
f.write("}\n")
f.write(stringarrayexpr)
f.write(filearrayexpr)
- f.write("file exec<'"+executable+"'>;\n")
+ f.write(stdinfilexpr)
+ f.write("file exec<\""+executable+"\">;\n")
if range_or_list == "list":
- f.write("file listfile<"+listfile+">;\n")
+ f.write("file listfile<\""+listfile+"\">;\n")
f.write("string list[] = readData(listfile);\n");
f.write("foreach l, i in list{\n")
f.write("file out <single_file_mapper; file=@strcat(\""+outloc+"\", \"/\", l, \".\", i, \".out\")>;\n")
f.write("file err <single_file_mapper; file=@strcat(\""+outloc+"\", \"/\", l, \".\", i, \".err\")>;\n")
- f.write("(out,err) = anapp(exec, i, stringargs, fileargs, stdinfile\n");
else:
f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
- f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
+ f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n")
f.write("}\n")
#build site
@@ -137,7 +143,7 @@
basedir=os.path.dirname(sys.argv[0])
else:
basedir='.'
- print "%s %s %s" %(basedir, sites, wdir)
+ #print "%s %s %s" %(basedir, sites, wdir)
subprocess.call([basedir+"/buildsite.sh", sites, wdir])
#build config
@@ -187,12 +193,79 @@
f.close()
+
+ #Invocation
+ if k_or_t == "swiftK":
+
+ #$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
+ with open(logfile,"wb") as errfile:
+ exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile)
+ print exitcode
+
+ else:
+
+ #print "Invoking T"
+ #EXITCODE=0
+
+ if sites == "localhost":
+
+ print "localhost"
+ stc=distutils.spawn.find_executable("stc")
+ turbine= distutils.spawn.find_executable("turbine")
+ with open(logfile, "wb") as errfile:
+ exitcode=subprocess.check_call([stc, "-j", "$(\which java)", wdir+"/script.swift"], stderr=errfile)
+
+ if exitcode == 0:
+ with open(logfile, "wb") as errfile:
+ exitcode=subprocess.check_call([turbine, "-n", n, wdir+"/script.swift"], stderr=errfile)
+
+
+ elif sites == "Vesta":
+
+ print "vesta"
+
+ elif sites == "Mira":
+
+ print "mira"
+
+ else:
+
+ print "site not recognized"
+ #STC=$(\which stc)
+ ##echo "got stc" >> /tmp/status.txt
+ #turbine=$(\which turbine)
+ ##echo "got turbine" >> /tmp/status.txt
+ #$STC -j $(\which java) $wdir/script.swift 2>$logfile
+ ##echo "done running stc" >> /tmp/status.txt
+ ##echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt
+ #$turbine -n $n $wdir/script.tcl "${swiftargs}" </dev/null
+ ##echo "done running turbine, for $n" >> /tmp/status.txt
+ #EXITCODE=$?
+ ##echo "exit code obtained" >> /tmp/status.txt
+ #;;
+ #"Vesta" )
+ #TURBINE_HOME=/home/wozniak/Public/sfw/turbine
+ #STC=/home/wozniak/Public/sfw/stc/bin/stc
+ #$STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile
+ #$TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl
+ #EXITCODE=$?
+ #;;
+ #"Mira" )
+ #TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine
+ #STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc
+ #$STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile
+ #$TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl
+ #EXITCODE=$?
+ #;;
+ #esac
+
#Populate output file
outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
f=open(outlistfile, "w")
f.write(outlist);
f.close()
- #dum ditty dum ditty dum dum dum
+
+
if __name__=='__main__':
main()
More information about the Swift-commit
mailing list