[Swift-commit] r7866 - in SwiftApps/swift-galaxy/swift: . old
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Mon May 12 14:45:47 CDT 2014
Author: ketan
Date: 2014-05-12 14:45:46 -0500 (Mon, 12 May 2014)
New Revision: 7866
Added:
SwiftApps/swift-galaxy/swift/old/swiftforeach.sh
SwiftApps/swift-galaxy/swift/old/swiftforeachlist.py
SwiftApps/swift-galaxy/swift/old/swiftforeachrange.py
SwiftApps/swift-galaxy/swift/old/swiftforeachrange.sh
SwiftApps/swift-galaxy/swift/old/test_exec/
Removed:
SwiftApps/swift-galaxy/swift/swiftforeach.sh
SwiftApps/swift-galaxy/swift/swiftforeachlist.py
SwiftApps/swift-galaxy/swift/swiftforeachrange.py
SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
SwiftApps/swift-galaxy/swift/test_exec/
Log:
clean
Copied: SwiftApps/swift-galaxy/swift/old/swiftforeach.sh (from rev 7862, SwiftApps/swift-galaxy/swift/swiftforeach.sh)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swiftforeach.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swiftforeach.sh 2014-05-12 19:45:46 UTC (rev 7866)
@@ -0,0 +1,282 @@
+#!/bin/bash
+# originally list
+
+#echo $@ > /tmp/args.txt
+#set workdir
+wdir=$(mktemp -d /tmp/swift-gal.XXXX)
+
+#determine loop mode: over a list or a range
+range_or_list=$1
+shift
+
+#determine invocation: Swift /K or /T
+k_or_t=$1
+shift
+
+#build commandline arguments
+
+sites=$1
+shift
+swiftargs=$1
+shift
+interpreter=$1
+shift
+executable=$1
+shift
+stdin=$1
+shift
+
+if [ "$range_or_list" = "list" ] ; then
+ listfile=$1
+ shift
+ n=$(wc -l < $listfile)
+else
+ rstart=$1
+ shift
+ rend=$1
+ shift
+ stepsize=$1
+ shift
+ n=$(( ($rend - $rstart) / $stepsize ))
+fi
+
+outloc=$1
+shift
+logfile=$1
+shift
+outlistfile=$1
+shift
+stringargs=$1
+shift
+
+swift=$(\which swift 2>/dev/null)
+BASEDIR=$(dirname $0)
+
+#workout the file args
+fileargs=()
+while [ $# -gt 0 ] ; do
+ fileargs+=("\"$1\"")
+ shift
+done
+
+#create outloc dir
+mkdir -p $outloc
+
+#workout the stringargs
+if [ "${stringargs}" ] ; then
+ stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
+else
+ stringarrayexpr='string stringargs[];'
+fi
+
+#workout standard input file
+if [ "${stdin}" ] ; then
+ if [ "${k_or_t}" = "swiftK" ] ; then
+ stdinfilexpr='file stdinfile<'\"${stdin}\"'>;'
+ else
+ stdinfilexpr='file stdinfile = input('\"${stdin}\"');'
+ fi
+else
+ if [ "${k_or_t}" = "swiftK" ] ; then
+ stdinfilexpr='file stdinfile<'\"/dev/null\"'>;'
+ else
+ stdinfilexpr='file stdinfile = input('\"/dev/null\"');'
+ fi
+fi
+
+#workout the array only if user adds args else make it blank
+if [ ${#fileargs[@]} -gt 0 ] ; then
+ tmp=$(printf -- '%s,' "${fileargs[@]}")
+ filearrayexpr='file fileargs[] <fixed_array_mapper; files="'$(echo $tmp | sed -e 's/,$//' -e 's/\"//g')'">;';
+else
+ filearrayexpr='file fileargs[];'
+fi
+
+#build swift source
+ if [ "$range_or_list" == "list" ] ; then ## list
+
+ cat << EOF > $wdir/script.swift
+ /*
+ *
+ * Swift source script is generated automatically via the Swift-Galaxy
+ * integration tool. Be careful before making any changes to it directly as
+ * it might be tied with other configuration and paramter values.
+ *
+ */
+ type file;
+
+ app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
+ $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err;
+ }
+ $stringarrayexpr
+ $filearrayexpr
+ $stdinfilexpr
+
+ file exec<"$executable">;
+ file listfile<"$listfile">;
+ string list[] = readData(listfile);
+ foreach l, i in list{
+ file out <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".out")>;
+ file err <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".err")>;
+ (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
+ }
+EOF
+
+else ## range
+
+cat << EOF > $wdir/script.swift
+ /*
+ *
+ * Swift source script is generated automatically via the Swift-Galaxy
+ * integration tool. Be careful before making any changes to it directly as
+ * it might be tied with other configuration and paramter values.
+ *
+ */
+type file;
+
+app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
+ $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err;
+}
+
+$stringarrayexpr
+$filearrayexpr
+$stdinfilexpr
+
+file exec<"$executable">;
+foreach i in [$rstart:$rend:$stepsize]{
+ file out <single_file_mapper; file=@strcat("$outloc", "/", i, ".out")>;
+ file err <single_file_mapper; file=@strcat("$outloc", "/", i, ".err")>;
+ (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
+}
+EOF
+
+fi
+
+touch None # Create a "None" file in case user does not specify any stdin file
+
+
+############# End common setup ################
+
+
+if [ "$k_or_t" = "swiftK" ] ; then #### invoke K
+
+$BASEDIR/buildsite.sh $sites $wdir
+
+#build config
+cat << EOF > $wdir/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+#build tc from sites and interpreter
+IFS=',' read -ra S <<< "$sites"
+for site in "${S[@]}" ; do
+cat << EOF >> $wdir/tc
+$site $interpreter $interpreter
+EOF
+done
+
+$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
+EXITCODE=$?
+
+else
+
+### Invoke T
+
+ #workout the array only if user adds args else make it blank
+ if [ ${#fileargs[@]} -gt 0 ] ; then
+ tmp=$(printf -- '%s,' "${fileargs[@]}")
+ filearrayexpr='file fileargs[] <fixed_array_mapper; files="'$(echo $tmp | sed -e 's/,$//' -e 's/\"//g')'">;';
+ else
+ filearrayexpr='file fileargs[];'
+ fi
+
+ cat << EOF > $wdir/script.swift
+ /*
+ *
+ * Swift source script is generated automatically via the Swift-Galaxy
+ * integration tool. Be careful before making any changes to it directly as
+ * it might be tied with other configuration and paramter values.
+ *
+ */
+
+ import files;
+ import string;
+ import io;
+
+ app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
+ "$interpreter" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err;
+ }
+
+main{
+ $stringarrayexpr
+ $filearrayexpr
+ $stdinfilexpr
+
+ file exec=input("$executable");
+ $( if [ "$range_or_list" = "list" ] ; then
+ echo "file listfile=input("$listfile");"
+ echo "string list[] = split(read(listfile),"\n");"
+ echo "foreach l, i in list{"
+ echo " file out <sprintf(\"$outloc/%s.%i.out\", l, i)>;"
+ echo " file err <sprintf(\"$outloc/%s.%i.err\", l, i)>;"
+ else
+ echo "foreach i in [$rstart:$rend:$stepsize]{"
+ echo " file out <sprintf(\"$outloc/%i.out\", i)>;"
+ echo " file err <sprintf(\"$outloc/%i.err\", i)>;"
+ fi
+ )
+ (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
+ }
+ printf("%s", "Done!");
+ }
+EOF
+
+EXITCODE=0
+case $sites in
+ "localhost" )
+ #echo "in localhost" >> /tmp/status.txt
+ STC=$(\which stc)
+ #echo "got stc" >> /tmp/status.txt
+ turbine=$(\which turbine)
+ #echo "got turbine" >> /tmp/status.txt
+ $STC -j $(\which java) $wdir/script.swift 2>$logfile
+ #echo "done running stc" >> /tmp/status.txt
+ #echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt
+ $turbine -n $n $wdir/script.tcl "${swiftargs}" </dev/null
+ #echo "done running turbine, for $n" >> /tmp/status.txt
+ EXITCODE=$?
+ #echo "exit code obtained" >> /tmp/status.txt
+ ;;
+"Vesta" )
+ TURBINE_HOME=/home/wozniak/Public/sfw/turbine
+ STC=/home/wozniak/Public/sfw/stc/bin/stc
+ $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile
+ $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl
+ EXITCODE=$?
+ ;;
+"Mira" )
+ TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine
+ STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc
+ $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile
+ $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl
+ EXITCODE=$?
+ ;;
+esac
+
+fi
+
+cat << EOF > $outlistfile
+$(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done)
+EOF
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+fi
+
+exit $EXITCODE
+
Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachlist.py (from rev 7862, SwiftApps/swift-galaxy/swift/swiftforeachlist.py)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swiftforeachlist.py (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swiftforeachlist.py 2014-05-12 19:45:46 UTC (rev 7866)
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+
+import subprocess
+import sys
+#import rpy
+import os
+
+def setwdir():
+ return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
+
+# numeric range
+def main():
+ print "%d" % (len(sys.argv))
+ print "%s" % (sys.argv)
+ try:
+ sites=sys.argv[1] # comma separated list of sites
+ swiftargs=sys.argv[2]
+ interpreter=sys.argv[3]
+ executable=sys.argv[4]
+ rstart=sys.argv[5]
+ rend=sys.argv[6]
+ stepsize=sys.argv[7]
+ outloc=sys.argv[8]
+ outlistfile=sys.argv[9]
+ logfile=sys.argv[10]
+ stringargs=sys.argv[11]
+ except:
+ #rpy.stop_error('Wrong number or type of args')
+ sys.exit(1)
+
+ print sites
+ print swiftargs
+ print interpreter
+
+ #workout the array only if user adds args else make it blank
+ if stringargs:
+ stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
+ else:
+ stringarrayexpr="string stringargs[];\n"
+
+ fileargs=list()
+ for files in sys.argv[12:]:
+ fileargs.append(files)
+
+ #workout the array only if user adds file args else make it blank
+ if fileargs:
+ filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
+ else:
+ filearrayexpr="file fileargs[];\n"
+
+ wdir=setwdir().rstrip()
+ #Build Swift source code
+ f=open(wdir+"/script.swift", 'w');
+ f.write("type file; \n")
+
+ f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
+ f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
+ f.write("}\n")
+
+ f.write(stringarrayexpr)
+ f.write(filearrayexpr)
+
+ f.write("file exec<'"+executable+"'>;\n")
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
+ f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
+ f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
+ f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
+ f.write("}\n")
+ f.close()
+
+ swift=subprocess.check_output(["which", "swift"]).rstrip()
+
+ #build site
+ if os.path.dirname(sys.argv[0]):
+ basedir=os.path.dirname(sys.argv[0])
+ else:
+ basedir='.'
+
+ print "%s %s %s" %(basedir, sites, wdir)
+ subprocess.call([basedir+"/buildsite.sh", sites, wdir])
+
+ #build config
+ f=open(wdir+"/cf", 'w');
+ f.write('use.provider.staging=true')
+ f.write('wrapperlog.always.transfer=true')
+ f.write('execution.retries=0')
+ f.write('provider.staging.pin.swiftfiles=false')
+ f.write('sitedir.keep=true')
+ f.write('lazy.errors=false')
+ f.close()
+
+
+ #build tc
+ f=open(wdir+"/tc", 'w');
+ for site in sites.split():
+ f.write(site+' '+interpreter+' '+interpreter)
+ f.close()
+
+ #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"])
+
+ #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
+ #Populate output file
+ outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
+ f=open(outlistfile)
+ f.write(outlist);
+ f.close()
+ #dum ditty dum ditty dum dum dum
+
+if __name__=='__main__':
+ main()
Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachrange.py (from rev 7862, SwiftApps/swift-galaxy/swift/swiftforeachrange.py)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swiftforeachrange.py (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swiftforeachrange.py 2014-05-12 19:45:46 UTC (rev 7866)
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+
+import subprocess
+import sys
+#import rpy
+import os
+
+def setwdir():
+ return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
+
+# numeric range
+def main():
+ print "%d" % (len(sys.argv))
+ print "%s" % (sys.argv)
+ try:
+ sites=sys.argv[1] # comma separated list of sites
+ swiftargs=sys.argv[2]
+ interpreter=sys.argv[3]
+ executable=sys.argv[4]
+ rstart=sys.argv[5]
+ rend=sys.argv[6]
+ stepsize=sys.argv[7]
+ outloc=sys.argv[8]
+ outlistfile=sys.argv[9]
+ logfile=sys.argv[10]
+ stringargs=sys.argv[11]
+ except:
+ #rpy.stop_error('Wrong number or type of args')
+ sys.exit(1)
+
+ print sites
+ print swiftargs
+ print interpreter
+
+ #workout the array only if user adds args else make it blank
+ if stringargs:
+ stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
+ else:
+ stringarrayexpr="string stringargs[];\n"
+
+ fileargs=list()
+ for files in sys.argv[12:]:
+ fileargs.append(files)
+
+ #workout the array only if user adds file args else make it blank
+ if fileargs:
+ filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
+ else:
+ filearrayexpr="file fileargs[];\n"
+
+ wdir=setwdir().rstrip()
+ #Build Swift source code
+ f=open(wdir+"/script.swift", 'w');
+ f.write("type file; \n")
+
+ f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
+ f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
+ f.write("}\n")
+
+ f.write(stringarrayexpr)
+ f.write(filearrayexpr)
+
+ f.write("file exec<'"+executable+"'>;\n")
+ f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
+ f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
+ f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
+ f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
+ f.write("}\n")
+ f.close()
+
+ swift=subprocess.check_output(["which", "swift"]).rstrip()
+
+ #build site
+ if os.path.dirname(sys.argv[0]):
+ basedir=os.path.dirname(sys.argv[0])
+ else:
+ basedir='.'
+
+ print "%s %s %s" %(basedir, sites, wdir)
+ subprocess.call([basedir+"/buildsite.sh", sites, wdir])
+
+ #build config
+ f=open(wdir+"/cf", 'w');
+ f.write('use.provider.staging=true')
+ f.write('wrapperlog.always.transfer=true')
+ f.write('execution.retries=0')
+ f.write('provider.staging.pin.swiftfiles=false')
+ f.write('sitedir.keep=true')
+ f.write('lazy.errors=false')
+ f.close()
+
+
+ #build tc
+ f=open(wdir+"/tc", 'w');
+ for site in sites.split():
+ f.write(site+' '+interpreter+' '+interpreter)
+ f.close()
+
+ #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"])
+
+ #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
+ #Populate output file
+ outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
+ f=open(outlistfile)
+ f.write(outlist);
+ f.close()
+ #dum ditty dum ditty dum dum dum
+
+if __name__=='__main__':
+ main()
Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachrange.sh (from rev 7862, SwiftApps/swift-galaxy/swift/swiftforeachrange.sh)
===================================================================
--- SwiftApps/swift-galaxy/swift/old/swiftforeachrange.sh (rev 0)
+++ SwiftApps/swift-galaxy/swift/old/swiftforeachrange.sh 2014-05-12 19:45:46 UTC (rev 7866)
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+echo $@ > /tmp/args.txt
+wdir=$(mktemp -d /tmp/swift-gal.XXXX)
+
+k_or_t=$1
+shift
+sites=$1 # comma separated list of sites
+shift
+swiftargs=$1
+shift
+interpreter=$1
+shift
+executable=$1
+shift
+stdin=$1
+shift
+rstart=$1
+shift
+rend=$1
+shift
+stepsize=$1
+shift
+outloc=$1
+shift
+logfile=$1
+shift
+outlistfile=$1
+shift
+stringargs=$1
+shift
+
+#workout the array only if user adds args else make it blank
+if [ "${stringargs}" ] ; then
+ stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
+else
+ stringarrayexpr='string stringargs[];'
+fi
+
+#workout standard input file
+if [ "${stdin}" ] ; then
+ stdinfilexpr='file stdinfile<'\"${stdin}\"'>;'
+else
+ stdinfilexpr='file stdinfile<'\"/dev/null\"'>;'
+fi
+
+fileargs=()
+while [ $# -gt 0 ] ; do
+ fileargs+=("\"$1\"")
+ shift
+done
+
+#workout the array only if user adds file args else make it blank
+if [ ${#fileargs[@]} -gt 0 ] ; then
+ tmp=$(printf -- '%s,' "${fileargs[@]}")
+ #file fileargs[] <fixed_array_mapper; files="/Users/ketan/galaxy-dist/database/files/000/dataset_308.dat, /Users/ketan/galaxy-dist/database/files/000/dataset_307.dat">;
+ filearrayexpr='file fileargs[] <fixed_array_mapper; files="'$(echo $tmp | sed -e 's/,$//' -e 's/\"//g')'">;';
+else
+ filearrayexpr='file fileargs[];'
+fi
+
+#Build Swift source code
+cat << EOF > $wdir/script.swift
+
+/*
+*
+* Swift source script is generated automatically via the Swift-Galaxy
+* integration tool. Be careful before making any changes to it directly as
+* it might be tied with other configuration and paramter values.
+*
+*/
+
+type file;
+
+app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
+ $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err;
+}
+
+$stringarrayexpr
+$filearrayexpr
+$stdinfilexpr
+
+file exec<"$executable">;
+foreach i in [$rstart:$rend:$stepsize]{
+ file out <single_file_mapper; file=@strcat("$outloc", "/", i, ".out")>;
+ file err <single_file_mapper; file=@strcat("$outloc", "/", i, ".err")>;
+ (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
+}
+EOF
+
+swift=$(which swift)
+
+#build site
+BASEDIR=$(dirname $0)
+$BASEDIR/buildsite.sh $sites $wdir
+
+#build config
+cat << EOF > $wdir/cf
+use.provider.staging=true
+wrapperlog.always.transfer=true
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+lazy.errors=false
+EOF
+
+#build tc
+IFS=',' read -ra S <<< "$sites"
+for site in "${S[@]}" ; do
+cat << EOF >> $wdir/tc
+$site $interpreter $interpreter
+EOF
+done
+
+touch None # Create a "None" file in case user does not specify any stdin file
+if [ "$k_or_t" = "swiftK" ] ; then
+ $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
+ EXITCODE=$?
+else
+ echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt
+fi
+
+#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
+#Populate output file
+cat << EOF > $outlistfile
+`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done`
+EOF
+
+#cleanup
+#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift
+
+if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+fi
+
+exit $EXITCODE
+
+#dum ditty dum ditty dum dum dum
Deleted: SwiftApps/swift-galaxy/swift/swiftforeach.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-12 19:44:03 UTC (rev 7865)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-12 19:45:46 UTC (rev 7866)
@@ -1,282 +0,0 @@
-#!/bin/bash
-# originally list
-
-#echo $@ > /tmp/args.txt
-#set workdir
-wdir=$(mktemp -d /tmp/swift-gal.XXXX)
-
-#determine loop mode: over a list or a range
-range_or_list=$1
-shift
-
-#determine invocation: Swift /K or /T
-k_or_t=$1
-shift
-
-#build commandline arguments
-
-sites=$1
-shift
-swiftargs=$1
-shift
-interpreter=$1
-shift
-executable=$1
-shift
-stdin=$1
-shift
-
-if [ "$range_or_list" = "list" ] ; then
- listfile=$1
- shift
- n=$(wc -l < $listfile)
-else
- rstart=$1
- shift
- rend=$1
- shift
- stepsize=$1
- shift
- n=$(( ($rend - $rstart) / $stepsize ))
-fi
-
-outloc=$1
-shift
-logfile=$1
-shift
-outlistfile=$1
-shift
-stringargs=$1
-shift
-
-swift=$(\which swift 2>/dev/null)
-BASEDIR=$(dirname $0)
-
-#workout the file args
-fileargs=()
-while [ $# -gt 0 ] ; do
- fileargs+=("\"$1\"")
- shift
-done
-
-#create outloc dir
-mkdir -p $outloc
-
-#workout the stringargs
-if [ "${stringargs}" ] ; then
- stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
-else
- stringarrayexpr='string stringargs[];'
-fi
-
-#workout standard input file
-if [ "${stdin}" ] ; then
- if [ "${k_or_t}" = "swiftK" ] ; then
- stdinfilexpr='file stdinfile<'\"${stdin}\"'>;'
- else
- stdinfilexpr='file stdinfile = input('\"${stdin}\"');'
- fi
-else
- if [ "${k_or_t}" = "swiftK" ] ; then
- stdinfilexpr='file stdinfile<'\"/dev/null\"'>;'
- else
- stdinfilexpr='file stdinfile = input('\"/dev/null\"');'
- fi
-fi
-
-#workout the array only if user adds args else make it blank
-if [ ${#fileargs[@]} -gt 0 ] ; then
- tmp=$(printf -- '%s,' "${fileargs[@]}")
- filearrayexpr='file fileargs[] <fixed_array_mapper; files="'$(echo $tmp | sed -e 's/,$//' -e 's/\"//g')'">;';
-else
- filearrayexpr='file fileargs[];'
-fi
-
-#build swift source
- if [ "$range_or_list" == "list" ] ; then ## list
-
- cat << EOF > $wdir/script.swift
- /*
- *
- * Swift source script is generated automatically via the Swift-Galaxy
- * integration tool. Be careful before making any changes to it directly as
- * it might be tied with other configuration and paramter values.
- *
- */
- type file;
-
- app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
- $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err;
- }
- $stringarrayexpr
- $filearrayexpr
- $stdinfilexpr
-
- file exec<"$executable">;
- file listfile<"$listfile">;
- string list[] = readData(listfile);
- foreach l, i in list{
- file out <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".out")>;
- file err <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".err")>;
- (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
- }
-EOF
-
-else ## range
-
-cat << EOF > $wdir/script.swift
- /*
- *
- * Swift source script is generated automatically via the Swift-Galaxy
- * integration tool. Be careful before making any changes to it directly as
- * it might be tied with other configuration and paramter values.
- *
- */
-type file;
-
-app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
- $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err;
-}
-
-$stringarrayexpr
-$filearrayexpr
-$stdinfilexpr
-
-file exec<"$executable">;
-foreach i in [$rstart:$rend:$stepsize]{
- file out <single_file_mapper; file=@strcat("$outloc", "/", i, ".out")>;
- file err <single_file_mapper; file=@strcat("$outloc", "/", i, ".err")>;
- (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
-}
-EOF
-
-fi
-
-touch None # Create a "None" file in case user does not specify any stdin file
-
-
-############# End common setup ################
-
-
-if [ "$k_or_t" = "swiftK" ] ; then #### invoke K
-
-$BASEDIR/buildsite.sh $sites $wdir
-
-#build config
-cat << EOF > $wdir/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-
-#build tc from sites and interpreter
-IFS=',' read -ra S <<< "$sites"
-for site in "${S[@]}" ; do
-cat << EOF >> $wdir/tc
-$site $interpreter $interpreter
-EOF
-done
-
-$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
-EXITCODE=$?
-
-else
-
-### Invoke T
-
- #workout the array only if user adds args else make it blank
- if [ ${#fileargs[@]} -gt 0 ] ; then
- tmp=$(printf -- '%s,' "${fileargs[@]}")
- filearrayexpr='file fileargs[] <fixed_array_mapper; files="'$(echo $tmp | sed -e 's/,$//' -e 's/\"//g')'">;';
- else
- filearrayexpr='file fileargs[];'
- fi
-
- cat << EOF > $wdir/script.swift
- /*
- *
- * Swift source script is generated automatically via the Swift-Galaxy
- * integration tool. Be careful before making any changes to it directly as
- * it might be tied with other configuration and paramter values.
- *
- */
-
- import files;
- import string;
- import io;
-
- app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
- "$interpreter" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err;
- }
-
-main{
- $stringarrayexpr
- $filearrayexpr
- $stdinfilexpr
-
- file exec=input("$executable");
- $( if [ "$range_or_list" = "list" ] ; then
- echo "file listfile=input("$listfile");"
- echo "string list[] = split(read(listfile),"\n");"
- echo "foreach l, i in list{"
- echo " file out <sprintf(\"$outloc/%s.%i.out\", l, i)>;"
- echo " file err <sprintf(\"$outloc/%s.%i.err\", l, i)>;"
- else
- echo "foreach i in [$rstart:$rend:$stepsize]{"
- echo " file out <sprintf(\"$outloc/%i.out\", i)>;"
- echo " file err <sprintf(\"$outloc/%i.err\", i)>;"
- fi
- )
- (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
- }
- printf("%s", "Done!");
- }
-EOF
-
-EXITCODE=0
-case $sites in
- "localhost" )
- #echo "in localhost" >> /tmp/status.txt
- STC=$(\which stc)
- #echo "got stc" >> /tmp/status.txt
- turbine=$(\which turbine)
- #echo "got turbine" >> /tmp/status.txt
- $STC -j $(\which java) $wdir/script.swift 2>$logfile
- #echo "done running stc" >> /tmp/status.txt
- #echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt
- $turbine -n $n $wdir/script.tcl "${swiftargs}" </dev/null
- #echo "done running turbine, for $n" >> /tmp/status.txt
- EXITCODE=$?
- #echo "exit code obtained" >> /tmp/status.txt
- ;;
-"Vesta" )
- TURBINE_HOME=/home/wozniak/Public/sfw/turbine
- STC=/home/wozniak/Public/sfw/stc/bin/stc
- $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile
- $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl
- EXITCODE=$?
- ;;
-"Mira" )
- TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine
- STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc
- $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile
- $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl
- EXITCODE=$?
- ;;
-esac
-
-fi
-
-cat << EOF > $outlistfile
-$(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done)
-EOF
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat $logfile >&2
-fi
-
-exit $EXITCODE
-
Deleted: SwiftApps/swift-galaxy/swift/swiftforeachlist.py
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.py 2014-05-12 19:44:03 UTC (rev 7865)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.py 2014-05-12 19:45:46 UTC (rev 7866)
@@ -1,110 +0,0 @@
-#!/usr/bin/env python
-
-import subprocess
-import sys
-#import rpy
-import os
-
-def setwdir():
- return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
-
-# numeric range
-def main():
- print "%d" % (len(sys.argv))
- print "%s" % (sys.argv)
- try:
- sites=sys.argv[1] # comma separated list of sites
- swiftargs=sys.argv[2]
- interpreter=sys.argv[3]
- executable=sys.argv[4]
- rstart=sys.argv[5]
- rend=sys.argv[6]
- stepsize=sys.argv[7]
- outloc=sys.argv[8]
- outlistfile=sys.argv[9]
- logfile=sys.argv[10]
- stringargs=sys.argv[11]
- except:
- #rpy.stop_error('Wrong number or type of args')
- sys.exit(1)
-
- print sites
- print swiftargs
- print interpreter
-
- #workout the array only if user adds args else make it blank
- if stringargs:
- stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
- else:
- stringarrayexpr="string stringargs[];\n"
-
- fileargs=list()
- for files in sys.argv[12:]:
- fileargs.append(files)
-
- #workout the array only if user adds file args else make it blank
- if fileargs:
- filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
- else:
- filearrayexpr="file fileargs[];\n"
-
- wdir=setwdir().rstrip()
- #Build Swift source code
- f=open(wdir+"/script.swift", 'w');
- f.write("type file; \n")
-
- f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
- f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
- f.write("}\n")
-
- f.write(stringarrayexpr)
- f.write(filearrayexpr)
-
- f.write("file exec<'"+executable+"'>;\n")
- f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
- f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
- f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
- f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
- f.write("}\n")
- f.close()
-
- swift=subprocess.check_output(["which", "swift"]).rstrip()
-
- #build site
- if os.path.dirname(sys.argv[0]):
- basedir=os.path.dirname(sys.argv[0])
- else:
- basedir='.'
-
- print "%s %s %s" %(basedir, sites, wdir)
- subprocess.call([basedir+"/buildsite.sh", sites, wdir])
-
- #build config
- f=open(wdir+"/cf", 'w');
- f.write('use.provider.staging=true')
- f.write('wrapperlog.always.transfer=true')
- f.write('execution.retries=0')
- f.write('provider.staging.pin.swiftfiles=false')
- f.write('sitedir.keep=true')
- f.write('lazy.errors=false')
- f.close()
-
-
- #build tc
- f=open(wdir+"/tc", 'w');
- for site in sites.split():
- f.write(site+' '+interpreter+' '+interpreter)
- f.close()
-
- #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"])
-
- #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
- #Populate output file
- outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
- f=open(outlistfile)
- f.write(outlist);
- f.close()
- #dum ditty dum ditty dum dum dum
-
-if __name__=='__main__':
- main()
Deleted: SwiftApps/swift-galaxy/swift/swiftforeachrange.py
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.py 2014-05-12 19:44:03 UTC (rev 7865)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.py 2014-05-12 19:45:46 UTC (rev 7866)
@@ -1,110 +0,0 @@
-#!/usr/bin/env python
-
-import subprocess
-import sys
-#import rpy
-import os
-
-def setwdir():
- return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"])
-
-# numeric range
-def main():
- print "%d" % (len(sys.argv))
- print "%s" % (sys.argv)
- try:
- sites=sys.argv[1] # comma separated list of sites
- swiftargs=sys.argv[2]
- interpreter=sys.argv[3]
- executable=sys.argv[4]
- rstart=sys.argv[5]
- rend=sys.argv[6]
- stepsize=sys.argv[7]
- outloc=sys.argv[8]
- outlistfile=sys.argv[9]
- logfile=sys.argv[10]
- stringargs=sys.argv[11]
- except:
- #rpy.stop_error('Wrong number or type of args')
- sys.exit(1)
-
- print sites
- print swiftargs
- print interpreter
-
- #workout the array only if user adds args else make it blank
- if stringargs:
- stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n"
- else:
- stringarrayexpr="string stringargs[];\n"
-
- fileargs=list()
- for files in sys.argv[12:]:
- fileargs.append(files)
-
- #workout the array only if user adds file args else make it blank
- if fileargs:
- filearrayexpr="file fileargs[]="+str(fileargs)+";\n"
- else:
- filearrayexpr="file fileargs[];\n"
-
- wdir=setwdir().rstrip()
- #Build Swift source code
- f=open(wdir+"/script.swift", 'w');
- f.write("type file; \n")
-
- f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n")
- f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n")
- f.write("}\n")
-
- f.write(stringarrayexpr)
- f.write(filearrayexpr)
-
- f.write("file exec<'"+executable+"'>;\n")
- f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n")
- f.write(" file out <single_file_mapper; file=@strcat("+outloc+", '/', i, '.out')>;\n")
- f.write(" file err <single_file_mapper; file=@strcat('"+outloc+"', '/', i, '.err')>;\n")
- f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n")
- f.write("}\n")
- f.close()
-
- swift=subprocess.check_output(["which", "swift"]).rstrip()
-
- #build site
- if os.path.dirname(sys.argv[0]):
- basedir=os.path.dirname(sys.argv[0])
- else:
- basedir='.'
-
- print "%s %s %s" %(basedir, sites, wdir)
- subprocess.call([basedir+"/buildsite.sh", sites, wdir])
-
- #build config
- f=open(wdir+"/cf", 'w');
- f.write('use.provider.staging=true')
- f.write('wrapperlog.always.transfer=true')
- f.write('execution.retries=0')
- f.write('provider.staging.pin.swiftfiles=false')
- f.write('sitedir.keep=true')
- f.write('lazy.errors=false')
- f.close()
-
-
- #build tc
- f=open(wdir+"/tc", 'w');
- for site in sites.split():
- f.write(site+' '+interpreter+' '+interpreter)
- f.close()
-
- #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"])
-
- #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
- #Populate output file
- outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"])
- f=open(outlistfile)
- f.write(outlist);
- f.close()
- #dum ditty dum ditty dum dum dum
-
-if __name__=='__main__':
- main()
Deleted: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-05-12 19:44:03 UTC (rev 7865)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-05-12 19:45:46 UTC (rev 7866)
@@ -1,138 +0,0 @@
-#!/bin/bash
-
-echo $@ > /tmp/args.txt
-wdir=$(mktemp -d /tmp/swift-gal.XXXX)
-
-k_or_t=$1
-shift
-sites=$1 # comma separated list of sites
-shift
-swiftargs=$1
-shift
-interpreter=$1
-shift
-executable=$1
-shift
-stdin=$1
-shift
-rstart=$1
-shift
-rend=$1
-shift
-stepsize=$1
-shift
-outloc=$1
-shift
-logfile=$1
-shift
-outlistfile=$1
-shift
-stringargs=$1
-shift
-
-#workout the array only if user adds args else make it blank
-if [ "${stringargs}" ] ; then
- stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
-else
- stringarrayexpr='string stringargs[];'
-fi
-
-#workout standard input file
-if [ "${stdin}" ] ; then
- stdinfilexpr='file stdinfile<'\"${stdin}\"'>;'
-else
- stdinfilexpr='file stdinfile<'\"/dev/null\"'>;'
-fi
-
-fileargs=()
-while [ $# -gt 0 ] ; do
- fileargs+=("\"$1\"")
- shift
-done
-
-#workout the array only if user adds file args else make it blank
-if [ ${#fileargs[@]} -gt 0 ] ; then
- tmp=$(printf -- '%s,' "${fileargs[@]}")
- #file fileargs[] <fixed_array_mapper; files="/Users/ketan/galaxy-dist/database/files/000/dataset_308.dat, /Users/ketan/galaxy-dist/database/files/000/dataset_307.dat">;
- filearrayexpr='file fileargs[] <fixed_array_mapper; files="'$(echo $tmp | sed -e 's/,$//' -e 's/\"//g')'">;';
-else
- filearrayexpr='file fileargs[];'
-fi
-
-#Build Swift source code
-cat << EOF > $wdir/script.swift
-
-/*
-*
-* Swift source script is generated automatically via the Swift-Galaxy
-* integration tool. Be careful before making any changes to it directly as
-* it might be tied with other configuration and paramter values.
-*
-*/
-
-type file;
-
-app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
- $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err;
-}
-
-$stringarrayexpr
-$filearrayexpr
-$stdinfilexpr
-
-file exec<"$executable">;
-foreach i in [$rstart:$rend:$stepsize]{
- file out <single_file_mapper; file=@strcat("$outloc", "/", i, ".out")>;
- file err <single_file_mapper; file=@strcat("$outloc", "/", i, ".err")>;
- (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
-}
-EOF
-
-swift=$(which swift)
-
-#build site
-BASEDIR=$(dirname $0)
-$BASEDIR/buildsite.sh $sites $wdir
-
-#build config
-cat << EOF > $wdir/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
-EOF
-
-#build tc
-IFS=',' read -ra S <<< "$sites"
-for site in "${S[@]}" ; do
-cat << EOF >> $wdir/tc
-$site $interpreter $interpreter
-EOF
-done
-
-touch None # Create a "None" file in case user does not specify any stdin file
-if [ "$k_or_t" = "swiftK" ] ; then
- $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
- EXITCODE=$?
-else
- echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt
-fi
-
-#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
-#Populate output file
-cat << EOF > $outlistfile
-`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done`
-EOF
-
-#cleanup
-#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift
-
-if [ "$EXITCODE" -ne "0" ]; then
- cat $logfile >&2
-fi
-
-exit $EXITCODE
-
-#dum ditty dum ditty dum dum dum
More information about the Swift-commit
mailing list