[Swift-commit] r7814 - SwiftApps/swift-galaxy/swift
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Mon May 5 20:07:20 CDT 2014
Author: ketan
Date: 2014-05-05 20:07:20 -0500 (Mon, 05 May 2014)
New Revision: 7814
Modified:
SwiftApps/swift-galaxy/swift/swiftforeach.sh
SwiftApps/swift-galaxy/swift/swiftforeach.xml
Log:
unify list and range
Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-05 22:01:47 UTC (rev 7813)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 01:07:20 UTC (rev 7814)
@@ -1,118 +1,238 @@
#!/bin/bash
-#ToDo:
-#
-#1. Add a variable number of arguments such that user can add and remove them.
-#
+# originally list
-# numeric range
-site=$1
+wdir=$(mktemp -d /tmp/swift-gal.XXXX)
+
+range_or_list=$1
shift
+
+k_or_t=$1
+shift
+
+sites=$1
+shift
+swiftargs=$1
+shift
interpreter=$1
shift
executable=$1
shift
-rstart=$1
+stdin=$1
shift
-rend=$1
+
+if [ "$range_or_list" == "list" ] ; then
+ listfile=$1
+ shift
+else
+ rstart=$1
+ shift
+ rend=$1
+ shift
+ stepsize=$1
+ shift
+fi
+
+outloc=$1
shift
-stepsize=$1
+logfile=$1
shift
-outloc=$1
-shift
outlistfile=$1
shift
-logfile=$1
-shift
+stringargs=$1
+shift
-a=()
-while [ $# -gt 0 ] ; do
- a+=("\"$1\"")
- shift
-done
-#workout the array only if user adds args else make it blank
-if [ ${#a[@]} -gt 0 ] ; then
- tmp=$(printf -- '%s,' "${a[@]}")
- arrayexpr='string args[]=['$(echo $tmp | sed s/,$//)'];';
+if [ "$k_or_t" = "swiftK" ] ; then
+ #invoke the k.sh
+ k "${@}"
else
- arrayexpr='string args[];'
+ #invoke the t.sh
+ t "${@}"
+ echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt
fi
-#Build Swift source code
-cat << EOF > /tmp/script.swift
-/*
-*
-* Swift source script is generated automatically via the Swift-Galaxy
-* integration tool. Be careful before making any changes to it directly as
-* it might be tied with other configuration and paramter values.
-*
-*/
-
-type file;
-
-app (file _out, file _err) anapp(file _exec, int _i, string _args[]){
- $interpreter @_exec _i _args stdout=@_out stderr=@_err;
-}
-$arrayexpr
-
-file exec<"$executable">;
-foreach i in [$rstart:$rend:$stepsize]{
- file out <single_file_mapper; file=@strcat("$outloc", "/", i, ".out")>;
- file err <single_file_mapper; file=@strcat("$outloc", "/", i, ".err")>;
- (out,err) = anapp(exec, i, args);
-}
+function k()
+{
+
+ #workout the array only if user adds args else make it blank
+ if [ "${stringargs}" ] ; then
+ stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
+ else
+ stringarrayexpr='string stringargs[];'
+ fi
+
+ #workout standard input file
+ if [ "${stdin}" ] ; then
+ stdinfilexpr='file stdinfile<'\"${stdin}\"'>;'
+ else
+ stdinfilexpr='file stdinfile<'\"/dev/null\"'>;'
+ fi
+
+ fileargs=()
+ while [ $# -gt 0 ] ; do
+ fileargs+=("\"$1\"")
+ shift
+ done
+
+ #workout the array only if user adds args else make it blank
+ if [ ${#fileargs[@]} -gt 0 ] ; then
+ tmp=$(printf -- '%s,' "${fileargs[@]}")
+ filearrayexpr='file fileargs[] <fixed_array_mapper; files="'$(echo $tmp | sed -e 's/,$//' -e 's/\"//g')'">;';
+ else
+ filearrayexpr='file fileargs[];'
+ fi
+
+ cat << EOF > $wdir/script.swift
+
+ /*
+ *
+ * Swift source script is generated automatically via the Swift-Galaxy
+ * integration tool. Be careful before making any changes to it directly as
+ * it might be tied with other configuration and paramter values.
+ *
+ */
+
+ type file;
+
+ app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
+ $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err;
+ }
+ $stringarrayexpr
+ $filearrayexpr
+ $stdinfilexpr
+
+ file exec<"$executable">;
+ file listfile<"$listfile">;
+ string list[] = readData(listfile);
+ foreach l, i in list{
+ file out <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".out")>;
+ file err <single_file_mapper; file=@strcat("$outloc", "/", l, ".", i, ".err")>;
+ (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
+ }
EOF
-
-swift=$(which swift)
-
-#build site
-BASEDIR=$(dirname $0)
-$BASEDIR/buildsite.sh $site
-
-#build config
-if [ $site -eq "localhost" ] ; then
-cat << EOF > /tmp/cf
-use.provider.staging=false
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
+
+ swift=$(which swift)
+
+ BASEDIR=$(dirname $0)
+ $BASEDIR/buildsite.sh $sites $wdir
+
+ #build config
+ cat << EOF > $wdir/cf
+ use.provider.staging=true
+ wrapperlog.always.transfer=true
+ execution.retries=0
+ provider.staging.pin.swiftfiles=false
+ sitedir.keep=true
+ lazy.errors=false
+ EOF
+
+ #build tc
+ IFS=',' read -ra S <<< "$sites"
+ for site in "${S[@]}" ; do
+ cat << EOF >> $wdir/tc
+ $site $interpreter $interpreter
EOF
-else
-cat << EOF > /tmp/cf
-use.provider.staging=true
-wrapperlog.always.transfer=true
-execution.retries=0
-provider.staging.pin.swiftfiles=false
-sitedir.keep=true
-lazy.errors=false
+
+ touch None # Create a "None" file in case user does not specify any stdin file
+ $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile
+ EXITCODE=$?
+
+ $(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done)
+
+ cat << EOF > $outlistfile
+ $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done)
EOF
-fi
+
+ if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+ fi
+ exit $EXITCODE
+
+ #dum ditty dum ditty dum dum dum
+}
-#build tc
-cat << EOF > /tmp/tc
-$site $interpreter $interpreter
-EOF
-$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/script.swift 2>$logfile
-EXITCODE=$?
+function t()
+{
+
+ #workout the array only if user adds args else make it blank
+ if [ "${stringargs}" ] ; then
+ stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];';
+ else
+ stringarrayexpr='string stringargs[];'
+ fi
+
+ #workout standard input file
+ if [ "${stdin}" ] ; then
+ stdinfilexpr='file stdinfile<'\"${stdin}\"'>;'
+ else
+ stdinfilexpr='file stdinfile<'\"/dev/null\"'>;'
+ fi
+
+ fileargs=()
+ while [ $# -gt 0 ] ; do
+ fileargs+=("\"$1\"")
+ shift
+ done
+
+ #workout the array only if user adds args else make it blank
+ if [ ${#fileargs[@]} -gt 0 ] ; then
+ tmp=$(printf -- '%s,' "${fileargs[@]}")
+ filearrayexpr='file fileargs[] <fixed_array_mapper; files="'$(echo $tmp | sed -e 's/,$//' -e 's/\"//g')'">;';
+ else
+ filearrayexpr='file fileargs[];'
+ fi
+
+ cat << EOF > $wdir/script.swift
+
+ /*
+ *
+ * Swift source script is generated automatically via the Swift-Galaxy
+ * integration tool. Be careful before making any changes to it directly as
+ * it might be tied with other configuration and paramter values.
+ *
+ */
+
+ import files;
+
+ app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){
+ $interpreter _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err;
+ }
-#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done`
-#Populate output file
-cat << EOF > $outlistfile
-`for i in \`find $outloc -type f\`; do echo "$i"; done`
+ $stringarrayexpr
+ $filearrayexpr
+ $stdinfilexpr
+
+ file exec<"$executable">;
+ file listfile<"$listfile">;
+ string list[] = split(read(input_file(listfile)),"\n");
+ foreach l, i in list{
+ file out <sprintf("$outloc/%s.%i.out", l, i)>;
+ file err <sprintf("$outloc/%s.%i.err", l, i)>;
+ (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);
+ }
EOF
+
+ stc=$(which stc)
+ turbine=$(which turbine)
+ touch None # Create a "None" file in case user does not specify any stdin file
+ $stc -j `which java` $wdir/script.swift "${swiftargs}" 2>$logfile
+
+ $turbine -n 3 $wdir/script.tcl
-#cleanup
-rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift
+ EXITCODE=$?
+
+ $(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done)
+
+ cat << EOF > $outlistfile
+ $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done)
+EOF
+
+ if [ "$EXITCODE" -ne "0" ]; then
+ cat $logfile >&2
+ fi
+ exit $EXITCODE
-if [ "$EXITCODE" -ne "0" ]; then
- cat $logfile >&2
-fi
-
-exit $EXITCODE
-
-#dum ditty dum ditty dum dum dum
+}
Modified: SwiftApps/swift-galaxy/swift/swiftforeach.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-05 22:01:47 UTC (rev 7813)
+++ SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-06 01:07:20 UTC (rev 7814)
@@ -11,9 +11,9 @@
<description>A generic tool to run executable via Swift foreach parallel construct</description>
<command interpreter="bash">
-
#if $rangeorlist.rl == "rng"
- swiftforeachrange.sh
+ swiftforeach.sh
+ "range"
"${k_or_t}"
"${site}"
"${swiftargs}"
@@ -31,7 +31,8 @@
${a.fileargname}
#end for
#elif $rangeorlist.rl == "lst"
- swiftforeachlist.sh
+ swiftforeach.sh
+ "list"
"${k_or_t}"
"${site}"
"${swiftargs}"
@@ -47,7 +48,8 @@
${a.fileargname}
#end for
#else
- swiftforeachtest.sh
+ swiftforeach.sh
+ "drytest"
"${k_or_t}"
"${site}"
"${swiftargs}"
More information about the Swift-commit
mailing list