From swift at ci.uchicago.edu Thu May 1 22:20:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 1 May 2014 22:20:03 -0500 (CDT) Subject: [Swift-commit] cog r3895 Message-ID: <20140502032003.B79328D0013C@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3895 | hategan | 2014-05-01 22:18:18 -0500 (Thu, 01 May 2014) | 1 line fixed npe on instantiation of SSH (and others) FileResource with default constructor (bug 1255) ------------------------------------------------------------------------ Index: modules/abstraction-common/src/org/globus/cog/abstraction/impl/common/task/ServiceImpl.java =================================================================== --- modules/abstraction-common/src/org/globus/cog/abstraction/impl/common/task/ServiceImpl.java (revision 3894) +++ modules/abstraction-common/src/org/globus/cog/abstraction/impl/common/task/ServiceImpl.java (working copy) @@ -44,9 +44,11 @@ this(); this.provider = provider; if (serviceContact == null) { - throw new NullPointerException("serviceContact"); + this.serviceContact = DEFAULT_CONTACT; } - this.serviceContact = serviceContact; + else { + this.serviceContact = serviceContact; + } this.securityContext = securityContext; } From davidk at ci.uchicago.edu Mon May 5 13:34:42 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 5 May 2014 13:34:42 -0500 (CDT) Subject: [Swift-commit] r7810 - SwiftApps Message-ID: <20140505183442.B80C79D04B@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-05 13:34:42 -0500 (Mon, 05 May 2014) New Revision: 7810 Added: SwiftApps/moab-wrappers/ Log: Location for moab wrappers From davidk at ci.uchicago.edu Mon May 5 13:35:48 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 5 May 2014 13:35:48 -0500 (CDT) Subject: [Swift-commit] r7811 - SwiftApps/moab-wrappers Message-ID: <20140505183548.013F99D04B@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-05 13:35:47 -0500 (Mon, 05 May 2014) New Revision: 7811 Added: SwiftApps/moab-wrappers/qdel SwiftApps/moab-wrappers/qstat SwiftApps/moab-wrappers/qsub Log: Moab wrappers for qdel, qstat, qsub Added: SwiftApps/moab-wrappers/qdel =================================================================== --- SwiftApps/moab-wrappers/qdel (rev 0) +++ SwiftApps/moab-wrappers/qdel 2014-05-05 18:35:47 UTC (rev 7811) @@ -0,0 +1,3 @@ +#!/bin/bash + +canceljob "$@" Property changes on: SwiftApps/moab-wrappers/qdel ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/moab-wrappers/qstat =================================================================== --- SwiftApps/moab-wrappers/qstat (rev 0) +++ SwiftApps/moab-wrappers/qstat 2014-05-05 18:35:47 UTC (rev 7811) @@ -0,0 +1,32 @@ +#!/usr/bin/python + +import sys +import subprocess +import getpass + +jobs = sys.argv[2:len(sys.argv)] + +username = getpass.getuser() +p = subprocess.Popen(['showq', '-w', 'user='+username], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +out, err = p.communicate() +out_list = out.rstrip().split('\n') + +for o in out_list: + try: + words = o.split() + jobid = words[0].rstrip() + jobstatus = words[2].rstrip() + + if jobid in jobs: + if jobstatus in ["Idle", "Deferred", "BatchHold", "SystemHold", "UserHold", "Deferred", "Migrated", "Staging", "Starting"]: + jobstatus = "Q" + elif jobstatus in ["Running"]: + jobstatus = "R" + elif jobstatus == ["Completed", "Vacated"]: + jobstatus = "C" + else: + continue + print "Job Id: " + jobid + print "\tjob_state = " + jobstatus + "\n" + except: + continue Property changes on: SwiftApps/moab-wrappers/qstat ___________________________________________________________________ Added: svn:executable + * Added: SwiftApps/moab-wrappers/qsub =================================================================== --- SwiftApps/moab-wrappers/qsub (rev 0) +++ SwiftApps/moab-wrappers/qsub 2014-05-05 18:35:47 UTC (rev 7811) @@ -0,0 +1,3 @@ +#!/bin/bash + +msub "$@" Property changes on: SwiftApps/moab-wrappers/qsub ___________________________________________________________________ Added: svn:executable + * From ketan at ci.uchicago.edu Mon May 5 15:54:01 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 5 May 2014 15:54:01 -0500 (CDT) Subject: [Swift-commit] r7812 - SwiftApps/swift-galaxy/swift Message-ID: <20140505205401.ED90D178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-05 15:54:01 -0500 (Mon, 05 May 2014) New Revision: 7812 Modified: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh Log: invoke Swift/T Modified: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-05-05 18:35:47 UTC (rev 7811) +++ SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-05-05 20:54:01 UTC (rev 7812) @@ -4,124 +4,239 @@ k_or_t=$1 shift -sites=$1 # comma separated list of sites -shift -swiftargs=$1 -shift -interpreter=$1 -shift -executable=$1 -shift -stdin=$1 -shift -listfile=$1 -shift -outloc=$1 -shift -logfile=$1 -shift -outlistfile=$1 -shift -stringargs=$1 -shift -#workout the array only if user adds args else make it blank -if [ "${stringargs}" ] ; then - stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; +if [ "$k_or_t" = "swiftK" ] ; then + #invoke the k.sh + k "${@}" else - stringarrayexpr='string stringargs[];' + #invoke the t.sh + t "${@}" + echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt fi -#workout standard input file -if [ "${stdin}" ] ; then - stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' -else - stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' -fi -fileargs=() -while [ $# -gt 0 ] ; do - fileargs+=("\"$1\"") - shift -done - -#workout the array only if user adds args else make it blank -if [ ${#fileargs[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${fileargs[@]}") - filearrayexpr='file fileargs[] ;'; -else - filearrayexpr='file fileargs[];' -fi - -cat << EOF > $wdir/script.swift - -/* -* -* Swift source script is generated automatically via the Swift-Galaxy -* integration tool. Be careful before making any changes to it directly as -* it might be tied with other configuration and paramter values. -* -*/ - -type file; - -app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ - $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; +function k() +{ + sites=$1 # comma separated list of sites + shift + swiftargs=$1 + shift + interpreter=$1 + shift + executable=$1 + shift + stdin=$1 + shift + listfile=$1 + shift + outloc=$1 + shift + logfile=$1 + shift + outlistfile=$1 + shift + stringargs=$1 + shift + + #workout the array only if user adds args else make it blank + if [ "${stringargs}" ] ; then + stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; + else + stringarrayexpr='string stringargs[];' + fi + + #workout standard input file + if [ "${stdin}" ] ; then + stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' + else + stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' + fi + + fileargs=() + while [ $# -gt 0 ] ; do + fileargs+=("\"$1\"") + shift + done + + #workout the array only if user adds args else make it blank + if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + filearrayexpr='file fileargs[] ;'; + else + filearrayexpr='file fileargs[];' + fi + + cat << EOF > $wdir/script.swift + + /* + * + * Swift source script is generated automatically via the Swift-Galaxy + * integration tool. Be careful before making any changes to it directly as + * it might be tied with other configuration and paramter values. + * + */ + + type file; + + app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; + } + $stringarrayexpr + $filearrayexpr + $stdinfilexpr + + file exec<"$executable">; + file listfile<"$listfile">; + string list[] = readData(listfile); + foreach l, i in list{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); + } +EOF + + swift=$(which swift) + + BASEDIR=$(dirname $0) + $BASEDIR/buildsite.sh $sites $wdir + + #build config + cat << EOF > $wdir/cf + use.provider.staging=true + wrapperlog.always.transfer=true + execution.retries=0 + provider.staging.pin.swiftfiles=false + sitedir.keep=true + lazy.errors=false + EOF + + #build tc + IFS=',' read -ra S <<< "$sites" + for site in "${S[@]}" ; do + cat << EOF >> $wdir/tc + $site $interpreter $interpreter +EOF + done + + touch None # Create a "None" file in case user does not specify any stdin file + $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile + EXITCODE=$? + + `for i in \`find $outloc -type f\`; do echo "\`basename $i\` $i"; done` + + cat << EOF > $outlistfile + `for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done` +EOF + + if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 + fi + exit $EXITCODE + + #dum ditty dum ditty dum dum dum } -$stringarrayexpr -$filearrayexpr -$stdinfilexpr -file exec<"$executable">; -file listfile<"$listfile">; -string list[] = readData(listfile); -foreach l, i in list{ - file out ; - file err ; - (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); -} -EOF -swift=$(which swift) +function t() +{ -BASEDIR=$(dirname $0) -$BASEDIR/buildsite.sh $sites $wdir + sites=$1 # comma separated list of sites + shift + swiftargs=$1 + shift + interpreter=$1 + shift + executable=$1 + shift + stdin=$1 + shift + listfile=$1 + shift + outloc=$1 + shift + logfile=$1 + shift + outlistfile=$1 + shift + stringargs=$1 + shift + + #workout the array only if user adds args else make it blank + if [ "${stringargs}" ] ; then + stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; + else + stringarrayexpr='string stringargs[];' + fi + + #workout standard input file + if [ "${stdin}" ] ; then + stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' + else + stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' + fi + + fileargs=() + while [ $# -gt 0 ] ; do + fileargs+=("\"$1\"") + shift + done + + #workout the array only if user adds args else make it blank + if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + filearrayexpr='file fileargs[] ;'; + else + filearrayexpr='file fileargs[];' + fi + + cat << EOF > $wdir/script.swift + + /* + * + * Swift source script is generated automatically via the Swift-Galaxy + * integration tool. Be careful before making any changes to it directly as + * it might be tied with other configuration and paramter values. + * + */ + + import files; + + app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + $interpreter _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; + } -#build config -cat << EOF > $wdir/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false + $stringarrayexpr + $filearrayexpr + $stdinfilexpr + + file exec<"$executable">; + file listfile<"$listfile">; + string list[] = split(read(input_file(listfile)),"\n"); + foreach l, i in list{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); + } EOF + + stc=$(which stc) + turbine=$(which turbine) + touch None # Create a "None" file in case user does not specify any stdin file + $stc -j `which java` $wdir/script.swift "${swiftargs}" 2>$logfile + + $turbine -n 3 $wdir/script.tcl -#build tc -IFS=',' read -ra S <<< "$sites" -for site in "${S[@]}" ; do -cat << EOF >> $wdir/tc -$site $interpreter $interpreter + EXITCODE=$? + + $(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done) + + cat << EOF > $outlistfile + $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) EOF -done + + if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 + fi + exit $EXITCODE -touch None # Create a "None" file in case user does not specify any stdin file -if [ "$k_or_t" = "swiftK" ] ; then - $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile - EXITCODE=$? -else - echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt -fi - -`for i in \`find $outloc -type f\`; do echo "\`basename $i\` $i"; done` - -cat << EOF > $outlistfile -`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done` -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 -fi -exit $EXITCODE - -#dum ditty dum ditty dum dum dum +} From ketan at ci.uchicago.edu Mon May 5 17:01:47 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 5 May 2014 17:01:47 -0500 (CDT) Subject: [Swift-commit] r7813 - SwiftApps/swift-galaxy/swift Message-ID: <20140505220147.C2BBF178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-05 17:01:47 -0500 (Mon, 05 May 2014) New Revision: 7813 Modified: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh Log: small fix Modified: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-05-05 20:54:01 UTC (rev 7812) +++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-05-05 22:01:47 UTC (rev 7813) @@ -27,7 +27,6 @@ shift outlistfile=$1 shift - stringargs=$1 shift From ketan at ci.uchicago.edu Mon May 5 20:07:20 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 5 May 2014 20:07:20 -0500 (CDT) Subject: [Swift-commit] r7814 - SwiftApps/swift-galaxy/swift Message-ID: <20140506010720.58A16178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-05 20:07:20 -0500 (Mon, 05 May 2014) New Revision: 7814 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh SwiftApps/swift-galaxy/swift/swiftforeach.xml Log: unify list and range Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-05 22:01:47 UTC (rev 7813) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 01:07:20 UTC (rev 7814) @@ -1,118 +1,238 @@ #!/bin/bash -#ToDo: -# -#1. Add a variable number of arguments such that user can add and remove them. -# +# originally list -# numeric range -site=$1 +wdir=$(mktemp -d /tmp/swift-gal.XXXX) + +range_or_list=$1 shift + +k_or_t=$1 +shift + +sites=$1 +shift +swiftargs=$1 +shift interpreter=$1 shift executable=$1 shift -rstart=$1 +stdin=$1 shift -rend=$1 + +if [ "$range_or_list" == "list" ] ; then + listfile=$1 + shift +else + rstart=$1 + shift + rend=$1 + shift + stepsize=$1 + shift +fi + +outloc=$1 shift -stepsize=$1 +logfile=$1 shift -outloc=$1 -shift outlistfile=$1 shift -logfile=$1 -shift +stringargs=$1 +shift -a=() -while [ $# -gt 0 ] ; do - a+=("\"$1\"") - shift -done -#workout the array only if user adds args else make it blank -if [ ${#a[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${a[@]}") - arrayexpr='string args[]=['$(echo $tmp | sed s/,$//)'];'; +if [ "$k_or_t" = "swiftK" ] ; then + #invoke the k.sh + k "${@}" else - arrayexpr='string args[];' + #invoke the t.sh + t "${@}" + echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt fi -#Build Swift source code -cat << EOF > /tmp/script.swift -/* -* -* Swift source script is generated automatically via the Swift-Galaxy -* integration tool. Be careful before making any changes to it directly as -* it might be tied with other configuration and paramter values. -* -*/ - -type file; - -app (file _out, file _err) anapp(file _exec, int _i, string _args[]){ - $interpreter @_exec _i _args stdout=@_out stderr=@_err; -} -$arrayexpr - -file exec<"$executable">; -foreach i in [$rstart:$rend:$stepsize]{ - file out ; - file err ; - (out,err) = anapp(exec, i, args); -} +function k() +{ + + #workout the array only if user adds args else make it blank + if [ "${stringargs}" ] ; then + stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; + else + stringarrayexpr='string stringargs[];' + fi + + #workout standard input file + if [ "${stdin}" ] ; then + stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' + else + stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' + fi + + fileargs=() + while [ $# -gt 0 ] ; do + fileargs+=("\"$1\"") + shift + done + + #workout the array only if user adds args else make it blank + if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + filearrayexpr='file fileargs[] ;'; + else + filearrayexpr='file fileargs[];' + fi + + cat << EOF > $wdir/script.swift + + /* + * + * Swift source script is generated automatically via the Swift-Galaxy + * integration tool. Be careful before making any changes to it directly as + * it might be tied with other configuration and paramter values. + * + */ + + type file; + + app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; + } + $stringarrayexpr + $filearrayexpr + $stdinfilexpr + + file exec<"$executable">; + file listfile<"$listfile">; + string list[] = readData(listfile); + foreach l, i in list{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); + } EOF - -swift=$(which swift) - -#build site -BASEDIR=$(dirname $0) -$BASEDIR/buildsite.sh $site - -#build config -if [ $site -eq "localhost" ] ; then -cat << EOF > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false + + swift=$(which swift) + + BASEDIR=$(dirname $0) + $BASEDIR/buildsite.sh $sites $wdir + + #build config + cat << EOF > $wdir/cf + use.provider.staging=true + wrapperlog.always.transfer=true + execution.retries=0 + provider.staging.pin.swiftfiles=false + sitedir.keep=true + lazy.errors=false + EOF + + #build tc + IFS=',' read -ra S <<< "$sites" + for site in "${S[@]}" ; do + cat << EOF >> $wdir/tc + $site $interpreter $interpreter EOF -else -cat << EOF > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false + + touch None # Create a "None" file in case user does not specify any stdin file + $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile + EXITCODE=$? + + $(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done) + + cat << EOF > $outlistfile + $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) EOF -fi + + if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 + fi + exit $EXITCODE + + #dum ditty dum ditty dum dum dum +} -#build tc -cat << EOF > /tmp/tc -$site $interpreter $interpreter -EOF -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/script.swift 2>$logfile -EXITCODE=$? +function t() +{ + + #workout the array only if user adds args else make it blank + if [ "${stringargs}" ] ; then + stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; + else + stringarrayexpr='string stringargs[];' + fi + + #workout standard input file + if [ "${stdin}" ] ; then + stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' + else + stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' + fi + + fileargs=() + while [ $# -gt 0 ] ; do + fileargs+=("\"$1\"") + shift + done + + #workout the array only if user adds args else make it blank + if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + filearrayexpr='file fileargs[] ;'; + else + filearrayexpr='file fileargs[];' + fi + + cat << EOF > $wdir/script.swift + + /* + * + * Swift source script is generated automatically via the Swift-Galaxy + * integration tool. Be careful before making any changes to it directly as + * it might be tied with other configuration and paramter values. + * + */ + + import files; + + app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + $interpreter _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; + } -#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` -#Populate output file -cat << EOF > $outlistfile -`for i in \`find $outloc -type f\`; do echo "$i"; done` + $stringarrayexpr + $filearrayexpr + $stdinfilexpr + + file exec<"$executable">; + file listfile<"$listfile">; + string list[] = split(read(input_file(listfile)),"\n"); + foreach l, i in list{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); + } EOF + + stc=$(which stc) + turbine=$(which turbine) + touch None # Create a "None" file in case user does not specify any stdin file + $stc -j `which java` $wdir/script.swift "${swiftargs}" 2>$logfile + + $turbine -n 3 $wdir/script.tcl -#cleanup -rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift + EXITCODE=$? + + $(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done) + + cat << EOF > $outlistfile + $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) +EOF + + if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 + fi + exit $EXITCODE -if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 -fi - -exit $EXITCODE - -#dum ditty dum ditty dum dum dum +} Modified: SwiftApps/swift-galaxy/swift/swiftforeach.xml =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-05 22:01:47 UTC (rev 7813) +++ SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-06 01:07:20 UTC (rev 7814) @@ -11,9 +11,9 @@ A generic tool to run executable via Swift foreach parallel construct - #if $rangeorlist.rl == "rng" - swiftforeachrange.sh + swiftforeach.sh + "range" "${k_or_t}" "${site}" "${swiftargs}" @@ -31,7 +31,8 @@ ${a.fileargname} #end for #elif $rangeorlist.rl == "lst" - swiftforeachlist.sh + swiftforeach.sh + "list" "${k_or_t}" "${site}" "${swiftargs}" @@ -47,7 +48,8 @@ ${a.fileargname} #end for #else - swiftforeachtest.sh + swiftforeach.sh + "drytest" "${k_or_t}" "${site}" "${swiftargs}" From ketan at ci.uchicago.edu Mon May 5 20:08:05 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 5 May 2014 20:08:05 -0500 (CDT) Subject: [Swift-commit] r7815 - SwiftApps/swift-galaxy/swift Message-ID: <20140506010805.80502178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-05 20:08:05 -0500 (Mon, 05 May 2014) New Revision: 7815 Removed: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh Log: clean up Deleted: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-05-06 01:07:20 UTC (rev 7814) +++ SwiftApps/swift-galaxy/swift/swiftforeachlist.sh 2014-05-06 01:08:05 UTC (rev 7815) @@ -1,242 +0,0 @@ -#!/bin/bash - -wdir=$(mktemp -d /tmp/swift-gal.XXXX) - -k_or_t=$1 -shift - -if [ "$k_or_t" = "swiftK" ] ; then - #invoke the k.sh - k "${@}" -else - #invoke the t.sh - t "${@}" - echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt -fi - - -function k() -{ - sites=$1 # comma separated list of sites - shift - swiftargs=$1 - shift - interpreter=$1 - shift - executable=$1 - shift - stdin=$1 - shift - listfile=$1 - shift - outloc=$1 - shift - logfile=$1 - shift - outlistfile=$1 - shift - stringargs=$1 - shift - - #workout the array only if user adds args else make it blank - if [ "${stringargs}" ] ; then - stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; - else - stringarrayexpr='string stringargs[];' - fi - - #workout standard input file - if [ "${stdin}" ] ; then - stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' - else - stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' - fi - - fileargs=() - while [ $# -gt 0 ] ; do - fileargs+=("\"$1\"") - shift - done - - #workout the array only if user adds args else make it blank - if [ ${#fileargs[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${fileargs[@]}") - filearrayexpr='file fileargs[] ;'; - else - filearrayexpr='file fileargs[];' - fi - - cat << EOF > $wdir/script.swift - - /* - * - * Swift source script is generated automatically via the Swift-Galaxy - * integration tool. Be careful before making any changes to it directly as - * it might be tied with other configuration and paramter values. - * - */ - - type file; - - app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ - $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; - } - $stringarrayexpr - $filearrayexpr - $stdinfilexpr - - file exec<"$executable">; - file listfile<"$listfile">; - string list[] = readData(listfile); - foreach l, i in list{ - file out ; - file err ; - (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); - } -EOF - - swift=$(which swift) - - BASEDIR=$(dirname $0) - $BASEDIR/buildsite.sh $sites $wdir - - #build config - cat << EOF > $wdir/cf - use.provider.staging=true - wrapperlog.always.transfer=true - execution.retries=0 - provider.staging.pin.swiftfiles=false - sitedir.keep=true - lazy.errors=false - EOF - - #build tc - IFS=',' read -ra S <<< "$sites" - for site in "${S[@]}" ; do - cat << EOF >> $wdir/tc - $site $interpreter $interpreter -EOF - done - - touch None # Create a "None" file in case user does not specify any stdin file - $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile - EXITCODE=$? - - `for i in \`find $outloc -type f\`; do echo "\`basename $i\` $i"; done` - - cat << EOF > $outlistfile - `for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done` -EOF - - if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 - fi - exit $EXITCODE - - #dum ditty dum ditty dum dum dum -} - - -function t() -{ - - sites=$1 # comma separated list of sites - shift - swiftargs=$1 - shift - interpreter=$1 - shift - executable=$1 - shift - stdin=$1 - shift - listfile=$1 - shift - outloc=$1 - shift - logfile=$1 - shift - outlistfile=$1 - shift - stringargs=$1 - shift - - #workout the array only if user adds args else make it blank - if [ "${stringargs}" ] ; then - stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; - else - stringarrayexpr='string stringargs[];' - fi - - #workout standard input file - if [ "${stdin}" ] ; then - stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' - else - stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' - fi - - fileargs=() - while [ $# -gt 0 ] ; do - fileargs+=("\"$1\"") - shift - done - - #workout the array only if user adds args else make it blank - if [ ${#fileargs[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${fileargs[@]}") - filearrayexpr='file fileargs[] ;'; - else - filearrayexpr='file fileargs[];' - fi - - cat << EOF > $wdir/script.swift - - /* - * - * Swift source script is generated automatically via the Swift-Galaxy - * integration tool. Be careful before making any changes to it directly as - * it might be tied with other configuration and paramter values. - * - */ - - import files; - - app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ - $interpreter _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; - } - - $stringarrayexpr - $filearrayexpr - $stdinfilexpr - - file exec<"$executable">; - file listfile<"$listfile">; - string list[] = split(read(input_file(listfile)),"\n"); - foreach l, i in list{ - file out ; - file err ; - (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); - } -EOF - - stc=$(which stc) - turbine=$(which turbine) - touch None # Create a "None" file in case user does not specify any stdin file - $stc -j `which java` $wdir/script.swift "${swiftargs}" 2>$logfile - - $turbine -n 3 $wdir/script.tcl - - EXITCODE=$? - - $(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done) - - cat << EOF > $outlistfile - $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) -EOF - - if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 - fi - exit $EXITCODE - -} From ketan at ci.uchicago.edu Mon May 5 20:18:00 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 5 May 2014 20:18:00 -0500 (CDT) Subject: [Swift-commit] r7816 - SwiftApps/swift-galaxy/swift Message-ID: <20140506011800.EA60D178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-05 20:18:00 -0500 (Mon, 05 May 2014) New Revision: 7816 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: document the unified code Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 01:08:05 UTC (rev 7815) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 01:18:00 UTC (rev 7816) @@ -2,14 +2,19 @@ # originally list +#set workdir wdir=$(mktemp -d /tmp/swift-gal.XXXX) +#determine loop mode: over a list or a range range_or_list=$1 shift +#determine invocation: Swift /K or /T k_or_t=$1 shift +#build commandline arguments + sites=$1 shift swiftargs=$1 @@ -42,20 +47,11 @@ stringargs=$1 shift +touch None # Create a "None" file in case user does not specify any stdin file -if [ "$k_or_t" = "swiftK" ] ; then - #invoke the k.sh - k "${@}" -else - #invoke the t.sh - t "${@}" - echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt -fi +if [ "$k_or_t" = "swiftK" ] +then # invoke /K - -function k() -{ - #workout the array only if user adds args else make it blank if [ "${stringargs}" ] ; then stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; @@ -135,27 +131,10 @@ $site $interpreter $interpreter EOF - touch None # Create a "None" file in case user does not specify any stdin file $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile EXITCODE=$? - - $(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done) - - cat << EOF > $outlistfile - $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) -EOF - - if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 - fi - exit $EXITCODE - - #dum ditty dum ditty dum dum dum -} - -function t() -{ +else # Invoke T #workout the array only if user adds args else make it blank if [ "${stringargs}" ] ; then @@ -217,22 +196,24 @@ stc=$(which stc) turbine=$(which turbine) - touch None # Create a "None" file in case user does not specify any stdin file + $stc -j `which java` $wdir/script.swift "${swiftargs}" 2>$logfile $turbine -n 3 $wdir/script.tcl EXITCODE=$? + +fi + + +$(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done) - $(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done) - - cat << EOF > $outlistfile - $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) +cat << EOF > $outlistfile +$(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) EOF - if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 - fi - exit $EXITCODE +if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 +fi +exit $EXITCODE -} From ketan at ci.uchicago.edu Mon May 5 20:43:46 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 5 May 2014 20:43:46 -0500 (CDT) Subject: [Swift-commit] r7817 - SwiftApps/swift-galaxy/swift Message-ID: <20140506014346.6F5AC178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-05 20:43:46 -0500 (Mon, 05 May 2014) New Revision: 7817 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: code clean Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 01:18:00 UTC (rev 7816) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 01:43:46 UTC (rev 7817) @@ -47,31 +47,35 @@ stringargs=$1 shift +#workout the file args +fileargs=() +while [ $# -gt 0 ] ; do + fileargs+=("\"$1\"") + shift +done + +#workout the stringargs +if [ "${stringargs}" ] ; then + stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; +else + stringarrayexpr='string stringargs[];' +fi + +#workout standard input file +if [ "${stdin}" ] ; then + stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' +else + stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' +fi + touch None # Create a "None" file in case user does not specify any stdin file +#### End common setup #### + + if [ "$k_or_t" = "swiftK" ] -then # invoke /K - - #workout the array only if user adds args else make it blank - if [ "${stringargs}" ] ; then - stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; - else - stringarrayexpr='string stringargs[];' - fi +then #### invoke K - #workout standard input file - if [ "${stdin}" ] ; then - stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' - else - stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' - fi - - fileargs=() - while [ $# -gt 0 ] ; do - fileargs+=("\"$1\"") - shift - done - #workout the array only if user adds args else make it blank if [ ${#fileargs[@]} -gt 0 ] ; then tmp=$(printf -- '%s,' "${fileargs[@]}") @@ -80,6 +84,8 @@ filearrayexpr='file fileargs[];' fi + if [ "$range_or_list" == "list" ] + then ## list cat << EOF > $wdir/script.swift /* @@ -108,7 +114,37 @@ (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); } EOF - + +else ## range + +cat << EOF > $wdir/script.swift + +/* +* +* Swift source script is generated automatically via the Swift-Galaxy +* integration tool. Be careful before making any changes to it directly as +* it might be tied with other configuration and paramter values. +* +*/ + +type file; + +app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; +} + +$stringarrayexpr +$filearrayexpr +$stdinfilexpr + +file exec<"$executable">; +foreach i in [$rstart:$rend:$stepsize]{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); +} +EOF + swift=$(which swift) BASEDIR=$(dirname $0) @@ -134,28 +170,10 @@ $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile EXITCODE=$? -else # Invoke T +else +#### Invoke T - #workout the array only if user adds args else make it blank - if [ "${stringargs}" ] ; then - stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; - else - stringarrayexpr='string stringargs[];' - fi - #workout standard input file - if [ "${stdin}" ] ; then - stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' - else - stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' - fi - - fileargs=() - while [ $# -gt 0 ] ; do - fileargs+=("\"$1\"") - shift - done - #workout the array only if user adds args else make it blank if [ ${#fileargs[@]} -gt 0 ] ; then tmp=$(printf -- '%s,' "${fileargs[@]}") @@ -197,7 +215,7 @@ stc=$(which stc) turbine=$(which turbine) - $stc -j `which java` $wdir/script.swift "${swiftargs}" 2>$logfile + $stc -j $(\which java) $wdir/script.swift "${swiftargs}" 2>$logfile $turbine -n 3 $wdir/script.tcl From ketan at ci.uchicago.edu Tue May 6 10:23:23 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 10:23:23 -0500 (CDT) Subject: [Swift-commit] r7818 - SwiftApps/swift-galaxy/swift Message-ID: <20140506152323.2A33E9D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 10:23:22 -0500 (Tue, 06 May 2014) New Revision: 7818 Removed: SwiftApps/swift-galaxy/swift/plotswiftlogs SwiftApps/swift-galaxy/swift/stochmodel.run SwiftApps/swift-galaxy/swift/wind_data.dat Modified: SwiftApps/swift-galaxy/swift/delog SwiftApps/swift-galaxy/swift/swiftforeach.sh SwiftApps/swift-galaxy/swift/swiftforeachtest.sh Log: test, fixes and cleanup Modified: SwiftApps/swift-galaxy/swift/delog =================================================================== --- SwiftApps/swift-galaxy/swift/delog 2014-05-06 01:43:46 UTC (rev 7817) +++ SwiftApps/swift-galaxy/swift/delog 2014-05-06 15:23:22 UTC (rev 7818) @@ -1 +1 @@ -rm -rfv *.d *.rlog *.log swift.work *.out outdir xhat_outdir obj_outdir /tmp/swift.work Results /scratch/midway/maheshwari/swift.workdir +rm -rfv *.d *.rlog *.log swift.work *.out outdir /tmp/swift.work Results /tmp/swift-gal.* Deleted: SwiftApps/swift-galaxy/swift/plotswiftlogs =================================================================== --- SwiftApps/swift-galaxy/swift/plotswiftlogs 2014-05-06 01:43:46 UTC (rev 7817) +++ SwiftApps/swift-galaxy/swift/plotswiftlogs 2014-05-06 15:23:22 UTC (rev 7818) @@ -1,97 +0,0 @@ -#!/bin/bash - -#usage: ./extract4plots - -SWIFTLOGFILE=$1 - -TMPDIR=`mktemp -d plotlog.XXX` - -grep -i ProgressTicker $SWIFTLOGFILE > $TMPDIR/swiftoutfile.out - -SWIFTOUTFILE=$TMPDIR/swiftoutfile.out - -#extract start time -TMPDATE=`grep -i progress $SWIFTOUTFILE | head -n 1 | cut -f1-2 -d ' '` -START_TIME=`date +%s -d "$TMPDATE"` - -#extract end time -TMPDATE=`grep -i progress $SWIFTOUTFILE | tail -n 1 | cut -f1-2 -d ' '` -END_TIME=`date +%s -d "$TMPDATE"` - -#duration -DIFFTIME=$((END_TIME - START_TIME)) - -#extract active runs in a file -grep -o -i "Active:[0-9]*" $SWIFTOUTFILE | awk -F: '{print $2}' > $TMPDIR/active.txt - -#extract successful completions in a file -grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE | awk -F: '{print $2}' > $TMPDIR/cumulative.txt - -#prepare tics -activelines=`wc -l $TMPDIR/active.txt | awk '{print $1}'` -cumulines=`wc -l $TMPDIR/cumulative.txt | awk '{print $1}'` - -activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` -seq 0 $activelinespertic $DIFFTIME > $TMPDIR/activetics.txt - -cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` -seq 0 $cumulinespertic $DIFFTIME > $TMPDIR/cumultics.txt - -#final plot data -paste $TMPDIR/activetics.txt $TMPDIR/active.txt > $TMPDIR/plot_active.txt -paste $TMPDIR/cumultics.txt $TMPDIR/cumulative.txt > $TMPDIR/plot_cumulative.txt - -cat << EOF > $TMPDIR/plotit.gp -set terminal png enhanced -#set term postscript eps enhanced -#set terminal svg enhanced size 1000 1000 -#set style line 1 linecolor rgb "blue" -set output "activeplot.png" -set nokey -set xlabel "Time in sec" -set ylabel "number of active jobs" -set title "Active jobs" -#set xrange [110000:160000] -plot "plot_active.txt" using 1:2 with line - -set output "cumulativeplot.png" -#set xlabel "Time in seconds" -set ylabel "number of completed jobs" -set title "Cumulative jobs" -plot "plot_cumulative.txt" using 1:2 with lines - -#set output "cumulativeplot-openmp.png" -#set xlabel "Time in seconds" -#set ylabel "number of completed jobs" -#set title "Cumulative jobs" -#plot "plot_cumulative.txt" using 1:($2*24) with lines - -#set output "scs_loss.png" -#set title "SciColSim evolution loss Results" -#set xlabel "Evolution" -#set ylabel "Value of loss(AR)" -#plot "anneal.data" using 1 with lines - -#set output "multiloss.png" -#set title "SciColSim evolution loss Results" -#set key auto -#set yrange [0:200] -#set xlabel "Evolution" -#set ylabel "loss" -#plot "multiloss.txt" using 3 with lines title "multiloss mean val", "multiloss.txt" using ($3+$4) with lines title "+stddev", "multiloss.txt" using ($3-$4) with lines title "-stddev" -EOF - -cd $TMPDIR - -gnuplot plotit.gp -cp *.png ../ - -cd .. - -#scicolsim specific -#grep "T =" $SWIFTOUTFILE | awk '{print $6}' | cut -c8- | sed 's/....$//' > T.data - -#grep multi_annealing $SWIFTOUTFILE | grep "1;30" | awk '{print $3}' | cut -c11- | sed 's/....$//' > anneal.data - -#grep returning $SWIFTOUTFILE | awk '{print $3, $4, $5, $6}' | sed -e 's/'ci='//' -e 's/'cj='//' -e 's/'r.loss='//' -e 's/'r.sdev='//' | sort -t' ' -k 1,2n > multiloss.txt - Deleted: SwiftApps/swift-galaxy/swift/stochmodel.run =================================================================== --- SwiftApps/swift-galaxy/swift/stochmodel.run 2014-05-06 01:43:46 UTC (rev 7817) +++ SwiftApps/swift-galaxy/swift/stochmodel.run 2014-05-06 15:23:22 UTC (rev 7818) @@ -1,69 +0,0 @@ -# ampl driver file for unit commitment -# V. M. Zavala - MCS/ANL 2012 - -reset; model edrevised.mod; -option presolve 0; -option solver cbc; -#option solver "/homes/ketan/cbc-2.2/2.2/bin/cbc"; - -# set model data -include set_data_ascii.inc; - -# assign reference bus -let ref_bus := 22671; - -# initialize variables -let{s in SCEN,t in TESTTIME, j in GEN} Pgen[s,t,j]:= np_cap[j]/2; -let{t in TESTTIME, j in GEN} yuc[t,j]:= 1; -let{s in SCEN,t in TESTTIME} slackp[s,t]:= 0; -let{s in SCEN,t in TESTTIME} slackm[s,t]:= 0; - -# fix first-stage variables if requested (for inference app) -if (fix_x == 1) then{ -read{t in TESTTIME,j in GEN} yuc[t,j] < stoch_model_x.in; -read{j in GEN} Pgen_initial[j] < stoch_model_x.in; - -close stoch_model_x.in; - -fix{t in TESTTIME,j in GEN} yuc[t,j]; - fix{j in GEN} Pgen_initial[j]; -} - -# data collection -param pricehis{s in SCEN,t in 0..TH}; -param costhisf{t in 0..TH}; -param costhiss{s in SCEN,t in 0..TH}; -param genhis{s in SCEN,t in 0..TH, j in GEN}; -param Phis{s in SCEN,t in 0..TH, j in LIN}; - -# start closed_loop -let idx_data := 24*90; -let idx_state := 0; - -for{idx_cl in 1..TH/T}{ - -# assign data for current horizon -let{s in SCEN,t in TESTTIME} wind_totalmh[s,t]:= wind_total[s,t+idx_data]; -fix{s in SCEN,t in TESTTIME, j in LOAD} Pload[s,t,j]:= loads[t+idx_data,j]; - -# compare total wind production to load -display wind_totalmh; - -# call solver -solve; - -# collect results over simulation horizon -let{s in SCEN,t in TESTTIME} pricehis[s,idx_state+t] := pfeq.dual[s,t]; -let{t in TESTTIME} costhisf[idx_state+t] := tot_start_cost[t]; -let{s in SCEN,t in TESTTIME} costhiss[s,idx_state+t] := tot_gen_cost[s,t]; -let{s in SCEN,t in TESTTIME,j in GEN} genhis[s,idx_state+t,j] := Pgen[s,t,j]; - -# shift horizon and set new initial conditions -let idx_data := idx_data + T; -let idx_state := idx_state + T; - -} - -# print results -include print_results_revised.inc; - Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 01:43:46 UTC (rev 7817) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 15:23:22 UTC (rev 7818) @@ -47,6 +47,9 @@ stringargs=$1 shift +swift=$(which swift) +BASEDIR=$(dirname $0) + #workout the file args fileargs=() while [ $# -gt 0 ] ; do @@ -68,26 +71,17 @@ stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' fi -touch None # Create a "None" file in case user does not specify any stdin file - -#### End common setup #### - - -if [ "$k_or_t" = "swiftK" ] -then #### invoke K +#workout the array only if user adds args else make it blank +if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + filearrayexpr='file fileargs[] ;'; +else + filearrayexpr='file fileargs[];' +fi - #workout the array only if user adds args else make it blank - if [ ${#fileargs[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${fileargs[@]}") - filearrayexpr='file fileargs[] ;'; - else - filearrayexpr='file fileargs[];' - fi + if [ "$range_or_list" == "list" ] ; then ## list - if [ "$range_or_list" == "list" ] - then ## list cat << EOF > $wdir/script.swift - /* * * Swift source script is generated automatically via the Swift-Galaxy @@ -95,7 +89,6 @@ * it might be tied with other configuration and paramter values. * */ - type file; app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ @@ -118,15 +111,13 @@ else ## range cat << EOF > $wdir/script.swift - -/* -* -* Swift source script is generated automatically via the Swift-Galaxy -* integration tool. Be careful before making any changes to it directly as -* it might be tied with other configuration and paramter values. -* -*/ - + /* + * + * Swift source script is generated automatically via the Swift-Galaxy + * integration tool. Be careful before making any changes to it directly as + * it might be tied with other configuration and paramter values. + * + */ type file; app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ @@ -145,35 +136,41 @@ } EOF - swift=$(which swift) +fi + +touch None # Create a "None" file in case user does not specify any stdin file + +#### End common setup #### + +if [ "$k_or_t" = "swiftK" ] ; then #### invoke K - BASEDIR=$(dirname $0) - $BASEDIR/buildsite.sh $sites $wdir +$BASEDIR/buildsite.sh $sites $wdir - #build config - cat << EOF > $wdir/cf - use.provider.staging=true - wrapperlog.always.transfer=true - execution.retries=0 - provider.staging.pin.swiftfiles=false - sitedir.keep=true - lazy.errors=false - EOF - - #build tc - IFS=',' read -ra S <<< "$sites" - for site in "${S[@]}" ; do - cat << EOF >> $wdir/tc - $site $interpreter $interpreter +#build config +cat << EOF > $wdir/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false EOF - $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile - EXITCODE=$? +#build tc from sites and interpreter +IFS=',' read -ra S <<< "$sites" +for site in "${S[@]}" ; do +cat << EOF >> $wdir/tc +$site $interpreter $interpreter +EOF +done +$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile +EXITCODE=$? + else -#### Invoke T + +### Invoke T - #workout the array only if user adds args else make it blank if [ ${#fileargs[@]} -gt 0 ] ; then tmp=$(printf -- '%s,' "${fileargs[@]}") @@ -183,7 +180,6 @@ fi cat << EOF > $wdir/script.swift - /* * * Swift source script is generated automatically via the Swift-Galaxy @@ -222,9 +218,6 @@ EXITCODE=$? fi - - -$(for i in $(find $outloc -type f); do echo "$(basename $i) $i"; done) cat << EOF > $outlistfile $(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) Modified: SwiftApps/swift-galaxy/swift/swiftforeachtest.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachtest.sh 2014-05-06 01:43:46 UTC (rev 7817) +++ SwiftApps/swift-galaxy/swift/swiftforeachtest.sh 2014-05-06 15:23:22 UTC (rev 7818) @@ -1,8 +1,6 @@ #!/bin/bash ->/tmp/args.txt -numargs=$# -for i in $(seq 0 $numargs) ; do - echo $1 >> /tmp/args.txt - shift -done +#!/bin/bash + +./swiftforeach.sh list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" Deleted: SwiftApps/swift-galaxy/swift/wind_data.dat =================================================================== --- SwiftApps/swift-galaxy/swift/wind_data.dat 2014-05-06 01:43:46 UTC (rev 7817) +++ SwiftApps/swift-galaxy/swift/wind_data.dat 2014-05-06 15:23:22 UTC (rev 7818) @@ -1,8761 +0,0 @@ - 1.4604638e+004 - 1.4634218e+004 - 1.4815815e+004 - 1.4727098e+004 - 1.4743168e+004 - 1.4483015e+004 - 1.2999350e+004 - 1.4630727e+004 - 1.4479719e+004 - 1.4306342e+004 - 1.4157402e+004 - 1.3937587e+004 - 1.3389845e+004 - 1.2486648e+004 - 1.1849529e+004 - 1.1162285e+004 - 1.0812127e+004 - 1.0708416e+004 - 6.8720418e+003 - 8.7812242e+003 - 9.5594627e+003 - 1.0758985e+004 - 1.1421796e+004 - 1.3009577e+004 - 1.4180183e+004 - 1.4329001e+004 - 1.4396551e+004 - 1.4341032e+004 - 1.4108325e+004 - 1.3147303e+004 - 8.8378060e+003 - 1.0591343e+004 - 9.0934965e+003 - 7.6853615e+003 - 6.1331303e+003 - 5.6353524e+003 - 4.4302037e+003 - 3.3891375e+003 - 3.0540820e+003 - 4.9563466e+003 - 6.7906722e+003 - 8.3109852e+003 - 6.4083670e+003 - 9.4898396e+003 - 1.0930867e+004 - 1.0099475e+004 - 1.0554804e+004 - 1.1486225e+004 - 1.2128493e+004 - 1.2527107e+004 - 1.3112883e+004 - 1.3659317e+004 - 1.3730192e+004 - 1.3472952e+004 - 1.1245654e+004 - 1.4206701e+004 - 1.4806646e+004 - 1.4734454e+004 - 1.4761135e+004 - 1.4911007e+004 - 1.4958264e+004 - 1.4989285e+004 - 1.4688689e+004 - 1.4345752e+004 - 1.4005039e+004 - 1.2135346e+004 - 6.6638895e+003 - 7.2567263e+003 - 5.9924644e+003 - 6.3092870e+003 - 2.1547187e+003 - 9.3036575e+003 - 1.3902825e+004 - 1.4328926e+004 - 1.4670176e+004 - 1.4878802e+004 - 1.5000000e+004 - 1.4878856e+004 - 1.3326524e+004 - 1.4198448e+004 - 1.4301489e+004 - 1.4415725e+004 - 1.4378444e+004 - 1.4498946e+004 - 1.4279060e+004 - 1.3914863e+004 - 1.2796871e+004 - 1.2393224e+004 - 1.0395986e+004 - 9.8298310e+003 - 1.0563066e+004 - 1.0364096e+004 - 8.7949911e+003 - 8.2902546e+003 - 8.4147112e+003 - 8.4594761e+003 - 7.2460111e+003 - 7.4637659e+003 - 7.5295553e+003 - 8.2846868e+003 - 8.4298085e+003 - 9.1106839e+003 - 1.0452194e+004 - 1.0058147e+004 - 1.0515954e+004 - 1.0791938e+004 - 1.0878239e+004 - 1.0984556e+004 - 1.2421442e+004 - 1.2526309e+004 - 1.2109814e+004 - 1.2187975e+004 - 1.3500910e+004 - 1.3209517e+004 - 1.0486120e+004 - 1.2258812e+004 - 1.3802451e+004 - 1.4032059e+004 - 1.3377330e+004 - 1.2489842e+004 - 1.1471203e+004 - 1.0850600e+004 - 1.1298938e+004 - 1.0912139e+004 - 1.0839806e+004 - 1.0915816e+004 - 1.1447102e+004 - 1.1111231e+004 - 1.1493451e+004 - 1.0918861e+004 - 9.6301889e+003 - 7.8502798e+003 - 7.4088057e+003 - 8.2662027e+003 - 8.8503890e+003 - 8.3600433e+003 - 7.6097422e+003 - 6.9887984e+003 - 7.2244809e+003 - 5.8163315e+003 - 5.8578837e+003 - 6.8142135e+003 - 8.0361268e+003 - 1.0156091e+004 - 1.1128796e+004 - 1.2739597e+004 - 1.2832609e+004 - 1.3409133e+004 - 1.3291405e+004 - 1.3147447e+004 - 9.8023123e+003 - 1.1100207e+004 - 1.2802876e+004 - 1.2815831e+004 - 1.3073433e+004 - 1.3921522e+004 - 1.4515886e+004 - 1.4488831e+004 - 1.4415337e+004 - 1.4255083e+004 - 1.4106908e+004 - 1.4375706e+004 - 1.1472189e+004 - 1.3094918e+004 - 1.4172600e+004 - 1.4286682e+004 - 1.4293421e+004 - 1.4197785e+004 - 1.4265690e+004 - 1.4333036e+004 - 1.4060515e+004 - 1.2991948e+004 - 1.1920505e+004 - 1.1908054e+004 - 8.4139266e+003 - 9.3300443e+003 - 9.0124664e+003 - 8.3614701e+003 - 7.5923326e+003 - 7.3969297e+003 - 7.4045002e+003 - 6.9294104e+003 - 6.9782450e+003 - 6.5853435e+003 - 5.6027888e+003 - 5.5153168e+003 - 5.5430526e+003 - 4.6971653e+003 - 5.1705340e+003 - 5.5306912e+003 - 6.5798117e+003 - 7.7000285e+003 - 8.6857097e+003 - 9.2988977e+003 - 1.0649614e+004 - 1.1233872e+004 - 1.1358126e+004 - 1.0732146e+004 - 8.6238285e+003 - 9.5651283e+003 - 9.9050952e+003 - 1.0010970e+004 - 1.1646976e+004 - 1.1860142e+004 - 1.1017663e+004 - 1.0123794e+004 - 1.0294310e+004 - 1.0328485e+004 - 1.0328735e+004 - 1.0259523e+004 - 9.2296678e+003 - 1.0185365e+004 - 1.0478528e+004 - 9.5400102e+003 - 1.0836937e+004 - 1.1211469e+004 - 1.1418778e+004 - 1.0803247e+004 - 9.6094019e+003 - 9.7373969e+003 - 1.0541534e+004 - 1.0092324e+004 - 7.8145481e+003 - 8.4654384e+003 - 7.3027336e+003 - 6.9402968e+003 - 7.6689952e+003 - 8.3595990e+003 - 7.6388666e+003 - 7.5505963e+003 - 8.0662858e+003 - 7.1923007e+003 - 5.7492060e+003 - 5.4911117e+003 - 5.9155381e+003 - 4.8508696e+003 - 5.2179338e+003 - 6.6292334e+003 - 6.9216687e+003 - 7.2410945e+003 - 7.6494041e+003 - 7.5113595e+003 - 7.8754058e+003 - 7.6942773e+003 - 8.0416971e+003 - 8.0673828e+003 - 6.5282983e+003 - 6.4137554e+003 - 8.0757559e+003 - 9.7571432e+003 - 1.1137879e+004 - 1.1103022e+004 - 1.2065013e+004 - 1.2489312e+004 - 1.2418812e+004 - 1.1777563e+004 - 1.0927431e+004 - 9.8937299e+003 - 7.9713155e+003 - 6.7819735e+003 - 6.8961700e+003 - 6.2566404e+003 - 5.7496646e+003 - 5.0932616e+003 - 1.8394195e+003 - 2.9395906e+003 - 4.4161939e+003 - 4.7198550e+003 - 5.1044737e+003 - 5.1703642e+003 - 7.5698444e+003 - 7.7816595e+003 - 9.4860846e+003 - 9.2244030e+003 - 9.5443543e+003 - 1.0250938e+004 - 9.4229737e+003 - 7.5854149e+003 - 5.1281729e+003 - 8.7431927e+003 - 9.3942612e+003 - 1.0925057e+004 - 1.2494781e+004 - 1.4617048e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4863523e+004 - 1.4676031e+004 - 1.4484506e+004 - 1.4524968e+004 - 1.4244404e+004 - 1.3808439e+004 - 1.2496348e+004 - 9.4803407e+003 - 9.7736709e+003 - 1.0771415e+004 - 9.7154778e+003 - 9.4543989e+003 - 9.0623354e+003 - 7.3890869e+003 - 6.2508752e+003 - 5.0886659e+003 - 2.0613230e+003 - 0.0000000e+000 - 1.0771810e+003 - 6.7979177e+003 - 7.1838819e+003 - 6.6479566e+003 - 8.1325891e+003 - 8.1190587e+003 - 6.9412771e+003 - 5.6077968e+003 - 5.8705515e+003 - 6.8481542e+003 - 7.6103731e+003 - 8.3501293e+003 - 9.0522048e+003 - 1.0180083e+004 - 1.2517236e+004 - 1.3366998e+004 - 1.2833310e+004 - 1.4022355e+004 - 1.4176963e+004 - 1.4322174e+004 - 1.4334444e+004 - 1.4279905e+004 - 1.4111912e+004 - 1.4002084e+004 - 1.4071263e+004 - 1.0732562e+004 - 1.3120899e+004 - 1.3241310e+004 - 1.3289913e+004 - 1.3153568e+004 - 1.2607650e+004 - 1.2600683e+004 - 1.2544957e+004 - 1.2725473e+004 - 1.2201818e+004 - 1.1474098e+004 - 1.1880459e+004 - 8.9270608e+003 - 1.0210032e+004 - 1.0925674e+004 - 1.1164600e+004 - 1.0764914e+004 - 1.0979607e+004 - 1.0592600e+004 - 9.8162198e+003 - 9.7583630e+003 - 9.8964683e+003 - 8.8958247e+003 - 8.7397326e+003 - 8.0608210e+003 - 8.1619070e+003 - 7.3504930e+003 - 8.0382088e+003 - 7.7118459e+003 - 7.2674756e+003 - 6.2334113e+003 - 5.8431547e+003 - 5.8203088e+003 - 5.7455026e+003 - 5.5800821e+003 - 6.8570401e+003 - 7.8564174e+003 - 7.7161606e+003 - 8.5653244e+003 - 9.5276556e+003 - 8.6756355e+003 - 9.5906841e+003 - 1.1230017e+004 - 1.2264045e+004 - 1.2399421e+004 - 1.1865131e+004 - 1.1451658e+004 - 1.2887561e+004 - 1.0243505e+004 - 1.1847173e+004 - 1.2894840e+004 - 1.3684599e+004 - 1.4127636e+004 - 1.4210979e+004 - 1.4242876e+004 - 1.4150656e+004 - 1.3203352e+004 - 1.3759379e+004 - 1.3974988e+004 - 1.3417503e+004 - 1.3000442e+004 - 1.1845492e+004 - 1.2277074e+004 - 1.4417319e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.2076322e+004 - 1.4897000e+004 - 1.4720062e+004 - 1.4653462e+004 - 1.4466740e+004 - 1.4172617e+004 - 1.4043972e+004 - 1.2890104e+004 - 1.1557656e+004 - 9.8829956e+003 - 8.7323179e+003 - 8.5367666e+003 - 6.1702612e+003 - 7.1326469e+003 - 6.5324079e+003 - 7.4564211e+003 - 8.5370752e+003 - 9.3851424e+003 - 1.0387025e+004 - 1.1620765e+004 - 1.2110906e+004 - 1.2775187e+004 - 1.2575969e+004 - 1.2504335e+004 - 1.0302945e+004 - 1.3906172e+004 - 1.2563315e+004 - 1.2838229e+004 - 1.2882414e+004 - 1.2687524e+004 - 1.2597265e+004 - 1.1614634e+004 - 1.0970847e+004 - 1.0758314e+004 - 1.0288128e+004 - 1.0114996e+004 - 9.0361650e+003 - 6.7228534e+003 - 6.7848430e+003 - 6.8220087e+003 - 7.1732749e+003 - 7.6893060e+003 - 8.9881171e+003 - 8.9253829e+003 - 8.1827589e+003 - 7.5418547e+003 - 8.0333735e+003 - 9.6364617e+003 - 1.0127739e+004 - 1.1646628e+004 - 1.2667632e+004 - 1.3201351e+004 - 1.3769993e+004 - 1.3552437e+004 - 1.3601327e+004 - 1.3217098e+004 - 1.2713829e+004 - 1.1374502e+004 - 1.1130578e+004 - 1.1186375e+004 - 7.9668857e+003 - 7.3172486e+003 - 7.2187024e+003 - 5.9896825e+003 - 6.2685310e+003 - 4.9945978e+003 - 5.6592245e+003 - 5.9005254e+003 - 6.5792015e+003 - 6.2338170e+003 - 6.5014534e+003 - 6.3499975e+003 - 7.5139668e+003 - 9.1379989e+003 - 9.5542845e+003 - 9.2687977e+003 - 8.5863183e+003 - 9.0758520e+003 - 9.4258051e+003 - 9.6298591e+003 - 9.7212965e+003 - 9.6183538e+003 - 9.4370213e+003 - 8.6267553e+003 - 6.6781890e+003 - 7.1908882e+003 - 7.4112830e+003 - 7.4141130e+003 - 8.0993674e+003 - 6.1384773e+003 - 5.8661528e+003 - 5.5360386e+003 - 6.0425172e+003 - 6.2331135e+003 - 7.3111632e+003 - 7.3912723e+003 - 5.7106273e+003 - 5.5063467e+003 - 5.8781842e+003 - 7.1501231e+003 - 7.4451899e+003 - 7.4308416e+003 - 7.3723454e+003 - 7.8567711e+003 - 7.6228960e+003 - 7.4459709e+003 - 6.9587345e+003 - 5.8638733e+003 - 5.8263665e+003 - 4.5055989e+003 - 2.8436927e+003 - 2.5815266e+003 - 1.4737478e+003 - 5.0386013e+003 - 5.8989621e+003 - 5.3344642e+003 - 4.9421662e+003 - 4.9012793e+003 - 4.9506939e+003 - 5.2855550e+003 - 5.9438141e+003 - 6.8177451e+003 - 7.7590123e+003 - 8.4481202e+003 - 7.5585712e+003 - 8.2980809e+003 - 9.9514887e+003 - 1.2905526e+004 - 1.4312394e+004 - 1.4292012e+004 - 1.4857261e+004 - 1.5000000e+004 - 1.3009420e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4693481e+004 - 1.4643895e+004 - 1.4966272e+004 - 1.4970012e+004 - 1.4787153e+004 - 1.4694029e+004 - 1.4653443e+004 - 1.4290510e+004 - 1.4237682e+004 - 1.2919130e+004 - 1.4474984e+004 - 1.4094082e+004 - 1.4171172e+004 - 1.4284654e+004 - 1.4200419e+004 - 1.3612478e+004 - 1.2867021e+004 - 1.2074154e+004 - 1.2093240e+004 - 1.1475811e+004 - 1.0668096e+004 - 7.2439294e+003 - 8.7303146e+003 - 8.3653813e+003 - 7.6986760e+003 - 7.2604085e+003 - 7.7792567e+003 - 8.1130047e+003 - 8.2974310e+003 - 8.3866661e+003 - 8.9405112e+003 - 8.5341046e+003 - 7.9927364e+003 - 7.7718509e+003 - 6.9899210e+003 - 6.9638300e+003 - 6.7659225e+003 - 7.1093142e+003 - 7.2991563e+003 - 6.8767903e+003 - 7.6934791e+003 - 7.7821522e+003 - 7.8216543e+003 - 7.8531126e+003 - 7.8233156e+003 - 6.9522294e+003 - 6.6591518e+003 - 6.6325518e+003 - 6.6580843e+003 - 6.3555215e+003 - 6.7890150e+003 - 6.8891207e+003 - 7.0642888e+003 - 6.8320286e+003 - 6.3737752e+003 - 5.5436397e+003 - 5.1576943e+003 - 6.4914504e+003 - 5.7387781e+003 - 6.3687839e+003 - 6.4308390e+003 - 6.2734878e+003 - 6.4876725e+003 - 6.4941664e+003 - 6.1584640e+003 - 6.3091637e+003 - 5.7402281e+003 - 4.7899058e+003 - 4.8102726e+003 - 4.8939982e+003 - 4.3956205e+003 - 3.9289686e+003 - 5.5061872e+003 - 5.5470573e+003 - 5.2700423e+003 - 5.4074628e+003 - 5.0278879e+003 - 5.2348388e+003 - 4.6101131e+003 - 4.4408374e+003 - 5.0240833e+003 - 6.1152071e+003 - 6.6835540e+003 - 6.3882847e+003 - 6.6668461e+003 - 7.8394472e+003 - 8.1415702e+003 - 7.2696697e+003 - 5.9848743e+003 - 4.6684139e+003 - 2.8942944e+003 - 2.8019687e+003 - 3.3070501e+003 - 2.5910214e+003 - 1.4314987e+003 - 3.8534992e+003 - 6.2982790e+003 - 5.2357355e+003 - 5.0272604e+003 - 5.3962594e+003 - 5.0584472e+003 - 5.1690184e+003 - 4.9319409e+003 - 2.3733825e+003 - 1.8674663e+003 - 3.9131230e+003 - 5.3270261e+003 - 4.7741690e+003 - 3.5228617e+003 - 4.8033253e+003 - 6.0465044e+003 - 6.4354184e+003 - 6.5445502e+003 - 7.1351961e+003 - 6.1159122e+003 - 5.6820687e+003 - 5.6851427e+003 - 6.6394371e+003 - 6.8051028e+003 - 8.7626018e+003 - 9.8700985e+003 - 9.7525488e+003 - 1.1186091e+004 - 1.2148091e+004 - 1.1977491e+004 - 1.1786886e+004 - 1.1670471e+004 - 1.1122932e+004 - 1.0955522e+004 - 8.5505749e+003 - 1.0314628e+004 - 1.0448709e+004 - 1.0385979e+004 - 1.1192427e+004 - 1.1504139e+004 - 1.3439511e+004 - 1.4104301e+004 - 1.4216387e+004 - 1.4231559e+004 - 1.4230738e+004 - 1.4282412e+004 - 9.2069812e+003 - 1.1617117e+004 - 1.2634287e+004 - 1.3755985e+004 - 1.4252133e+004 - 1.4163865e+004 - 1.4057767e+004 - 1.4223056e+004 - 1.4281525e+004 - 1.4292509e+004 - 1.4182438e+004 - 1.4491449e+004 - 1.0433180e+004 - 1.2111644e+004 - 1.3685567e+004 - 1.3778457e+004 - 1.4193909e+004 - 1.4108127e+004 - 1.3645883e+004 - 1.2539153e+004 - 1.0983064e+004 - 9.9111817e+003 - 9.4714199e+003 - 8.9182732e+003 - 6.2315458e+003 - 7.1504760e+003 - 7.3816534e+003 - 6.3503045e+003 - 5.9743150e+003 - 6.1233743e+003 - 7.0836795e+003 - 7.6316002e+003 - 7.4750709e+003 - 7.5902643e+003 - 7.0252588e+003 - 6.9375868e+003 - 7.7537233e+003 - 9.7011463e+003 - 1.0347442e+004 - 9.8876541e+003 - 1.0082942e+004 - 1.0287851e+004 - 1.0031402e+004 - 1.0167991e+004 - 1.0413256e+004 - 1.0861076e+004 - 1.1213174e+004 - 1.1601803e+004 - 9.2470352e+003 - 1.0854291e+004 - 1.1090546e+004 - 1.2188771e+004 - 1.2154009e+004 - 1.1914423e+004 - 1.1501796e+004 - 1.1092450e+004 - 1.1602299e+004 - 1.4502074e+004 - 1.4980428e+004 - 1.4413361e+004 - 9.9192635e+003 - 1.1077996e+004 - 1.1246221e+004 - 1.4389340e+004 - 1.4406407e+004 - 1.4364214e+004 - 1.4087054e+004 - 1.3412246e+004 - 1.3100242e+004 - 1.2514518e+004 - 1.1336575e+004 - 1.0800852e+004 - 8.6264598e+003 - 8.8385719e+003 - 9.9604824e+003 - 1.0153907e+004 - 9.3941784e+003 - 1.0691830e+004 - 1.2315834e+004 - 1.3213231e+004 - 1.3502457e+004 - 1.3767249e+004 - 1.2988381e+004 - 1.2892412e+004 - 1.1108399e+004 - 1.2727293e+004 - 1.3211493e+004 - 1.2949342e+004 - 1.3082253e+004 - 1.2638907e+004 - 1.2021355e+004 - 1.1617553e+004 - 1.1970265e+004 - 1.2122193e+004 - 1.2169804e+004 - 1.2037302e+004 - 1.0492064e+004 - 1.3778985e+004 - 1.3739473e+004 - 1.1934559e+004 - 1.3949745e+004 - 1.4459238e+004 - 1.4609580e+004 - 1.4512488e+004 - 1.4463631e+004 - 1.4192982e+004 - 1.3897833e+004 - 1.4080478e+004 - 1.2234312e+004 - 1.3003037e+004 - 1.4294994e+004 - 1.4470062e+004 - 1.4361884e+004 - 1.4415235e+004 - 1.4389634e+004 - 1.4345621e+004 - 1.4262627e+004 - 1.4222277e+004 - 1.4081542e+004 - 1.3636272e+004 - 1.1659217e+004 - 1.2622429e+004 - 1.3567385e+004 - 1.4032255e+004 - 1.4015588e+004 - 1.4025452e+004 - 1.3969795e+004 - 1.4004818e+004 - 1.3707871e+004 - 1.3197849e+004 - 1.2464983e+004 - 1.1130557e+004 - 9.4842278e+003 - 1.0956735e+004 - 1.1154607e+004 - 1.1102103e+004 - 1.1272361e+004 - 1.0487036e+004 - 9.8683403e+003 - 9.4703844e+003 - 9.0549051e+003 - 8.4705467e+003 - 7.4133889e+003 - 6.7553473e+003 - 5.5238722e+003 - 5.7548244e+003 - 5.6875661e+003 - 1.6051225e+003 - 4.2026773e+002 - 8.9539686e+002 - 1.9395485e+003 - 3.6587482e+003 - 5.3701823e+003 - 5.5331971e+003 - 7.4278833e+003 - 8.5178197e+003 - 9.2523895e+003 - 9.9117717e+003 - 1.1013095e+004 - 1.2062219e+004 - 1.2852101e+004 - 1.3831542e+004 - 1.4118424e+004 - 1.4145167e+004 - 1.4321831e+004 - 1.4426432e+004 - 1.4425608e+004 - 1.4479994e+004 - 1.4010351e+004 - 1.4118594e+004 - 1.3761216e+004 - 1.1712989e+004 - 1.0712392e+004 - 9.7974893e+003 - 8.6829814e+003 - 7.5767150e+003 - 6.4081320e+003 - 5.8829048e+003 - 4.9227422e+003 - 4.6993801e+003 - 5.0885306e+003 - 4.1873780e+003 - 5.8107984e+003 - 5.9665712e+003 - 6.5201013e+003 - 8.3094094e+003 - 8.7357251e+003 - 8.1988957e+003 - 8.9073769e+003 - 9.3771058e+003 - 8.7278003e+003 - 9.2382234e+003 - 8.5998370e+003 - 9.4934231e+003 - 1.0776880e+004 - 1.1458334e+004 - 1.0406241e+004 - 1.1029115e+004 - 1.2097817e+004 - 1.3024363e+004 - 1.2949831e+004 - 1.2541418e+004 - 1.4188205e+004 - 1.4503364e+004 - 9.9500972e+003 - 1.4111632e+004 - 1.4270307e+004 - 1.4707024e+004 - 1.4800199e+004 - 1.4351261e+004 - 1.4409142e+004 - 1.4768074e+004 - 1.4378121e+004 - 1.4424706e+004 - 1.4144187e+004 - 1.3636573e+004 - 9.6444768e+003 - 9.5041747e+003 - 1.0009163e+004 - 1.0935002e+004 - 1.0667843e+004 - 1.2365242e+004 - 1.3204490e+004 - 1.3699200e+004 - 1.3683974e+004 - 1.4012554e+004 - 1.2984965e+004 - 1.3961703e+004 - 1.0137793e+004 - 1.1918972e+004 - 1.2531256e+004 - 1.2594269e+004 - 1.3464707e+004 - 1.2834511e+004 - 1.0959183e+004 - 8.8585548e+003 - 7.1860950e+003 - 7.0195908e+003 - 6.6644474e+003 - 5.8859625e+003 - 4.9916209e+003 - 5.9825393e+003 - 5.9594466e+003 - 6.7251193e+003 - 3.9156185e+003 - 5.0988736e+003 - 6.1351253e+003 - 7.2079983e+003 - 8.4281654e+003 - 8.3617024e+003 - 7.0216475e+003 - 7.0912922e+003 - 6.7809001e+003 - 6.9705054e+003 - 7.5679247e+003 - 8.4736683e+003 - 8.3794527e+003 - 8.7731583e+003 - 9.6065591e+003 - 1.1053568e+004 - 1.2049796e+004 - 1.2838092e+004 - 1.2986587e+004 - 1.3334437e+004 - 1.4062354e+004 - 1.4521259e+004 - 1.4451756e+004 - 1.4572678e+004 - 1.4510068e+004 - 1.4594316e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4800121e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4267581e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4643474e+004 - 1.4404255e+004 - 1.4067538e+004 - 9.6880369e+003 - 9.5288890e+003 - 8.1137043e+003 - 9.0937782e+003 - 8.5854274e+003 - 8.2528275e+003 - 7.5572465e+003 - 8.1722396e+003 - 9.0987208e+003 - 1.0357802e+004 - 1.1228729e+004 - 1.3052231e+004 - 9.4805002e+003 - 9.5240684e+003 - 9.8184787e+003 - 9.2381123e+003 - 8.1361840e+003 - 1.0231635e+004 - 1.1941217e+004 - 1.3170271e+004 - 1.4215765e+004 - 1.4439739e+004 - 1.4725206e+004 - 1.4709380e+004 - 1.1901456e+004 - 1.4328621e+004 - 1.4801070e+004 - 1.4569778e+004 - 1.4509730e+004 - 1.4561757e+004 - 1.4645148e+004 - 1.4526179e+004 - 1.4241119e+004 - 1.4146712e+004 - 1.4173931e+004 - 1.4113412e+004 - 1.2158668e+004 - 1.4289328e+004 - 1.4317921e+004 - 1.4016582e+004 - 1.4529492e+004 - 1.4629424e+004 - 1.4650339e+004 - 1.4576891e+004 - 1.4529512e+004 - 1.4403845e+004 - 1.4020125e+004 - 1.2875908e+004 - 9.9692489e+003 - 1.0429161e+004 - 8.6418236e+003 - 7.7577650e+003 - 7.1825943e+003 - 7.1630558e+003 - 7.6362135e+003 - 7.8691510e+003 - 7.9296263e+003 - 8.0154153e+003 - 6.7209134e+003 - 6.1843787e+003 - 5.6140957e+003 - 6.5164118e+003 - 7.7940088e+003 - 7.3504529e+003 - 7.6546009e+003 - 8.1981775e+003 - 8.7925540e+003 - 5.5695932e+003 - 5.3862940e+003 - 6.6689093e+003 - 7.5352257e+003 - 8.4738248e+003 - 7.9110218e+003 - 9.4590426e+003 - 9.0735452e+003 - 9.7768640e+003 - 1.0310611e+004 - 1.0361878e+004 - 1.0695676e+004 - 1.0986598e+004 - 1.0736037e+004 - 1.0717525e+004 - 1.0586572e+004 - 1.0824228e+004 - 9.1727177e+003 - 9.0529054e+003 - 8.8206118e+003 - 8.6730362e+003 - 1.0022275e+004 - 1.0738376e+004 - 1.1152733e+004 - 1.1201020e+004 - 1.1509478e+004 - 1.1592040e+004 - 1.1160838e+004 - 9.9301227e+003 - 6.9899958e+003 - 8.5092482e+003 - 8.2148459e+003 - 8.6370951e+003 - 8.6102654e+003 - 8.0137262e+003 - 8.0508292e+003 - 7.8329203e+003 - 7.7650416e+003 - 7.3820555e+003 - 7.7287472e+003 - 8.1888309e+003 - 8.1132039e+003 - 7.9290762e+003 - 7.7094138e+003 - 6.8262619e+003 - 5.9207827e+003 - 7.0217579e+003 - 6.6644715e+003 - 6.0004366e+003 - 6.2932051e+003 - 6.9010053e+003 - 6.5315695e+003 - 6.3663386e+003 - 5.8259128e+003 - 5.1309686e+003 - 4.8077354e+003 - 3.1566475e+003 - 2.9443549e+003 - 1.9698490e+003 - 0.0000000e+000 - 3.0803787e+003 - 5.1792297e+003 - 6.0360971e+003 - 7.6424085e+003 - 8.0098398e+003 - 7.2371363e+003 - 7.8566266e+003 - 9.0892714e+003 - 8.4858187e+003 - 1.0515779e+004 - 1.1632925e+004 - 1.2363693e+004 - 1.3896894e+004 - 1.4284029e+004 - 1.4268253e+004 - 1.3508070e+004 - 1.2725922e+004 - 1.0430121e+004 - 1.2075224e+004 - 1.3326251e+004 - 1.3708150e+004 - 1.3606815e+004 - 1.3333076e+004 - 1.2320067e+004 - 1.1612730e+004 - 1.0551017e+004 - 1.0272285e+004 - 1.0265439e+004 - 1.0550800e+004 - 8.0674553e+003 - 8.9121610e+003 - 9.8175919e+003 - 8.0322053e+003 - 8.3964183e+003 - 8.8729891e+003 - 9.4233535e+003 - 1.0556272e+004 - 1.0844777e+004 - 1.1518843e+004 - 1.1554839e+004 - 1.1634080e+004 - 9.1246987e+003 - 9.6649536e+003 - 8.8475278e+003 - 6.4252507e+003 - 2.5030974e+003 - 0.0000000e+000 - 4.8223132e+003 - 5.9091647e+003 - 6.3095408e+003 - 9.2739423e+003 - 1.2091638e+004 - 1.4002867e+004 - 9.5414956e+003 - 1.2942629e+004 - 1.3189751e+004 - 1.3649273e+004 - 1.4366966e+004 - 1.4636004e+004 - 1.4770926e+004 - 1.4842580e+004 - 1.4849564e+004 - 1.4938460e+004 - 1.5000000e+004 - 1.4964678e+004 - 1.3861763e+004 - 1.4873335e+004 - 1.4858591e+004 - 1.4641649e+004 - 1.4466881e+004 - 1.4355976e+004 - 1.4231878e+004 - 1.4061419e+004 - 1.3499465e+004 - 1.2970548e+004 - 1.2473375e+004 - 1.2198572e+004 - 9.4989568e+003 - 9.9775846e+003 - 9.7012179e+003 - 9.4648198e+003 - 9.9017691e+003 - 9.7581187e+003 - 9.6560768e+003 - 9.4750273e+003 - 1.0040307e+004 - 1.0126242e+004 - 1.0159096e+004 - 1.0035799e+004 - 6.0853452e+003 - 7.7743236e+003 - 7.6168651e+003 - 7.5760682e+003 - 6.2833726e+003 - 5.1195741e+003 - 4.3444223e+003 - 3.4019572e+003 - 1.4048059e+003 - 2.6381346e+003 - 3.2159074e+003 - 4.9083576e+003 - 4.8543078e+003 - 5.3196828e+003 - 6.0733839e+003 - 5.8981501e+003 - 7.3894451e+003 - 8.1131159e+003 - 7.7056799e+003 - 7.3681339e+003 - 7.5035738e+003 - 7.2107157e+003 - 7.2424915e+003 - 7.8543538e+003 - 7.9705821e+003 - 8.7023244e+003 - 1.0107390e+004 - 9.6761206e+003 - 9.4715963e+003 - 9.4045555e+003 - 9.6628513e+003 - 8.7524292e+003 - 1.1176901e+004 - 1.4236794e+004 - 1.4736664e+004 - 1.5000000e+004 - 1.1465203e+004 - 1.4946616e+004 - 1.4995395e+004 - 1.4665277e+004 - 1.4807208e+004 - 1.4781769e+004 - 1.4614677e+004 - 1.4655981e+004 - 1.4658028e+004 - 1.4667642e+004 - 1.4621946e+004 - 1.4344220e+004 - 1.0935302e+004 - 1.3621966e+004 - 1.4234808e+004 - 1.4165539e+004 - 1.4101912e+004 - 1.4054979e+004 - 1.3464645e+004 - 1.2828780e+004 - 1.2101428e+004 - 1.1485596e+004 - 1.1123652e+004 - 1.0902801e+004 - 9.2434772e+003 - 8.6906192e+003 - 8.6448217e+003 - 8.7698894e+003 - 9.9557753e+003 - 1.0710070e+004 - 1.1141701e+004 - 1.1433754e+004 - 1.1059784e+004 - 1.0268694e+004 - 1.0120759e+004 - 9.8481759e+003 - 7.8345036e+003 - 8.4266113e+003 - 8.4287481e+003 - 7.1434853e+003 - 6.9707444e+003 - 6.3603010e+003 - 5.7699056e+003 - 5.5568299e+003 - 5.7163069e+003 - 5.9495520e+003 - 5.6191076e+003 - 4.5413240e+003 - 2.5676679e+003 - 2.0135028e+003 - 2.4955947e+003 - 2.4802356e+003 - 5.4545188e+003 - 6.0614815e+003 - 6.9120892e+003 - 8.1157475e+003 - 8.8766452e+003 - 9.6636250e+003 - 9.9976361e+003 - 9.9915715e+003 - 9.3086075e+003 - 1.1065778e+004 - 1.2298101e+004 - 1.2846225e+004 - 1.3351264e+004 - 1.3290772e+004 - 1.2793474e+004 - 1.2407574e+004 - 1.3033798e+004 - 1.2946688e+004 - 1.3004449e+004 - 1.3214582e+004 - 1.3188911e+004 - 1.3880751e+004 - 1.3344874e+004 - 1.2264422e+004 - 1.2827873e+004 - 1.2784096e+004 - 1.3106839e+004 - 1.2594570e+004 - 1.2451453e+004 - 1.2047485e+004 - 1.1567536e+004 - 1.1569071e+004 - 9.2547492e+003 - 1.0441054e+004 - 1.2046080e+004 - 1.2288846e+004 - 1.1666194e+004 - 1.1710763e+004 - 1.1771927e+004 - 1.1442414e+004 - 1.0872409e+004 - 1.0644470e+004 - 1.0148674e+004 - 9.8374366e+003 - 6.8668282e+003 - 7.5542725e+003 - 6.8828798e+003 - 6.8931399e+003 - 7.8866138e+003 - 9.8719819e+003 - 1.1082496e+004 - 1.1129404e+004 - 1.0736206e+004 - 1.0061079e+004 - 9.8078823e+003 - 8.7029149e+003 - 7.3990094e+003 - 9.2700761e+003 - 8.8012180e+003 - 8.4360514e+003 - 7.7027747e+003 - 7.1016189e+003 - 6.6235625e+003 - 6.8712639e+003 - 8.2716918e+003 - 8.8870305e+003 - 1.0195663e+004 - 9.5568632e+003 - 8.8355765e+003 - 9.9249718e+003 - 1.0348319e+004 - 9.3396604e+003 - 1.3080695e+004 - 1.4094140e+004 - 1.1229962e+004 - 1.4019446e+004 - 5.7537004e+003 - 1.4168118e+004 - 1.2409819e+004 - 1.1198443e+004 - 7.5510963e+003 - 1.0476067e+004 - 1.2449274e+004 - 1.4459436e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4779109e+004 - 1.5000000e+004 - 1.4973127e+004 - 1.4396088e+004 - 1.3932624e+004 - 1.1404258e+004 - 1.2490379e+004 - 1.3124220e+004 - 1.3285245e+004 - 1.2154323e+004 - 1.1719791e+004 - 1.1405729e+004 - 1.0803569e+004 - 1.0979757e+004 - 1.0842750e+004 - 9.4148279e+003 - 8.9447294e+003 - 1.0474580e+004 - 1.0654170e+004 - 1.2073502e+004 - 9.8241618e+003 - 1.0735108e+004 - 1.1393666e+004 - 8.0101436e+003 - 6.4510267e+003 - 6.8135170e+003 - 6.8185015e+003 - 6.9813701e+003 - 6.4062007e+003 - 6.4544616e+003 - 5.7861479e+003 - 6.2556668e+003 - 7.1665808e+003 - 8.5007547e+003 - 1.0166471e+004 - 1.0408297e+004 - 9.6927255e+003 - 8.6889407e+003 - 8.7834211e+003 - 1.0125432e+004 - 9.9263118e+003 - 8.1671862e+003 - 7.2932426e+003 - 8.4357286e+003 - 9.0819169e+003 - 1.0383249e+004 - 1.1034219e+004 - 1.1524295e+004 - 1.2521236e+004 - 1.1814155e+004 - 1.2221289e+004 - 1.2604720e+004 - 1.2927573e+004 - 8.2841820e+003 - 1.1474974e+004 - 1.0852282e+004 - 1.0843922e+004 - 1.0291204e+004 - 9.5123974e+003 - 8.4371387e+003 - 8.0675826e+003 - 7.4476846e+003 - 7.5308305e+003 - 6.7131646e+003 - 6.4980960e+003 - 6.8570256e+003 - 7.3573735e+003 - 7.0123758e+003 - 7.0713789e+003 - 6.7994975e+003 - 6.3093348e+003 - 7.1318035e+003 - 8.2868390e+003 - 5.4611297e+003 - 4.7964524e+003 - 5.2555576e+003 - 6.9087118e+003 - 8.2341951e+003 - 7.7071670e+003 - 7.6673284e+003 - 7.4659879e+003 - 7.5914242e+003 - 7.3816621e+003 - 7.2481117e+003 - 6.8707318e+003 - 6.4761274e+003 - 6.0537112e+003 - 6.5645932e+003 - 7.8205565e+003 - 7.9643481e+003 - 8.4133415e+003 - 1.0531654e+004 - 9.1855043e+003 - 8.5728379e+003 - 8.8171523e+003 - 8.7963033e+003 - 9.1893302e+003 - 9.2374726e+003 - 9.7593068e+003 - 9.8509366e+003 - 1.0739447e+004 - 9.7097732e+003 - 9.7198021e+003 - 1.0693049e+004 - 1.1383915e+004 - 1.2439196e+004 - 1.3533839e+004 - 1.4181894e+004 - 1.4305717e+004 - 1.4229583e+004 - 1.4089279e+004 - 1.4134168e+004 - 1.4281123e+004 - 1.3076306e+004 - 1.4187114e+004 - 1.4187817e+004 - 1.4176408e+004 - 1.4135421e+004 - 1.4090997e+004 - 1.3502228e+004 - 1.3233510e+004 - 1.2525716e+004 - 1.1769479e+004 - 1.2250798e+004 - 1.1958145e+004 - 1.0973872e+004 - 1.1881686e+004 - 1.1473418e+004 - 1.0833930e+004 - 1.1196950e+004 - 1.1900890e+004 - 1.2338879e+004 - 1.3326834e+004 - 1.3645038e+004 - 1.3821350e+004 - 1.3940791e+004 - 1.4140934e+004 - 9.9930929e+003 - 1.4077074e+004 - 1.4497044e+004 - 1.4753759e+004 - 1.4677194e+004 - 1.4442834e+004 - 1.3938667e+004 - 1.2907764e+004 - 1.2380220e+004 - 1.2847234e+004 - 1.3579206e+004 - 1.1359191e+004 - 1.1802584e+004 - 1.3090014e+004 - 1.4135285e+004 - 1.3535498e+004 - 1.3254817e+004 - 1.3452397e+004 - 1.2904570e+004 - 1.2649231e+004 - 1.2312797e+004 - 1.1480006e+004 - 1.0671593e+004 - 1.0519124e+004 - 9.3888725e+003 - 7.9363198e+003 - 6.8886571e+003 - 7.8537176e+003 - 7.6754894e+003 - 7.8931987e+003 - 8.4899389e+003 - 7.8401602e+003 - 7.8560037e+003 - 8.9715145e+003 - 9.9519428e+003 - 1.0580305e+004 - 6.4441669e+003 - 7.7639910e+003 - 8.6309266e+003 - 7.4221209e+003 - 7.2698584e+003 - 8.1423531e+003 - 8.1718451e+003 - 8.5642397e+003 - 8.2973969e+003 - 7.8242125e+003 - 6.5184638e+003 - 5.2189921e+003 - 6.4355588e+003 - 7.0334125e+003 - 7.0321991e+003 - 4.4188598e+003 - 3.2297928e+003 - 3.5285907e+003 - 4.6457330e+003 - 5.5306687e+003 - 2.5399096e+003 - 7.1858130e+003 - 9.1080593e+003 - 7.3359031e+003 - 8.1763323e+003 - 1.0732346e+004 - 8.9290812e+003 - 7.6960552e+003 - 7.0271170e+003 - 6.8914436e+003 - 8.0677804e+003 - 8.7445213e+003 - 9.7155992e+003 - 1.0737119e+004 - 9.8512669e+003 - 9.0484760e+003 - 6.0517899e+003 - 6.8466248e+003 - 8.1888250e+003 - 8.1071709e+003 - 8.0731130e+003 - 8.0280502e+003 - 1.0483030e+004 - 1.3518132e+004 - 1.1172404e+004 - 1.0316610e+004 - 8.0378472e+003 - 8.7738759e+003 - 5.8568716e+003 - 9.0746575e+003 - 9.0963285e+003 - 8.0703178e+003 - 9.9411019e+003 - 1.1217857e+004 - 1.0760294e+004 - 9.7667395e+003 - 8.4648519e+003 - 8.2984604e+003 - 7.7178833e+003 - 7.7580898e+003 - 7.4717085e+003 - 6.0676257e+003 - 3.4951801e+003 - 0.0000000e+000 - 2.3380512e+003 - 3.9846646e+003 - 4.0235272e+003 - 5.2688290e+003 - 4.4797040e+003 - 2.8788073e+003 - 4.7908594e+003 - 5.9793720e+003 - 5.2025107e+003 - 7.7421494e+003 - 7.8324247e+003 - 9.7963825e+003 - 1.1518941e+004 - 1.2570508e+004 - 1.2867317e+004 - 1.3821599e+004 - 1.3857268e+004 - 1.3930694e+004 - 1.3478814e+004 - 1.3730738e+004 - 9.3564515e+003 - 9.4812371e+003 - 9.0548790e+003 - 1.0403499e+004 - 9.0645645e+003 - 7.2126478e+003 - 4.7089765e+003 - 4.7891849e+003 - 6.6100268e+003 - 7.3447948e+003 - 6.4376428e+003 - 7.6185749e+003 - 2.1110874e+002 - 1.1373506e+003 - 4.3757936e+003 - 4.9497412e+003 - 6.2684726e+003 - 6.0406612e+003 - 5.3421910e+003 - 5.9610803e+003 - 7.9200691e+003 - 1.1499305e+004 - 1.4576091e+004 - 1.5000000e+004 - 1.4402641e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4909002e+004 - 1.4467209e+004 - 1.4716336e+004 - 1.4679317e+004 - 1.4650656e+004 - 1.4609993e+004 - 1.4795064e+004 - 1.4828855e+004 - 1.4605051e+004 - 1.4540515e+004 - 1.4201620e+004 - 1.3258656e+004 - 1.2017414e+004 - 1.1291449e+004 - 1.1914902e+004 - 1.0340472e+004 - 9.8174494e+003 - 9.5299013e+003 - 9.4850546e+003 - 9.2803055e+003 - 9.0472652e+003 - 8.9645641e+003 - 8.1458484e+003 - 7.0661672e+003 - 4.2546276e+003 - 1.4108127e+003 - 5.8335512e+002 - 1.0146889e+003 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 7.4131086e+002 - 1.6022106e+003 - 2.0770799e+003 - 2.1974531e+003 - 4.1840292e+003 - 5.6402774e+003 - 4.5223642e+003 - 5.5014722e+003 - 6.9221280e+003 - 7.7283190e+003 - 7.7721459e+003 - 8.0817321e+003 - 7.7778470e+003 - 7.2838175e+003 - 7.7189904e+003 - 8.1249837e+003 - 7.8472547e+003 - 7.7363691e+003 - 6.2543381e+003 - 7.7466277e+003 - 7.2656586e+003 - 8.2225310e+003 - 8.8950049e+003 - 9.3533840e+003 - 9.7748716e+003 - 1.0809136e+004 - 1.1618329e+004 - 1.1970662e+004 - 1.1314808e+004 - 1.0556664e+004 - 6.6345732e+003 - 6.6463632e+003 - 6.5032022e+003 - 7.2746443e+003 - 7.4977044e+003 - 8.1142193e+003 - 7.8889939e+003 - 7.1863198e+003 - 6.4160775e+003 - 6.4400400e+003 - 7.0156107e+003 - 7.3654879e+003 - 8.8244128e+003 - 7.8582791e+003 - 8.6853513e+003 - 8.8249663e+003 - 1.0370174e+004 - 1.0941245e+004 - 1.1922483e+004 - 1.1623202e+004 - 1.1243085e+004 - 1.1133588e+004 - 1.1053421e+004 - 1.1290972e+004 - 9.2377126e+003 - 8.8026095e+003 - 8.9373533e+003 - 9.1757044e+003 - 8.5725449e+003 - 9.7099979e+003 - 9.7403763e+003 - 9.2271907e+003 - 9.1182220e+003 - 9.0043299e+003 - 8.3937215e+003 - 8.2742474e+003 - 9.6549021e+003 - 7.9081203e+003 - 8.2856014e+003 - 7.8383385e+003 - 9.5395080e+003 - 1.1176383e+004 - 1.2221771e+004 - 1.2785948e+004 - 1.2887618e+004 - 1.2716079e+004 - 1.2184075e+004 - 1.1533334e+004 - 9.8282771e+003 - 1.1269068e+004 - 1.0978624e+004 - 1.0794367e+004 - 1.0556955e+004 - 1.0769979e+004 - 1.0828544e+004 - 1.0209513e+004 - 9.5137719e+003 - 9.0121265e+003 - 8.1567994e+003 - 8.0307162e+003 - 1.0168467e+004 - 1.0131980e+004 - 1.1256369e+004 - 9.8236835e+003 - 1.1013015e+004 - 1.2507868e+004 - 1.4021151e+004 - 1.4280011e+004 - 1.4407910e+004 - 1.4445569e+004 - 1.4308222e+004 - 1.4420381e+004 - 1.1232421e+004 - 1.3378237e+004 - 1.3411923e+004 - 1.3012345e+004 - 1.2492584e+004 - 1.2298654e+004 - 1.2386837e+004 - 1.2179670e+004 - 1.1168429e+004 - 1.0902152e+004 - 1.1486993e+004 - 1.2063918e+004 - 1.1177430e+004 - 1.0963057e+004 - 1.1435959e+004 - 9.6789524e+003 - 1.1476199e+004 - 1.1546764e+004 - 1.0733248e+004 - 1.0504587e+004 - 1.0406529e+004 - 1.0638678e+004 - 1.0868653e+004 - 1.0263272e+004 - 8.6091671e+003 - 1.1066959e+004 - 1.0953674e+004 - 9.4429814e+003 - 8.4179404e+003 - 8.2217248e+003 - 8.3601949e+003 - 9.0282996e+003 - 9.1982073e+003 - 8.6569087e+003 - 8.6585671e+003 - 7.8009205e+003 - 6.4178891e+003 - 6.7415206e+003 - 5.8559976e+003 - 5.9117999e+003 - 3.9722836e+003 - 6.0296701e+003 - 7.4618641e+003 - 8.8313588e+003 - 9.9238990e+003 - 1.0898124e+004 - 1.1418445e+004 - 1.0722496e+004 - 8.7333012e+003 - 1.1216738e+004 - 9.9782002e+003 - 9.7126705e+003 - 9.5776298e+003 - 1.0067521e+004 - 1.0404540e+004 - 1.0176616e+004 - 9.6210804e+003 - 9.4973669e+003 - 8.6921398e+003 - 7.4706247e+003 - 6.1252518e+003 - 6.1409395e+003 - 6.5952187e+003 - 7.3020032e+003 - 7.2527959e+003 - 7.3933468e+003 - 7.5749785e+003 - 6.5162795e+003 - 6.4908732e+003 - 7.0001880e+003 - 6.8496104e+003 - 6.0681274e+003 - 5.3155844e+003 - 4.0722825e+003 - 5.3203594e+003 - 5.8607589e+003 - 6.6859924e+003 - 8.0999583e+003 - 8.9846017e+003 - 8.7020661e+003 - 8.8184534e+003 - 8.9254723e+003 - 8.7249484e+003 - 9.5903230e+003 - 8.8941581e+003 - 1.0232725e+004 - 1.0706478e+004 - 1.0521954e+004 - 1.1170978e+004 - 1.2512466e+004 - 1.2289820e+004 - 1.3044446e+004 - 1.4138890e+004 - 1.4536861e+004 - 1.4415462e+004 - 1.4839216e+004 - 1.3999461e+004 - 1.4718569e+004 - 1.4585079e+004 - 1.4362790e+004 - 1.4226527e+004 - 1.3863471e+004 - 1.3005983e+004 - 1.2267237e+004 - 1.1705881e+004 - 1.0603914e+004 - 9.5383822e+003 - 9.0802436e+003 - 8.0364191e+003 - 7.3809675e+003 - 6.7710724e+003 - 6.5323381e+003 - 7.2133261e+003 - 6.9420851e+003 - 6.7515115e+003 - 7.2353404e+003 - 7.2918229e+003 - 7.4334467e+003 - 7.7776140e+003 - 7.6357015e+003 - 5.4136383e+003 - 6.9567967e+003 - 7.2914571e+003 - 8.3746646e+003 - 8.3797689e+003 - 8.0609962e+003 - 7.9441481e+003 - 8.4004479e+003 - 8.5951888e+003 - 8.6105226e+003 - 8.5546344e+003 - 8.3593580e+003 - 8.1901362e+003 - 9.0765431e+003 - 9.1195072e+003 - 1.0019727e+004 - 1.2812407e+004 - 1.4195852e+004 - 1.3134091e+004 - 1.4120804e+004 - 1.4408752e+004 - 1.4875879e+004 - 1.4845344e+004 - 1.5000000e+004 - 1.4395046e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4638207e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4653973e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4681622e+004 - 1.4266024e+004 - 1.4815745e+004 - 1.4983997e+004 - 1.4154553e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4973464e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.3780294e+004 - 9.7010363e+003 - 8.1917712e+003 - 1.1310936e+004 - 9.8965791e+003 - 1.3507641e+004 - 1.3554012e+004 - 1.3765462e+004 - 1.4101219e+004 - 1.4272395e+004 - 1.4416189e+004 - 1.4688652e+004 - 1.4826086e+004 - 1.4670777e+004 - 1.4402169e+004 - 1.2496872e+004 - 1.2241314e+004 - 1.2404736e+004 - 1.3741628e+004 - 1.3500178e+004 - 1.3979235e+004 - 1.4143004e+004 - 1.4192295e+004 - 1.4321612e+004 - 1.4128956e+004 - 1.3829451e+004 - 1.3514849e+004 - 9.9970621e+003 - 1.1707246e+004 - 9.8970467e+003 - 8.9752415e+003 - 7.1486200e+003 - 6.2692908e+003 - 6.0784196e+003 - 5.9619368e+003 - 5.9730174e+003 - 6.2039048e+003 - 5.9723830e+003 - 3.8351775e+003 - 1.6122503e+003 - 3.0080328e+003 - 3.1533220e+003 - 5.8748542e+003 - 7.6174181e+003 - 8.7155469e+003 - 1.0193757e+004 - 1.1121322e+004 - 1.1171004e+004 - 1.1535169e+004 - 1.2083812e+004 - 1.1691971e+004 - 1.0325971e+004 - 1.1501575e+004 - 1.1621857e+004 - 1.1512075e+004 - 1.1001734e+004 - 1.0224501e+004 - 9.2666648e+003 - 8.6597987e+003 - 9.2477156e+003 - 1.0439182e+004 - 1.0597366e+004 - 1.1124649e+004 - 9.4253225e+003 - 1.0670077e+004 - 1.0919176e+004 - 1.0402610e+004 - 1.0135706e+004 - 9.6732621e+003 - 9.7695968e+003 - 9.6926672e+003 - 9.6194394e+003 - 1.0365399e+004 - 9.2423537e+003 - 8.0166389e+003 - 6.1149169e+003 - 7.0112661e+003 - 8.9560465e+003 - 8.6875461e+003 - 7.7678205e+003 - 7.1471131e+003 - 6.3106523e+003 - 7.3594311e+003 - 7.1696882e+003 - 7.3800598e+003 - 7.5454383e+003 - 8.0115755e+003 - 5.7496217e+003 - 7.2951002e+003 - 8.0890098e+003 - 9.1902366e+003 - 9.6392392e+003 - 8.7087991e+003 - 9.1081456e+003 - 9.6922402e+003 - 9.7020514e+003 - 9.3872463e+003 - 9.4720711e+003 - 9.7554888e+003 - 8.4477821e+003 - 9.4027607e+003 - 1.0538707e+004 - 1.0151953e+004 - 1.0863096e+004 - 1.1911000e+004 - 1.2228654e+004 - 1.2114524e+004 - 1.2299824e+004 - 1.1786188e+004 - 1.1336688e+004 - 1.1215392e+004 - 1.1269589e+004 - 1.2427636e+004 - 1.4272886e+004 - 1.4160130e+004 - 1.4095589e+004 - 1.3735754e+004 - 1.2900201e+004 - 1.1633518e+004 - 1.0698299e+004 - 9.8901335e+003 - 8.4908450e+003 - 7.2511629e+003 - 7.4197300e+003 - 7.7631189e+003 - 6.5190722e+003 - 6.4703384e+003 - 7.0509953e+003 - 6.9060394e+003 - 6.3509707e+003 - 7.4017998e+003 - 7.7841830e+003 - 8.4215456e+003 - 9.1903027e+003 - 1.0378403e+004 - 7.3133303e+003 - 1.0189031e+004 - 1.0539813e+004 - 1.2521716e+004 - 1.3380934e+004 - 1.4167081e+004 - 1.4398173e+004 - 1.4752845e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4121779e+004 - 1.4646284e+004 - 1.4910063e+004 - 1.4669098e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4811656e+004 - 1.4745843e+004 - 1.4669903e+004 - 1.4548940e+004 - 1.3769466e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4974665e+004 - 1.4617172e+004 - 1.4746468e+004 - 1.4771972e+004 - 1.4777292e+004 - 1.4855458e+004 - 1.4925520e+004 - 1.4889594e+004 - 1.4512723e+004 - 1.1195171e+004 - 1.3210868e+004 - 1.2472851e+004 - 1.3127953e+004 - 1.3010754e+004 - 1.4069465e+004 - 1.4621238e+004 - 1.4998415e+004 - 1.4976641e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4720855e+004 - 1.4529343e+004 - 1.4285345e+004 - 1.4588616e+004 - 1.4991823e+004 - 1.4827456e+004 - 1.4877676e+004 - 1.4956095e+004 - 1.5000000e+004 - 1.4784350e+004 - 1.4536478e+004 - 1.4382463e+004 - 1.3781386e+004 - 1.1777683e+004 - 1.1233448e+004 - 1.0268559e+004 - 8.9035038e+003 - 8.2587982e+003 - 7.6550959e+003 - 7.9435538e+003 - 9.1523550e+003 - 9.0167633e+003 - 6.3455941e+003 - 3.4077947e+003 - 3.6531293e+003 - 5.6488491e+003 - 5.4296927e+003 - 5.0411011e+003 - 6.3584433e+003 - 6.5824564e+003 - 7.9359933e+003 - 7.6170518e+003 - 6.8850285e+003 - 6.8817227e+003 - 6.6294968e+003 - 6.6850377e+003 - 7.1037350e+003 - 7.1479244e+003 - 8.5531852e+003 - 1.0199440e+004 - 9.6071085e+003 - 9.6609307e+003 - 8.7744688e+003 - 9.1489221e+003 - 1.0130569e+004 - 9.4514436e+003 - 9.2088144e+003 - 8.8188430e+003 - 8.1513160e+003 - 6.1715184e+003 - 6.3983365e+003 - 7.8589017e+003 - 8.6600640e+003 - 8.9167077e+003 - 8.9093442e+003 - 9.3809166e+003 - 9.4957980e+003 - 9.1720008e+003 - 9.0786986e+003 - 8.0819689e+003 - 6.7059840e+003 - 6.2571183e+003 - 7.0325435e+003 - 7.1425390e+003 - 7.1184741e+003 - 7.4887276e+003 - 6.7769911e+003 - 6.4787666e+003 - 6.8245185e+003 - 7.4141759e+003 - 7.5973919e+003 - 7.1652653e+003 - 7.8404290e+003 - 5.7802248e+003 - 6.5005778e+003 - 7.5021835e+003 - 8.3194442e+003 - 9.1070989e+003 - 9.4338715e+003 - 8.5041363e+003 - 9.2970958e+003 - 1.0188705e+004 - 1.2417032e+004 - 1.2459151e+004 - 1.2942270e+004 - 9.1233726e+003 - 1.1727934e+004 - 1.3984109e+004 - 1.2889472e+004 - 1.3244862e+004 - 1.4308968e+004 - 1.4731769e+004 - 1.4746503e+004 - 1.4656047e+004 - 1.4303445e+004 - 1.3865946e+004 - 1.2714763e+004 - 1.2000960e+004 - 1.3560890e+004 - 1.2995361e+004 - 1.3422039e+004 - 1.4535890e+004 - 1.4777465e+004 - 1.4967070e+004 - 1.4879856e+004 - 1.4910732e+004 - 1.4782258e+004 - 1.4695056e+004 - 1.4687357e+004 - 1.4121511e+004 - 1.4759418e+004 - 1.5000000e+004 - 1.4965401e+004 - 1.4758242e+004 - 1.4581716e+004 - 1.4444208e+004 - 1.4376421e+004 - 1.4335061e+004 - 1.4350635e+004 - 1.4364710e+004 - 1.4347222e+004 - 1.2114426e+004 - 1.3872644e+004 - 1.4874553e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4979334e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4783654e+004 - 1.4211571e+004 - 1.4716196e+004 - 1.4786293e+004 - 1.4835413e+004 - 1.4827877e+004 - 1.4597823e+004 - 1.4238281e+004 - 1.3727348e+004 - 1.2957858e+004 - 1.2502349e+004 - 1.2132159e+004 - 1.1644598e+004 - 1.0970398e+004 - 1.1668630e+004 - 1.1020949e+004 - 1.1402423e+004 - 1.0558907e+004 - 1.0490039e+004 - 1.0706292e+004 - 1.0850644e+004 - 1.0799384e+004 - 1.0126717e+004 - 9.3871949e+003 - 8.4693499e+003 - 7.0085363e+003 - 7.2415602e+003 - 7.2292194e+003 - 7.8275692e+003 - 6.4402378e+003 - 5.9890921e+003 - 5.9320782e+003 - 6.2308728e+003 - 6.8136383e+003 - 9.3276627e+003 - 1.0341470e+004 - 1.2197048e+004 - 6.9205376e+003 - 9.9778044e+003 - 1.1350595e+004 - 1.3609267e+004 - 1.4041094e+004 - 1.4192064e+004 - 1.4167130e+004 - 1.4176939e+004 - 1.4088837e+004 - 1.3888439e+004 - 1.2814708e+004 - 1.1596045e+004 - 9.2511066e+003 - 9.0477129e+003 - 8.4430163e+003 - 7.8586893e+003 - 7.8376359e+003 - 8.0917059e+003 - 7.4358508e+003 - 7.0982087e+003 - 6.3395532e+003 - 6.3941027e+003 - 6.4247922e+003 - 6.3935672e+003 - 6.1854927e+003 - 6.8903284e+003 - 6.3522680e+003 - 7.4846730e+003 - 8.1266749e+003 - 8.1469858e+003 - 8.3237821e+003 - 7.7033255e+003 - 8.5741890e+003 - 7.6528532e+003 - 7.5833375e+003 - 7.7325730e+003 - 8.3871074e+003 - 9.9255606e+003 - 9.5239558e+003 - 8.9170313e+003 - 8.6002920e+003 - 8.7554196e+003 - 9.7421154e+003 - 1.1203120e+004 - 1.1292763e+004 - 1.0542719e+004 - 1.0256276e+004 - 9.8525464e+003 - 1.0033602e+004 - 1.0860940e+004 - 9.0378211e+003 - 8.8191962e+003 - 8.8037314e+003 - 8.6550895e+003 - 9.4754528e+003 - 9.5530391e+003 - 9.4883842e+003 - 9.4697984e+003 - 9.3662235e+003 - 9.0572360e+003 - 7.4380150e+003 - 8.2599767e+003 - 7.8665648e+003 - 7.4989007e+003 - 7.3549964e+003 - 6.5577604e+003 - 7.2480244e+003 - 7.8794547e+003 - 7.0517735e+003 - 7.6100286e+003 - 8.0429919e+003 - 7.9515827e+003 - 5.8749805e+003 - 5.7532693e+003 - 5.5580964e+003 - 4.5027769e+003 - 2.1999071e+003 - 4.4327089e+003 - 5.1465416e+003 - 5.2304857e+003 - 5.3689367e+003 - 5.8632625e+003 - 6.3480527e+003 - 6.5561440e+003 - 4.5541650e+003 - 6.1566234e+003 - 8.2088203e+003 - 9.0900066e+003 - 8.1685436e+003 - 8.5454832e+003 - 7.8245753e+003 - 8.3920945e+003 - 8.6716270e+003 - 8.1360994e+003 - 7.4933140e+003 - 8.0284841e+003 - 1.0812000e+004 - 9.3677352e+003 - 8.6221530e+003 - 9.4356538e+003 - 1.0018119e+004 - 1.1037641e+004 - 1.1755321e+004 - 1.1791536e+004 - 1.1849172e+004 - 1.2442525e+004 - 1.0985696e+004 - 9.6724684e+003 - 7.9805620e+003 - 8.5344147e+003 - 9.2306408e+003 - 1.0019384e+004 - 9.8203235e+003 - 9.8452106e+003 - 9.6707453e+003 - 9.1928141e+003 - 8.5436622e+003 - 8.6397967e+003 - 8.4160791e+003 - 6.2885912e+003 - 8.5566525e+003 - 8.7598886e+003 - 9.2746980e+003 - 8.5801115e+003 - 9.0054750e+003 - 1.0192231e+004 - 1.1036706e+004 - 1.0721682e+004 - 1.1205569e+004 - 1.1174279e+004 - 1.0908404e+004 - 1.0499884e+004 - 9.3894396e+003 - 1.0080836e+004 - 8.6986021e+003 - 7.1700107e+003 - 6.2546192e+003 - 5.4877778e+003 - 5.7101842e+003 - 5.8420469e+003 - 6.5916981e+003 - 7.1928085e+003 - 7.7545833e+003 - 7.8107923e+003 - 6.1881292e+003 - 6.2236700e+003 - 6.9613381e+003 - 9.1102774e+003 - 1.0215285e+004 - 1.1015769e+004 - 1.1340707e+004 - 1.0581202e+004 - 1.0141809e+004 - 1.0292459e+004 - 9.9494756e+003 - 9.1300539e+003 - 6.7316930e+003 - 6.7469802e+003 - 6.3323985e+003 - 7.0103758e+003 - 6.8523683e+003 - 6.8727941e+003 - 7.2459330e+003 - 6.7233994e+003 - 6.1920621e+003 - 6.1866381e+003 - 6.3122711e+003 - 6.7220368e+003 - 3.2082342e+003 - 2.2880789e+003 - 2.6630021e+003 - 0.0000000e+000 - 1.5670377e+003 - 5.8932968e+003 - 7.3475465e+003 - 7.6742968e+003 - 8.1732699e+003 - 8.4452682e+003 - 8.5323467e+003 - 8.4917884e+003 - 7.1823961e+003 - 7.5608578e+003 - 8.7644028e+003 - 9.6566351e+003 - 9.1752699e+003 - 9.2910945e+003 - 8.7737259e+003 - 8.6738995e+003 - 8.3935537e+003 - 8.1964652e+003 - 8.7631510e+003 - 8.3801290e+003 - 9.4438511e+003 - 8.0818650e+003 - 6.6076349e+003 - 8.2119992e+003 - 9.5355494e+003 - 9.7703666e+003 - 9.6181619e+003 - 9.9868457e+003 - 9.7474863e+003 - 9.2769174e+003 - 9.4036885e+003 - 9.6872708e+003 - 7.7019979e+003 - 8.0180679e+003 - 7.4097706e+003 - 7.8521285e+003 - 7.8862599e+003 - 6.9756760e+003 - 7.0458814e+003 - 7.0818630e+003 - 7.0469740e+003 - 7.3942750e+003 - 7.6212107e+003 - 8.0331130e+003 - 5.8774297e+003 - 6.7858793e+003 - 5.0526846e+003 - 6.3296262e+003 - 6.6286394e+003 - 5.7507044e+003 - 4.8187842e+003 - 5.6265098e+003 - 6.1350865e+003 - 6.3377813e+003 - 5.3438440e+003 - 5.0777790e+003 - 5.7758229e+003 - 7.1658929e+003 - 6.8816661e+003 - 6.7885425e+003 - 6.2921146e+003 - 6.5498023e+003 - 6.6064213e+003 - 6.2183867e+003 - 5.0498603e+003 - 4.6634948e+003 - 4.5687261e+003 - 4.3857684e+003 - 5.0285132e+003 - 2.1155539e+003 - 0.0000000e+000 - 2.7416907e+002 - 0.0000000e+000 - 0.0000000e+000 - 4.9088419e+002 - 1.7638303e+003 - 3.8605084e+003 - 8.2135056e+003 - 8.7801030e+003 - 5.9497100e+003 - 4.0424478e+003 - 5.7662183e+003 - 4.8092876e+003 - 6.0083239e+003 - 6.8251731e+003 - 5.9522311e+003 - 8.2892291e+003 - 9.1881319e+003 - 9.7250825e+003 - 8.8972293e+003 - 9.0742703e+003 - 9.1758180e+003 - 9.0004363e+003 - 9.4690063e+003 - 9.6097636e+003 - 9.9118848e+003 - 1.0552070e+004 - 1.0937108e+004 - 1.1796701e+004 - 1.2074522e+004 - 1.2548494e+004 - 1.2534556e+004 - 1.1906156e+004 - 1.0657603e+004 - 9.9997481e+003 - 9.8693954e+003 - 9.3460985e+003 - 7.6985595e+003 - 6.1220872e+003 - 5.3906247e+003 - 5.2934532e+003 - 5.3205076e+003 - 6.2453775e+003 - 7.3703333e+003 - 6.9521713e+003 - 1.1312021e+004 - 6.9335426e+003 - 9.6296636e+003 - 1.3122951e+004 - 1.4613797e+004 - 1.4656108e+004 - 1.4862599e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.2926132e+004 - 1.4332965e+004 - 1.4697240e+004 - 1.4862313e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4990098e+004 - 1.4940247e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4060591e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4751323e+004 - 1.1684594e+004 - 9.1645237e+003 - 7.2940883e+003 - 6.1109924e+003 - 6.0846096e+003 - 6.6383473e+003 - 7.2848805e+003 - 8.3240326e+003 - 7.6836168e+003 - 7.3149665e+003 - 7.3222876e+003 - 8.3529365e+003 - 8.8919206e+003 - 8.5622460e+003 - 8.5157977e+003 - 1.0213160e+004 - 1.1473742e+004 - 1.1416144e+004 - 1.0262146e+004 - 9.8474701e+003 - 9.6073067e+003 - 9.3334956e+003 - 8.1114094e+003 - 6.3438393e+003 - 4.0561582e+002 - 4.9338711e+003 - 7.6173277e+003 - 8.1979965e+003 - 9.6515621e+003 - 9.9401838e+003 - 1.0555439e+004 - 1.1204735e+004 - 1.2438476e+004 - 1.2991111e+004 - 1.3726179e+004 - 1.3853785e+004 - 1.3384780e+004 - 1.4689700e+004 - 1.4351914e+004 - 1.4352144e+004 - 1.3893380e+004 - 1.3939271e+004 - 1.4075510e+004 - 1.4510647e+004 - 1.4691427e+004 - 1.4697897e+004 - 1.4591968e+004 - 1.4143373e+004 - 1.1993411e+004 - 1.4062098e+004 - 1.4355202e+004 - 1.4496242e+004 - 1.4251580e+004 - 1.3691010e+004 - 1.3787219e+004 - 1.3348001e+004 - 1.2868016e+004 - 1.2121213e+004 - 1.1597200e+004 - 1.1197107e+004 - 1.0874213e+004 - 1.0800695e+004 - 8.6047889e+003 - 8.7931705e+003 - 1.0162987e+004 - 1.1157604e+004 - 1.2127598e+004 - 1.3331766e+004 - 1.4088376e+004 - 1.4241923e+004 - 1.4298649e+004 - 1.3983031e+004 - 1.2426142e+004 - 1.2821723e+004 - 1.0648820e+004 - 1.0788457e+004 - 1.1565678e+004 - 1.0675574e+004 - 1.0052872e+004 - 1.0062980e+004 - 9.9391197e+003 - 1.1037139e+004 - 1.0427495e+004 - 9.3659423e+003 - 9.8900697e+003 - 1.0357397e+004 - 1.0785399e+004 - 1.0799019e+004 - 1.1000265e+004 - 1.0543268e+004 - 1.0143810e+004 - 9.7901431e+003 - 9.0954554e+003 - 8.6694209e+003 - 9.7264622e+003 - 1.0464018e+004 - 8.4039424e+003 - 9.2706552e+003 - 9.8937533e+003 - 9.7688013e+003 - 1.0387232e+004 - 1.0884452e+004 - 1.2140833e+004 - 1.1592473e+004 - 1.0549975e+004 - 1.1517130e+004 - 1.0276514e+004 - 9.4995134e+003 - 1.1074524e+004 - 1.0266440e+004 - 1.0003780e+004 - 9.4539517e+003 - 1.0130070e+004 - 9.1334953e+003 - 9.4781231e+003 - 8.7930478e+003 - 1.1818054e+004 - 1.4232598e+004 - 1.4272504e+004 - 1.4130687e+004 - 1.0264280e+004 - 1.2858109e+004 - 1.4357514e+004 - 1.5000000e+004 - 1.4880674e+004 - 1.4071591e+004 - 1.0816174e+004 - 1.3903217e+004 - 7.6465652e+003 - 7.2759868e+003 - 1.1878430e+004 - 7.9363899e+003 - 1.0012931e+004 - 8.3717781e+003 - 9.8052206e+003 - 1.0887308e+004 - 1.0384128e+004 - 9.0317571e+003 - 9.0035754e+003 - 9.3831353e+003 - 9.5517033e+003 - 9.9428520e+003 - 9.9763147e+003 - 1.0281617e+004 - 7.1627889e+003 - 8.4401671e+003 - 1.0010852e+004 - 1.0799497e+004 - 1.0757979e+004 - 1.0022172e+004 - 9.2195862e+003 - 8.9633090e+003 - 9.0556752e+003 - 8.7844813e+003 - 8.2308721e+003 - 8.3282462e+003 - 8.9225324e+003 - 8.3367010e+003 - 1.0200315e+004 - 9.5009000e+003 - 8.2646443e+003 - 8.0715231e+003 - 6.5503256e+003 - 7.2821816e+003 - 7.4417689e+003 - 7.3749177e+003 - 8.0189642e+003 - 8.0100006e+003 - 8.5355483e+003 - 8.4134563e+003 - 8.8888674e+003 - 9.7086060e+003 - 1.0389594e+004 - 1.0714743e+004 - 9.5721915e+003 - 7.9565683e+003 - 9.0114754e+003 - 9.4010374e+003 - 9.1089941e+003 - 8.4832686e+003 - 1.0275837e+004 - 8.4921531e+003 - 8.6300718e+003 - 7.9358269e+003 - 9.8119300e+003 - 1.1489482e+004 - 1.1847258e+004 - 1.2046636e+004 - 1.1030807e+004 - 1.1506255e+004 - 1.1150376e+004 - 1.2732117e+004 - 1.2249684e+004 - 1.3175625e+004 - 1.1492777e+004 - 1.2040457e+004 - 1.3115363e+004 - 1.3116650e+004 - 1.0825734e+004 - 1.0265123e+004 - 1.0946416e+004 - 9.9381093e+003 - 6.8152515e+003 - 5.7137921e+003 - 5.9031640e+003 - 5.3094664e+003 - 5.2097611e+003 - 5.3048974e+003 - 5.2859667e+003 - 5.2217390e+003 - 5.7591864e+003 - 7.5012790e+003 - 9.4523013e+003 - 1.0614623e+004 - 1.1955298e+004 - 1.2006303e+004 - 1.1038886e+004 - 1.1795523e+004 - 1.2380650e+004 - 1.3152396e+004 - 1.3418746e+004 - 1.3160889e+004 - 1.2299129e+004 - 1.2013338e+004 - 1.1741069e+004 - 1.1893269e+004 - 1.1584606e+004 - 1.1147106e+004 - 1.0761704e+004 - 1.2353932e+004 - 1.2179779e+004 - 1.2336318e+004 - 1.1347633e+004 - 1.1096587e+004 - 1.0899695e+004 - 1.1064539e+004 - 1.0730557e+004 - 1.0857225e+004 - 1.1160569e+004 - 1.0652600e+004 - 9.4004273e+003 - 8.3736762e+003 - 7.6853930e+003 - 7.1318434e+003 - 8.4332247e+003 - 8.4782871e+003 - 8.5971295e+003 - 8.6871340e+003 - 8.4365704e+003 - 8.5786831e+003 - 8.6365783e+003 - 8.5494723e+003 - 1.0489354e+004 - 7.6192296e+003 - 7.9697233e+003 - 9.5136252e+003 - 1.0947586e+004 - 1.1125370e+004 - 1.0969058e+004 - 1.1619407e+004 - 1.1480734e+004 - 1.1331045e+004 - 1.1085992e+004 - 1.0504574e+004 - 8.2885307e+003 - 7.1116444e+003 - 5.3099486e+003 - 5.4104594e+003 - 7.1939045e+003 - 7.8115645e+003 - 8.0941088e+003 - 8.5691029e+003 - 8.5428885e+003 - 8.2565212e+003 - 7.6313954e+003 - 7.4785229e+003 - 6.4507925e+003 - 7.3010808e+003 - 5.7756984e+003 - 7.4007447e+003 - 8.3370762e+003 - 9.3143142e+003 - 9.3536507e+003 - 9.4141175e+003 - 9.2036282e+003 - 8.8145538e+003 - 8.9239289e+003 - 9.3804717e+003 - 6.2709081e+003 - 6.1114121e+003 - 6.2738219e+003 - 5.6129922e+003 - 4.7757788e+003 - 4.3727289e+003 - 5.1782805e+003 - 5.4239609e+003 - 5.5214974e+003 - 4.8746007e+003 - 5.0404383e+003 - 5.5573708e+003 - 5.4923791e+003 - 4.9899502e+003 - 2.2982894e+003 - 4.2297189e+003 - 5.4411159e+003 - 6.2842951e+003 - 6.7031059e+003 - 8.1185574e+003 - 8.4948276e+003 - 8.1088164e+003 - 7.6992566e+003 - 7.5837450e+003 - 7.5886235e+003 - 7.7050497e+003 - 8.3973342e+003 - 8.9813138e+003 - 8.3544307e+003 - 8.1024005e+003 - 8.3723648e+003 - 8.9836052e+003 - 8.7945796e+003 - 8.7470606e+003 - 9.1922825e+003 - 9.5241727e+003 - 9.1467564e+003 - 9.6344040e+003 - 9.9588518e+003 - 1.0519693e+004 - 1.1436568e+004 - 1.1880399e+004 - 1.2471748e+004 - 1.3461235e+004 - 1.4294578e+004 - 1.4343208e+004 - 1.2798949e+004 - 1.2732037e+004 - 1.0217775e+004 - 1.0970581e+004 - 1.0073755e+004 - 1.1582556e+004 - 1.2277242e+004 - 1.2719678e+004 - 1.2128965e+004 - 1.2331973e+004 - 1.1861323e+004 - 1.0967865e+004 - 1.0032578e+004 - 9.7760533e+003 - 8.1447286e+003 - 8.9166745e+003 - 1.0109226e+004 - 1.1228582e+004 - 1.1381985e+004 - 1.1970990e+004 - 1.2398492e+004 - 1.3443496e+004 - 1.4066596e+004 - 1.2492656e+004 - 1.1226688e+004 - 1.2020566e+004 - 8.2054384e+003 - 6.9616154e+003 - 5.7974308e+003 - 6.4327344e+003 - 6.7351741e+003 - 6.5627760e+003 - 8.1136570e+003 - 9.4613752e+003 - 8.5925193e+003 - 7.2255361e+003 - 7.9589936e+003 - 8.5371920e+003 - 7.3700956e+003 - 5.8595577e+003 - 6.8845730e+003 - 6.7791940e+003 - 8.2558450e+003 - 1.1292907e+004 - 1.1580723e+004 - 1.0303704e+004 - 1.0512163e+004 - 1.0907818e+004 - 1.1204047e+004 - 1.0924884e+004 - 7.5308779e+003 - 8.2331017e+003 - 9.1181047e+003 - 9.3105271e+003 - 8.5745396e+003 - 7.6183117e+003 - 5.8037402e+003 - 4.9676050e+003 - 5.6976352e+003 - 6.1163937e+003 - 6.3299585e+003 - 6.6673479e+003 - 3.7720009e+003 - 1.2907931e+003 - 3.8050762e+003 - 3.5581705e+003 - 4.2191732e+003 - 3.0935467e+003 - 7.8517051e+002 - 4.5230470e+002 - 0.0000000e+000 - 1.6051884e+002 - 7.7393419e+003 - 7.9471793e+003 - 3.4814704e+003 - 6.9684122e+003 - 8.1945817e+003 - 7.9942471e+003 - 9.5009542e+003 - 9.9566565e+003 - 1.4230842e+004 - 1.3027073e+004 - 1.4054149e+004 - 1.4219180e+004 - 1.4109562e+004 - 1.4099178e+004 - 1.1406177e+004 - 1.4182420e+004 - 1.3930990e+004 - 1.4016152e+004 - 1.3402675e+004 - 1.2406548e+004 - 1.2344375e+004 - 1.2679775e+004 - 1.3540816e+004 - 1.3766233e+004 - 1.3628327e+004 - 1.2999719e+004 - 9.4274555e+003 - 8.7006771e+003 - 6.9992221e+003 - 5.3572917e+003 - 4.5776297e+003 - 3.0814996e+003 - 4.9574130e+003 - 6.1368574e+003 - 6.9490544e+003 - 7.2498410e+003 - 6.6022853e+003 - 6.0844322e+003 - 6.9187796e+003 - 8.2769576e+003 - 1.0125024e+004 - 1.0262458e+004 - 9.8215287e+003 - 1.0097065e+004 - 9.7316288e+003 - 9.0815458e+003 - 8.3111465e+003 - 7.7877204e+003 - 7.7417598e+003 - 7.1922222e+003 - 5.9405829e+003 - 6.0208822e+003 - 5.2262047e+003 - 5.5931817e+003 - 5.5231809e+003 - 5.7824516e+003 - 6.1202902e+003 - 6.0829500e+003 - 5.3967209e+003 - 4.7342316e+003 - 3.4466146e+003 - 2.6059355e+003 - 1.5067015e+003 - 0.0000000e+000 - 2.2589853e+003 - 3.8027231e+003 - 3.8883399e+003 - 3.2429762e+003 - 2.1907548e+003 - 2.4572208e+003 - 4.2492064e+003 - 4.8484941e+003 - 5.6113452e+003 - 6.0872126e+003 - 5.8661149e+003 - 5.9887086e+003 - 7.2430355e+003 - 7.4775572e+003 - 7.6548130e+003 - 8.5438716e+003 - 9.0262212e+003 - 8.7213279e+003 - 8.7849481e+003 - 9.2445689e+003 - 1.0463584e+004 - 1.0851014e+004 - 1.1244613e+004 - 9.9222886e+003 - 1.1400914e+004 - 1.2771548e+004 - 1.4351597e+004 - 1.4669362e+004 - 1.4750682e+004 - 1.4907726e+004 - 1.4903037e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 9.6945335e+003 - 1.4235141e+004 - 1.4558910e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4998788e+004 - 1.4838116e+004 - 1.4780061e+004 - 1.4551851e+004 - 1.3909302e+004 - 1.3598276e+004 - 1.4487232e+004 - 8.7927681e+003 - 1.0226854e+004 - 1.0384457e+004 - 9.6039296e+003 - 8.1925527e+003 - 5.2707829e+003 - 4.4195401e+003 - 2.9587139e+003 - 0.0000000e+000 - 1.2717178e+003 - 5.3838182e+003 - 6.1523695e+003 - 1.0218047e+004 - 1.2344060e+004 - 1.4307026e+004 - 1.4693372e+004 - 1.4538001e+004 - 1.4256473e+004 - 1.3881682e+004 - 1.4549841e+004 - 1.4747555e+004 - 1.4969578e+004 - 1.5000000e+004 - 1.4623904e+004 - 1.0738703e+004 - 1.1906195e+004 - 1.3115463e+004 - 1.0614817e+004 - 7.2261616e+003 - 7.3784938e+003 - 8.4224594e+003 - 9.9926385e+003 - 9.7966311e+003 - 9.2890147e+003 - 8.2290849e+003 - 7.5383569e+003 - 2.2519468e+003 - 2.4927724e+003 - 5.6970359e+003 - 7.4595587e+003 - 8.1395470e+003 - 8.7517767e+003 - 9.0394558e+003 - 8.5998846e+003 - 9.6424600e+003 - 8.8426168e+003 - 8.8433162e+003 - 9.6074805e+003 - 1.2378596e+004 - 1.1291806e+004 - 1.2874125e+004 - 1.1967205e+004 - 9.8177551e+003 - 1.0863792e+004 - 9.8493212e+003 - 9.4457432e+003 - 9.6908406e+003 - 9.7060549e+003 - 7.1828683e+003 - 7.4402109e+003 - 4.9753699e+003 - 7.0477806e+003 - 7.2878056e+003 - 7.0312899e+003 - 1.4422677e+004 - 1.4862382e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4664915e+004 - 1.4860431e+004 - 1.5000000e+004 - 1.3167820e+004 - 1.4553562e+004 - 1.4424759e+004 - 1.4369247e+004 - 1.4510113e+004 - 1.4715426e+004 - 1.4821807e+004 - 1.4779634e+004 - 1.4773623e+004 - 1.4665011e+004 - 1.4497329e+004 - 1.4228307e+004 - 1.0349298e+004 - 1.0231707e+004 - 9.0755604e+003 - 6.9963721e+003 - 5.4214200e+003 - 5.1842575e+003 - 5.5043389e+003 - 6.0176365e+003 - 6.0730345e+003 - 5.9990182e+003 - 6.3205371e+003 - 6.6522761e+003 - 5.7592750e+003 - 3.9618116e+003 - 4.7188334e+003 - 3.8302327e+003 - 4.8734338e+003 - 5.1174204e+003 - 4.2626353e+003 - 3.9377523e+003 - 2.6386477e+003 - 3.0096234e+003 - 4.8132301e+003 - 6.0570486e+003 - 6.4175356e+003 - 7.5031616e+003 - 7.2262474e+003 - 7.6295466e+003 - 7.4976582e+003 - 6.9635814e+003 - 7.9384315e+003 - 8.8953825e+003 - 9.5185395e+003 - 1.0330471e+004 - 1.1349655e+004 - 1.2470088e+004 - 1.3779902e+004 - 1.4065696e+004 - 1.4388848e+004 - 1.4601822e+004 - 1.4622679e+004 - 1.4579627e+004 - 1.4419865e+004 - 1.4297374e+004 - 1.4191509e+004 - 1.4050337e+004 - 1.4047227e+004 - 1.3268590e+004 - 1.0842803e+004 - 1.0856715e+004 - 1.0763559e+004 - 1.1008671e+004 - 1.0584061e+004 - 1.0834087e+004 - 1.0659918e+004 - 1.0818468e+004 - 1.0845614e+004 - 1.0985154e+004 - 1.1090202e+004 - 1.0924856e+004 - 9.6969443e+003 - 7.9065884e+003 - 8.1800297e+003 - 8.3632823e+003 - 9.0186720e+003 - 9.8467959e+003 - 1.1016602e+004 - 1.2789440e+004 - 1.3928034e+004 - 1.4321691e+004 - 1.4499065e+004 - 1.4191363e+004 - 1.3850833e+004 - 1.4453261e+004 - 1.4702270e+004 - 1.4574877e+004 - 1.4401424e+004 - 1.4472576e+004 - 1.4527548e+004 - 1.4498455e+004 - 1.4551654e+004 - 1.4663920e+004 - 1.4462211e+004 - 1.4366495e+004 - 1.4205637e+004 - 1.4045071e+004 - 1.3780095e+004 - 1.4468147e+004 - 1.4584918e+004 - 1.4839587e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4103580e+004 - 1.4780416e+004 - 1.4511864e+004 - 1.4251713e+004 - 1.4010091e+004 - 1.3771909e+004 - 1.3718037e+004 - 1.2941120e+004 - 1.2348602e+004 - 1.1332261e+004 - 1.0755099e+004 - 9.9317289e+003 - 8.4146989e+003 - 7.3312072e+003 - 8.5987428e+003 - 1.0575563e+004 - 9.7730287e+003 - 9.8465886e+003 - 9.9801805e+003 - 1.0713912e+004 - 1.2316672e+004 - 1.1628361e+004 - 1.0222359e+004 - 1.1185287e+004 - 7.1952274e+003 - 8.1462252e+003 - 7.1832873e+003 - 6.5941512e+003 - 5.7905771e+003 - 6.3441888e+003 - 6.9921466e+003 - 7.4044132e+003 - 7.5974582e+003 - 8.4721387e+003 - 9.1518057e+003 - 9.7677916e+003 - 9.1428252e+003 - 7.1795296e+003 - 9.2744816e+003 - 8.3609313e+003 - 8.0079662e+003 - 7.6190240e+003 - 7.4483544e+003 - 7.3982961e+003 - 7.6194396e+003 - 6.9536670e+003 - 4.9022526e+003 - 5.6161008e+003 - 5.0917111e+003 - 5.1245516e+003 - 5.4174584e+003 - 5.6520863e+003 - 5.9454104e+003 - 6.0045630e+003 - 5.9210494e+003 - 5.2389732e+003 - 3.8591481e+003 - 1.4467601e+003 - 3.0417928e+003 - 2.9433412e+003 - 6.4794727e+003 - 7.6529045e+003 - 8.1381275e+003 - 5.0191317e+003 - 4.8657277e+003 - 2.9020809e+003 - 4.3824444e+003 - 5.0371205e+003 - 2.1037217e+003 - 3.3177935e+003 - 4.6513657e+003 - 5.2011133e+003 - 5.9202731e+003 - 6.6907524e+003 - 5.5995211e+003 - 5.3336612e+003 - 5.9147189e+003 - 7.5846374e+003 - 8.4814278e+003 - 8.5951282e+003 - 8.4993330e+003 - 8.3402980e+003 - 7.2545869e+003 - 5.3112997e+003 - 6.4486672e+003 - 4.7763944e+003 - 6.1593969e+003 - 6.8076768e+003 - 6.5509355e+003 - 6.0067295e+003 - 6.2819191e+003 - 5.7548141e+003 - 5.2695458e+003 - 5.2348433e+003 - 5.0356579e+003 - 5.2923127e+003 - 3.1332352e+003 - 6.1057148e+003 - 6.4533302e+003 - 8.0497698e+003 - 7.9455172e+003 - 8.2137588e+003 - 8.3572595e+003 - 8.5122692e+003 - 9.1443293e+003 - 9.3060565e+003 - 9.8158368e+003 - 1.0478419e+004 - 1.1646779e+004 - 9.2837239e+003 - 1.0183691e+004 - 1.0957920e+004 - 1.1322106e+004 - 1.1273801e+004 - 1.1588634e+004 - 1.1234634e+004 - 1.1145033e+004 - 1.1251659e+004 - 1.1084729e+004 - 1.0745217e+004 - 1.0080793e+004 - 1.1191130e+004 - 1.0367737e+004 - 9.5894096e+003 - 1.0238764e+004 - 1.0304305e+004 - 9.9091002e+003 - 1.0046645e+004 - 8.5734719e+003 - 8.7977642e+003 - 8.1851105e+003 - 5.2578662e+003 - 6.6571882e+003 - 6.0686277e+003 - 6.3070662e+003 - 6.1018019e+003 - 6.7428190e+003 - 6.7637043e+003 - 6.5940733e+003 - 6.4179907e+003 - 5.7490083e+003 - 5.9589102e+003 - 6.8095689e+003 - 7.2564983e+003 - 4.6593157e+003 - 7.2394601e+003 - 7.8106759e+003 - 8.1899386e+003 - 8.5651090e+003 - 8.5033792e+003 - 8.2711352e+003 - 7.3121470e+003 - 6.8207499e+003 - 7.3849363e+003 - 8.4384262e+003 - 1.0405006e+004 - 7.5278498e+003 - 8.8948559e+003 - 8.6513568e+003 - 9.4766073e+003 - 1.0265654e+004 - 1.1138318e+004 - 1.0004118e+004 - 1.0014276e+004 - 9.6773035e+003 - 9.2577794e+003 - 9.6536494e+003 - 8.9305790e+003 - 6.3995071e+003 - 7.8741632e+003 - 8.2033798e+003 - 6.7964818e+003 - 7.1534205e+003 - 7.4988049e+003 - 7.8277198e+003 - 7.5563407e+003 - 7.4225807e+003 - 8.6526650e+003 - 7.4091893e+003 - 6.7803344e+003 - 6.0273998e+003 - 8.8065513e+003 - 9.4010440e+003 - 9.4169298e+003 - 8.7370657e+003 - 9.2979916e+003 - 9.4322621e+003 - 8.5011357e+003 - 8.0025332e+003 - 8.5687934e+003 - 9.4051043e+003 - 9.5558041e+003 - 6.6547020e+003 - 4.6573601e+003 - 2.5193775e+003 - 2.7473859e+003 - 4.0446006e+003 - 5.4193625e+003 - 7.0747943e+003 - 8.0697881e+003 - 7.9641116e+003 - 8.5241692e+003 - 8.7522679e+003 - 9.5272462e+003 - 9.9581737e+003 - 8.2782298e+003 - 9.0375052e+003 - 1.0328521e+004 - 1.0773521e+004 - 1.0865363e+004 - 1.0751728e+004 - 1.0122695e+004 - 9.0702557e+003 - 8.3015051e+003 - 7.5808901e+003 - 6.1856114e+003 - 5.1589270e+003 - 2.8245546e+003 - 0.0000000e+000 - 3.8834380e+003 - 7.7036154e+003 - 9.1131257e+003 - 1.0157199e+004 - 1.0459716e+004 - 9.9694679e+003 - 9.9187957e+003 - 9.0103822e+003 - 9.6560498e+003 - 8.8128617e+003 - 5.4104040e+003 - 8.1617268e+003 - 1.1239039e+004 - 1.3006974e+004 - 1.2982015e+004 - 1.2787861e+004 - 1.3251519e+004 - 1.3642529e+004 - 1.3484369e+004 - 1.3334273e+004 - 1.3469060e+004 - 1.1373561e+004 - 1.1458739e+004 - 9.9091026e+003 - 7.9893865e+003 - 7.0067381e+003 - 6.4598191e+003 - 6.6988439e+003 - 6.1887904e+003 - 5.7090384e+003 - 4.3731349e+003 - 2.3750084e+003 - 3.0234599e+003 - 1.1766045e+003 - 3.5745828e+003 - 6.2534313e+003 - 7.3481995e+003 - 6.6570226e+003 - 7.3509847e+003 - 9.1271248e+003 - 1.0324436e+004 - 1.2345007e+004 - 1.3442423e+004 - 1.4155620e+004 - 1.3754551e+004 - 1.0406521e+004 - 1.2816353e+004 - 1.2599886e+004 - 1.2021014e+004 - 1.0130577e+004 - 9.7117472e+003 - 8.7699330e+003 - 7.8535861e+003 - 7.7425062e+003 - 7.0244407e+003 - 5.6131164e+003 - 7.2183845e+003 - 7.7125816e+003 - 1.0143119e+004 - 1.2252463e+004 - 1.2125642e+004 - 1.0142400e+004 - 9.1455574e+003 - 7.7235463e+003 - 7.2803145e+003 - 7.1167510e+003 - 7.8016304e+003 - 8.4389021e+003 - 8.5366661e+003 - 7.5667628e+003 - 9.0872544e+003 - 6.8845001e+003 - 6.1899389e+003 - 6.7479612e+003 - 7.6967947e+003 - 7.3090887e+003 - 7.1469689e+003 - 8.3283654e+003 - 7.3473692e+003 - 7.0048256e+003 - 7.0912599e+003 - 1.9713676e+003 - 2.3842790e+003 - 1.5824960e+003 - 0.0000000e+000 - 6.4342466e+002 - 4.8269062e+003 - 5.8250748e+003 - 6.8101249e+003 - 7.1916506e+003 - 8.1950589e+003 - 9.0687165e+003 - 9.9904914e+003 - 5.4050319e+003 - 5.8251943e+003 - 8.4884213e+003 - 1.1620470e+004 - 1.3012971e+004 - 1.3447326e+004 - 1.2912711e+004 - 1.2250688e+004 - 1.1236535e+004 - 9.8294648e+003 - 9.4850794e+003 - 8.1851149e+003 - 7.3622724e+003 - 6.4167929e+003 - 7.8333624e+003 - 6.8216244e+003 - 5.8865947e+003 - 5.8376621e+003 - 6.4972995e+003 - 6.1574145e+003 - 6.4825886e+003 - 6.8777054e+003 - 6.5615306e+003 - 6.1711141e+003 - 4.7640420e+003 - 5.1067154e+003 - 4.7805015e+003 - 4.3092449e+003 - 5.5870241e+003 - 5.8390378e+003 - 7.2819950e+003 - 7.2913504e+003 - 7.2789419e+003 - 7.2806319e+003 - 6.9130886e+003 - 7.6665121e+003 - 5.4523266e+003 - 2.8719313e+003 - 4.5680029e+003 - 3.1745745e+003 - 3.9838718e+003 - 3.9009663e+003 - 4.2923389e+003 - 4.5294279e+003 - 3.8545680e+003 - 2.5409848e+003 - 2.8636222e+003 - 3.6677657e+003 - 5.7449924e+003 - 6.9465220e+003 - 6.1407558e+003 - 7.1463738e+003 - 7.1974219e+003 - 6.7406301e+003 - 6.4771015e+003 - 7.0134470e+003 - 6.2257799e+003 - 6.7958021e+003 - 6.0997263e+003 - 6.0029926e+003 - 7.7918150e+003 - 6.0998418e+003 - 8.2846717e+003 - 1.0162078e+004 - 1.0335985e+004 - 1.0398186e+004 - 1.0994938e+004 - 1.0746735e+004 - 1.0954441e+004 - 1.0907056e+004 - 1.1026725e+004 - 1.0564338e+004 - 8.8677528e+003 - 9.9587755e+003 - 9.7356965e+003 - 1.0051236e+004 - 1.0629361e+004 - 1.0720792e+004 - 1.0221790e+004 - 9.2497245e+003 - 8.4315678e+003 - 9.2424888e+003 - 9.1586627e+003 - 9.3637077e+003 - 6.5017165e+003 - 7.2404906e+003 - 6.5052638e+003 - 9.9293934e+003 - 1.2565271e+004 - 1.2901982e+004 - 1.3041471e+004 - 1.2996395e+004 - 1.0961241e+004 - 1.1649050e+004 - 1.2537911e+004 - 1.2072589e+004 - 1.4410087e+003 - 2.0367254e+003 - 6.3571656e+003 - 7.8128982e+003 - 8.1644409e+003 - 7.9625535e+003 - 8.1115099e+003 - 7.2439469e+003 - 7.3689942e+003 - 6.9361593e+003 - 7.2144664e+003 - 8.5767925e+003 - 5.2476323e+003 - 4.4913888e+003 - 6.4680599e+003 - 6.8294158e+003 - 7.4294059e+003 - 7.8053774e+003 - 8.7951187e+003 - 9.4409060e+003 - 1.0153159e+004 - 9.5702221e+003 - 1.4026322e+004 - 1.4156905e+004 - 8.8657290e+003 - 1.0246489e+004 - 1.2590538e+004 - 1.4422736e+004 - 1.2713618e+004 - 1.3046080e+004 - 1.3301771e+004 - 1.2647591e+004 - 1.2796771e+004 - 1.3716864e+004 - 1.2726277e+004 - 1.0539755e+004 - 5.7693609e+003 - 8.5675077e+003 - 1.1349817e+004 - 1.0410482e+004 - 1.0099439e+004 - 1.1517140e+004 - 1.2124079e+004 - 1.3122642e+004 - 1.2324279e+004 - 1.2158293e+004 - 1.2205180e+004 - 1.1892311e+004 - 8.9143136e+003 - 9.7568659e+003 - 8.1227673e+003 - 6.3500495e+003 - 5.4058499e+003 - 4.5472374e+003 - 6.0866360e+003 - 5.5470453e+003 - 1.6602197e+002 - 1.8578068e+003 - 3.7883452e+003 - 3.3225216e+003 - 1.9742115e+003 - 5.1385953e+002 - 3.9077731e+003 - 1.8933235e+003 - 2.2054272e+003 - 6.3156260e+003 - 4.1913372e+003 - 6.9488198e+003 - 7.7643293e+003 - 1.0266475e+004 - 6.9127114e+003 - 5.2511505e+003 - 5.8870647e+001 - 1.0240969e+003 - 2.6163590e+003 - 5.1724745e+003 - 5.5372000e+003 - 5.2604604e+003 - 5.2838191e+003 - 5.6754571e+003 - 5.8584137e+003 - 5.2987272e+003 - 4.5076933e+003 - 3.8788050e+003 - 3.4151248e+003 - 3.5225775e+003 - 2.8733259e+003 - 1.8093918e+003 - 0.0000000e+000 - 1.0795535e+003 - 2.7754500e+002 - 0.0000000e+000 - 9.0553162e+002 - 0.0000000e+000 - 2.7707268e+003 - 5.1900788e+003 - 5.7874955e+003 - 6.5378102e+003 - 7.2404172e+003 - 6.9584685e+003 - 7.6042900e+003 - 7.2979851e+003 - 6.6436143e+003 - 6.5802680e+003 - 7.2929043e+003 - 7.1330503e+003 - 7.5207578e+003 - 7.7095669e+003 - 6.0941090e+003 - 6.1375898e+003 - 6.7324730e+003 - 6.3539827e+003 - 6.7199163e+003 - 7.3856206e+003 - 6.9099188e+003 - 6.2791681e+003 - 5.7336970e+003 - 6.1250121e+003 - 4.9989824e+003 - 3.7517717e+003 - 2.0270446e+003 - 4.1754971e+003 - 3.3284127e+003 - 1.7522747e+003 - 2.9663492e+003 - 2.2665690e+003 - 2.1946821e+003 - 4.7015155e+003 - 4.7593250e+003 - 5.1262892e+003 - 5.2956802e+003 - 4.0256523e+003 - 6.9406231e+002 - 0.0000000e+000 - 1.3529806e+003 - 4.8545542e+001 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 1.9105744e+003 - 5.7566395e+003 - 6.7927206e+003 - 4.4084449e+003 - 9.0032342e+003 - 9.2858955e+003 - 7.9040366e+003 - 6.6612651e+003 - 3.4000540e+003 - 5.1593503e+003 - 5.2678603e+003 - 5.5953629e+003 - 6.7201199e+003 - 5.9258621e+003 - 5.5936188e+003 - 5.1160557e+003 - 3.6117174e+003 - 1.6301069e+003 - 1.8176182e+003 - 2.6925144e+003 - 3.6755757e+003 - 2.9776107e+003 - 1.4714214e+003 - 7.1961512e+002 - 6.4631935e+002 - 1.3784135e+003 - 9.0936796e+002 - 0.0000000e+000 - 3.7065335e+003 - 5.4657806e+003 - 5.0072228e+003 - 5.8812034e+003 - 6.3856518e+003 - 5.4793683e+003 - 5.9842340e+003 - 6.1433556e+003 - 6.5785460e+003 - 5.1688039e+003 - 3.5269882e+003 - 4.5670886e+003 - 5.5525570e+003 - 2.0728713e+003 - 3.8527724e+003 - 5.0305078e+003 - 4.9060143e+003 - 4.2802594e+003 - 5.1909190e+003 - 5.5657233e+003 - 5.4868548e+003 - 5.5248154e+003 - 5.6573546e+003 - 5.6130026e+003 - 5.0661694e+003 - 5.4837302e+003 - 5.4849214e+003 - 5.9098361e+003 - 5.4620551e+003 - 6.3312340e+003 - 3.1756159e+003 - 2.1098569e+003 - 4.3778587e+003 - 5.3733805e+003 - 5.5877967e+003 - 6.3841769e+003 - 6.7416745e+003 - 5.8575543e+003 - 7.8369288e+003 - 9.5787071e+003 - 9.5863616e+003 - 1.0115756e+004 - 1.0336706e+004 - 1.0297212e+004 - 1.0415543e+004 - 1.0099925e+004 - 1.0082155e+004 - 1.0545828e+004 - 8.1380135e+003 - 8.8193941e+003 - 8.0032709e+003 - 7.1685078e+003 - 7.6135448e+003 - 8.1851669e+003 - 7.8635292e+003 - 8.0855428e+003 - 9.0333803e+003 - 9.6257221e+003 - 9.9277903e+003 - 9.5930641e+003 - 7.7066137e+003 - 8.0910999e+003 - 9.8946754e+003 - 9.4830051e+003 - 8.5376690e+003 - 6.5759429e+003 - 7.7289846e+003 - 8.6976175e+003 - 1.0036760e+004 - 1.1666412e+004 - 1.1616603e+004 - 1.3064265e+004 - 4.1354681e+003 - 3.3577367e+003 - 5.1410283e+003 - 5.3071700e+003 - 3.8658578e+003 - 1.5108627e+003 - 1.7223051e+001 - 2.3983935e+003 - 4.0122291e+003 - 3.1450738e+003 - 1.9015533e+003 - 1.5889253e+003 - 1.7059994e+003 - 0.0000000e+000 - 4.5913009e+002 - 3.8454060e+002 - 1.3740268e+003 - 4.5888329e+003 - 5.9313580e+003 - 6.3368913e+003 - 6.5389834e+003 - 8.0079893e+003 - 7.2733464e+003 - 6.2922643e+003 - 6.9324580e+003 - 9.0884543e+003 - 9.2501225e+003 - 9.6300279e+003 - 8.4199893e+003 - 1.0679424e+004 - 1.1288787e+004 - 6.6741081e+003 - 7.1241351e+003 - 8.9936766e+003 - 8.2553715e+003 - 8.2626896e+003 - 5.0835977e+003 - 6.3241808e+003 - 7.0707636e+003 - 8.1504495e+003 - 8.1898525e+003 - 1.4767667e+004 - 9.8976934e+003 - 1.1943930e+004 - 1.0355243e+004 - 9.9232964e+003 - 9.2338121e+003 - 9.0760019e+003 - 7.7107337e+003 - 8.1354205e+003 - 7.6092748e+003 - 7.7099566e+003 - 8.0319901e+003 - 7.1813213e+003 - 6.4175565e+003 - 8.5573093e+003 - 8.3248106e+003 - 7.2845923e+003 - 8.3599213e+003 - 6.2220891e+003 - 8.2891906e+003 - 7.9116589e+003 - 8.0517064e+003 - 8.9145604e+003 - 9.8391029e+003 - 9.3974746e+003 - 1.0051104e+004 - 1.0737941e+004 - 9.9615112e+003 - 1.1239958e+004 - 1.1281650e+004 - 1.0532324e+004 - 9.8471216e+003 - 9.8222243e+003 - 8.6986757e+003 - 9.3830351e+003 - 8.5736062e+003 - 8.6084393e+003 - 8.0770468e+003 - 7.9935944e+003 - 8.1157801e+003 - 8.1151703e+003 - 8.2616322e+003 - 7.4956381e+003 - 6.2352758e+003 - 5.8959343e+003 - 6.5368028e+003 - 6.7409952e+003 - 6.7792682e+003 - 7.3127334e+003 - 7.4745658e+003 - 7.6090101e+003 - 6.7380284e+003 - 5.9900257e+003 - 5.0695933e+003 - 3.1288433e+003 - 5.6078846e+003 - 4.5385090e+003 - 7.6509109e+002 - 5.5717259e+002 - 1.1732229e+003 - 2.1972670e+003 - 4.1585390e+002 - 2.1927157e+003 - 2.4746003e+003 - 3.0887979e+003 - 0.0000000e+000 - 3.8216367e+003 - 7.5230860e+003 - 9.2476041e+003 - 6.1601714e+003 - 7.2145547e+003 - 8.3901803e+003 - 9.9004312e+003 - 9.0669439e+003 - 8.3979605e+003 - 9.9188135e+003 - 9.3720552e+003 - 8.6272044e+003 - 8.9535088e+003 - 8.1727367e+003 - 8.3266831e+003 - 7.9830464e+003 - 9.2392207e+003 - 8.5760059e+003 - 7.7162674e+003 - 7.7887734e+003 - 6.7516880e+003 - 6.1944475e+003 - 5.8675081e+003 - 4.1581110e+003 - 3.8564988e+003 - 2.2141436e+003 - 2.8429817e+003 - 2.5847702e+003 - 9.3644845e+002 - 2.4883095e+003 - 2.6087179e+003 - 1.7287660e+003 - 2.8552824e+003 - 5.0847389e+003 - 5.6747216e+003 - 5.5030866e+003 - 5.0631323e+003 - 6.1120144e+003 - 6.4617729e+003 - 5.6395520e+003 - 6.4217439e+003 - 7.3185571e+003 - 6.9522616e+003 - 6.5553289e+003 - 7.5717204e+003 - 7.5227937e+003 - 7.2179354e+003 - 6.1251516e+003 - 5.3120072e+003 - 6.4272348e+003 - 5.3138488e+003 - 5.4044837e+003 - 5.6653468e+003 - 5.8702991e+003 - 5.9703083e+003 - 6.3890118e+003 - 7.4351706e+003 - 8.6857612e+003 - 9.2276656e+003 - 9.2702176e+003 - 9.8048538e+003 - 2.6152022e+003 - 4.5711136e+003 - 4.4905163e+003 - 5.6068393e+003 - 5.5306206e+003 - 5.6161037e+003 - 5.5625188e+003 - 6.3023223e+003 - 6.1484242e+003 - 4.8717551e+003 - 3.1150243e+003 - 3.6059683e+003 - 3.3726779e+003 - 1.9004160e+003 - 3.8964321e+002 - 1.7545885e+003 - 2.4606839e+003 - 4.4542562e+003 - 5.6779564e+003 - 6.6094187e+003 - 7.1349968e+003 - 1.4308249e+004 - 8.5103093e+003 - 9.6022785e+003 - 2.9507782e+003 - 5.4110663e+003 - 2.6728750e+003 - 4.1088924e+003 - 3.5104532e+003 - 5.1870171e+003 - 7.6663404e+003 - 5.5474271e+003 - 3.8663124e+003 - 6.3706671e+002 - 5.9133141e+003 - 7.4740093e+003 - 2.6901149e+003 - 5.1306972e+003 - 5.1092424e+003 - 4.6151011e+003 - 3.7477441e+003 - 7.6072783e+003 - 8.6949678e+003 - 8.0007780e+003 - 8.6120852e+003 - 8.9808666e+003 - 9.7124296e+003 - 9.6761321e+003 - 6.7570433e+003 - 8.4884199e+003 - 7.7006251e+003 - 7.9392441e+003 - 6.8280380e+003 - 8.5589116e+003 - 8.8220993e+003 - 7.6247765e+003 - 6.9773111e+003 - 6.7552273e+003 - 7.1007963e+003 - 6.7492893e+003 - 4.0456122e+003 - 3.7308786e+002 - 0.0000000e+000 - 1.2617616e+003 - 3.4709102e+003 - 5.5699170e+003 - 5.2390305e+003 - 5.5814255e+003 - 6.2784441e+003 - 7.8487709e+003 - 6.8106837e+003 - 6.1722759e+003 - 1.2882737e+003 - 5.6374066e+003 - 6.7666036e+003 - 6.4044011e+003 - 6.1926965e+003 - 7.4527327e+003 - 7.7745913e+003 - 8.0129269e+003 - 9.5576000e+003 - 1.1091750e+004 - 1.3087854e+004 - 1.2803123e+004 - 1.2008456e+004 - 1.2125610e+004 - 1.3413773e+004 - 1.3152577e+004 - 1.3214561e+004 - 1.4148770e+004 - 1.3147302e+004 - 1.2895442e+004 - 1.4378279e+004 - 1.4734349e+004 - 1.4559626e+004 - 1.4143941e+004 - 1.1352517e+004 - 1.3049280e+004 - 1.2563819e+004 - 1.1947592e+004 - 1.0072417e+004 - 9.8663881e+003 - 1.2550155e+004 - 1.4089469e+004 - 1.4221670e+004 - 1.4122945e+004 - 1.4007315e+004 - 1.3585524e+004 - 1.0671743e+004 - 1.0531281e+004 - 1.1816532e+004 - 1.4294193e+004 - 1.4850294e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4972497e+004 - 1.4923080e+004 - 1.4849811e+004 - 1.4578036e+004 - 1.2700639e+004 - 1.4317148e+004 - 1.3925392e+004 - 1.2230665e+004 - 1.0558172e+004 - 9.9140437e+003 - 9.7760140e+003 - 1.0673951e+004 - 1.1006136e+004 - 1.1797015e+004 - 1.2104783e+004 - 1.2477481e+004 - 1.0546639e+004 - 1.1083246e+004 - 1.4326341e+004 - 1.4447058e+004 - 1.4407826e+004 - 1.4323477e+004 - 1.4343003e+004 - 1.4519662e+004 - 1.4712139e+004 - 1.4799313e+004 - 1.4680448e+004 - 1.4576413e+004 - 1.1806693e+004 - 1.3122693e+004 - 1.2933308e+004 - 1.2232749e+004 - 1.2273590e+004 - 1.1562848e+004 - 1.1281326e+004 - 1.1157287e+004 - 1.1721403e+004 - 1.1603909e+004 - 1.2306861e+004 - 1.1710049e+004 - 9.2397281e+003 - 1.2037147e+004 - 1.2909439e+004 - 1.3295225e+004 - 1.3806301e+004 - 1.3957378e+004 - 1.4017477e+004 - 1.4005151e+004 - 1.4012014e+004 - 1.3572802e+004 - 1.2752857e+004 - 1.1374821e+004 - 1.0153828e+004 - 1.1778491e+004 - 1.2911783e+004 - 1.1920576e+004 - 1.2762397e+004 - 1.2455245e+004 - 1.2241557e+004 - 1.0901359e+004 - 1.0069315e+004 - 9.4443427e+003 - 8.3266568e+003 - 8.3331980e+003 - 6.8647294e+003 - 8.7714991e+003 - 7.9320258e+003 - 9.4644069e+003 - 1.0534225e+004 - 1.0607815e+004 - 1.0451568e+004 - 1.0228923e+004 - 9.7361868e+003 - 9.2832224e+003 - 8.6847002e+003 - 7.9327451e+003 - 7.4417248e+003 - 8.3701672e+003 - 8.2197891e+003 - 7.4887390e+003 - 6.6065643e+003 - 6.6685270e+003 - 6.6872881e+003 - 6.2746222e+003 - 5.6517581e+003 - 5.1326215e+003 - 5.3242084e+003 - 5.9364864e+003 - 5.6449955e+003 - 6.4779212e+003 - 7.4852756e+003 - 7.8167260e+003 - 7.6819011e+003 - 8.1733732e+003 - 7.8396676e+003 - 7.1357197e+003 - 6.7507416e+003 - 6.7767261e+003 - 6.6558331e+003 - 6.9054847e+003 - 6.9934084e+003 - 6.7077666e+003 - 4.5141515e+003 - 4.0838158e+003 - 4.1676578e+003 - 5.4400710e+003 - 5.6341860e+003 - 5.5547820e+003 - 2.5698700e+003 - 1.5022833e+003 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 1.1477677e+003 - 0.0000000e+000 - 1.8894553e+003 - 0.0000000e+000 - 2.6108040e+002 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 9.5495003e+002 - 9.4006028e+002 - 0.0000000e+000 - 3.5370562e+003 - 3.9566627e+003 - 3.0929742e+003 - 4.9011007e+003 - 1.8903528e+003 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 2.2071740e+003 - 3.9380997e+003 - 2.3659637e+003 - 4.5819484e+003 - 7.8283526e+003 - 8.3395475e+003 - 8.2555568e+003 - 7.4249946e+003 - 7.0053788e+003 - 7.3615225e+003 - 7.8190695e+003 - 7.3652682e+003 - 6.4071854e+003 - 5.6562926e+003 - 5.5819296e+003 - 5.4997161e+003 - 4.3778493e+003 - 4.2852124e+003 - 3.6996386e+003 - 4.1962655e+003 - 5.3572614e+003 - 6.1529617e+003 - 6.9828832e+003 - 6.4557735e+003 - 6.2853946e+003 - 6.7592074e+003 - 6.9318648e+003 - 7.3361642e+003 - 8.6591450e+003 - 6.8195745e+003 - 6.0680803e+003 - 7.6203908e+003 - 7.5128838e+003 - 7.8285502e+003 - 7.9947436e+003 - 8.4166739e+003 - 9.0386918e+003 - 9.6255894e+003 - 9.5458869e+003 - 9.2030754e+003 - 6.6768717e+003 - 6.7033235e+003 - 5.6917905e+003 - 5.4640444e+003 - 5.1686694e+003 - 5.0262107e+003 - 5.7329716e+003 - 7.3859021e+003 - 5.8516838e+003 - 4.5590723e+003 - 2.6981644e+003 - 2.0504300e+003 - 5.3858967e+003 - 4.9134822e+003 - 4.6603703e+003 - 5.9036473e+003 - 7.6810899e+003 - 8.1997977e+003 - 8.0106951e+003 - 8.1596697e+003 - 8.2479784e+003 - 7.9965925e+003 - 7.5024288e+003 - 6.8502876e+003 - 6.0533001e+003 - 6.2002184e+003 - 6.0958282e+003 - 5.6691136e+003 - 4.2681847e+003 - 6.5703859e+003 - 6.8901403e+003 - 7.5029552e+003 - 7.5444821e+003 - 7.3894798e+003 - 7.6846132e+003 - 8.0774156e+003 - 0.0000000e+000 - 0.0000000e+000 - 8.8731706e+002 - 1.2785911e+003 - 3.8026027e+002 - 2.0614489e+003 - 1.6214557e+003 - 7.8504105e+003 - 7.2713330e+003 - 7.2829772e+003 - 7.9999313e+003 - 7.8578117e+003 - 0.0000000e+000 - 2.5632195e+003 - 5.3873109e+003 - 8.6432428e+003 - 7.8386396e+003 - 7.8621048e+003 - 7.0017132e+003 - 6.7518025e+003 - 7.1735400e+003 - 5.4733881e+003 - 4.6497026e+003 - 2.6160870e+003 - 7.0548515e+003 - 8.2487991e+003 - 8.0543010e+003 - 8.3473312e+003 - 8.4405522e+003 - 7.7185410e+003 - 8.9730573e+003 - 8.3117900e+003 - 7.5650959e+003 - 9.3203575e+003 - 1.0766107e+004 - 1.0506120e+004 - 8.2404517e+003 - 9.3112087e+003 - 8.6444914e+003 - 7.9584098e+003 - 7.4190556e+003 - 8.3069102e+003 - 8.7101999e+003 - 9.0762521e+003 - 8.8499354e+003 - 8.7581015e+003 - 8.4550577e+003 - 8.5654165e+003 - 1.0108675e+004 - 1.0153487e+004 - 9.8892965e+003 - 1.0336175e+004 - 1.1190767e+004 - 1.1337847e+004 - 1.0699500e+004 - 1.1279998e+004 - 1.1353812e+004 - 1.1806177e+004 - 1.2046242e+004 - 1.1586266e+004 - 8.1553968e+003 - 9.7107256e+003 - 9.6347892e+003 - 9.8535087e+003 - 1.0241629e+004 - 1.0625407e+004 - 1.0808044e+004 - 1.0408890e+004 - 1.0504415e+004 - 8.0974393e+003 - 8.1872064e+003 - 7.5799017e+003 - 8.3155986e+003 - 6.9353920e+003 - 7.7001135e+003 - 6.2024939e+003 - 4.3010875e+003 - 4.0428118e+003 - 4.6734249e+003 - 4.2398119e+003 - 2.8420659e+003 - 2.3937756e+003 - 2.1968649e+003 - 3.4247098e+003 - 5.7593676e+003 - 6.4822557e+003 - 5.4086234e+003 - 3.5863455e+003 - 6.5186845e+003 - 1.0174982e+004 - 6.2258884e+003 - 6.2735088e+003 - 7.2463818e+003 - 7.8019508e+003 - 8.0914774e+003 - 8.0273003e+003 - 6.5042407e+003 - 7.1130267e+003 - 8.0390038e+003 - 9.1553699e+003 - 9.1063926e+003 - 1.0104636e+004 - 9.7730171e+003 - 1.0835057e+004 - 1.1252954e+004 - 1.0593947e+004 - 1.0261236e+004 - 1.0178175e+004 - 8.0030699e+003 - 7.9421457e+003 - 6.5004948e+003 - 5.0355241e+003 - 4.0448523e+003 - 5.5471107e+003 - 6.1952646e+003 - 6.1735064e+003 - 5.2142062e+003 - 4.2632449e+003 - 5.3247919e+003 - 6.6243626e+003 - 0.0000000e+000 - 1.0626700e+003 - 5.5930531e+003 - 6.6048446e+003 - 6.9432292e+003 - 9.0784543e+003 - 8.8071665e+003 - 4.6108863e+003 - 2.0320442e+003 - 1.7529495e+003 - 4.2818276e+002 - 0.0000000e+000 - 1.4657715e+003 - 2.8733436e+003 - 1.1113761e+003 - 8.2026959e+002 - 3.2031411e+003 - 5.4578303e+003 - 6.1476515e+003 - 5.1612321e+003 - 5.5145236e+003 - 5.8632761e+003 - 5.9789478e+003 - 6.0100731e+003 - 2.8119170e+003 - 1.4753276e+003 - 0.0000000e+000 - 1.1645733e+003 - 3.4029069e+003 - 4.6166264e+003 - 5.3362747e+003 - 5.0644515e+003 - 4.4729336e+003 - 3.1215161e+003 - 2.8050234e+003 - 2.4127796e+003 - 6.2574085e+001 - 0.0000000e+000 - 0.0000000e+000 - 2.8797743e+003 - 4.0707793e+003 - 6.2025962e+003 - 7.3729939e+003 - 8.0943202e+003 - 8.7145014e+003 - 8.0080964e+003 - 8.5488228e+003 - 8.7599510e+003 - 1.1881966e+004 - 1.2007864e+004 - 1.2566659e+004 - 1.2708876e+004 - 1.3455895e+004 - 1.4052810e+004 - 1.3674358e+004 - 1.2998792e+004 - 1.3524610e+004 - 1.4075571e+004 - 1.3970034e+004 - 1.4565383e+004 - 1.1776112e+004 - 1.3784832e+004 - 1.3391414e+004 - 1.4176030e+004 - 1.3156822e+004 - 1.2128830e+004 - 1.1109645e+004 - 9.5154339e+003 - 8.0810609e+003 - 6.7481260e+003 - 6.9190229e+003 - 7.8567433e+003 - 9.5370169e+003 - 1.3076686e+004 - 1.4083823e+004 - 1.3602359e+004 - 1.1822029e+004 - 1.2437776e+004 - 1.2526091e+004 - 1.3312266e+004 - 1.3354330e+004 - 1.3324940e+004 - 1.2930313e+004 - 1.2219589e+004 - 1.0347340e+004 - 1.2503860e+004 - 1.0987501e+004 - 1.0359392e+004 - 8.9019367e+003 - 8.8707211e+003 - 7.9610484e+003 - 7.3082613e+003 - 7.5009937e+003 - 7.5070157e+003 - 6.4086248e+003 - 6.5885396e+003 - 1.0160313e+004 - 7.2605065e+003 - 8.6941476e+003 - 1.1691884e+004 - 1.2858932e+004 - 1.4066665e+004 - 1.4389738e+004 - 1.4553906e+004 - 1.4534616e+004 - 1.4536238e+004 - 1.4532307e+004 - 1.4419266e+004 - 1.3565756e+004 - 1.4536834e+004 - 1.3934880e+004 - 1.2604112e+004 - 1.1548268e+004 - 1.0939456e+004 - 1.0584223e+004 - 1.0371914e+004 - 1.0265391e+004 - 1.0704553e+004 - 1.1566453e+004 - 1.1408185e+004 - 1.2508518e+004 - 1.4352964e+004 - 1.4210721e+004 - 1.4159789e+004 - 1.4148719e+004 - 1.4364776e+004 - 1.4297846e+004 - 1.4465736e+004 - 1.4459171e+004 - 1.4444024e+004 - 1.4095202e+004 - 1.2566492e+004 - 1.1047449e+004 - 1.1939240e+004 - 1.1649965e+004 - 1.2192118e+004 - 1.1706108e+004 - 1.1143421e+004 - 1.0827084e+004 - 1.0681743e+004 - 9.4032011e+003 - 8.4956601e+003 - 8.8424794e+003 - 9.1107555e+003 - 9.1775315e+003 - 9.0893828e+003 - 9.8194462e+003 - 9.9996784e+003 - 9.5577876e+003 - 1.0215585e+004 - 1.0346600e+004 - 1.0159862e+004 - 9.3719283e+003 - 9.8465513e+003 - 9.5547432e+003 - 9.6763764e+003 - 7.2230865e+003 - 7.1347127e+003 - 7.5650666e+003 - 7.3806092e+003 - 8.3559304e+003 - 8.0864770e+003 - 7.6486417e+003 - 6.8091315e+003 - 6.7052913e+003 - 6.2480900e+003 - 6.4172527e+003 - 6.8336305e+003 - 5.0828604e+003 - 5.8992184e+003 - 5.8221517e+003 - 5.4765914e+003 - 4.4040706e+003 - 2.6548177e+003 - 2.7952422e+003 - 1.7821496e+003 - 1.8672479e+003 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 2.4444499e+003 - 3.3633107e+003 - 4.2376038e+003 - 5.7767493e+003 - 6.4128512e+003 - 6.5488570e+003 - 6.5035884e+003 - 6.2188076e+003 - 6.4609732e+003 - 6.8132900e+003 - 7.7303846e+003 - 7.9623514e+003 - 7.9183496e+003 - 6.8808556e+003 - 8.6255178e+003 - 8.6000044e+003 - 8.7458709e+003 - 9.0742932e+003 - 8.6634135e+003 - 8.6738839e+003 - 8.9289682e+003 - 8.8524505e+003 - 8.9733887e+003 - 8.9253610e+003 - 7.2014395e+003 - 6.8398964e+003 - 7.0270889e+003 - 5.3659379e+003 - 5.9222096e+003 - 6.6846536e+003 - 6.5958651e+003 - 7.0643611e+003 - 6.3064577e+003 - 6.5389800e+003 - 6.7931725e+003 - 5.9972918e+003 - 6.4241124e+003 - 5.3563267e+003 - 5.9733319e+003 - 6.2930395e+003 - 5.6327730e+003 - 5.2531108e+003 - 3.7128078e+003 - 4.5538938e+003 - 5.5351034e+003 - 6.0940877e+003 - 7.0292030e+003 - 7.1063534e+003 - 3.8284812e+003 - 3.3488307e+003 - 2.9444053e+003 - 6.4797021e+002 - 2.5122984e+003 - 2.6265758e+003 - 1.8083527e+003 - 3.2267532e+003 - 4.2393282e+003 - 4.0888360e+003 - 2.2768930e+003 - 2.2836113e+003 - 2.5109225e+003 - 3.1859473e+003 - 1.2710600e+003 - 3.1366806e+003 - 4.0157416e+003 - 5.1189722e+003 - 4.4843395e+003 - 1.6144730e+003 - 0.0000000e+000 - 4.5732753e+003 - 5.5599506e+003 - 5.6723745e+003 - 5.6073942e+003 - 6.8107067e+003 - 6.3042161e+003 - 6.0348055e+003 - 6.6608160e+003 - 6.1826184e+003 - 6.3623836e+003 - 6.2256643e+003 - 6.0451574e+003 - 6.2148039e+003 - 6.3462415e+003 - 6.1456435e+003 - 3.5846413e+003 - 2.5739341e+003 - 9.4450583e+002 - 3.0889885e+003 - 4.7997977e+003 - 4.8693042e+003 - 5.1698751e+003 - 5.5960284e+003 - 5.6517376e+003 - 6.1233882e+003 - 6.3522560e+003 - 5.7405811e+003 - 2.5993199e+003 - 5.4638363e+003 - 5.5471539e+003 - 6.4629692e+003 - 6.9976269e+003 - 7.2131170e+003 - 7.1983480e+003 - 7.1404862e+003 - 6.0510438e+003 - 6.9985492e+003 - 7.1085467e+003 - 7.8331410e+003 - 5.8834640e+003 - 6.3762216e+003 - 7.3401193e+003 - 8.8201549e+003 - 9.2383681e+003 - 1.0278857e+004 - 1.1100604e+004 - 1.1966643e+004 - 1.3413119e+004 - 1.3819395e+004 - 1.2424903e+004 - 1.1904587e+004 - 9.0157868e+003 - 1.3279007e+004 - 1.3220215e+004 - 1.4298650e+004 - 1.2617323e+004 - 1.1164287e+004 - 1.0353945e+004 - 1.0130607e+004 - 9.8825520e+003 - 8.5915722e+003 - 6.5857254e+003 - 6.8821880e+003 - 6.7446958e+003 - 6.6509499e+003 - 1.0193421e+004 - 1.1739819e+004 - 1.1535379e+004 - 1.2180969e+004 - 1.3345418e+004 - 1.3615411e+004 - 1.3348182e+004 - 1.1565162e+004 - 1.3698943e+004 - 1.3029112e+004 - 9.9551260e+003 - 1.0763619e+004 - 1.0549552e+004 - 1.1013599e+004 - 1.0364306e+004 - 1.1028403e+004 - 9.5178317e+003 - 9.1667881e+003 - 8.2795001e+003 - 8.1078401e+003 - 8.2202564e+003 - 8.9934709e+003 - 1.0274814e+004 - 8.9252330e+003 - 9.3773803e+003 - 1.1804999e+004 - 1.2113585e+004 - 1.1559838e+004 - 1.0593637e+004 - 1.1287984e+004 - 1.0475303e+004 - 1.0541109e+004 - 1.0908007e+004 - 1.0249198e+004 - 6.1529706e+003 - 6.0935898e+003 - 6.2652916e+003 - 6.5007676e+003 - 7.3582511e+003 - 6.1801771e+003 - 7.0134822e+003 - 3.6288112e+003 - 4.5586934e+003 - 5.5976878e+003 - 6.2029795e+003 - 7.4451693e+003 - 7.3777627e+003 - 6.1712209e+003 - 7.4637218e+003 - 9.2102922e+003 - 1.0274244e+004 - 1.0954009e+004 - 1.0866417e+004 - 1.1324523e+004 - 1.1836344e+004 - 1.1514071e+004 - 1.2535753e+004 - 1.1528844e+004 - 7.7377714e+003 - 8.6786008e+003 - 8.5255593e+003 - 6.1684493e+003 - 4.4285909e+003 - 5.0377109e+003 - 5.4061638e+003 - 5.5049577e+003 - 6.6037472e+003 - 6.9336391e+003 - 6.4109303e+003 - 5.9195186e+003 - 7.0944024e+003 - 7.5582800e+003 - 8.3012190e+003 - 1.1166341e+004 - 1.1299093e+004 - 1.0557624e+004 - 1.0498080e+004 - 1.0785689e+004 - 1.1072526e+004 - 1.0573615e+004 - 9.8143695e+003 - 9.4596037e+003 - 9.3518218e+003 - 9.8870819e+003 - 9.0467028e+003 - 8.7880165e+003 - 9.3161241e+003 - 7.7260787e+003 - 5.7779613e+003 - 4.0173668e+003 - 0.0000000e+000 - 0.0000000e+000 - 2.0349063e+003 - 4.4535083e+003 - 5.3760972e+003 - 4.8468214e+003 - 4.9434670e+003 - 7.2279781e+003 - 7.2882867e+003 - 5.7551835e+003 - 5.7226935e+003 - 5.5472680e+003 - 5.9673108e+003 - 6.4493170e+003 - 6.0596171e+003 - 4.7756622e+003 - 7.5709617e+003 - 9.1739222e+003 - 8.3555257e+003 - 7.2097817e+003 - 4.7081130e+003 - 5.3603382e+003 - 4.1465399e+003 - 4.9537368e+003 - 5.2346486e+003 - 6.0700313e+003 - 3.9568124e+003 - 5.3899786e+003 - 4.9310969e+003 - 5.8825847e+003 - 5.8654853e+003 - 7.8877575e+003 - 8.9138831e+003 - 9.8613647e+003 - 9.5793666e+003 - 9.0664274e+003 - 9.6940153e+003 - 9.7512241e+003 - 9.8523584e+003 - 1.0337797e+004 - 7.5918334e+003 - 8.5713856e+003 - 7.4845574e+003 - 5.6235969e+003 - 5.8237300e+003 - 6.2623144e+003 - 7.1909566e+003 - 5.4336173e+003 - 5.2598829e+003 - 6.8257552e+003 - 7.1605744e+003 - 6.9151052e+003 - 3.5521632e+003 - 3.9139148e+003 - 4.4599288e+003 - 5.9519629e+003 - 6.6357104e+003 - 6.6248078e+003 - 7.4514145e+003 - 7.1365405e+003 - 8.2300653e+003 - 8.3304531e+003 - 8.4534022e+003 - 9.3858545e+003 - 9.1189711e+003 - 1.0007885e+004 - 9.9829376e+003 - 1.0591772e+004 - 1.2813041e+004 - 1.4363911e+004 - 1.4073040e+004 - 1.2315978e+004 - 1.4089120e+004 - 1.4125442e+004 - 1.4033364e+004 - 1.1589221e+004 - 1.0631415e+004 - 1.0954366e+004 - 1.2177986e+004 - 1.3353017e+004 - 1.2761991e+004 - 1.2633734e+004 - 1.2990509e+004 - 1.2728662e+004 - 1.2674109e+004 - 1.2735031e+004 - 1.1685820e+004 - 1.1053840e+004 - 8.5524742e+003 - 9.3246320e+003 - 8.1247159e+003 - 6.0150872e+003 - 5.3598934e+003 - 5.1411224e+003 - 5.5232704e+003 - 7.4889039e+003 - 7.6695049e+003 - 6.6142665e+003 - 6.6109702e+003 - 7.0680483e+003 - 7.3945468e+003 - 6.2695651e+003 - 6.3174936e+003 - 7.7640201e+003 - 8.6138896e+003 - 9.3973384e+003 - 9.6834026e+003 - 1.0240379e+004 - 1.1430414e+004 - 1.2520215e+004 - 1.2417081e+004 - 1.1416349e+004 - 1.1805839e+004 - 1.4246599e+004 - 1.4182743e+004 - 1.4032671e+004 - 1.4089797e+004 - 1.4416118e+004 - 1.4075613e+004 - 1.4012769e+004 - 1.3434182e+004 - 1.2431042e+004 - 1.0230039e+004 - 9.5355291e+003 - 6.7898887e+003 - 4.4146866e+003 - 5.1461040e+003 - 5.9618071e+003 - 6.3559881e+003 - 6.7300501e+003 - 8.0858920e+003 - 1.0094225e+004 - 1.0624917e+004 - 1.0802007e+004 - 1.0609083e+004 - 1.0119989e+004 - 7.9402254e+003 - 8.6304183e+003 - 7.2813145e+003 - 5.3816246e+003 - 4.0770919e+003 - 3.4348965e+003 - 4.9313878e+003 - 5.7502706e+003 - 6.7566936e+003 - 7.7965326e+003 - 8.3827771e+003 - 7.2951984e+003 - 5.4848842e+003 - 5.9779613e+003 - 7.3257492e+003 - 1.1081163e+004 - 1.0688916e+004 - 1.1044403e+004 - 1.1096059e+004 - 1.1524599e+004 - 1.1097401e+004 - 1.0486339e+004 - 1.0858550e+004 - 1.0406341e+004 - 8.9542643e+003 - 9.9270114e+003 - 9.1880195e+003 - 8.5413152e+003 - 8.1198252e+003 - 7.6858942e+003 - 6.4090910e+003 - 6.1523516e+003 - 6.8745151e+003 - 7.1066583e+003 - 6.7331301e+003 - 5.5979951e+003 - 1.7755961e+003 - 1.3562948e+002 - 0.0000000e+000 - 1.6042664e+003 - 2.9107272e+003 - 4.3536463e+003 - 5.5000389e+003 - 5.6206158e+003 - 6.1148893e+003 - 6.7583248e+003 - 6.8001422e+003 - 7.0468471e+003 - 7.0191319e+003 - 7.7175985e+003 - 3.9444491e+003 - 2.5202439e+003 - 5.1432144e+003 - 6.1400909e+003 - 6.1928574e+003 - 6.0092150e+003 - 5.2317011e+003 - 4.3729533e+003 - 5.5194336e+003 - 5.6428980e+003 - 9.1242384e+001 - 0.0000000e+000 - 0.0000000e+000 - 1.4242304e+003 - 2.7538591e+003 - 2.0447649e+003 - 2.2813065e+003 - 6.6479513e+002 - 0.0000000e+000 - 2.3218507e+003 - 2.9178129e+003 - 1.0075762e+003 - 7.6042190e+002 - 2.3123902e+003 - 0.0000000e+000 - 1.6249207e+003 - 6.8478921e+003 - 6.2787220e+003 - 6.5264535e+003 - 5.5105023e+003 - 6.0055324e+003 - 8.3570253e+003 - 9.8056794e+003 - 9.9738140e+003 - 1.1661279e+004 - 1.2647040e+004 - 1.2696660e+004 - 1.3074444e+004 - 1.2692282e+004 - 1.3493621e+004 - 1.2119390e+004 - 1.2520317e+004 - 1.1427051e+004 - 1.1733847e+004 - 1.2096095e+004 - 1.2298656e+004 - 1.0735676e+004 - 1.0577124e+004 - 1.1337851e+004 - 1.2168440e+004 - 1.3819441e+004 - 1.3409225e+004 - 1.4338177e+004 - 1.3999631e+004 - 1.2840154e+004 - 1.3407145e+004 - 1.4147617e+004 - 1.2962800e+004 - 9.2962153e+003 - 1.0163658e+004 - 1.0560186e+004 - 1.1198147e+004 - 1.1848334e+004 - 1.1691601e+004 - 1.3411732e+004 - 1.4166846e+004 - 1.4250871e+004 - 1.4426263e+004 - 1.4451948e+004 - 1.4271562e+004 - 1.1220364e+004 - 1.4294909e+004 - 1.4304854e+004 - 1.3861962e+004 - 1.3402458e+004 - 1.2674533e+004 - 9.1137565e+003 - 7.6905048e+003 - 6.1986860e+003 - 7.8157356e+003 - 8.0786646e+003 - 7.7383208e+003 - 8.5632873e+003 - 8.9716496e+003 - 8.5197489e+003 - 6.6292700e+003 - 6.5788965e+003 - 5.8446458e+003 - 5.1343637e+003 - 3.7804675e+003 - 3.6646527e+003 - 4.5559739e+003 - 4.0801894e+003 - 3.0889203e+003 - 5.1245560e+003 - 5.7252473e+003 - 4.7450424e+003 - 3.4205074e+003 - 1.5560902e+002 - 1.3233490e+003 - 2.4701306e+003 - 5.0005302e+003 - 5.3416377e+003 - 4.7058088e+003 - 6.2278430e+003 - 7.6958709e+003 - 7.4274587e+003 - 5.5337376e+003 - 5.4062043e+003 - 7.6138492e+003 - 7.9586383e+003 - 9.5649518e+003 - 1.0415792e+004 - 1.1005070e+004 - 1.0685923e+004 - 1.1177141e+004 - 1.1048467e+004 - 1.0264085e+004 - 6.1634786e+003 - 5.3477874e+003 - 4.6652114e+003 - 5.2390204e+003 - 6.4925152e+003 - 7.2199672e+003 - 6.9288452e+003 - 6.2525698e+003 - 6.1091925e+003 - 5.8054677e+003 - 6.5173447e+003 - 7.8372233e+003 - 3.7428617e+003 - 5.5320730e+003 - 5.1086385e+003 - 6.3817160e+003 - 6.3764770e+003 - 6.3543721e+003 - 5.8388769e+003 - 5.4736171e+003 - 5.3580691e+003 - 5.8288554e+003 - 5.4950842e+003 - 4.9193925e+003 - 1.7864913e+003 - 1.2784254e+003 - 0.0000000e+000 - 2.6019563e+003 - 5.0684133e+003 - 5.0112085e+003 - 6.7487924e+003 - 6.7260251e+003 - 7.1570118e+003 - 6.6810402e+003 - 6.7275574e+003 - 6.4126151e+003 - 8.7849196e+003 - 6.5088512e+003 - 5.4787562e+003 - 5.9438236e+003 - 5.6150140e+003 - 5.3430394e+003 - 5.4510156e+003 - 5.4051302e+003 - 5.3003643e+003 - 3.4611066e+003 - 1.8131724e+003 - 3.8491530e+003 - 5.9584132e+003 - 6.5455652e+003 - 7.2901078e+003 - 8.4735461e+003 - 7.1315714e+003 - 7.1478182e+003 - 7.5210884e+003 - 7.9298573e+003 - 8.7818406e+003 - 9.2178110e+003 - 9.6075744e+003 - 1.0212956e+004 - 1.1103043e+004 - 1.0080205e+004 - 9.6075676e+003 - 9.6804039e+003 - 9.5340997e+003 - 9.0131537e+003 - 8.6393029e+003 - 8.3575252e+003 - 9.5716271e+003 - 9.7100297e+003 - 9.9368813e+003 - 9.6374942e+003 - 9.8624549e+003 - 1.1571313e+004 - 1.1356635e+004 - 1.0260043e+004 - 1.0080624e+004 - 1.0337064e+004 - 1.0286647e+004 - 1.0491590e+004 - 9.8008242e+003 - 9.5531995e+003 - 8.7477598e+003 - 9.4514705e+003 - 1.1701180e+004 - 1.0124664e+004 - 1.0049546e+004 - 9.9697416e+003 - 9.2959513e+003 - 9.3465605e+003 - 9.9154663e+003 - 1.0888258e+004 - 1.1299865e+004 - 1.1313257e+004 - 1.1241781e+004 - 1.0626215e+004 - 1.1346379e+004 - 1.2023596e+004 - 1.2416685e+004 - 1.0951767e+004 - 9.5043326e+003 - 9.8288361e+003 - 1.0452497e+004 - 9.9805882e+003 - 1.0079133e+004 - 1.0418755e+004 - 1.0844063e+004 - 1.0727602e+004 - 9.1250847e+003 - 9.1223078e+003 - 8.5980677e+003 - 8.9959900e+003 - 1.0241263e+004 - 1.1460452e+004 - 1.2588604e+004 - 1.3015737e+004 - 1.4360157e+004 - 1.4146275e+004 - 1.4242740e+004 - 1.4137522e+004 - 1.1699640e+004 - 1.2905070e+004 - 1.2927750e+004 - 1.2549948e+004 - 1.3005659e+004 - 1.2005557e+004 - 1.2019268e+004 - 1.1641720e+004 - 1.1393746e+004 - 1.1255862e+004 - 1.0498864e+004 - 9.3811903e+003 - 8.0861282e+003 - 9.4912147e+003 - 9.9878486e+003 - 9.9054268e+003 - 1.1843300e+004 - 1.1801218e+004 - 1.2287729e+004 - 1.0696577e+004 - 1.1750753e+004 - 1.1491259e+004 - 1.0603689e+004 - 1.0257701e+004 - 9.3381541e+003 - 9.9545168e+003 - 8.8723682e+003 - 6.8255067e+003 - 2.0467247e+003 - 3.2452102e+003 - 3.2844154e+003 - 2.8742408e+003 - 2.4197311e+003 - 3.1021524e+003 - 5.1387699e+003 - 6.1293638e+003 - 3.2921338e+003 - 2.6175485e+003 - 9.5475362e+002 - 1.7208089e+003 - 9.8905437e+002 - 4.8163437e+003 - 5.6220877e+003 - 6.4602328e+003 - 6.3463214e+003 - 6.0038047e+003 - 5.6737485e+003 - 5.7971183e+003 - 7.0524877e+003 - 7.3112267e+003 - 5.5435371e+003 - 5.5898473e+003 - 6.8604283e+003 - 7.5036297e+003 - 8.9290214e+003 - 1.0678985e+004 - 1.0005482e+004 - 9.0648224e+003 - 8.2275582e+003 - 6.5318957e+003 - 8.0080259e+003 - 7.2424398e+003 - 6.1778658e+003 - 8.6715757e+003 - 9.9686107e+003 - 1.0526597e+004 - 1.0522531e+004 - 1.0278738e+004 - 9.3666762e+003 - 8.2623541e+003 - 7.9241349e+003 - 7.4759145e+003 - 7.4522610e+003 - 7.1598904e+003 - 6.2912572e+003 - 6.6056157e+003 - 7.7455674e+003 - 1.0291074e+004 - 9.8840953e+003 - 8.7280162e+003 - 7.5299759e+003 - 7.4856318e+003 - 7.4214820e+003 - 7.9481593e+003 - 8.8461479e+003 - 7.5754030e+003 - 9.3611912e+003 - 1.0200861e+004 - 1.1782434e+004 - 1.3913160e+004 - 1.4102198e+004 - 1.4158658e+004 - 1.4165130e+004 - 1.4568294e+004 - 1.1465353e+004 - 5.5524416e+003 - 6.3579182e+003 - 6.5862465e+003 - 7.7525256e+003 - 8.2555798e+003 - 6.4698932e+003 - 6.0917484e+003 - 7.4246595e+003 - 6.0211783e+003 - 6.5209044e+003 - 6.7477006e+003 - 6.3979170e+003 - 6.2706980e+003 - 1.1071541e+004 - 7.9770924e+003 - 8.9163736e+003 - 1.1779147e+004 - 1.3465730e+004 - 1.3905087e+004 - 1.3506625e+004 - 1.3933348e+004 - 1.4048030e+004 - 1.3813297e+004 - 1.3473137e+004 - 1.3075889e+004 - 1.1855773e+004 - 1.1373837e+004 - 1.1865900e+004 - 1.3738178e+004 - 1.3035083e+004 - 1.0516397e+004 - 1.1929626e+004 - 1.2500826e+004 - 1.2104122e+004 - 1.3187354e+004 - 1.2531593e+004 - 1.2700832e+004 - 8.5353899e+003 - 1.0701652e+004 - 1.0242972e+004 - 1.2172264e+004 - 1.1418584e+004 - 1.1862941e+004 - 1.2096041e+004 - 1.3026156e+004 - 1.2785229e+004 - 1.3089958e+004 - 1.2466738e+004 - 1.3257661e+004 - 1.0696104e+004 - 1.1611353e+004 - 1.1171632e+004 - 1.0225592e+004 - 1.0164262e+004 - 1.0155083e+004 - 1.0011354e+004 - 9.5255369e+003 - 8.7373264e+003 - 8.5013683e+003 - 8.7933974e+003 - 8.4143412e+003 - 7.0507934e+003 - 6.3916097e+003 - 6.1650165e+003 - 7.3843829e+003 - 7.2161497e+003 - 7.1621673e+003 - 7.0889671e+003 - 6.4102311e+003 - 5.9403004e+003 - 5.6635497e+003 - 4.9922642e+003 - 3.6887101e+003 - 4.8210330e+003 - 3.7355896e+003 - 5.3336306e+003 - 5.8268423e+003 - 6.8754428e+003 - 6.6773179e+003 - 6.5487486e+003 - 6.9358329e+003 - 7.5025574e+003 - 7.1960818e+003 - 7.4126055e+003 - 6.9100766e+003 - 5.6613124e+003 - 4.8489918e+003 - 1.7269751e+003 - 3.0279125e+003 - 5.3521975e+003 - 4.2586573e+003 - 4.9853727e+003 - 5.9623042e+003 - 6.6820201e+003 - 7.0023943e+003 - 6.8191137e+003 - 6.8626877e+003 - 5.7127696e+003 - 5.7708439e+003 - 6.1544987e+003 - 7.3928439e+003 - 8.0067932e+003 - 8.0691231e+003 - 8.5611692e+003 - 8.9560219e+003 - 9.2034840e+003 - 8.3471842e+003 - 7.9740992e+003 - 7.9022327e+003 - 9.7158311e+003 - 8.6733999e+003 - 8.0950930e+003 - 1.0467051e+004 - 1.1002515e+004 - 1.1893679e+004 - 1.2190123e+004 - 1.2543451e+004 - 1.2961643e+004 - 1.3331308e+004 - 1.3194671e+004 - 1.3149113e+004 - 1.3622721e+004 - 1.4000775e+004 - 1.4148801e+004 - 1.4243414e+004 - 1.4040078e+004 - 1.3401777e+004 - 1.2793529e+004 - 1.2709770e+004 - 1.2111972e+004 - 1.1556052e+004 - 1.0353812e+004 - 1.0196181e+004 - 1.1557895e+004 - 8.3720478e+003 - 7.2123969e+003 - 6.4660890e+003 - 5.8333630e+003 - 4.5341357e+003 - 3.1885234e+003 - 3.0965204e+003 - 1.8477836e+003 - 1.5328572e+003 - 0.0000000e+000 - 0.0000000e+000 - 6.0856256e+003 - 7.7175818e+003 - 8.4323380e+003 - 7.3505589e+003 - 5.6816125e+003 - 6.8651622e+003 - 6.7632769e+003 - 6.4644485e+003 - 6.6207425e+003 - 7.6519791e+003 - 8.0228471e+003 - 8.4057987e+003 - 7.2619523e+003 - 6.5469679e+003 - 6.6473868e+003 - 7.7771868e+003 - 6.7019790e+003 - 7.5136010e+003 - 7.7771910e+003 - 7.4751975e+003 - 7.1655259e+003 - 7.5865310e+003 - 7.3629939e+003 - 7.9053798e+003 - 5.7023132e+003 - 4.1720901e+003 - 5.7631431e+003 - 3.4444684e+003 - 2.8333692e+002 - 0.0000000e+000 - 3.2311695e+003 - 5.6239611e+003 - 6.8355006e+003 - 6.7346058e+003 - 6.3348782e+003 - 6.4251261e+003 - 7.7023908e+003 - 7.5435400e+003 - 9.3690638e+003 - 8.5957303e+003 - 7.7655711e+003 - 7.5134421e+003 - 7.4827400e+003 - 6.8404314e+003 - 5.8673574e+003 - 5.5755444e+003 - 5.9534875e+003 - 6.0340229e+003 - 5.6055019e+003 - 6.2848840e+003 - 7.8403307e+003 - 5.8927328e+003 - 5.3340355e+003 - 4.7376558e+003 - 2.9155967e+003 - 6.7248136e+002 - 0.0000000e+000 - 3.9756141e+002 - 1.9913700e+003 - 3.9429972e+003 - 2.7672044e+003 - 3.9120234e+001 - 2.8866462e+003 - 6.5069457e+003 - 8.3936392e+003 - 8.8051260e+003 - 9.3150939e+003 - 9.0347247e+003 - 8.8919679e+003 - 9.3955879e+003 - 1.0137611e+004 - 1.1213669e+004 - 1.0142556e+004 - 1.3015694e+004 - 1.2949099e+004 - 1.2639363e+004 - 1.1081784e+004 - 9.5317693e+003 - 8.4108351e+003 - 8.1734045e+003 - 8.9084618e+003 - 1.0104325e+004 - 9.2020499e+003 - 9.3650406e+003 - 1.1015573e+004 - 1.2203650e+004 - 1.2454921e+004 - 1.2136892e+004 - 1.3425550e+004 - 1.3380674e+004 - 1.3172389e+004 - 1.3214882e+004 - 1.3231588e+004 - 1.3157438e+004 - 1.2316119e+004 - 1.2244812e+004 - 1.1893163e+004 - 1.2629505e+004 - 1.2627421e+004 - 1.1956578e+004 - 1.1200288e+004 - 1.0308203e+004 - 9.3572223e+003 - 8.7986871e+003 - 8.7686771e+003 - 7.1976928e+003 - 7.9248401e+003 - 6.6938447e+003 - 6.8421381e+003 - 6.7646100e+003 - 6.8972671e+003 - 7.5058083e+003 - 6.9228792e+003 - 6.3113691e+003 - 6.1901866e+003 - 7.0202441e+003 - 7.0869902e+003 - 5.9566826e+003 - 6.6213433e+003 - 6.6805158e+003 - 4.3547422e+003 - 6.2437921e+003 - 5.7185409e+003 - 5.1433910e+003 - 5.4420468e+003 - 5.6063917e+003 - 5.1371758e+003 - 5.5334798e+003 - 5.2412203e+003 - 4.2829118e+003 - 4.1111588e+003 - 3.3603315e+003 - 1.4546636e+003 - 1.4753341e+003 - 1.8736653e+003 - 3.1513548e+003 - 2.4947502e+003 - 8.1765964e+002 - 0.0000000e+000 - 3.1628737e+002 - 1.4731189e+003 - 1.9561781e+002 - 5.4329983e+002 - 3.3794641e+003 - 4.3948104e+003 - 6.4756637e+003 - 7.2252005e+003 - 7.8634085e+003 - 7.1527382e+003 - 7.1463869e+003 - 6.7356234e+003 - 5.9201916e+003 - 5.3802162e+003 - 5.4066411e+003 - 5.4605831e+003 - 5.3928410e+003 - 6.6368335e+003 - 5.3446629e+003 - 4.3842820e+003 - 6.7018121e+003 - 6.0942797e+003 - 5.6988930e+003 - 5.6471843e+003 - 6.3339357e+003 - 6.9601090e+003 - 6.4297262e+003 - 6.2285395e+003 - 6.8991757e+003 - 6.8831852e+003 - 7.0522489e+003 - 7.6735353e+003 - 7.4946333e+003 - 7.0111891e+003 - 7.0719382e+003 - 6.4044419e+003 - 5.7499819e+003 - 5.3874755e+003 - 5.7531262e+003 - 6.2837777e+003 - 5.3746341e+003 - 7.2295377e+003 - 6.7849463e+003 - 5.5859468e+003 - 6.8770004e+003 - 6.9349532e+003 - 6.1946925e+003 - 6.4860080e+003 - 7.3195551e+003 - 7.4739297e+003 - 6.7558567e+003 - 6.6256697e+003 - 6.8173731e+003 - 7.1520776e+003 - 7.7239487e+003 - 7.1591009e+003 - 7.3949447e+003 - 7.1517824e+003 - 6.6525829e+003 - 6.3600641e+003 - 5.7020142e+003 - 5.0247887e+003 - 5.6059380e+003 - 5.4208366e+003 - 4.1454077e+003 - 5.8279458e+003 - 5.3459109e+003 - 3.5919892e+003 - 2.4283349e+003 - 2.9171519e+003 - 2.5236476e+003 - 3.1207679e+003 - 5.1609892e+003 - 5.9772193e+003 - 5.4471092e+003 - 4.9726340e+003 - 5.4033178e+003 - 4.4045428e+003 - 5.1217141e+003 - 5.6081278e+003 - 5.9255166e+003 - 6.3887153e+003 - 5.2974961e+003 - 4.3325883e+003 - 5.0084821e+003 - 4.2860880e+003 - 1.4428545e+003 - 0.0000000e+000 - 0.0000000e+000 - 2.6847624e+003 - 1.6929017e+003 - 1.7015647e+003 - 4.7194808e+003 - 1.6010556e+002 - 2.0684930e+002 - 0.0000000e+000 - 9.8498008e+002 - 1.1500522e+003 - 2.1509832e+003 - 2.9809805e+003 - 4.6906098e+003 - 2.0618810e+003 - 2.5563985e+003 - 3.1613569e+003 - 5.0542917e+003 - 6.1138826e+003 - 6.0037971e+003 - 6.6370659e+003 - 6.7404988e+003 - 6.8883136e+003 - 6.4440831e+003 - 4.4293969e+003 - 2.5157663e+003 - 8.2078022e+002 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 1.2629965e+003 - 1.6348842e+003 - 1.4696652e+003 - 2.0216463e+003 - 2.9282693e+003 - 4.6233346e+003 - 5.1797581e+003 - 4.0452479e+003 - 4.5469460e+003 - 6.6180962e+003 - 7.3562709e+003 - 6.8527974e+003 - 7.1642161e+003 - 6.6491125e+003 - 6.0071041e+003 - 6.0427055e+003 - 6.3061845e+003 - 6.5424780e+003 - 5.8198567e+003 - 5.5646029e+003 - 5.2141471e+003 - 3.5006897e+003 - 3.9349788e+003 - 4.5350766e+003 - 4.9754126e+003 - 5.9381811e+003 - 5.7298213e+003 - 4.3764239e+003 - 3.9046456e+003 - 4.7244611e+003 - 3.0394915e+003 - 5.4050673e+003 - 5.2420422e+003 - 5.7328425e+003 - 6.8474918e+003 - 7.3277473e+003 - 7.0234694e+003 - 5.6626872e+003 - 5.5440130e+003 - 5.0373464e+003 - 3.8189696e+003 - 1.2819947e+003 - 1.1193130e+003 - 3.2716457e+003 - 1.2114837e+003 - 1.2363369e+003 - 1.7868256e+003 - 5.8966503e+003 - 6.3040690e+003 - 6.5548409e+003 - 6.0545161e+003 - 6.7619804e+003 - 6.8052642e+003 - 6.7290090e+003 - 6.6268361e+003 - 7.5249381e+003 - 7.0614225e+003 - 6.2391479e+003 - 5.4482959e+003 - 5.9830959e+003 - 5.4034926e+003 - 5.4341370e+003 - 5.1748092e+003 - 5.6930914e+003 - 5.6282818e+003 - 5.0665427e+003 - 5.0923014e+003 - 7.7845481e+003 - 6.3461004e+003 - 5.8966994e+003 - 5.3817612e+003 - 7.1757568e+003 - 7.5880187e+003 - 7.9628631e+003 - 7.9109051e+003 - 7.8051045e+003 - 8.0569675e+003 - 8.0583386e+003 - 7.7833438e+003 - 6.4726596e+003 - 6.3669373e+003 - 7.1941393e+003 - 6.0032010e+003 - 6.6578199e+003 - 6.8981226e+003 - 6.5066082e+003 - 6.7767023e+003 - 6.5407841e+003 - 6.1920033e+003 - 5.7720228e+003 - 5.6160769e+003 - 6.2813778e+003 - 5.7778780e+003 - 4.2056496e+003 - 4.8976687e+003 - 5.4001703e+003 - 5.5266017e+003 - 4.8179281e+003 - 1.6186274e+003 - 1.4140156e+003 - 2.2384059e+003 - 4.5070155e+003 - 5.1041150e+003 - 4.9297212e+003 - 5.9568958e+003 - 7.0551771e+003 - 7.0182106e+003 - 8.2167292e+003 - 7.5298268e+003 - 7.2007881e+003 - 6.4093961e+003 - 6.3768111e+003 - 4.8989579e+003 - 3.4140760e+003 - 1.1862434e+003 - 5.4902117e+003 - 5.0564323e+003 - 4.1143366e+003 - 3.8700616e+003 - 4.0314558e+003 - 5.1110656e+003 - 4.3260579e+003 - 5.5224389e+003 - 4.6664149e+003 - 5.7148885e+003 - 6.1185484e+003 - 6.0828831e+003 - 7.4441446e+003 - 8.0745498e+003 - 8.1886847e+003 - 7.7038662e+003 - 6.3823664e+003 - 6.8949858e+003 - 5.8904503e+003 - 6.5400522e+003 - 5.2945407e+003 - 5.7014842e+003 - 5.7832989e+003 - 5.6279009e+003 - 5.6479845e+003 - 5.5947674e+003 - 3.7591365e+003 - 2.2404735e+003 - 4.0842101e+003 - 6.5154913e+003 - 7.5129659e+003 - 8.2986438e+003 - 8.7714860e+003 - 8.9753103e+003 - 7.5533746e+003 - 6.7391163e+003 - 6.0783802e+003 - 6.7927731e+003 - 8.0082974e+003 - 7.4411921e+003 - 8.0062390e+003 - 7.7523470e+003 - 7.5746421e+003 - 6.5309933e+003 - 6.0644969e+003 - 6.3111076e+003 - 5.5924293e+003 - 5.6003373e+003 - 5.7491696e+003 - 6.0111737e+003 - 4.5828826e+003 - 2.1131576e+003 - 4.2724522e+003 - 3.9074538e+003 - 5.5211476e+003 - 5.8780455e+003 - 6.4156199e+003 - 6.2099804e+003 - 5.3523288e+003 - 4.6574471e+003 - 1.4440184e+003 - 1.5581470e+003 - 9.2609764e+002 - 1.1419822e+003 - 2.2756133e+003 - 5.3081573e+003 - 5.1707306e+003 - 5.5734725e+003 - 4.4890838e+003 - 3.4739327e+003 - 3.8113807e+003 - 4.5561658e+003 - 4.3859226e+002 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 9.9445126e+002 - 3.0877438e+003 - 4.1360264e+003 - 4.9050481e+003 - 3.1701019e+003 - 2.0889323e+003 - 1.3595259e+003 - 0.0000000e+000 - 6.2171268e+002 - 1.4059197e+002 - 0.0000000e+000 - 1.6166397e+003 - 7.7659513e+003 - 6.1965499e+003 - 5.9305908e+003 - 6.1478009e+003 - 5.4262135e+003 - 6.0231700e+003 - 5.9973827e+003 - 5.3999670e+003 - 5.5996362e+003 - 2.2963567e+003 - 1.6861951e+003 - 8.7962968e+002 - 1.0721115e+003 - 3.0830485e+003 - 3.3150742e+003 - 3.7337179e+003 - 4.6907809e+003 - 4.8529294e+003 - 4.4349967e+003 - 2.4788204e+003 - 4.9028505e+003 - 5.9014371e+003 - 6.3237684e+003 - 5.2311981e+003 - 5.5068333e+003 - 5.7913854e+003 - 5.9287723e+003 - 5.4825711e+003 - 5.5425848e+003 - 6.1923594e+003 - 7.1284284e+003 - 1.9029299e+003 - 2.9779409e+003 - 3.8533472e+003 - 3.0121360e+003 - 4.4596195e+003 - 5.3600812e+003 - 5.4191545e+003 - 5.2210800e+003 - 5.1841177e+003 - 5.2667045e+003 - 5.7230346e+003 - 6.0235878e+003 - 5.1593669e+003 - 5.4819734e+003 - 6.5482079e+003 - 7.2227162e+003 - 7.7241498e+003 - 8.0096402e+003 - 9.5184703e+003 - 9.9312287e+003 - 1.0126630e+004 - 1.0026173e+004 - 8.9253352e+003 - 8.0347065e+003 - 8.2601065e+003 - 8.4570318e+003 - 8.2739836e+003 - 7.7423649e+003 - 7.6658436e+003 - 6.6431721e+003 - 5.9534412e+003 - 5.6552223e+003 - 5.2838699e+003 - 5.3838761e+003 - 7.3628541e+003 - 6.1463259e+003 - 5.1333730e+003 - 5.4215809e+003 - 6.0809128e+003 - 6.1497039e+003 - 6.4129313e+003 - 7.2086361e+003 - 6.6693384e+003 - 6.5827846e+003 - 6.7676094e+003 - 6.4582399e+003 - 5.9693084e+003 - 5.5667379e+003 - 6.3917989e+003 - 4.6197094e+003 - 2.0582144e+003 - 2.7739645e+003 - 3.9116055e+003 - 4.0103260e+003 - 2.2206564e+003 - 5.7510759e+002 - 9.4293667e+002 - 0.0000000e+000 - 1.7840079e+002 - 1.9056219e+003 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 2.8164920e+002 - 1.3748946e+003 - 3.3948110e+003 - 4.1770420e+003 - 3.9842667e+003 - 5.4345177e+003 - 6.5563901e+003 - 5.6211929e+003 - 5.3348421e+003 - 5.0284895e+003 - 4.7485429e+003 - 3.1884757e+003 - 1.2273884e+003 - 5.5059316e+002 - 0.0000000e+000 - 1.2339220e+001 - 0.0000000e+000 - 1.7924013e+002 - 2.0563663e+002 - 0.0000000e+000 - 2.0629046e+003 - 1.0814038e+003 - 3.0227182e+003 - 3.8368946e+003 - 4.2333775e+003 - 5.2021061e+003 - 8.0817691e+003 - 7.4449576e+003 - 7.2705424e+003 - 6.8208161e+003 - 6.1957500e+003 - 6.0151908e+003 - 6.4278665e+003 - 6.5942335e+003 - 4.4702894e+003 - 5.3068886e+003 - 6.2331258e+003 - 5.7593768e+003 - 5.1854448e+003 - 5.6538226e+003 - 6.1359739e+003 - 5.7940790e+003 - 5.3278138e+003 - 5.6586897e+003 - 6.5798191e+003 - 6.1394346e+003 - 6.3416878e+003 - 6.1090622e+003 - 6.6410843e+003 - 7.4290954e+003 - 7.8466318e+003 - 7.6127870e+003 - 6.6080198e+003 - 4.9226787e+003 - 5.6649077e+003 - 6.2347282e+003 - 6.7926851e+003 - 7.2331258e+003 - 6.2577961e+003 - 5.6694478e+003 - 5.4949786e+003 - 6.7775967e+003 - 8.4277524e+003 - 8.9116380e+003 - 9.2928723e+003 - 8.8488694e+003 - 8.8803614e+003 - 8.8003555e+003 - 1.0417705e+004 - 7.0185104e+003 - 7.5979808e+003 - 7.7078957e+003 - 9.3302393e+003 - 9.4029060e+003 - 9.2390964e+003 - 9.4659933e+003 - 8.9856114e+003 - 8.1719434e+003 - 5.9918342e+003 - 6.6261480e+003 - 6.7999154e+003 - 7.1308364e+003 - 7.3128113e+003 - 7.5193912e+003 - 8.5210706e+003 - 9.3254540e+003 - 1.0732447e+004 - 1.2434714e+004 - 1.2000442e+004 - 1.3016679e+004 - 1.3095939e+004 - 1.3237607e+004 - 1.3030819e+004 - 6.4853805e+003 - 7.4253209e+003 - 8.0817879e+003 - 8.1700274e+003 - 8.8228409e+003 - 8.1720795e+003 - 6.9504931e+003 - 6.4051843e+003 - 5.3577722e+003 - 2.4575940e+003 - 1.1739589e+003 - 2.4555003e+003 - 2.0589599e+003 - 1.6405148e+003 - 3.3450473e+003 - 4.3305585e+003 - 5.5395649e+003 - 4.9029067e+003 - 1.7796430e+003 - 0.0000000e+000 - 2.0959832e+003 - 2.9977463e+003 - 1.9019925e+003 - 2.2605578e+003 - 0.0000000e+000 - 3.1871903e+003 - 2.5507246e+003 - 3.8058820e+003 - 2.5641560e+003 - 2.0218466e+003 - 7.7664404e+002 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 2.5978085e+003 - 5.6850570e+003 - 2.8738494e+003 - 1.1115691e+003 - 6.9749098e+002 - 1.2981572e+002 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 3.0570176e+003 - 5.5122908e+003 - 3.5197659e+002 - 9.6677682e+002 - 9.5143080e+002 - 1.3016139e+003 - 5.3023113e+003 - 6.9934825e+003 - 5.4087831e+003 - 5.2674010e+003 - 5.6089676e+003 - 5.4951330e+003 - 5.6734416e+003 - 4.9210630e+003 - 5.1550331e+003 - 3.1524421e+003 - 4.5368092e+003 - 5.0268175e+003 - 5.0464719e+003 - 5.6254666e+003 - 6.1402536e+003 - 5.8269427e+003 - 7.2813390e+003 - 7.4501963e+003 - 6.8797458e+003 - 6.8003631e+003 - 5.2229272e+003 - 6.2046183e+003 - 6.0118434e+003 - 5.8835168e+003 - 5.6148331e+003 - 5.5574420e+003 - 5.7094511e+003 - 5.2777154e+003 - 3.1803041e+003 - 1.5118207e+003 - 0.0000000e+000 - 5.9225056e+002 - 7.2948423e+003 - 8.3644402e+003 - 8.1247348e+003 - 7.5153621e+003 - 6.1456482e+003 - 7.8781595e+003 - 4.8661157e+003 - 2.7730139e+003 - 0.0000000e+000 - 1.3692556e+003 - 4.7206366e+002 - 3.5031926e+002 - 3.0979744e+003 - 5.2077832e+003 - 4.8174986e+003 - 6.6942692e+003 - 5.6920756e+003 - 5.3744609e+003 - 5.5210659e+003 - 5.4395164e+003 - 5.3927569e+003 - 5.1274323e+003 - 5.0898692e+003 - 3.9406011e+003 - 5.2050328e+003 - 4.8321027e+003 - 1.9695651e+003 - 3.8724860e+003 - 5.4854755e+003 - 5.7334233e+003 - 4.3493759e+003 - 4.1830491e+003 - 5.2125472e+003 - 5.2771030e+003 - 5.4501323e+003 - 4.3940276e+003 - 5.4322602e+003 - 6.5893793e+003 - 6.8160553e+003 - 7.6479811e+003 - 7.9050673e+003 - 8.1022775e+003 - 7.9983864e+003 - 8.2061460e+003 - 8.2638857e+003 - 9.0189938e+003 - 9.6504657e+003 - 1.0005628e+004 - 1.1412526e+004 - 1.2164635e+004 - 1.1687337e+004 - 1.2602792e+004 - 1.3297914e+004 - 1.3689648e+004 - 1.4151800e+004 - 1.4207331e+004 - 1.4610773e+004 - 1.4957426e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4696926e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4316020e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4662642e+004 - 1.4954699e+004 - 1.4194284e+004 - 1.4767609e+004 - 1.4952645e+004 - 1.4018945e+004 - 1.4534642e+004 - 1.4448436e+004 - 1.4181473e+004 - 1.3978627e+004 - 1.3222085e+004 - 1.3411780e+004 - 1.2471997e+004 - 1.1708367e+004 - 1.0580281e+004 - 1.1003804e+004 - 1.0877809e+004 - 1.0322387e+004 - 9.5954167e+003 - 9.1729322e+003 - 9.1473957e+003 - 8.5763077e+003 - 7.8103292e+003 - 6.9858035e+003 - 6.1168932e+003 - 6.3702576e+003 - 5.9672019e+003 - 6.5223330e+003 - 6.6260396e+003 - 6.9229174e+003 - 7.2464936e+003 - 6.8647237e+003 - 7.1675063e+003 - 6.0360554e+003 - 6.1175536e+003 - 6.7336623e+003 - 5.7560299e+003 - 5.8602074e+003 - 7.0873941e+003 - 6.7609128e+003 - 6.7921665e+003 - 6.8702686e+003 - 8.4752600e+003 - 8.1463933e+003 - 9.0246364e+003 - 9.4597181e+003 - 1.0318698e+004 - 1.0967717e+004 - 1.2101441e+004 - 1.2153791e+004 - 8.6356752e+003 - 1.1966658e+004 - 1.2944558e+004 - 1.2857528e+004 - 1.2666923e+004 - 1.2403407e+004 - 1.3049728e+004 - 1.3180148e+004 - 1.3114272e+004 - 1.4222574e+004 - 1.4260084e+004 - 1.4325616e+004 - 1.0976626e+004 - 1.3165985e+004 - 1.4506695e+004 - 1.3955862e+004 - 1.4436755e+004 - 1.4555105e+004 - 1.4695783e+004 - 1.5000000e+004 - 1.3222011e+004 - 1.4841159e+004 - 1.5000000e+004 - 1.5000000e+004 - 8.2173949e+003 - 8.4847403e+003 - 7.4698844e+003 - 7.6789078e+003 - 7.2305135e+003 - 8.9171536e+003 - 9.9065098e+003 - 1.0939268e+004 - 1.0621789e+004 - 9.4237668e+003 - 9.6240995e+003 - 1.0469106e+004 - 6.4078653e+003 - 5.3461163e+003 - 4.6850295e+003 - 4.8372339e+003 - 5.4194062e+003 - 5.3089505e+003 - 6.4268435e+003 - 5.7804106e+003 - 6.9797974e+003 - 6.4299853e+003 - 9.6226511e+003 - 1.2467356e+004 - 6.7712962e+003 - 7.9248497e+003 - 9.3151535e+003 - 1.0085636e+004 - 1.0147420e+004 - 8.9951102e+003 - 9.9091571e+003 - 1.1642743e+004 - 1.1700057e+004 - 1.0425793e+004 - 9.1562497e+003 - 9.6663366e+003 - 8.0042721e+003 - 7.3568177e+003 - 8.4068014e+003 - 1.0219124e+004 - 1.0887920e+004 - 1.0851768e+004 - 1.1037744e+004 - 1.1325030e+004 - 1.0881386e+004 - 1.0749443e+004 - 9.9498587e+003 - 9.7435453e+003 - 8.1783665e+003 - 9.6435539e+003 - 9.4577928e+003 - 8.5567057e+003 - 7.5892480e+003 - 8.2518112e+003 - 9.0487260e+003 - 8.7831170e+003 - 8.7174295e+003 - 9.5873414e+003 - 8.6072456e+003 - 8.4018478e+003 - 8.3214364e+003 - 8.2080383e+003 - 8.4919721e+003 - 8.7725860e+003 - 8.6849076e+003 - 9.5541535e+003 - 9.6507978e+003 - 9.4835377e+003 - 9.4475613e+003 - 8.7227082e+003 - 8.9118537e+003 - 8.5411034e+003 - 5.8997332e+003 - 5.9055423e+003 - 5.2136739e+003 - 5.2847993e+003 - 4.5993360e+003 - 3.4901932e+003 - 3.5498222e+003 - 3.6042675e+003 - 2.8310621e+003 - 2.5213362e+003 - 8.3615086e+002 - 0.0000000e+000 - 2.7318461e+003 - 2.7563325e+003 - 2.5664120e+003 - 3.0715774e+003 - 5.2981928e+003 - 5.7880972e+003 - 5.3332870e+003 - 4.9269530e+003 - 3.8220028e+003 - 4.0341363e+003 - 5.6430991e+003 - 8.4219815e+003 - 6.4900996e+003 - 7.9735105e+003 - 8.5340289e+003 - 8.8095234e+003 - 9.9476293e+003 - 1.0073817e+004 - 1.0424993e+004 - 9.4713840e+003 - 1.1219678e+004 - 1.1619110e+004 - 9.7605285e+003 - 8.5219678e+003 - 8.3360250e+003 - 6.7428396e+003 - 4.6351759e+003 - 4.9320568e+003 - 8.2151328e+003 - 1.2515897e+004 - 1.4303586e+004 - 1.4735261e+004 - 1.4716127e+004 - 1.4824866e+004 - 1.4721656e+004 - 1.4565963e+004 - 1.4221860e+004 - 1.4705759e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4507079e+004 - 1.4396400e+004 - 1.3644260e+004 - 1.3086407e+004 - 1.2197427e+004 - 1.0136187e+004 - 8.9287669e+003 - 7.7589076e+003 - 6.4332317e+003 - 6.7680824e+003 - 6.4501980e+003 - 6.9652749e+003 - 9.1506872e+003 - 1.0823517e+004 - 1.2185235e+004 - 1.3216763e+004 - 1.4164465e+004 - 1.4237102e+004 - 1.4005279e+004 - 1.3682191e+004 - 8.5839202e+003 - 8.4315178e+003 - 7.6467575e+003 - 8.7082007e+003 - 7.4805065e+003 - 7.1282142e+003 - 6.4398718e+003 - 5.4742430e+003 - 5.4238763e+003 - 6.5274469e+003 - 6.3097353e+003 - 9.7437229e+003 - 6.0475931e+003 - 6.3554896e+003 - 2.9133611e+003 - 1.1504990e+003 - 6.8040226e+003 - 8.5763764e+003 - 9.3027835e+003 - 9.4178973e+003 - 8.8552093e+003 - 7.9389411e+003 - 7.5158553e+003 - 8.7700745e+003 - 7.9826577e+003 - 8.7632201e+003 - 7.4780363e+003 - 8.2100350e+003 - 9.0048657e+003 - 8.8556761e+003 - 9.0429944e+003 - 9.1620640e+003 - 9.1525854e+003 - 8.6142446e+003 - 8.5521287e+003 - 8.5319084e+003 - 7.4003934e+003 - 6.7825743e+003 - 7.1724267e+003 - 7.4660327e+003 - 6.6877976e+003 - 7.5859773e+003 - 9.2123735e+003 - 9.5149269e+003 - 9.2623814e+003 - 9.1159724e+003 - 9.2936618e+003 - 9.4038871e+003 - 7.0349421e+003 - 8.0949154e+003 - 7.5066224e+003 - 5.7406204e+003 - 5.5086134e+003 - 5.5304440e+003 - 5.8952209e+003 - 5.9899934e+003 - 6.6911446e+003 - 1.0122217e+004 - 1.1024627e+004 - 1.1579969e+004 - 1.1221878e+004 - 1.3331358e+004 - 1.2356294e+004 - 1.2052109e+004 - 1.0862029e+004 - 1.0303977e+004 - 1.0365205e+004 - 1.0359478e+004 - 1.0951119e+004 - 1.0983809e+004 - 1.0491018e+004 - 9.6035460e+003 - 7.6767401e+003 - 8.2892729e+003 - 8.2801080e+003 - 8.3342996e+003 - 8.2710579e+003 - 7.7671303e+003 - 7.7769709e+003 - 7.0797850e+003 - 7.0259542e+003 - 6.2744285e+003 - 6.2045692e+003 - 5.8443636e+003 - 6.9740070e+003 - 5.5070749e+003 - 4.6829467e+003 - 5.0816949e+003 - 3.4093937e+003 - 2.1094457e+003 - 2.0420227e+003 - 3.2283722e+003 - 4.3418792e+003 - 4.0507156e+003 - 4.1019593e+003 - 5.3344924e+003 - 2.1648101e+003 - 2.2728210e+003 - 4.3947053e+003 - 4.6767554e+003 - 5.0950169e+003 - 5.8727648e+003 - 4.3214421e+003 - 6.1234279e+002 - 1.1804406e+003 - 1.1929998e+003 - 4.1897831e+003 - 5.2370970e+003 - 5.1190211e+003 - 5.1660950e+003 - 5.4647338e+003 - 5.5155667e+003 - 6.5996263e+003 - 6.9684127e+003 - 5.6261129e+003 - 1.8381287e+003 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 5.4776100e+003 - 8.1800953e+003 - 7.4535447e+003 - 7.7761855e+003 - 7.5552787e+003 - 7.4256751e+003 - 7.6105024e+003 - 7.9168255e+003 - 8.5201799e+003 - 7.9427089e+003 - 7.4554443e+003 - 5.6157724e+003 - 6.9630792e+003 - 7.6932765e+003 - 7.3456133e+003 - 6.9729421e+003 - 7.0760439e+003 - 6.9676838e+003 - 6.5517380e+003 - 7.1909632e+003 - 7.1104170e+003 - 6.6833622e+003 - 6.4006330e+003 - 6.1337757e+003 - 6.3979589e+003 - 5.5594768e+003 - 6.3601978e+003 - 6.0498012e+003 - 6.3149615e+003 - 6.6374567e+003 - 7.4413136e+003 - 7.3576290e+003 - 6.9949594e+003 - 7.1236270e+003 - 7.6452271e+003 - 5.6925313e+003 - 6.1566268e+003 - 6.3217040e+003 - 8.0486104e+003 - 8.4958653e+003 - 9.4619897e+003 - 9.7052377e+003 - 9.1069443e+003 - 8.4658526e+003 - 7.9609607e+003 - 6.8776257e+003 - 6.4398980e+003 - 7.0934445e+003 - 8.3853568e+003 - 9.4684526e+003 - 1.0075574e+004 - 9.3185898e+003 - 8.2293815e+003 - 6.8109563e+003 - 8.0968716e+003 - 8.1219652e+003 - 6.6452039e+003 - 5.8063498e+003 - 4.3312717e+003 - 4.2182452e+003 - 7.0991009e+003 - 6.8156501e+003 - 6.9388041e+003 - 4.2453291e+003 - 3.2440046e+003 - 3.0738538e+003 - 3.2727368e+003 - 2.9425068e+003 - 2.6064224e+003 - 2.3901037e+003 - 2.4992386e+003 - 3.4849103e+003 - 5.0818266e+003 - 5.4011282e+003 - 6.1131908e+003 - 6.8600006e+003 - 7.8854328e+003 - 7.4621859e+003 - 6.3617678e+003 - 5.9848306e+003 - 6.6823063e+003 - 6.3239802e+003 - 5.7991782e+003 - 3.6498423e+003 - 2.5347859e+003 - 5.2572831e+003 - 6.0198489e+003 - 5.8484611e+003 - 6.2704238e+003 - 5.8769197e+003 - 5.3361740e+003 - 5.5285551e+003 - 5.7525642e+003 - 7.2207674e+003 - 6.4197555e+003 - 6.3724792e+003 - 3.8649419e+003 - 6.1326291e+003 - 6.0219614e+003 - 6.5016140e+003 - 6.9994260e+003 - 6.8765450e+003 - 6.6086661e+003 - 7.9010464e+003 - 7.8683424e+003 - 7.7782454e+003 - 8.5296193e+003 - 8.8206632e+003 - 8.1978418e+003 - 7.7307222e+003 - 8.0264325e+003 - 8.3835673e+003 - 8.5269342e+003 - 8.9054883e+003 - 8.3814781e+003 - 8.0928745e+003 - 7.9500547e+003 - 7.0942781e+003 - 6.4445111e+003 - 5.8863182e+003 - 2.8031650e+003 - 2.4356351e+003 - 3.9402771e+003 - 5.8025060e+003 - 7.1818275e+003 - 7.2318949e+003 - 7.9729002e+003 - 8.7035659e+003 - 9.7801719e+003 - 1.0786477e+004 - 1.1100724e+004 - 1.1519484e+004 - 1.1185413e+004 - 1.1287507e+004 - 1.2886716e+004 - 1.3071407e+004 - 1.4054179e+004 - 1.4222627e+004 - 1.4403408e+004 - 1.4442751e+004 - 1.4544456e+004 - 1.4584555e+004 - 1.4502668e+004 - 1.4592728e+004 - 1.0334188e+004 - 1.4211989e+004 - 1.4546937e+004 - 1.4455925e+004 - 1.4336482e+004 - 1.4134154e+004 - 1.3429594e+004 - 1.2326987e+004 - 1.1760410e+004 - 1.2061839e+004 - 1.2500606e+004 - 1.2245617e+004 - 8.2655803e+003 - 8.7309205e+003 - 9.1305818e+003 - 8.4128893e+003 - 6.7350017e+003 - 6.2227305e+003 - 6.2624930e+003 - 5.6653901e+003 - 5.4428207e+003 - 4.1760847e+003 - 2.0358527e+003 - 3.5722682e+003 - 1.8301911e+003 - 4.6998732e+003 - 6.8465506e+003 - 8.2619534e+003 - 8.9153791e+003 - 9.2376504e+003 - 9.2414628e+003 - 9.2391090e+003 - 9.5181399e+003 - 6.8448284e+003 - 7.2845764e+003 - 9.3930271e+003 - 7.9746440e+003 - 7.2318372e+003 - 7.4801543e+003 - 5.9752537e+003 - 5.9060442e+003 - 6.8245411e+003 - 7.5077476e+003 - 8.1376765e+003 - 8.0662850e+003 - 8.4547265e+003 - 7.7854312e+003 - 7.7538850e+003 - 5.3991422e+003 - 5.4404675e+003 - 6.3204464e+003 - 7.2593961e+003 - 7.2804696e+003 - 7.9656057e+003 - 7.2606322e+003 - 6.7078421e+003 - 4.1798043e+003 - 3.3833419e+003 - 3.8154037e+003 - 5.3286300e+003 - 7.8737472e+003 - 6.8192825e+003 - 6.7455772e+003 - 7.3959928e+003 - 9.3236225e+003 - 1.0507159e+004 - 1.0906161e+004 - 1.2232260e+004 - 1.2148698e+004 - 1.3022422e+004 - 1.1489517e+004 - 9.9359215e+003 - 8.4360144e+003 - 1.1222499e+004 - 9.7947010e+003 - 1.0395324e+004 - 1.0905962e+004 - 1.0653146e+004 - 1.0726526e+004 - 9.8016998e+003 - 1.0207280e+004 - 1.1354212e+004 - 1.1429272e+004 - 1.0483560e+004 - 1.0997644e+004 - 1.1650747e+004 - 1.1833354e+004 - 1.2315291e+004 - 1.3621037e+004 - 1.4119984e+004 - 1.4262125e+004 - 1.4206777e+004 - 1.3317693e+004 - 1.3309426e+004 - 1.3507703e+004 - 1.4075858e+004 - 1.2512776e+004 - 1.4811687e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4867827e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4823335e+004 - 1.1556068e+004 - 1.3417042e+004 - 1.4087129e+004 - 1.4168998e+004 - 1.4235831e+004 - 1.4079482e+004 - 1.4583530e+004 - 1.4858485e+004 - 1.4854364e+004 - 1.4528486e+004 - 1.4057816e+004 - 1.3011969e+004 - 9.8385404e+003 - 1.1825110e+004 - 1.2510194e+004 - 1.3536656e+004 - 1.3095381e+004 - 1.1520288e+004 - 1.2874967e+004 - 1.3019951e+004 - 1.2619576e+004 - 1.1266786e+004 - 1.0826468e+004 - 9.5982624e+003 - 6.8778831e+003 - 8.6616137e+003 - 7.6771449e+003 - 7.8008145e+003 - 7.9894875e+003 - 8.8016993e+003 - 9.4063364e+003 - 8.9377300e+003 - 7.7824054e+003 - 7.2823412e+003 - 7.0161408e+003 - 5.7601852e+003 - 7.9047308e+003 - 9.6424302e+003 - 9.6934752e+003 - 8.8286192e+003 - 8.3690940e+003 - 9.1814400e+003 - 9.3118182e+003 - 9.2434294e+003 - 8.5874078e+003 - 7.9319946e+003 - 8.9715121e+003 - 1.0204434e+004 - 5.3144578e+003 - 6.1229146e+003 - 5.4283860e+003 - 4.6520040e+003 - 5.3880605e+003 - 5.3650611e+003 - 5.8750843e+003 - 5.7096377e+003 - 5.5227528e+003 - 5.5002884e+003 - 5.5142816e+003 - 5.6307109e+003 - 6.9449320e+003 - 8.8480462e+003 - 9.8821917e+003 - 9.9156206e+003 - 1.0033452e+004 - 1.0834247e+004 - 1.0747492e+004 - 1.0175973e+004 - 9.4331453e+003 - 8.5438366e+003 - 8.3912263e+003 - 8.2878476e+003 - 4.4956714e+003 - 5.1466654e+003 - 6.2688815e+003 - 6.8308835e+003 - 6.8956086e+003 - 7.5436097e+003 - 7.5836712e+003 - 8.2025505e+003 - 9.0613333e+003 - 9.9931828e+003 - 8.7386469e+003 - 6.1763915e+003 - 5.2497803e+003 - 5.0380541e+003 - 2.9299883e+003 - 4.5879082e+003 - 4.8972300e+003 - 6.6820488e+003 - 6.8431977e+003 - 7.5871958e+003 - 7.8408747e+003 - 8.9435668e+003 - 8.3872562e+003 - 8.8040651e+003 - 8.0266848e+003 - 1.0127819e+004 - 1.1307804e+004 - 9.8327250e+003 - 1.0772157e+004 - 1.1441995e+004 - 1.2705131e+004 - 1.2487697e+004 - 1.2222480e+004 - 1.2052018e+004 - 1.2262223e+004 - 1.2465880e+004 - 8.1135460e+003 - 8.8010011e+003 - 1.0396560e+004 - 1.0933333e+004 - 1.1263575e+004 - 1.0149281e+004 - 1.0142376e+004 - 9.0366303e+003 - 8.0723367e+003 - 8.3245788e+003 - 9.2440759e+003 - 8.4529768e+003 - 7.0740447e+003 - 8.7289570e+003 - 8.7878085e+003 - 8.4808207e+003 - 8.3801839e+003 - 8.8354445e+003 - 9.6164879e+003 - 9.0760913e+003 - 1.0100582e+004 - 1.1383009e+004 - 1.1212801e+004 - 1.2134560e+004 - 8.9347836e+003 - 1.1142941e+004 - 1.3322105e+004 - 1.3551012e+004 - 1.3310149e+004 - 1.4010992e+004 - 1.4021666e+004 - 1.4104404e+004 - 1.3763623e+004 - 1.3247540e+004 - 1.3660137e+004 - 1.3889468e+004 - 1.1294418e+004 - 1.2103737e+004 - 1.4395596e+004 - 1.4629684e+004 - 1.4967872e+004 - 1.5000000e+004 - 1.4623302e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4865756e+004 - 1.4891453e+004 - 1.4620953e+004 - 1.1784641e+004 - 1.4588375e+004 - 1.4286414e+004 - 1.4412371e+004 - 1.4033645e+004 - 1.3401735e+004 - 1.3238423e+004 - 1.2542659e+004 - 1.3512386e+004 - 1.4172996e+004 - 1.4748230e+004 - 1.4949178e+004 - 1.1875865e+004 - 1.4895319e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4853666e+004 - 1.4718655e+004 - 1.4692975e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4802535e+004 - 1.3272526e+004 - 1.3692940e+004 - 1.2242988e+004 - 1.4355671e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4414343e+004 - 1.4341184e+004 - 1.5000000e+004 - 1.4685462e+004 - 1.4599022e+004 - 1.4537431e+004 - 1.4559949e+004 - 1.4551811e+004 - 1.3884032e+004 - 1.4603349e+004 - 1.4115257e+004 - 1.4106073e+004 - 1.4140307e+004 - 1.2973570e+004 - 1.3411626e+004 - 1.3665887e+004 - 1.3700611e+004 - 1.3043086e+004 - 1.1805899e+004 - 9.9131107e+003 - 7.4991295e+003 - 6.8132153e+003 - 4.6794405e+003 - 1.9823224e+002 - 6.2965908e+002 - 4.8531320e+003 - 6.5759814e+003 - 7.7801119e+003 - 8.3046726e+003 - 8.4287226e+003 - 9.0399325e+003 - 9.9251611e+003 - 9.5085475e+003 - 9.9040034e+003 - 9.8367973e+003 - 9.4694639e+003 - 1.0925941e+004 - 1.2060486e+004 - 1.2308514e+004 - 1.2500359e+004 - 1.2491119e+004 - 1.2346487e+004 - 1.1391662e+004 - 1.0806994e+004 - 9.0756405e+003 - 9.0416329e+003 - 7.7871365e+003 - 6.8184074e+003 - 5.9365681e+003 - 6.1782070e+003 - 7.3528925e+003 - 1.1042672e+004 - 1.1880352e+004 - 1.1504791e+004 - 1.0785477e+004 - 1.0700104e+004 - 1.0131450e+004 - 1.2199688e+004 - 1.1980281e+004 - 1.0571114e+004 - 1.0281845e+004 - 1.0297735e+004 - 1.1128600e+004 - 1.1694741e+004 - 1.1884568e+004 - 1.2722472e+004 - 1.2395753e+004 - 1.3329513e+004 - 1.0656431e+004 - 1.3118487e+004 - 1.2145510e+004 - 1.2612328e+004 - 1.1789607e+004 - 1.0845578e+004 - 9.9575331e+003 - 9.0284838e+003 - 8.1174937e+003 - 7.2917601e+003 - 6.1150027e+003 - 5.2875798e+003 - 3.7187808e+003 - 1.8878923e+003 - 0.0000000e+000 - 2.6353629e+003 - 5.1195240e+003 - 6.3532473e+003 - 7.4359613e+003 - 8.2783813e+003 - 8.7127947e+003 - 9.3210671e+003 - 1.0418584e+004 - 1.0730893e+004 - 9.9346252e+003 - 1.0895413e+004 - 1.1234775e+004 - 1.1816047e+004 - 1.1716867e+004 - 1.2109045e+004 - 1.1776069e+004 - 1.0462066e+004 - 9.1852810e+003 - 8.1221290e+003 - 7.0134665e+003 - 4.9225654e+003 - 6.0855393e+003 - 8.1815508e+003 - 1.0353301e+004 - 1.0356309e+004 - 1.0784069e+004 - 1.1356969e+004 - 1.2627577e+004 - 1.2727905e+004 - 1.3313116e+004 - 1.2733812e+004 - 1.1776632e+004 - 1.1623052e+004 - 7.5770686e+003 - 9.2204548e+003 - 9.4619889e+003 - 9.0626182e+003 - 7.7922636e+003 - 6.6181763e+003 - 6.3208759e+003 - 6.2892103e+003 - 6.2343793e+003 - 5.6255002e+003 - 6.2588468e+003 - 5.6078262e+003 - 4.1708612e+003 - 5.5244542e+003 - 6.2289131e+003 - 6.2951739e+003 - 5.8751219e+003 - 7.2790609e+003 - 7.9316271e+003 - 8.4277355e+003 - 9.1067280e+003 - 9.7450270e+003 - 1.0807030e+004 - 1.1721592e+004 - 9.6146068e+003 - 1.0640881e+004 - 1.0851923e+004 - 1.1651608e+004 - 1.2517318e+004 - 1.3473528e+004 - 1.3382612e+004 - 1.3537044e+004 - 1.3614426e+004 - 1.3681931e+004 - 1.3911280e+004 - 1.4062071e+004 - 1.0821924e+004 - 1.4125654e+004 - 1.4151468e+004 - 1.4286110e+004 - 1.4173656e+004 - 1.4618062e+004 - 1.4503884e+004 - 1.4686577e+004 - 1.4628925e+004 - 1.4588656e+004 - 1.4673029e+004 - 1.4784433e+004 - 1.2252563e+004 - 1.4810115e+004 - 1.4728487e+004 - 1.4620800e+004 - 1.4325483e+004 - 1.4220812e+004 - 1.4304914e+004 - 1.3941514e+004 - 1.2650455e+004 - 1.2320906e+004 - 1.1842037e+004 - 1.0765490e+004 - 9.3810669e+003 - 7.9973356e+003 - 8.2197087e+003 - 9.3693315e+003 - 8.6204755e+003 - 8.8787996e+003 - 9.0276740e+003 - 8.9857178e+003 - 8.6367811e+003 - 7.5960597e+003 - 7.1373715e+003 - 6.0213223e+003 - 4.9606339e+003 - 2.0251961e+003 - 3.2233065e+003 - 1.6992731e+003 - 3.1912470e+003 - 4.5050657e+003 - 5.3506785e+003 - 6.7241904e+003 - 8.8993623e+003 - 1.0011600e+004 - 1.0833658e+004 - 1.1182707e+004 - 1.0399275e+004 - 1.0574507e+004 - 1.1152425e+004 - 1.0935680e+004 - 1.0317606e+004 - 1.0826712e+004 - 1.0715578e+004 - 9.6744369e+003 - 1.1095373e+004 - 1.2787674e+004 - 1.2219585e+004 - 1.1742696e+004 - 8.8870302e+003 - 1.2192333e+004 - 1.2361145e+004 - 1.1842017e+004 - 1.1955942e+004 - 1.1433316e+004 - 1.0266510e+004 - 9.5703957e+003 - 9.6174630e+003 - 9.7612734e+003 - 9.6515392e+003 - 1.0007616e+004 - 7.7612534e+003 - 8.3035449e+003 - 8.9049017e+003 - 7.5842812e+003 - 6.2487320e+003 - 5.6785973e+003 - 6.2410254e+003 - 6.2044892e+003 - 6.1865979e+003 - 5.6246744e+003 - 4.9114881e+003 - 5.1630116e+003 - 7.7873136e+003 - 7.5224850e+003 - 7.1171818e+003 - 8.3823642e+003 - 8.3840973e+003 - 8.4598232e+003 - 8.6088400e+003 - 8.2343045e+003 - 8.2475222e+003 - 8.4276597e+003 - 8.8854238e+003 - 9.4218736e+003 - 6.9783476e+003 - 7.0176553e+003 - 7.6935227e+003 - 7.8321128e+003 - 8.1217139e+003 - 5.9977112e+003 - 6.3403458e+003 - 6.8605886e+003 - 6.7188709e+003 - 6.2680774e+003 - 6.5573511e+003 - 7.1927024e+003 - 6.4824031e+003 - 7.8562382e+003 - 8.0599809e+003 - 8.5368191e+003 - 7.9897591e+003 - 8.3682930e+003 - 8.8611666e+003 - 8.3085342e+003 - 8.3245558e+003 - 7.9686390e+003 - 8.1487728e+003 - 7.8398278e+003 - 8.4654749e+003 - 8.4643540e+003 - 7.9403524e+003 - 7.8457254e+003 - 8.1367199e+003 - 7.8860161e+003 - 9.2751191e+003 - 9.9976158e+003 - 1.0330024e+004 - 1.0892778e+004 - 1.0629697e+004 - 1.0700399e+004 - 8.9128977e+003 - 9.2639283e+003 - 9.6032591e+003 - 1.0917082e+004 - 1.1492970e+004 - 1.2646978e+004 - 1.2826383e+004 - 1.3291833e+004 - 1.3820755e+004 - 1.3808865e+004 - 1.3432736e+004 - 1.3353035e+004 - 1.4027808e+004 - 1.4387238e+004 - 1.4104953e+004 - 1.4136789e+004 - 1.4067905e+004 - 1.4207896e+004 - 1.4034042e+004 - 1.4133785e+004 - 1.4410175e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4940238e+004 - 1.2718204e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4922235e+004 - 1.4616712e+004 - 1.4449723e+004 - 1.4174409e+004 - 1.3320212e+004 - 1.3657265e+004 - 1.3582301e+004 - 1.2496910e+004 - 1.1859223e+004 - 1.0152449e+004 - 1.2723281e+004 - 1.2675245e+004 - 1.2751640e+004 - 1.2435756e+004 - 1.2632841e+004 - 1.2757560e+004 - 1.2025055e+004 - 1.0946839e+004 - 1.0406643e+004 - 9.5467677e+003 - 8.2489435e+003 - 8.1835940e+003 - 1.0046730e+004 - 1.0000027e+004 - 8.4541262e+003 - 7.1049530e+003 - 6.2915147e+003 - 7.2411477e+003 - 8.0121166e+003 - 8.1454782e+003 - 7.4781379e+003 - 7.7620821e+003 - 8.8870823e+003 - 5.6594602e+003 - 6.9996298e+003 - 7.6000931e+003 - 8.3545882e+003 - 8.5272729e+003 - 1.0431292e+004 - 1.0885185e+004 - 1.0819912e+004 - 1.1467976e+004 - 1.1745796e+004 - 1.0283164e+004 - 1.0181007e+004 - 6.7818400e+003 - 8.5494268e+003 - 9.5238453e+003 - 9.0662438e+003 - 9.0180732e+003 - 9.2270320e+003 - 9.6859774e+003 - 9.9164793e+003 - 9.5317936e+003 - 8.7221920e+003 - 7.8512347e+003 - 7.2898235e+003 - 6.7492205e+003 - 5.6880052e+003 - 6.3508833e+003 - 5.5409463e+003 - 4.4627936e+003 - 4.4778260e+003 - 5.0426947e+003 - 4.9808647e+003 - 3.3684808e+003 - 5.1537654e+003 - 5.9260481e+003 - 7.2579928e+003 - 6.3989463e+003 - 6.4496612e+003 - 6.8836099e+003 - 7.6740454e+003 - 7.7629658e+003 - 7.9867851e+003 - 8.3486878e+003 - 8.4462219e+003 - 7.9024098e+003 - 7.9810279e+003 - 7.9940842e+003 - 8.0718303e+003 - 6.2196168e+003 - 6.8664702e+003 - 7.9234127e+003 - 8.8856314e+003 - 7.6563791e+003 - 8.7470152e+003 - 1.0288779e+004 - 1.0685573e+004 - 1.0193268e+004 - 9.9524226e+003 - 9.9518596e+003 - 9.9782388e+003 - 8.4927817e+003 - 1.0485659e+004 - 1.2139918e+004 - 1.1488226e+004 - 1.1693826e+004 - 1.1450334e+004 - 1.1707138e+004 - 1.1472874e+004 - 1.1046821e+004 - 1.0988219e+004 - 1.0266586e+004 - 9.5012927e+003 - 1.0525125e+004 - 1.0643669e+004 - 1.1772746e+004 - 1.1264066e+004 - 1.0535157e+004 - 1.0575322e+004 - 1.0263374e+004 - 1.0238810e+004 - 1.0364559e+004 - 1.1177416e+004 - 1.1306058e+004 - 1.1099297e+004 - 9.0592327e+003 - 1.0913017e+004 - 1.1070730e+004 - 1.0601287e+004 - 9.7067147e+003 - 8.2404809e+003 - 7.9272085e+003 - 8.3657110e+003 - 7.9955623e+003 - 7.6907269e+003 - 8.2667200e+003 - 8.9488888e+003 - 8.2271937e+003 - 8.8478152e+003 - 9.6570623e+003 - 1.0383155e+004 - 9.9699347e+003 - 9.3187616e+003 - 9.0646578e+003 - 9.4499136e+003 - 9.8932764e+003 - 9.4027845e+003 - 8.4209375e+003 - 8.3851623e+003 - 6.4781754e+003 - 6.2488606e+003 - 6.3032680e+003 - 7.0174331e+003 - 7.5251108e+003 - 7.8357363e+003 - 8.4807872e+003 - 6.9745611e+003 - 5.4510584e+003 - 3.1569579e+003 - 1.9581557e+003 - 0.0000000e+000 - 4.1361609e+003 - 2.1622807e+003 - 2.1973831e+003 - 3.8776383e+003 - 3.4827115e+003 - 1.5692641e+003 - 4.4365857e+003 - 5.3216632e+003 - 5.1580715e+003 - 5.5886359e+003 - 4.7856576e+003 - 4.6019656e+003 - 5.3837013e+003 - 5.8026190e+003 - 5.4042069e+003 - 6.0251994e+003 - 6.5403587e+003 - 6.5027559e+003 - 6.6719144e+003 - 6.0726101e+003 - 5.2372376e+003 - 4.4800352e+003 - 3.9911030e+003 - 5.7004603e+003 - 6.9833864e+003 - 5.9965623e+003 - 5.2337832e+003 - 5.5980423e+003 - 4.5043840e+003 - 5.6284706e+003 - 7.0152064e+003 - 8.2758979e+003 - 8.1873050e+003 - 7.5664604e+003 - 7.1198067e+003 - 6.6165636e+003 - 5.9673113e+003 - 5.4207125e+003 - 3.8522297e+003 - 2.8145002e+003 - 3.3736327e+003 - 5.9505896e+003 - 6.4316213e+003 - 6.9096980e+003 - 7.4753270e+003 - 7.0954315e+003 - 7.6634242e+003 - 7.6494835e+003 - 8.8269527e+003 - 8.1969882e+003 - 8.8447985e+003 - 9.4840747e+003 - 9.5967282e+003 - 8.8663561e+003 - 9.3218328e+003 - 9.5963247e+003 - 1.1362314e+004 - 1.1525552e+004 - 1.3334699e+004 - 1.3548153e+004 - 8.1934758e+003 - 1.1507558e+004 - 1.2468513e+004 - 1.3237703e+004 - 1.3620296e+004 - 1.4199941e+004 - 1.3272316e+004 - 1.3085348e+004 - 1.2220774e+004 - 1.0700335e+004 - 1.2340356e+004 - 1.1376885e+004 - 1.0465304e+004 - 1.1367084e+004 - 1.2271068e+004 - 1.1792266e+004 - 1.1495788e+004 - 1.2385901e+004 - 1.2931141e+004 - 1.3207900e+004 - 1.2086631e+004 - 1.2476402e+004 - 1.2632056e+004 - 1.2407289e+004 - 1.0719221e+004 - 1.2866402e+004 - 1.2833877e+004 - 1.1980885e+004 - 1.1995089e+004 - 1.3270733e+004 - 1.4124120e+004 - 1.3988345e+004 - 1.2367008e+004 - 1.1293875e+004 - 1.1303024e+004 - 1.0031057e+004 - 9.3351698e+003 - 9.8501488e+003 - 9.9037710e+003 - 8.2365539e+003 - 8.1877232e+003 - 8.7060416e+003 - 9.7142189e+003 - 9.9187846e+003 - 1.0400487e+004 - 1.0631118e+004 - 1.0647037e+004 - 9.7660250e+003 - 7.6360388e+003 - 9.2086962e+003 - 1.0544309e+004 - 9.5429184e+003 - 9.6418384e+003 - 9.3524162e+003 - 1.0067624e+004 - 1.0069148e+004 - 9.8313546e+003 - 8.5050244e+003 - 8.2247991e+003 - 7.6364324e+003 - 8.7839250e+003 - 9.7247523e+003 - 1.0412423e+004 - 1.0176744e+004 - 9.4436136e+003 - 1.0392367e+004 - 8.1571148e+003 - 8.2925115e+003 - 7.2755936e+003 - 6.1263830e+003 - 7.9962540e+003 - 9.0865979e+003 - 6.8667200e+003 - 7.3811132e+003 - 7.9762261e+003 - 7.2080326e+003 - 7.4286902e+003 - 8.6522583e+003 - 8.9122617e+003 - 8.6372205e+003 - 6.7640870e+003 - 8.4705275e+003 - 1.0049812e+004 - 1.0254002e+004 - 9.1283484e+003 - 8.2060875e+003 - 8.4081139e+003 - 1.0608388e+004 - 1.0671249e+004 - 1.1785592e+004 - 1.2096886e+004 - 1.2077758e+004 - 1.3188593e+004 - 1.3379630e+004 - 1.4364724e+004 - 1.4730658e+004 - 1.2850476e+004 - 1.4984018e+004 - 1.4723322e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4961506e+004 - 1.4905479e+004 - 1.4836107e+004 - 1.4896295e+004 - 1.4789047e+004 - 1.4599403e+004 - 1.2606663e+004 - 1.4316576e+004 - 1.4617536e+004 - 1.4052037e+004 - 1.4067355e+004 - 1.3315640e+004 - 1.2026114e+004 - 1.1058062e+004 - 1.1812421e+004 - 1.1369317e+004 - 1.0155953e+004 - 9.4705645e+003 - 8.2681861e+003 - 8.7932317e+003 - 8.4022172e+003 - 7.1561565e+003 - 6.4141078e+003 - 6.0687719e+003 - 4.5991697e+003 - 3.6721879e+003 - 4.0250027e+003 - 3.5355895e+003 - 4.2863135e+003 - 5.7726998e+003 - 3.4543634e+003 - 6.1282939e+003 - 6.4295839e+003 - 7.6161846e+003 - 7.0481479e+003 - 7.9064700e+003 - 8.7447786e+003 - 9.2819045e+003 - 9.7037519e+003 - 9.7105788e+003 - 9.3318068e+003 - 9.7694690e+003 - 8.7550488e+003 - 8.7268760e+003 - 8.9168627e+003 - 8.9174942e+003 - 8.3993095e+003 - 8.3170976e+003 - 8.1747617e+003 - 7.5471645e+003 - 6.9103981e+003 - 6.1242050e+003 - 6.2065964e+003 - 6.5763785e+003 - 5.0285396e+003 - 2.1729596e+003 - 9.6468707e+002 - 5.8891797e+002 - 0.0000000e+000 - 0.0000000e+000 - 0.0000000e+000 - 2.9955456e+003 - 5.8815407e+003 - 6.9207737e+003 - 6.9781408e+003 - 7.5562398e+003 - 7.3091159e+003 - 8.1595196e+003 - 8.7499659e+003 - 8.2375036e+003 - 8.2734150e+003 - 8.4853271e+003 - 9.5687654e+003 - 1.0673308e+004 - 1.0150519e+004 - 1.0474492e+004 - 1.0527526e+004 - 1.0866457e+004 - 8.4985254e+003 - 1.2016135e+004 - 1.1833896e+004 - 1.1475508e+004 - 1.0824698e+004 - 1.2081122e+004 - 1.3150616e+004 - 1.2930457e+004 - 1.2685177e+004 - 1.1910429e+004 - 1.1504104e+004 - 1.1891623e+004 - 8.1937017e+003 - 9.1861685e+003 - 7.6695567e+003 - 6.3901822e+003 - 6.2345339e+003 - 6.7983982e+003 - 7.2913670e+003 - 7.5137339e+003 - 7.4296602e+003 - 6.9808140e+003 - 6.5265951e+003 - 6.8322154e+003 - 7.4606780e+003 - 8.2442046e+003 - 9.6940577e+003 - 1.0201618e+004 - 9.4767404e+003 - 9.4781196e+003 - 1.0059078e+004 - 1.1103791e+004 - 1.1610210e+004 - 1.1715099e+004 - 1.2305306e+004 - 1.2846171e+004 - 1.1126415e+004 - 1.3280540e+004 - 1.2138000e+004 - 1.1645108e+004 - 1.0097359e+004 - 8.9867809e+003 - 8.3078091e+003 - 7.3198606e+003 - 7.6184490e+003 - 8.1279724e+003 - 8.4491918e+003 - 9.0588419e+003 - 6.4728071e+003 - 6.7957448e+003 - 6.8198674e+003 - 7.0263385e+003 - 6.6966588e+003 - 7.2377898e+003 - 8.4875808e+003 - 8.6020818e+003 - 9.8731001e+003 - 9.6379546e+003 - 9.3144768e+003 - 9.1017257e+003 - 6.7035182e+003 - 7.2354796e+003 - 1.0080525e+004 - 1.1774817e+004 - 1.3366058e+004 - 1.3062401e+004 - 1.2058036e+004 - 1.0937884e+004 - 1.1100433e+004 - 1.0946695e+004 - 1.1066259e+004 - 1.1843795e+004 - 8.9862379e+003 - 1.0957139e+004 - 1.1223407e+004 - 1.1340532e+004 - 1.1731978e+004 - 1.2163871e+004 - 1.3020081e+004 - 1.3389849e+004 - 1.3305756e+004 - 1.2612009e+004 - 1.2859894e+004 - 1.2583202e+004 - 9.2096540e+003 - 1.1523719e+004 - 1.1821795e+004 - 1.1379864e+004 - 1.2273141e+004 - 1.1627550e+004 - 1.1105007e+004 - 1.0446525e+004 - 1.0530605e+004 - 1.0769622e+004 - 1.0794506e+004 - 9.6961177e+003 - 6.5992768e+003 - 6.8393081e+003 - 7.4969232e+003 - 7.6221348e+003 - 8.4338777e+003 - 9.1497322e+003 - 9.9651156e+003 - 1.0229916e+004 - 1.0424314e+004 - 1.0371634e+004 - 1.0825729e+004 - 1.1741721e+004 - 9.3508150e+003 - 1.1715560e+004 - 1.0439883e+004 - 1.1118577e+004 - 1.1796279e+004 - 1.2243624e+004 - 1.1456869e+004 - 1.1132808e+004 - 1.0756835e+004 - 1.0284278e+004 - 9.8459938e+003 - 9.6277061e+003 - 8.1001721e+003 - 8.8634695e+003 - 9.7920796e+003 - 1.0154686e+004 - 1.0992151e+004 - 1.0094231e+004 - 9.8579314e+003 - 9.8526067e+003 - 1.0276526e+004 - 9.3266332e+003 - 8.5841083e+003 - 8.8421805e+003 - 9.6316486e+003 - 9.7904155e+003 - 1.0432630e+004 - 1.0817028e+004 - 1.0542244e+004 - 1.0249457e+004 - 9.5140570e+003 - 8.7435717e+003 - 8.7497927e+003 - 8.7831897e+003 - 8.7049529e+003 - 8.0628457e+003 - 7.0360671e+003 - 8.1396566e+003 - 7.7644194e+003 - 8.7971831e+003 - 8.9114182e+003 - 9.3327653e+003 - 9.8615082e+003 - 1.0097022e+004 - 9.4708197e+003 - 8.4409194e+003 - 7.7953777e+003 - 8.1808153e+003 - 6.7629218e+003 - 6.2199216e+003 - 5.1283727e+003 - 3.3393389e+003 - 8.8887754e+002 - 0.0000000e+000 - 0.0000000e+000 - 4.8360458e+002 - 4.5301439e+003 - 6.8613899e+003 - 8.3216931e+003 - 8.9978785e+003 - 8.1505786e+003 - 8.8262728e+003 - 9.4902688e+003 - 9.2232027e+003 - 9.1329966e+003 - 8.6481879e+003 - 8.6992312e+003 - 8.1410475e+003 - 7.4783403e+003 - 7.2672574e+003 - 6.9964605e+003 - 8.2045212e+003 - 8.6188153e+003 - 9.2879649e+003 - 9.6800031e+003 - 8.8242201e+003 - 9.3919228e+003 - 9.1541994e+003 - 8.9368869e+003 - 8.8663711e+003 - 8.8637237e+003 - 1.0358561e+004 - 1.0679750e+004 - 9.8509788e+003 - 7.8335861e+003 - 8.9938577e+003 - 9.0954655e+003 - 8.8174609e+003 - 9.1541933e+003 - 9.0907840e+003 - 8.6601509e+003 - 7.8282473e+003 - 7.3958913e+003 - 7.1758793e+003 - 6.7362010e+003 - 6.0320283e+003 - 3.1097025e+003 - 5.1605822e+003 - 4.5721668e+003 - 5.0344872e+003 - 6.4508479e+003 - 6.9148480e+003 - 6.7113728e+003 - 6.4727297e+003 - 6.1835343e+003 - 5.9823431e+003 - 6.5574710e+003 - 7.0343944e+003 - 6.8192733e+003 - 8.0025942e+003 - 8.7662442e+003 - 9.0243644e+003 - 9.1820604e+003 - 9.6583570e+003 - 1.0550306e+004 - 1.1225924e+004 - 1.1415506e+004 - 1.1499066e+004 - 1.2873280e+004 - 1.4217651e+004 - 1.4028468e+004 - 1.4827162e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.1703600e+004 - 1.3809439e+004 - 1.4130240e+004 - 1.3941559e+004 - 1.3555537e+004 - 1.4003123e+004 - 1.4011376e+004 - 1.4014546e+004 - 1.3863351e+004 - 1.3838598e+004 - 1.3368868e+004 - 1.2754049e+004 - 8.7494374e+003 - 1.0847553e+004 - 1.0870989e+004 - 1.0570287e+004 - 1.0772084e+004 - 1.1126820e+004 - 1.1368669e+004 - 1.2276198e+004 - 1.1134128e+004 - 1.2099371e+004 - 1.2014335e+004 - 1.2664943e+004 - 1.0252912e+004 - 1.4040026e+004 - 1.4298735e+004 - 1.4251431e+004 - 1.4323999e+004 - 1.4158562e+004 - 1.3378844e+004 - 1.2500144e+004 - 1.1979175e+004 - 1.1219153e+004 - 9.7814032e+003 - 8.7403902e+003 - 7.2926161e+003 - 6.5947065e+003 - 6.1163001e+003 - 6.4892988e+003 - 6.3350131e+003 - 5.7185015e+003 - 6.2740702e+003 - 6.3811268e+003 - 6.8367820e+003 - 6.8219511e+003 - 7.3524658e+003 - 7.8586939e+003 - 8.2212175e+003 - 8.4121312e+003 - 8.4145047e+003 - 8.9618037e+003 - 9.9811505e+003 - 1.0064305e+004 - 1.0365332e+004 - 1.0820447e+004 - 1.0481384e+004 - 1.0813482e+004 - 1.1624816e+004 - 1.1342820e+004 - 1.1740587e+004 - 9.4637925e+003 - 1.0721595e+004 - 1.1262185e+004 - 9.1741819e+003 - 9.7944427e+003 - 1.0006041e+004 - 1.0895648e+004 - 1.0787664e+004 - 1.1883270e+004 - 1.3137309e+004 - 1.2246955e+004 - 9.3195923e+003 - 1.1442503e+004 - 1.1561216e+004 - 8.7571380e+003 - 7.0314465e+003 - 5.7975115e+003 - 6.1673775e+003 - 6.3803548e+003 - 6.7330141e+003 - 7.2316942e+003 - 8.8556598e+003 - 9.9826608e+003 - 7.1568118e+003 - 7.7579420e+003 - 8.2825238e+003 - 9.4256578e+003 - 8.5604542e+003 - 7.7292329e+003 - 7.0894812e+003 - 6.1563469e+003 - 6.2621843e+003 - 7.4636487e+003 - 7.3151418e+003 - 7.2637823e+003 - 8.3902280e+003 - 9.3544348e+003 - 1.0241565e+004 - 9.6909043e+003 - 1.0729626e+004 - 9.4570092e+003 - 9.5437564e+003 - 8.1031195e+003 - 7.8905012e+003 - 8.9203080e+003 - 8.8224632e+003 - 1.0109554e+004 - 9.7302707e+003 - 1.1830709e+004 - 1.2153244e+004 - 1.3317954e+004 - 1.3693128e+004 - 1.4393790e+004 - 1.4444875e+004 - 1.4517855e+004 - 1.4665332e+004 - 1.4489033e+004 - 1.4199576e+004 - 1.3266151e+004 - 1.1752882e+004 - 1.2748247e+004 - 1.2911859e+004 - 1.2143087e+004 - 1.2226110e+004 - 1.2613103e+004 - 1.1616002e+004 - 1.0870676e+004 - 1.0486755e+004 - 1.1157880e+004 - 1.1188622e+004 - 1.0602125e+004 - 1.0338139e+004 - 1.1913235e+004 - 1.2067774e+004 - 1.2107985e+004 - 1.2099986e+004 - 1.0846354e+004 - 1.0577616e+004 - 1.0394067e+004 - 1.1023946e+004 - 1.0766400e+004 - 9.5689698e+003 - 8.8559422e+003 - 6.9918654e+003 - 6.9387704e+003 - 6.7037375e+003 - 6.1689638e+003 - 6.3645941e+003 - 6.1152664e+003 - 5.9813192e+003 - 5.5421437e+003 - 6.1669290e+003 - 7.6526330e+003 - 9.1337777e+003 - 1.0089135e+004 - 9.4533359e+003 - 8.8552893e+003 - 9.3505003e+003 - 8.0424792e+003 - 8.4926750e+003 - 8.1740595e+003 - 7.5840074e+003 - 7.9379792e+003 - 7.8730491e+003 - 7.2658867e+003 - 6.4560034e+003 - 6.3920533e+003 - 5.2324499e+003 - 6.9571298e+003 - 7.1218523e+003 - 7.6627410e+003 - 7.8576139e+003 - 7.1514743e+003 - 6.9253850e+003 - 6.7103714e+003 - 6.7553730e+003 - 7.2774352e+003 - 7.1694017e+003 - 8.2030791e+003 - 8.1626441e+003 - 8.2055416e+003 - 7.8405581e+003 - 7.7634046e+003 - 8.3429194e+003 - 7.2842556e+003 - 6.5048361e+003 - 5.8830421e+003 - 5.9515169e+003 - 5.0519174e+003 - 3.6266518e+003 - 3.6239845e+003 - 5.3111137e+003 - 5.8364330e+003 - 7.0492798e+003 - 6.9505260e+003 - 6.5972883e+003 - 6.2043409e+003 - 7.3406386e+003 - 7.4697800e+003 - 6.6336917e+003 - 6.3560296e+003 - 6.8747105e+003 - 7.2586838e+003 - 5.6507618e+003 - 5.5364460e+003 - 6.5221528e+003 - 6.7980852e+003 - 6.5531339e+003 - 6.0596823e+003 - 5.0947957e+003 - 5.0510786e+003 - 5.6177765e+003 - 6.0390833e+003 - 6.4766888e+003 - 7.2429973e+003 - 7.1270059e+003 - 7.5706741e+003 - 9.1528855e+003 - 1.0278827e+004 - 9.9261362e+003 - 9.4376097e+003 - 8.4871343e+003 - 7.9123192e+003 - 8.0345550e+003 - 9.7293659e+003 - 1.0096585e+004 - 9.7864746e+003 - 7.7825390e+003 - 7.6814826e+003 - 8.2674284e+003 - 8.9266038e+003 - 9.0746692e+003 - 8.5442182e+003 - 8.3506215e+003 - 8.1510291e+003 - 8.1636070e+003 - 7.7765691e+003 - 7.2268307e+003 - 6.6937740e+003 - 5.5167300e+003 - 6.2409802e+003 - 6.4174921e+003 - 5.2713486e+003 - 3.8863184e+003 - 8.2659032e+001 - 1.5890210e+002 - 4.5208973e+003 - 5.5057891e+003 - 6.1988629e+003 - 6.2511919e+003 - 6.0507334e+003 - 6.6346265e+003 - 6.9009708e+003 - 7.4838811e+003 - 6.7797713e+003 - 6.7505008e+003 - 6.6481965e+003 - 6.0139479e+003 - 5.2906993e+003 - 3.7620722e+003 - 2.8596057e+003 - 1.8834228e+003 - 3.3565102e+003 - 2.8757731e+002 - 2.9799645e+003 - 2.7825702e+003 - 5.2657561e+003 - 6.0255571e+003 - 5.5676304e+003 - 5.7391395e+003 - 5.7511047e+003 - 5.3763780e+003 - 5.7204045e+003 - 6.1185761e+003 - 6.9954768e+003 - 5.9093653e+003 - 5.0862762e+003 - 1.8183489e+003 - 3.0705794e+003 - 2.6084604e+003 - 3.7137336e+003 - 4.9215512e+003 - 4.0920101e+003 - 3.7668166e+003 - 3.1356594e+003 - 1.8926704e+003 - 3.0574233e+003 - 4.1764733e+003 - 5.8626431e+003 - 7.6043353e+003 - 8.4165993e+003 - 9.0816317e+003 - 9.8344583e+003 - 1.0249931e+004 - 1.0303469e+004 - 1.0636419e+004 - 1.1351723e+004 - 1.1230485e+004 - 1.0959601e+004 - 9.4192105e+003 - 1.2026964e+004 - 1.2396320e+004 - 1.2933653e+004 - 1.3204015e+004 - 1.3301412e+004 - 1.2939778e+004 - 1.2613446e+004 - 1.2051031e+004 - 1.1649087e+004 - 1.2871084e+004 - 1.3015694e+004 - 1.0044640e+004 - 1.3092703e+004 - 1.3494530e+004 - 1.3603892e+004 - 1.3986177e+004 - 1.4235129e+004 - 1.4083700e+004 - 1.4044218e+004 - 1.4040194e+004 - 1.3851744e+004 - 1.3329832e+004 - 1.3297219e+004 - 1.2414261e+004 - 1.4320152e+004 - 1.4699493e+004 - 1.4401055e+004 - 1.4730959e+004 - 1.4948897e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4145817e+004 - 1.4936071e+004 - 1.4995060e+004 - 1.5000000e+004 - 1.5000000e+004 - 1.4987099e+004 - 1.4808753e+004 - 1.4372910e+004 - 1.4189271e+004 - 1.3286968e+004 - 1.2578629e+004 - 1.1788479e+004 - 7.3866594e+003 - 8.2360516e+003 - 8.0231207e+003 - 9.0101810e+003 - 8.6523929e+003 - 8.9970225e+003 - 9.0536483e+003 - 9.6593325e+003 - 1.0202091e+004 - 1.1052202e+004 - 1.0329222e+004 - 1.0863809e+004 - 6.9227114e+003 - 1.0062115e+004 - 1.1384535e+004 - 1.2248356e+004 - 1.2935084e+004 - 1.2222968e+004 - 1.2013331e+004 - 1.1459208e+004 - 1.1724537e+004 - 1.1560507e+004 - 1.2708470e+004 - 1.4005176e+004 - 1.2689722e+004 - 1.4688497e+004 - 1.5000000e+004 - 1.4878740e+004 - 1.4904220e+004 - 1.4407671e+004 - 1.4295174e+004 - 1.4327414e+004 - 1.4132944e+004 - 1.4371335e+004 - 1.4364555e+004 - 1.4183109e+004 - 9.2753222e+003 - 1.1466913e+004 - 9.4138163e+003 - 9.8752098e+003 - 8.8095095e+003 - 9.2966746e+003 - 8.2091595e+003 - 7.8979952e+003 - 7.3221497e+003 - 6.9919695e+003 - 7.0668830e+003 - 6.1203090e+003 - 5.8040369e+003 - 5.6700062e+003 - 5.6160506e+003 - 6.3435073e+003 - 6.4889347e+003 - 6.3405873e+003 - 6.3700074e+003 - 5.7910466e+003 - 5.8485272e+003 - 6.8462642e+003 - 6.8803586e+003 - 8.1586741e+003 - 5.8493850e+003 - 5.4484592e+003 - 6.1277692e+003 - 6.8770244e+003 - 7.0062926e+003 - 6.6084435e+003 - 7.0887198e+003 - 7.9520997e+003 - 8.0155114e+003 - 8.6635010e+003 - 9.9807829e+003 - 1.0785841e+004 - 6.6340975e+003 - 8.1283461e+003 - 9.8179592e+003 - 1.0756762e+004 - 1.0296159e+004 - 1.0573318e+004 - 1.0729068e+004 - 9.9593970e+003 - 9.5174493e+003 - 9.3200381e+003 - 1.0666613e+004 - 1.0964625e+004 - 9.8860632e+003 - 1.1319071e+004 - 1.2509703e+004 - 1.2661196e+004 - 1.3328256e+004 - 1.3297630e+004 - 1.3155342e+004 - 1.3450837e+004 - 1.3504028e+004 - 1.3903319e+004 - 1.4118087e+004 - 1.4315980e+004 - 1.4001304e+004 - 1.4146675e+004 - 1.4275379e+004 - 1.4363251e+004 - 1.4286439e+004 - 1.4287179e+004 - 1.4295394e+004 - 1.4115738e+004 - 1.3707113e+004 - 1.3210478e+004 - 1.2512895e+004 - 1.2633922e+004 - 1.2125882e+004 - 1.2740344e+004 - 1.0953583e+004 - 9.5622254e+003 - 8.5113293e+003 - 8.0084280e+003 - 7.9704872e+003 - 6.2140435e+003 - 5.5391930e+003 - 5.3799182e+003 - 4.3642110e+003 - 2.8249640e+003 - 0.0000000e+000 - 4.7758066e+002 - 2.6377083e+003 - 5.7771548e+003 - 5.4674808e+003 - 6.6943525e+003 - 7.9820343e+003 - 7.9081290e+003 - 8.5047664e+003 - 9.0581851e+003 - 9.9137355e+003 - 1.0516250e+004 - 9.2366671e+003 - 1.0234123e+004 - 1.0279045e+004 - 1.2318085e+004 - 1.2499570e+004 - 1.2293419e+004 - 1.2354663e+004 - 1.2336548e+004 - 1.1697450e+004 - 1.0724265e+004 - 1.1057731e+004 - 1.1483937e+004 - 1.1444137e+004 - 1.1145743e+004 - 1.0879900e+004 - 1.0958261e+004 - 1.1395920e+004 - 1.0888674e+004 - 1.0412822e+004 - 9.9142688e+003 - 9.5805745e+003 - 9.0602204e+003 - 8.9447336e+003 - 8.3338607e+003 - 5.2623176e+003 - 6.1992126e+003 - 6.6080458e+003 - 8.9456742e+003 - 1.0699771e+004 - 1.0954132e+004 - 1.1644000e+004 - 1.2029956e+004 - 1.1895722e+004 - 1.2017569e+004 - 1.2144519e+004 - 1.2458809e+004 - 1.0662727e+004 - 1.1603767e+004 - 1.2397715e+004 - 1.2339432e+004 - 1.2364182e+004 - 1.3599119e+004 - 1.4144389e+004 - 1.4187817e+004 - 1.3812512e+004 - 1.3107417e+004 - 1.2306262e+004 - 1.2775950e+004 - 8.9080654e+003 - 1.0985420e+004 - 1.0995708e+004 - 1.0460706e+004 - 1.0746053e+004 - 1.0667141e+004 - 1.0622371e+004 From ketan at ci.uchicago.edu Tue May 6 10:27:53 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 10:27:53 -0500 (CDT) Subject: [Swift-commit] r7819 - SwiftApps/swift-galaxy/swift Message-ID: <20140506152753.871CE9D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 10:27:53 -0500 (Tue, 06 May 2014) New Revision: 7819 Modified: SwiftApps/swift-galaxy/swift/delog Log: minor Modified: SwiftApps/swift-galaxy/swift/delog =================================================================== --- SwiftApps/swift-galaxy/swift/delog 2014-05-06 15:23:22 UTC (rev 7818) +++ SwiftApps/swift-galaxy/swift/delog 2014-05-06 15:27:53 UTC (rev 7819) @@ -1 +1 @@ -rm -rfv *.d *.rlog *.log swift.work *.out outdir /tmp/swift.work Results /tmp/swift-gal.* +rm -rfv *.d *.rlog *.log swift.work *.out outdir /tmp/swift.work Results /tmp/swift-gal.* tmpout* From ketan at ci.uchicago.edu Tue May 6 10:54:36 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 10:54:36 -0500 (CDT) Subject: [Swift-commit] r7820 - SwiftApps/swift-galaxy/swift Message-ID: <20140506155436.993F09D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 10:54:36 -0500 (Tue, 06 May 2014) New Revision: 7820 Added: SwiftApps/swift-galaxy/swift/test.sh Removed: SwiftApps/swift-galaxy/swift/swiftforeachtest.sh Log: rename Deleted: SwiftApps/swift-galaxy/swift/swiftforeachtest.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachtest.sh 2014-05-06 15:27:53 UTC (rev 7819) +++ SwiftApps/swift-galaxy/swift/swiftforeachtest.sh 2014-05-06 15:54:36 UTC (rev 7820) @@ -1,6 +0,0 @@ -#!/bin/bash - -#!/bin/bash - -./swiftforeach.sh list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" -./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" Copied: SwiftApps/swift-galaxy/swift/test.sh (from rev 7819, SwiftApps/swift-galaxy/swift/swiftforeachtest.sh) =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 15:54:36 UTC (rev 7820) @@ -0,0 +1,12 @@ +#!/bin/bash + +#!/bin/bash + +echo "List, no input parameters and files" +./swiftforeach.sh list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" + +echo "Range, no input parameters and files" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" + +echo "List, string input parameters" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there!" From ketan at ci.uchicago.edu Tue May 6 11:00:41 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 11:00:41 -0500 (CDT) Subject: [Swift-commit] r7821 - SwiftApps/swift-galaxy/swift Message-ID: <20140506160041.7E2029D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 11:00:41 -0500 (Tue, 06 May 2014) New Revision: 7821 Modified: SwiftApps/swift-galaxy/swift/test.sh Log: more tests Modified: SwiftApps/swift-galaxy/swift/test.sh =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 15:54:36 UTC (rev 7820) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 16:00:41 UTC (rev 7821) @@ -9,4 +9,10 @@ ./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" echo "List, string input parameters" -./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there!" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there list!" + +echo "Range, string input parameters" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hi there range!" + +echo "List, no input parameters and files stdin /dev/null" +./swiftforeach.sh list swiftK localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" From ketan at ci.uchicago.edu Tue May 6 11:22:32 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 11:22:32 -0500 (CDT) Subject: [Swift-commit] r7822 - SwiftApps/swift-galaxy/swift Message-ID: <20140506162232.7FEB49D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 11:22:32 -0500 (Tue, 06 May 2014) New Revision: 7822 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh SwiftApps/swift-galaxy/swift/test.sh Log: T test Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 16:00:41 UTC (rev 7821) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 16:22:32 UTC (rev 7822) @@ -140,8 +140,10 @@ touch None # Create a "None" file in case user does not specify any stdin file -#### End common setup #### +############# End common setup ################ + + if [ "$k_or_t" = "swiftK" ] ; then #### invoke K $BASEDIR/buildsite.sh $sites $wdir Modified: SwiftApps/swift-galaxy/swift/test.sh =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 16:00:41 UTC (rev 7821) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 16:22:32 UTC (rev 7822) @@ -16,3 +16,6 @@ echo "List, no input parameters and files stdin /dev/null" ./swiftforeach.sh list swiftK localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" + +echo "T" +./swiftforeach.sh list swiftT localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" From ketan at ci.uchicago.edu Tue May 6 12:04:31 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 12:04:31 -0500 (CDT) Subject: [Swift-commit] r7823 - SwiftApps/swift-galaxy/swift Message-ID: <20140506170431.4A2E89D018@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 12:04:31 -0500 (Tue, 06 May 2014) New Revision: 7823 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh SwiftApps/swift-galaxy/swift/test.sh Log: improve T script gen Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 16:22:32 UTC (rev 7822) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 17:04:31 UTC (rev 7823) @@ -191,23 +191,28 @@ */ import files; + import string; + import io; app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ - $interpreter _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; + "$interpreter" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; } +main{ $stringarrayexpr $filearrayexpr $stdinfilexpr - file exec<"$executable">; - file listfile<"$listfile">; + file exec=input_file("$executable"); + file listfile=input_file("$listfile"); + string list[] = split(read(input_file(listfile)),"\n"); foreach l, i in list{ file out ; file err ; (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); } + } EOF stc=$(which stc) Modified: SwiftApps/swift-galaxy/swift/test.sh =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 16:22:32 UTC (rev 7822) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 17:04:31 UTC (rev 7823) @@ -2,6 +2,20 @@ #!/bin/bash +cat << EOF > list.txt +a +b +c +EOF + +cat << EOF > x.sh +#!/bin/bash + +echo $@ +EOF + +chmod 755 x.sh + echo "List, no input parameters and files" ./swiftforeach.sh list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" From ketan at ci.uchicago.edu Tue May 6 12:26:22 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 12:26:22 -0500 (CDT) Subject: [Swift-commit] r7824 - SwiftApps/swift-galaxy/swift Message-ID: <20140506172622.C2CDA9D018@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 12:26:22 -0500 (Tue, 06 May 2014) New Revision: 7824 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: Swift T works for simple case Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 17:04:31 UTC (rev 7823) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 17:26:22 UTC (rev 7824) @@ -66,9 +66,17 @@ #workout standard input file if [ "${stdin}" ] ; then - stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' + if [ "${k_or_t}" = "swiftK" ] ; then + stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' + else + stdinfilexpr='file stdinfile = input('\"${stdin}\"');' + fi else - stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' + if [ "${k_or_t}" = "swiftK" ] ; then + stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' + else + stdinfilexpr='file stdinfile = input('\"/dev/null\"');' + fi fi #workout the array only if user adds args else make it blank @@ -203,15 +211,16 @@ $filearrayexpr $stdinfilexpr - file exec=input_file("$executable"); - file listfile=input_file("$listfile"); + file exec=input("$executable"); + file listfile=input("$listfile"); - string list[] = split(read(input_file(listfile)),"\n"); + string list[] = split(read(listfile),"\n"); foreach l, i in list{ file out ; file err ; (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); } + printf("%s", "Done!"); } EOF From ketan at ci.uchicago.edu Tue May 6 13:30:03 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 13:30:03 -0500 (CDT) Subject: [Swift-commit] r7825 - SwiftApps/swift-galaxy/swift Message-ID: <20140506183003.AAF729D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 13:30:03 -0500 (Tue, 06 May 2014) New Revision: 7825 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: add numproc Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 17:26:22 UTC (rev 7824) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 18:30:03 UTC (rev 7825) @@ -29,6 +29,7 @@ if [ "$range_or_list" == "list" ] ; then listfile=$1 shift + n=$(wc -l < $listfile) else rstart=$1 shift @@ -36,6 +37,7 @@ shift stepsize=$1 shift + n=$(( ($rend - $rstart) / $stepsize )) fi outloc=$1 @@ -229,7 +231,7 @@ $stc -j $(\which java) $wdir/script.swift "${swiftargs}" 2>$logfile - $turbine -n 3 $wdir/script.tcl + $turbine -n $n $wdir/script.tcl EXITCODE=$? From ketan at ci.uchicago.edu Tue May 6 13:50:56 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 13:50:56 -0500 (CDT) Subject: [Swift-commit] r7826 - SwiftApps/swift-galaxy/swift Message-ID: <20140506185056.68D219D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 13:50:56 -0500 (Tue, 06 May 2014) New Revision: 7826 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: take care of list and range for T Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 18:30:03 UTC (rev 7825) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 18:50:56 UTC (rev 7826) @@ -26,7 +26,7 @@ stdin=$1 shift -if [ "$range_or_list" == "list" ] ; then +if [ "$range_or_list" = "list" ] ; then listfile=$1 shift n=$(wc -l < $listfile) @@ -214,12 +214,18 @@ $stdinfilexpr file exec=input("$executable"); - file listfile=input("$listfile"); - - string list[] = split(read(listfile),"\n"); - foreach l, i in list{ - file out ; - file err ; + $( if [ "$range_or_list" = "list" ] ; then + echo "file listfile=input("$listfile");" + echo "string list[] = split(read(listfile),"\n");" + echo "foreach l, i in list{" + echo "file out ;" + echo "file err ;" + else + echo "foreach i in [$rstart:$rend:$stepsize]{" + echo "file out ;" + echo "file err ;" + fi + ) (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); } printf("%s", "Done!"); From ketan at ci.uchicago.edu Tue May 6 13:57:43 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 13:57:43 -0500 (CDT) Subject: [Swift-commit] r7827 - SwiftApps/swift-galaxy/swift Message-ID: <20140506185743.14F739D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 13:57:42 -0500 (Tue, 06 May 2014) New Revision: 7827 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh SwiftApps/swift-galaxy/swift/test.sh Log: test and format Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 18:50:56 UTC (rev 7826) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 18:57:42 UTC (rev 7827) @@ -218,12 +218,12 @@ echo "file listfile=input("$listfile");" echo "string list[] = split(read(listfile),"\n");" echo "foreach l, i in list{" - echo "file out ;" - echo "file err ;" + echo " file out ;" + echo " file err ;" else echo "foreach i in [$rstart:$rend:$stepsize]{" - echo "file out ;" - echo "file err ;" + echo " file out ;" + echo " file err ;" fi ) (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); Modified: SwiftApps/swift-galaxy/swift/test.sh =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 18:50:56 UTC (rev 7826) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 18:57:42 UTC (rev 7827) @@ -31,5 +31,8 @@ echo "List, no input parameters and files stdin /dev/null" ./swiftforeach.sh list swiftK localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" -echo "T" +echo "T list" ./swiftforeach.sh list swiftT localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" + +echo "T range" +./swiftforeach.sh range swiftT localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" From ketan at ci.uchicago.edu Tue May 6 14:11:07 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 14:11:07 -0500 (CDT) Subject: [Swift-commit] r7828 - SwiftApps/swift-galaxy/swift Message-ID: <20140506191107.E34639D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 14:11:07 -0500 (Tue, 06 May 2014) New Revision: 7828 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.xml SwiftApps/swift-galaxy/swift/test.sh Log: add vesta and mira Modified: SwiftApps/swift-galaxy/swift/swiftforeach.xml =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-06 18:57:42 UTC (rev 7827) +++ SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-06 19:11:07 UTC (rev 7828) @@ -88,6 +88,8 @@ + + Modified: SwiftApps/swift-galaxy/swift/test.sh =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 18:57:42 UTC (rev 7827) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-06 19:11:07 UTC (rev 7828) @@ -17,22 +17,22 @@ chmod 755 x.sh echo "List, no input parameters and files" -./swiftforeach.sh list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" +./swiftforeach.sh list swiftK Localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" echo "Range, no input parameters and files" -./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" +./swiftforeach.sh range swiftK Localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" echo "List, string input parameters" -./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there list!" +./swiftforeach.sh range swiftK Localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there list!" echo "Range, string input parameters" -./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hi there range!" +./swiftforeach.sh range swiftK Localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hi there range!" echo "List, no input parameters and files stdin /dev/null" -./swiftforeach.sh list swiftK localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" +./swiftforeach.sh list swiftK Localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" echo "T list" -./swiftforeach.sh list swiftT localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" +./swiftforeach.sh list swiftT Localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" echo "T range" -./swiftforeach.sh range swiftT localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" +./swiftforeach.sh range swiftT Localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" From ketan at ci.uchicago.edu Tue May 6 14:42:57 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 14:42:57 -0500 (CDT) Subject: [Swift-commit] r7829 - SwiftApps/swift-galaxy/swift Message-ID: <20140506194257.D5DC19D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 14:42:57 -0500 (Tue, 06 May 2014) New Revision: 7829 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: T on Mira and Vesta Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 19:11:07 UTC (rev 7828) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 19:42:57 UTC (rev 7829) @@ -231,16 +231,31 @@ printf("%s", "Done!"); } EOF - - stc=$(which stc) - turbine=$(which turbine) - $stc -j $(\which java) $wdir/script.swift "${swiftargs}" 2>$logfile - - $turbine -n $n $wdir/script.tcl +case {$sites} in + "Localhost" ) + STC=$(which stc) + turbine=$(which turbine) + $STC -j $(\which java) $wdir/script.swift 2>$logfile + $turbine -n $n $wdir/script.tcl "${swiftargs}" + EXITCODE=$? + ;; +"Vesta" ) + TURBINE_HOME=/home/wozniak/Public/sfw/turbine + STC=/home/wozniak/Public/sfw/stc/bin/stc + $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift "${swiftargs}" 2>$logfile + $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl + EXITCODE=$? + ;; +"Mira" ) + TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine + STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc + $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift "${swiftargs}" 2>$logfile + $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl + EXITCODE=$? + ;; +esac - EXITCODE=$? - fi cat << EOF > $outlistfile From ketan at ci.uchicago.edu Tue May 6 16:47:59 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 6 May 2014 16:47:59 -0500 (CDT) Subject: [Swift-commit] r7830 - SwiftApps/swift-galaxy/swift Message-ID: <20140506214759.1D8B59D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-06 16:47:58 -0500 (Tue, 06 May 2014) New Revision: 7830 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: swift/T+Galaxy+Mira close to working but not yet Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 19:42:57 UTC (rev 7829) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-06 21:47:58 UTC (rev 7830) @@ -49,7 +49,7 @@ stringargs=$1 shift -swift=$(which swift) +swift=$(\which swift 2>/dev/null) BASEDIR=$(dirname $0) #workout the file args @@ -232,10 +232,11 @@ } EOF -case {$sites} in +EXITCODE=0 +case $sites in "Localhost" ) - STC=$(which stc) - turbine=$(which turbine) + STC=$(\which stc) + turbine=$(\which turbine) $STC -j $(\which java) $wdir/script.swift 2>$logfile $turbine -n $n $wdir/script.tcl "${swiftargs}" EXITCODE=$? @@ -251,7 +252,7 @@ TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift "${swiftargs}" 2>$logfile - $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl + $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl EXITCODE=$? ;; esac From davidk at ci.uchicago.edu Wed May 7 08:58:57 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 7 May 2014 08:58:57 -0500 (CDT) Subject: [Swift-commit] r7831 - trunk/docs/userguide Message-ID: <20140507135857.66D3C9D00D@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-07 08:58:56 -0500 (Wed, 07 May 2014) New Revision: 7831 Modified: trunk/docs/userguide/configuration Log: Formatting Modified: trunk/docs/userguide/configuration =================================================================== --- trunk/docs/userguide/configuration 2014-05-06 21:47:58 UTC (rev 7830) +++ trunk/docs/userguide/configuration 2014-05-07 13:58:56 UTC (rev 7831) @@ -401,8 +401,8 @@ |true, false |false |When provider staging is enabled and provider.staging.pin.swiftfiles -|is set, cache some small files needed by Swift to avoid the cost -|of staging more than once. + is set, cache some small files needed by Swift to avoid the cost + of staging more than once. |sitedir.keep |true, false From ketan at ci.uchicago.edu Wed May 7 10:04:37 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Wed, 7 May 2014 10:04:37 -0500 (CDT) Subject: [Swift-commit] r7832 - SwiftApps/swift-galaxy/swift Message-ID: <20140507150437.466D3178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-07 10:04:37 -0500 (Wed, 07 May 2014) New Revision: 7832 Added: SwiftApps/swift-galaxy/swift/script.swift Modified: SwiftApps/swift-galaxy/swift/test.sh Log: swift source for debug Added: SwiftApps/swift-galaxy/swift/script.swift =================================================================== --- SwiftApps/swift-galaxy/swift/script.swift (rev 0) +++ SwiftApps/swift-galaxy/swift/script.swift 2014-05-07 15:04:37 UTC (rev 7832) @@ -0,0 +1,21 @@ +import files; +import string; +import io; + +app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + "bash" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; +} + +main{ + string stringargs[]=["hello","there","T"]; + file fileargs[]; + file stdinfile = input("/dev/null"); + + file exec=input("x.sh"); + foreach i in [0:99:1]{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); + } + printf("%s", "Done!"); +} Modified: SwiftApps/swift-galaxy/swift/test.sh =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh 2014-05-07 13:58:56 UTC (rev 7831) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-07 15:04:37 UTC (rev 7832) @@ -11,7 +11,7 @@ cat << EOF > x.sh #!/bin/bash -echo $@ +echo \$\@ EOF chmod 755 x.sh From davidk at ci.uchicago.edu Wed May 7 10:06:39 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 7 May 2014 10:06:39 -0500 (CDT) Subject: [Swift-commit] r7833 - in trunk: bin docs/userguide Message-ID: <20140507150639.F0E01178884@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-07 10:06:39 -0500 (Wed, 07 May 2014) New Revision: 7833 Modified: trunk/bin/swiftrun trunk/docs/userguide/configuration Log: Add and document site property stagingMethod Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2014-05-07 15:04:37 UTC (rev 7832) +++ trunk/bin/swiftrun 2014-05-07 15:06:39 UTC (rev 7833) @@ -51,6 +51,7 @@ 'providerattributes' => 'globus.providerAttributes', 'remotemonitorenabled' => 'globus.remoteMonitorEnabled', 'slurm' => 'globus.slurm', + 'stagingmethod' => 'swift.stagingMethod', 'taskdir' => 'globus.scratch', 'tasksperworker' => 'globus.jobsPerNode', 'taskthrottle' => 'karajan.jobThrottle', Modified: trunk/docs/userguide/configuration =================================================================== --- trunk/docs/userguide/configuration 2014-05-07 15:04:37 UTC (rev 7832) +++ trunk/docs/userguide/configuration 2014-05-07 15:06:39 UTC (rev 7833) @@ -227,6 +227,16 @@ the line "#SBATCH --mail-user=username". |site.midway.slurm.mail-user=username +|stagingMethod +|When provider staging is enabled, this option will specify the staging mechanism for +use for each site. If set to 'file', staging is done from a filesystem accessible to +the coaster service (typically running on the head node). If set to 'proxy', staging +is done from a filesystem accessible to the client machine that swift is running on, +and is proxied through the coaster service. If set to 'sfs' (short for "shared filesystem"), +staging is done by copying files to and from a filesystem accessible by the compute node +(such as an NFS or GPFS mount) +|site.osg.stagingMethod=file + |taskDir |Tasks will be run from this directory. In the absence of a taskDir definition, Swift will run the task from workdir. From davidk at ci.uchicago.edu Wed May 7 10:18:02 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 7 May 2014 10:18:02 -0500 (CDT) Subject: [Swift-commit] r7834 - branches/release-0.95/bin Message-ID: <20140507151802.024FA178884@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-07 10:18:01 -0500 (Wed, 07 May 2014) New Revision: 7834 Modified: branches/release-0.95/bin/swiftrun Log: stagingmethod Modified: branches/release-0.95/bin/swiftrun =================================================================== --- branches/release-0.95/bin/swiftrun 2014-05-07 15:06:39 UTC (rev 7833) +++ branches/release-0.95/bin/swiftrun 2014-05-07 15:18:01 UTC (rev 7834) @@ -51,6 +51,7 @@ 'providerattributes' => 'globus.providerAttributes', 'remotemonitorenabled' => 'globus.remoteMonitorEnabled', 'slurm' => 'globus.slurm', + 'stagingmethod' => 'swift.stagingMethod', 'taskdir' => 'globus.scratch', 'tasksperworker' => 'globus.jobsPerNode', 'taskthrottle' => 'karajan.jobThrottle', From swift at ci.uchicago.edu Wed May 7 15:30:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 7 May 2014 15:30:03 -0500 (CDT) Subject: [Swift-commit] cog r3897 Message-ID: <20140507203003.EDE6E8D0013C@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3897 | hategan | 2014-05-07 15:29:40 -0500 (Wed, 07 May 2014) | 1 line deal better with errors in the lead job when a soft image is requested (i.e. signal all workers and jobs that a failure has occurred) ------------------------------------------------------------------------ Index: modules/provider-coaster/resources/worker.pl =================================================================== --- modules/provider-coaster/resources/worker.pl (revision 3896) +++ modules/provider-coaster/resources/worker.pl (working copy) @@ -145,8 +145,14 @@ my $SOFT_IMAGE_CREATE_LOCK; my $SOFT_IMAGE_USE_LOCK; my $SOFT_IMAGE_DIR; +# true if this is the first worker on a node my $SOFT_IMAGE_LEAD_PROCESS = 0; +# true if this is the first job in this worker my $SOFT_IMAGE_FIRST_IN_PROCESS = 1; +# keep track of the job that stages in the soft image +# any errors that occur with this job should cause the worker +# to signal all other workers on this node to fail and then quit +my $SOFT_IMAGE_JOB_ID; use constant BUFSZ => 2048; use constant IOBUFSZ => 32768; @@ -1088,6 +1094,9 @@ sub queueJobStatusCmd { my ($jobid, $statusCode, $errorCode, $msg) = @_; + if ($statusCode == FAILED) { + checkSoftimageJobFailure($jobid, $msg); + } queueCmd((nullCB(), "JOBSTATUS", $jobid, encodeInt($statusCode), encodeInt($errorCode), $msg, NULL_TIMESTAMP)); } @@ -1095,6 +1104,9 @@ sub queueJobStatusCmdExt { my ($jobid, $statusCode, $errorCode, $msg, $out, $err) = @_; + if ($statusCode == FAILED) { + checkSoftimageJobFailure($jobid, $msg); + } queueCmd((nullCB(), "JOBSTATUS", $jobid, encodeInt($statusCode), encodeInt($errorCode), $msg, NULL_TIMESTAMP, $out, $err)); } @@ -1989,6 +2001,12 @@ $SOFT_IMAGE_CREATE_LOCK = writeLock("$SOFT_IMAGE_DIR/.create"); unlock($SOFT_IMAGE_USE_LOCK); $SOFT_IMAGE_USE_LOCK = readLock("$SOFT_IMAGE_DIR/.use"); + + # make sure no errors from previous runs are there + if (-f "$SOFT_IMAGE_DIR/.error") { + unlink("$SOFT_IMAGE_DIR/.error"); + } + return 1; } else { @@ -2044,6 +2062,17 @@ } } +sub checkSoftimageJobFailure { + my ($JOBID, $err) = @_; + + if ($JOBID == $SOFT_IMAGE_JOB_ID) { + $SOFT_IMAGE_JOB_ID = -1; + open(my $ERRF, ">$SOFT_IMAGE_DIR/.error"); + print $ERRF $err; + close($ERRF); + unlock($SOFT_IMAGE_CREATE_LOCK); + } +} sub cleanSoftImage { if (!defined $SOFT_IMAGE_DIR) { @@ -2137,6 +2166,7 @@ push @STAGEIND, $dest; $SOFTIMAGE = $dest; } + $SOFT_IMAGE_JOB_ID = $JOBID; } else { # prevent job from trying to unpack the image @@ -2182,7 +2212,8 @@ $JOB{$pair[0]} = $pair[1]; } } - if (checkJob($tag, $JOBID, \%JOB)) { + my $err = checkJob($tag, $JOBID, \%JOB); + if ($err eq "") { $JOBDATA{$JOBID} = { stagein => \@STAGEIN, stageind => \@STAGEIND, @@ -2201,6 +2232,10 @@ stagein($JOBID); } + else { + queueError($tag, ($err)); + checkSoftimageJobFailure($JOBID, $err); + } } sub checkJob() { @@ -2214,12 +2249,10 @@ wlog DEBUG, "$JOBID Job details $ds\n"; - queueError($tag, ("Missing job identity")); - return 0; + return "Missing job identity"; } elsif (!(defined $executable)) { - queueError($tag, ("Missing executable")); - return 0; + return "Missing executable"; } else { my $dir = $$JOB{directory}; @@ -2231,8 +2264,7 @@ my $c; foreach $c (@$cleanup) { if (substr($c, 0, $dirlen) ne $dir) { - queueError($tag, ("Cannot clean up outside of the job directory (cleanup: $c, jobdir: $dir)")); - return 0; + return "Cannot clean up outside of the job directory (cleanup: $c, jobdir: $dir)"; } } chdir $dir; @@ -2240,7 +2272,7 @@ wlog DEBUG, "$JOBID Sending submit reply (tag=$tag)\n"; queueReply($tag, ("OK")); wlog DEBUG, "$JOBID Submit reply sent (tag=$tag)\n"; - return 1; + return ""; } } @@ -2438,13 +2470,20 @@ wlog DEBUG, "Got soft image\n"; # no need to hold lock after that unlock($createLock); + if (-f "$SOFT_IMAGE_DIR/.error") { + open(my $ERRF, "<$SOFT_IMAGE_DIR/.error"); + my $err = ""; + while (<$ERRF>) { + $err .= $_; + } + dieNicely("Soft image deployment failed: $err"); + } $ENV{SOFTIMAGE} = $SOFT_IMAGE_DIR; } my $cwd = getcwd(); - # wlog DEBUG, "CWD: $cwd\n"; - # wlog DEBUG, "Running $executable\n"; + my $ename; foreach $ename (keys %$JOBENV) { $ENV{$ename} = $$JOBENV{$ename}; @@ -2473,7 +2512,10 @@ open STDERR, ">$serr" or dieNicely("Cannot redirect STDERR"); } close STDIN; - + + #wlog DEBUG, "CWD: $cwd\n"; + #wlog DEBUG, "Running $executable\n"; + exec { $executable } @$JOBARGS or print $WR "Could not execute $executable: $!\n"; die "Could not execute $executable: $!"; } From ketan at ci.uchicago.edu Thu May 8 16:31:10 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 8 May 2014 16:31:10 -0500 (CDT) Subject: [Swift-commit] r7835 - SwiftApps/swift-galaxy/swift Message-ID: <20140508213110.CEAFB9D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-08 16:31:09 -0500 (Thu, 08 May 2014) New Revision: 7835 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh SwiftApps/swift-galaxy/swift/swiftforeach.xml SwiftApps/swift-galaxy/swift/test.sh Log: test for T Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-07 15:18:01 UTC (rev 7834) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-08 21:31:09 UTC (rev 7835) @@ -1,7 +1,7 @@ #!/bin/bash - # originally list +#echo $@ > /tmp/args.txt #set workdir wdir=$(mktemp -d /tmp/swift-gal.XXXX) @@ -234,24 +234,31 @@ EXITCODE=0 case $sites in - "Localhost" ) + "localhost" ) + echo "in localhost" >> /tmp/status.txt STC=$(\which stc) + echo "got stc" >> /tmp/status.txt turbine=$(\which turbine) + echo "got turbine" >> /tmp/status.txt $STC -j $(\which java) $wdir/script.swift 2>$logfile + echo "done running stc" >> /tmp/status.txt + echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt $turbine -n $n $wdir/script.tcl "${swiftargs}" + echo "done running turbine, for $n" >> /tmp/status.txt EXITCODE=$? + echo "exit code obtained" >> /tmp/status.txt ;; "Vesta" ) TURBINE_HOME=/home/wozniak/Public/sfw/turbine STC=/home/wozniak/Public/sfw/stc/bin/stc - $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift "${swiftargs}" 2>$logfile + $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl EXITCODE=$? ;; "Mira" ) TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc - $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift "${swiftargs}" 2>$logfile + $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl EXITCODE=$? ;; @@ -266,5 +273,6 @@ if [ "$EXITCODE" -ne "0" ]; then cat $logfile >&2 fi + exit $EXITCODE Modified: SwiftApps/swift-galaxy/swift/swiftforeach.xml =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-07 15:18:01 UTC (rev 7834) +++ SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-08 21:31:09 UTC (rev 7835) @@ -158,7 +158,7 @@ - + Modified: SwiftApps/swift-galaxy/swift/test.sh =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh 2014-05-07 15:18:01 UTC (rev 7834) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-08 21:31:09 UTC (rev 7835) @@ -17,22 +17,25 @@ chmod 755 x.sh echo "List, no input parameters and files" -./swiftforeach.sh list swiftK Localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" +./swiftforeach.sh list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" echo "Range, no input parameters and files" -./swiftforeach.sh range swiftK Localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" echo "List, string input parameters" -./swiftforeach.sh range swiftK Localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there list!" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there list!" echo "Range, string input parameters" -./swiftforeach.sh range swiftK Localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hi there range!" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hi there range!" echo "List, no input parameters and files stdin /dev/null" -./swiftforeach.sh list swiftK Localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" +./swiftforeach.sh list swiftK localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" echo "T list" -./swiftforeach.sh list swiftT Localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" +./swiftforeach.sh list swiftT localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" echo "T range" -./swiftforeach.sh range swiftT Localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" +./swiftforeach.sh range swiftT localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" + +echo "Call from Galaxy" +bash /nfs2/ketan/galaxy-dist/tools/swift/swiftforeach.sh "range" "swiftT" "localhost" "" "sh" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_1.dat" "None" "0" "9" "1" "$HOME/swift-sandbox" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_12.dat" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_13.dat" "hello how are you" From ketan at ci.uchicago.edu Thu May 8 16:41:39 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 8 May 2014 16:41:39 -0500 (CDT) Subject: [Swift-commit] r7836 - SwiftApps/Swift-MapRed/Paper Message-ID: <20140508214139.67EA09D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-08 16:41:39 -0500 (Thu, 08 May 2014) New Revision: 7836 Modified: SwiftApps/Swift-MapRed/Paper/.bibtex.out SwiftApps/Swift-MapRed/Paper/swifthadoop.blg Log: _ Modified: SwiftApps/Swift-MapRed/Paper/.bibtex.out =================================================================== --- SwiftApps/Swift-MapRed/Paper/.bibtex.out 2014-05-08 21:31:09 UTC (rev 7835) +++ SwiftApps/Swift-MapRed/Paper/.bibtex.out 2014-05-08 21:41:39 UTC (rev 7836) @@ -1,4 +1,4 @@ -This is BibTeX, Version 0.99d (TeX Live 2013/MacPorts 2013_2) +This is BibTeX, Version 0.99d (TeX Live 2013) The top-level auxiliary file: swifthadoop.aux The style file: IEEEtran.bst Database file #1: ref.bib Modified: SwiftApps/Swift-MapRed/Paper/swifthadoop.blg =================================================================== --- SwiftApps/Swift-MapRed/Paper/swifthadoop.blg 2014-05-08 21:31:09 UTC (rev 7835) +++ SwiftApps/Swift-MapRed/Paper/swifthadoop.blg 2014-05-08 21:41:39 UTC (rev 7836) @@ -1,4 +1,4 @@ -This is BibTeX, Version 0.99d (TeX Live 2013/MacPorts 2013_2) +This is BibTeX, Version 0.99d (TeX Live 2013) Capacity: max_strings=35307, hash_size=35307, hash_prime=30011 The top-level auxiliary file: swifthadoop.aux The style file: IEEEtran.bst From ketan at ci.uchicago.edu Thu May 8 16:42:10 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 8 May 2014 16:42:10 -0500 (CDT) Subject: [Swift-commit] r7837 - SwiftApps/Swift-MapRed/Paper Message-ID: <20140508214210.57CD49D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-08 16:42:10 -0500 (Thu, 08 May 2014) New Revision: 7837 Removed: SwiftApps/Swift-MapRed/Paper/.bibtex.out SwiftApps/Swift-MapRed/Paper/swifthadoop.blg Log: removing unnencessary files from svn Deleted: SwiftApps/Swift-MapRed/Paper/.bibtex.out =================================================================== --- SwiftApps/Swift-MapRed/Paper/.bibtex.out 2014-05-08 21:41:39 UTC (rev 7836) +++ SwiftApps/Swift-MapRed/Paper/.bibtex.out 2014-05-08 21:42:10 UTC (rev 7837) @@ -1,9 +0,0 @@ -This is BibTeX, Version 0.99d (TeX Live 2013) -The top-level auxiliary file: swifthadoop.aux -The style file: IEEEtran.bst -Database file #1: ref.bib --- IEEEtran.bst version 1.12 (2007/01/11) by Michael Shell. --- http://www.michaelshell.org/tex/ieeetran/bibtex/ --- See the "IEEEtran_bst_HOWTO.pdf" manual for usage information. - -Done. Deleted: SwiftApps/Swift-MapRed/Paper/swifthadoop.blg =================================================================== --- SwiftApps/Swift-MapRed/Paper/swifthadoop.blg 2014-05-08 21:41:39 UTC (rev 7836) +++ SwiftApps/Swift-MapRed/Paper/swifthadoop.blg 2014-05-08 21:42:10 UTC (rev 7837) @@ -1,56 +0,0 @@ -This is BibTeX, Version 0.99d (TeX Live 2013) -Capacity: max_strings=35307, hash_size=35307, hash_prime=30011 -The top-level auxiliary file: swifthadoop.aux -The style file: IEEEtran.bst -Reallocated singl_function (elt_size=4) to 100 items from 50. -Reallocated singl_function (elt_size=4) to 100 items from 50. -Reallocated singl_function (elt_size=4) to 100 items from 50. -Reallocated wiz_functions (elt_size=4) to 6000 items from 3000. -Reallocated singl_function (elt_size=4) to 100 items from 50. -Database file #1: ref.bib --- IEEEtran.bst version 1.12 (2007/01/11) by Michael Shell. --- http://www.michaelshell.org/tex/ieeetran/bibtex/ --- See the "IEEEtran_bst_HOWTO.pdf" manual for usage information. - -Done. -You've used 10 entries, - 4024 wiz_defined-function locations, - 874 strings with 9291 characters, -and the built_in function-call counts, 9202 in all, are: -= -- 714 -> -- 220 -< -- 80 -+ -- 119 -- -- 36 -* -- 469 -:= -- 1335 -add.period$ -- 23 -call.type$ -- 10 -change.case$ -- 10 -chr.to.int$ -- 201 -cite$ -- 10 -duplicate$ -- 629 -empty$ -- 778 -format.name$ -- 45 -if$ -- 2142 -int.to.chr$ -- 0 -int.to.str$ -- 10 -missing$ -- 111 -newline$ -- 63 -num.names$ -- 10 -pop$ -- 269 -preamble$ -- 1 -purify$ -- 0 -quote$ -- 2 -skip$ -- 699 -stack$ -- 0 -substring$ -- 476 -swap$ -- 525 -text.length$ -- 13 -text.prefix$ -- 0 -top$ -- 5 -type$ -- 10 -warning$ -- 0 -while$ -- 42 -width$ -- 12 -write$ -- 133 From ketan at ci.uchicago.edu Thu May 8 16:57:35 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Thu, 8 May 2014 16:57:35 -0500 (CDT) Subject: [Swift-commit] r7838 - SwiftApps/swift-galaxy/swift Message-ID: <20140508215735.938439D00D@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-08 16:57:35 -0500 (Thu, 08 May 2014) New Revision: 7838 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: verbose Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-08 21:42:10 UTC (rev 7837) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-08 21:57:35 UTC (rev 7838) @@ -243,7 +243,7 @@ $STC -j $(\which java) $wdir/script.swift 2>$logfile echo "done running stc" >> /tmp/status.txt echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt - $turbine -n $n $wdir/script.tcl "${swiftargs}" + $turbine -V -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt echo "done running turbine, for $n" >> /tmp/status.txt EXITCODE=$? echo "exit code obtained" >> /tmp/status.txt From swift at ci.uchicago.edu Thu May 8 18:50:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 8 May 2014 18:50:03 -0500 (CDT) Subject: [Swift-commit] cog r3898 Message-ID: <20140508235003.8F7238D0013C@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3898 | hategan | 2014-05-08 18:48:41 -0500 (Thu, 08 May 2014) | 1 line patch to use compression in GSS streams when 1.7 style constructor for DeflaterOutputStream is available but still allow compilation on 1.6 ------------------------------------------------------------------------ Index: modules/provider-coaster/src/org/globus/cog/coaster/channels/GSSChannel.java =================================================================== --- modules/provider-coaster/src/org/globus/cog/coaster/channels/GSSChannel.java (revision 3897) +++ modules/provider-coaster/src/org/globus/cog/coaster/channels/GSSChannel.java (working copy) @@ -12,6 +12,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.lang.reflect.Constructor; import java.net.URI; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; @@ -36,11 +37,21 @@ public class GSSChannel extends AbstractTCPChannel { private static final Logger logger = Logger.getLogger(GSSChannel.class); - + private static final boolean streamCompression; static { - streamCompression = "true".equals(System.getProperty("gss.channel.compression.enabled")); + boolean st; + try { + DeflaterOutputStream.class.getConstructor(OutputStream.class, boolean.class); + st = true; + } + catch (Exception e) { + logger.info("Exception caught trying to find DeflaterOutputStream(OutputStream, boolean) " + + "constructor. Disabling stream compression."); + st = false; + } + streamCompression = st; } private GssSocket socket; @@ -178,7 +189,21 @@ @Override protected void setOutputStream(OutputStream outputStream) { if (streamCompression) { - super.setOutputStream(new DeflaterOutputStream(outputStream, true)); + try { + /* + * Instantiate DeflaterOutputStream(out, syncFlush) using reflection + * since it is only available in 1.7, but we still want this + * to compile on 1.6. + */ + Constructor cons = + DeflaterOutputStream.class.getConstructor(OutputStream.class, boolean.class); + DeflaterOutputStream dos = cons.newInstance(outputStream, true); + super.setOutputStream(dos); + } + catch (Exception e) { + logger.warn("Failed to instantiate DeflaterOutputStream", e); + throw new RuntimeException(e); + } } else { super.setOutputStream(outputStream); From swift at ci.uchicago.edu Thu May 8 23:20:04 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 8 May 2014 23:20:04 -0500 (CDT) Subject: [Swift-commit] cog r3900 Message-ID: <20140509042004.C91B68D0013C@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3900 | hategan | 2014-05-08 23:18:41 -0500 (Thu, 08 May 2014) | 1 line allow null service contact ------------------------------------------------------------------------ Index: modules/provider-local/src/org/globus/cog/abstraction/impl/file/local/FileResourceImpl.java =================================================================== --- modules/provider-local/src/org/globus/cog/abstraction/impl/file/local/FileResourceImpl.java (revision 3899) +++ modules/provider-local/src/org/globus/cog/abstraction/impl/file/local/FileResourceImpl.java (working copy) @@ -55,7 +55,7 @@ } public FileResourceImpl(String name) { - super(name, "local", new ServiceContactImpl("localhost"), null); + super(name, "local", null, null); } protected FileResourceImpl(String name, String protocol) { From swift at ci.uchicago.edu Thu May 8 23:25:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 8 May 2014 23:25:03 -0500 (CDT) Subject: [Swift-commit] cog r3903 Message-ID: <20140509042503.8F7DB8D0013C@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3903 | hategan | 2014-05-08 23:24:09 -0500 (Thu, 08 May 2014) | 1 line misc improvements ------------------------------------------------------------------------ Index: modules/karajan/src/k/thr/LWThread.java =================================================================== --- modules/karajan/src/k/thr/LWThread.java (revision 3902) +++ modules/karajan/src/k/thr/LWThread.java (working copy) @@ -462,6 +462,10 @@ } return cid; } + + public int getForkID() { + return forkId; + } public final synchronized void sleep(long delay) { sleep(delay, 0); @@ -551,7 +555,20 @@ return child; } + + public final synchronized LWThread fork(int id, KRunnable r) { + if (DEBUG) + System.out.println(this + " fork(" + state + ")"); + LWThread child = new LWThread(id, r, stack.copy()); + + if (DEBUG) { + Exception e = new Exception(); + System.out.println(child + " forked by " + e.getStackTrace()[1].getClassName()); + } + return child; + } + public final void die() { if (DEBUG) System.out.println(this + " die()"); Index: modules/karajan/src/k/thr/Scheduler.java =================================================================== --- modules/karajan/src/k/thr/Scheduler.java (revision 3902) +++ modules/karajan/src/k/thr/Scheduler.java (working copy) @@ -47,8 +47,12 @@ Clock.init(); } - public synchronized void awake(final LWThread thread) { - if (sleeping.remove(thread)) { + public void awake(final LWThread thread) { + boolean wasSleeping; + synchronized(sleeping) { + wasSleeping = sleeping.remove(thread); + } + if (wasSleeping) { //System.err.println("awake(" + thread + ")"); schedule(thread); } @@ -85,10 +89,12 @@ return workers.getActiveCount() > 0 || !queue.isEmpty(); } - public synchronized void putToSleep(final LWThread thread) { + public void putToSleep(final LWThread thread) { if (thread.isSleeping()) { //System.err.println("putToSleep(" + thread + ")"); - sleeping.add(thread); + synchronized(sleeping) { + sleeping.add(thread); + } } } @@ -147,6 +153,8 @@ } public List getSleepingThreads() { - return new ArrayList(sleeping); + synchronized(sleeping) { + return new ArrayList(sleeping); + } } } From swift at ci.uchicago.edu Thu May 8 23:30:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 8 May 2014 23:30:03 -0500 (CDT) Subject: [Swift-commit] cog r3905 Message-ID: <20140509043003.D11A28D0013C@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3905 | hategan | 2014-05-08 23:28:53 -0500 (Thu, 08 May 2014) | 1 line an improved stack frame allocation scheme and made some things allocate a frame (particularly allocateHost, since most of its execution time is waiting for a free host, so having its own frame means that the allocation happens only when the host is in use ------------------------------------------------------------------------ Index: modules/karajan/src/org/globus/cog/karajan/analyzer/ChannelRef.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/analyzer/ChannelRef.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/analyzer/ChannelRef.java (working copy) @@ -15,6 +15,7 @@ import k.rt.Channel; import k.rt.EmptyChannel; import k.rt.FixedArgChannel; +import k.rt.FixedArgChannelDebug; import k.rt.FixedFutureArgChannel; import k.rt.FutureMemoryChannel; import k.rt.MemoryChannel; @@ -489,11 +490,15 @@ @Override public void create(Stack stack) { - FixedArgChannel c = new FixedArgChannel(stack.top(), first, last); + FixedArgChannel c; + if (CompilerSettings.DEBUG) { + c = new FixedArgChannelDebug(stack.top(), first, last); + c.setNames(names); + } + else { + c = new FixedArgChannel(stack.top(), first, last); + } stack.top().set(index, c); - if (CompilerSettings.DEBUG) { - c.setNames(names); - } } @Override Index: modules/karajan/src/org/globus/cog/karajan/analyzer/Scope.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/analyzer/Scope.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/analyzer/Scope.java (working copy) @@ -9,6 +9,7 @@ */ package org.globus.cog.karajan.analyzer; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -26,6 +27,14 @@ import org.globus.cog.karajan.parser.WrapperNode; public class Scope { + public static class Checkpoint { + protected final ArrayList map; + + public Checkpoint(ArrayList map) { + this.map = new ArrayList(map); + } + } + public abstract static class Def { public abstract Node newInstance(); } @@ -118,6 +127,14 @@ return parent.getContainerScope(); } + public Checkpoint checkpoint() { + return getContainerScope().checkpoint(); + } + + public void restore(Checkpoint c) { + getContainerScope().restore(c); + } + public boolean hasParams() { return params; } @@ -494,19 +511,40 @@ if (args == null || args.isEmpty()) { return; } - int index = getContainerScope().allocateContiguous(args.size()); for (Param p : args) { Var v = new ParamWrapperVar.Positional(p); + addVarNoIndex(v); + } + } + + public void allocatePositionalParams(List args) { + if (args == null || args.isEmpty()) { + return; + } + int index = getContainerScope().allocateContiguous(args.size(), owner); + for (Param p : args) { + Var v = new ParamWrapperVar.Positional(p); addVar(v, index++); } } - public ParamWrapperVar.IndexRange addOptionalParams(List args) { + public void addOptionalParams(List args) { if (args == null || args.isEmpty()) { + return; + } + + for (Param p : args) { + Var v = new ParamWrapperVar.Optional(p, null); + addVarNoIndex(v); + } + } + + public ParamWrapperVar.IndexRange allocateOptionalParams(List args) { + if (args == null || args.isEmpty()) { return null; } - int index = getContainerScope().allocateContiguous(args.size()); + int index = getContainerScope().allocateContiguous(args.size(), owner); ParamWrapperVar.IndexRange ir = new ParamWrapperVar.IndexRange(index, args.size()); for (Param p : args) { Var v = new ParamWrapperVar.Optional(p, ir); @@ -524,7 +562,11 @@ } public int addParams(List channels, List optional, List positional) { - int index = getContainerScope().allocateContiguous(channels.size() + optional.size() + positional.size()); + return addParams(channels, optional, positional, null); + } + + public int addParams(List channels, List optional, List positional, Object owner) { + int index = getContainerScope().allocateContiguous(channels.size() + optional.size() + positional.size(), owner); int first = index; for (Param c : channels) { Var.Channel v = new Var.Channel(c.name); Index: modules/karajan/src/org/globus/cog/karajan/analyzer/ContainerScope.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/analyzer/ContainerScope.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/analyzer/ContainerScope.java (working copy) @@ -33,6 +33,24 @@ return this; } + @Override + public Checkpoint checkpoint() { + if (CompilerSettings.DEBUG_STACK_ALLOCATION) { + List l = Collections.emptyList(); + dump("checkpoint", l); + } + return new Checkpoint(map); + } + + @Override + public void restore(Checkpoint c) { + this.map = c.map; + if (CompilerSettings.DEBUG_STACK_ALLOCATION) { + List l = Collections.emptyList(); + dump("restore", l); + } + } + public int allocate(Var ref) { int index = findUnused(); if (index == -1) { @@ -159,33 +177,37 @@ } public int allocateContiguous(int size) { - int pos = findHole(size); - if (pos != -1) { - if (CompilerSettings.DEBUG_STACK_ALLOCATION) { - dump("allocateContiguous", range(pos, pos + size - 1)); - } - return pos; - } - else { - int tailFree = map.size(); - for (int i = map.size() - 1; i >= 0; i--) { - if (map.get(i)) { - break; - } - else { - tailFree = i; - } - } - for (int i = 0; i < size; i++) { - set(i + tailFree); - } - if (CompilerSettings.DEBUG_STACK_ALLOCATION) { - dump("allocateContiguous", range(tailFree, tailFree + size - 1)); - } - return tailFree; - } + return allocateContiguous(size, null); } + public int allocateContiguous(int size, Object who) { + int pos = findHole(size); + if (pos != -1) { + if (CompilerSettings.DEBUG_STACK_ALLOCATION) { + dump("allocateContiguous(" + size + ") - " + who, range(pos, pos + size - 1)); + } + return pos; + } + else { + int tailFree = map.size(); + for (int i = map.size() - 1; i >= 0; i--) { + if (map.get(i)) { + break; + } + else { + tailFree = i; + } + } + for (int i = 0; i < size; i++) { + set(i + tailFree); + } + if (CompilerSettings.DEBUG_STACK_ALLOCATION) { + dump("allocateContiguous(" + size + ") - " + who, range(tailFree, tailFree + size - 1)); + } + return tailFree; + } + } + private int findHole(int size) { int index = 0; while (index + size < map.size()) { Index: modules/karajan/src/org/globus/cog/karajan/analyzer/VarRef.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/analyzer/VarRef.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/analyzer/VarRef.java (working copy) @@ -53,7 +53,7 @@ } public static class DynamicLocal extends VarRef { - protected final int index; + public final int index; public final String name; public DynamicLocal(String name, int index) { Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Import.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Import.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Import.java (working copy) @@ -38,6 +38,7 @@ import org.globus.cog.karajan.parser.ParsingException; import org.globus.cog.karajan.parser.WrapperNode; import org.globus.cog.karajan.util.KarajanProperties; +import org.globus.cog.karajan.util.Pair; public class Import extends InternalFunction { public static final Logger logger = Logger.getLogger(Import.class); @@ -137,13 +138,26 @@ } } } + + @Override + protected void scanNamed(WrapperNode w, Scope scope, List params, List> dynamicOptimized) + throws CompilationException { + // override to allow imports of non-identifier but static names + } @Override + protected void optimizePositional(WrapperNode w, Scope scope, List params, List> dynamicOptimized) + throws CompilationException { + // see above + } + + @Override + protected void scanNotSet(WrapperNode w, Scope scope, List optional) + throws CompilationException { + } + + @Override public Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { - if ("import @ swift.k, line: 137".equals(this.toString())) { - System.out.print(""); - } - if (name == null && file.getValue() == null) { throw new CompilationException(w, "Could not statically determine file name"); } @@ -158,7 +172,7 @@ context = scope.getVarRef("#context"); Resolved r; - try{ + try { r = resolve(name, props.getValue(), fileDir.getValue()); } catch (IOException ee) { Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/InternalFunction.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/InternalFunction.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/InternalFunction.java (working copy) @@ -36,7 +36,9 @@ import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.StaticChannel; import org.globus.cog.karajan.analyzer.Var; +import org.globus.cog.karajan.analyzer.VarRef; import org.globus.cog.karajan.parser.WrapperNode; +import org.globus.cog.karajan.util.Pair; public abstract class InternalFunction extends Sequential { protected static final int DYNAMIC = -1; @@ -149,6 +151,8 @@ catch (IllegalArgumentException e) { throw new ExecutionException(this, e.getMessage()); } + i = Integer.MAX_VALUE; + case Integer.MAX_VALUE: try { runBody(thr); } @@ -186,7 +190,7 @@ protected void initializeOptional(Stack stack) { if (firstOptionalIndex != -1) { - Arrays.fill(stack.top().getAll(), firstOptionalIndex, lastOptionalIndex, null); + Arrays.fill(stack.top().getAll(), firstOptionalIndex, lastOptionalIndex + 1, null); } } @@ -197,39 +201,55 @@ public final List optional; public final List positional; public final List channelParams; + public final List> dynamicOptimized; public final ParamWrapperVar.IndexRange ir; public ArgInfo(LinkedList blocks, Var.Channel vargs, StaticChannel vargValues, - List optional, List positional, List channelParams, ParamWrapperVar.IndexRange ir) { + List optional, List positional, List channelParams, + List> dynamicOptimized, + ParamWrapperVar.IndexRange ir) { this.blocks = blocks; this.vargs = vargs; this.vargValues = vargValues; this.optional = optional; this.positional = positional; this.channelParams = channelParams; + this.dynamicOptimized = dynamicOptimized; this.ir = ir; } } protected ArgInfo compileArgs(WrapperNode w, Signature sig, Scope scope) throws CompilationException { + if (w.toString().contains("stageIn @ swift-int-staging.k, line: 121")) { + System.out.print(""); + } resolveChannelReturns(w, sig, scope); resolveParamReturns(w, sig, scope); List channels = getChannelParams(sig); List optional = getOptionalParams(sig); List positional = getPositionalParams(sig); + // optionals first if (optional != null || positional != null) { scope.setParams(true); } - ParamWrapperVar.IndexRange ir = addParams(w, sig, scope, channels, optional, positional); + List> dynamicOptimized = new LinkedList>(); + + addParams(w, sig, scope, channels, optional, positional); + scanNamed(w, scope, optional, dynamicOptimized); + optimizePositional(w, scope, positional, dynamicOptimized); + scanNamed(w, scope, positional, dynamicOptimized); + scanNotSet(w, scope, optional); + + ParamWrapperVar.IndexRange ir = allocateParams(w, sig, scope, channels, optional, positional); LinkedList blocks = checkBlockArgs(w, sig); Var.Channel vargs = null; ArgMappingChannel amc = null; - if (!sig.getParams().isEmpty() || hasVargs) { + if (!positional.isEmpty() || !optional.isEmpty() || hasVargs) { if (hasVargs) { vargs = scope.lookupChannel("..."); vargs.setValue(amc = new ArgMappingChannel(w, positional, true)); @@ -239,29 +259,149 @@ } } - return new ArgInfo(blocks, vargs, amc, optional, positional, channels, ir); + return new ArgInfo(blocks, vargs, amc, optional, positional, channels, dynamicOptimized, ir); } - protected ParamWrapperVar.IndexRange addParams(WrapperNode w, Signature sig, Scope scope, List channels, + protected void scanNotSet(WrapperNode w, Scope scope, List optional) throws CompilationException { + if (w.nodeCount() == 0) { + for (Param p : optional) { + setArg(w, p, new ArgRef.Static(p.value)); + } + optional.clear(); + } + } + + protected void scanNamed(WrapperNode w, Scope scope, List params, + List> dynamicOptimized) throws CompilationException { + Iterator i = w.nodes().iterator(); + while (i.hasNext()) { + WrapperNode c = i.next(); + if (c.getNodeType().equals("k:named")) { + if (optimizeNamed(w, c, scope, params, dynamicOptimized)) { + i.remove(); + } + } + } + } + + private boolean optimizeNamed(WrapperNode w, WrapperNode c, Scope scope, List params, + List> dynamicOptimized) throws CompilationException { + if (c.nodeCount() == 2) { + WrapperNode nameNode = c.getNode(0); + if (nameNode.getNodeType().equals("k:var")) { + String name = nameNode.getText(); + Param p = getParam(params, name); + if (p != null) { + WrapperNode valueNode = c.getNode(1); + if (setValue(w, scope, p, valueNode, dynamicOptimized)) { + params.remove(p); + return true; + } + } + } + } + return false; + } + + protected void optimizePositional(WrapperNode w, Scope scope, List params, + List> dynamicOptimized) throws CompilationException { + int index = 0; + Iterator i = w.nodes().iterator(); + while (i.hasNext() && index < params.size()) { + WrapperNode c = i.next(); + Param p = params.get(index); + if (p.type == Param.Type.IDENTIFIER) { + return; + } + if (setValue(w, scope, p, c, dynamicOptimized)) { + params.remove(p); + i.remove(); + index--; + } + index++; + } + } + + private boolean setValue(WrapperNode w, Scope scope, Param p, WrapperNode c, + List> dynamicOptimized) throws CompilationException { + if (c.getNodeType().equals("k:var")) { + VarRef ref = scope.getVarRef(c.getText()); + if (ref.isStatic()) { + p.setValue(ref.getValue()); + setArg(w, p, new ArgRef.Static(ref.getValue())); + return true; + } + else if (ref instanceof VarRef.DynamicLocal) { + VarRef.DynamicLocal ref2 = (VarRef.DynamicLocal) ref; + setArg(w, p, new ArgRef.Dynamic(ref2.index)); + dynamicOptimized.add(new Pair(p, c.getText())); + return true; + } + else { + return false; + } + } + else if (c.getNodeType().equals("k:num")) { + // there is probably a better way to handle these cases + // perhaps by compiling children and then allocating entries on the frame + Object value; + if (c.getText().indexOf(".") >= 0) { + value = Double.parseDouble(c.getText()); + } + else { + value = Integer.parseInt(c.getText()); + } + p.setValue(value); + setArg(w, p, new ArgRef.Static(value)); + return true; + } + else if (c.getNodeType().equals("k:str")) { + p.setValue(c.getText()); + setArg(w, p, new ArgRef.Static(c.getText())); + return true; + } + else { + return false; + } + } + + private Param getParam(List params, String name) { + for (Param p : params) { + if (p.name.equals(name)) { + return p; + } + } + return null; + } + + protected void addParams(WrapperNode w, Signature sig, Scope scope, List channels, List optional, List positional) throws CompilationException { processIdentifierArgs(w, sig); prepareChannelParams(scope, channels); scope.addChannelParams(channels); scope.addPositionalParams(positional); - return scope.addOptionalParams(optional); + scope.addOptionalParams(optional); } + protected ParamWrapperVar.IndexRange allocateParams(WrapperNode w, Signature sig, Scope scope, List channels, + List optional, List positional) throws CompilationException { + scope.allocatePositionalParams(positional); + return scope.allocateOptionalParams(optional); + } + @Override - protected final Node compileChildren(WrapperNode w, Scope scope) throws CompilationException { - + protected final Node compileChildren(WrapperNode w, Scope scope) throws CompilationException { Signature sig = getSignature(); Scope argScope = new Scope(w, scope); - + Scope.Checkpoint chk = argScope.checkpoint(); + ArgInfo ai = compileArgs(w, sig, argScope); Node n = super.compileChildren(w, argScope); + checkStaticOptimizedParams(w, scope, ai.dynamicOptimized); + if (ai.ir != null) { // unused optional parameters scope.releaseRange(ai.ir.currentIndex(), ai.ir.lastIndex()); @@ -273,7 +413,9 @@ compileBlocks(w, sig, ai.blocks, scope); - if (compileBody(w, argScope, scope) == null && n == null && ai.blocks == null) { + Node self = compileBody(w, argScope, scope); + + if (self == null && n == null && ai.blocks == null) { argScope.close(); return null; } @@ -281,13 +423,22 @@ argScope.close(); if (this instanceof Pure && ai.blocks == null) { - return n; + return self; } else { return this; } } + private void checkStaticOptimizedParams(WrapperNode w, Scope scope, List> dynamicOptimized) throws CompilationException { + for (Pair p : dynamicOptimized) { + VarRef ref = scope.getVarRef(p.t); + if (ref.isStatic()) { + setArg(w, p.s, new ArgRef.Static(ref.getValue())); + } + } + } + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { return this; } Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/CompoundNode.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/CompoundNode.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/CompoundNode.java (working copy) @@ -47,10 +47,14 @@ protected Node compileChildren(WrapperNode w, Scope scope) throws CompilationException { for (WrapperNode c : w.nodes()) { + Scope.Checkpoint sc = scope.checkpoint(); Node n = compileChild(c, scope); if (n != null) { addChild(n); } + else { + scope.restore(sc); + } } if (childCount() == 0) { return null; Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/IncludedMain.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/IncludedMain.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/IncludedMain.java (working copy) @@ -23,7 +23,6 @@ import org.globus.cog.karajan.analyzer.CompilationException; import org.globus.cog.karajan.analyzer.NamedValue; import org.globus.cog.karajan.analyzer.Param; -import org.globus.cog.karajan.analyzer.ParamWrapperVar; import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.Var; @@ -63,20 +62,19 @@ scope.addVar(Namespace.VAR_NAME, ""); super.addLocals(scope); } + + @Override - protected ParamWrapperVar.IndexRange addParams(WrapperNode w, Signature sig, Scope scope, List channels, + protected void addParams(WrapperNode w, Signature sig, Scope scope, List channels, List optional, List positional) throws CompilationException { - ParamWrapperVar.IndexRange ir = super.addParams(w, sig, scope, channels, optional, positional); - + super.addParams(w, sig, scope, channels, optional, positional); Scope prev = scope.getRoot().getImportScope(); - - cr_export = bind(prev, scope, "export"); - cr_stdout = bind(prev, scope, "stdout"); - cr_stderr = bind(prev, scope, "stderr"); - cr_vargs = bind(prev, scope, "..."); - - return ir; + + cr_export = bind(prev, scope, "export"); + cr_stdout = bind(prev, scope, "stdout"); + cr_stderr = bind(prev, scope, "stderr"); + cr_vargs = bind(prev, scope, "..."); } private ChannelRef bind(Scope scope, Scope prev, String name) { Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/DefNode.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/DefNode.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/DefNode.java (working copy) @@ -9,15 +9,19 @@ */ package org.globus.cog.karajan.compiled.nodes; +import java.util.List; + import org.apache.log4j.Logger; import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Param; import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.Scope.JavaDef; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.Var; import org.globus.cog.karajan.parser.WrapperNode; +import org.globus.cog.karajan.util.Pair; public class DefNode extends InternalFunction { private static final Logger logger = Logger.getLogger(DefNode.class); @@ -65,4 +69,9 @@ } return null; } + + @Override + protected void optimizePositional(WrapperNode w, Scope scope, List params, List> dynamicOptimized) + throws CompilationException { + } } \ No newline at end of file Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/grid/AllocateHost.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/grid/AllocateHost.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/grid/AllocateHost.java (working copy) @@ -8,10 +8,7 @@ import java.io.IOException; import java.io.PrintStream; -import java.util.Collections; -import java.util.HashMap; import java.util.LinkedList; -import java.util.Map; import k.rt.ConditionalYield; import k.rt.Context; @@ -27,6 +24,7 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.CompilationException; import org.globus.cog.karajan.analyzer.CompilerSettings; +import org.globus.cog.karajan.analyzer.ContainerScope; import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.VarRef; @@ -46,6 +44,7 @@ private VarRef context; private VarRef var; + private int frameSize; @Override protected Signature getSignature() { @@ -72,10 +71,14 @@ allocateHost(thr); i++; case 2: + stack.enter(this, frameSize); + i++; + case 3: if (CompilerSettings.PERFORMANCE_COUNTERS) { startCount++; } body.run(thr); + stack.leave(); _finally(thr.getStack()); } } @@ -94,8 +97,10 @@ @Override protected void compileBlocks(WrapperNode w, Signature sig, LinkedList blocks, Scope scope) throws CompilationException { - var = scope.getVarRef(scope.addVar(name)); - super.compileBlocks(w, sig, blocks, scope); + var = scope.getVarRef(scope.addVar(name)); + ContainerScope cs = new ContainerScope(w, scope); + super.compileBlocks(w, sig, blocks, cs); + frameSize = cs.size(); } protected void allocateHost(LWThread thr) { Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Export.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Export.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Export.java (working copy) @@ -9,16 +9,20 @@ */ package org.globus.cog.karajan.compiled.nodes; +import java.util.List; + import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.CompilationException; import org.globus.cog.karajan.analyzer.NamedValue; +import org.globus.cog.karajan.analyzer.Param; import org.globus.cog.karajan.analyzer.Scope; -import org.globus.cog.karajan.analyzer.VarRef; import org.globus.cog.karajan.analyzer.Scope.Def; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.Var; +import org.globus.cog.karajan.analyzer.VarRef; import org.globus.cog.karajan.parser.WrapperNode; +import org.globus.cog.karajan.util.Pair; public class Export extends InternalFunction { private ChannelRef cr_export; @@ -63,4 +67,10 @@ return this; } } + + @Override + protected void optimizePositional(WrapperNode w, Scope scope, List params, List> dynamicOptimized) + throws CompilationException { + // do not optimize positionals + } } Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/UParallel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/UParallel.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/UParallel.java (working copy) @@ -10,20 +10,28 @@ import k.rt.KRunnable; import k.rt.Stack; import k.thr.LWThread; -import k.thr.ThreadSet; +import k.thr.ThreadSetFixed; import k.thr.Yield; import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.ContainerScope; import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.TrackingScope; import org.globus.cog.karajan.parser.WrapperNode; public class UParallel extends CompoundNode { + private int[] frameSizes; @Override protected Node compileChildren(WrapperNode w, Scope scope) throws CompilationException { + frameSizes = new int[w.nodes().size()]; + // TODO Allocate separate frames for each sub-thread. It's likely that most sub-threads + // will finish long before a straggling one. This unnecessarily eats all of the space + // that was allocated on the frame for the finished threads + int index = 0; for (WrapperNode c : w.nodes()) { - TrackingScope ts = new TrackingScope(scope); + ContainerScope cs = new ContainerScope(c, scope); + TrackingScope ts = new TrackingScope(cs); ts.setTrackChannels(false); ts.setAllowChannelReturns(true); ts.setTrackNamed(false); @@ -32,6 +40,7 @@ Node n = compileChild(c, ts); if (n != null) { addChild(n); + frameSizes[index++] = cs.size(); } } if (childCount() == 0) { @@ -47,7 +56,7 @@ public void run(LWThread thr) throws ExecutionException { int state = thr.checkSliceAndPopState(); Stack stack = thr.getStack(); - ThreadSet ts = (ThreadSet) thr.popState(); + ThreadSetFixed ts = (ThreadSetFixed) thr.popState(); int fc = thr.popIntState(); try { switch (state) { @@ -55,13 +64,13 @@ fc = stack.frameCount(); state++; case 1: - final ThreadSet tsf = new ThreadSet(); + int ec = childCount(); + final ThreadSetFixed tsf = new ThreadSetFixed(ec); ts = tsf; final int fcf = fc; - int ec = childCount(); for (int i = 0; i < ec; i++) { final int fi = i; - LWThread ct = thr.fork(new KRunnable() { + LWThread ct = thr.fork(i, new KRunnable() { @Override public void run(LWThread thr) { try { @@ -83,6 +92,8 @@ } }); tsf.add(ct); + Stack cs = ct.getStack(); + cs.enter(this, frameSizes[i]); } ts.startAll(); state++; Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Main.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Main.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/Main.java (working copy) @@ -73,11 +73,6 @@ setVarCount(cs.parent.size()); return fn; } - - @Override - public void run(LWThread thr) { - super.run(thr); - } @Override protected void initializeArgs(Stack stack) { Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/functions/ConstantOp.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/functions/ConstantOp.java (revision 0) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/functions/ConstantOp.java (revision 3905) @@ -0,0 +1,39 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Dec 18, 2012 + */ +package org.globus.cog.karajan.compiled.nodes.functions; + +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.parser.WrapperNode; + +public abstract class ConstantOp extends AbstractFunction { + @Override + public R function(Stack stack) { + return value(); + } + + protected abstract R value(); + + @Override + protected Signature getSignature() { + return new Signature(params()); + } + + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) + throws CompilationException { + staticReturn(scope, value()); + return null; + } +} Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/user/Function.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/user/Function.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/user/Function.java (working copy) @@ -19,7 +19,6 @@ import k.thr.LWThread; import k.thr.Yield; -import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.CompilationException; import org.globus.cog.karajan.analyzer.CompilerSettings; @@ -123,6 +122,7 @@ protected void bindArgs(Frame src, Frame dst, int firstIndex, int argCount) { System.arraycopy(src.getAll(), firstIndex, dst.getAll(), 0, argCount); + //Arrays.fill(src.getAll(), firstIndex, firstIndex + argCount, null); } public void bindChannels(Stack parent, Stack def, List> wrapperChannels) { Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/user/InvocationWrapper.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/user/InvocationWrapper.java (revision 3904) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/user/InvocationWrapper.java (working copy) @@ -32,6 +32,7 @@ import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.parser.WrapperNode; +import org.globus.cog.karajan.util.Pair; public class InvocationWrapper extends InternalFunction { @@ -84,15 +85,35 @@ } @Override - protected ParamWrapperVar.IndexRange addParams(WrapperNode w, Signature sig, Scope scope, List channels, + protected void addParams(WrapperNode w, Signature sig, Scope scope, List channels, List optional, List positional) throws CompilationException { + } + + @Override + protected ParamWrapperVar.IndexRange allocateParams(WrapperNode w, Signature sig, Scope scope, List channels, + List optional, List positional) throws CompilationException { prepareChannelParams(scope, channels); - firstIndex = scope.addParams(channels, optional, positional); + firstIndex = scope.addParams(channels, optional, positional, this); argCount = channels.size() + optional.size() + positional.size(); return null; } + + @Override + protected void scanNotSet(WrapperNode w, Scope scope, List optional) + throws CompilationException { + } + @Override + protected void scanNamed(WrapperNode w, Scope scope, List params, + List> dynamicOptimized) throws CompilationException { + } + + @Override + protected void optimizePositional(WrapperNode w, Scope scope, List params, + List> dynamicOptimized) throws CompilationException { + } + @SuppressWarnings("unchecked") @Override protected void setArg(WrapperNode w, Param r, Object value) throws CompilationException { From hategan at ci.uchicago.edu Thu May 8 23:46:37 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:46:37 -0500 (CDT) Subject: [Swift-commit] r7840 - trunk/src/org/griphyn/vdl/engine Message-ID: <20140509044637.B10679D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:46:37 -0500 (Thu, 08 May 2014) New Revision: 7840 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: intern fields Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2014-05-09 04:35:05 UTC (rev 7839) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2014-05-09 04:46:37 UTC (rev 7840) @@ -81,8 +81,36 @@ new HashMap(); Map functionsMap = new HashMap(); - Map typesMap = new HashMap(); + Map typesMap = new HashMap(); + Set nonMappedTypes = new HashSet(); + private class InternedField { + public final String name, type; + + public InternedField(String name, String type) { + this.name = name; + this.type = type; + } + + @Override + public int hashCode() { + return name.hashCode() + type.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof InternedField) { + InternedField o = (InternedField) obj; + return name.equals(o.name) && type.equals(o.type); + } + else { + return false; + } + } + } + + Map usedFields = new HashMap(); + List variables = new ArrayList(); public static final String TEMPLATE_FILE_NAME = "Karajan.stg"; @@ -177,6 +205,13 @@ proceduresMap = ProcedureSignature.makeProcedureSignatures(); // Built-in functions functionsMap = ProcedureSignature.makeFunctionSignatures(); + + // used by some templates + addInternedField("temp", "int"); + addInternedField("const", "int"); + addInternedField("const", "float"); + addInternedField("const", "string"); + addInternedField("const", "boolean"); } void setTemplateGroup(StringTemplateGroup tempGroup) { @@ -263,15 +298,22 @@ } TypeStructure ts = theType.getTypestructure(); + boolean allPrimitive = ts.sizeOfMemberArray() > 0; for (int j = 0; j < ts.sizeOfMemberArray(); j++) { TypeRow tr = ts.getMemberArray(j); StringTemplate stMember = template("memberdefinition"); stMember.setAttribute("name", tr.getMembername()); stMember.setAttribute("type", tr.getMembertype()); + if (!isPrimitiveOrArrayOfPrimitive(tr.getMembertype())) { + allPrimitive = false; + } st.setAttribute("members", stMember); } + if (allPrimitive) { + nonMappedTypes.add(typeName); + } scope.bodyTemplate.setAttribute("types", st); } } @@ -374,6 +416,7 @@ for (Program program : importList) statements(program, scope); + generateInternedFields(scope.bodyTemplate); generateInternedConstants(scope.bodyTemplate); scope.analyzeWriters(); @@ -393,12 +436,18 @@ FormalParameter param = proc.getOutputArray(i); StringTemplate paramST = parameter(param, innerScope); procST.setAttribute("outputs", paramST); + if (!this.isPrimitiveOrArrayOfPrimitive(param.getType().getLocalPart())) { + procST.setAttribute("stageouts", paramST); + } addArg(procST, param, paramST, true, innerScope); } for (int i = 0; i < proc.sizeOfInputArray(); i++) { FormalParameter param = proc.getInputArray(i); StringTemplate paramST = parameter(param, outerScope); procST.setAttribute("inputs", paramST); + if (!this.isPrimitiveOrArrayOfPrimitive(param.getType().getLocalPart())) { + procST.setAttribute("stageins", paramST); + } addArg(procST, param, paramST, false, outerScope); outerScope.addWriter(param.getName(), WriteType.FULL, proc, procST); } @@ -458,6 +507,7 @@ StringTemplate variableST = template("variable"); variableST.setAttribute("name", var.getName()); variableST.setAttribute("type", var.getType().getLocalPart()); + variableST.setAttribute("field", addInternedField(var.getName(), var.getType().getLocalPart())); variableST.setAttribute("isGlobal", Boolean.valueOf(var.getIsGlobal())); variableST.setAttribute("line", getLine(var)); variables.add(variableST); @@ -542,7 +592,7 @@ paramST.setAttribute("expr", expressionToKarajan(param.getAbstractExpression(), scope)); } else { - String parameterVariableName="swift#mapper#"+(internedIDCounter++); + String parameterVariableName="swift.mapper." + (internedIDCounter++); // make template for variable declaration (need to compute type of this variable too?) StringTemplate variableDeclarationST = template("variable"); // TODO factorise this and other code in variable()? @@ -560,6 +610,8 @@ String paramValueType = datatype(paramValueST); scope.addVariable(parameterVariableName, paramValueType, "Variable", param); variableDeclarationST.setAttribute("type", paramValueType); + variableDeclarationST.setAttribute("field", addInternedField(parameterVariableName, paramValueType)); + StringTemplate variableReferenceST = template("id"); variableReferenceST.setAttribute("var",parameterVariableName); StringTemplate variableAssignmentST = template("assign"); @@ -568,7 +620,7 @@ scope.appendStatement(variableAssignmentST); if (param.getAbstractExpression().getDomNode().getNodeName().equals("stringConstant")) { StringTemplate valueST = template("sConst"); - valueST.setAttribute("innervalue", param.getAbstractExpression().getDomNode().getFirstChild().getNodeValue()); + valueST.setAttribute("value", param.getAbstractExpression().getDomNode().getFirstChild().getNodeValue()); paramST.setAttribute("expr", valueST); } else { @@ -591,10 +643,15 @@ return t.isPrimitive() || (t.isArray() && t.itemType().isPrimitive()); } catch (NoSuchTypeException e) { - return false; + if (nonMappedTypes.contains(type)) { + return true; + } + else { + return false; + } } } - + public void assign(Assign assign, VariableScope scope) throws CompilationException { try { XmlObject value = assign.getAbstractExpressionArray(1); @@ -653,7 +710,6 @@ } } - private boolean isProcedureCall(XmlObject value) { if (value instanceof Call) { Call call = (Call) value; @@ -664,7 +720,6 @@ } } - private boolean isAnyType(String type) { return ProcedureSignature.ANY.equals(type); } @@ -1106,8 +1161,8 @@ innerScope.addVariable(foreach.getVar(), itemType, "Iteration variable", foreach); innerScope.addWriter(foreach.getVar(), WriteType.FULL, foreach, foreachST); foreachST.setAttribute("indexVar", foreach.getIndexVar()); - foreachST.setAttribute("indexVarType", keyType); if (foreach.getIndexVar() != null) { + foreachST.setAttribute("indexVarField", addInternedField(foreach.getIndexVar(), keyType)); innerScope.addVariable(foreach.getIndexVar(), keyType, "Iteration variable", foreach); innerScope.addWriter(foreach.getIndexVar(), WriteType.FULL, foreach, foreachST); } @@ -1427,7 +1482,7 @@ Integer iobj = Integer.valueOf(i); String internedID; if(intInternMap.get(iobj) == null) { - internedID = "swift#int#" + i; + internedID = "swift.int." + i; intInternMap.put(iobj, internedID); } else { internedID = intInternMap.get(iobj); @@ -1442,7 +1497,7 @@ Float fobj = new Float(f); String internedID; if(floatInternMap.get(fobj) == null) { - internedID = "swift#float#" + (internedIDCounter++); + internedID = "swift.float." + (internedIDCounter++); floatInternMap.put(fobj, internedID); } else { internedID = floatInternMap.get(fobj); @@ -1455,14 +1510,14 @@ XmlString xmlString = (XmlString) expression; String s = xmlString.getStringValue(); String internedID; - if(stringInternMap.get(s) == null) { - internedID = "swift#string#" + (internedIDCounter++); - stringInternMap.put(s,internedID); + if (stringInternMap.get(s) == null) { + internedID = "swift.string." + (internedIDCounter++); + stringInternMap.put(s, internedID); } else { internedID = stringInternMap.get(s); } StringTemplate st = template("id"); - st.setAttribute("var",internedID); + st.setAttribute("var", internedID); st.setAttribute("datatype", "string"); return st; } else if (expressionQName.equals(COND_EXPR)) { @@ -1702,7 +1757,7 @@ c.addNewOutput(); VariableScope subscope = new VariableScope(this, scope, c); VariableReferenceDocument ref = VariableReferenceDocument.Factory.newInstance(); - ref.setVariableReference("swift#callintermediate"); + ref.setVariableReference("swift.callintermediate"); c.getOutputArray(0).set(ref); String name = c.getProc().getLocalPart(); ProcedureSignature funcSignature = proceduresMap.get(name); @@ -1725,9 +1780,14 @@ } } - subscope.addInternalVariable("swift#callintermediate", type, null); + if (!isPrimitiveOrArrayOfPrimitive(type)) { + call.setAttribute("mapping", true); + } + + subscope.addInternalVariable("swift.callintermediate", type, null); call.setAttribute("datatype", type); + call.setAttribute("field", addInternedField("swift.callintermediate", type)); call.setAttribute("call", call(c, subscope, true)); if (!isPrimitiveOrArrayOfPrimitive(type)) { call.setAttribute("prefix", UUIDGenerator.getInstance().generateRandomBasedUUID().toString()); @@ -1971,42 +2031,62 @@ throw new RuntimeException("Could not find root for abstract expression."); } } + + public void generateInternedFields(StringTemplate programTemplate) { + for (Map.Entry e : usedFields.entrySet()) { + StringTemplate st = template("fieldConst"); + st.setAttribute("name", e.getValue()); + st.setAttribute("id", e.getKey().name); + st.setAttribute("type", e.getKey().type); + programTemplate.setAttribute("constants", st); + } + } - public void generateInternedConstants(StringTemplate programTemplate) { + private String internedFieldName(InternedField f) { + String v = usedFields.get(f); + if (v == null) { + throw new IllegalArgumentException("No such interned field: " + f); + } + return v; + } + + private int fieldCounter = 1; + private String addInternedField(String name, String type) { + String v = "swift.field." + name + "." + type.replace("[", ".array.").replace("]", ""); + usedFields.put(new InternedField(name, type), v); + return v; + } - for (String key : stringInternMap.keySet()) { - String variableName = stringInternMap.get(key); - StringTemplate st = template("sConst"); - st.setAttribute("innervalue",escape(key)); - StringTemplate vt = template("globalConstant"); - vt.setAttribute("name",variableName); - vt.setAttribute("expr",st); - programTemplate.setAttribute("constants",vt); - } - - for (Integer key : intInternMap.keySet()) { - String variableName = intInternMap.get(key); - StringTemplate st = template("iConst"); - st.setAttribute("value",key); - StringTemplate vt = template("globalConstant"); - vt.setAttribute("name",variableName); - vt.setAttribute("expr",st); - programTemplate.setAttribute("constants",vt); - } + public void generateInternedConstants(StringTemplate programTemplate) { + generateInternedConstants(programTemplate, stringInternMap, "sConst"); + generateInternedConstants(programTemplate, intInternMap, "iConst"); + generateInternedConstants(programTemplate, floatInternMap, "fConst"); + } - for (Float key : floatInternMap.keySet()) { - String variableName = floatInternMap.get(key); - StringTemplate st = template("fConst"); - st.setAttribute("value",key); - StringTemplate vt = template("globalConstant"); - vt.setAttribute("name",variableName); - vt.setAttribute("expr",st); - programTemplate.setAttribute("constants",vt); - } + + private void generateInternedConstants(StringTemplate programTemplate, Map map, + String cTemplate) { + for (Object key : map.keySet()) { + String variableName = map.get(key); + StringTemplate st = template(cTemplate); + st.setAttribute("value", toKarajanValue(key)); + StringTemplate vt = template("globalConstant"); + vt.setAttribute("name", variableName); + vt.setAttribute("expr", st); + programTemplate.setAttribute("constants", vt); + } + } + + private String toKarajanValue(Object o) { + if (o instanceof String) { + return escape((String) o); + } + else { + return String.valueOf(o); + } } - - String escape(String in) { + private String escape(String in) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < in.length(); i++) { char c = in.charAt(i); From hategan at ci.uchicago.edu Thu May 8 23:48:06 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:48:06 -0500 (CDT) Subject: [Swift-commit] r7841 - in trunk/src/org/griphyn/vdl/type: . impl Message-ID: <20140509044806.F1EE99D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:48:06 -0500 (Thu, 08 May 2014) New Revision: 7841 Modified: trunk/src/org/griphyn/vdl/type/Field.java trunk/src/org/griphyn/vdl/type/Type.java trunk/src/org/griphyn/vdl/type/impl/FieldImpl.java trunk/src/org/griphyn/vdl/type/impl/ImmutableField.java trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java Log: support fields that are indexed and a bunch of other things Modified: trunk/src/org/griphyn/vdl/type/Field.java =================================================================== --- trunk/src/org/griphyn/vdl/type/Field.java 2014-05-09 04:46:37 UTC (rev 7840) +++ trunk/src/org/griphyn/vdl/type/Field.java 2014-05-09 04:48:06 UTC (rev 7841) @@ -55,8 +55,7 @@ * of Field. */ - public static final class Factory - { + public static final class Factory { private static final Map fieldCache = new HashMap(); public static Field newInstance() { return new FieldImpl(); @@ -77,6 +76,14 @@ return cached; } } - } + } + + public static final Field GENERIC_STRING = new FieldImpl("?", Types.STRING); + public static final Field GENERIC_ANY = new FieldImpl("?", Types.ANY); + public static final Field GENERIC_BOOLEAN = new FieldImpl("?", Types.BOOLEAN); + public static final Field GENERIC_INT = new FieldImpl("?", Types.INT); + public static final Field GENERIC_FLOAT = new FieldImpl("?", Types.FLOAT); + + public static final Field GENERIC_STRING_ARRAY = new FieldImpl("?", Types.STRING.arrayType()); } Modified: trunk/src/org/griphyn/vdl/type/Type.java =================================================================== --- trunk/src/org/griphyn/vdl/type/Type.java 2014-05-09 04:46:37 UTC (rev 7840) +++ trunk/src/org/griphyn/vdl/type/Type.java 2014-05-09 04:48:06 UTC (rev 7841) @@ -111,7 +111,9 @@ * @return Field * @throws NoSuchFieldException */ - public Field getField(String name) throws NoSuchFieldException; + public Field getField(String name) throws NoSuchFieldException; + + public int getFieldIndex(String name) throws NoSuchFieldException; /** * get a list of field names @@ -187,7 +189,7 @@ } - public boolean hasNonPrimitiveComponents(); + public boolean hasMappedComponents(); public boolean hasArrayComponents(); } Modified: trunk/src/org/griphyn/vdl/type/impl/FieldImpl.java =================================================================== --- trunk/src/org/griphyn/vdl/type/impl/FieldImpl.java 2014-05-09 04:46:37 UTC (rev 7840) +++ trunk/src/org/griphyn/vdl/type/impl/FieldImpl.java 2014-05-09 04:48:06 UTC (rev 7841) @@ -23,21 +23,13 @@ public class FieldImpl implements Field { private Comparable id; private Type type; - private boolean array = false; - + public FieldImpl() { } - - public FieldImpl(Comparable id, Type type, boolean array) { - this.id = id; - this.type = type; - this.array = array; - } public FieldImpl(Comparable id, Type type) { this.id = id; this.type = type; - this.array = false; } public Comparable getId() { @@ -48,14 +40,6 @@ this.id = id; } - public boolean isArray() { - return array; - } - - public void setArray() { - array = true; - } - public Type getType() { return type; } Modified: trunk/src/org/griphyn/vdl/type/impl/ImmutableField.java =================================================================== --- trunk/src/org/griphyn/vdl/type/impl/ImmutableField.java 2014-05-09 04:46:37 UTC (rev 7840) +++ trunk/src/org/griphyn/vdl/type/impl/ImmutableField.java 2014-05-09 04:48:06 UTC (rev 7841) @@ -23,11 +23,6 @@ } @Override - public void setArray() { - throw new UnsupportedOperationException(); - } - - @Override public void setType(Type type) { throw new UnsupportedOperationException(); } Modified: trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java =================================================================== --- trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java 2014-05-09 04:46:37 UTC (rev 7840) +++ trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java 2014-05-09 04:48:06 UTC (rev 7841) @@ -30,9 +30,10 @@ public class TypeImpl extends UnresolvedType { private boolean primitive; - private Map fields; + private Map fields; + private Map fieldIndices; private Type baseType; - private Boolean hasNonPrimitiveComponents, hasArrayComponents; + private Boolean hasMappedComponents, hasArrayComponents; public TypeImpl() { this((URI) null, null, false); @@ -71,7 +72,11 @@ throw new DuplicateFieldException(name); } else { - fields.put(name, field); + fields.put(name, field); + if (fieldIndices == null) { + fieldIndices = new HashMap(); + } + fieldIndices.put(name, fieldIndices.size()); } } @@ -92,6 +97,17 @@ else { throw new NoSuchFieldException(name); } + } + + public int getFieldIndex(String name) throws NoSuchFieldException { + if (fieldIndices == null) { + throw new NoSuchFieldException("The type " + this + " has no fields"); + } + Integer i = fieldIndices.get(name); + if (i == null) { + throw new NoSuchFieldException("The type " + this + " has no field named '" + name + "'"); + } + return i; } public List getFieldNames() { @@ -132,37 +148,37 @@ return isArray() || !fields.isEmpty(); } - public synchronized boolean hasNonPrimitiveComponents() { - if (hasNonPrimitiveComponents == null) { + public synchronized boolean hasMappedComponents() { + if (hasMappedComponents == null) { if (isPrimitive()) { - hasNonPrimitiveComponents = false; + hasMappedComponents = false; } else if (!isComposite()) { // mapped - hasNonPrimitiveComponents = true; + hasMappedComponents = true; } else if (isArray()) { - if (keyType().hasNonPrimitiveComponents()) { - hasNonPrimitiveComponents = true; + if (keyType().hasMappedComponents()) { + hasMappedComponents = true; } - else if (itemType().hasNonPrimitiveComponents()) { - hasNonPrimitiveComponents = true; + else if (itemType().hasMappedComponents()) { + hasMappedComponents = true; } else { - hasNonPrimitiveComponents = false; + hasMappedComponents = false; } } else { // struct for (Field f : getFields()) { - if (f.getType().hasNonPrimitiveComponents()) { - return hasNonPrimitiveComponents = true; + if (f.getType().hasMappedComponents()) { + return hasMappedComponents = true; } } - hasNonPrimitiveComponents = false; + hasMappedComponents = false; } } - return hasNonPrimitiveComponents; + return hasMappedComponents; } public synchronized boolean hasArrayComponents() { Modified: trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java =================================================================== --- trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java 2014-05-09 04:46:37 UTC (rev 7840) +++ trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java 2014-05-09 04:48:06 UTC (rev 7841) @@ -85,9 +85,14 @@ } @Override - public boolean hasNonPrimitiveComponents() { + public int getFieldIndex(String name) throws NoSuchFieldException { throw new UnsupportedOperationException(); } + + @Override + public boolean hasMappedComponents() { + throw new UnsupportedOperationException(); + } @Override public boolean hasArrayComponents() { From hategan at ci.uchicago.edu Thu May 8 23:48:47 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:48:47 -0500 (CDT) Subject: [Swift-commit] r7842 - trunk/src/org/griphyn/vdl/util Message-ID: <20140509044847.5A3139D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:48:47 -0500 (Thu, 08 May 2014) New Revision: 7842 Modified: trunk/src/org/griphyn/vdl/util/FQN.java Log: a FQN constructor with no version Modified: trunk/src/org/griphyn/vdl/util/FQN.java =================================================================== --- trunk/src/org/griphyn/vdl/util/FQN.java 2014-05-09 04:48:06 UTC (rev 7841) +++ trunk/src/org/griphyn/vdl/util/FQN.java 2014-05-09 04:48:47 UTC (rev 7842) @@ -39,6 +39,10 @@ this.version = version; this.hashCode = _hashCode(); } + + public FQN(String namespace, String name) { + this(namespace, name, null); + } public FQN(String fqn) { /* From hategan at ci.uchicago.edu Thu May 8 23:57:10 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:57:10 -0500 (CDT) Subject: [Swift-commit] r7844 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20140509045710.A05389D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:57:10 -0500 (Thu, 08 May 2014) New Revision: 7844 Modified: trunk/src/org/griphyn/vdl/karajan/HangChecker.java Log: fixed hang checker Modified: trunk/src/org/griphyn/vdl/karajan/HangChecker.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/HangChecker.java 2014-05-09 04:56:39 UTC (rev 7843) +++ trunk/src/org/griphyn/vdl/karajan/HangChecker.java 2014-05-09 04:57:10 UTC (rev 7844) @@ -39,7 +39,6 @@ import k.rt.Context; import k.rt.ExecutionException; -import k.rt.Stack; import k.thr.LWThread; import k.thr.Scheduler; @@ -415,7 +414,7 @@ } Object prev = c.getLast(); for (Object o : c) { - if (o instanceof Stack) { + if (o instanceof LWThread) { if (prev != null) { ps.println("\t" + varWithLine((DSHandle) prev) + " is needed by: "); } From hategan at ci.uchicago.edu Thu May 8 23:57:23 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:57:23 -0500 (CDT) Subject: [Swift-commit] r7845 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20140509045723.BBBB99D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:57:23 -0500 (Thu, 08 May 2014) New Revision: 7845 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: fixed import Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-05-09 04:57:10 UTC (rev 7844) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-05-09 04:57:23 UTC (rev 7845) @@ -62,7 +62,7 @@ import org.griphyn.vdl.karajan.lib.Log; import org.griphyn.vdl.karajan.lib.New; import org.griphyn.vdl.karajan.monitor.MonitorAppender; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.toolkit.VDLt2VDLx; import org.griphyn.vdl.toolkit.VDLt2VDLx.IncorrectInvocationException; import org.griphyn.vdl.toolkit.VDLt2VDLx.ParsingException; From hategan at ci.uchicago.edu Thu May 8 23:57:56 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:57:56 -0500 (CDT) Subject: [Swift-commit] r7846 - trunk/src/org/globus/swift/data Message-ID: <20140509045756.03D059D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:57:55 -0500 (Thu, 08 May 2014) New Revision: 7846 Modified: trunk/src/org/globus/swift/data/Query.java Log: query is a constant op Modified: trunk/src/org/globus/swift/data/Query.java =================================================================== --- trunk/src/org/globus/swift/data/Query.java 2014-05-09 04:57:23 UTC (rev 7845) +++ trunk/src/org/globus/swift/data/Query.java 2014-05-09 04:57:55 UTC (rev 7846) @@ -23,7 +23,7 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Param; import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; -import org.globus.cog.karajan.compiled.nodes.functions.NullaryOp; +import org.globus.cog.karajan.compiled.nodes.functions.ConstantOp; import org.globus.swift.data.policy.Policy; import org.griphyn.vdl.mapping.AbsFile; @@ -79,7 +79,7 @@ Obtain the CDM policy file given on the command-line, conventionally "fs.data". If not set, returns an empty String. */ - public static class File extends NullaryOp { + public static class File extends ConstantOp { @Override protected String value() { String file = ""; From hategan at ci.uchicago.edu Thu May 8 23:58:58 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:58:58 -0500 (CDT) Subject: [Swift-commit] r7847 - trunk/src/org/globus/swift/catalog/site Message-ID: <20140509045858.DC3F79D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:58:58 -0500 (Thu, 08 May 2014) New Revision: 7847 Added: trunk/src/org/globus/swift/catalog/site/SwiftContact.java Log: added a swift specific contact Added: trunk/src/org/globus/swift/catalog/site/SwiftContact.java =================================================================== --- trunk/src/org/globus/swift/catalog/site/SwiftContact.java (rev 0) +++ trunk/src/org/globus/swift/catalog/site/SwiftContact.java 2014-05-09 04:58:58 UTC (rev 7847) @@ -0,0 +1,36 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.globus.swift.catalog.site; + +import java.util.HashMap; +import java.util.Map; + +import org.globus.cog.karajan.util.BoundContact; +import org.griphyn.vdl.util.FQN; + +public class SwiftContact extends BoundContact { + private Map profiles = new HashMap(); + + public SwiftContact() { + super(); + } + + public SwiftContact(String host) { + super(host); + } + + public void addProfile(FQN fqn, String value) { + profiles.put(fqn, value); + } + + public Map getProfiles() { + return profiles; + } +} From hategan at ci.uchicago.edu Fri May 9 00:01:20 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 9 May 2014 00:01:20 -0500 (CDT) Subject: [Swift-commit] r7848 - trunk/resources Message-ID: <20140509050120.63D999D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-09 00:01:20 -0500 (Fri, 09 May 2014) New Revision: 7848 Modified: trunk/resources/swiftscript.g Log: I think this was arbitrary string app declaration stuff Modified: trunk/resources/swiftscript.g =================================================================== --- trunk/resources/swiftscript.g 2014-05-09 04:58:58 UTC (rev 7847) +++ trunk/resources/swiftscript.g 2014-05-09 05:01:20 UTC (rev 7848) @@ -273,6 +273,12 @@ : id:ID {code=text(id.getText());} ; +appDeclarator returns [StringTemplate code=null] + : id:ID {code=text(id.getText());} + | s:STRING_LITERAL {code=text(s.getText());} + ; + + varInitializer returns [StringTemplate code=null] : ASSIGN code=expression ; @@ -432,8 +438,10 @@ RPAREN LCURLY ( appProfile[app] )* - exec=declarator - {app.setAttribute("exec",exec);} + (exec=appDeclarator) + { + app.setAttribute("exec", exec); + } ( appArg[app] )* SEMI {code.setAttribute("config",app);} RCURLY From hategan at ci.uchicago.edu Fri May 9 00:01:52 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 9 May 2014 00:01:52 -0500 (CDT) Subject: [Swift-commit] r7849 - trunk/resources Message-ID: <20140509050152.6C0979D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-09 00:01:52 -0500 (Fri, 09 May 2014) New Revision: 7849 Modified: trunk/resources/Karajan.stg Log: interned fields, karajan generation part Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2014-05-09 05:01:20 UTC (rev 7848) +++ trunk/resources/Karajan.stg 2014-05-09 05:01:52 UTC (rev 7849) @@ -16,8 +16,8 @@ $types;separator="\n"$ ) ) - $endif$ +$typeConstants;separator="\n"$ $constants;separator="\n"$ $declarations;separator="\n"$ $procedures;separator="\n"$ @@ -58,13 +58,13 @@ xs:element("$name$", "$type$") >> -procedure(name, outputs, inputs, arguments, optargs, binding, declarations, statements, config, line, initWaitCounts, cleanups, closes) ::= << +procedure(name, outputs, inputs, stageins, stageouts, arguments, optargs, binding, declarations, statements, config, line, initWaitCounts, cleanups, closes) ::= << $name$ := function($if(optargs)$$proc_oargs(args=optargs)$$if(arguments)$, $endif$$endif$$if(arguments)$$proc_args(args=arguments)$$endif$) { $swift_log_inputs(inputs)$ $swift_log_outputs(outputs)$ $if(binding)$ - $swift_execute(outputs=outputs,inputs=inputs,application=binding.application, name=name,line=line)$ + $swift_execute(outputs=outputs,inputs=inputs,stageins=stageins,stageouts=stageouts,application=binding.application, name=name,line=line)$ $else$ $compound(outputs=outputs, inputs=inputs, declarations=declarations, statements=statements, name=name, initWaitCounts=initWaitCounts, cleanups=cleanups)$ $endif$ @@ -110,13 +110,13 @@ $it.name$=$if(it.default)$$it.default$$else$null$endif$ >> -swift_execute(outputs,inputs,attributes,application,name,line) ::= << +swift_execute(outputs,inputs,stageins,stageouts,attributes,application,name,line) ::= << swift:unitStart("PROCEDURE", name="$name$", line=$line$, outputs="$outputs:list();separator=","$") swift:execute( $attributes$ swift:tr("$application.exec$") - $inputs:swift_stagein();separator="\n"$ - $outputs:swift_stageout();separator="\n"$ + $stageins:swift_stagein();separator="\n"$ + $stageouts:swift_stageout();separator="\n"$ $swift_arguments(attributes=application.attributes,arguments=application.arguments, stdin=application.stdin,stdout=application.stdout,stderr=application.stderr)$ ) $outputs:swift_closedataset();separator="\n"$ @@ -205,13 +205,13 @@ } >> -foreach(var, in, indexVar, indexVarType, declarations, statements, line, refs, selfClose, cleanups) ::= << +foreach(var, in, indexVar, indexVarField, declarations, statements, line, refs, selfClose, cleanups) ::= << swift:tParallelFor(\$, _traceline=$line$$if(indexVar)$, _kvar="$indexVar$"$endif$, _vvar="$var$"$if(selfClose)$, selfClose=true$endif$$if(refs)$, refs="$refs;separator=" "$"$endif$, $! The iterator !$ getArrayIterator($in$)) { (\$\$, $var$) := each(\$) $if(indexVar)$ - $indexVar$ := swift:new("$indexVarType$", value=\$\$) + $indexVar$ := swift:new($indexVarField$, value=\$\$) $endif$ swift:unitStart("FOREACH_IT", line=$line$) @@ -285,19 +285,19 @@ $name$ := $expr$ >> -variable(name, type, expr, mapping, nil, file, waitCount, input, datatype, isGlobal, line) ::= << +variable(name, type, field, expr, mapping, nil, file, waitCount, input, datatype, isGlobal, line) ::= << $name$ := $if(mapping)$ - swift:new("$type$", dbgname="$name$"$if(waitCount)$, waitCount=$waitCount$$endif$$if(line)$, _defline=$line$$endif$$if(input)$, input=true$endif$ + swift:new($field$$if(waitCount)$, waitCount=$waitCount$$endif$$if(line)$, _defline=$line$$endif$$if(input)$, input=true$endif$ $swift_mapping(mapping=mapping, file=file)$ ) $else$ $if(file)$ - swift:new("$type$", dbgname="$name$"$if(waitCount)$, waitCount=$waitCount$$endif$$if(line)$, _defline=$line$$endif$$if(input)$, input=true$endif$ + swift:new($field$$if(waitCount)$, waitCount=$waitCount$$endif$$if(line)$, _defline=$line$$endif$$if(input)$, input=true$endif$ $swift_mapping(mapping=mapping, file=file)$ ) $else$ - swift:new("$type$", dbgname="$name$"$if(waitCount)$, waitCount=$waitCount$$endif$$if(line)$, _defline=$line$$endif$$if(input)$, input=true$endif$) + swift:new($field$$if(waitCount)$, waitCount=$waitCount$$endif$$if(line)$, _defline=$line$$endif$$if(input)$, input=true$endif$) $endif$ $endif$ $variable_log()$ @@ -345,15 +345,15 @@ $endif$ >> -callexpr(call, datatype, prefix) ::= << +callexpr(call, datatype, field, prefix, mapping) ::= << sequential( - swift#callintermediate := swift:new("$datatype$", dbgname="swift#callintermediate", + swift.callintermediate := swift:new($field$$if(mapping)$, swift:mapping("concurrent_mapper", swift:parameter("prefix", "_callintermediate-$prefix$") ) - ) + $endif$) $call$ - swift#callintermediate + swift.callintermediate ) >> @@ -473,7 +473,7 @@ ] unaryNegation(exp, datatype) ::= << -swiftop:product(swift:new("int", value = -1), $exp$) +swiftop:product(swift:new(swift.field.temp.int, value = -1), $exp$) >> binaryop(op,left,right,datatype) ::= << @@ -511,19 +511,22 @@ iConst(value, datatype) ::= << -swift:new("int", value=$value$) +swift:new(swift.field.const.int, value=$value$) >> fConst(value, datatype) ::= << -swift:new("float", value=$value$) +swift:new(swift.field.const.float, value=$value$) >> bConst(value, datatype) ::= << -swift:new("boolean", value=$value$) +swift:new(swift.field.const.boolean, value=$value$) >> -// Why is there a value attribute? -sConst(value,innervalue,datatype) ::= << -swift:new("string", value="$innervalue$") +sConst(value, datatype) ::= << +swift:new(swift.field.const.string, value="$value$") >> +fieldConst(name, id, type) ::= << +$name$ := swift:field("$id$", "$type$") +>> + From hategan at ci.uchicago.edu Thu May 8 23:35:05 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:35:05 -0500 (CDT) Subject: [Swift-commit] r7839 - in trunk/src/org/griphyn/vdl/mapping: . file nodes Message-ID: <20140509043505.C79CB9D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:35:05 -0500 (Thu, 08 May 2014) New Revision: 7839 Added: trunk/src/org/griphyn/vdl/mapping/nodes/ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedMappedSingleDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedNonCompositeDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedPrimitiveDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedStructDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureMappedSingleDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureNonCompositeDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFuturePrimitiveDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureStructDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/ArrayHandle.java trunk/src/org/griphyn/vdl/mapping/nodes/ClosedArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/ClosedMappedSingleDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/ClosedPrimitiveDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/ExternalDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/FutureArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/FutureMappedSingleDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/FuturePrimitiveDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/FutureStructDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java trunk/src/org/griphyn/vdl/mapping/nodes/NodeFactory.java trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedMappedSingleDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedPrimitiveDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureMappedSingleDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureStructDataNode.java Removed: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java trunk/src/org/griphyn/vdl/mapping/ArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/DataNode.java trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/RootDataNode.java Modified: trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java trunk/src/org/griphyn/vdl/mapping/DSHandle.java trunk/src/org/griphyn/vdl/mapping/DependentException.java trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java trunk/src/org/griphyn/vdl/mapping/InvalidPathException.java trunk/src/org/griphyn/vdl/mapping/Mapper.java trunk/src/org/griphyn/vdl/mapping/MappingParamSet.java trunk/src/org/griphyn/vdl/mapping/MissingDataException.java trunk/src/org/griphyn/vdl/mapping/NotCompositeException.java trunk/src/org/griphyn/vdl/mapping/NullMapper.java trunk/src/org/griphyn/vdl/mapping/OpenArrayEntries.java trunk/src/org/griphyn/vdl/mapping/Path.java trunk/src/org/griphyn/vdl/mapping/RootHandle.java trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/CSVMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/ExternalMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/SimpleFileMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/SingleFileMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapperParams.java trunk/src/org/griphyn/vdl/mapping/file/TestMapperParams.java Log: re-organized the data node business in a more OO-proper way (though this is one place where multiple inheritance would be good) Deleted: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -1,843 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jun 6, 2006 - */ -package org.griphyn.vdl.mapping; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import k.rt.FutureListener; -import k.rt.FutureValue; -import k.thr.Yield; - -import org.apache.log4j.Logger; -import org.globus.cog.karajan.compiled.nodes.Node; -import org.globus.cog.karajan.futures.FutureNotYetAvailable; -import org.griphyn.vdl.karajan.Loader; -import org.griphyn.vdl.karajan.WaitingThreadsMonitor; -import org.griphyn.vdl.karajan.lib.Tracer; -import org.griphyn.vdl.type.Field; -import org.griphyn.vdl.type.Type; -import org.griphyn.vdl.util.VDL2Config; - - - -public abstract class AbstractDataNode implements DSHandle, FutureValue { - public static final Object FILE_VALUE = new Object() { - public String toString() { - return ""; - } - }; - - static final String DATASET_URI_PREFIX = "dataset:"; - - public static final Logger logger = Logger.getLogger(AbstractDataNode.class); - - /** - * Datasets are identified within a run by this sequence number and the - * partial ID field. The initial value is chosen to aid human recognition of - * sequence numbers in the wild. There is no requirement that it start at - * this (or any other) particular value. Note that this introduces a maximum - * on the number of datasets which can be dealt with in any run to be about - * 2^62. - */ - private static long datasetIDCounter = 720000000000l; - - /** - * This is used to provide a (hopefully) globally unique identifier for each - * time the datasetIDCounter is reset (whenever this class is loaded, which - * will usually happen once per JVM). No meaning should be inferred from - * this value - it exists purely for making unique URIs. - */ - private static final String datasetIDPartialID = Loader.getUUID(); - - public static boolean provenance = false; - static { - try { - provenance = VDL2Config.getConfig().getProvenanceLog(); - } - catch (IOException e) { - } - } - - private Field field; - private Map, DSHandle> handles; - private Object value; - private boolean closed; - private String identifier; - private Path pathFromRoot; - - private int writeRefCount; - private List listeners; - - protected static final Tracer variableTracer = Tracer.getTracer("VARIABLE"); - - protected AbstractDataNode(Field field) { - this.field = field; - if (field.getType().isComposite()) { - handles = new HashMap, DSHandle>(); - } - else { - handles = Collections.emptyMap(); - } - } - - protected void populateStructFields() { - for (String name : getType().getFieldNames()) { - try { - createField(name); - } - catch (NoSuchFieldException e) { - throw new RuntimeException("Internal inconsistency found: field '" + name - + "' is listed by the type but createField() claims it is invalid"); - } - } - } - - public String getName() { - return getRoot().getName(); - } - - @Override - public void setName(String name) { - throw new UnsupportedOperationException("setName can only be called on a root variable"); - } - - public Type getType() { - return field.getType(); - } - - public boolean isPrimitive() { - return field.getType().isPrimitive(); - } - - public boolean isRestartable() { - return !isPrimitive(); - } - - protected Field getField() { - return field; - } - - protected abstract AbstractDataNode getParentNode(); - - /** - * create a String representation of this node. If the node has a value, - * then uses the String representation of that value. Otherwise, generates a - * text description. - */ - public String toString() { - return toDisplayableString(); - } - - private String toDisplayableString() { - StringBuilder sb = new StringBuilder(); - sb.append(getDisplayableName()); - Path p = getPathFromRoot(); - if (!p.isEmpty()) { - if (!p.isArrayIndex(0)) { - sb.append("."); - } - sb.append(p.toString()); - } - sb.append(":"); - Type type = getType(); - String strtype = type.toString(); - if (type.isArray() && closed) { - strtype = strtype.replace("[]", "[" + this.getHandles().size() + "]"); - } - sb.append(strtype); - if (value != null) { - sb.append(" = "); - if (value instanceof Throwable) { - sb.append(value.getClass().getName()); - } - else { - sb.append(value); - } - } - if (closed) { - sb.append(" - Closed"); - } - else { - sb.append(" - Open"); - } - return sb.toString(); - } - - public String getIdentifyingString() { - StringBuffer sb = new StringBuffer(); - sb.append(this.getClass().getName()); - - sb.append(" identifier "); - sb.append(this.getIdentifier()); - - sb.append(" type "); - sb.append(getType()); - - if (value == null) { - sb.append(" with no value at dataset="); - } - else if (value instanceof Throwable) { - sb.append(" containing throwable "); - sb.append(value.getClass()); - sb.append(" dataset="); - } - else { - sb.append(" value="); - sb.append(this.value.toString()); - sb.append(" dataset="); - } - - sb.append(getDisplayableName()); - - if (!Path.EMPTY_PATH.equals(getPathFromRoot())) { - sb.append(" path="); - sb.append(getPathFromRoot().toString()); - } - - if (closed) { - sb.append(" (closed)"); - } - else { - sb.append(" (not closed)"); - } - - return sb.toString(); - } - - public String getDisplayableName() { - return getName(); - } - - public String getFullName() { - String name = getDisplayableName(); - Path p = getPathFromRoot(); - if (p.isEmpty()) { - return name; - } - else { - return name + "." + p; - } - } - - public DSHandle getField(Path path) throws InvalidPathException { - if (path.isEmpty()) { - return this; - } - try { - DSHandle handle = getField(path.getFirst()); - - if (path.size() > 1) { - return handle.getField(path.butFirst()); - } - else { - return handle; - } - } - catch (NoSuchFieldException e) { - throw new InvalidPathException(path, this); - } - } - - public Collection getFields(Path path) throws InvalidPathException { - List fields = new ArrayList(); - getFields(fields, path); - return fields; - } - - protected void getFields(List fields, Path path) throws InvalidPathException { - if (path.isEmpty()) { - fields.add(this); - } - else { - if (path.isWildcard(0)) { - throw new InvalidPathException("getFields([*]) only applies to arrays"); - } - try { - ((AbstractDataNode) getField(path.getFirst())).getFields(fields, path.butFirst()); - } - catch (NoSuchFieldException e) { - throw new InvalidPathException(path, this); - } - } - } - - public void set(DSHandle handle) { - // TODO check type - if (closed) { - throw new IllegalArgumentException(this.getDisplayableName() + " is already assigned"); - } - if (getParent() == null) { - /* - * AbstractDataNode node = (AbstractDataNode)handle; field = - * node.getField(); handles = node.getHandles(); closed = - * node.isClosed(); value = node.getValue(); - */ - throw new RuntimeException("Can't set root data node!"); - } - ((AbstractDataNode) getParent()).setField(field.getId(), handle); - } - - protected synchronized void setField(Comparable id, DSHandle handle) { - handles.put(id, handle); - } - - public synchronized DSHandle getField(Comparable key) throws NoSuchFieldException { - DSHandle handle = handles.get(key); - if (handle == null) { - if (closed) { - throw new NoSuchFieldException(key.toString()); - } - handle = createField(key); - } - return handle; - } - - protected synchronized boolean isHandlesEmpty() { - return handles.isEmpty(); - } - - public DSHandle createField(Comparable key) throws NoSuchFieldException { - if (closed) { - throw new RuntimeException("Cannot write to closed handle: " + this + " (" + key + ")"); - } - - if (!getType().isComposite()) { - throw new NotCompositeException(this); - } - - return addHandle(key, newNode(getChildField(key))); - } - - @Override - public DSHandle createField(Path path) throws InvalidPathException { - if (path.size() != 1) { - throw new InvalidPathException("Expected a path of size 1: " + path); - } - try { - return createField(path.getFirst()); - } - catch (NoSuchFieldException e) { - throw new InvalidPathException("Invalid path (" + path + ") for " + this); - } - } - - protected synchronized DSHandle addHandle(Comparable id, DSHandle handle) { - Object o = handles.put(id, handle); - if (o != null) { - throw new RuntimeException("Trying to create a handle that already exists (" + id + ") in " + this); - } - return handle; - } - - protected AbstractDataNode newNode(Field f) { - if (f.getType().isArray()) { - return new ArrayDataNode(f, getRoot(), this); - } - else { - DataNode dn = new DataNode(f, getRoot(), this); - if (field.getType().isComposite()) { - dn.populateStructFields(); - } - return dn; - } - } - - protected Field getChildField(Comparable key) throws NoSuchFieldException { - if (getType().isArray()) { - return Field.Factory.createField(key, getType().itemType()); - } - else { - return Field.Factory.getImmutableField(key, getType().getField((String) key).getType()); - } - } - - protected Field getArrayChildField(Comparable key) { - return Field.Factory.createField(key, getType().itemType()); - } - - protected void checkDataException() { - if (value instanceof DependentException) { - throw (DependentException) value; - } - } - - protected void checkMappingException() { - if (value instanceof MappingDependentException) { - throw (MappingDependentException) value; - } - } - - public synchronized Object getValue() { - checkNoValue(); - checkDataException(); - if (field.getType().isArray()) { - return handles; - } - return value; - } - - public Map, DSHandle> getArrayValue() { - checkDataException(); - if (!field.getType().isArray()) { - throw new RuntimeException("getArrayValue called on a non-array: " + this); - } - return handles; - } - - public boolean isArray() { - return false; - } - - public void setValue(Object value) { - synchronized(this) { - if (this.closed) { - throw new IllegalArgumentException(this.getFullName() - + " is closed with a value of " + this.value); - } - if (this.value != null) { - throw new IllegalArgumentException(this.getFullName() - + " is already assigned with a value of " + this.value); - } - - this.value = value; - this.closed = true; - } - postCloseActions(); - } - - public Collection getFringePaths() throws HandleOpenException { - List list = new ArrayList(); - getFringePaths(list, Path.EMPTY_PATH); - return list; - } - - public void getFringePaths(List list, Path parentPath) - throws HandleOpenException { - checkMappingException(); - if (getType().getBaseType() != null) { - list.add(Path.EMPTY_PATH); - } - else { - for (Field field : getType().getFields()) { - AbstractDataNode child; - String name = (String) field.getId(); - try { - child = (AbstractDataNode) this.getField(name); - } - catch (NoSuchFieldException e) { - throw new RuntimeException("Inconsistency between type declaration and " + - "handle for field '" + name + "'"); - } - Path fullPath = parentPath.addLast(name); - Type type = child.getType(); - if (!type.isPrimitive() && !child.isArray() && type.getFields().size() == 0) { - list.add(fullPath); - } - else { - child.getFringePaths(list, fullPath); - } - } - } - } - - public Collection getLeaves() throws HandleOpenException { - Type t = getType(); - if (t.isPrimitive()) { - return Collections.emptyList(); - } - if (!t.isComposite()) { - return Collections.singletonList((DSHandle) this); - } - List list = new ArrayList(); - getLeaves(list); - return list; - } - - public void getLeaves(List list) throws HandleOpenException { - checkMappingException(); - if (getType().getBaseType() != null) { - list.add(this); - } - else { - for (Field field : getType().getFields()) { - AbstractDataNode child; - String name = (String) field.getId(); - try { - child = (AbstractDataNode) this.getField(name); - } - catch (NoSuchFieldException e) { - throw new RuntimeException("Inconsistency between type declaration and " + - "handle for field '" + name + "'"); - } - Type type = child.getType(); - if (!type.isPrimitive() && !child.isArray() && type.getFields().size() == 0) { - list.add(child); - } - else { - child.getLeaves(list); - } - } - } - } - - public void closeShallow() { - synchronized(this) { - if (this.closed) { - return; - } - this.closed = true; - } - postCloseActions(); - } - - private void postCloseActions() { - // closed - notifyListeners(); - if (logger.isDebugEnabled()) { - logger.debug("closed " + this.getIdentifyingString()); - } - // so because its closed, we can dump the contents - - try { - if(provenance) { - logContent(); - } - } - catch (Exception e) { - logger.warn("Exception whilst logging dataset content for " + this, e); - } - // TODO record retrospective provenance information for this dataset here - // we should do it at closing time because that's the point at which we - // know the dataset has its values (all the way down the tree) assigned. - - // provenance-id for this dataset should have been assigned at creation time, - // though, so that we can refer to this dataset elsewhere before it is closed. - - // is this method the only one called to set this.closed? or do subclasses - // or other methods ever change it? - } - - public synchronized void logContent() { - String identifier = this.getIdentifier(); - Path pathFromRoot = this.getPathFromRoot(); - if (this.getPathFromRoot() != null) { - if (logger.isInfoEnabled()) { - logger.info("ROOTPATH dataset=" + identifier + " path=" + pathFromRoot); - if (this.getType().isPrimitive()) { - logger.info("VALUE dataset=" + identifier + " VALUE=" + this.toString()); - } - } - - Mapper m = getActualMapper(); - - - if (m != null) { - // TODO proper type here - // Not sure catching exception here is really the right thing to - // do here - // anyway - should perhaps only be trying to map leafnodes? - // Mapping - // non-leaf stuff is giving wierd paths anyway - - // TODO this is perhaps an unpleasant way of finding if this is a file-backed - // leaf node or not - boolean filemapped = true; - Type type = this.getType(); - if(type.getName().equals("external")) { - filemapped = false; - } - if(type.isPrimitive()) { - filemapped = false; - } - if(type.isArray()) { - filemapped = false; - } - if(handles.size()>0) { - filemapped = false; - } - - try { - if (filemapped) { - Object path = map(); - if (logger.isInfoEnabled()) { - logger.info("FILENAME dataset=" + identifier + " filename=" + path); - } - } - } - catch (Exception e) { - if (logger.isInfoEnabled()) { - logger.info("NOFILENAME dataset=" + identifier); - } - } - } - } - - synchronized (this) { - //Iterator i = handles.entrySet().iterator(); - //while (i.hasNext()) { - // Map.Entry e = (Map.Entry) i.next(); - for (DSHandle handle : handles.values()) { - AbstractDataNode node = (AbstractDataNode) handle; - if (logger.isInfoEnabled()) { - logger.info("CONTAINMENT parent=" + identifier + " child=" + node.getIdentifier()); - } - node.logContent(); - } - } - } - - public Mapper getActualMapper() { - return null; - } - - public boolean isClosed() { - return closed; - } - - public synchronized void closeDeep() { - if (!this.closed) { - closeShallow(); - } - for (DSHandle handle : handles.values()) { - AbstractDataNode mapper = (AbstractDataNode) handle; - mapper.closeDeep(); - } - } - - /** - * Recursively closes arrays through a tree of arrays and complex types. - */ - public void closeArraySizes() { - if (!this.closed && this.getType().isArray()) { - closeShallow(); - } - synchronized (this) { - for (DSHandle handle : handles.values()) { - AbstractDataNode child = (AbstractDataNode) handle; - if (child.getType().isArray() || child.getType().getFields().size() > 0) { - child.closeArraySizes(); - } - } - } - } - - public Path getPathFromRoot() { - if (pathFromRoot == null) { - AbstractDataNode parent = (AbstractDataNode) this.getParent(); - Path myPath; - if (parent != null) { - myPath = parent.getPathFromRoot(); - pathFromRoot = myPath.addLast(getField().getId(), parent.getField().getType().isArray()); - } - else { - pathFromRoot = Path.EMPTY_PATH; - } - } - return pathFromRoot; - } - - public Mapper getMapper() { - return ((AbstractDataNode) getRoot()).getMapper(); - } - - @Override - public PhysicalFormat map(Path path) { - Mapper m = getMapper(); - if (m == null) { - return null; - } - else { - Path p = getPathFromRoot().append(path); - return m.map(p); - } - } - - @Override - public PhysicalFormat map() { - return map(Path.EMPTY_PATH); - } - - protected Map, DSHandle> getHandles() { - return handles; - } - - public synchronized String getIdentifier() { - if (identifier == null) { - identifier = makeIdentifierURIString(); - } - return identifier; - } - - String makeIdentifierURIString() { - datasetIDCounter++; - return DATASET_URI_PREFIX + datasetIDPartialID + ":" + datasetIDCounter; - } - - public synchronized void waitFor(Node who) { - if (!closed) { - if (logger.isDebugEnabled()) { - logger.debug("Waiting for " + this); - } - - Yield y = new FutureNotYetAvailable(this); - y.getState().addTraceElement(who); - throw y; - } - else { - if (logger.isDebugEnabled()) { - logger.debug("Do not need to wait for " + this); - } - checkNoValue(); - if (value instanceof RuntimeException) { - throw (RuntimeException) value; - } - } - } - - public synchronized void waitFor() throws OOBYield { - if (!closed) { - if (logger.isDebugEnabled()) { - logger.debug("Waiting for " + this); - } - - throw new OOBYield(new FutureNotYetAvailable(this), this); - } - else { - if (logger.isDebugEnabled()) { - logger.debug("Do not need to wait for " + this); - } - checkNoValue(); - if (value instanceof RuntimeException) { - throw (RuntimeException) value; - } - } - } - - protected void checkNoValue() { - if (value == null) { - if (getType().isComposite()) { - // composite types (arrays, structs) don't usually have a value - return; - } - AbstractDataNode parent = getParentNode(); - if (parent != null && parent.getType().isArray()) { - throw new IndexOutOfBoundsException("Invalid index [" + field.getId() + "] for " + parent.getFullName()); - } - else if (getType().isPrimitive()) { - throw new RuntimeException(getFullName() + " has no value"); - } - else { - throw new MissingDataException(this, map()); - } - } - } - - public void addListener(DSHandleListener listener) { - throw new UnsupportedOperationException(); - } - - public void addListener(FutureListener l) { - addListener(l, true); - } - - public void addListener(FutureListener l, boolean partialUpdates) { - boolean closed; - WaitingThreadsMonitor.addThread(l, this); - synchronized(this) { - closed = addListener0(l); - } - if (closed) { - notifyListeners(); - } - } - - protected boolean addListener0(FutureListener l) { - if (this.listeners == null) { - this.listeners = new ArrayList(); - } - this.listeners.add(l); - return this.closed; - } - - protected void notifyListeners() { - List l; - synchronized(this) { - l = this.listeners; - this.listeners = null; - } - if (l != null) { - for (FutureListener ll : l) { - ll.futureUpdated(this); - WaitingThreadsMonitor.removeThread(ll); - } - } - } - - public synchronized void clean() { - if (!handles.isEmpty()) { - for (DSHandle h : handles.values()) { - ((AbstractDataNode) h).clean(); - } - } - else if (!getType().isArray() && !getType().isPrimitive()) { - Mapper mapper = getMapper(); - if (mapper != null) { - mapper.clean(getPathFromRoot()); - } - } - field = null; - handles = null; - value = null; - pathFromRoot = null; - } - - @Override - public synchronized void setWriteRefCount(int count) { - this.writeRefCount = count; - } - - @Override - public synchronized int updateWriteRefCount(int delta) { - this.writeRefCount += delta; - - if (this.writeRefCount < 0) { - throw new IllegalArgumentException("Reference count mismatch for " + this + ". Count is " + this.writeRefCount); - } - - if (logger.isDebugEnabled()) { - logger.debug(this + " writeRefCount " + this.writeRefCount); - } - if (this.writeRefCount == 0) { - if (variableTracer.isEnabled()) { - RootHandle root = getRoot(); - variableTracer.trace(root.getThread(), root.getLine(), getDisplayableName() + " CLOSE write ref count is zero"); - } - closeDeep(); - } - return this.writeRefCount; - } -} Modified: trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -24,6 +24,7 @@ import org.apache.log4j.Logger; import org.griphyn.vdl.mapping.file.FileGarbageCollector; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Type; /** AbstractMapper provides an implementation of the Mapper interface to be Deleted: trunk/src/org/griphyn/vdl/mapping/ArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/ArrayDataNode.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/ArrayDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -1,196 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jun 30, 2006 - */ -package org.griphyn.vdl.mapping; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -import k.rt.FutureListener; - -import org.globus.cog.karajan.futures.FutureNotYetAvailable; -import org.griphyn.vdl.karajan.WaitingThreadsMonitor; -import org.griphyn.vdl.type.Field; - -public class ArrayDataNode extends DataNode { - - private List> keyList; - - protected ArrayDataNode(Field field, RootHandle root, AbstractDataNode parent) { - super(field, root, parent); - } - - public void getFringePaths(List list, Path parentPath) throws HandleOpenException { - checkMappingException(); - if (!isClosed()) { - throw new FutureNotYetAvailable(this); - } - Map, DSHandle> handles = getHandles(); - synchronized (this) { - for (Map.Entry, DSHandle> e : handles.entrySet()) { - AbstractDataNode mapper = (AbstractDataNode) e.getValue(); - Path fullPath = parentPath.addLast(e.getKey(), getType().isArray()); - if (mapper.getType().isComposite()) { - mapper.getFringePaths(list, fullPath); - } - else if (!mapper.getType().isPrimitive()) { - list.add(fullPath); - } - } - } - } - - /** - * Get leaves and make sure they are sorted based on the indices - */ - public void getLeaves(List list) throws HandleOpenException { - checkMappingException(); - if (!isClosed()) { - throw new HandleOpenException(this); - } - Map, DSHandle> handles = getHandles(); - Collection values = handles.values(); - for (DSHandle child : values) { - ((AbstractDataNode) child).getLeaves(list); - } - } - - /** Recursively closes arrays through a tree of arrays and complex - types. */ - public void closeDeep() { - assert(this.getType().isArray()); - closeShallow(); - Map, DSHandle> handles = getHandles(); - synchronized (this) { - for (Map.Entry, DSHandle> e : handles.entrySet()) { - AbstractDataNode child = (AbstractDataNode) e.getValue(); - child.closeDeep(); - } - } - } - - public boolean isArray() { - return true; - } - - public int size() { - return getHandles().size(); - } - - @Override - protected void setField(Comparable id, DSHandle handle) { - synchronized(this) { - super.setField(id, handle); - // Operations on the handles and the wrapper keys need to be synchronized. - // When a wrapper is created, it populates its list of keys with the handles - // available. If this happens concurrently with a call being exactly at this - // point in the code, then it's possible for a key to have both been added - // as part of the creation of the wrapper as well as due to the next call, - // causing duplicate iterations. - addKey(id); - } - } - - private void addKey(Comparable key) { - synchronized(this) { - if (keyList != null) { - keyList.add(key); - } - } - notifyListeners(); - } - - @Override - public void addListener(FutureListener l, boolean partialUpdates) { - boolean shouldNotify; - WaitingThreadsMonitor.addThread(l, this); - synchronized(this) { - shouldNotify = addListener0(l); - if (keyList != null && partialUpdates) { - shouldNotify = true; - } - } - if (shouldNotify) { - notifyListeners(); - } - } - - @Override - public synchronized DSHandle createField(Comparable key) throws NoSuchFieldException { - DSHandle h = super.createField(key); - addKey(key); - return h; - } - - public Iterable> entryList() { - synchronized(this) { - if (isClosed()) { - return new ClosedArrayEntries(getArrayValue()); - } - else { - keyList = new ArrayList>(getArrayValue().keySet()); - return new OpenArrayEntries(keyList, getArrayValue(), this); - } - } - } - - @Override - public void closeShallow() { - super.closeShallow(); - synchronized(this) { - keyList = null; - } - } - - protected void getFields(List fields, Path path) throws InvalidPathException { - if (path.isEmpty()) { - fields.add(this); - } - else { - Path rest = path.butFirst(); - if (path.isWildcard(0)) { - if (!isClosed()) { - throw new FutureNotYetAvailable(this); - } - for (DSHandle handle : getHandles().values()) { - ((AbstractDataNode) handle).getFields(fields, rest); - } - } - else { - try { - ((AbstractDataNode) getField(path.getKey(0))).getFields( - fields, path.butFirst()); - } - catch (NoSuchFieldException e) { - throw new InvalidPathException(path.getKey(0), this); - } - } - } - } - - @Override - protected void checkNoValue() { - // lack of a value in an array node does not indicate an error - } - - -} Modified: trunk/src/org/griphyn/vdl/mapping/DSHandle.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/DSHandle.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/DSHandle.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -71,17 +71,12 @@ * * @return a Collection of DSHandle objects */ - public Collection getFields(Path path) throws InvalidPathException, HandleOpenException; + public Collection getAllFields() throws InvalidPathException, HandleOpenException; public Object getValue(); public void setValue(Object value); - - /** create a new logical component */ - public DSHandle createField(Comparable key) throws NoSuchFieldException; - public DSHandle createField(Path path) throws InvalidPathException; - // special file oriented methods, not sure if these apply to // all datasets @@ -98,8 +93,6 @@ public Path getPathFromRoot(); - public void set(DSHandle svar); - public boolean isClosed(); public void addListener(DSHandleListener listener); Deleted: trunk/src/org/griphyn/vdl/mapping/DataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/DataNode.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/DataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -1,54 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jun 15, 2006 - */ -package org.griphyn.vdl.mapping; - -import org.griphyn.vdl.type.Field; - - -public class DataNode extends AbstractDataNode { - private RootHandle root; - private AbstractDataNode parent; - - protected DataNode(Field field, RootHandle root, AbstractDataNode parent) { - super(field); - if (parent != null && field.getId() == null) { - throw new IllegalArgumentException("Internal error: field id is null"); - } - this.root = root; - this.parent = parent; - } - - public RootHandle getRoot() { - return root; - } - - public DSHandle getParent() { - return parent; - } - - public AbstractDataNode getParentNode() { - return parent; - } - - public void setParent(AbstractDataNode parent) { - this.parent = parent; - } -} Modified: trunk/src/org/griphyn/vdl/mapping/DependentException.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/DependentException.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/DependentException.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -25,10 +25,13 @@ */ public abstract class DependentException extends RuntimeException { private DSHandle handle; + private String name; public DependentException(DSHandle handle, Exception prev) { super(prev); this.handle = handle; + // store the name here since it might get set to null during handle.cleanup() + this.name = handle.getRoot().getName(); } public DependentException(DSHandle handle) { @@ -44,7 +47,7 @@ public String getVariableInfo() { RootHandle root = handle.getRoot(); - return root.getName() + ", line " + root.getLine(); + return name + ", line " + root.getLine(); } public String toString() { Modified: trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -15,6 +15,7 @@ import java.util.Map; import org.apache.log4j.Logger; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.util.VDL2Config; import org.griphyn.vdl.util.VDL2ConfigProperties; Deleted: trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -1,163 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.griphyn.vdl.mapping; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import k.thr.LWThread; - -import org.apache.log4j.Logger; -import org.griphyn.vdl.karajan.Loader; -import org.griphyn.vdl.type.Types; -import org.griphyn.vdl.type.impl.FieldImpl; - -public class ExternalDataNode extends AbstractDataNode implements RootHandle { - - static final String DATASET_URI_PREFIX = "dataset:external:"; - - public static final Logger logger = Logger.getLogger(ExternalDataNode.class); - - private static long datasetIDCounter = 850000000000l; - - private static final String datasetIDPartialID = Loader.getUUID(); - - // previously in mapper params - private int line = -1; - private LWThread thread; - private boolean input; - - - public ExternalDataNode(String name) { - super(new FieldImpl(name, Types.EXTERNAL)); - } - - public int getLine() { - return line; - } - - public void setLine(int line) { - this.line = line; - } - - public boolean isInput() { - return input; - } - - public void setInput(boolean input) { - this.input = input; - } - - public void setThread(LWThread thread) { - this.thread = thread; - } - - public LWThread getThread() { - return thread; - } - - public String getName() { - return (String) getField().getId(); - } - - @Override - public void setName(String name) { - getField().setId(name); - } - - @Override - public void init(Mapper mapper) { - } - - public boolean isRestartable() { - return true; - } - - public RootHandle getRoot() { - return this; - } - - public DSHandle getField(Path path) throws InvalidPathException { - if (path.isEmpty()) { - return this; - } - else { - throw new InvalidPathException(path, this); - } - } - - protected void getFields(List fields, Path path) throws InvalidPathException { - // nothing - } - - public void set(DSHandle handle) { - throw new UnsupportedOperationException(this.getDisplayableName() + " is an external dataset and cannot be set"); - } - - public Map, DSHandle> getArrayValue() { - throw new UnsupportedOperationException("cannot get value of external dataset"); - } - - public boolean isArray() { - return false; - } - - public Collection getFringePaths() throws HandleOpenException { - return Collections.singletonList(Path.EMPTY_PATH); - } - - public Path getPathFromRoot() { - return Path.EMPTY_PATH; - } - - public Mapper getMapper() { - return null; - } - - protected String makeIdentifierURIString() { - datasetIDCounter++; - return DATASET_URI_PREFIX + datasetIDPartialID + ":" + datasetIDCounter; - } - - public DSHandle createDSHandle(String fieldName) { - throw new UnsupportedOperationException("cannot create new field in external dataset"); - } - - public DSHandle getParent() { - return null; - } - - @Override - protected AbstractDataNode getParentNode() { - return null; - } - - @Override - public synchronized void closeDeep() { - if (!this.isClosed()) { - /* - * Need to override this and set a value since - * this is skipped by the normal stageout mechanism which - * does that - */ - this.setValue(FILE_VALUE); - } - } -} Modified: trunk/src/org/griphyn/vdl/mapping/InvalidPathException.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/InvalidPathException.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/InvalidPathException.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -20,7 +20,9 @@ */ package org.griphyn.vdl.mapping; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; + public class InvalidPathException extends Exception { public InvalidPathException(String path, DSHandle source) { super(getMessage(path, source)); Modified: trunk/src/org/griphyn/vdl/mapping/Mapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/Mapper.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/Mapper.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -21,6 +21,7 @@ import java.util.Map; import java.util.Set; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Type; /** This interface must be implemented by a Java class that represents Modified: trunk/src/org/griphyn/vdl/mapping/MappingParamSet.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/MappingParamSet.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/MappingParamSet.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -7,6 +7,7 @@ import java.util.Map; import org.griphyn.vdl.karajan.lib.Tracer; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public abstract class MappingParamSet { Modified: trunk/src/org/griphyn/vdl/mapping/MissingDataException.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/MissingDataException.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/MissingDataException.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -9,6 +9,8 @@ */ package org.griphyn.vdl.mapping; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; + public class MissingDataException extends RuntimeException { public MissingDataException(AbstractDataNode n, PhysicalFormat pf) { super("File not found for variable '" + n.getFullName() + "': " + pf); Modified: trunk/src/org/griphyn/vdl/mapping/NotCompositeException.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/NotCompositeException.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/NotCompositeException.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -9,6 +9,8 @@ */ package org.griphyn.vdl.mapping; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; + public class NotCompositeException extends RuntimeException { private AbstractDataNode node; Modified: trunk/src/org/griphyn/vdl/mapping/NullMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/NullMapper.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/NullMapper.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Set; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Type; Modified: trunk/src/org/griphyn/vdl/mapping/OpenArrayEntries.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/OpenArrayEntries.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/OpenArrayEntries.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -17,13 +17,14 @@ import k.rt.ConditionalYield; import org.griphyn.vdl.karajan.Pair; +import org.griphyn.vdl.mapping.nodes.ArrayHandle; public class OpenArrayEntries implements Iterable> { private List> keyList; private Map, DSHandle> array; - private ArrayDataNode source; + private ArrayHandle source; - public OpenArrayEntries(List> keyList, Map, DSHandle> array, ArrayDataNode source) { + public OpenArrayEntries(List> keyList, Map, DSHandle> array, ArrayHandle source) { this.keyList = keyList; this.array = array; this.source = source; Modified: trunk/src/org/griphyn/vdl/mapping/Path.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/Path.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/Path.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -178,6 +178,10 @@ public Comparable getKey(int index) { return entries.get(index).key; } + + public Entry getEntry(int index) { + return entries.get(index); + } public int size() { return entries == null ? 0 : entries.size(); @@ -198,6 +202,10 @@ public Path butFirst() { return subPath(1); } + + public Path butLast() { + return subPath(0, entries.size() - 1); + } public Path subPath(int fromIndex) { return subPath(fromIndex, entries.size()); Deleted: trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -1,189 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.griphyn.vdl.mapping; - -import k.rt.Future; -import k.rt.FutureListener; -import k.thr.LWThread; - -import org.apache.log4j.Logger; -import org.globus.cog.karajan.futures.FutureNotYetAvailable; -import org.griphyn.vdl.karajan.lib.Tracer; -import org.griphyn.vdl.type.Field; -import org.griphyn.vdl.type.Type; - -public class RootArrayDataNode extends ArrayDataNode implements FutureListener, RootHandle { - - Logger logger = Logger.getLogger(RootArrayDataNode.class); - - private boolean initialized = false; - private Mapper mapper; - private AbstractDataNode waitingMapperParam; - private DuplicateMappingChecker dmc; - - // previously in mapper params - private int line = -1; - private LWThread thread; - private boolean input; - - - public RootArrayDataNode(Type type) { - this("?", type); - } - /** - * Instantiate a root array data node with specified type. - */ - public RootArrayDataNode(String name, Type type) { - super(Field.Factory.getImmutableField(name, type), null, null); - } - - public RootArrayDataNode(String name, Type type, DuplicateMappingChecker dmc) { - this(name, type); - this.dmc = dmc; - } - - public int getLine() { - return line; - } - - public void setLine(int line) { - this.line = line; - } - - public boolean isInput() { - return input; - } - - public void setInput(boolean input) { - this.input = input; - } - - public void setThread(LWThread thread) { - this.thread = thread; - } - - public LWThread getThread() { - return thread; - } - - public String getName() { - return (String) getField().getId(); - } - - @Override - public void setName(String name) { - getField().setId(name); - } - - @Override - public void init(Mapper mapper) { - this.mapper = mapper; - if (this.mapper == null) { - initialized(); - if (isInput()) { - // Empty array. Clearly only in test cases. - closeDeep(); - } - } - else { - innerInit(); - } - } - - private synchronized void innerInit() { - if (logger.isDebugEnabled()) { - logger.debug("innerInit: " + this); - } - - waitingMapperParam = mapper.getFirstOpenParameter(); - if (waitingMapperParam != null) { - waitingMapperParam.addListener(this, false); - if (variableTracer.isEnabled()) { - variableTracer.trace(getThread(), getLine(), getName() + " WAIT " - + Tracer.getVarName(waitingMapperParam)); - } - return; - } - - mapper.initialize(this); - initialized(); - checkInputs(); - if (isClosed()) { - notifyListeners(); - } - } - - private void checkInputs() { - try { - RootDataNode.checkInputs(this, dmc); - } - catch (DependentException e) { - setValue(new MappingDependentException(this, e)); - } - } - - public void futureUpdated(Future f) { - try { - innerInit(); - } - catch (Exception e) { - this.setValue(new MappingException(this, e)); - } - } - - public RootHandle getRoot() { - return this; - } - - public DSHandle getParent() { - return null; - } - - public synchronized Mapper getMapper() { - if (initialized) { - return mapper; - } - if (waitingMapperParam == null) { - return null; - } - else { - throw new FutureNotYetAvailable(waitingMapperParam); - } - } - - public Mapper getActualMapper() { - return mapper; - } - - public boolean isArray() { - return true; - } - - public void setValue(Object value) { - super.setValue(value); - initialized(); - } - - private synchronized void initialized() { - initialized = true; - waitingMapperParam = null; - if (variableTracer.isEnabled()) { - variableTracer.trace(getThread(), getLine(), getName() + " INITIALIZED " + mapper); - } - } -} Deleted: trunk/src/org/griphyn/vdl/mapping/RootDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -1,319 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jun 15, 2006 - */ -package org.griphyn.vdl.mapping; - -import java.util.Collection; - -import k.rt.Future; -import k.rt.FutureListener; -import k.thr.LWThread; - -import org.apache.log4j.Logger; -import org.globus.cog.karajan.futures.FutureNotYetAvailable; -import org.griphyn.vdl.karajan.lib.Tracer; -import org.griphyn.vdl.type.Field; -import org.griphyn.vdl.type.Type; - -public class RootDataNode extends AbstractDataNode implements FutureListener, RootHandle { - - static Logger logger = Logger.getLogger(RootDataNode.class); - - private boolean initialized = false; - private Mapper mapper; - private AbstractDataNode waitingMapperParam; - private DuplicateMappingChecker dmc; - - // previously in mapper params - private int line = -1; - private LWThread thread; - private boolean input; - - - public RootDataNode(String name, Type type, DuplicateMappingChecker dmc) { - super(Field.Factory.getImmutableField(name, type)); - this.dmc = dmc; - } - - public RootDataNode(Type type, Object value) { - this("?", type, value); - } - - public RootDataNode(String name, Type type) { - this(name, type, null); - } - - public RootDataNode(String name, Type type, Object value) { - this(name, type); - initialized(); - setValue(value); - } - - public int getLine() { - return line; - } - - public void setLine(int line) { - this.line = line; - } - - public boolean isInput() { - return input; - } - - public void setInput(boolean input) { - this.input = input; - } - - public void setThread(LWThread thread) { - this.thread = thread; - } - - public LWThread getThread() { - return thread; - } - - public String getName() { - return (String) getField().getId(); - } - - @Override - public void setName(String name) { - getField().setId(name); - } - - public void init(Mapper mapper) { - this.mapper = mapper; - if (mapper == null) { - initialized(); - } - else { - innerInit(); - } - } - - /** must have this.params set to the appropriate parameters before - being called. - * @throws HandleOpenException */ - private synchronized void innerInit() { - waitingMapperParam = mapper.getFirstOpenParameter(); - if (waitingMapperParam != null) { - waitingMapperParam.addListener(this); - if (variableTracer.isEnabled()) { - variableTracer.trace(getThread(), getLine(), getDisplayableName() + " WAIT " - + Tracer.getVarName(waitingMapperParam)); - } - return; - } - - // initialized means that this data has its mapper initialized - // this should be called before checkInputs because the latter - // may trigger calls to things that try to access this data node's - // mapper - mapper.initialize(this); - initialized(); - checkInputs(); - - if (isClosed()) { - notifyListeners(); - } - } - - private void checkInputs() { - try { - checkInputs(this, dmc); - } - catch (DependentException e) { - setValue(new MappingDependentException(this, e)); - } - } - - public void futureUpdated(Future f) { - try { - innerInit(); - } - catch (Exception e) { - this.setValue(new MappingException(this, e)); - } - } - - - static protected void checkInputs(RootHandle root, DuplicateMappingChecker dmc) { - Mapper mapper = root.getActualMapper(); - if (root.isInput()) { - addExisting(mapper.existing(), mapper, root, root); - checkConsistency(root, root, dmc); - } - else if (mapper.isStatic()) { - if (root.isClosed()) { - // this means that code that would have used this variable is already done - // which can happen in cases such as if(false) {a = ...} - return; - } - if (!root.getType().isComposite()) { - return; - } - // Static mappers are (array) mappers which know the size of - // an array statically. A good example is the fixed array mapper - if (logger.isDebugEnabled()) { - logger.debug("mapper: " + mapper); - } - for (Path p : mapper.existing()) { - try { - // Try to get the path in order to check that the - // path is valid - we'll get an exception if not - DSHandle h = root.getField(p); - if (variableTracer.isEnabled()) { - variableTracer.trace(root.getThread(), root.getLine(), - root.getName() + " MAPPING " + p + ", " + mapper.map(p)); - } - } - catch (InvalidPathException e) { - throw new IllegalStateException - ("mapper.existing() returned a path " + - " that it cannot subsequently map: " + - " root: " + root + " path: " + p); - } - } - if (root.isArray()) { - root.closeArraySizes(); - } - checkConsistency(root, root, dmc); - } - } - - public static void addExisting(Collection existing, Mapper mapper, RootHandle root, DSHandle var) { - boolean any = false; - for (Path p : existing) { - try { - DSHandle field = var.getField(p); - field.setValue(FILE_VALUE); - if (variableTracer.isEnabled()) { - variableTracer.trace(root.getThread(), root.getLine(), - root.getName() + " MAPPING " + p + ", " + mapper.map(p)); - } - any = true; - } - catch (InvalidPathException e) { - throw new IllegalStateException("Structure of mapped data is " + - "incompatible with the mapped variable type: " + e.getMessage()); - } - catch (NotCompositeException e) { - throw new IllegalStateException("Cannot map multiple files to variable '" + - e.getDataNode().getFullName() + "' of non composite type '" + - e.getDataNode().getType() + "'"); - } - } - var.closeDeep(); - if (!any && variableTracer.isEnabled()) { - variableTracer.trace(root.getThread(), root.getLine(), - root.getName() + " MAPPING no files found"); - } - } - - public static void checkConsistency(RootHandle root, DSHandle handle, DuplicateMappingChecker dmc) { - if (handle.getType().isArray()) { - // any number of indices is ok - try { - for (DSHandle dh : handle.getFields(Path.CHILDREN)) { - checkConsistency(root, dh, dmc); - } - } - catch (HandleOpenException e) { - // TODO init() should throw some checked exception - throw new RuntimeException("Mapper consistency check failed for " + handle - + ". A HandleOpenException was thrown during consistency checking for "+e.getSource(), e); - } - catch (InvalidPathException e) { - e.printStackTrace(); - } - } - else { - // all fields must be present - Type type = handle.getType(); - if (!type.isPrimitive() && !type.isComposite()) { - // mapped. Feed the DMC. - PhysicalFormat f = root.getActualMapper().map(handle.getPathFromRoot()); - if (root.isInput()) { - dmc.addRead(f, handle); - } - else { - dmc.addWrite(f, handle); - } - } - for (String fieldName : type.getFieldNames()) { - Path fieldPath = Path.parse(fieldName); - try { - checkConsistency(root, handle.getField(fieldPath), dmc); - } - catch (InvalidPathException e) { - throw new RuntimeException("Data set initialization failed for " + handle - + ". Missing required field: " + fieldName); - } - } - } - } - - public RootHandle getRoot() { - return this; - } - - public DSHandle getParent() { - return null; - } - - @Override - protected AbstractDataNode getParentNode() { - return null; - } - - public synchronized Mapper getMapper() { - if (initialized) { - return mapper; - } - if (waitingMapperParam == null) { - return null; - } - else { - throw new FutureNotYetAvailable(waitingMapperParam); - } - } - - public Mapper getActualMapper() { - return mapper; - } - - public boolean isArray() { - return false; - } - - public void setValue(Object value) { - super.setValue(value); - initialized(); - } - - private synchronized void initialized() { - initialized = true; - waitingMapperParam = null; - if (variableTracer.isEnabled()) { - variableTracer.trace(getThread(), getLine(), getName() + " INITIALIZED " + mapper); - } - } -} Modified: trunk/src/org/griphyn/vdl/mapping/RootHandle.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/RootHandle.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/RootHandle.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -26,4 +26,6 @@ Mapper getActualMapper(); boolean isArray(); void closeArraySizes(); + + void mapperInitialized(Mapper mapper); } Modified: trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; Modified: trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.DSHandle; Modified: trunk/src/org/griphyn/vdl/mapping/file/CSVMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/CSVMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/CSVMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.DSHandle; Modified: trunk/src/org/griphyn/vdl/mapping/file/ExternalMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/ExternalMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/ExternalMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -6,8 +6,8 @@ import java.util.List; import java.util.Map; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class ExternalMapperParams extends MappingParamSet { Modified: trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.DSHandle; Modified: trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -132,7 +132,7 @@ } private static final List IMPORTS = Arrays.asList("java.util.Arrays", "java.util.Collection", - "java.util.List", "org.griphyn.vdl.mapping.AbstractDataNode", "org.griphyn.vdl.mapping.MappingParamSet"); + "java.util.List", "org.griphyn.vdl.mapping.nodes.AbstractDataNode", "org.griphyn.vdl.mapping.MappingParamSet"); private static void writeFile(File nf, String pkg, List params, Map opts) throws IOException { String name = nf.getName().substring(0, nf.getName().lastIndexOf('.')); Modified: trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.DSHandle; Modified: trunk/src/org/griphyn/vdl/mapping/file/SimpleFileMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/SimpleFileMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/SimpleFileMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; Modified: trunk/src/org/griphyn/vdl/mapping/file/SingleFileMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/SingleFileMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/SingleFileMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -5,8 +5,8 @@ import java.util.List; import org.griphyn.vdl.mapping.AbsFile; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class SingleFileMapperParams extends MappingParamSet { Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -69,7 +69,7 @@ Collection output = new ArrayList(); Collection sourceFields; try { - sourceFields = sourceHandle.getFields(Path.CHILDREN); + sourceFields = sourceHandle.getAllFields(); } catch (InvalidPathException ipe) { return Collections.emptyList(); Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.DSHandle; Modified: trunk/src/org/griphyn/vdl/mapping/file/TestMapperParams.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/TestMapperParams.java 2014-05-08 21:57:35 UTC (rev 7838) +++ trunk/src/org/griphyn/vdl/mapping/file/TestMapperParams.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -3,7 +3,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.mapping.MappingParamSet; Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedArrayDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedArrayDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,158 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.AbstractMap; +import java.util.AbstractSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.griphyn.vdl.mapping.ClosedArrayEntries; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.type.Field; +import org.griphyn.vdl.type.Types; + + +public abstract class AbstractClosedArrayDataNode extends AbstractClosedDataNode implements ArrayHandle { + private DSHandle[] values; + + public AbstractClosedArrayDataNode(Field field, List values) { + super(field); + setValues(values); + } + + private void setValues(List values) { + int sz = values.size(); + this.values = new DSHandle[sz]; + int index = 0; + Iterator i = values.iterator(); + while (i.hasNext()) { + Object n = i.next(); + if (n instanceof DSHandle) { + this.values[index] = (DSHandle) n; + } + else if (n instanceof String) { + this.values[index] = NodeFactory.newNode(Field.Factory.createField(index, Types.STRING), getRoot(), this, n); + } + else { + throw new RuntimeException( + "An array variable can only be initialized by a list of DSHandle values"); + } + index++; + } + } + + @Override + protected Object getRawValue() { + return null; + } + + @Override + public synchronized DSHandle getField(Comparable key) throws NoSuchFieldException { + int index = ((Integer) key).intValue(); + if (index < 0 || index >= values.length) { + throw new NoSuchFieldException(String.valueOf(index)); + } + return values[index]; + } + + @Override + public Object getValue() { + return getArrayValue(); + } + + @Override + public Map, DSHandle> getArrayValue() { + return new AbstractMap, DSHandle>() { + @Override + public Set, DSHandle>> entrySet() { + return new AbstractSet, DSHandle>>() { + @Override + public Iterator, DSHandle>> iterator() { + return new Iterator, DSHandle>>() { + private int index = 0; + + @Override + public boolean hasNext() { + return index < values.length; + } + + @Override + public Map.Entry, DSHandle> next() { + final int i = index; + index++; + return new Map.Entry, DSHandle>() { + @Override + public Comparable getKey() { + return Integer.valueOf(i); + } + + @Override + public DSHandle getValue() { + return values[i]; + } + + @Override + public DSHandle setValue(DSHandle value) { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public int size() { + return values.length; + } + }; + } + }; + } + + @Override + public Iterable> entryList() { + return new ClosedArrayEntries(getArrayValue()); + } + + @Override + public void getFringePaths(List list, Path myPath) throws HandleOpenException { + for (int i = 0; i < values.length; i++) { + Path fullPath = myPath.addLast(i, true); + ((AbstractDataNode) values[i]).getFringePaths(list, fullPath); + } + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + for (DSHandle h : values) { + AbstractDataNode child = (AbstractDataNode) h; + child.getLeaves(list); + } + } + + public boolean isArray() { + return true; + } + + @Override + public int arraySize() { + return values.length; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,72 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import k.rt.FutureListener; + +import org.globus.cog.karajan.compiled.nodes.Node; +import org.griphyn.vdl.mapping.OOBYield; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractClosedDataNode extends AbstractDataNode { + + public AbstractClosedDataNode(Field field) { + super(field); + } + + @Override + public boolean isClosed() { + return true; + } + + @Override + public void closeShallow() { + // already closed + } + + @Override + public void closeDeep() { + // already closed + } + + public void setValue(Object value) { + throw new IllegalArgumentException(this.getFullName() + + " is closed with a value of " + this.getValue()); + } + + @Override + protected boolean addListener0(FutureListener l) { + throw new UnsupportedOperationException("Cannot add listener to closed node"); + } + + @Override + protected void notifyListeners() { + } + + @Override + public void setWriteRefCount(int count) { + if (count != 0) { + throw new UnsupportedOperationException("Attempt to set non-zero write ref count on read-only node"); + } + } + + @Override + public int updateWriteRefCount(int delta) { + throw new UnsupportedOperationException("Attempt to update write ref count on read-only node"); + } + + @Override + public synchronized void waitFor(Node who) { + } + + @Override + public synchronized void waitFor() throws OOBYield { + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedMappedSingleDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedMappedSingleDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedMappedSingleDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,33 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractClosedMappedSingleDataNode extends AbstractClosedNonCompositeDataNode { + + public AbstractClosedMappedSingleDataNode(Field field, Object value) { + super(field, value); + } + + public void getFringePaths(List list, Path myPath) throws HandleOpenException { + list.add(myPath); + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + list.add(this); + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedNonCompositeDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedNonCompositeDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedNonCompositeDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,42 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import org.griphyn.vdl.mapping.DataDependentException; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractClosedNonCompositeDataNode extends AbstractClosedDataNode { + private Object value; + + public AbstractClosedNonCompositeDataNode(Field field, Object value) { + super(field); + this.value = value; + } + + public AbstractClosedNonCompositeDataNode(Field field, DependentException e) { + super(field); + this.value = new DataDependentException(this, e); + } + + @Override + public synchronized Object getValue() { + return value; + } + + @Override + protected Object getRawValue() { + return value; + } + + public boolean isArray() { + return false; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedPrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedPrimitiveDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedPrimitiveDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,38 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractClosedPrimitiveDataNode extends AbstractClosedNonCompositeDataNode { + + public AbstractClosedPrimitiveDataNode(Field field, Object value) { + super(field, value); + } + + public AbstractClosedPrimitiveDataNode(Field field, DependentException e) { + super(field, e); + } + + public void getFringePaths(List list, Path parentPath) throws HandleOpenException { + // only mappable paths + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + // only mappable paths + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedStructDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedStructDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedStructDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,80 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.type.Field; +import org.griphyn.vdl.type.Type; + +public abstract class AbstractClosedStructDataNode extends AbstractClosedDataNode { + private DSHandle[] fields; + + public AbstractClosedStructDataNode(Field field) { + super(field); + fields = new DSHandle[field.getType().getFields().size()]; + } + + protected void setField(String name, DSHandle n) throws NoSuchFieldException { + fields[field.getType().getFieldIndex(name)] = n; + } + + @Override + public synchronized DSHandle getField(Comparable key) throws NoSuchFieldException { + return fields[field.getType().getFieldIndex((String) key)]; + } + + @Override + public Collection getAllFields() throws InvalidPathException, HandleOpenException { + return Arrays.asList(fields); + } + + @Override + public void getFringePaths(List list, Path myPath) throws HandleOpenException { + Type t = field.getType(); + for (Field f : t.getFields()) { + AbstractDataNode child; + try { + child = (AbstractDataNode) getField(f.getId()); + } + catch (Exception e) { + throw new RuntimeException("Structure inconsistency detected for field " + f); + } + Path fullPath = myPath.addLast(f.getId()); + child.getFringePaths(list, fullPath); + } + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + Type t = field.getType(); + for (Field f : t.getFields()) { + AbstractDataNode child; + try { + child = (AbstractDataNode) getField(f.getId()); + } + catch (Exception e) { + throw new RuntimeException("Structure inconsistency detected for field " + f); + } + child.getLeaves(list); + } + } + + + public boolean isArray() { + return false; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,485 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* + * Created on Jun 6, 2006 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import k.rt.FutureListener; +import k.rt.FutureValue; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.griphyn.vdl.karajan.Loader; +import org.griphyn.vdl.karajan.WaitingThreadsMonitor; +import org.griphyn.vdl.karajan.lib.Tracer; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DSHandleListener; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.OOBYield; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.type.Field; +import org.griphyn.vdl.type.Type; +import org.griphyn.vdl.util.VDL2Config; + + + +public abstract class AbstractDataNode implements DSHandle, FutureValue { + public static final Object FILE_VALUE = new Object() { + public String toString() { + return ""; + } + }; + + static final String DATASET_URI_PREFIX = "dataset:"; + + public static final Logger logger = Logger.getLogger(AbstractDataNode.class); + + /** + * Datasets are identified within a run by this sequence number and the + * partial ID field. The initial value is chosen to aid human recognition of + * sequence numbers in the wild. There is no requirement that it start at + * this (or any other) particular value. Note that this introduces a maximum + * on the number of datasets which can be dealt with in any run to be about + * 2^62. + */ + private static long datasetIDCounter = 720000000000l; + + /** + * This is used to provide a (hopefully) globally unique identifier for each + * time the datasetIDCounter is reset (whenever this class is loaded, which + * will usually happen once per JVM). No meaning should be inferred from + * this value - it exists purely for making unique URIs. + */ + private static final String datasetIDPartialID = Loader.getUUID(); + + public static boolean provenance = false; + static { + try { + provenance = VDL2Config.getConfig().getProvenanceLog(); + } + catch (IOException e) { + } + } + + protected Field field; + private String identifier; + + protected static final Tracer variableTracer = Tracer.getTracer("VARIABLE"); + + protected AbstractDataNode(Field field) { + this.field = field; + } + + public void initialize() { + } + + public String getName() { + return getRoot().getName(); + } + + @Override + public void setName(String name) { + throw new UnsupportedOperationException("setName can only be called on a root variable"); + } + + public Type getType() { + return field.getType(); + } + + public boolean isPrimitive() { + return field.getType().isPrimitive(); + } + + public boolean isRestartable() { + return !isPrimitive(); + } + + public Field getField() { + return field; + } + + protected abstract AbstractDataNode getParentNode(); + + /** + * create a String representation of this node. If the node has a value, + * then uses the String representation of that value. Otherwise, generates a + * text description. + */ + public String toString() { + return toDisplayableString(); + } + + private String toDisplayableString() { + StringBuilder sb = new StringBuilder(); + sb.append(getDisplayableName()); + Path p = getPathFromRoot(); + if (!p.isEmpty()) { + if (!p.isArrayIndex(0)) { + sb.append("."); + } + sb.append(p.toString()); + } + sb.append(":"); + Type type = getType(); + String strtype = type.toString(); + if (type.isArray() && isClosed()) { + strtype = strtype.replace("[]", "[" + this.arraySize() + "]"); + } + sb.append(strtype); + Object value = getRawValue(); + + if (value != null) { + sb.append(" = "); + if (value instanceof Throwable) { + sb.append(value.getClass().getName()); + } + else { + sb.append(value); + } + } + if (isClosed()) { + sb.append(" - Closed"); + } + else { + sb.append(" - Open"); + } + return sb.toString(); + } + + protected abstract Object getRawValue(); + + protected int arraySize() { + return -1; + } + + public String getIdentifyingString() { + StringBuffer sb = new StringBuffer(); + sb.append(this.getClass().getName()); + + sb.append(" identifier "); + sb.append(this.getIdentifier()); + + sb.append(" type "); + sb.append(getType()); + + Object value = getRawValue(); + if (value == null) { + sb.append(" with no value at dataset="); + } + else if (value instanceof Throwable) { + sb.append(" containing throwable "); + sb.append(value.getClass()); + sb.append(" dataset="); + } + else { + sb.append(" value="); + sb.append(value.toString()); + sb.append(" dataset="); + } + + sb.append(getDisplayableName()); + + if (!Path.EMPTY_PATH.equals(getPathFromRoot())) { + sb.append(" path="); + sb.append(getPathFromRoot().toString()); + } + + if (isClosed()) { + sb.append(" (closed)"); + } + else { + sb.append(" (not closed)"); + } + + return sb.toString(); + } + + public String getDisplayableName() { + return getName(); + } + + public String getFullName() { + String name = getDisplayableName(); + Path p = getPathFromRoot(); + if (p.isEmpty()) { + return name; + } + else { + return name + "." + p; + } + } + + public DSHandle getField(Path path) throws InvalidPathException { + if (path.isEmpty()) { + return this; + } + try { + DSHandle handle = getField(path.getFirst()); + + if (path.size() > 1) { + return handle.getField(path.butFirst()); + } + else { + return handle; + } + } + catch (NoSuchFieldException e) { + throw new InvalidPathException(path, this); + } + } + + @Override + public Collection getAllFields() throws InvalidPathException, HandleOpenException { + throw new InvalidPathException("No fields"); + } + + @Override + public DSHandle getField(Comparable key) throws NoSuchFieldException { + throw new NoSuchFieldException("No fields"); + } + + public Map, DSHandle> getArrayValue() { + throw new RuntimeException("getArrayValue called on a non-array: " + this); + } + + @Override + public Collection getFringePaths() throws HandleOpenException { + List list = new ArrayList(); + getFringePaths(list, Path.EMPTY_PATH); + return list; + } + + protected abstract void getFringePaths(List list, Path myPath) throws HandleOpenException; + + public Collection getLeaves() throws HandleOpenException { + List list = new ArrayList(); + getLeaves(list); + return list; + } + + protected abstract void getLeaves(List list) throws HandleOpenException; + + protected void postCloseActions() { + // closed + notifyListeners(); + if (logger.isDebugEnabled()) { + logger.debug("closed " + this.getIdentifyingString()); + } + // so because its closed, we can dump the contents + + try { + if(provenance) { + logContent(); + } + } + catch (Exception e) { + logger.warn("Exception whilst logging dataset content for " + this, e); + } + // TODO record retrospective provenance information for this dataset here + // we should do it at closing time because that's the point at which we + // know the dataset has its values (all the way down the tree) assigned. + + // provenance-id for this dataset should have been assigned at creation time, + // though, so that we can refer to this dataset elsewhere before it is closed. + + // is this method the only one called to set this.closed? or do subclasses + // or other methods ever change it? + } + + public synchronized void logContent() { + String identifier = this.getIdentifier(); + Path pathFromRoot = this.getPathFromRoot(); + Type type = this.getType(); + if (this.getPathFromRoot() != null) { + if (logger.isInfoEnabled()) { + logger.info("ROOTPATH dataset=" + identifier + " path=" + pathFromRoot); + if (this.getType().isPrimitive()) { + logger.info("VALUE dataset=" + identifier + " VALUE=" + this.toString()); + } + } + + Mapper m = getActualMapper(); + + if (m != null) { + // TODO proper type here + // Not sure catching exception here is really the right thing to + // do here + // anyway - should perhaps only be trying to map leafnodes? + // Mapping + // non-leaf stuff is giving wierd paths anyway + + // TODO this is perhaps an unpleasant way of finding if this is a file-backed + // leaf node or not + boolean filemapped = true; + + if (type.getName().equals("external")) { + filemapped = false; + } + if (type.isPrimitive()) { + filemapped = false; + } + if (type.isComposite()) { + filemapped = false; + } + + try { + if (filemapped) { + Object path = map(); + if (logger.isInfoEnabled()) { + logger.info("FILENAME dataset=" + identifier + " filename=" + path); + } + } + } + catch (Exception e) { + if (logger.isInfoEnabled()) { + logger.info("NOFILENAME dataset=" + identifier); + } + } + } + } + + if (type.isComposite()) { + synchronized (this) { + try { + for (DSHandle handle : this.getAllFields()) { + AbstractDataNode node = (AbstractDataNode) handle; + if (logger.isInfoEnabled()) { + logger.info("CONTAINMENT parent=" + identifier + " child=" + node.getIdentifier()); + } + node.logContent(); + } + } + catch (Exception e) { + logger.warn("Exception caught while trying to log provenance data", e); + } + } + } + } + + private Mapper getActualMapper() { + return getRoot().getActualMapper(); + } + + protected Path calculatePathFromRoot() { + AbstractDataNode parent = (AbstractDataNode) this.getParent(); + Path myPath; + if (parent != null) { + myPath = parent.getPathFromRoot(); + return myPath.addLast(getField().getId(), parent.getField().getType().isArray()); + } + else { + return Path.EMPTY_PATH; + } + } + + public Mapper getMapper() { + return ((AbstractDataNode) getRoot()).getMapper(); + } + + protected InitMapper getInitMapper() { + Mapper m = getActualMapper(); + if (m instanceof InitMapper) { + return (InitMapper) m; + } + else { + throw new IllegalStateException("Cannot get initialization mapper on initialized node"); + } + } + + public boolean isInput() { + return getInitMapper().isInput(); + } + + public void setInput(boolean input) { + getInitMapper().setInput(input); + } + + @Override + public PhysicalFormat map(Path path) { + Mapper m = getMapper(); + if (m == null) { + return null; + } + else { + Path p = getPathFromRoot().append(path); + return m.map(p); + } + } + + @Override + public PhysicalFormat map() { + return map(Path.EMPTY_PATH); + } + + public synchronized String getIdentifier() { + if (identifier == null) { + identifier = makeIdentifierURIString(); + } + return identifier; + } + + String makeIdentifierURIString() { + datasetIDCounter++; + return DATASET_URI_PREFIX + datasetIDPartialID + ":" + datasetIDCounter; + } + + public void addListener(DSHandleListener listener) { + throw new UnsupportedOperationException(); + } + + public void addListener(FutureListener l) { + addListener(l, true); + } + + public void addListener(FutureListener l, boolean partialUpdates) { + boolean closed; + WaitingThreadsMonitor.addThread(l, this); + synchronized(this) { + closed = addListener0(l); + } + if (closed) { + notifyListeners(); + } + } + + protected abstract boolean addListener0(FutureListener l); + + protected abstract void notifyListeners(); + + public synchronized void clean() { + clean0(); + } + + protected void clean0() { + field = null; + } + + public abstract void waitFor(Node who); + + public abstract void waitFor() throws OOBYield; +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,232 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import k.rt.FutureListener; + +import org.globus.cog.karajan.futures.FutureNotYetAvailable; +import org.griphyn.vdl.karajan.WaitingThreadsMonitor; +import org.griphyn.vdl.mapping.ClosedArrayEntries; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.MappingDependentException; +import org.griphyn.vdl.mapping.OpenArrayEntries; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractFutureArrayDataNode extends AbstractFutureDataNode implements ArrayHandle { + private Map, DSHandle> handles; + private RuntimeException exception; + private List> keyList; + + public AbstractFutureArrayDataNode(Field field) { + super(field); + handles = new HashMap, DSHandle>(); + } + + public void addField(Comparable key, DSHandle n) { + synchronized(this) { + handles.put(key, n); + addKey(key); + } + notifyListeners(); + } + + private void addKey(Comparable key) { + if (keyList != null) { + keyList.add(key); + } + } + + + @Override + public RootHandle getRoot() { + return null; + } + + @Override + public DSHandle getParent() { + return null; + } + + @Override + public Path getPathFromRoot() { + return null; + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public void addListener(FutureListener l, boolean partialUpdates) { + boolean shouldNotify; + WaitingThreadsMonitor.addThread(l, this); + synchronized(this) { + shouldNotify = addListener0(l); + if (keyList != null && partialUpdates) { + shouldNotify = true; + } + } + if (shouldNotify) { + notifyListeners(); + } + } + + @Override + public Iterable> entryList() { + synchronized(this) { + if (isClosed()) { + return new ClosedArrayEntries(getArrayValue()); + } + else { + keyList = new ArrayList>(getArrayValue().keySet()); + return new OpenArrayEntries(keyList, getArrayValue(), this); + } + } + } + + @Override + protected Object getRawValue() { + return handles; + } + + public synchronized DSHandle getField(Comparable key) throws NoSuchFieldException { + DSHandle handle = handles.get(key); + if (handle == null) { + if (isClosed()) { + throw new NoSuchFieldException(key.toString()); + } + handle = createField(key); + } + return handle; + } + + private DSHandle createField(Comparable key) { + DSHandle h = NodeFactory.newNode(Field.Factory.createField(key, field.getType().itemType()), getRoot(), this); + synchronized(this) { + handles.put(key, h); + addKey(key); + } + notifyListeners(); + return h; + } + + @Override + public Collection getAllFields() throws InvalidPathException, HandleOpenException { + synchronized(this) { + if (!isClosed()) { + throw new HandleOpenException(this); + } + } + return handles.values(); + } + + protected void checkDataException() { + if (exception instanceof DependentException) { + throw (DependentException) exception; + } + } + + protected void checkMappingException() { + if (exception instanceof MappingDependentException) { + throw (MappingDependentException) exception; + } + } + + public synchronized Object getValue() { + checkDataException(); + return handles; + } + + public Map, DSHandle> getArrayValue() { + checkDataException(); + return handles; + } + + @Override + public synchronized void closeShallow() { + super.closeShallow(); + this.keyList = null; + if (handles.isEmpty()) { + handles = Collections.emptyMap(); + } + } + + @Override + public void closeDeep() { + closeShallow(); + // closeShallow provides a synchronization point, so we know handles cannot be modified after that + for (DSHandle handle : handles.values()) { + handle.closeDeep(); + } + } + + @Override + public void getFringePaths(List list, Path myPath) throws HandleOpenException { + checkMappingException(); + if (!isClosed()) { + throw new FutureNotYetAvailable(this); + } + for (Map.Entry, DSHandle> e : handles.entrySet()) { + AbstractDataNode child = (AbstractDataNode) e.getValue(); + Path fullPath = myPath.addLast(e.getKey(), true); + child.getFringePaths(list, fullPath); + } + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + checkMappingException(); + if (!isClosed()) { + throw new HandleOpenException(this); + } + for (DSHandle h : handles.values()) { + AbstractDataNode child = (AbstractDataNode) h; + child.getLeaves(list); + } + } + + public boolean isArray() { + return true; + } + + @Override + protected void clean0() { + for (DSHandle h : handles.values()) { + ((AbstractDataNode) h).clean0(); + } + handles = null; + super.clean0(); + } + + @Override + protected void checkNoValue() { + if (exception != null) { + throw exception; + } + } + + @Override + public int arraySize() { + return handles.size(); + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,155 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.ArrayList; +import java.util.List; + +import k.rt.FutureListener; +import k.thr.Yield; + +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; +import org.griphyn.vdl.karajan.WaitingThreadsMonitor; +import org.griphyn.vdl.mapping.OOBYield; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractFutureDataNode extends AbstractDataNode { + private boolean closed; + private List listeners; + private int writeRefCount; + + public AbstractFutureDataNode(Field field) { + super(field); + } + + @Override + protected boolean addListener0(FutureListener l) { + if (this.listeners == null) { + this.listeners = new ArrayList(); + } + this.listeners.add(l); + return this.isClosed(); + } + + @Override + protected void notifyListeners() { + List l; + synchronized(this) { + l = this.listeners; + this.listeners = null; + } + if (l != null) { + for (FutureListener ll : l) { + ll.futureUpdated(this); + WaitingThreadsMonitor.removeThread(ll); + } + } + } + + @Override + public boolean isClosed() { + return closed; + } + + protected void setClosed(boolean closed) { + this.closed = closed; + } + + public void closeShallow() { + synchronized(this) { + if (this.closed) { + return; + } + this.closed = true; + } + postCloseActions(); + } + + public void closeDeep() { + closeShallow(); + } + + public void setValue(Object value) { + throw new IllegalArgumentException(this.getFullName() + + " cannot set the value of a " + this.getType()); + } + + @Override + protected void clean0() { + listeners = null; + super.clean0(); + } + + @Override + public synchronized void setWriteRefCount(int count) { + this.writeRefCount = count; + } + + @Override + public synchronized int updateWriteRefCount(int delta) { + this.writeRefCount += delta; + + if (this.writeRefCount < 0) { + throw new IllegalArgumentException("Reference count mismatch for " + this + ". Count is " + this.writeRefCount); + } + + if (logger.isDebugEnabled()) { + logger.debug(this + " writeRefCount " + this.writeRefCount); + } + if (this.writeRefCount == 0) { + if (variableTracer.isEnabled()) { + RootHandle root = getRoot(); + variableTracer.trace(root.getThread(), root.getLine(), getDisplayableName() + " CLOSE write ref count is zero"); + } + closeDeep(); + } + return this.writeRefCount; + } + + @Override + public synchronized void waitFor(Node who) { + if (!closed) { + if (logger.isDebugEnabled()) { + logger.debug("Waiting for " + this); + } + + Yield y = new FutureNotYetAvailable(this); + y.getState().addTraceElement(who); + throw y; + } + else { + if (logger.isDebugEnabled()) { + logger.debug("Do not need to wait for " + this); + } + checkNoValue(); + } + } + + @Override + public synchronized void waitFor() throws OOBYield { + if (!closed) { + if (logger.isDebugEnabled()) { + logger.debug("Waiting for " + this); + } + + throw new OOBYield(new FutureNotYetAvailable(this), this); + } + else { + if (logger.isDebugEnabled()) { + logger.debug("Do not need to wait for " + this); + } + checkNoValue(); + } + } + + protected abstract void checkNoValue(); +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureMappedSingleDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureMappedSingleDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureMappedSingleDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,43 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractFutureMappedSingleDataNode extends AbstractFutureNonCompositeDataNode { + + public AbstractFutureMappedSingleDataNode(Field field) { + super(field); + } + + public void getFringePaths(List list, Path myPath) throws HandleOpenException { + list.add(myPath); + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + list.add(this); + } + + @Override + protected void clean0() { + Mapper mapper = getMapper(); + if (mapper != null) { + mapper.clean(getPathFromRoot()); + } + super.clean0(); + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureNonCompositeDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureNonCompositeDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureNonCompositeDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,94 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.MappingDependentException; +import org.griphyn.vdl.mapping.MissingDataException; +import org.griphyn.vdl.type.Field; + + +public abstract class AbstractFutureNonCompositeDataNode extends AbstractFutureDataNode { + private Object value; + + public AbstractFutureNonCompositeDataNode(Field field) { + super(field); + } + + @Override + protected Object getRawValue() { + return value; + } + + protected void checkDataException() { + if (value instanceof DependentException) { + throw (DependentException) value; + } + } + + protected void checkMappingException() { + if (value instanceof MappingDependentException) { + throw (MappingDependentException) value; + } + } + + @Override + public synchronized Object getValue() { + checkNoValue(); + checkDataException(); + return value; + } + + @Override + protected void checkNoValue() { + if (value == null) { + AbstractDataNode parent = getParentNode(); + if (parent != null && parent.getType().isArray()) { + throw new IndexOutOfBoundsException("Invalid index [" + field.getId() + "] for " + parent.getFullName()); + } + else if (getType().isPrimitive()) { + throw new RuntimeException(getFullName() + " has no value"); + } + else { + throw new MissingDataException(this, map()); + } + } + if (value instanceof RuntimeException) { + throw (RuntimeException) value; + } + } + + public void setValue(Object value) { + synchronized(this) { + if (isClosed()) { + throw new IllegalArgumentException(this.getFullName() + + " is closed with a value of " + this.value); + } + if (this.value != null) { + throw new IllegalArgumentException(this.getFullName() + + " is already assigned with a value of " + this.value); + } + + this.value = value; + this.setClosed(true); + } + postCloseActions(); + } + + public boolean isArray() { + return false; + } + + @Override + protected void clean0() { + value = null; + super.clean0(); + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFuturePrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFuturePrimitiveDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFuturePrimitiveDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,33 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractFuturePrimitiveDataNode extends AbstractFutureNonCompositeDataNode { + + public AbstractFuturePrimitiveDataNode(Field field) { + super(field); + } + + public void getFringePaths(List list, Path parentPath) throws HandleOpenException { + // only mappable paths + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + // only mappable paths + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureStructDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureStructDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureStructDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,142 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.Arrays; +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.MappingDependentException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; +import org.griphyn.vdl.type.Type; + +public abstract class AbstractFutureStructDataNode extends AbstractFutureDataNode { + private DSHandle[] fields; + private RuntimeException exception; + + public AbstractFutureStructDataNode(Field field, RootHandle root) { + super(field); + fields = new DSHandle[field.getType().getFields().size()]; + } + + @Override + public void initialize() { + super.initialize(); + createFieldNodes(); + } + + private void createFieldNodes() { + for (Field f : field.getType().getFields()) { + try { + setField((String) f.getId(), NodeFactory.newNode(f, getRoot(), this)); + } + catch (NoSuchFieldException e) { + throw new RuntimeException("Type inconsistency detected for field " + f); + } + } + } + + protected void setField(String name, DSHandle n) throws NoSuchFieldException { + fields[field.getType().getFieldIndex(name)] = n; + } + + @Override + protected Object getRawValue() { + return null; + } + + @Override + public Object getValue() { + return Arrays.asList(fields); + } + + protected void checkDataException() { + if (exception instanceof DependentException) { + throw (DependentException) exception; + } + } + + protected void checkMappingException() { + if (exception instanceof MappingDependentException) { + throw (MappingDependentException) exception; + } + } + + @Override + public synchronized DSHandle getField(Comparable key) throws NoSuchFieldException { + return fields[field.getType().getFieldIndex((String) key)]; + } + + @Override + public void closeDeep() { + closeShallow(); + for (DSHandle handle : fields) { + handle.closeDeep(); + } + } + + @Override + public void getFringePaths(List list, Path myPath) throws HandleOpenException { + checkMappingException(); + Type t = field.getType(); + for (Field f : t.getFields()) { + AbstractDataNode child; + try { + child = (AbstractDataNode) getField(f.getId()); + } + catch (Exception e) { + throw new RuntimeException("Structure inconsistency detected for field " + f); + } + Path fullPath = myPath.addLast(f.getId()); + child.getFringePaths(list, fullPath); + } + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + checkMappingException(); + Type t = field.getType(); + for (Field f : t.getFields()) { + AbstractDataNode child; + try { + child = (AbstractDataNode) getField(f.getId()); + } + catch (Exception e) { + throw new RuntimeException("Structure inconsistency detected for field " + f); + } + child.getLeaves(list); + } + } + + public boolean isArray() { + return false; + } + + @Override + protected void clean0() { + for (DSHandle h : fields) { + ((AbstractDataNode) h).clean0(); + } + fields = null; + exception = null; + super.clean0(); + } + + @Override + protected void checkNoValue() { + if (exception != null) { + throw exception; + } + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/ArrayHandle.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/ArrayHandle.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/ArrayHandle.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,22 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 30, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import k.rt.Future; + +import org.griphyn.vdl.mapping.DSHandle; + +public interface ArrayHandle extends DSHandle, Future { + Iterable> entryList(); + + public int arraySize(); +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/ClosedArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/ClosedArrayDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/ClosedArrayDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,47 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class ClosedArrayDataNode extends AbstractClosedArrayDataNode { + private RootHandle root; + private AbstractDataNode parent; + private Path pathFromRoot; + + protected ClosedArrayDataNode(Field field, RootHandle root, AbstractDataNode parent, List values) { + super(field, values); + this.root = root; + this.parent = parent; + this.pathFromRoot = parent.getPathFromRoot().addLast(field.getId(), parent.getType().isArray()); + } + + public RootHandle getRoot() { + return root; + } + + public DSHandle getParent() { + return parent; + } + + public AbstractDataNode getParentNode() { + return parent; + } + + @Override + public Path getPathFromRoot() { + return pathFromRoot; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/ClosedMappedSingleDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/ClosedMappedSingleDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/ClosedMappedSingleDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,45 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class ClosedMappedSingleDataNode extends AbstractClosedMappedSingleDataNode { + private RootHandle root; + private AbstractDataNode parent; + private Path pathFromRoot; + + protected ClosedMappedSingleDataNode(Field field, RootHandle root, AbstractDataNode parent) { + super(field, AbstractDataNode.FILE_VALUE); + this.root = root; + this.parent = parent; + this.pathFromRoot = parent.getPathFromRoot().addLast(field.getId(), parent.getType().isArray()); + } + + public RootHandle getRoot() { + return root; + } + + public DSHandle getParent() { + return parent; + } + + public AbstractDataNode getParentNode() { + return parent; + } + + @Override + public Path getPathFromRoot() { + return pathFromRoot; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/ClosedPrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/ClosedPrimitiveDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/ClosedPrimitiveDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,45 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class ClosedPrimitiveDataNode extends AbstractClosedMappedSingleDataNode { + private RootHandle root; + private AbstractDataNode parent; + private Path pathFromRoot; + + protected ClosedPrimitiveDataNode(Field field, RootHandle root, AbstractDataNode parent, Object value) { + super(field, value); + this.root = root; + this.parent = parent; + this.pathFromRoot = parent.getPathFromRoot().addLast(field.getId(), parent.getType().isArray()); + } + + public RootHandle getRoot() { + return root; + } + + public DSHandle getParent() { + return parent; + } + + public AbstractDataNode getParentNode() { + return parent; + } + + @Override + public Path getPathFromRoot() { + return pathFromRoot; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/ExternalDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/ExternalDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/ExternalDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,195 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.griphyn.vdl.mapping.nodes; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import k.thr.LWThread; + +import org.apache.log4j.Logger; +import org.griphyn.vdl.karajan.Loader; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; +import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.type.impl.FieldImpl; + +public class ExternalDataNode extends AbstractFutureNonCompositeDataNode implements RootHandle { + + static final String DATASET_URI_PREFIX = "dataset:external:"; + + public static final Logger logger = Logger.getLogger(ExternalDataNode.class); + + private static long datasetIDCounter = 850000000000l; + + private static final String datasetIDPartialID = Loader.getUUID(); + + // previously in mapper params + private int line = -1; + private LWThread thread; + private boolean input; + + + public ExternalDataNode(String name) { + super(new FieldImpl(name, Types.EXTERNAL)); + } + + public ExternalDataNode(Field field) { + super(field); + } + + public int getLine() { + return line; + } + + public void setLine(int line) { + this.line = line; + } + + public boolean isInput() { + return input; + } + + public void setInput(boolean input) { + this.input = input; + } + + public void setThread(LWThread thread) { + this.thread = thread; + } + + public LWThread getThread() { + return thread; + } + + public String getName() { + return (String) getField().getId(); + } + + @Override + public void setName(String name) { + getField().setId(name); + } + + @Override + public void init(Mapper mapper) { + } + + @Override + public void mapperInitialized(Mapper mapper) { + } + + public boolean isRestartable() { + return true; + } + + public RootHandle getRoot() { + return this; + } + + public DSHandle getField(Path path) throws InvalidPathException { + if (path.isEmpty()) { + return this; + } + else { + throw new InvalidPathException(path, this); + } + } + + protected void getFields(List fields, Path path) throws InvalidPathException { + // nothing + } + + public void set(DSHandle handle) { + throw new UnsupportedOperationException(this.getDisplayableName() + " is an external dataset and cannot be set"); + } + + public Map, DSHandle> getArrayValue() { + throw new UnsupportedOperationException("cannot get value of external dataset"); + } + + public boolean isArray() { + return false; + } + + public Collection getFringePaths() throws HandleOpenException { + return Collections.singletonList(Path.EMPTY_PATH); + } + + public Path getPathFromRoot() { + return Path.EMPTY_PATH; + } + + public Mapper getMapper() { + return null; + } + + protected String makeIdentifierURIString() { + datasetIDCounter++; + return DATASET_URI_PREFIX + datasetIDPartialID + ":" + datasetIDCounter; + } + + public DSHandle createDSHandle(String fieldName) { + throw new UnsupportedOperationException("cannot create new field in external dataset"); + } + + public DSHandle getParent() { + return null; + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public synchronized void closeDeep() { + if (!this.isClosed()) { + /* + * Need to override this and set a value since + * this is skipped by the normal stageout mechanism which + * does that + */ + this.setValue(FILE_VALUE); + } + } + + @Override + public Mapper getActualMapper() { + return null; + } + + @Override + public void closeArraySizes() { + } + + @Override + protected void getFringePaths(List list, Path myPath) throws HandleOpenException { + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/FutureArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/FutureArrayDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/FutureArrayDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,53 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class FutureArrayDataNode extends AbstractFutureArrayDataNode { + private RootHandle root; + private AbstractDataNode parent; + private Path pathFromRoot; + + protected FutureArrayDataNode(Field field, RootHandle root, AbstractDataNode parent) { + super(field); + this.root = root; + this.parent = parent; + this.pathFromRoot = calculatePathFromRoot(); + } + + public RootHandle getRoot() { + return root; + } + + public DSHandle getParent() { + return parent; + } + + public AbstractDataNode getParentNode() { + return parent; + } + + @Override + public Path getPathFromRoot() { + return pathFromRoot; + } + + @Override + protected void clean0() { + super.clean0(); + root = null; + parent = null; + pathFromRoot = null; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/FutureMappedSingleDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/FutureMappedSingleDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/FutureMappedSingleDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,66 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class FutureMappedSingleDataNode extends AbstractFutureNonCompositeDataNode { + private RootHandle root; + private AbstractDataNode parent; + private Path pathFromRoot; + + protected FutureMappedSingleDataNode(Field field, RootHandle root, AbstractDataNode parent) { + super(field); + this.root = root; + this.parent = parent; + this.pathFromRoot = calculatePathFromRoot(); + } + + public RootHandle getRoot() { + return root; + } + + public DSHandle getParent() { + return parent; + } + + public AbstractDataNode getParentNode() { + return parent; + } + + @Override + public Path getPathFromRoot() { + return pathFromRoot; + } + + @Override + protected void getFringePaths(List list, Path myPath) throws HandleOpenException { + list.add(myPath); + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + list.add(this); + } + + @Override + protected void clean0() { + super.clean0(); + root = null; + parent = null; + pathFromRoot = null; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/FuturePrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/FuturePrimitiveDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/FuturePrimitiveDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,64 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class FuturePrimitiveDataNode extends AbstractFutureNonCompositeDataNode { + private RootHandle root; + private AbstractDataNode parent; + private Path pathFromRoot; + + protected FuturePrimitiveDataNode(Field field, RootHandle root, AbstractDataNode parent) { + super(field); + this.root = root; + this.parent = parent; + this.pathFromRoot = calculatePathFromRoot(); + } + + public RootHandle getRoot() { + return root; + } + + public DSHandle getParent() { + return parent; + } + + public AbstractDataNode getParentNode() { + return parent; + } + + @Override + public Path getPathFromRoot() { + return pathFromRoot; + } + + @Override + protected void getFringePaths(List list, Path myPath) throws HandleOpenException { + } + + @Override + protected void getLeaves(List list) throws HandleOpenException { + } + + @Override + protected void clean0() { + super.clean0(); + root = null; + parent = null; + pathFromRoot = null; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/FutureStructDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/FutureStructDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/FutureStructDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,53 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class FutureStructDataNode extends AbstractFutureStructDataNode { + private RootHandle root; + private AbstractDataNode parent; + private Path pathFromRoot; + + protected FutureStructDataNode(Field field, RootHandle root, AbstractDataNode parent) { + super(field, root); + this.root = root; + this.parent = parent; + this.pathFromRoot = calculatePathFromRoot(); + } + + public RootHandle getRoot() { + return root; + } + + public DSHandle getParent() { + return parent; + } + + public AbstractDataNode getParentNode() { + return parent; + } + + @Override + public Path getPathFromRoot() { + return pathFromRoot; + } + + @Override + protected void clean0() { + super.clean0(); + root = null; + parent = null; + pathFromRoot = null; + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,292 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 30, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.Collection; +import java.util.Map; +import java.util.Set; + +import k.rt.Future; +import k.rt.FutureListener; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.futures.FutureNotYetAvailable; +import org.griphyn.vdl.karajan.lib.Tracer; +import org.griphyn.vdl.mapping.AbsFile; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.DuplicateMappingChecker; +import org.griphyn.vdl.mapping.FileSystemLister; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.MappingDependentException; +import org.griphyn.vdl.mapping.NotCompositeException; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Type; + +public class InitMapper implements Mapper, FutureListener { + public static final Logger logger = Logger.getLogger(InitMapper.class); + + protected static final Tracer variableTracer = Tracer.getTracer("VARIABLE"); + + private Mapper mapper; + private boolean input; + private AbstractDataNode waitingMapperParam; + private RootHandle node; + private DuplicateMappingChecker dmc; + + public InitMapper(DuplicateMappingChecker dmc) { + this.dmc = dmc; + } + + @Override + public PhysicalFormat map(Path path) { + return null; + } + + @Override + public boolean exists(Path path) { + return false; + } + + @Override + public Collection existing() { + return null; + } + + @Override + public Collection existing(FileSystemLister l) { + return null; + } + + @Override + public boolean isStatic() { + return false; + } + + @Override + public boolean canBeRemapped(Path path) { + return false; + } + + @Override + public void remap(Path path, Mapper sourceMapper, Path sourcePath) { + } + + @Override + public void clean(Path paths) { + } + + @Override + public boolean isPersistent(Path path) { + return false; + } + + @Override + public Set getSupportedParamNames() { + return null; + } + + @Override + public void setParameters(Map params) { + } + + @Override + public void initialize(RootHandle node) { + synchronized(node) { + this.node = node; + waitingMapperParam = mapper.getFirstOpenParameter(); + if (waitingMapperParam != null) { + waitingMapperParam.addListener(this); + if (variableTracer.isEnabled()) { + variableTracer.trace(node.getThread(), node.getLine(), node.getName() + " WAIT " + + Tracer.getVarName(waitingMapperParam)); + } + return; + } + + // initialized means that this data has its mapper initialized + // this should be called before checkInputs because the latter + // may trigger calls to things that try to access this data node's + // mapper + mapper.initialize(node); + node.mapperInitialized(mapper); + try { + checkInputs(node, dmc); + } + catch (DependentException e) { + node.setValue(new MappingDependentException(node, e)); + } + } + } + + @Override + public void futureUpdated(Future fv) { + initialize(node); + } + + @Override + public void setBaseDir(String baseDir) { + mapper.setBaseDir(baseDir); + } + + @Override + public AbstractDataNode getFirstOpenParameter() { + return null; + } + + @Override + public Collection getPattern(Path path, Type type) { + return null; + } + + public boolean isInput() { + return input; + } + + public void setInput(boolean input) { + this.input = input; + } + + public Mapper getMapper() { + synchronized(node) { + if (waitingMapperParam != null) { + throw new FutureNotYetAvailable(waitingMapperParam); + } + else { + return mapper; + } + } + } + + public void setMapper(Mapper mapper) { + this.mapper = mapper; + } + + public void checkInputs(RootHandle root, DuplicateMappingChecker dmc) { + Mapper mapper = root.getActualMapper(); + if (input) { + addExisting(mapper.existing(), mapper, root, root); + checkConsistency(root, root, dmc); + } + else if (mapper.isStatic()) { + if (root.isClosed()) { + // this means that code that would have used this variable is already done + // which can happen in cases such as if(false) {a = ...} + return; + } + if (!root.getType().isComposite()) { + return; + } + // Static mappers are (array) mappers which know the size of + // an array statically. A good example is the fixed array mapper + if (logger.isDebugEnabled()) { + logger.debug("mapper: " + mapper); + } + for (Path p : mapper.existing()) { + try { + // Try to get the path in order to check that the + // path is valid - we'll get an exception if not + DSHandle h = root.getField(p); + if (variableTracer.isEnabled()) { + variableTracer.trace(root.getThread(), root.getLine(), + root.getName() + " MAPPING " + p + ", " + mapper.map(p)); + } + } + catch (InvalidPathException e) { + throw new IllegalStateException + ("mapper.existing() returned a path " + + " that it cannot subsequently map: " + + " root: " + root + " path: " + p); + } + } + if (root.isArray()) { + root.closeArraySizes(); + } + checkConsistency(root, root, dmc); + } + } + + public static void addExisting(Collection existing, Mapper mapper, RootHandle root, DSHandle var) { + boolean any = false; + for (Path p : existing) { + try { + DSHandle field = var.getField(p); + field.setValue(AbstractDataNode.FILE_VALUE); + if (variableTracer.isEnabled()) { + variableTracer.trace(root.getThread(), root.getLine(), + root.getName() + " MAPPING " + p + ", " + mapper.map(p)); + } + any = true; + } + catch (InvalidPathException e) { + throw new IllegalStateException("Structure of mapped data is " + + "incompatible with the mapped variable type: " + e.getMessage()); + } + catch (NotCompositeException e) { + throw new IllegalStateException("Cannot map multiple files to variable '" + + e.getDataNode().getFullName() + "' of non composite type '" + + e.getDataNode().getType() + "'"); + } + } + var.closeDeep(); + if (!any && variableTracer.isEnabled()) { + variableTracer.trace(root.getThread(), root.getLine(), + root.getName() + " MAPPING no files found"); + } + } + + public void checkConsistency(RootHandle root, DSHandle handle, DuplicateMappingChecker dmc) { + if (handle.getType().isArray()) { + // any number of indices is ok + try { + for (DSHandle dh : handle.getAllFields()) { + checkConsistency(root, dh, dmc); + } + } + catch (HandleOpenException e) { + // TODO init() should throw some checked exception + throw new RuntimeException("Mapper consistency check failed for " + handle + + ". A HandleOpenException was thrown during consistency checking for "+e.getSource(), e); + } + catch (InvalidPathException e) { + e.printStackTrace(); + } + } + else { + // all fields must be present + Type type = handle.getType(); + if (!type.isPrimitive() && !type.isComposite()) { + // mapped. Feed the DMC. + PhysicalFormat f = root.getActualMapper().map(handle.getPathFromRoot()); + if (input) { + dmc.addRead(f, handle); + } + else { + dmc.addWrite(f, handle); + } + } + for (String fieldName : type.getFieldNames()) { + Path fieldPath = Path.parse(fieldName); + try { + checkConsistency(root, handle.getField(fieldPath), dmc); + } + catch (InvalidPathException e) { + throw new RuntimeException("Data set initialization failed for " + handle + + ". Missing required field: " + fieldName); + } + } + } + } + +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/NodeFactory.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/NodeFactory.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/NodeFactory.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,99 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.DuplicateMappingChecker; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; +import org.griphyn.vdl.type.Type; + + +public class NodeFactory { + public static AbstractDataNode newNode(Field f, RootHandle root, AbstractDataNode parent) { + AbstractDataNode n; + Type t = f.getType(); + if (t.isPrimitive()) { + n = new FuturePrimitiveDataNode(f, root, parent); + } + else if (!t.isComposite()) { + n = new FutureMappedSingleDataNode(f, root, parent); + } + else if (t.isArray()) { + n = new FutureArrayDataNode(f, root, parent); + } + else { + n = new FutureStructDataNode(f, root, parent); + } + n.initialize(); + return n; + } + + public static AbstractDataNode newNode(Field f, RootHandle root, AbstractDataNode parent, Object value) { + AbstractDataNode n; + Type t = f.getType(); + if (t.isPrimitive()) { + return new ClosedPrimitiveDataNode(f, root, parent, value); + } + else { + throw new IllegalArgumentException(); + } + } + + public static AbstractDataNode newRoot(Field f, Object value) { + Type t = f.getType(); + if (t.isPrimitive()) { + return new RootClosedPrimitiveDataNode(f, value); + } + else if (t.isArray() && t.itemType().isPrimitive()){ + return new RootClosedArrayDataNode(f, (List) value, null); + } + else { + throw new IllegalArgumentException(); + } + } + + public static AbstractDataNode newRoot(Field f, DependentException e) { + Type t = f.getType(); + if (t.isPrimitive()) { + return new RootClosedPrimitiveDataNode(f, e); + } + else { + throw new IllegalArgumentException(); + } + } + + public static RootHandle newOpenRoot(Field f, DuplicateMappingChecker dmc) { + Type t = f.getType(); + if (t.isPrimitive()) { + return new RootFuturePrimitiveDataNode(f); + } + else if (t.isArray()) { + return new RootFutureArrayDataNode(f, dmc); + } + else if (t.isComposite()) { + return new RootFutureStructDataNode(f, dmc); + } + else { + return new RootFutureMappedSingleDataNode(f, dmc); + } + } + + public static AbstractDataNode newRoot(Type type, Object value) { + if (type.isPrimitive()) { + return new RootClosedPrimitiveDataNode(Field.Factory.getImmutableField("?", type), value); + } + else { + throw new IllegalArgumentException(); + } + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedArrayDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedArrayDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,127 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import java.util.List; + +import k.thr.LWThread; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DuplicateMappingChecker; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class RootClosedArrayDataNode extends AbstractClosedArrayDataNode implements RootHandle { + private int line = -1; + private LWThread thread; + private Mapper mapper; + + public RootClosedArrayDataNode(Field field, List values, DuplicateMappingChecker dmChecker) { + super(field, values); + if (getType().itemType().hasMappedComponents()) { + this.mapper = new InitMapper(dmChecker); + } + } + + @Override + public RootHandle getRoot() { + return this; + } + + @Override + public DSHandle getParent() { + return null; + } + + @Override + public Path getPathFromRoot() { + return Path.EMPTY_PATH; + } + + @Override + public void init(Mapper mapper) { + if (!getType().itemType().hasMappedComponents()) { + return; + } + if (mapper == null) { + initialized(); + } + else { + this.getInitMapper().setMapper(mapper); + this.mapper.initialize(this); + } + } + + @Override + public void mapperInitialized(Mapper mapper) { + synchronized(this) { + this.mapper = mapper; + } + initialized(); + } + + protected void initialized() { + if (variableTracer.isEnabled()) { + variableTracer.trace(thread, line, getName() + " INITIALIZED " + mapper); + } + } + + public synchronized Mapper getMapper() { + if (mapper instanceof InitMapper) { + return ((InitMapper) mapper).getMapper(); + } + else { + return mapper; + } + } + + + @Override + public int getLine() { + return line; + } + + @Override + public void setLine(int line) { + this.line = line; + } + + @Override + public void setThread(LWThread thread) { + this.thread = thread; + } + + @Override + public LWThread getThread() { + return thread; + } + + @Override + public String getName() { + return (String) getField().getId(); + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public Mapper getActualMapper() { + return mapper; + } + + @Override + public void closeArraySizes() { + // already closed + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedMappedSingleDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedMappedSingleDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedMappedSingleDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,119 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import k.thr.LWThread; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DuplicateMappingChecker; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class RootClosedMappedSingleDataNode extends AbstractClosedMappedSingleDataNode implements RootHandle { + private int line = -1; + private LWThread thread; + private Mapper mapper; + + public RootClosedMappedSingleDataNode(Field field, Object value, DuplicateMappingChecker dmc) { + super(field, value); + this.mapper = new InitMapper(dmc); + } + + @Override + public RootHandle getRoot() { + return this; + } + + @Override + public DSHandle getParent() { + return null; + } + + @Override + public Path getPathFromRoot() { + return Path.EMPTY_PATH; + } + + @Override + public void init(Mapper mapper) { + if (mapper == null) { + initialized(); + } + else { + this.getInitMapper().setMapper(mapper); + this.mapper.initialize(this); + } + } + + @Override + public void mapperInitialized(Mapper mapper) { + synchronized(this) { + this.mapper = mapper; + } + initialized(); + } + + protected void initialized() { + if (variableTracer.isEnabled()) { + variableTracer.trace(thread, line, getName() + " INITIALIZED " + mapper); + } + } + + public synchronized Mapper getMapper() { + if (mapper instanceof InitMapper) { + return ((InitMapper) mapper).getMapper(); + } + else { + return mapper; + } + } + + @Override + public int getLine() { + return line; + } + + @Override + public void setLine(int line) { + this.line = line; + } + + @Override + public void setThread(LWThread thread) { + this.thread = thread; + } + + @Override + public LWThread getThread() { + return thread; + } + + @Override + public String getName() { + return (String) getField().getId(); + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public Mapper getActualMapper() { + return mapper; + } + + @Override + public void closeArraySizes() { + // does not apply + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedPrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedPrimitiveDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedPrimitiveDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,108 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import k.thr.LWThread; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class RootClosedPrimitiveDataNode extends AbstractClosedPrimitiveDataNode implements RootHandle { + private int line = -1; + private LWThread thread; + + public RootClosedPrimitiveDataNode(Field field, Object value) { + super(field, value); + } + + public RootClosedPrimitiveDataNode(Field field, DependentException e) { + super(field, e); + } + + @Override + public RootHandle getRoot() { + return this; + } + + @Override + public DSHandle getParent() { + return null; + } + + @Override + public Path getPathFromRoot() { + return Path.EMPTY_PATH; + } + + @Override + public void init(Mapper mapper) { + } + + @Override + public void mapperInitialized(Mapper mapper) { + } + + @Override + public int getLine() { + return line; + } + + @Override + public void setLine(int line) { + this.line = line; + } + + @Override + public boolean isInput() { + // maybe not strictly true, but the only place where this matters + // is when mapping input data, and there is no mapping for this + // node + return false; + } + + @Override + public void setInput(boolean input) { + // noop + } + + @Override + public void setThread(LWThread thread) { + this.thread = thread; + } + + @Override + public LWThread getThread() { + return thread; + } + + @Override + public String getName() { + return (String) getField().getId(); + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public Mapper getActualMapper() { + return null; + } + + @Override + public void closeArraySizes() { + // does not apply + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureArrayDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureArrayDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,123 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import k.thr.LWThread; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DuplicateMappingChecker; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class RootFutureArrayDataNode extends AbstractFutureArrayDataNode implements RootHandle { + private int line = -1; + private LWThread thread; + private Mapper mapper; + + public RootFutureArrayDataNode(Field field, DuplicateMappingChecker dmc) { + super(field); + this.mapper = new InitMapper(dmc); + } + + @Override + public RootHandle getRoot() { + return this; + } + + @Override + public DSHandle getParent() { + return null; + } + + @Override + public Path getPathFromRoot() { + return Path.EMPTY_PATH; + } + + @Override + public void init(Mapper mapper) { + if (!getType().itemType().hasMappedComponents()) { + return; + } + if (mapper == null) { + initialized(); + } + else { + this.getInitMapper().setMapper(mapper); + this.mapper.initialize(this); + } + } + + @Override + public void mapperInitialized(Mapper mapper) { + synchronized(this) { + this.mapper = mapper; + } + initialized(); + } + + protected void initialized() { + if (variableTracer.isEnabled()) { + variableTracer.trace(thread, line, getName() + " INITIALIZED " + mapper); + } + } + + public synchronized Mapper getMapper() { + if (mapper instanceof InitMapper) { + return ((InitMapper) mapper).getMapper(); + } + else { + return mapper; + } + } + + + @Override + public int getLine() { + return line; + } + + @Override + public void setLine(int line) { + this.line = line; + } + + @Override + public void setThread(LWThread thread) { + this.thread = thread; + } + + @Override + public LWThread getThread() { + return thread; + } + + @Override + public String getName() { + return (String) getField().getId(); + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public Mapper getActualMapper() { + return mapper; + } + + @Override + public void closeArraySizes() { + closeShallow(); + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureMappedSingleDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureMappedSingleDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureMappedSingleDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,126 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import k.thr.LWThread; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DuplicateMappingChecker; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class RootFutureMappedSingleDataNode extends AbstractFutureMappedSingleDataNode implements RootHandle { + private int line = -1; + + private LWThread thread; + + private Mapper mapper; + + public RootFutureMappedSingleDataNode(Field field, DuplicateMappingChecker dmc) { + super(field); + if (field == null) { + throw new NullPointerException(); + } + this.mapper = new InitMapper(dmc); + } + + @Override + public RootHandle getRoot() { + return this; + } + + @Override + public DSHandle getParent() { + return null; + } + + @Override + public Path getPathFromRoot() { + return Path.EMPTY_PATH; + } + + + @Override + public void init(Mapper mapper) { + if (mapper == null) { + initialized(); + } + else { + this.getInitMapper().setMapper(mapper); + this.mapper.initialize(this); + } + } + + @Override + public void mapperInitialized(Mapper mapper) { + synchronized(this) { + this.mapper = mapper; + } + initialized(); + } + + protected void initialized() { + if (variableTracer.isEnabled()) { + variableTracer.trace(thread, line, getName() + " INITIALIZED " + mapper); + } + } + + public synchronized Mapper getMapper() { + if (mapper instanceof InitMapper) { + return ((InitMapper) mapper).getMapper(); + } + else { + return mapper; + } + } + + + @Override + public int getLine() { + return line; + } + + @Override + public void setLine(int line) { + this.line = line; + } + + @Override + public void setThread(LWThread thread) { + this.thread = thread; + } + + @Override + public LWThread getThread() { + return thread; + } + + @Override + public String getName() { + return (String) getField().getId(); + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public Mapper getActualMapper() { + return mapper; + } + + @Override + public void closeArraySizes() { + // does not apply + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,100 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import k.thr.LWThread; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class RootFuturePrimitiveDataNode extends AbstractFuturePrimitiveDataNode implements RootHandle { + private int line = -1; + private LWThread thread; + + public RootFuturePrimitiveDataNode(Field field) { + super(field); + } + + @Override + public RootHandle getRoot() { + return this; + } + + @Override + public DSHandle getParent() { + return null; + } + + @Override + public Path getPathFromRoot() { + return Path.EMPTY_PATH; + } + + @Override + public void init(Mapper mapper) { + } + + @Override + public void mapperInitialized(Mapper mapper) { + } + + @Override + public int getLine() { + return line; + } + + @Override + public void setLine(int line) { + this.line = line; + } + + @Override + public boolean isInput() { + return false; + } + + @Override + public void setInput(boolean input) { + // doesn't matter + } + + @Override + public void setThread(LWThread thread) { + this.thread = thread; + } + + @Override + public LWThread getThread() { + return thread; + } + + @Override + public String getName() { + return (String) getField().getId(); + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public Mapper getActualMapper() { + return null; + } + + @Override + public void closeArraySizes() { + // does not apply + } +} Added: trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureStructDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureStructDataNode.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureStructDataNode.java 2014-05-09 04:35:05 UTC (rev 7839) @@ -0,0 +1,137 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 28, 2014 + */ +package org.griphyn.vdl.mapping.nodes; + +import k.thr.LWThread; + +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DuplicateMappingChecker; +import org.griphyn.vdl.mapping.Mapper; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.type.Field; + +public class RootFutureStructDataNode extends AbstractFutureStructDataNode implements RootHandle { + private int line = -1; + private LWThread thread; + private boolean input; + private Mapper mapper; + + public RootFutureStructDataNode(Field field, DuplicateMappingChecker dmc) { + super(field, null); + if (getType().itemType().hasMappedComponents()) { + this.mapper = new InitMapper(dmc); + } + } + + @Override + public RootHandle getRoot() { + return this; + } + + @Override + public DSHandle getParent() { + return null; + } + + @Override + public Path getPathFromRoot() { + return Path.EMPTY_PATH; + } + + @Override + public void init(Mapper mapper) { + initialize(); + if (!getType().itemType().hasMappedComponents()) { + return; + } + if (mapper == null) { + initialized(); + } + else { + this.getInitMapper().setMapper(mapper); + this.mapper.initialize(this); + } + } + + @Override + public void mapperInitialized(Mapper mapper) { + synchronized(this) { + this.mapper = mapper; + } + initialized(); + } + + protected void initialized() { + if (variableTracer.isEnabled()) { + variableTracer.trace(thread, line, getName() + " INITIALIZED " + mapper); + } + } + + public synchronized Mapper getMapper() { + if (mapper instanceof InitMapper) { + return ((InitMapper) mapper).getMapper(); + } + else { + return mapper; + } + } + + + @Override + public int getLine() { + return line; + } + + @Override + public void setLine(int line) { + this.line = line; + } + + @Override + public boolean isInput() { + return input; + } + + @Override + public void setInput(boolean input) { + this.input = input; + } + + @Override + public void setThread(LWThread thread) { + this.thread = thread; + } + + @Override + public LWThread getThread() { + return thread; + } + + @Override + public String getName() { + return (String) getField().getId(); + } + + @Override + protected AbstractDataNode getParentNode() { + return null; + } + + @Override + public Mapper getActualMapper() { + return mapper; + } + + @Override + public void closeArraySizes() { + // does not apply + } +} From hategan at ci.uchicago.edu Thu May 8 23:56:39 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 8 May 2014 23:56:39 -0500 (CDT) Subject: [Swift-commit] r7843 - in trunk/src/org/griphyn/vdl/karajan/lib: . swiftscript Message-ID: <20140509045639.C74F29D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-08 23:56:39 -0500 (Thu, 08 May 2014) New Revision: 7843 Added: trunk/src/org/griphyn/vdl/karajan/lib/GetFieldConst.java trunk/src/org/griphyn/vdl/karajan/lib/StringCache.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/AbstractSingleValuedSwiftFunction.java Modified: trunk/src/org/griphyn/vdl/karajan/lib/AppendArray.java trunk/src/org/griphyn/vdl/karajan/lib/CleanDataset.java trunk/src/org/griphyn/vdl/karajan/lib/CloseDataset.java trunk/src/org/griphyn/vdl/karajan/lib/CreateArray.java trunk/src/org/griphyn/vdl/karajan/lib/Execute.java trunk/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java trunk/src/org/griphyn/vdl/karajan/lib/FileName.java trunk/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java trunk/src/org/griphyn/vdl/karajan/lib/GetFieldSubscript.java trunk/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java trunk/src/org/griphyn/vdl/karajan/lib/GetStagingInfo.java trunk/src/org/griphyn/vdl/karajan/lib/IsFileBound.java trunk/src/org/griphyn/vdl/karajan/lib/Mark.java trunk/src/org/griphyn/vdl/karajan/lib/New.java trunk/src/org/griphyn/vdl/karajan/lib/Operators.java trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java trunk/src/org/griphyn/vdl/karajan/lib/Range.java trunk/src/org/griphyn/vdl/karajan/lib/RemoteFileNames.java trunk/src/org/griphyn/vdl/karajan/lib/SetDatasetValues.java trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java trunk/src/org/griphyn/vdl/karajan/lib/SliceArray.java trunk/src/org/griphyn/vdl/karajan/lib/Stagein.java trunk/src/org/griphyn/vdl/karajan/lib/Stageout.java trunk/src/org/griphyn/vdl/karajan/lib/SwiftBinaryOp.java trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java trunk/src/org/griphyn/vdl/karajan/lib/SwiftUnaryOp.java trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java trunk/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java trunk/src/org/griphyn/vdl/karajan/lib/WaitFieldValue.java trunk/src/org/griphyn/vdl/karajan/lib/While.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java Log: interned fields; data type reorganization; propagation of lazy failures in internal functions Modified: trunk/src/org/griphyn/vdl/karajan/lib/AppendArray.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/AppendArray.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/AppendArray.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -21,8 +21,8 @@ import k.rt.Stack; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class AppendArray extends SetFieldValue { Modified: trunk/src/org/griphyn/vdl/karajan/lib/CleanDataset.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/CleanDataset.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/CleanDataset.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -24,8 +24,8 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.file.FileGarbageCollector; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class CleanDataset extends SwiftFunction { private ArgRef var; Modified: trunk/src/org/griphyn/vdl/karajan/lib/CloseDataset.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/CloseDataset.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/CloseDataset.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -26,10 +26,10 @@ import org.apache.log4j.Logger; import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class CloseDataset extends SwiftFunction { public static final Logger logger = Logger.getLogger(CloseDataset.class); Modified: trunk/src/org/griphyn/vdl/karajan/lib/CreateArray.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/CreateArray.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/CreateArray.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -27,10 +27,10 @@ import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.futures.FutureFault; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.RootArrayDataNode; import org.griphyn.vdl.mapping.RootHandle; import org.griphyn.vdl.mapping.file.ConcurrentMapper; +import org.griphyn.vdl.mapping.nodes.RootClosedArrayDataNode; +import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; public class CreateArray extends SetFieldValue { @@ -53,43 +53,18 @@ Type type = checkTypes((List) value); - RootHandle handle = new RootArrayDataNode("arrayexpr", type.arrayType()); - if (type.hasNonPrimitiveComponents()) { + RootHandle handle = new RootClosedArrayDataNode(Field.Factory.getImmutableField("arrayexpr", type.arrayType()), + (List) value, null); + if (type.hasMappedComponents()) { handle.init(new ConcurrentMapper()); } else { handle.init(null); } - - if (logger.isInfoEnabled()) { - logger.info("CREATEARRAY START array=" + handle.getIdentifier()); - } - - int index = 0; - for (Object o : (List) value) { - // TODO check type consistency of elements with - // the type of the array - DSHandle n = (DSHandle) o; - // we know this DSHandle cast will work because we checked - // it in the previous scan of the array contents - Path p = Path.EMPTY_PATH.addLast(index, true); - - DSHandle dst = handle.getField(p); - - deepCopy(dst, n, stack); - - if (logger.isInfoEnabled()) { - logger.info("CREATEARRAY MEMBER array=" + handle.getIdentifier() - + " index=" + index + " member=" + n.getIdentifier()); - } - index++; - } - handle.closeShallow(); - if (logger.isInfoEnabled()) { - logger.info("CREATEARRAY COMPLETED array=" + handle.getIdentifier()); - } + logger.info("CREATEARRAY array=" + handle.getIdentifier()); + } return handle; } Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -20,6 +20,12 @@ */ package org.griphyn.vdl.karajan.lib; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + import k.rt.Abort; import k.rt.Channel; import k.rt.ConditionalYield; @@ -32,6 +38,7 @@ import org.globus.cog.abstraction.interfaces.Status; import org.globus.cog.abstraction.interfaces.Task; import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.CompilationException; import org.globus.cog.karajan.analyzer.Param; import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.Signature; @@ -59,14 +66,35 @@ @Override protected Signature getSignature() { Signature sig = super.getSignature(); - sig.getParams().add(0, new Param("progress", Param.Type.POSITIONAL)); - sig.getParams().add(optional("replicationGroup", null)); - sig.getParams().add(optional("replicationChannel", null)); - sig.getParams().add(optional("jobid", null)); + List params = sig.getParams(); + params.add(0, new Param("progress", Param.Type.POSITIONAL)); + params.add(optional("replicationGroup", null)); + params.add(optional("replicationChannel", null)); + params.add(optional("jobid", null)); + removeParams(params, "stdout", "stderr", "stdoutLocation", "stderrLocation", + "stdin", "provider", "securityContext", "nativespec", + "delegation", "batch"); return sig; } - @Override + private void removeParams(List params, String... names) { + Set snames = new HashSet(Arrays.asList(names)); + Iterator i = params.iterator(); + while (i.hasNext()) { + Param p = i.next(); + if (snames.contains(p.name)) { + try { + setArg(null, p, new ArgRef.Static(p.value)); + } + catch (CompilationException e) { + throw new RuntimeException("Failed to remove parameter " + p.name, e); + } + i.remove(); + } + } + } + + @Override protected void addLocals(Scope scope) { super.addLocals(scope); context = scope.getVarRef("#context"); Modified: trunk/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -29,9 +29,10 @@ import org.apache.log4j.Logger; import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.PathElementComparator; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.ArrayHandle; public class ExpandArguments extends SwiftFunction { public static final Logger logger = Logger.getLogger(ExpandArguments.class); @@ -58,15 +59,17 @@ private Object expandArrays(Channel items) { ArrayList l = new ArrayList(); for (Object item : items) { - if (item instanceof ArrayDataNode) { - ArrayDataNode array = (ArrayDataNode) item; + AbstractDataNode h = (AbstractDataNode) item; + h.waitFor(this); + if (item instanceof ArrayHandle) { + ArrayHandle array = (ArrayHandle) item; Map, DSHandle> m = array.getArrayValue(); SortedMap, DSHandle> sorted = new TreeMap, DSHandle>(new PathElementComparator()); sorted.putAll(m); l.addAll(m.values()); } else { - l.add((DSHandle) item); + l.add(h); } // TODO this does not correctly handle structs or // externals - at the moment, probably neither of @@ -85,7 +88,7 @@ throw new ExecutionException(this, "Cannot handle argument implemented by " + item.getClass()); } - if (item instanceof ArrayDataNode) { + if (item instanceof ArrayHandle) { arraySeen = true; } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/FileName.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/FileName.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/FileName.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -38,6 +38,7 @@ @Override public Object function(Stack stack) { - return argList(filename(var.getValue(stack)), true); + DSHandle var = this.var.getValue(stack); + return var.map(var.getPathFromRoot()); } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -24,10 +24,10 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.PairSet; -import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.nodes.ArrayHandle; public class GetArrayIterator extends SwiftFunction { public static final Logger logger = Logger.getLogger(GetArrayIterator.class); @@ -67,7 +67,7 @@ if (logger.isDebugEnabled()) { logger.debug("Using future iterator for " + var); } - return ((ArrayDataNode) var).entryList(); + return ((ArrayHandle) var).entryList(); } } } Added: trunk/src/org/griphyn/vdl/karajan/lib/GetFieldConst.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/GetFieldConst.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/GetFieldConst.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -0,0 +1,55 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* + * Created on Dec 26, 2006 + */ +package org.griphyn.vdl.karajan.lib; + +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.Param; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; +import org.griphyn.vdl.type.Field; +import org.griphyn.vdl.type.NoSuchTypeException; +import org.griphyn.vdl.type.Types; + +public class GetFieldConst extends AbstractSingleValuedFunction { + private ArgRef name; + private ArgRef type; + + + @Override + protected Param[] getParams() { + return params("name", "type"); + } + + + @Override + public Object function(Stack stack) { + String name = this.name.getValue(stack); + String type = this.type.getValue(stack); + try { + return Field.Factory.getImmutableField(name, Types.getType(type)); + } + catch (NoSuchTypeException e) { + throw new ExecutionException("No such type: " + name, e); + } + } +} Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetFieldSubscript.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/GetFieldSubscript.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/GetFieldSubscript.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -22,12 +22,12 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.OOBYield; import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class GetFieldSubscript extends SwiftFunction { private ArgRef var; @@ -48,7 +48,7 @@ indexh.waitFor(); Object index = indexh.getValue(); if ("*".equals(index)) { - return var.getFields(Path.CHILDREN); + return var.getAllFields(); } else { return var.getField(Path.EMPTY_PATH.addFirst((Comparable) index, true)); Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -27,8 +27,8 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.futures.FutureFault; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class GetFieldValue extends SwiftFunction { public static final Logger logger = Logger.getLogger(GetFieldValue.class); @@ -52,7 +52,8 @@ try { Path path = parsePath(this.path.getValue(stack)); if (path.hasWildcards()) { - return var.getFields(path).toArray(); + // TODO we should clarify whether we allow generic selectors + return var.getField(path.butLast()).getAllFields().toArray(); } else { var = (AbstractDataNode) var.getField(path); Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetStagingInfo.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/GetStagingInfo.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/GetStagingInfo.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -85,6 +85,9 @@ private void addPaths(Info info, Collection vars, boolean out) throws HandleOpenException { for (DSHandle var : vars) { + if (!var.getType().hasMappedComponents()) { + continue; + } Mapper m = getMapper(var); if (out && !m.isStatic() && var.getType().hasArrayComponents()) { Collection patterns = m.getPattern(var.getPathFromRoot(), var.getType()); Modified: trunk/src/org/griphyn/vdl/karajan/lib/IsFileBound.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/IsFileBound.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/IsFileBound.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -24,8 +24,8 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class IsFileBound extends SwiftFunction { private ArgRef var; Modified: trunk/src/org/griphyn/vdl/karajan/lib/Mark.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Mark.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/Mark.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -28,10 +28,10 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DataDependentException; import org.griphyn.vdl.mapping.MappingDependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class Mark extends SwiftFunction { private ArgRef> restarts; Modified: trunk/src/org/griphyn/vdl/karajan/lib/New.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/New.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/New.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -20,7 +20,7 @@ */ package org.griphyn.vdl.karajan.lib; -import java.util.Iterator; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -37,17 +37,23 @@ import org.globus.cog.karajan.analyzer.VarRef; import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.parser.WrapperNode; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DuplicateMappingChecker; -import org.griphyn.vdl.mapping.ExternalDataNode; import org.griphyn.vdl.mapping.InvalidMapperException; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.MapperFactory; import org.griphyn.vdl.mapping.NullMapper; -import org.griphyn.vdl.mapping.RootArrayDataNode; -import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.ExternalDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.mapping.nodes.RootClosedArrayDataNode; +import org.griphyn.vdl.mapping.nodes.RootClosedPrimitiveDataNode; +import org.griphyn.vdl.mapping.nodes.RootFutureArrayDataNode; +import org.griphyn.vdl.mapping.nodes.RootFutureMappedSingleDataNode; +import org.griphyn.vdl.mapping.nodes.RootFuturePrimitiveDataNode; +import org.griphyn.vdl.mapping.nodes.RootFutureStructDataNode; +import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.NoSuchTypeException; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; @@ -57,10 +63,9 @@ private static final Mapper NULL_MAPPER = new NullMapper(); - private ArgRef type; + private ArgRef field; private ArgRef> mapping; private ArgRef value; - private ArgRef dbgname; private ArgRef waitCount; private ArgRef input; private ArgRef _defline; @@ -70,8 +75,8 @@ @Override protected Signature getSignature() { - return new Signature(params("type", optional("mapping", null), optional("value", null), - optional("dbgname", null), optional("waitCount", null), optional("input", Boolean.FALSE), optional("_defline", null))); + return new Signature(params("field", optional("mapping", null), optional("value", null), + optional("waitCount", null), optional("input", Boolean.FALSE), optional("_defline", null))); } private Tracer tracer; @@ -96,22 +101,22 @@ @Override public Object function(Stack stack) { - String typename = this.type.getValue(stack); + Field field = this.field.getValue(stack); Object value = this.value.getValue(stack); Map mapping = this.mapping.getValue(stack); - String dbgname = this.dbgname.getValue(stack); Number waitCount = this.waitCount.getValue(stack); boolean input = this.input.getValue(stack); Integer line = this._defline.getValue(stack); - if (typename == null && value == null) { - throw new ExecutionException("You must specify either a type or a value"); + if (field == null && value == null) { + throw new ExecutionException("You must specify either a field or a value"); } Mapper mapper; + Type type = field.getType(); + String dbgname = (String) field.getId(); - Type type = getType(typename); - if (type.hasNonPrimitiveComponents()) { + if (type.hasMappedComponents()) { String desc = (String) mapping.remove("swift#descriptor"); try { mapper = MapperFactory.getMapper(desc); @@ -143,32 +148,36 @@ try { DSHandle handle; - if (typename.equals("external")) { + if (Types.EXTERNAL.equals(type)) { if (tracer.isEnabled()) { tracer.trace(thread, dbgname + " = external"); } - handle = initHandle(new ExternalDataNode(dbgname), mapper, thread, line, input); + handle = initHandle(new ExternalDataNode(field), mapper, thread, line, input); } else if (type.isArray()) { // dealing with array variable if (value != null) { - if (value instanceof RootArrayDataNode) { + if (value instanceof RootHandle) { + handle = (RootHandle) value; if (tracer.isEnabled()) { - tracer.trace(thread, dbgname + " = " + Tracer.getVarName((RootDataNode) value)); + tracer.trace(thread, dbgname + " = " + Tracer.getVarName(handle)); } - handle = (RootArrayDataNode) value; + handle.closeShallow(); } else { - handle = initHandle(createArrayFromList(stack, thread, dbgname, type), + handle = initHandle(createArrayFromList(stack, thread, field, value), mapper, thread, line, input); } - handle.closeShallow(); } + else if (input && !type.hasMappedComponents()) { + handle = initHandle(createArrayFromList(stack, thread, field, Collections.emptyList()), + mapper, thread, line, input); + } else { if (tracer.isEnabled()) { tracer.trace(thread, dbgname); } - handle = initHandle(createEmptyArray(stack, thread, dbgname, type), + handle = initHandle(createEmptyArray(stack, thread, field), mapper, thread, line, input); } } @@ -180,19 +189,25 @@ handle = (DSHandle) value; } else { - handle = initHandle(new RootDataNode(dbgname, type, getDMChecker(stack)), - mapper, thread, line, input); - if (value != null) { - if (tracer.isEnabled()) { - tracer.trace(thread, dbgname + " = " + value); - } - handle.setValue(internalValue(type, value)); - } - else { - if (tracer.isEnabled()) { + if (value == null) { + handle = initHandle(NodeFactory.newOpenRoot(field, getDMChecker(stack)), + mapper, thread, line, input); + if (tracer.isEnabled()) { tracer.trace(thread, dbgname + " " + mapper); } - } + } + else { + if (type.isPrimitive()) { + handle = initHandle(new RootClosedPrimitiveDataNode(field, internalValue(type, value)), + mapper, thread, line, input); + } + else { + throw new ExecutionException("Cannot create non-primitive data node with value '" + value + "'"); + } + if (tracer.isEnabled()) { + tracer.trace(thread, dbgname + " = " + value); + } + } } if (AbstractDataNode.provenance && logger.isDebugEnabled()) { @@ -211,41 +226,27 @@ if (line != null) { handle.setLine(line); } - handle.setInput(input); + if (mapper != NULL_MAPPER) { + handle.setInput(input); + } handle.init(mapper); return handle; } - private RootHandle createArrayFromList(Stack stack, LWThread thread, String dbgname, Type type) + private RootHandle createArrayFromList(Stack stack, LWThread thread, Field field, Object value) throws NoSuchFieldException { - RootHandle handle = new RootArrayDataNode(dbgname, type, getDMChecker(stack)); if (!(value instanceof List)) { throw new ExecutionException("An array variable can only be initialized with a list of values"); } if (tracer.isEnabled()) { - tracer.trace(thread, dbgname + " = " + formatList((List) value)); + tracer.trace(thread, field.getId() + " = " + formatList((List) value)); } - int index = 0; - Iterator i = ((List) value).iterator(); - while (i.hasNext()) { - // TODO check type consistency of elements with - // the type of the array - Object n = i.next(); - if (n instanceof DSHandle) { - handle.getField(index).set((DSHandle) n); - } - else { - throw new RuntimeException( - "An array variable can only be initialized by a list of DSHandle values"); - } - index++; - } - return handle; + return new RootClosedArrayDataNode(field, (List) value, getDMChecker(stack)); } - private RootHandle createEmptyArray(Stack stack, LWThread thread, String dbgname, Type type) + private RootHandle createEmptyArray(Stack stack, LWThread thread, Field field) throws NoSuchFieldException { - return new RootArrayDataNode(dbgname, type, getDMChecker(stack)); + return new RootFutureArrayDataNode(field, getDMChecker(stack)); } private Type getType(String typename) { Modified: trunk/src/org/griphyn/vdl/karajan/lib/Operators.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Operators.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/Operators.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -21,9 +21,11 @@ import org.apache.log4j.Logger; import org.globus.cog.karajan.compiled.nodes.Node; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; import org.griphyn.vdl.util.VDL2Config; @@ -101,17 +103,22 @@ protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { Type t = type(v1, v2); DSHandle r; - if (t == Types.STRING) { - r = new RootDataNode(Types.STRING, (String.valueOf(v1.getValue()) + String.valueOf(v2.getValue()))); + try { + if (t == Types.STRING) { + r = NodeFactory.newRoot(Field.GENERIC_STRING, StringCache.get((String.valueOf(v1.getValue()) + String.valueOf(v2.getValue())))); + } + else if (t == Types.INT) { + r = NodeFactory.newRoot(Field.GENERIC_INT, getInt(this, v1) + getInt(this, v2)); + } + else { + r = NodeFactory.newRoot(Field.GENERIC_FLOAT, getFloat(this, v1) + getFloat(this, v2)); + } + logBinaryProvenance("sum", v1, v2, r); + return r; } - else if (t == Types.INT) { - r = new RootDataNode(Types.INT, getInt(this, v1) + getInt(this, v2)); + catch (DependentException e) { + return NodeFactory.newRoot(Field.Factory.getImmutableField("?", t), e); } - else { - r = new RootDataNode(Types.FLOAT, getFloat(this, v1) + getFloat(this, v2)); - } - logBinaryProvenance("sum", v1, v2, r); - return r; } } @@ -120,14 +127,19 @@ protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { Type t = type(v1, v2); DSHandle r; - if (t == Types.INT) { - r = new RootDataNode(Types.INT, getInt(this, v1) - getInt(this, v2)); + try { + if (t == Types.INT) { + r = NodeFactory.newRoot(Field.GENERIC_INT, getInt(this, v1) - getInt(this, v2)); + } + else { + r = NodeFactory.newRoot(Field.GENERIC_FLOAT, getFloat(this, v1) - getFloat(this, v2)); + } + logBinaryProvenance("difference", v1, v2, r); + return r; } - else { - r = new RootDataNode(Types.FLOAT, getFloat(this, v1) - getFloat(this, v2)); + catch (DependentException e) { + return NodeFactory.newRoot(Field.Factory.getImmutableField("?", t), e); } - logBinaryProvenance("difference", v1, v2, r); - return r; } } @@ -136,21 +148,26 @@ protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { Type t = type(v1, v2); DSHandle r; - if (t == Types.INT) { - r = new RootDataNode(Types.INT, getInt(this, v1) * getInt(this, v2)); + try { + if (t == Types.INT) { + r = NodeFactory.newRoot(Field.GENERIC_INT, getInt(this, v1) * getInt(this, v2)); + } + else { + r = NodeFactory.newRoot(Field.GENERIC_FLOAT, getFloat(this, v1) * getFloat(this, v2)); + } + logBinaryProvenance("product", v1, v2, r); + return r; } - else { - r = new RootDataNode(Types.FLOAT, getFloat(this, v1) * getFloat(this, v2)); + catch (DependentException e) { + return NodeFactory.newRoot(Field.Factory.getImmutableField("?", t), e); } - logBinaryProvenance("product", v1, v2, r); - return r; } } public static class FQuotient extends SwiftBinaryOp { @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.FLOAT, getFloat(this, v1) / getFloat(this, v2)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_FLOAT, getFloat(this, v1) / getFloat(this, v2)); logBinaryProvenance("fquotient", v1, v2, r); return r; } @@ -162,7 +179,7 @@ public static class IQuotient extends SwiftBinaryOp { @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.INT, getInt(this, v1) / getInt(this, v2)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_INT, getInt(this, v1) / getInt(this, v2)); logBinaryProvenance("iquotient", v1, v2, r); return r; } @@ -173,102 +190,157 @@ protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { Type t = type(v1, v2); DSHandle r; - if (t == Types.INT) { - r = new RootDataNode(Types.INT, getInt(this, v1) % getInt(this, v2)); + try { + if (t == Types.INT) { + r = NodeFactory.newRoot(Field.GENERIC_INT, getInt(this, v1) % getInt(this, v2)); + } + else { + r = NodeFactory.newRoot(Field.GENERIC_FLOAT, getFloat(this, v1) % getFloat(this, v2)); + } + logBinaryProvenance("remainder", v1, v2, r); + return r; } - else { - r = new RootDataNode(Types.FLOAT, getFloat(this, v1) % getFloat(this, v2)); + catch (DependentException e) { + return NodeFactory.newRoot(Field.Factory.getImmutableField("?", t), e); } - logBinaryProvenance("remainder", v1, v2, r); - return r; } } public static class LE extends SwiftBinaryOp { @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + + @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.BOOLEAN, getFloat(this, v1) <= getFloat(this, v2)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, getFloat(this, v1) <= getFloat(this, v2)); logBinaryProvenance("le", v1, v2, r); return r; } } public static class GE extends SwiftBinaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.BOOLEAN, getFloat(this, v1) >= getFloat(this, v2)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, getFloat(this, v1) >= getFloat(this, v2)); logBinaryProvenance("ge", v1, v2, r); return r; } } public static class LT extends SwiftBinaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.BOOLEAN, getFloat(this, v1) < getFloat(this, v2)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, getFloat(this, v1) < getFloat(this, v2)); logBinaryProvenance("lt", v1, v2, r); return r; } } public static class GT extends SwiftBinaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.BOOLEAN, getFloat(this, v1) > getFloat(this, v2)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, getFloat(this, v1) > getFloat(this, v2)); logBinaryProvenance("gt", v1, v2, r); return r; } } public static class EQ extends SwiftBinaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.BOOLEAN, v1.getValue().equals(v2.getValue())); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, v1.getValue().equals(v2.getValue())); logBinaryProvenance("eq", v1, v2, r); return r; } } public static class NE extends SwiftBinaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.BOOLEAN, !v1.getValue().equals(v2.getValue())); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, !v1.getValue().equals(v2.getValue())); logBinaryProvenance("ne", v1, v2, r); return r; } } public static class And extends SwiftBinaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.BOOLEAN, getBool(this, v1) && getBool(this, v2)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, getBool(this, v1) && getBool(this, v2)); logBinaryProvenance("and", v1, v2, r); return r; } } public static class Or extends SwiftBinaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + @Override protected DSHandle value(AbstractDataNode v1, AbstractDataNode v2) { - DSHandle r = new RootDataNode(Types.BOOLEAN, getBool(this, v1) || getBool(this, v2)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, getBool(this, v1) || getBool(this, v2)); logBinaryProvenance("or", v1, v2, r); return r; } } public static class Not extends SwiftUnaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_BOOLEAN; + } + @Override protected DSHandle value(AbstractDataNode v) { - DSHandle r = new RootDataNode(Types.BOOLEAN, !getBool(this, v)); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, !getBool(this, v)); logUnaryProvenance("not", v, r); return r; } } public static class Inc extends SwiftUnaryOp { + @Override + protected Field getReturnType() { + return Field.GENERIC_INT; + } + @Override protected DSHandle value(AbstractDataNode v) { - DSHandle r = new RootDataNode(Types.INT, getInt(this, v) + 1); + DSHandle r = NodeFactory.newRoot(Field.GENERIC_INT, getInt(this, v) + 1); return r; } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -31,12 +31,12 @@ import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.mapping.AbsFile; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.FileSystemLister; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.InitMapper; public class ProcessStageouts extends SwiftFunction { public static final Logger logger = Logger.getLogger(ProcessStageouts.class); @@ -78,7 +78,7 @@ Mapper m = var.getMapper(); if (!m.isStatic() && var.getType().hasArrayComponents()) { Collection found = m.existing(new FileSystemLister.FileList(collectList)); - RootDataNode.addExisting(found, m, var.getRoot(), var); + InitMapper.addExisting(found, m, var.getRoot(), var); logOnly(var, log); } else { Modified: trunk/src/org/griphyn/vdl/karajan/lib/Range.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Range.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/Range.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -33,12 +33,13 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.RootArrayDataNode; -import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.mapping.nodes.RootFutureArrayDataNode; +import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; @@ -49,7 +50,7 @@ @Override protected Signature getSignature() { - return new Signature(params("from", "to", optional("step", new RootDataNode(Types.FLOAT, 1)))); + return new Signature(params("from", "to", optional("step", NodeFactory.newRoot(Field.GENERIC_FLOAT, 1)))); } @Override @@ -70,8 +71,9 @@ // only deal with int and float try { final AbstractDataNode handle; + final Field valueField = Field.Factory.getImmutableField("rangeItem", type); - handle = new RootArrayDataNode(type.arrayType()) { + handle = new RootFutureArrayDataNode(Field.Factory.getImmutableField("range", type.arrayType()), null) { final DSHandle h = this; { closeShallow(); @@ -87,7 +89,7 @@ } else { int index = (Integer) path.getFirst(); - DSHandle value = new RootDataNode(type, new Double(start + incr * index)); + DSHandle value = NodeFactory.newRoot(valueField, new Double(start + incr * index)); return Collections.singletonList(value); } } @@ -113,10 +115,10 @@ { if (type == Types.INT) { - value = new RootDataNode(Types.INT, (int) crt); + value = NodeFactory.newRoot(Field.GENERIC_INT, (int) crt); } else { - value = new RootDataNode(Types.FLOAT, Double.valueOf(crt)); + value = NodeFactory.newRoot(Field.GENERIC_FLOAT, Double.valueOf(crt)); } key = index; } Modified: trunk/src/org/griphyn/vdl/karajan/lib/RemoteFileNames.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/RemoteFileNames.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/RemoteFileNames.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -43,7 +43,6 @@ Collection files = this.files.getValue(stack); List ret = new ArrayList(); for (AbsFile f : files) { - String path = null; if ("file".equals(f.getProtocol())) { ret.add(PathUtils.remotePathName(f.getPath())); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetDatasetValues.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SetDatasetValues.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/SetDatasetValues.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -28,8 +28,8 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class SetDatasetValues extends SwiftFunction { private ArgRef> stageouts; Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -43,10 +43,11 @@ import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.karajan.PairSet; import org.griphyn.vdl.karajan.WaitingThreadsMonitor; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DataDependentException; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Type; public class SetFieldValue extends SwiftFunction { @@ -111,8 +112,6 @@ tracer = Tracer.getTracer(this); return fn; } - - @Override protected void initializeArgs(Stack stack) { @@ -186,9 +185,15 @@ return sb.toString(); } - protected void deepCopy(DSHandle dest, DSHandle source, Stack stack) { + protected void deepCopy(DSHandle dest, DSHandle source, Stack stack) { // don't create a state if only a non-composite is copied - ((AbstractDataNode) source).waitFor(this); + try { + ((AbstractDataNode) source).waitFor(this); + } + catch (DataDependentException e) { + dest.setValue(new DataDependentException(dest, e)); + return; + } if (source.getType().isPrimitive()) { dest.setValue(source.getValue()); } @@ -208,7 +213,13 @@ /** make dest look like source - if its a simple value, copy that and if its an array then recursively copy */ public void deepCopy(DSHandle dest, DSHandle source, State state, int level) { - ((AbstractDataNode) source).waitFor(this); + try { + ((AbstractDataNode) source).waitFor(this); + } + catch (DataDependentException e) { + dest.setValue(new DataDependentException(dest, e)); + return; + } if (source.getType().isPrimitive()) { dest.setValue(source.getValue()); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -27,6 +27,8 @@ import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.util.ContactSet; import org.globus.swift.catalog.site.SiteCatalogParser; +import org.globus.swift.catalog.site.SwiftContact; +import org.griphyn.vdl.util.FQN; import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; @@ -120,7 +122,7 @@ return null; } String name = poolName(n, v); - BoundContact bc = new BoundContact(name); + SwiftContact bc = new SwiftContact(name); String sysinfo = attr(n, "sysinfo", null); if (sysinfo != null) { @@ -252,14 +254,14 @@ return s; } - private void env(BoundContact bc, Node n) { + private void env(SwiftContact bc, Node n) { String key = attr(n, "key"); String value = text(n); - bc.addProperty("env:" + key, value); + bc.addProfile(new FQN("env", key), value); } - private void profile(BoundContact bc, Node n) { + private void profile(SwiftContact bc, Node n) { String ns = attr(n, "namespace"); String key = attr(n, "key"); String value = text(n); @@ -271,7 +273,7 @@ bc.addProperty(key, value); } else { - bc.addProperty(ns + ":" + key, value); + bc.addProfile(new FQN(ns, key), value); } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/SliceArray.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SliceArray.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/SliceArray.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -28,11 +28,11 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.PairSet; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.RootArrayDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.RootFutureArrayDataNode; import org.griphyn.vdl.type.NoSuchTypeException; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; @@ -77,7 +77,7 @@ String destinationTypeName = this.type.getValue(stack); Type destinationType = Types.getType(destinationTypeName); - RootArrayDataNode destinationArray = new RootArrayDataNode(destinationType); + RootFutureArrayDataNode destinationArray = new RootFutureArrayDataNode(sourceArray.getField(), null); Path cutPath = Path.EMPTY_PATH.addLast(this.path.getValue(stack), false); @@ -89,8 +89,8 @@ DSHandle sourceElement = (DSHandle) pair.get(1); DSHandle n = sourceElement.getField(cutPath); - - destinationArray.getField((Comparable) index).set(n); + + destinationArray.addField((Comparable) index, n); } // all of the inputs should be closed, so @@ -122,9 +122,6 @@ catch(NoSuchTypeException nste) { throw new ExecutionException("No such type",nste); } - catch (NoSuchFieldException e) { - throw new ExecutionException(this, e); - } catch (InvalidPathException e) { throw new ExecutionException(this, e); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/Stagein.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Stagein.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/Stagein.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -37,10 +37,10 @@ import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.futures.FutureFault; import org.globus.cog.karajan.parser.WrapperNode; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DependentException; import org.griphyn.vdl.mapping.MappingDependentException; import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class Stagein extends SwiftFunction { public static final Logger logger = Logger.getLogger(Stagein.class); Modified: trunk/src/org/griphyn/vdl/karajan/lib/Stageout.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Stageout.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/Stageout.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -27,9 +27,9 @@ import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.VarRef; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.MappingDependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class Stageout extends SwiftFunction { public static final Logger logger = Logger.getLogger(Stageout.class); Added: trunk/src/org/griphyn/vdl/karajan/lib/StringCache.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/StringCache.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/StringCache.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -0,0 +1,28 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Apr 5, 2014 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.Map; +import java.util.WeakHashMap; + +public class StringCache { + private static Map cache = new WeakHashMap(); + + public synchronized static String get(String s) { + String sc = cache.get(s); + if (sc == null) { + cache.put(s, s); + return s; + } + else { + return sc; + } + } +} Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftBinaryOp.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SwiftBinaryOp.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/SwiftBinaryOp.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -12,17 +12,29 @@ import k.rt.Stack; import org.globus.cog.karajan.compiled.nodes.functions.BinaryOp; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; public abstract class SwiftBinaryOp extends BinaryOp { @Override public DSHandle function(Stack stack) { - AbstractDataNode v1 = this.v1.getValue(stack); - v1.waitFor(this); - AbstractDataNode v2 = this.v2.getValue(stack); - v2.waitFor(this); - return value(v1, v2); + try { + AbstractDataNode v1 = this.v1.getValue(stack); + v1.waitFor(this); + AbstractDataNode v2 = this.v2.getValue(stack); + v2.waitFor(this); + return value(v1, v2); + } + catch (DependentException e) { + return NodeFactory.newRoot(getReturnType(), e); + } } + + protected Field getReturnType() { + return Field.GENERIC_ANY; + } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -50,7 +50,6 @@ import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.karajan.functions.ConfigProperty; import org.griphyn.vdl.mapping.AbsFile; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DependentException; import org.griphyn.vdl.mapping.GeneralizedFileFormat; @@ -61,6 +60,9 @@ import org.griphyn.vdl.mapping.PathComparator; import org.griphyn.vdl.mapping.PhysicalFormat; import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; import org.griphyn.vdl.util.FQN; @@ -103,8 +105,8 @@ @Override public void runBody(LWThread thr) { + Stack stack = thr.getStack(); try { - Stack stack = thr.getStack(); ret(stack, function(stack)); } catch (AssertFailedException e) { @@ -112,11 +114,13 @@ throw e; } catch (DependentException e) { - // This would not be the primal fault so in non-lazy errors mode it - // should not matter - throw new ExecutionException("Wrapping a dependent exception in VDLFunction.post() - errors in data dependencies",e); + ret(stack, NodeFactory.newRoot(getReturnType(), e)); } } + + protected Field getReturnType() { + return Field.GENERIC_ANY; + } /* * This will likely break if the engine changes in fundamental ways. It also @@ -330,9 +334,14 @@ // Close the future handle.closeShallow(); // Mark all leaves - for (DSHandle child : handle.getFields(Path.CHILDREN)) { - child.closeShallow(); - } + try { + for (DSHandle child : handle.getAllFields()) { + child.closeShallow(); + } + } + catch (HandleOpenException e) { + throw new RuntimeException("Handle not closed after closeShallow()"); + } } public static void waitForAll(Node who, Channel vargs) throws ExecutionException { Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftUnaryOp.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SwiftUnaryOp.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/SwiftUnaryOp.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -12,15 +12,27 @@ import k.rt.Stack; import org.globus.cog.karajan.compiled.nodes.functions.UnaryOp; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; public abstract class SwiftUnaryOp extends UnaryOp { @Override public DSHandle function(Stack stack) { - AbstractDataNode v1 = this.v1.getValue(stack); - v1.waitFor(this); - return value(v1); + try { + AbstractDataNode v1 = this.v1.getValue(stack); + v1.waitFor(this); + return value(v1); + } + catch (DependentException e) { + return NodeFactory.newRoot(getReturnType(), e); + } } + + protected Field getReturnType() { + return Field.GENERIC_ANY; + } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -37,8 +37,8 @@ import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.VarRef; import org.globus.cog.karajan.compiled.nodes.functions.Map.Entry; -import org.globus.cog.karajan.util.BoundContact; import org.globus.swift.catalog.TCEntry; +import org.globus.swift.catalog.site.SwiftContact; import org.globus.swift.catalog.util.Profile; import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.util.FQN; @@ -46,7 +46,7 @@ public class TCProfile extends SwiftFunction { public static final Logger logger = Logger.getLogger(TCProfile.class); - private ArgRef host; + private ArgRef host; /** Allows for dynamic attributes from the SwiftScript profile statements. @@ -87,9 +87,9 @@ Map dynamicAttributes = readDynamicAttributes(stack); - BoundContact bc = this.host.getValue(stack); + SwiftContact bc = this.host.getValue(stack); - Map attrs = null; + Map attrs = null; attrs = attributesFromHost(bc, attrs, stack); TCEntry tce = null; @@ -176,12 +176,11 @@ public static final String PROFILE_GLOBUS_PREFIX = (Profile.GLOBUS + "::").toLowerCase(); - private void addEnvironment(Stack stack, BoundContact bc) { - Map props = bc.getProperties(); - if (props != null) { - for (Map.Entry e : props.entrySet()) { - String name = e.getKey(); - FQN fqn = new FQN(name); + private void addEnvironment(Stack stack, SwiftContact bc) { + Map profiles = bc.getProfiles(); + if (profiles != null) { + for (Map.Entry e : profiles.entrySet()) { + FQN fqn = e.getKey(); String value = (String) e.getValue(); if (Profile.ENV.equalsIgnoreCase(fqn.getNamespace())) { cr_environment.append(stack, new Entry(fqn.getName(), value)); @@ -235,11 +234,11 @@ Inserts namespace=globus attributes from BoundContact bc into given attrs */ - private Map attributesFromHost(BoundContact bc, Map attrs, Stack stack) { - Map props = bc.getProperties(); - if (props != null) { - for (Map.Entry e : props.entrySet()) { - FQN fqn = new FQN(e.getKey()); + private Map attributesFromHost(SwiftContact bc, Map attrs, Stack stack) { + Map profiles = bc.getProfiles(); + if (profiles != null) { + for (Map.Entry e : profiles.entrySet()) { + FQN fqn = e.getKey(); if (Profile.GLOBUS.equalsIgnoreCase(fqn.getNamespace())) { Attr a = ATTR_TYPES.get(fqn.getName()); if (a == null) { Modified: trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -19,11 +19,11 @@ import org.apache.log4j.Logger; import org.globus.cog.karajan.analyzer.VariableNotFoundException; import org.globus.cog.karajan.compiled.nodes.Node; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.RootHandle; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Types; import org.griphyn.vdl.util.VDL2Config; Modified: trunk/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -30,8 +30,8 @@ import org.apache.log4j.Logger; import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class UnwrapClosedList extends SwiftFunction { public static final Logger logger = Logger.getLogger(UnwrapClosedList.class); Modified: trunk/src/org/griphyn/vdl/karajan/lib/WaitFieldValue.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/WaitFieldValue.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/WaitFieldValue.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -22,9 +22,9 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; public class WaitFieldValue extends SwiftFunction { private ArgRef var; Modified: trunk/src/org/griphyn/vdl/karajan/lib/While.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/While.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/While.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -28,8 +28,8 @@ import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.parser.WrapperNode; -import org.griphyn.vdl.mapping.RootDataNode; -import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; public class While extends InternalFunction { @@ -85,7 +85,7 @@ switch(i) { case 0: drefs = RefCount.build(stack, srefs); - var.setValue(stack, new RootDataNode(Types.INT, 0)); + var.setValue(stack, NodeFactory.newRoot(Field.GENERIC_INT, 0)); c_next.create(stack); RefCount.incRefs(drefs); next = (SingleValueChannel) c_next.get(stack); Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/AbstractSingleValuedSwiftFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/AbstractSingleValuedSwiftFunction.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/AbstractSingleValuedSwiftFunction.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -0,0 +1,37 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Apr 17, 2014 + */ +package org.griphyn.vdl.karajan.lib.swiftscript; + +import k.rt.Stack; +import k.thr.LWThread; + +import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; + +public abstract class AbstractSingleValuedSwiftFunction extends AbstractSingleValuedFunction { + + + @Override + public void runBody(LWThread thr) { + Stack stack = thr.getStack(); + try { + ret(stack, function(stack)); + } + catch (DependentException e) { + ret(stack, NodeFactory.newRoot(getFieldType(), e)); + } + } + + protected Field getFieldType() { + return Field.GENERIC_ANY; + } +} Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -26,8 +26,9 @@ import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.AssertFailedException; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Types; /** @@ -58,18 +59,25 @@ hmessage = fargs.get(1); } - String message; - if (hmessage != null) { - hmessage.waitFor(this); - message = (String) hmessage.getValue(); + try { + String message; + if (hmessage != null) { + hmessage.waitFor(this); + message = (String) hmessage.getValue(); + } + else { + message = "Assertion failed"; + } + AbstractDataNode hvalue = fargs.get(0); + hvalue.waitFor(this); + + checkAssert(hvalue, message); } - else { - message = "Assertion failed"; + catch (DependentException e) { + if (logger.isInfoEnabled()) { + logger.info(this + " cannot check assertion: data exception"); + } } - AbstractDataNode hvalue = fargs.get(0); - hvalue.waitFor(this); - - checkAssert(hvalue, message); return null; } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -28,10 +28,11 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.RootDataNode; -import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; public class ExtractFloat extends SwiftFunction { @@ -53,12 +54,15 @@ BufferedReader breader = new BufferedReader(freader); String str = breader.readLine(); freader.close(); - DSHandle result = new RootDataNode(Types.FLOAT, Double.parseDouble(str)); + DSHandle result = NodeFactory.newRoot(Field.GENERIC_FLOAT, Double.parseDouble(str)); int provid = SwiftFunction.nextProvenanceID(); SwiftFunction.logProvenanceResult(provid, result, "extractfloat"); SwiftFunction.logProvenanceParameter(provid, handle, "filename"); return result; } + catch (DependentException e) { + return NodeFactory.newRoot(Field.GENERIC_FLOAT, e); + } catch (IOException ioe) { throw new ExecutionException("Reading integer content of file", ioe); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -28,10 +28,11 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.RootDataNode; -import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; public class ExtractInt extends SwiftFunction { @@ -53,12 +54,15 @@ BufferedReader breader = new BufferedReader(freader); String str = breader.readLine(); freader.close(); - DSHandle result = new RootDataNode(Types.INT, Integer.parseInt(str)); + DSHandle result = NodeFactory.newRoot(Field.GENERIC_INT, Integer.parseInt(str)); int provid = SwiftFunction.nextProvenanceID(); SwiftFunction.logProvenanceResult(provid, result, "extractint"); SwiftFunction.logProvenanceParameter(provid, handle, "filename"); return result; } + catch (DependentException e) { + return NodeFactory.newRoot(Field.GENERIC_INT, e); + } catch (IOException ioe) { throw new ExecutionException("Reading integer content of file", ioe); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -22,10 +22,10 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.RootDataNode; -import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; public class FileName extends SwiftFunction { private ArgRef var; @@ -39,7 +39,7 @@ public Object function(Stack stack) { AbstractDataNode var = this.var.getValue(stack); String s = argList(filename(var), true); - DSHandle result = new RootDataNode(Types.STRING, s); + DSHandle result = NodeFactory.newRoot(Field.GENERIC_STRING, s); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); SwiftFunction.logProvenanceParameter(provid, var, "input"); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -20,18 +20,17 @@ */ package org.griphyn.vdl.karajan.lib.swiftscript; -import k.rt.ExecutionException; +import java.util.Arrays; + import k.rt.Stack; import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.InvalidPathException; -import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.RootArrayDataNode; -import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; public class FileNames extends SwiftFunction { private ArgRef var; @@ -45,16 +44,10 @@ public Object function(Stack stack) { AbstractDataNode var = this.var.getValue(stack); String[] f = filename(var); - DSHandle returnArray = new RootArrayDataNode(Types.STRING.arrayType()); - try { - for (int i = 0; i < f.length; i++) { - Path p = parsePath("["+i+"]"); - DSHandle h = returnArray.getField(p); - h.setValue(relativize(f[i])); - } - } catch (InvalidPathException e) { - throw new ExecutionException("Unexpected invalid path exception",e); + for (int i = 0; i < f.length; i++) { + f[i] = relativize(f[i]); } + DSHandle returnArray = NodeFactory.newRoot(Field.GENERIC_STRING_ARRAY, Arrays.asList(f)); returnArray.closeShallow(); if (PROVENANCE_ENABLED) { Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -33,10 +33,10 @@ import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.VarRef; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.RootDataNode; -import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; public class FnArg extends SwiftFunction { @@ -48,6 +48,11 @@ protected Signature getSignature() { return new Signature(params("...")); } + + @Override + protected Field getReturnType() { + return Field.GENERIC_STRING; + } @Override protected void addLocals(Scope scope) { @@ -87,7 +92,7 @@ throw new ExecutionException("Missing command line argument: " + name); } else { - DSHandle result = new RootDataNode(Types.STRING, value); + DSHandle result = NodeFactory.newRoot(Field.GENERIC_STRING, value); if (PROVENANCE_ENABLED) { int provid = nextProvenanceID(); logProvenanceResult(provid, result, "arg"); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -30,7 +30,8 @@ import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; /** Formatted file output.
@@ -55,26 +56,43 @@ @Override public Object function(Stack stack) { - AbstractDataNode hfilename = this.filename.getValue(stack); - AbstractDataNode hspec = this.spec.getValue(stack); - hfilename.waitFor(this); - hspec.waitFor(this); - Channel args = c_vargs.get(stack); - waitForAll(this, args); - String filename = (String) hfilename.getValue(); - String spec = (String) hspec.getValue(); - - StringBuilder output = new StringBuilder(); try { - Sprintf.format(spec, args, output); + AbstractDataNode hfilename = this.filename.getValue(stack); + AbstractDataNode hspec = this.spec.getValue(stack); + hfilename.waitFor(this); + String filename = (String) hfilename.getValue(); + try { + hspec.waitFor(this); + Channel args = c_vargs.get(stack); + waitForAll(this, args); + String spec = (String) hspec.getValue(); + + StringBuilder output = new StringBuilder(); + try { + Sprintf.format(spec, args, output); + } + catch (RuntimeException e) { + throw new ExecutionException(this, e.getMessage()); + } + String msg = output.toString(); + + if (logger.isDebugEnabled()) { + logger.debug("file: " + filename + " msg: " + msg); + } + write(filename, msg); + } + catch (DependentException e) { + if (logger.isDebugEnabled()) { + logger.debug("file: " + filename + " msg: "); + } + write(filename, ""); + } } - catch (RuntimeException e) { - throw new ExecutionException(this, e.getMessage()); + catch (DependentException e) { + if (logger.isDebugEnabled()) { + logger.debug(""); + } } - String msg = output.toString(); - - logger.debug("file: " + filename + " msg: " + msg); - write(filename, msg); return null; } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -28,8 +28,8 @@ import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; -import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; @@ -48,6 +48,7 @@ AbstractDataNode hlib = this.lib.getValue(stack); AbstractDataNode hname = this.name.getValue(stack); Channel args = this.c_vargs.get(stack); + hlib.waitFor(this); hname.waitFor(this); waitForAll(this, args); @@ -56,7 +57,7 @@ Object[] p = convertInputs(method, args); Type type = returnType(method); Object value = invoke(method, p); - return new RootDataNode(type, value); + return NodeFactory.newRoot(type, value); } /** Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -17,10 +17,10 @@ package org.griphyn.vdl.karajan.lib.swiftscript; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; -import java.io.BufferedReader; - +import java.util.Arrays; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -35,14 +35,15 @@ import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.compiled.nodes.InternalFunction; -import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; import org.globus.cog.karajan.util.TypeUtil; +import org.griphyn.vdl.karajan.lib.StringCache; import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.RootArrayDataNode; -import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Types; import org.griphyn.vdl.util.VDL2Config; @@ -76,27 +77,39 @@ @Override protected void runBody(LWThread thr) { Channel vargs = c_vargs.get(thr.getStack()); - SwiftFunction.waitForAll(this, vargs); + StringBuilder buf = new StringBuilder(); + try { + Tracef.waitForAll(this, vargs); + buf.append("SwiftScript trace: "); + boolean first = true; + for (AbstractDataNode n : vargs) { + if (!first) { + buf.append(", "); + } + else { + first = false; + } + //buf.append(v == null ? args[i] : v); + prettyPrint(buf, n); + } + } + catch (DependentException e) { + buf.append("SwiftScript trace: "); + } - StringBuilder buf = new StringBuilder(); - buf.append("SwiftScript trace: "); - boolean first = true; - for (AbstractDataNode n : vargs) { - if (!first) { - buf.append(", "); - } - else { - first = false; - } - //buf.append(v == null ? args[i] : v); - prettyPrint(buf, n); - } traceLogger.warn(buf); } } private static void prettyPrint(StringBuilder buf, DSHandle h) { - Object o = h.getValue(); + Object o; + try { + o = h.getValue(); + } + catch (DependentException e) { + buf.append(""); + return; + } if (o == null) { buf.append(h); } @@ -131,7 +144,7 @@ } } - public static class StrCat extends AbstractSingleValuedFunction { + public static class StrCat extends AbstractSingleValuedSwiftFunction { private ChannelRef c_vargs; @Override @@ -140,6 +153,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { Channel vargs = c_vargs.get(stack); Channel args = SwiftFunction.unwrapAll(this, vargs); @@ -150,7 +168,7 @@ buf.append(TypeUtil.toString(o)); } - DSHandle handle = new RootDataNode(Types.STRING, buf.toString()); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING, buf.toString()); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -164,7 +182,7 @@ } } - public static class Exists extends AbstractSingleValuedFunction { + public static class Exists extends AbstractSingleValuedSwiftFunction { private ArgRef file; @Override @@ -173,6 +191,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_BOOLEAN; + } + + @Override public Object function(Stack stack) { AbstractDataNode dn = file.getValue(stack); String filename; @@ -187,7 +210,7 @@ if (logger.isDebugEnabled()) { logger.debug("exists: " + file); } - DSHandle handle = new RootDataNode(Types.BOOLEAN, file.exists()); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_BOOLEAN, file.exists()); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -199,7 +222,7 @@ } } - public static class StrCut extends AbstractSingleValuedFunction { + public static class StrCut extends AbstractSingleValuedSwiftFunction { private ArgRef input; private ArgRef pattern; @@ -209,6 +232,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { AbstractDataNode hinput = this.input.getValue(stack); String input = SwiftFunction.unwrap(this, hinput); @@ -235,7 +263,7 @@ if (logger.isDebugEnabled()) { logger.debug("strcut matched '" + group + "'"); } - DSHandle handle = new RootDataNode(Types.STRING, group); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING, group); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -247,7 +275,7 @@ } } - public static class StrStr extends AbstractSingleValuedFunction { + public static class StrStr extends AbstractSingleValuedSwiftFunction { private ArgRef input; private ArgRef pattern; @@ -257,6 +285,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { AbstractDataNode hinput = this.input.getValue(stack); String input = SwiftFunction.unwrap(this, hinput); @@ -267,7 +300,7 @@ logger.debug("strstr will search '" + input + "' for pattern '" + pattern + "'"); } - DSHandle result = new RootDataNode(Types.INT, input.indexOf(pattern)); + DSHandle result = NodeFactory.newRoot(Field.GENERIC_INT, input.indexOf(pattern)); if (PROVENANCE_ENABLED) { @@ -280,62 +313,63 @@ } } - public static class ExecSystem extends AbstractSingleValuedFunction { + public static class ExecSystem extends AbstractSingleValuedSwiftFunction { private ArgRef input; - //private ArgRef pattern; - + @Override protected Signature getSignature() { return new Signature(params("input")); - } + } @Override - public Object function(Stack stack) { - AbstractDataNode hinput = this.input.getValue(stack); - String input = SwiftFunction.unwrap(this, hinput); - - DSHandle handle = new RootArrayDataNode(Types.STRING.arrayType()); - StringBuffer out = new StringBuffer(); - Process proc; - int i = 0; - - try { - proc = Runtime.getRuntime().exec(new String[] {"bash", "-c", input}); - proc.waitFor(); - int exitcode = proc.exitValue(); - // If the shell returned a non-zero exit code, attempt to print stderr - if ( exitcode != 0 ) { - BufferedReader reader = new BufferedReader( new InputStreamReader(proc.getErrorStream()) ); - String line = ""; - StringBuffer stderr = new StringBuffer(); + public Object function(Stack stack) { + AbstractDataNode hinput = this.input.getValue(stack); + String input = SwiftFunction.unwrap(this, hinput); + + DSHandle handle = NodeFactory.newOpenRoot(Field.GENERIC_STRING_ARRAY, null); + + StringBuffer out = new StringBuffer(); + Process proc; + int i = 0; + + try { + proc = Runtime.getRuntime().exec(new String[] {"bash", "-c", input}); + proc.waitFor(); + int exitcode = proc.exitValue(); + // If the shell returned a non-zero exit code, attempt to print stderr + if ( exitcode != 0 ) { + BufferedReader reader = new BufferedReader( new InputStreamReader(proc.getErrorStream()) ); + String line = ""; + StringBuffer stderr = new StringBuffer(); + while ( (line = reader.readLine()) != null ) { + stderr.append(line); + } + logger.warn("swift:system returned exitcode :" + exitcode); + logger.warn("swift:system stderr:\n " + stderr ); + } + BufferedReader reader = new BufferedReader( new InputStreamReader(proc.getInputStream()) ); + String line = ""; while ( (line = reader.readLine()) != null ) { - stderr.append(line); + DSHandle el; + el = handle.getField(i++); + el.setValue(line); } - logger.warn("swift:system returned exitcode :" + exitcode); - logger.warn("swift:system stderr:\n " + stderr ); - } - BufferedReader reader = new BufferedReader( new InputStreamReader(proc.getInputStream()) ); - String line = ""; - while ( (line = reader.readLine()) != null ) { - DSHandle el; - el = handle.getField(i++); - el.setValue(line); - } - } catch (Exception e) { - e.printStackTrace(); - } - handle.closeDeep(); - - if (PROVENANCE_ENABLED) { - int provid = SwiftFunction.nextProvenanceID(); - SwiftFunction.logProvenanceResult(provid, handle, "system"); - SwiftFunction.logProvenanceParameter(provid, hinput, "input"); - } - return handle; + } catch (Exception e) { + e.printStackTrace(); + } + handle.closeDeep(); + + if (PROVENANCE_ENABLED) { + int provid = SwiftFunction.nextProvenanceID(); + SwiftFunction.logProvenanceResult(provid, handle, "system"); + SwiftFunction.logProvenanceParameter(provid, hinput, "input"); + } + return handle; } } - public static class StrSplit extends AbstractSingleValuedFunction { + public static class StrSplit extends AbstractSingleValuedSwiftFunction { + private ArgRef input; private ArgRef pattern; @@ -345,6 +379,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING_ARRAY; + } + + @Override public Object function(Stack stack) { AbstractDataNode hinput = this.input.getValue(stack); String input = SwiftFunction.unwrap(this, hinput); @@ -353,17 +392,7 @@ String[] split = input.split(pattern); - DSHandle handle = new RootArrayDataNode(Types.STRING.arrayType()); - for (int i = 0; i < split.length; i++) { - DSHandle el; - try { - el = handle.getField(i); - el.setValue(split[i]); - } - catch (NoSuchFieldException e) { - throw new ExecutionException(this, e); - } - } + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING_ARRAY, Arrays.asList(split)); handle.closeDeep(); if (PROVENANCE_ENABLED) { @@ -382,7 +411,7 @@ * @return DSHandle representing the resulting string * @throws ExecutionException */ - public static class StrJoin extends AbstractSingleValuedFunction { + public static class StrJoin extends AbstractSingleValuedSwiftFunction { private ArgRef array; private ArgRef delim; @@ -392,6 +421,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { AbstractDataNode harray = this.array.getValue(stack); Map, DSHandle> arrayValues = SwiftFunction.waitForArray(this, harray); @@ -411,7 +445,7 @@ result.append(h.getValue()); } - DSHandle handle = new RootDataNode(Types.STRING, result.toString()); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING, result.toString()); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -423,7 +457,7 @@ } } - public static class Regexp extends AbstractSingleValuedFunction { + public static class Regexp extends AbstractSingleValuedSwiftFunction { private ArgRef input; private ArgRef pattern; private ArgRef transform; @@ -434,6 +468,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { AbstractDataNode hinput = this.input.getValue(stack); String input = SwiftFunction.unwrap(this, hinput); @@ -461,7 +500,7 @@ if (logger.isDebugEnabled()) { logger.debug("regexp replacement produced '" + group + "'"); } - DSHandle handle = new RootDataNode(Types.STRING, group); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING, group); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -474,7 +513,7 @@ } } - public static class ToInt extends AbstractSingleValuedFunction { + public static class ToInt extends AbstractSingleValuedSwiftFunction { private ArgRef str; @Override @@ -483,11 +522,16 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_INT; + } + + @Override public Object function(Stack stack) { AbstractDataNode hstr = str.getValue(stack); String str = SwiftFunction.unwrap(this, hstr); - DSHandle handle = new RootDataNode(Types.INT, Integer.valueOf(str)); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_INT, Integer.valueOf(str)); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -499,7 +543,7 @@ } } - public static class ToFloat extends AbstractSingleValuedFunction { + public static class ToFloat extends AbstractSingleValuedSwiftFunction { private ArgRef str; @Override @@ -508,6 +552,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_FLOAT; + } + + @Override public Object function(Stack stack) { AbstractDataNode hstr = str.getValue(stack); Object obj = SwiftFunction.unwrap(this, hstr); @@ -515,10 +564,10 @@ DSHandle handle; if (obj instanceof String) { - handle = new RootDataNode(Types.FLOAT, Double.valueOf((String) obj)); + handle = NodeFactory.newRoot(Field.GENERIC_FLOAT, Double.valueOf((String) obj)); } else if (obj instanceof Number) { - handle = new RootDataNode(Types.FLOAT, ((Number) obj).doubleValue()); + handle = NodeFactory.newRoot(Field.GENERIC_FLOAT, ((Number) obj).doubleValue()); } else { throw new ExecutionException("Expected a string or int. Got " + obj); @@ -538,7 +587,7 @@ /* * Takes in a float and formats to desired precision and returns a string */ - public static class Format extends AbstractSingleValuedFunction { + public static class Format extends AbstractSingleValuedSwiftFunction { private ChannelRef c_vargs; @Override @@ -547,6 +596,11 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { Channel vargs = c_vargs.get(stack); Channel args = SwiftFunction.unwrapAll(this, vargs); @@ -558,7 +612,7 @@ String format = (String) args.get(0); Object[] a = args.subChannel(1).toArray(); - DSHandle handle = new RootDataNode(Types.STRING, String.format(format, a)); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING, String.format(format, a)); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -571,7 +625,7 @@ /* * Takes in an int and pads zeros to the left and returns a string */ - public static class Pad extends AbstractSingleValuedFunction { + public static class Pad extends AbstractSingleValuedSwiftFunction { private ArgRef size; private ArgRef value; @@ -581,13 +635,18 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { AbstractDataNode hsize = this.size.getValue(stack); Integer size = SwiftFunction.unwrap(this, hsize); AbstractDataNode hvalue = this.value.getValue(stack); Integer value = SwiftFunction.unwrap(this, hvalue); - DSHandle handle = new RootDataNode(Types.STRING, + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING, String.format("%0" + size + "d", value)); if (PROVENANCE_ENABLED) { @@ -600,7 +659,7 @@ } } - public static class ToString extends AbstractSingleValuedFunction { + public static class ToString extends AbstractSingleValuedSwiftFunction { private ArgRef value; @Override @@ -609,13 +668,18 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { AbstractDataNode hvalue = this.value.getValue(stack); hvalue.waitFor(this); StringBuilder sb = new StringBuilder(); prettyPrint(sb, hvalue); - DSHandle handle = new RootDataNode(Types.STRING, sb.toString()); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING, StringCache.get(sb.toString())); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -634,7 +698,7 @@ * Good for debugging because array needs to be closed * before the length is determined */ - public static class Dirname extends AbstractSingleValuedFunction { + public static class Dirname extends AbstractSingleValuedSwiftFunction { private ArgRef file; @Override @@ -643,12 +707,17 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { AbstractDataNode dn = file.getValue(stack); String name = SwiftFunction.filename(dn)[0]; String result = new AbsFile(name).getDirectory(); - DSHandle handle = new RootDataNode(Types.STRING, result); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_STRING, result); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); @@ -660,7 +729,7 @@ } } - public static class Length extends AbstractSingleValuedFunction { + public static class Length extends AbstractSingleValuedSwiftFunction { private ArgRef array; @Override @@ -669,11 +738,16 @@ } @Override + protected Field getFieldType() { + return Field.GENERIC_INT; + } + + @Override public Object function(Stack stack) { AbstractDataNode harray = this.array.getValue(stack); harray.waitFor(this); - DSHandle handle = new RootDataNode(Types.INT, Integer.valueOf(harray.getArrayValue().size())); + DSHandle handle = NodeFactory.newRoot(Field.GENERIC_INT, Integer.valueOf(harray.getArrayValue().size())); if (PROVENANCE_ENABLED) { int provid = SwiftFunction.nextProvenanceID(); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -36,9 +36,11 @@ import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DataDependentException; +import org.griphyn.vdl.mapping.DependentException; import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; @@ -59,7 +61,13 @@ public Object function(Stack stack) { AbstractDataNode dest = this.dest.getValue(stack); AbstractDataNode src = this.src.getValue(stack); - src.waitFor(this); + try { + src.waitFor(this); + } + catch (DependentException e) { + dest.setValue(new DataDependentException(dest, e)); + return null; + } if (src.getType().equals(Types.STRING)) { readData(dest, (String) src.getValue()); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -33,10 +33,12 @@ import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DataDependentException; +import org.griphyn.vdl.mapping.DependentException; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Types; public class ReadStructured extends SwiftFunction { @@ -57,7 +59,13 @@ public Object function(Stack stack) { AbstractDataNode dest = this.dest.getValue(stack); AbstractDataNode src = this.src.getValue(stack); - src.waitFor(this); + try { + src.waitFor(this); + } + catch (DependentException e) { + dest.setValue(new DataDependentException(dest, e)); + return null; + } if (src.getType().equals(Types.STRING)) { readData(dest, (String) src.getValue()); dest.closeDeep(); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -26,11 +26,12 @@ import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; -import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.ArrayHandle; +import org.griphyn.vdl.mapping.nodes.NodeFactory; +import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Types; /** @@ -63,8 +64,12 @@ return new Signature(params("spec", "...")); } - @Override + protected Field getReturnType() { + return Field.GENERIC_STRING; + } + + @Override public Object function(Stack stack) { AbstractDataNode hspec = this.spec.getValue(stack); hspec.waitFor(this); @@ -77,9 +82,9 @@ logger.debug("generated: " + msg); } - return new RootDataNode(Types.STRING, msg); + return NodeFactory.newRoot(Field.GENERIC_STRING, msg); } - + public static String format(String spec, Channel args) { logger.debug("spec: " + spec); StringBuilder output = new StringBuilder(); @@ -197,11 +202,11 @@ } private static void append_q(DSHandle arg, StringBuilder output) { - if (arg instanceof ArrayDataNode) { - ArrayDataNode node = (ArrayDataNode) arg; + if (arg instanceof ArrayHandle) { + ArrayHandle node = (ArrayHandle) arg; output.append("["); try { - int size = node.size(); + int size = node.arraySize(); for (int i = 0; i < size; i++) { String entry = "["+i+"]"; DSHandle handle = Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -24,8 +24,10 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.Node; import org.griphyn.vdl.karajan.lib.SwiftFunction; -import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.DependentException; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; /** Formatted trace output.
@@ -52,14 +54,32 @@ @Override public Object function(Stack stack) { AbstractDataNode hspec = this.spec.getValue(stack); - hspec.waitFor(this); - Channel args = c_vargs.get(stack); - waitForAll(this, args); - String spec = (String) hspec.getValue(); - - String msg = Sprintf.format(spec, args); + String msg; + try { + hspec.waitFor(this); + Channel args = c_vargs.get(stack); + waitForAll(this, args); + String spec = (String) hspec.getValue(); + + msg = Sprintf.format(spec, args); + } + catch (DependentException e) { + msg = ""; + } logger.info(msg); System.out.print(msg); return null; } + + public static void waitForAll(Node who, Channel vargs) { + for (AbstractDataNode n : vargs) { + try { + n.waitFor(who); + } + catch (DependentException e) { + // ignore here, will print special message in trace + } + } + } + } Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java 2014-05-09 04:48:47 UTC (rev 7842) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java 2014-05-09 04:56:39 UTC (rev 7843) @@ -32,11 +32,13 @@ import org.globus.cog.karajan.analyzer.Signature; import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; -import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DataDependentException; +import org.griphyn.vdl.mapping.DependentException; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; @@ -62,7 +64,16 @@ // src can be any of several forms of value AbstractDataNode src = this.src.getValue(stack); - src.waitFor(this); + try { + src.waitFor(this); + } + catch (DependentException e) { + if (logger.isInfoEnabled()) { + logger.info(this + " caught dependent exception"); + } + dest.setValue(new DataDependentException(dest, e)); + return null; + } if (dest.getType().equals(Types.STRING)) { writeData((String)dest.getValue(), src); From ketan at ci.uchicago.edu Fri May 9 09:35:52 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 9 May 2014 09:35:52 -0500 (CDT) Subject: [Swift-commit] r7850 - SwiftApps/swift-galaxy/swift Message-ID: <20140509143552.C5710178884@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-09 09:35:51 -0500 (Fri, 09 May 2014) New Revision: 7850 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: stdin from /dev/null and create outloc dir Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-09 05:01:52 UTC (rev 7849) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-09 14:35:51 UTC (rev 7850) @@ -59,6 +59,9 @@ shift done +#create outloc dir +mkdir -p $outloc + #workout the stringargs if [ "${stringargs}" ] ; then stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; @@ -235,18 +238,18 @@ EXITCODE=0 case $sites in "localhost" ) - echo "in localhost" >> /tmp/status.txt + #echo "in localhost" >> /tmp/status.txt STC=$(\which stc) - echo "got stc" >> /tmp/status.txt + #echo "got stc" >> /tmp/status.txt turbine=$(\which turbine) - echo "got turbine" >> /tmp/status.txt + #echo "got turbine" >> /tmp/status.txt $STC -j $(\which java) $wdir/script.swift 2>$logfile - echo "done running stc" >> /tmp/status.txt - echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt - $turbine -V -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt - echo "done running turbine, for $n" >> /tmp/status.txt + #echo "done running stc" >> /tmp/status.txt + #echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt + $turbine -n $n $wdir/script.tcl "${swiftargs}" > /tmp/status.txt EXITCODE=$? - echo "exit code obtained" >> /tmp/status.txt + #echo "exit code obtained" >> /tmp/status.txt ;; "Vesta" ) TURBINE_HOME=/home/wozniak/Public/sfw/turbine From hategan at ci.uchicago.edu Sat May 10 13:05:54 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 10 May 2014 13:05:54 -0500 (CDT) Subject: [Swift-commit] r7851 - trunk/libexec Message-ID: <20140510180554.2FFEF9D05C@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-10 13:05:51 -0500 (Sat, 10 May 2014) New Revision: 7851 Modified: trunk/libexec/swift-int-staging.k trunk/libexec/swift-int-wrapper-staging.k trunk/libexec/swift-int.k trunk/libexec/swift-lib.k trunk/libexec/swift.k Log: interned fields; data type reorganization; propagation of lazy failures in internal functions Modified: trunk/libexec/swift-int-staging.k =================================================================== --- trunk/libexec/swift-int-staging.k 2014-05-09 14:35:51 UTC (rev 7850) +++ trunk/libexec/swift-int-staging.k 2014-05-10 18:05:51 UTC (rev 7851) @@ -20,11 +20,10 @@ PROVENANCE_GRAPH_ENABLED := (configProperty("pgraph") != "false") CLEANUP_ENABLED := (configProperty("sitedir.keep") != "true") -namespace(swift) { +DEBUG_DIR := "{SWIFT:DEBUG_DIR_PREFIX}{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d" +CDM_FILE := cdm:file() - initDDir := function() { - "{SWIFT:DEBUG_DIR_PREFIX}{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d" - } +namespace(swift) { fileSizes := function(files) { math:sum( @@ -60,140 +59,136 @@ } export(execute2, - function(progress, tr, stagein, stageout, + function(rhost, progress, tr, stagein, stageout, replicationGroup, replicationChannel, arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { - - allocateHost(rhost, constraints = jobConstraints(tr, stagein = stagein)) { - ddir := initDDir() + uid := UID() + jobdir := substring(uid, 0, to=1) + jobid := "{tr}-{uid}" - uid := UID() - jobdir := substring(uid, 0, to=1) - jobid := "{tr}-{uid}" + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") + + wfdir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" + tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid), + + (fileDirs, inFiles, outFiles, outCollect) := getStagingInfo(stagein, stageout) - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") - - wfdir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" - tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) - cdmfile := cdm:file(), - - (fileDirs, inFiles, outFiles, outCollect) := getStagingInfo(stagein, stageout) + try { + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), + " tmpdir={tmpdir} host={rhost}") - try { - log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), - " tmpdir={tmpdir} host={rhost}") + setProgress(progress, "Submitting") - setProgress(progress, "Submitting") + swift:execute( + progress, + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + "_swiftwrap.staging", + "-e", executable(tr, rhost), + "-out", if(stdout == null, "stdout.txt", getFieldValue(stdout)), + "-err", if(stderr == null, "stderr.txt", getFieldValue(stderr)), + "-i", if (stdin != null, getFieldValue(stdin)), + "-d", str:join(fileDirs, "|"), + "-if", str:join(remoteFileNames(inFiles), "|"), + "-of", str:join(remoteFileNames(outFiles), "|"), + "-cf", str:join(remoteFileNames(outCollect), "|"), + "-cdmfile", CDM_FILE, + "-status", "provider" + "-a", if (arguments != null, each(arguments)) + ) + directory = tmpdir + redirect = false + host = rhost - swift:execute( - progress, - siteProfile(rhost, "swift:wrapperInterpreter"), - list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), - "_swiftwrap.staging", - "-e", executable(tr, rhost), - "-out", if(stdout == null, "stdout.txt", getFieldValue(stdout)), - "-err", if(stderr == null, "stderr.txt", getFieldValue(stderr)), - "-i", if (stdin != null, getFieldValue(stdin)), - "-d", str:join(fileDirs, "|"), - "-if", str:join(remoteFileNames(inFiles), "|"), - "-of", str:join(remoteFileNames(outFiles), "|"), - "-cf", str:join(remoteFileNames(outCollect), "|"), - "-cdmfile", cdmfile, - "-status", "provider" - "-a", if (arguments != null, each(arguments)) - ) - directory = tmpdir - redirect = false - host = rhost + TCProfile(rhost, if (attributes != null, attributes = attributes), tr = tr) + replicationGroup = replicationGroup + replicationChannel = replicationChannel + jobid = jobid + + stagingMethod := siteProfile(rhost, "swift:stagingMethod", default="proxy") - TCProfile(rhost, if (attributes != null, attributes = attributes), tr = tr) - replicationGroup = replicationGroup - replicationChannel = replicationChannel - jobid = jobid - - stagingMethod := siteProfile(rhost, "swift:stagingMethod", default="proxy") + stageIn("{PIN}{stagingMethod}://localhost/{SWIFT:HOME}/libexec/_swiftwrap.staging", "_swiftwrap.staging") - stageIn("{PIN}{stagingMethod}://localhost/{SWIFT:HOME}/libexec/_swiftwrap.staging", "_swiftwrap.staging") + if (CDM_FILE != "") { + d := swift:dirname(CDM_FILE) + file := basename(CDM_FILE) + dir := if (d == "", "./", "{d}/") + loc := "{PIN}{stagingMethod}://localhost/" + stageIn("{loc}{dir}{file}", CDM_FILE) + stageIn("{loc}{SWIFT:HOME}/libexec/cdm.pl", "cdm.pl") + stageIn("{loc}{SWIFT:HOME}/libexec/cdm_lib.sh", "cdm_lib.sh") + } - if (cdmfile != "") { - d := swift:dirname(cdmfile) - file := basename(cdmfile) - dir := if (d == "", "./", "{d}/") - loc := "{PIN}{stagingMethod}://localhost/" - stageIn("{loc}{dir}{file}", cdmfile) - stageIn("{loc}{SWIFT:HOME}/libexec/cdm.pl", "cdm.pl") - stageIn("{loc}{SWIFT:HOME}/libexec/cdm_lib.sh", "cdm_lib.sh") - } + appStageins(jobid, inFiles, stagingMethod) - appStageins(jobid, inFiles, stagingMethod) - - stageOut("wrapper.log", "{stagingMethod}://localhost/{ddir}/{jobid}.info", - mode = WRAPPER_TRANSFER_MODE) - - if (stdout == null) { - // if not stdout requested, only stage on error - stageOut("stdout.txt", "{stagingMethod}://localhost/{ddir}/{jobid}.stdout", - mode = STAGING_MODE:ON_ERROR + STAGING_MODE:IF_PRESENT) - } - else { - stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}", - mode = STAGING_MODE:IF_PRESENT) - } - if (stderr == null) { - stageOut("stderr.txt", "{stagingMethod}://localhost/{ddir}/{jobid}.stderr", - mode = STAGING_MODE:ON_ERROR + STAGING_MODE:IF_PRESENT) - } - else { - stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}", - mode = STAGING_MODE:IF_PRESENT) - } - stageOut("wrapper.error", "{stagingMethod}://localhost/{ddir}/{jobid}.error", - mode = STAGING_MODE:IF_PRESENT) - appStageouts(jobid, outFiles, outCollect, stagingMethod) - if (!isEmpty(outCollect)) { - stageOut("_collect", "{stagingMethod}://localhost/{ddir}/{jobid}.collect", - mode = STAGING_MODE:ALWAYS) - } - - if (CLEANUP_ENABLED) { - task:cleanUp(".") - } - ) + stageOut("wrapper.log", "{stagingMethod}://localhost/{DEBUG_DIR}/{jobid}.info", + mode = WRAPPER_TRANSFER_MODE) - - if (!isEmpty(outCollect)) { - readCollectList("{ddir}/{jobid}.collect") + if (stdout == null) { + // if not stdout requested, only stage on error + stageOut("stdout.txt", "{stagingMethod}://localhost/{DEBUG_DIR}/{jobid}.stdout", + mode = STAGING_MODE:ON_ERROR + STAGING_MODE:IF_PRESENT) } else { - [] + stdoutf := getFieldValue(stdout) + stageOut(stdoutf, "{stagingMethod}://localhost/{DEBUG_DIR}/{stdoutf}", + mode = STAGING_MODE:IF_PRESENT) } - - log(LOG:DEBUG, "JOB_END jobid={jobid}") - } - else catch(prev) { - if (matches(prev, "^Abort$")) { - log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") - throw(prev) + if (stderr == null) { + stageOut("stderr.txt", "{stagingMethod}://localhost/{DEBUG_DIR}/{jobid}.stderr", + mode = STAGING_MODE:ON_ERROR + STAGING_MODE:IF_PRESENT) } else { - setProgress(progress, "Failed but can retry") - exception := try(exception(readErrorFiles(ddir, jobid, stdout, stderr), prev), prev) - log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + stderrf := getFieldValue(stderr) + stageOut(stderrf, "{stagingMethod}://localhost/{DEBUG_DIR}/{stderrf}", + mode = STAGING_MODE:IF_PRESENT) + } + stageOut("wrapper.error", "{stagingMethod}://localhost/{DEBUG_DIR}/{jobid}.error", + mode = STAGING_MODE:IF_PRESENT) + appStageouts(jobid, outFiles, outCollect, stagingMethod) + if (!isEmpty(outCollect)) { + stageOut("_collect", "{stagingMethod}://localhost/{DEBUG_DIR}/{jobid}.collect", + mode = STAGING_MODE:ALWAYS) + } - throw( - exception( - concat( - "Exception in {tr}:", - if (arguments != null, "\n Arguments: {arguments}") - "\n Host: {rhost}", - "\n Directory: {tmpdir}" - ) - exception + if (CLEANUP_ENABLED) { + task:cleanUp(".") + } + ) + + + if (!isEmpty(outCollect)) { + readCollectList("{DEBUG_DIR}/{jobid}.collect") + } + else { + [] + } + + log(LOG:DEBUG, "JOB_END jobid={jobid}") + } + else catch(prev) { + if (matches(prev, "^Abort$")) { + log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") + throw(prev) + } + else { + setProgress(progress, "Failed but can retry") + exception := try(exception(readErrorFiles(DEBUG_DIR, jobid, stdout, stderr), prev), prev) + log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + + throw( + exception( + concat( + "Exception in {tr}:", + if (arguments != null, "\n Arguments: {arguments}") + "\n Host: {rhost}", + "\n Directory: {tmpdir}" ) + exception ) - } + ) } } } Modified: trunk/libexec/swift-int-wrapper-staging.k =================================================================== --- trunk/libexec/swift-int-wrapper-staging.k 2014-05-09 14:35:51 UTC (rev 7850) +++ trunk/libexec/swift-int-wrapper-staging.k 2014-05-10 18:05:51 UTC (rev 7851) @@ -129,178 +129,175 @@ export(execute2, - function(progress, tr, stagein, stageout, + function(rhost, progress, tr, stagein, stageout, replicationGroup, replicationChannel, arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { - allocateHost(rhost, constraints=jobConstraints(tr, stagein = stagein)) { + ddir := initDDir() + wfdir := try { + initSharedDir(rhost) + } + else catch(exception) { + throw(exception("Could not initialize shared directory on {rhost}", exception)) + } - ddir := initDDir() - wfdir := try { - initSharedDir(rhost) - } - else catch(exception) { - throw(exception("Could not initialize shared directory on {rhost}", exception)) - } + uid := UID() + jobid := "{tr}-{uid}" + + jobdir := concat(ddir, "/jobs/", substring(uid, from=0, to=1), "/{jobid}/") - uid := UID() - jobid := "{tr}-{uid}" - - jobdir := concat(ddir, "/jobs/", substring(uid, from=0, to=1), "/{jobid}/") + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") + statusMode := configProperty("status.mode", host = rhost) + wrapperMode := configProperty("wrapper.parameter.mode", host = rhost) - statusMode := configProperty("status.mode", host = rhost) - wrapperMode := configProperty("wrapper.parameter.mode", host = rhost) + wrapfile := "{jobdir}/_paramfile" - wrapfile := "{jobdir}/_paramfile" + stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout)) + stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)), + + (localFileDirs, remoteFileDirs, inFiles, outFiles) := getStagingInfo(stagein, stageout) - stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout)) - stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)), - - (localFileDirs, remoteFileDirs, inFiles, outFiles) := getStagingInfo(stagein, stageout) + os := siteProfile(rhost, "SYSINFO:OS") + + scratch := siteProfile(rhost, "scratch") - os := siteProfile(rhost, "SYSINFO:OS") - - scratch := siteProfile(rhost, "scratch") + if(wrapperMode == "files") { + file:write(wrapfile, + "-e ",vdl:executable(tr, rhost), + "\n-out ", stdout, + "\n-err ", stderr, + "\n-i ", if (stdin != null, getFieldValue(stdin)), + "\n-d ", str:join(remoteFileDirs, "|"), + "\n-if ", str:join(remoteFileNames(inFiles), "|"), + "\n-of ", str:join(remoteFileNames(outFiles), "|"), + "\n-wt", WRAPPERLOG_ALWAYS_TRANSFER, + "\n-sk", SITEDIR_KEEP, + "\n-cdmfile ", cdm:file(), + "\n-status ", statusMode, + for(a, arguments) { + "\n-a ", a + } + ) + } + - if(wrapperMode == "files") { - file:write(wrapfile, - "-e ",vdl:executable(tr, rhost), - "\n-out ", stdout, - "\n-err ", stderr, - "\n-i ", if (stdin != null, getFieldValue(stdin)), - "\n-d ", str:join(remoteFileDirs, "|"), - "\n-if ", str:join(remoteFileNames(inFiles), "|"), - "\n-of ", str:join(remoteFileNames(outFiles), "|"), - "\n-wt", WRAPPERLOG_ALWAYS_TRANSFER, - "\n-sk", SITEDIR_KEEP, - "\n-cdmfile ", cdm:file(), - "\n-status ", statusMode, - for(a, arguments) { - "\n-a ", a - } - ) + setProgress(progress, "Stage in") + + try { + if (wrapperMode == "files") { + stageWrapperParams(jobid, wrapfile, wfdir, rhost) } - - setProgress(progress, "Stage in") + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), " host={rhost}") - try { - if (wrapperMode == "files") { - stageWrapperParams(jobid, wrapfile, wfdir, rhost) - } + setProgress(progress, "Submitting") - log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), " host={rhost}") - - setProgress(progress, "Submitting") - - if (wrapperMode == "files") { - swift:execute( - progress, - siteProfile(rhost, "swift:wrapperInterpreter"), - list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), - "_swiftwrap.wrapperstaging", - jobid, - "-urlprefix", URL_PREFIX, - "-jobdir", jobdir, - "-scratch", scratch, - "-p", wrapfile - ) - directory=wfdir - redirect=false - host=rhost - TCProfile(rhost, if (attributes != null, attributes = attributes), tr=tr) - replicationGroup=replicationGroup - replicationChannel=replicationChannel - jobid=jobid + if (wrapperMode == "files") { + swift:execute( + progress, + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + "_swiftwrap.wrapperstaging", + jobid, + "-urlprefix", URL_PREFIX, + "-jobdir", jobdir, + "-scratch", scratch, + "-p", wrapfile ) - } - if (wrapperMode == "args") { - swift:execute( - siteProfile(rhost, "swift:wrapperInterpreter"), - list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), - "_swiftwrap.wrapperstaging", - jobid, - "-urlprefix", URL_PREFIX, - "-jobdir", jobdir, - "-scratch", scratch, - "-e", vdl:executable(tr, rhost), - "-out", stdout, - "-err", stderr, - "-i", if (stdin != null, getFieldValue(stdin)), - "-d", str:join(remoteFileDirs, "|"), - "-if", str:join(remoteFileNames(inFiles), "|"), - "-of", str:join(remoteFileNames(outFiles), "|"), - "-wt", WRAPPERLOG_ALWAYS_TRANSFER, - "-sk", SITEDIR_KEEP, - "-cdmfile", cdm:file(), - "-status", statusMode, - "-a", if (arguments != null, each(arguments)) - ) - directory=wfdir - redirect=false - host=rhost - TCProfile(rhost, if(attributes != null, attributes = attributes), tr=tr) - replicationGroup=replicationGroup - replicationChannel=replicationChannel - jobid=jobid + directory=wfdir + redirect=false + host=rhost + TCProfile(rhost, if (attributes != null, attributes = attributes), tr=tr) + replicationGroup=replicationGroup + replicationChannel=replicationChannel + jobid=jobid + ) + } + if (wrapperMode == "args") { + swift:execute( + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + "_swiftwrap.wrapperstaging", + jobid, + "-urlprefix", URL_PREFIX, + "-jobdir", jobdir, + "-scratch", scratch, + "-e", vdl:executable(tr, rhost), + "-out", stdout, + "-err", stderr, + "-i", if (stdin != null, getFieldValue(stdin)), + "-d", str:join(remoteFileDirs, "|"), + "-if", str:join(remoteFileNames(inFiles), "|"), + "-of", str:join(remoteFileNames(outFiles), "|"), + "-wt", WRAPPERLOG_ALWAYS_TRANSFER, + "-sk", SITEDIR_KEEP, + "-cdmfile", cdm:file(), + "-status", statusMode, + "-a", if (arguments != null, each(arguments)) ) - } - - setProgress(progress, "Checking status") - if (statusMode == "files") { - checkJobStatus(jobdir, jobid, tr) - } + directory=wfdir + redirect=false + host=rhost + TCProfile(rhost, if(attributes != null, attributes = attributes), tr=tr) + replicationGroup=replicationGroup + replicationChannel=replicationChannel + jobid=jobid + ) + } + + setProgress(progress, "Checking status") + if (statusMode == "files") { + checkJobStatus(jobdir, jobid, tr) + } - if (wrapperMode == "files") { - file:remove(wrapfile) - } + if (wrapperMode == "files") { + file:remove(wrapfile) + } - log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") + log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") - /* need to stage the files to upper scratch area in case they are not transfered to another site - before all the files get cleaned out */ + /* need to stage the files to upper scratch area in case they are not transfered to another site + before all the files get cleaned out */ - setProgress(progress, "Stage out") - doRestartlog(stageout) - - log(LOG:DEBUG, "JOB_END jobid={jobid}") + setProgress(progress, "Stage out") + doRestartlog(stageout) + + log(LOG:DEBUG, "JOB_END jobid={jobid}") + } + else catch(prev) { + if (matches(prev, "^Abort$")) { + log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") + throw(prev) } - else catch(prev) { - if (matches(prev, "^Abort$")) { - log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") - throw(prev) + else { + setProgress(progress, "Failed but can retry") + exception := try(exception(checkErrorFile(jobdir, jobid)), prev) + + log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + + if (matches(exception,".*executable bit.*")) { + throw(exception) } - else { - setProgress(progress, "Failed but can retry") - exception := try(exception(checkErrorFile(jobdir, jobid)), prev) - log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) - - if (matches(exception,".*executable bit.*")) { - throw(exception) - } - - outs := readStandardFiles(jobdir, stdout, stderr) - - throw( - exception( - concat( - "Exception in {tr}:", - if (arguments != null, "\n Arguments: {arguments}") - "\n Host: {rhost}", - "\n Directory: {tmpdir}", - "{outs}", - ) - exception + outs := readStandardFiles(jobdir, stdout, stderr) + + throw( + exception( + concat( + "Exception in {tr}:", + if (arguments != null, "\n Arguments: {arguments}") + "\n Host: {rhost}", + "\n Directory: {tmpdir}", + "{outs}", ) + exception ) - } + ) } } } Modified: trunk/libexec/swift-int.k =================================================================== --- trunk/libexec/swift-int.k 2014-05-09 14:35:51 UTC (rev 7850) +++ trunk/libexec/swift-int.k 2014-05-10 18:05:51 UTC (rev 7851) @@ -10,6 +10,14 @@ SWIFT:HOME := contextAttribute("SWIFT:HOME") SWIFT:DEBUG_DIR_PREFIX := contextAttribute("SWIFT:DEBUG_DIR_PREFIX") +RUN_DIR := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" +SHARED_DIR := dircat(RUN_DIR, "shared") +DEBUG_DIR := "{SWIFT:DEBUG_DIR_PREFIX}{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d" + +if (!file:exists(DEBUG_DIR)) { + task:dir:make(DEBUG_DIR) +} + namespace(swift) { rmdir := function(dir, host) { @@ -32,12 +40,12 @@ dir:make(dc, host=host) } - checkErrorFile := function(rhost, wfdir, jobid, jobdir) { - if (file:exists("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost)) { + checkErrorFile := function(rhost, jobid, jobdir) { + if (file:exists("{RUN_DIR}/status/{jobdir}/{jobid}-error", host=rhost)) { log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found") - task:transfer("{jobid}-error", srchost=rhost, srcdir="{wfdir}/status/{jobdir}") + task:transfer("{jobid}-error", srchost=rhost, srcdir="{RUN_DIR}/status/{jobdir}") error := parallel( - file:remove("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost) + file:remove("{RUN_DIR}/status/{jobdir}/{jobid}-error", host=rhost) sequential( str:strip(file:read("{jobid}-error")) file:remove("{jobid}-error") @@ -51,15 +59,15 @@ } } - checkJobStatus := function(rhost, wfdir, jobid, tr, jobdir) { + checkJobStatus := function(rhost, jobid, tr, jobdir) { log(LOG:DEBUG, "START jobid={jobid}") try { - file:remove("{wfdir}/status/{jobdir}/{jobid}-success", host=rhost) + file:remove("{RUN_DIR}/status/{jobdir}/{jobid}-success", host=rhost) log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") } else { msg := try { - checkErrorFile(rhost, wfdir, jobid, jobdir) + checkErrorFile(rhost, jobid, jobdir) } else { log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing") @@ -74,43 +82,29 @@ setProgress(progress, "Initializing site shared directory") log(LOG:INFO, "START host={rhost} - Initializing shared directory") - wfdir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" - sharedDir := dircat(wfdir, "shared") + dir:make(SHARED_DIR, host = rhost) + transfer(siteProfile(rhost, "swift:wrapperScript"), srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) + transfer("_swiftseq", srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) + dir:make(dircat(RUN_DIR, "kickstart"), host=rhost) - dir:make(sharedDir, host = rhost) - transfer(siteProfile(rhost, "swift:wrapperScript"), srcdir="{SWIFT:HOME}/libexec/", destdir=sharedDir, desthost=rhost) - transfer("_swiftseq", srcdir="{SWIFT:HOME}/libexec/", destdir=sharedDir, desthost=rhost) - dir:make(dircat(wfdir, "kickstart"), host=rhost) - - statusMode := configProperty("status.mode",host=rhost) + statusMode := configProperty("status.mode", host=rhost) if (statusMode == "files") { - dir:make(dircat(wfdir, "status"), host=rhost) + dir:make(dircat(RUN_DIR, "status"), host=rhost) } - wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) + wrapperMode := configProperty("wrapper.parameter.mode", host=rhost) if (wrapperMode == "files") { - dir:make(dircat(wfdir, "parameters"), host=rhost) + dir:make(dircat(RUN_DIR, "parameters"), host=rhost) } - dir:make(dircat(wfdir, "info"), host=rhost) - wfdir, sharedDir + dir:make(dircat(RUN_DIR, "info"), host=rhost) + //we send the cleanup data to vdl:main() - to(cleanup, list(wfdir, rhost)) + to(cleanup, list(RUN_DIR, rhost)) log(LOG:INFO, "END host={rhost} - Done initializing shared directory") } } - initDDir := function() { - ddir := "{SWIFT:DEBUG_DIR_PREFIX}{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d" - - once(ddir) { - if(!file:exists(ddir)) { - task:dir:make(ddir) - } - } - ddir - } - createDirSet := function(jobid, destdir, host, dirs) { /* * Ideally this would be done by creating a tree of the directories @@ -308,11 +302,11 @@ } - transferWrapperLog := function(rhost, wfdir, jobid, jobdir) { + transferWrapperLog := function(rhost, jobid, jobdir) { recfile := "{jobid}-info" - srcdir := dircat("{wfdir}/info/", jobdir) + srcdir := dircat("{RUN_DIR}/info/", jobdir) try { - task:transfer(recfile, srchost=rhost, srcdir=srcdir, destdir="{SWIFT:DEBUG_DIR_PREFIX}{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d/") + task:transfer(recfile, srchost=rhost, srcdir=srcdir, destdir=DEBUG_DIR) } else catch (exception) { maybe(file:remove(recfile)) @@ -339,196 +333,192 @@ ) export(execute2, - function(progress, tr, stagein, stageout, + function(rhost, progress, tr, stagein, stageout, replicationGroup, replicationChannel, arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { - - allocateHost(rhost, constraints = jobConstraints(tr, stagein = stagein)) { - ddir := initDDir(), - (wfdir, sharedDir) := - try { - initSharedDir(progress, rhost) - } - else catch(exception) { - throw(exception("Could not initialize shared directory on {rhost}", exception)) - } - - uid := UID() - jobdir := substring(uid, 0, to=1) - jobid := "{tr}-{uid}" - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") + try { + initSharedDir(progress, rhost) + } + else catch(exception) { + throw(exception("Could not initialize shared directory on {rhost}", exception)) + } + + uid := UID() + jobdir := substring(uid, 0, to=1) + jobid := "{tr}-{uid}" - statusMode := configProperty("status.mode",host=rhost) - wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") - wrapfile := "{ddir}/param-{jobid}" + statusMode := configProperty("status.mode",host=rhost) + wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) - stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout)) - stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)), - - (fileDirs, inFiles, outFiles, outCollect) := getStagingInfo(stagein, stageout) + wrapfile := "{DEBUG_DIR}/param-{jobid}" - os := siteProfile(rhost, "SYSINFO:OS") - - if(wrapperMode == "files") { - sys:file:write(wrapfile) { - "-scratch ", try(siteProfile(rhost, "scratch"), ""), - "\n-e ", executable(tr, rhost), - "\n-out ", stdout, - "\n-err ", stderr, - "\n-i ", if (stdin != null, getFieldValue(stdin)), - "\n-d ", str:join(fileDirs, "|"), - "\n-if ", str:join(remoteFileNames(inFiles), "|"), - "\n-of ", str:join(remoteFileNames(outFiles), "|"), - "\n-cf ", str:join(remoteFileNames(outCollect), "|"), - "\n-cdmfile ", cdm:file(), - "\n-status ", statusMode, - for(a, arguments) { - "\n-a ", a - } + stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout)) + stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)), + + (fileDirs, inFiles, outFiles, outCollect) := getStagingInfo(stagein, stageout) + + os := siteProfile(rhost, "SYSINFO:OS") + + if (wrapperMode == "files") { + sys:file:write(wrapfile) { + "-scratch ", try(siteProfile(rhost, "scratch"), ""), + "\n-e ", executable(tr, rhost), + "\n-out ", stdout, + "\n-err ", stderr, + "\n-i ", if (stdin != null, getFieldValue(stdin)), + "\n-d ", str:join(fileDirs, "|"), + "\n-if ", str:join(remoteFileNames(inFiles), "|"), + "\n-of ", str:join(remoteFileNames(outFiles), "|"), + "\n-cf ", str:join(remoteFileNames(outCollect), "|"), + "\n-cdmfile ", cdm:file(), + "\n-status ", statusMode, + for(a, arguments) { + "\n-a ", a } } + } - setProgress(progress, "Stage in") - tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) + setProgress(progress, "Stage in") + tmpdir := dircat("{RUN_DIR}/jobs/{jobdir}", jobid) - try { - createDirSet(jobid, sharedDir, rhost, fileDirs) - doStagein(jobid, sharedDir, rhost, inFiles) - if(wrapperMode == "files") { - stageWrapperParams(jobid, jobdir, wrapfile, wfdir, rhost) - } + try { + createDirSet(jobid, SHARED_DIR, rhost, fileDirs) + doStagein(jobid, SHARED_DIR, rhost, inFiles) + if (wrapperMode == "files") { + stageWrapperParams(jobid, jobdir, wrapfile, RUN_DIR, rhost) + } - log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), " tmpdir={tmpdir} host={rhost}") + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), " tmpdir={tmpdir} host={rhost}") - setProgress(progress, "Submitting") + setProgress(progress, "Submitting") - if (wrapperMode == "files") { - swift:execute( - progress, - siteProfile(rhost, "swift:wrapperInterpreter"), - list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), - dircat("shared", siteProfile(rhost, "swift:wrapperScript"), os=os), - jobid, "-p", jobdir - ) - directory = wfdir - redirect = false - host = rhost - TCProfile(rhost, if (attributes != null, attributes = attributes), tr = tr) - replicationGroup = replicationGroup - replicationChannel = replicationChannel - jobid = jobid + if (wrapperMode == "files") { + swift:execute( + progress, + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + dircat("shared", siteProfile(rhost, "swift:wrapperScript"), os=os), + jobid, "-p", jobdir ) - } - else if (wrapperMode == "args") { - swift:execute( - progress, - siteProfile(rhost, "swift:wrapperInterpreter"), - list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), - dircat("shared", siteProfile(rhost, "swift:wrapperScript"), os=os), - jobid, - "-jobdir", jobdir, - "-scratch", try(siteProfile(rhost, "scratch"), "") - "-e", executable(tr, rhost), - "-out", stdout, - "-err", stderr, - "-i", if (stdin != null, getFieldValue(stdin)), - "-d", str:join(fileDirs, "|"), - "-if", str:join(remoteFileNames(inFiles), "|"), - "-of", str:join(remoteFileNames(outFiles), "|"), - "-cf", str:join(remoteFileNames(outCollect), "|"), - "-cdmfile", cdm:file(), - "-status", statusMode, - "-a", if (arguments != null, each(arguments)) - ) - directory = wfdir - redirect = false - host = rhost - TCProfile(rhost, if(attributes != null, attributes = attributes), tr = tr) - replicationGroup = replicationGroup - replicationChannel = replicationChannel - jobid = jobid + directory = RUN_DIR + redirect = false + host = rhost + TCProfile(rhost, if (attributes != null, attributes = attributes), tr = tr) + replicationGroup = replicationGroup + replicationChannel = replicationChannel + jobid = jobid + ) + } + else if (wrapperMode == "args") { + swift:execute( + progress, + siteProfile(rhost, "swift:wrapperInterpreter"), + list( + siteProfile(rhost, "swift:wrapperInterpreterOptions"), + dircat("shared", siteProfile(rhost, "swift:wrapperScript"), os=os), + jobid, + "-jobdir", jobdir, + "-scratch", try(siteProfile(rhost, "scratch"), "") + "-e", executable(tr, rhost), + "-out", stdout, + "-err", stderr, + "-i", if (stdin != null, getFieldValue(stdin)), + "-d", str:join(fileDirs, "|"), + "-if", str:join(remoteFileNames(inFiles), "|"), + "-of", str:join(remoteFileNames(outFiles), "|"), + "-cf", str:join(remoteFileNames(outCollect), "|"), + "-cdmfile", cdm:file(), + "-status", statusMode, + "-a", if (arguments != null, each(arguments)) ) - } + directory = RUN_DIR + redirect = false + host = rhost + TCProfile(rhost, if(attributes != null, attributes = attributes), tr = tr) + replicationGroup = replicationGroup + replicationChannel = replicationChannel + jobid = jobid + ) + } - setProgress(progress, "Checking status") - if (statusMode == "files") { - checkJobStatus(rhost, wfdir, jobid, tr, jobdir) - } + setProgress(progress, "Checking status") + if (statusMode == "files") { + checkJobStatus(rhost, jobid, tr, jobdir) + } - if (wrapperMode == "files") { - file:remove(wrapfile) - } + if (wrapperMode == "files") { + file:remove(wrapfile) + } - log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") + log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") - /* need to stage the files to upper scratch area in case they are not transfered to another site - before all the files get cleaned out */ + /* need to stage the files to upper scratch area in case they are not transfered to another site + before all the files get cleaned out */ - setProgress(progress, "Stage out") - if (isEmpty(outCollect)) { - doStageout(jobid, sharedDir, rhost, outFiles) - [] // empty collect list - } - else { - doStageoutCollect(jobid, sharedDir, rhost, outFiles) - } + setProgress(progress, "Stage out") + if (isEmpty(outCollect)) { + doStageout(jobid, SHARED_DIR, rhost, outFiles) + [] // empty collect list + } + else { + doStageoutCollect(jobid, SHARED_DIR, rhost, outFiles) + } + + if (configProperty("wrapperlog.always.transfer") == "true") { + discard(transferWrapperLog(rhost, RUN_DIR, jobid, jobdir)) + } + + cacheUnlockFiles(inFiles, SHARED_DIR, rhost) { + cleanupFiles(cacheFilesToRemove, rhost) + } - if (configProperty("wrapperlog.always.transfer") == "true") { - discard(transferWrapperLog(rhost, wfdir, jobid, jobdir)) - } - - cacheUnlockFiles(inFiles, sharedDir, rhost) { + log(LOG:DEBUG, "JOB_END jobid={jobid}") + } + else catch(prev) { + if (matches(prev, "^Abort$")) { + log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") + cacheUnlockFiles(inFiles, SHARED_DIR, rhost, force=false) { cleanupFiles(cacheFilesToRemove, rhost) } - - log(LOG:DEBUG, "JOB_END jobid={jobid}") + throw(prev) } - else catch(prev) { - if (matches(prev, "^Abort$")) { - log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}") - cacheUnlockFiles(inFiles, sharedDir, rhost, force=false) { - cleanupFiles(cacheFilesToRemove, rhost) - } - throw(prev) + else { + setProgress(progress, "Failed but can retry") + exception := try(exception(checkErrorFile(rhost, jobid, jobdir), prev), prev) + + log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) + + if (matches(exception,".*executable bit.*")) { + throw(exception) } - else { - setProgress(progress, "Failed but can retry") - exception := try(exception(checkErrorFile(rhost, wfdir, jobid, jobdir), prev), prev) - - log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception) - if (matches(exception,".*executable bit.*")) { - throw(exception) - } + cacheUnlockFiles(inFiles, SHARED_DIR, rhost, force=false) { + cleanupFiles(cacheFilesToRemove, rhost) + } - cacheUnlockFiles(inFiles, sharedDir, rhost, force=false) { - cleanupFiles(cacheFilesToRemove, rhost) - } + outs := transferStandardFiles(rhost, tmpdir, jobid, stdout, stderr) - outs := transferStandardFiles(rhost, tmpdir, jobid, stdout, stderr) + discard(maybe(transferWrapperLog(rhost, jobid, jobdir))) - discard(maybe(transferWrapperLog(rhost, wfdir, jobid, jobdir))) - - - throw( - exception( - concat( - "Exception in {tr}:", - if (arguments != null, "\n Arguments: {arguments}") - "\n Host: {rhost}", - "\n Directory: {tmpdir}", - "{outs}", - ) - exception + + throw( + exception( + concat( + "Exception in {tr}:", + if (arguments != null, "\n Arguments: {arguments}") + "\n Host: {rhost}", + "\n Directory: {tmpdir}", + "{outs}", ) + exception ) - } + ) } } } Modified: trunk/libexec/swift-lib.k =================================================================== --- trunk/libexec/swift-lib.k 2014-05-09 14:35:51 UTC (rev 7850) +++ trunk/libexec/swift-lib.k 2014-05-10 18:05:51 UTC (rev 7851) @@ -52,6 +52,7 @@ export(initProgressState, def("org.griphyn.vdl.karajan.lib.RuntimeStats$InitProgressState")) export(setProgress, def("org.griphyn.vdl.karajan.lib.RuntimeStats$SetProgress")) export(new, def("org.griphyn.vdl.karajan.lib.New")) + export(field, def("org.griphyn.vdl.karajan.lib.GetFieldConst")) export(createArray, def("org.griphyn.vdl.karajan.lib.CreateArray")) /* used from VDL2 for arguments to apps and returns relative paths */ export(fileName, def("org.griphyn.vdl.karajan.lib.FileName")) Modified: trunk/libexec/swift.k =================================================================== --- trunk/libexec/swift.k 2014-05-09 14:35:51 UTC (rev 7850) +++ trunk/libexec/swift.k 2014-05-10 18:05:51 UTC (rev 7851) @@ -179,7 +179,7 @@ attributes = null, deperror = false, mdeperror = false, channel(stagein), channel(stageout)) { - + progress := initProgressState() done := isDone(stageout) @@ -199,12 +199,14 @@ replicationGroup := UID() parallelFor(i, replicationChannel) { try { - execute2( - progress, - tr, if(arguments != null, arguments = unwrapClosedList(arguments)), - stdin=stdin, stdout=stdout, stderr=stderr, attributes=attributes, - stagein, stageout, replicationGroup, replicationChannel - ) + allocateHost(rhost, constraints = jobConstraints(tr, stagein = stagein)) { + execute2( + rhost, progress, + tr, if(arguments != null, arguments = unwrapClosedList(arguments)), + stdin=stdin, stdout=stdout, stderr=stderr, attributes=attributes, + stagein, stageout, replicationGroup, replicationChannel + ) + } } else catch(exception) { if (matches(exception, "^Abort$")) { @@ -218,12 +220,14 @@ } else { try { - execute2( - progress, - tr, if(arguments != null, arguments = unwrapClosedList(arguments)), - stdin=stdin, stdout=stdout, stderr=stderr, attributes=attributes, - stagein, stageout, null, null - ) + allocateHost(rhost, constraints = jobConstraints(tr, stagein = stagein)) { + execute2( + rhost, progress, + tr, if(arguments != null, arguments = unwrapClosedList(arguments)), + stdin=stdin, stdout=stdout, stderr=stderr, attributes=attributes, + stagein, stageout, null, null + ) + } } else catch(exception) { if (matches(exception, "^Abort$")) { From ketan at ci.uchicago.edu Sun May 11 14:32:50 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Sun, 11 May 2014 14:32:50 -0500 (CDT) Subject: [Swift-commit] r7852 - SwiftApps/swift-galaxy/swift Message-ID: <20140511193250.1B45C9D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-11 14:32:48 -0500 (Sun, 11 May 2014) New Revision: 7852 Added: SwiftApps/swift-galaxy/swift/swiftforeach.py Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh Log: translate python driver script to python Added: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py (rev 0) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-11 19:32:48 UTC (rev 7852) @@ -0,0 +1,191 @@ +#!/usr/bin/env python + +import subprocess +import sys +import os +import distutils. + +def setwdir(): + return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) + +def genargs(args): + for a in args: + yield a + +def buildcfg(): + f=open(wdir+"/cf", 'w'); + f.write('use.provider.staging=true') + f.write('wrapperlog.always.transfer=true') + f.write('execution.retries=0') + f.write('provider.staging.pin.swiftfiles=false') + f.write('sitedir.keep=true') + f.write('lazy.errors=false') + f.close() + +# numeric range +def main(): + myargs = genargs(sys.argv) + try: + range_or_list = next(myargs) + k_or_t = next(myargs) + sites = next(myargs) + swiftargs = next(myargs) + interpreter = next(myargs) + executable = next(myargs) + stdin = next(myargs) + + if range_or_list == "list": + listfile = next(myargs) + n = sum(1 for line in open(listfile)) + else: + rstart = next(myargs) + rend = next(myargs) + stepsize = next(myargs) + + rstart = next(myargs) + rend = next(myargs) + stepsize = next(myargs) + outloc = next(myargs) + outlistfile = next(myargs) + logfile = next(myargs) + stringargs = next(myargs) + except: + sys.exit(1) + + swift = distutils.spawn.find_executable("swift") + BASEDIR = os.path.dirname(os.path.realpath(__file__)) + + # workout the file args + fileargs=[] + for f in myargs: + fileargs.append(f) + + #workout the file array only if user adds args else make it blank + if fileargs: + if k_or_t == "swiftK": + filearrayexpr="file fileargs[] ;" + else: #T + filearrayexpr="file fileargs[]="+str(fileargs)+";\n" + else: + filearrayexpr="file fileargs[];" + + # create outloc dir + if not os.path.exists(directory): + os.makedirs(outloc) + + # workout the stringargs + if stringargs: + stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n" + else: + stringarrayexpr="string stringargs[];\n" + + wdir = setwdir().rstrip() + + #workout standard input file + if stdin: + if k_or_t == "swiftK": + stdinfilexpr = "file stdinfile<"+stdin+">;" + else: + stdinfilexpr = "file stdinfile = input("+stdin+");" + else: + if k_or_t == "swiftK": + stdinfilexpr = "file stdinfile<\"/dev/null\">;" + else: + stdinfilexpr = "file stdinfile = input(\"/dev/null\");" + + + #Build Swift source code + f = open(wdir+"/script.swift", 'w'); + + if k_or_t == "swiftK": + + f.write("type file; \n") + f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n") + f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n") + f.write("}\n") + f.write(stringarrayexpr) + f.write(filearrayexpr) + f.write("file exec<'"+executable+"'>;\n") + + if range_or_list == "list": + + f.write("file listfile<"+listfile+">;\n") + f.write("string list[] = readData(listfile);\n"); + f.write("foreach l, i in list{\n") + f.write("file out ;\n") + f.write("file err ;\n") + f.write("(out,err) = anapp(exec, i, stringargs, fileargs, stdinfile\n"); + + else: + + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n") + + f.write("}\n") + + #build site + if os.path.dirname(sys.argv[0]): + basedir=os.path.dirname(sys.argv[0]) + else: + basedir='.' + print "%s %s %s" %(basedir, sites, wdir) + subprocess.call([basedir+"/buildsite.sh", sites, wdir]) + + #build config + buildcfg() + + #build tc + tc=open(wdir+"/tc", 'w'); + for site in sites.split(): + tc.write(site+' '+interpreter+' '+interpreter) + + tc.close() + + else: # T + + f.write("import files;\n"); + f.write("import string;\n"); + f.write("import io;\n"); + f.write("\n"); + f.write(" app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){\n"); + f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n"); + f.write("}\n"); + f.write("main{\n"); + f.write(stringarrayexpr + " \n"); + f.write(filearrayexpr + " \n"); + f.write(stdinfilexpr + " \n"); + f.write("file exec = input("+executable+");\n"); + + if range_or_list == "list": + + f.write(" \n"); + f.write("file listfile=input("$listfile");\n") + f.write("string list[] = split(read(listfile),);\n"); + f.write("foreach l, i in list{"\n) + f.write(" file out ;\n") + f.write(" file err ;\n") + + else: + + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{ \n"); + f.write(" file out ; \n"); + f.write(" file err ; \n"); + + f.write("(out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") + f.write("}\n"); + f.write("printf("%s", "Done!");\n"); + f.write("}\n"); + + f.close() + + #Populate output file + outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) + f=open(outlistfile) + f.write(outlist); + f.close() + #dum ditty dum ditty dum dum dum + +if __name__=='__main__': + main() Property changes on: SwiftApps/swift-galaxy/swift/swiftforeach.py ___________________________________________________________________ Added: svn:executable + * Modified: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-10 18:05:51 UTC (rev 7851) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-11 19:32:48 UTC (rev 7852) @@ -92,8 +92,9 @@ filearrayexpr='file fileargs[];' fi +#build swift source if [ "$range_or_list" == "list" ] ; then ## list - + cat << EOF > $wdir/script.swift /* * From ketan at ci.uchicago.edu Sun May 11 18:09:31 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Sun, 11 May 2014 18:09:31 -0500 (CDT) Subject: [Swift-commit] r7853 - SwiftApps/swift-galaxy/swift Message-ID: <20140511230931.E9C509D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-11 18:09:31 -0500 (Sun, 11 May 2014) New Revision: 7853 Added: SwiftApps/swift-galaxy/swift/testpy.sh Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py Log: test and fixes for py Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-11 19:32:48 UTC (rev 7852) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-11 23:09:31 UTC (rev 7853) @@ -3,7 +3,8 @@ import subprocess import sys import os -import distutils. +import distutils.spawn +import traceback def setwdir(): return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) @@ -12,7 +13,7 @@ for a in args: yield a -def buildcfg(): +def buildcfg(wdir): f=open(wdir+"/cf", 'w'); f.write('use.provider.staging=true') f.write('wrapperlog.always.transfer=true') @@ -26,13 +27,21 @@ def main(): myargs = genargs(sys.argv) try: + this=next(myargs) range_or_list = next(myargs) + #print "range or list=> "+ range_or_list k_or_t = next(myargs) + #print "k or t=> "+ k_or_t sites = next(myargs) + #print "sites=> " + sites swiftargs = next(myargs) + #print "swiftargs=> " + swiftargs interpreter = next(myargs) + #print "interpreter=> " + interpreter executable = next(myargs) + #print "executable=> " + executable stdin = next(myargs) + #print "stdin => " + stdin if range_or_list == "list": listfile = next(myargs) @@ -41,17 +50,15 @@ rstart = next(myargs) rend = next(myargs) stepsize = next(myargs) - - rstart = next(myargs) - rend = next(myargs) - stepsize = next(myargs) + outloc = next(myargs) outlistfile = next(myargs) + #print "outlistfile=>"+outlistfile logfile = next(myargs) stringargs = next(myargs) except: + traceback.print_exc() sys.exit(1) - swift = distutils.spawn.find_executable("swift") BASEDIR = os.path.dirname(os.path.realpath(__file__)) @@ -70,7 +77,7 @@ filearrayexpr="file fileargs[];" # create outloc dir - if not os.path.exists(directory): + if not os.path.exists(outloc): os.makedirs(outloc) # workout the stringargs @@ -134,7 +141,7 @@ subprocess.call([basedir+"/buildsite.sh", sites, wdir]) #build config - buildcfg() + buildcfg(wdir) #build tc tc=open(wdir+"/tc", 'w'); @@ -161,9 +168,9 @@ if range_or_list == "list": f.write(" \n"); - f.write("file listfile=input("$listfile");\n") + f.write("file listfile=input(\""+listfile+"\");\n") f.write("string list[] = split(read(listfile),);\n"); - f.write("foreach l, i in list{"\n) + f.write("foreach l, i in list{\n") f.write(" file out ;\n") f.write(" file err ;\n") @@ -175,14 +182,14 @@ f.write("(out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") f.write("}\n"); - f.write("printf("%s", "Done!");\n"); + f.write("printf(\"%s\", \"Done!\");\n"); f.write("}\n"); f.close() #Populate output file outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) - f=open(outlistfile) + f=open(outlistfile, "w") f.write(outlist); f.close() #dum ditty dum ditty dum dum dum Added: SwiftApps/swift-galaxy/swift/testpy.sh =================================================================== --- SwiftApps/swift-galaxy/swift/testpy.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/testpy.sh 2014-05-11 23:09:31 UTC (rev 7853) @@ -0,0 +1,41 @@ +#!/bin/bash + +#!/bin/bash + +cat << EOF > list.txt +a +b +c +EOF + +cat << EOF > x.sh +#!/bin/bash + +echo \$\@ +EOF + +chmod 755 x.sh + +echo "List, no input parameters and files" +./swiftforeach.py list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" + +echo "Range, no input parameters and files" +./swiftforeach.py range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" + +echo "List, string input parameters" +./swiftforeach.py range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there list!" + +echo "Range, string input parameters" +./swiftforeach.py range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hi there range!" + +echo "List, no input parameters and files stdin /dev/null" +./swiftforeach.py list swiftK localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" + +echo "T list" +./swiftforeach.py list swiftT localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" + +echo "T range" +./swiftforeach.py range swiftT localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" + +echo "Call from Galaxy" +python /nfs2/ketan/galaxy-dist/tools/swift/swiftforeach.py "range" "swiftT" "localhost" "" "sh" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_1.dat" "None" "0" "9" "1" "$HOME/swift-sandbox" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_12.dat" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_13.dat" "hello how are you" Property changes on: SwiftApps/swift-galaxy/swift/testpy.sh ___________________________________________________________________ Added: svn:executable + * From ketan at ci.uchicago.edu Sun May 11 22:08:22 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Sun, 11 May 2014 22:08:22 -0500 (CDT) Subject: [Swift-commit] r7854 - SwiftApps/swift-galaxy/swift Message-ID: <20140512030822.0453A9D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-11 22:08:21 -0500 (Sun, 11 May 2014) New Revision: 7854 Modified: SwiftApps/swift-galaxy/swift/buildsite.sh SwiftApps/swift-galaxy/swift/swiftforeach.py Log: fixes in python and test for T Modified: SwiftApps/swift-galaxy/swift/buildsite.sh =================================================================== --- SwiftApps/swift-galaxy/swift/buildsite.sh 2014-05-11 23:09:31 UTC (rev 7853) +++ SwiftApps/swift-galaxy/swift/buildsite.sh 2014-05-12 03:08:21 UTC (rev 7854) @@ -116,10 +116,11 @@ localhost ) cat << EOF >> $wdir/pool.xml - + 0.07 10000 + local /tmp/swift.work EOF @@ -129,10 +130,11 @@ echo "defaulting to localhost" cat << EOF >> $wdir/pool.xml - + 0.07 10000 + local /tmp/swift.work EOF Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-11 23:09:31 UTC (rev 7853) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 03:08:21 UTC (rev 7854) @@ -5,6 +5,7 @@ import os import distutils.spawn import traceback +import fnmatch def setwdir(): return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) @@ -15,12 +16,12 @@ def buildcfg(wdir): f=open(wdir+"/cf", 'w'); - f.write('use.provider.staging=true') - f.write('wrapperlog.always.transfer=true') - f.write('execution.retries=0') - f.write('provider.staging.pin.swiftfiles=false') - f.write('sitedir.keep=true') - f.write('lazy.errors=false') + f.write("use.provider.staging=true\n") + f.write("wrapperlog.always.transfer=true\n") + f.write("execution.retries=0\n") + f.write("provider.staging.pin.swiftfiles=false\n") + f.write("sitedir.keep=true\n") + f.write("lazy.errors=false\n") f.close() # numeric range @@ -50,6 +51,7 @@ rstart = next(myargs) rend = next(myargs) stepsize = next(myargs) + n = (rend - rstart)/stepsize outloc = next(myargs) outlistfile = next(myargs) @@ -70,11 +72,11 @@ #workout the file array only if user adds args else make it blank if fileargs: if k_or_t == "swiftK": - filearrayexpr="file fileargs[] ;" + filearrayexpr="file fileargs[] ;\n" else: #T filearrayexpr="file fileargs[]="+str(fileargs)+";\n" else: - filearrayexpr="file fileargs[];" + filearrayexpr="file fileargs[];\n" # create outloc dir if not os.path.exists(outloc): @@ -90,15 +92,19 @@ #workout standard input file if stdin: + #print "stdin" if k_or_t == "swiftK": - stdinfilexpr = "file stdinfile<"+stdin+">;" + #print "stdinK" + stdinfilexpr = "file stdinfile<"+stdin+">;\n" else: - stdinfilexpr = "file stdinfile = input("+stdin+");" + #print "stdinT" + stdinfilexpr = "file stdinfile = input("+stdin+");\n" else: + #print "no stdin" if k_or_t == "swiftK": - stdinfilexpr = "file stdinfile<\"/dev/null\">;" + stdinfilexpr = "file stdinfile<\"/dev/null\">;\n" else: - stdinfilexpr = "file stdinfile = input(\"/dev/null\");" + stdinfilexpr = "file stdinfile = input(\"/dev/null\");\n" #Build Swift source code @@ -107,29 +113,29 @@ if k_or_t == "swiftK": f.write("type file; \n") - f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n") - f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n") + f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ \n") + f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; \n") f.write("}\n") f.write(stringarrayexpr) f.write(filearrayexpr) - f.write("file exec<'"+executable+"'>;\n") + f.write(stdinfilexpr) + f.write("file exec<\""+executable+"\">;\n") if range_or_list == "list": - f.write("file listfile<"+listfile+">;\n") + f.write("file listfile<\""+listfile+"\">;\n") f.write("string list[] = readData(listfile);\n"); f.write("foreach l, i in list{\n") f.write("file out ;\n") f.write("file err ;\n") - f.write("(out,err) = anapp(exec, i, stringargs, fileargs, stdinfile\n"); else: f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") f.write(" file out ;\n") f.write(" file err ;\n") - f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n") + f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") f.write("}\n") #build site @@ -137,7 +143,7 @@ basedir=os.path.dirname(sys.argv[0]) else: basedir='.' - print "%s %s %s" %(basedir, sites, wdir) + #print "%s %s %s" %(basedir, sites, wdir) subprocess.call([basedir+"/buildsite.sh", sites, wdir]) #build config @@ -187,12 +193,79 @@ f.close() + + #Invocation + if k_or_t == "swiftK": + + #$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile + with open(logfile,"wb") as errfile: + exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile) + print exitcode + + else: + + #print "Invoking T" + #EXITCODE=0 + + if sites == "localhost": + + print "localhost" + stc=distutils.spawn.find_executable("stc") + turbine= distutils.spawn.find_executable("turbine") + with open(logfile, "wb") as errfile: + exitcode=subprocess.check_call([stc, "-j", "$(\which java)", wdir+"/script.swift"], stderr=errfile) + + if exitcode == 0: + with open(logfile, "wb") as errfile: + exitcode=subprocess.check_call([turbine, "-n", n, wdir+"/script.swift"], stderr=errfile) + + + elif sites == "Vesta": + + print "vesta" + + elif sites == "Mira": + + print "mira" + + else: + + print "site not recognized" + #STC=$(\which stc) + ##echo "got stc" >> /tmp/status.txt + #turbine=$(\which turbine) + ##echo "got turbine" >> /tmp/status.txt + #$STC -j $(\which java) $wdir/script.swift 2>$logfile + ##echo "done running stc" >> /tmp/status.txt + ##echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt + #$turbine -n $n $wdir/script.tcl "${swiftargs}" > /tmp/status.txt + #EXITCODE=$? + ##echo "exit code obtained" >> /tmp/status.txt + #;; + #"Vesta" ) + #TURBINE_HOME=/home/wozniak/Public/sfw/turbine + #STC=/home/wozniak/Public/sfw/stc/bin/stc + #$STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile + #$TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl + #EXITCODE=$? + #;; + #"Mira" ) + #TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine + #STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc + #$STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile + #$TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl + #EXITCODE=$? + #;; + #esac + #Populate output file outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) f=open(outlistfile, "w") f.write(outlist); f.close() - #dum ditty dum ditty dum dum dum + + if __name__=='__main__': main() From wozniak at ci.uchicago.edu Mon May 12 09:43:29 2014 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Mon, 12 May 2014 09:43:29 -0500 (CDT) Subject: [Swift-commit] r7855 - www/papers Message-ID: <20140512144329.D0A859CCF4@svn.ci.uchicago.edu> Author: wozniak Date: 2014-05-12 09:43:28 -0500 (Mon, 12 May 2014) New Revision: 7855 Added: www/papers/Wozniak-KISTI-report_2013.pdf Log: Adding Wozniak-KISTI-report_2013.pdf Added: www/papers/Wozniak-KISTI-report_2013.pdf =================================================================== (Binary files differ) Property changes on: www/papers/Wozniak-KISTI-report_2013.pdf ___________________________________________________________________ Added: svn:mime-type + application/octet-stream From davidk at ci.uchicago.edu Mon May 12 11:01:47 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 12 May 2014 11:01:47 -0500 (CDT) Subject: [Swift-commit] r7856 - branches/release-0.94/docs/siteguide Message-ID: <20140512160147.6A3929D587@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-12 11:01:47 -0500 (Mon, 12 May 2014) New Revision: 7856 Added: branches/release-0.94/docs/siteguide/blues Modified: branches/release-0.94/docs/siteguide/fusion branches/release-0.94/docs/siteguide/siteguide.txt Log: Add a site guide entries for blues, update fusion guide Added: branches/release-0.94/docs/siteguide/blues =================================================================== --- branches/release-0.94/docs/siteguide/blues (rev 0) +++ branches/release-0.94/docs/siteguide/blues 2014-05-12 16:01:47 UTC (rev 7856) @@ -0,0 +1,126 @@ +Blues (x86 cluster) +------------------- + +Fusion is a 310-node computing cluster for the Argonne +National Laboratory community. The primary goal of the LCRC is to +facilitate mid-range computing in all of the scientific programs of +Argonne and the University of Chicago. + +This section will walk you through running a simple Swift script +on Fusion. + +Requesting Access +~~~~~~~~~~~~~~~~~ +If you do not already have a Blues account, you can request one at +https://accounts.lcrc.anl.gov/request.php. Email support at lcrc.anl.gov +for additional help. + +Projects +~~~~~~~~ +In order to run a job on a Blues compute node, you must first be associated +with a project. + +Each project has one or more Primary Investigators, or PIs. These PIs are +responsible for adding and removing users to a project. Contact the PI of +your project to be added. + +More information on this process can be found at +http://www.lcrc.anl.gov/info/Projects. + +SSH Keys +~~~~~~~~ +Before accessing Blues, be sure to have your SSH keys configured correctly. +SSH keys are required to access fusion. You should see information about +this when you request your account. Email support at lcrc.anl.gov for +additional help. + +Connecting to a login node +~~~~~~~~~~~~~~~~~~~~~~~~~~ +Once your keys are configured, you should be able to access a Fusion login +node with the following command: + +----- +ssh yourusername at blues.lcrc.anl.gov +----- + +Creating sites.xml +~~~~~~~~~~~~~~~~~~ +This section will provide a working configuration file which +you can copy and paste to get running quickly. The sites.xml file +tells Swift how to submit jobs, where working directories are +located, and various other configuration information. More +information on sites.xml can be found in the Swift User's Guide. + +The first step is to paste the text below into a file named sites.xml. + +----- + + + + + + 3600 + 16 + 16 + shared + 1 + 1 + 1 + 1.00 + 10000 + /home/{env.USER}/swiftwork + + +----- + +Creating tc.data +~~~~~~~~~~~~~~~~ +The tc.data configuration file gives information about the applications +that will be called by Swift. More information about the format +of tc.data can be found in the Swift User's guide. + +Paste the following example into a file named tc.data + +----- +blues hostname /bin/hostname +----- + +Create a Swift Script +~~~~~~~~~~~~~~~~~~~~~ +The following script runs /bin/hostname utility 100 times and writes +the outputs. Paste the following script into a file called hostname.swift. + +----- +type file; + +app (file o) hostname () +{ + hostname stdout=@o; +} + +foreach i in [1:100] { + file hostname_log ; + hostname_log = hostname(); +} +----- + +Run Swift +~~~~~~~~~ +Finally, run the script +----- +$ swift -sites.file sites.xml -tc.file tc.data hostname.swift +----- + +When this runs, you should see 100 new text files get created, named *.hostname.log. If +you see these files, then you have succesfully run Swift on Blues! + +Queues +~~~~~~ +Blues has multiple queues available. The command "qstat -q" will print a list. + +To edit the queue, modify the following line in sites.xml + +----- +batch +----- + Modified: branches/release-0.94/docs/siteguide/fusion =================================================================== --- branches/release-0.94/docs/siteguide/fusion 2014-05-12 14:43:28 UTC (rev 7855) +++ branches/release-0.94/docs/siteguide/fusion 2014-05-12 16:01:47 UTC (rev 7856) @@ -1,6 +1,5 @@ Fusion (x86 cluster) ------------------- - Fusion is a 320-node computing cluster for the Argonne National Laboratory community. The primary goal of the LCRC is to facilitate mid-range computing in all of the scientific programs of @@ -55,16 +54,25 @@ The first step is to paste the text below into a file named sites.xml. ----- -include::../../tests/providers/fusion/coasters/sites.template.xml[] + + + + + 3600 + 8 + 1 + 2 + 2 + shared + 5.99 + 10000 + 1000 + 1000 + /homes/{env.USER}/swiftwork + + ----- -This file will require one customization. Create a -directory called swiftwork. Modify \_WORK_ in sites.xml -to point to this new directory. For example ------ -/home/myhome/swiftwork ------ - Creating tc.data ~~~~~~~~~~~~~~~~ The tc.data configuration file gives information about the applications @@ -74,30 +82,36 @@ Paste the following example into a file named tc.data ----- -include::../../tests/providers/fusion/coasters/tc.template.data[] +fusion hostname /bin/hostname ----- -Copy a Swift Script -~~~~~~~~~~~~~~~~~~~~ -Within the Swift directory is an examples directory which contains -several introductory Swift scripts. The example we will use in this -section is called catsn.swift. Copy this script to the same directory -that your sites.xml and tc.data files are located. +Create a Swift Script +~~~~~~~~~~~~~~~~~~~~~ +The following script runs /bin/hostname utility 100 times and writes +the outputs. Paste the following script into a file called hostname.swift. ----- -$ cp ~/swift/examples/misc/catsn.swift . -$ cp ~/swift/examples/misc/data.txt . +type file; + +app (file o) hostname () +{ + hostname stdout=@o; +} + +foreach i in [1:100] { + file hostname_log ; + hostname_log = hostname(); +} ----- -TIP: The location of your swift directory may vary depending on how you installed it. Change this to the examples/misc directory of your installation as needed. Run Swift ~~~~~~~~~ Finally, run the script ----- -$ swift -sites.file sites.xml -tc.file tc.data catsn.swift +$ swift -sites.file sites.xml -tc.file tc.data hostname.swift ----- -You should see 10 new text files get created, named catsn*.out. If +When this runs, you should see 100 new text files get created, named *.hostname.log. If you see these files, then you have succesfully run Swift on Fusion! Queues @@ -115,10 +129,3 @@ More information on Fusion queues can be found at http://www.lcrc.anl.gov/info/BatchJobs. - -More Help -~~~~~~~~~ -The best place for additional help is the Swift user mailing list. You can subscribe to this list at -https://lists.ci.uchicago.edu/cgi-bin/mailman/listinfo/swift-user. When submitting information, please send your sites.xml file, your tc.data, and any Swift log files that were created during your attempt. - - Modified: branches/release-0.94/docs/siteguide/siteguide.txt =================================================================== --- branches/release-0.94/docs/siteguide/siteguide.txt 2014-05-12 14:43:28 UTC (rev 7855) +++ branches/release-0.94/docs/siteguide/siteguide.txt 2014-05-12 16:01:47 UTC (rev 7856) @@ -12,6 +12,8 @@ include::beagle[] +include::blues[] + include::fusion[] include::futuregrid[] From ketan at ci.uchicago.edu Mon May 12 11:05:37 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 11:05:37 -0500 (CDT) Subject: [Swift-commit] r7857 - SwiftApps/swift-galaxy/swift Message-ID: <20140512160537.C23B69D587@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 11:05:37 -0500 (Mon, 12 May 2014) New Revision: 7857 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py Log: python version of Swift-Galaxy driver passes tests. So, will switch to python from now. This will give advantage working with existing Galaxy tools Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 16:01:47 UTC (rev 7856) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 16:05:37 UTC (rev 7857) @@ -6,6 +6,7 @@ import distutils.spawn import traceback import fnmatch +import re def setwdir(): return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) @@ -51,7 +52,7 @@ rstart = next(myargs) rend = next(myargs) stepsize = next(myargs) - n = (rend - rstart)/stepsize + n = (int(rend) - int(rstart))/int(stepsize) outloc = next(myargs) outlistfile = next(myargs) @@ -84,7 +85,7 @@ # workout the stringargs if stringargs: - stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n" + stringarrayexpr="string stringargs[]="+re.sub('\'','"',str(stringargs.split()))+";\n" else: stringarrayexpr="string stringargs[];\n" @@ -95,10 +96,10 @@ #print "stdin" if k_or_t == "swiftK": #print "stdinK" - stdinfilexpr = "file stdinfile<"+stdin+">;\n" + stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n" else: #print "stdinT" - stdinfilexpr = "file stdinfile = input("+stdin+");\n" + stdinfilexpr = "file stdinfile = input(\""+stdin+"\");\n" else: #print "no stdin" if k_or_t == "swiftK": @@ -169,7 +170,7 @@ f.write(stringarrayexpr + " \n"); f.write(filearrayexpr + " \n"); f.write(stdinfilexpr + " \n"); - f.write("file exec = input("+executable+");\n"); + f.write("file exec = input(\""+executable+"\");\n"); if range_or_list == "list": @@ -213,11 +214,11 @@ stc=distutils.spawn.find_executable("stc") turbine= distutils.spawn.find_executable("turbine") with open(logfile, "wb") as errfile: - exitcode=subprocess.check_call([stc, "-j", "$(\which java)", wdir+"/script.swift"], stderr=errfile) + exitcode=subprocess.check_call([stc, "-j", "/homes/ketan/jdk1.7.0_07/bin/java", wdir+"/script.swift"], stderr=errfile) if exitcode == 0: with open(logfile, "wb") as errfile: - exitcode=subprocess.check_call([turbine, "-n", n, wdir+"/script.swift"], stderr=errfile) + exitcode=subprocess.check_call([turbine, "-n", str(n), wdir+"/script.tcl"], stderr=errfile) elif sites == "Vesta": @@ -266,6 +267,5 @@ f.close() - if __name__=='__main__': main() From davidk at ci.uchicago.edu Mon May 12 11:07:35 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 12 May 2014 11:07:35 -0500 (CDT) Subject: [Swift-commit] r7858 - branches/release-0.94/docs/siteguide Message-ID: <20140512160735.D21A69D587@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-12 11:07:35 -0500 (Mon, 12 May 2014) New Revision: 7858 Modified: branches/release-0.94/docs/siteguide/blues Log: Fix names Modified: branches/release-0.94/docs/siteguide/blues =================================================================== --- branches/release-0.94/docs/siteguide/blues 2014-05-12 16:05:37 UTC (rev 7857) +++ branches/release-0.94/docs/siteguide/blues 2014-05-12 16:07:35 UTC (rev 7858) @@ -1,14 +1,10 @@ Blues (x86 cluster) ------------------- -Fusion is a 310-node computing cluster for the Argonne -National Laboratory community. The primary goal of the LCRC is to -facilitate mid-range computing in all of the scientific programs of -Argonne and the University of Chicago. +Blues is a 310-node computing cluster for the Argonne +National Laboratory community. This section will walk +you through running a simple Swift script on Blues. -This section will walk you through running a simple Swift script -on Fusion. - Requesting Access ~~~~~~~~~~~~~~~~~ If you do not already have a Blues account, you can request one at @@ -30,13 +26,13 @@ SSH Keys ~~~~~~~~ Before accessing Blues, be sure to have your SSH keys configured correctly. -SSH keys are required to access fusion. You should see information about +SSH keys are required to access Blues. You should see information about this when you request your account. Email support at lcrc.anl.gov for additional help. Connecting to a login node ~~~~~~~~~~~~~~~~~~~~~~~~~~ -Once your keys are configured, you should be able to access a Fusion login +Once your keys are configured, you should be able to access a Blues login node with the following command: ----- From ketan at ci.uchicago.edu Mon May 12 11:16:18 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 11:16:18 -0500 (CDT) Subject: [Swift-commit] r7859 - SwiftApps/swift-galaxy/swift Message-ID: <20140512161618.1E6B99D587@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 11:16:17 -0500 (Mon, 12 May 2014) New Revision: 7859 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py Log: Set stdin as /dev/null to the input to the subprocess call of turbine Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 16:07:35 UTC (rev 7858) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 16:16:17 UTC (rev 7859) @@ -217,8 +217,8 @@ exitcode=subprocess.check_call([stc, "-j", "/homes/ketan/jdk1.7.0_07/bin/java", wdir+"/script.swift"], stderr=errfile) if exitcode == 0: - with open(logfile, "wb") as errfile: - exitcode=subprocess.check_call([turbine, "-n", str(n), wdir+"/script.tcl"], stderr=errfile) + with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile: + exitcode=subprocess.check_call([turbine, "-n", str(n), wdir+"/script.tcl"], stdin=infile, stderr=errfile) elif sites == "Vesta": From davidk at ci.uchicago.edu Mon May 12 11:20:28 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 12 May 2014 11:20:28 -0500 (CDT) Subject: [Swift-commit] r7860 - in trunk: . bin Message-ID: <20140512162028.EC8D99D587@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-12 11:20:28 -0500 (Mon, 12 May 2014) New Revision: 7860 Added: trunk/bin/moab-qdel trunk/bin/moab-qstat trunk/bin/moab-qsub Modified: trunk/build.xml Log: Add moab wrapper scripts Added: trunk/bin/moab-qdel =================================================================== --- trunk/bin/moab-qdel (rev 0) +++ trunk/bin/moab-qdel 2014-05-12 16:20:28 UTC (rev 7860) @@ -0,0 +1,3 @@ +#!/bin/bash + +canceljob "$@" Property changes on: trunk/bin/moab-qdel ___________________________________________________________________ Added: svn:executable + * Added: trunk/bin/moab-qstat =================================================================== --- trunk/bin/moab-qstat (rev 0) +++ trunk/bin/moab-qstat 2014-05-12 16:20:28 UTC (rev 7860) @@ -0,0 +1,32 @@ +#!/usr/bin/python + +import sys +import subprocess +import getpass + +jobs = sys.argv[2:len(sys.argv)] + +username = getpass.getuser() +p = subprocess.Popen(['showq', '-w', 'user='+username], stdout=subprocess.PIPE, stderr=subprocess.PIPE) +out, err = p.communicate() +out_list = out.rstrip().split('\n') + +for o in out_list: + try: + words = o.split() + jobid = words[0].rstrip() + jobstatus = words[2].rstrip() + + if jobid in jobs: + if jobstatus in ["Idle", "Deferred", "BatchHold", "SystemHold", "UserHold", "Deferred", "Migrated", "Staging", "Starting"]: + jobstatus = "Q" + elif jobstatus in ["Running"]: + jobstatus = "R" + elif jobstatus == ["Completed", "Vacated"]: + jobstatus = "C" + else: + continue + print "Job Id: " + jobid + print "\tjob_state = " + jobstatus + "\n" + except: + continue Property changes on: trunk/bin/moab-qstat ___________________________________________________________________ Added: svn:executable + * Added: trunk/bin/moab-qsub =================================================================== --- trunk/bin/moab-qsub (rev 0) +++ trunk/bin/moab-qsub 2014-05-12 16:20:28 UTC (rev 7860) @@ -0,0 +1,3 @@ +#!/bin/bash + +msub "$@" Property changes on: trunk/bin/moab-qsub ___________________________________________________________________ Added: svn:executable + * Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2014-05-12 16:16:17 UTC (rev 7859) +++ trunk/build.xml 2014-05-12 16:20:28 UTC (rev 7860) @@ -87,6 +87,9 @@ + + + From ketan at ci.uchicago.edu Mon May 12 12:49:25 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 12:49:25 -0500 (CDT) Subject: [Swift-commit] r7861 - SwiftApps/swift-galaxy/swift Message-ID: <20140512174926.001E19D587@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 12:49:25 -0500 (Mon, 12 May 2014) New Revision: 7861 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py Log: vesta case Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 16:20:28 UTC (rev 7860) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 17:49:25 UTC (rev 7861) @@ -207,23 +207,31 @@ #print "Invoking T" #EXITCODE=0 + stc=distutils.spawn.find_executable("stc") + java=distutils.spawn.find_executable("java") + turbine= distutils.spawn.find_executable("turbine") if sites == "localhost": print "localhost" - stc=distutils.spawn.find_executable("stc") - turbine= distutils.spawn.find_executable("turbine") with open(logfile, "wb") as errfile: - exitcode=subprocess.check_call([stc, "-j", "/homes/ketan/jdk1.7.0_07/bin/java", wdir+"/script.swift"], stderr=errfile) + exitcode=subprocess.check_call([stc, "-j", java, wdir+"/script.swift"], stderr=errfile) if exitcode == 0: with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile: - exitcode=subprocess.check_call([turbine, "-n", str(n), wdir+"/script.tcl"], stdin=infile, stderr=errfile) + exitcode=subprocess.check_call([turbine, "-n", str(n), wdir+"/script.tcl", swiftargs], stdin=infile, stderr=errfile) - elif sites == "Vesta": print "vesta" + pwd=os.path.getcwd() + turbine_home="/home/wozniak/Public/sfw/turbine" + with open(logfile, "wb") as errfile: + exitcode=subprocess.check_call([stc, "-t", "checkpointing", "-r", pwd, "-j", java, wdir+"/script.swift"], stderr=errfile) + + if exitcode == 0: + with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile: + exitcode=subprocess.check_call([turbine_home+"/scripts/submit/cobalt/turbine-cobalt-run.zsh", "-n", str(n), wdir+"/script.tcl"], stdin=infile, stderr=errfile) elif sites == "Mira": @@ -232,20 +240,7 @@ else: print "site not recognized" - #STC=$(\which stc) - ##echo "got stc" >> /tmp/status.txt - #turbine=$(\which turbine) - ##echo "got turbine" >> /tmp/status.txt - #$STC -j $(\which java) $wdir/script.swift 2>$logfile - ##echo "done running stc" >> /tmp/status.txt - ##echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt - #$turbine -n $n $wdir/script.tcl "${swiftargs}" > /tmp/status.txt - #EXITCODE=$? - ##echo "exit code obtained" >> /tmp/status.txt - #;; #"Vesta" ) - #TURBINE_HOME=/home/wozniak/Public/sfw/turbine #STC=/home/wozniak/Public/sfw/stc/bin/stc #$STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile #$TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl @@ -266,6 +261,5 @@ f.write(outlist); f.close() - if __name__=='__main__': main() From ketan at ci.uchicago.edu Mon May 12 12:51:08 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 12:51:08 -0500 (CDT) Subject: [Swift-commit] r7862 - SwiftApps/swift-galaxy/swift Message-ID: <20140512175108.E7FAA9D587@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 12:51:08 -0500 (Mon, 12 May 2014) New Revision: 7862 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.xml Log: python interpreter Modified: SwiftApps/swift-galaxy/swift/swiftforeach.xml =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-12 17:49:25 UTC (rev 7861) +++ SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-12 17:51:08 UTC (rev 7862) @@ -10,9 +10,9 @@ --> A generic tool to run executable via Swift foreach parallel construct - + #if $rangeorlist.rl == "rng" - swiftforeach.sh + swiftforeach.py "range" "${k_or_t}" "${site}" @@ -31,7 +31,7 @@ ${a.fileargname} #end for #elif $rangeorlist.rl == "lst" - swiftforeach.sh + swiftforeach.py "list" "${k_or_t}" "${site}" @@ -48,7 +48,7 @@ ${a.fileargname} #end for #else - swiftforeach.sh + swiftforeach.py "drytest" "${k_or_t}" "${site}" From ketan at ci.uchicago.edu Mon May 12 14:10:34 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 14:10:34 -0500 (CDT) Subject: [Swift-commit] r7863 - SwiftApps/swift-galaxy/swift Message-ID: <20140512191034.9E0EA9CCF4@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 14:10:34 -0500 (Mon, 12 May 2014) New Revision: 7863 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py Log: small fix Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 17:51:08 UTC (rev 7862) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 19:10:34 UTC (rev 7863) @@ -92,8 +92,8 @@ wdir = setwdir().rstrip() #workout standard input file - if stdin: - #print "stdin" + if stdin != "None": + print stdin if k_or_t == "swiftK": #print "stdinK" stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n" @@ -133,8 +133,8 @@ else: f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") + f.write(" file out ;\n") + f.write(" file err ;\n") f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") f.write("}\n") From ketan at ci.uchicago.edu Mon May 12 14:42:49 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 14:42:49 -0500 (CDT) Subject: [Swift-commit] r7864 - in SwiftApps/swift-galaxy/swift: . old Message-ID: <20140512194249.A14679CCF4@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 14:42:49 -0500 (Mon, 12 May 2014) New Revision: 7864 Added: SwiftApps/swift-galaxy/swift/old/ SwiftApps/swift-galaxy/swift/old/cf SwiftApps/swift-galaxy/swift/old/face-it/ SwiftApps/swift-galaxy/swift/old/genswift.sh SwiftApps/swift-galaxy/swift/old/swift-mysim Removed: SwiftApps/swift-galaxy/swift/cf SwiftApps/swift-galaxy/swift/face-it/ SwiftApps/swift-galaxy/swift/genswift.sh SwiftApps/swift-galaxy/swift/swift-mysim Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py Log: move old stuff in old Deleted: SwiftApps/swift-galaxy/swift/cf =================================================================== --- SwiftApps/swift-galaxy/swift/cf 2014-05-12 19:10:34 UTC (rev 7863) +++ SwiftApps/swift-galaxy/swift/cf 2014-05-12 19:42:49 UTC (rev 7864) @@ -1,7 +0,0 @@ -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -use.wrapper.staging=false Deleted: SwiftApps/swift-galaxy/swift/genswift.sh =================================================================== --- SwiftApps/swift-galaxy/swift/genswift.sh 2014-05-12 19:10:34 UTC (rev 7863) +++ SwiftApps/swift-galaxy/swift/genswift.sh 2014-05-12 19:42:49 UTC (rev 7864) @@ -1,113 +0,0 @@ -#!/bin/bash - -echo $@ > /tmp/args.txt -wdir=$(mktemp -d /tmp/swift-gal.XXXX) - -# numeric range -sites=$1 # comma separated list of sites -shift -swiftargs=$1 -shift -interpreter=$1 -shift -executable=$1 -shift -rstart=$1 -shift -rend=$1 -shift -stepsize=$1 -shift -outloc=$1 -shift -outlistfile=$1 -shift -logfile=$1 -shift - -stringargs=$1 -shift - -#workout the array only if user adds args else make it blank -if [ "${stringargs}" ] ; then - stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; -else - stringarrayexpr='string stringargs[];' -fi - -fileargs=() -while [ $# -gt 0 ] ; do - fileargs+=("\"$1\"") - shift -done - -#workout the array only if user adds file args else make it blank -if [ ${#fileargs[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${fileargs[@]}") - filearrayexpr='file fileargs[]=['$(echo $tmp | sed s/,$//)'];'; -else - filearrayexpr='file fileargs[];' -fi - -#Build Swift source code -cat << EOF > $wdir/script.swift -type file; - -app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ - $interpreter @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; -} - -$stringarrayexpr -$filearrayexpr - -file exec<"$executable">; -foreach i in [$rstart:$rend:$stepsize]{ - file out ; - file err ; - (out,err) = anapp(exec, i, stringargs, fileargs); -} -EOF - -swift=$(which swift) - -#build site -BASEDIR=$(dirname $0) -$BASEDIR/buildsite.sh $sites $wdir - -#build config -cat << EOF > $wdir/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF - -#build tc -IFS=',' read -ra S <<< "$sites" -for site in "${S[@]}" ; do -cat << EOF >> $wdir/tc -$site $interpreter $interpreter -EOF -done - -$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile -EXITCODE=$? - -#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` -#Populate output file -cat << EOF > $outlistfile -`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done` -EOF - -#cleanup -#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift - -if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 -fi - -exit $EXITCODE - -#dum ditty dum ditty dum dum dum Copied: SwiftApps/swift-galaxy/swift/old/cf (from rev 7862, SwiftApps/swift-galaxy/swift/cf) =================================================================== --- SwiftApps/swift-galaxy/swift/old/cf (rev 0) +++ SwiftApps/swift-galaxy/swift/old/cf 2014-05-12 19:42:49 UTC (rev 7864) @@ -0,0 +1,7 @@ +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +use.wrapper.staging=false Copied: SwiftApps/swift-galaxy/swift/old/genswift.sh (from rev 7862, SwiftApps/swift-galaxy/swift/genswift.sh) =================================================================== --- SwiftApps/swift-galaxy/swift/old/genswift.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/old/genswift.sh 2014-05-12 19:42:49 UTC (rev 7864) @@ -0,0 +1,113 @@ +#!/bin/bash + +echo $@ > /tmp/args.txt +wdir=$(mktemp -d /tmp/swift-gal.XXXX) + +# numeric range +sites=$1 # comma separated list of sites +shift +swiftargs=$1 +shift +interpreter=$1 +shift +executable=$1 +shift +rstart=$1 +shift +rend=$1 +shift +stepsize=$1 +shift +outloc=$1 +shift +outlistfile=$1 +shift +logfile=$1 +shift + +stringargs=$1 +shift + +#workout the array only if user adds args else make it blank +if [ "${stringargs}" ] ; then + stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; +else + stringarrayexpr='string stringargs[];' +fi + +fileargs=() +while [ $# -gt 0 ] ; do + fileargs+=("\"$1\"") + shift +done + +#workout the array only if user adds file args else make it blank +if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + filearrayexpr='file fileargs[]=['$(echo $tmp | sed s/,$//)'];'; +else + filearrayexpr='file fileargs[];' +fi + +#Build Swift source code +cat << EOF > $wdir/script.swift +type file; + +app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ + $interpreter @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; +} + +$stringarrayexpr +$filearrayexpr + +file exec<"$executable">; +foreach i in [$rstart:$rend:$stepsize]{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs); +} +EOF + +swift=$(which swift) + +#build site +BASEDIR=$(dirname $0) +$BASEDIR/buildsite.sh $sites $wdir + +#build config +cat << EOF > $wdir/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF + +#build tc +IFS=',' read -ra S <<< "$sites" +for site in "${S[@]}" ; do +cat << EOF >> $wdir/tc +$site $interpreter $interpreter +EOF +done + +$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile +EXITCODE=$? + +#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` +#Populate output file +cat << EOF > $outlistfile +`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done` +EOF + +#cleanup +#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift + +if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 +fi + +exit $EXITCODE + +#dum ditty dum ditty dum dum dum Copied: SwiftApps/swift-galaxy/swift/old/swift-mysim (from rev 7862, SwiftApps/swift-galaxy/swift/swift-mysim) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-mysim (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-mysim 2014-05-12 19:42:49 UTC (rev 7864) @@ -0,0 +1,282 @@ +#!/bin/bash + +site=$1 +nsim=$2 +steps=$3 +range=$4 +values=$5 +outdir=$6 +logfile=$7 +htmlfile=$8 + +mkdir -p $outdir + +swift=`which swift` +# use "here" document for configs + +cat << EOF > /tmp/mysim.swift +type file; + +app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values) { + simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log; +} + +app (file out, file log) analyze (file s[]) { + stats @filenames(s) stdout=@out stderr=@log; +} + +int nsim = @toInt(@arg("nsim", "10")); +int steps = @toInt(@arg("steps", "1")); +int range = @toInt(@arg("range", "100")); +int values = @toInt(@arg("values", "5")); + +file sims[]; + +foreach i in [0:nsim-1] { + file simout ; + file simlog ; + (simout,simlog) = simulation(steps,range,values); + sims[i] = simout; +} + +file stats_out<"output/average.out">; +file stats_log<"output/average.log">; +(stats_out, stats_log) = analyze(sims); +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF + +cat << 'EOF' > /tmp/tc +swiftgal echo echo +swiftgal cat cat +swiftgal simulate simulate.sh +swiftgal stats stats.sh +EOF + +case "$site" in +midway ) +cat << 'EOF' > /tmp/sites.xml + + + + + sandyb + 16 + 02:00:00 + 7500 + 100 + 100 + 4 + 2 + 2 + 1.27 + 10000 + /scratch/midway/maheshwari/swift.work + + +EOF + +;; + +uc3 ) +cat << 'EOF' > /tmp/sites.xml + + + + + + .09 + 1000 + 1 + 3600 + 1 + 100 + 100 + 1 + UidDomain == "osg-gk.mwt2.org" + 1 + "group_friends.{env.USER}" + nonshared + + + /tmp/swift.work + + +EOF + +;; + +stampede ) +cat << 'EOF' > /tmp/sites.xml + + + + + + 16 + 16 + TG-STA110005S + normal + ketancmaheshwari at gmail.com + ALL + 02:00:00 + 8000 + 100 + 100 + 1 + 1 + 1 + .15 + 10000 + /tmp/swift.work + + +EOF + +;; + +localhost ) +cat << 'EOF' > /tmp/sites.xml + + + + + + + /tmp/swift.work + 0.31 + + +EOF +;; + +* ) +echo "defaulting to localhost" + +cat << 'EOF' > /tmp/sites.xml + + + + + + + /tmp/swift.work + 0.31 + + +EOF +;; +esac + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/mysim.swift -nsim=$nsim -steps=$steps -range=$range -values=$values 2> std.err +EXITCODE=$? + +swiftlog=`\ls -t *.log | head -1` + +cat << 'EOF' > /tmp/plotswiftlogs +#!/bin/bash + +#usage: ./plotswiftlogs.ketan + +SWIFTLOGFILE=$1 + +#TMPDIR=`mktemp -d plotlog.XXX` + +grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out + +SWIFTOUTFILE=swiftoutfile.out + +#extract start time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` +START_TIME=`date +%s -d "$TMPDATE"` + +#extract end time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` +END_TIME=`date +%s -d "$TMPDATE"` + +#duration +DIFFTIME=$((END_TIME - START_TIME)) + +#extract active runs in a file +(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt) + +#extract successful completions in a file +(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) + +#prepare tics +activelines=`wc -l active.txt | awk '{print $1}'` +cumulines=`wc -l cumulative.txt | awk '{print $1}'` + +if [ $activelines -ne 0 ] +then + activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` +fi + +seq 0 $activelinespertic $DIFFTIME > activetics.txt + +if [ $cumulines -ne 0 ] +then + cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` +fi + +seq 0 $cumulinespertic $DIFFTIME > cumultics.txt + +#final plot data +paste activetics.txt active.txt > plot_active.txt +paste cumultics.txt cumulative.txt > plot_cumulative.txt + +cat << EOF1 > plotit.gp +set terminal png enhanced +set nokey +set output "cumulativeplot.png" +set xlabel "Time in sec" +set ylabel "number of completed apps" +set title "Cumulative Apps Completed" +plot "plot_cumulative.txt" using 1:2 with lines +set output "activeplot.png" +set xlabel "Time in sec" +set ylabel "number of active apps" +set title "Active Apps" +plot "plot_active.txt" using 1:2 with line +EOF1 + +gnuplot plotit.gp 2>/dev/null +rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt + +EOF + +sh /tmp/plotswiftlogs $swiftlog +cp *.png $outdir/ + +cat << EOF > $htmlfile + + + + + Application output + + +

Swift run trace plots

+ +active plot +cumulative plot + +

Browse output of application

+`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` + + +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat std.err >&2 +fi +cp std.err $logfile +rm std.err +exit $EXITCODE Deleted: SwiftApps/swift-galaxy/swift/swift-mysim =================================================================== --- SwiftApps/swift-galaxy/swift/swift-mysim 2014-05-12 19:10:34 UTC (rev 7863) +++ SwiftApps/swift-galaxy/swift/swift-mysim 2014-05-12 19:42:49 UTC (rev 7864) @@ -1,282 +0,0 @@ -#!/bin/bash - -site=$1 -nsim=$2 -steps=$3 -range=$4 -values=$5 -outdir=$6 -logfile=$7 -htmlfile=$8 - -mkdir -p $outdir - -swift=`which swift` -# use "here" document for configs - -cat << EOF > /tmp/mysim.swift -type file; - -app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values) { - simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log; -} - -app (file out, file log) analyze (file s[]) { - stats @filenames(s) stdout=@out stderr=@log; -} - -int nsim = @toInt(@arg("nsim", "10")); -int steps = @toInt(@arg("steps", "1")); -int range = @toInt(@arg("range", "100")); -int values = @toInt(@arg("values", "5")); - -file sims[]; - -foreach i in [0:nsim-1] { - file simout ; - file simlog ; - (simout,simlog) = simulation(steps,range,values); - sims[i] = simout; -} - -file stats_out<"output/average.out">; -file stats_log<"output/average.log">; -(stats_out, stats_log) = analyze(sims); -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF - -cat << 'EOF' > /tmp/tc -swiftgal echo echo -swiftgal cat cat -swiftgal simulate simulate.sh -swiftgal stats stats.sh -EOF - -case "$site" in -midway ) -cat << 'EOF' > /tmp/sites.xml - - - - - sandyb - 16 - 02:00:00 - 7500 - 100 - 100 - 4 - 2 - 2 - 1.27 - 10000 - /scratch/midway/maheshwari/swift.work - - -EOF - -;; - -uc3 ) -cat << 'EOF' > /tmp/sites.xml - - - - - - .09 - 1000 - 1 - 3600 - 1 - 100 - 100 - 1 - UidDomain == "osg-gk.mwt2.org" - 1 - "group_friends.{env.USER}" - nonshared - - - /tmp/swift.work - - -EOF - -;; - -stampede ) -cat << 'EOF' > /tmp/sites.xml - - - - - - 16 - 16 - TG-STA110005S - normal - ketancmaheshwari at gmail.com - ALL - 02:00:00 - 8000 - 100 - 100 - 1 - 1 - 1 - .15 - 10000 - /tmp/swift.work - - -EOF - -;; - -localhost ) -cat << 'EOF' > /tmp/sites.xml - - - - - - - /tmp/swift.work - 0.31 - - -EOF -;; - -* ) -echo "defaulting to localhost" - -cat << 'EOF' > /tmp/sites.xml - - - - - - - /tmp/swift.work - 0.31 - - -EOF -;; -esac - -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/mysim.swift -nsim=$nsim -steps=$steps -range=$range -values=$values 2> std.err -EXITCODE=$? - -swiftlog=`\ls -t *.log | head -1` - -cat << 'EOF' > /tmp/plotswiftlogs -#!/bin/bash - -#usage: ./plotswiftlogs.ketan - -SWIFTLOGFILE=$1 - -#TMPDIR=`mktemp -d plotlog.XXX` - -grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out - -SWIFTOUTFILE=swiftoutfile.out - -#extract start time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` -START_TIME=`date +%s -d "$TMPDATE"` - -#extract end time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` -END_TIME=`date +%s -d "$TMPDATE"` - -#duration -DIFFTIME=$((END_TIME - START_TIME)) - -#extract active runs in a file -(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt) - -#extract successful completions in a file -(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) - -#prepare tics -activelines=`wc -l active.txt | awk '{print $1}'` -cumulines=`wc -l cumulative.txt | awk '{print $1}'` - -if [ $activelines -ne 0 ] -then - activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` -fi - -seq 0 $activelinespertic $DIFFTIME > activetics.txt - -if [ $cumulines -ne 0 ] -then - cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` -fi - -seq 0 $cumulinespertic $DIFFTIME > cumultics.txt - -#final plot data -paste activetics.txt active.txt > plot_active.txt -paste cumultics.txt cumulative.txt > plot_cumulative.txt - -cat << EOF1 > plotit.gp -set terminal png enhanced -set nokey -set output "cumulativeplot.png" -set xlabel "Time in sec" -set ylabel "number of completed apps" -set title "Cumulative Apps Completed" -plot "plot_cumulative.txt" using 1:2 with lines -set output "activeplot.png" -set xlabel "Time in sec" -set ylabel "number of active apps" -set title "Active Apps" -plot "plot_active.txt" using 1:2 with line -EOF1 - -gnuplot plotit.gp 2>/dev/null -rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt - -EOF - -sh /tmp/plotswiftlogs $swiftlog -cp *.png $outdir/ - -cat << EOF > $htmlfile - - - - - Application output - - -

Swift run trace plots

- -active plot -cumulative plot - -

Browse output of application

-`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` - - -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat std.err >&2 -fi -cp std.err $logfile -rm std.err -exit $EXITCODE Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 19:10:34 UTC (rev 7863) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 19:42:49 UTC (rev 7864) @@ -1,5 +1,12 @@ #!/usr/bin/env python +""" +This is the Swift foreach driver for Galaxy. It receives inputs from the Swift +"foreach" Galaxy tool and depending on the options specified, builds a +canonical Swift/K or Swift/T script and runs over specified computational +environment. +""" + import subprocess import sys import os @@ -25,7 +32,6 @@ f.write("lazy.errors=false\n") f.close() -# numeric range def main(): myargs = genargs(sys.argv) try: @@ -46,9 +52,11 @@ #print "stdin => " + stdin if range_or_list == "list": + #list of items listfile = next(myargs) n = sum(1 for line in open(listfile)) else: + # numeric range rstart = next(myargs) rend = next(myargs) stepsize = next(myargs) @@ -62,7 +70,10 @@ except: traceback.print_exc() sys.exit(1) + + #which swift swift = distutils.spawn.find_executable("swift") + BASEDIR = os.path.dirname(os.path.realpath(__file__)) # workout the file args @@ -93,7 +104,6 @@ #workout standard input file if stdin != "None": - print stdin if k_or_t == "swiftK": #print "stdinK" stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n" From ketan at ci.uchicago.edu Mon May 12 14:44:03 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 14:44:03 -0500 (CDT) Subject: [Swift-commit] r7865 - in SwiftApps/swift-galaxy/swift: . old Message-ID: <20140512194403.97F0A9CCF4@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 14:44:03 -0500 (Mon, 12 May 2014) New Revision: 7865 Added: SwiftApps/swift-galaxy/swift/old/2stage.xml SwiftApps/swift-galaxy/swift/old/dogftp.sh SwiftApps/swift-galaxy/swift/old/edrevised.mod SwiftApps/swift-galaxy/swift/old/gftplist.swift SwiftApps/swift-galaxy/swift/old/mysim.xml SwiftApps/swift-galaxy/swift/old/swift-2stage SwiftApps/swift-galaxy/swift/old/swift-3stage SwiftApps/swift-galaxy/swift/old/swift-skeleton.sh SwiftApps/swift-galaxy/swift/old/swift-skeleton.xml SwiftApps/swift-galaxy/swift/old/tc Removed: SwiftApps/swift-galaxy/swift/2stage.xml SwiftApps/swift-galaxy/swift/dogftp.sh SwiftApps/swift-galaxy/swift/edrevised.mod SwiftApps/swift-galaxy/swift/gftplist.swift SwiftApps/swift-galaxy/swift/mysim.xml SwiftApps/swift-galaxy/swift/swift-2stage SwiftApps/swift-galaxy/swift/swift-3stage SwiftApps/swift-galaxy/swift/swift-skeleton.sh SwiftApps/swift-galaxy/swift/swift-skeleton.xml SwiftApps/swift-galaxy/swift/tc Log: move old stuff in old Deleted: SwiftApps/swift-galaxy/swift/2stage.xml =================================================================== --- SwiftApps/swift-galaxy/swift/2stage.xml 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/2stage.xml 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,23 +0,0 @@ - - A 2 stage Swift workflow - - - swift-2stage "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file" - - - - - - - - - - - - - - - - - - Deleted: SwiftApps/swift-galaxy/swift/dogftp.sh =================================================================== --- SwiftApps/swift-galaxy/swift/dogftp.sh 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/dogftp.sh 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,16 +0,0 @@ -#!/bin/bash -x - -#stagein -#eg. globus-url-copy file:///work/01739/ketan/JSC_AdaptiveSchedule/workflow/sample.txt gsiftp://gridftp.stampede.tacc.utexas.edu:2811/scratch/01739/ketan/storage/ - -src=$1 -dest=$2 -token=$3 - -#echo $src > $token -#echo $dest >> $token - -globus-url-copy $src $dest -#ssh -i /home1/01739/ketan/.ssh/id_rsa maheshwari at swift.rcc.uchicago.edu globus-url-copy gsiftp://trestles-dm2.sdsc.edu:2811/home/ketan/storage/$src gsiftp://gridftp.stampede.tacc.utexas.edu:2811/work/01739/ketan/JSC_AdaptiveSchedule/workflow/dir/$dest - -echo $? > $token Deleted: SwiftApps/swift-galaxy/swift/edrevised.mod =================================================================== --- SwiftApps/swift-galaxy/swift/edrevised.mod 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/edrevised.mod 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,115 +0,0 @@ -# ampl model declaration -# V. M. Zavala - MCS/ANL 2012 - -# scenarios -param NS; # number of scenarios -set SCEN := 1..NS; # set of scenarios - -# dispatch horizon -param T; # horizon length -param TH; # simulation horizon (hours) -param dt; # length of time step [hr] -set HORIZON := 0..T-1; # horizon minus one -set TESTTIME:=0..T; # set of test times -param idx_data; # auxiliary parameter -param idx_state; # auxiliary parameter - -# lines -set LIN; # set of lines -param status{LIN}, symbolic; # line status -param snd_bus{LIN}; # sending bus -param rec_bus{LIN}; # receiving bus -param X{LIN}; # reactance - Ohm -param V{LIN}; # voltage drop - kV -param Pmax{LIN}; # maximum line capacity - MW -param install_cost{LIN}; # annualized installation cost $/yr - -# buses -set BUS; # set of buses -param lat_bus{BUS}; # latitude of bus -param lon_bus{BUS}; # longitude of bus -param ref_bus, symbolic; # reference bus - -# generators -set GEN; -param bus_gen{GEN}; # generator bus -param np_cap{GEN}; # name place capacity - MW -param sum_cap{GEN}; # summer capacity - MW -param win_cap{GEN}; # winter capacity - MW -param min_cap{GEN}; # minimum capacity - MW -param fuel{GEN}, symbolic; # type of fuel for generator -param min_hrate{GEN}; # heat rate at minimum capacity - BTU/kWh -param min_power{GEN}; # power at minimum capacity - MW -param max_ur{GEN}; # minimum up ramp - MW -param max_dr{GEN}; # maximum down ramp - MW -param lat_gen{GEN}; # latitude of generation bus -param lon_gen{GEN}; # longitude of generation bus -param gen_cost{GEN}; # generation cost $/MWhr -param Pgen_init{GEN}; # initial conditions generators MW -param clgen_cost{TESTTIME}; # closed-loop generation cost $/MWh - -# fuels -set FUEL; -param Description{FUEL}, symbolic; # fuel description -param HV{FUEL}; # heating value (Mbtu/unit) -param Unitprice{FUEL}; # unit price ($/unit) -param Units{FUEL}, symbolic; # units - -# loads -set TIME; # set of times for loads -set LOAD; # set of loads -param bus_load{LOAD}; # load bus -param loads{TIME,LOAD}; # loads in space and time - MW -param wind_totalmh{SCEN,TESTTIME}; # total wind power - MW - -# wind data -set WIND; -param wind_total{SCEN,TIME}; # total wind power - MW -param bus_wind{WIND}; # wind bus -param wind_share{WIND}; # wind share of total power MW/MW - -# variables -var Pgen{SCEN,TESTTIME,GEN}; # generation levels - MW -var Pload{SCEN,TESTTIME,LOAD}; # load levels - MW -var yuc{TESTTIME,j in GEN} >= 0, <= 1; # ON/OFF Status generators -var slackp{s in SCEN, t in TESTTIME}>=0; # slack - for modeling purposes only -var slackm{s in SCEN, t in TESTTIME}>=0; # slack - for modeling purposes only -var Pgen_initial{GEN}; # initial state for generators - MW - -# params for inference -param fix_x; - -# power flow equations -pfeq{s in SCEN, t in TESTTIME}: - +(sum{i in GEN} Pgen[s,t,i]) - -(sum{i in LOAD} Pload[s,t,i]) - + wind_totalmh[s,t] - = slackp[s,t]-slackm[s,t]; - -# upper bound and lower bounds for generation flows [MW] -Pgenequb{s in SCEN,t in TESTTIME,i in GEN}: Pgen[s,t,i] <= np_cap[i]*yuc[t,i]; -Pgeneqlb{s in SCEN,t in TESTTIME,i in GEN}: Pgen[s,t,i] >= 0; - -Pgenequbinit{i in GEN}: Pgen_initial[i] <= np_cap[i]*yuc[1,i]; -Pgeneqlbinit{i in GEN}: Pgen_initial[i] >= 0; - -# upper and lower bounds on ramps -upramp{s in SCEN,t in HORIZON,i in GEN}: Pgen[s,t+1,i] - Pgen[s,t,i] <= +max_ur[i]; -loramp{s in SCEN,t in HORIZON,i in GEN}: Pgen[s,t,i] - Pgen[s,t+1,i] >= -max_dr[i]; - -upramp0{s in SCEN,i in GEN}: Pgen[s,1,i] - Pgen_initial[i] <= +max_ur[i]; -loramp0{s in SCEN,i in GEN}: Pgen_initial[i] - Pgen[s,1,i] >= -max_dr[i]; - -# time-dependent generation costs -var tot_gen_cost{s in SCEN,t in TESTTIME} = 1e+0*sum{i in GEN} (gen_cost[i]*Pgen[s,t,i]); - -# time-dependent startup/shutdown costs -var tot_start_cost{t in TESTTIME} = 1e+3*(sum{i in GEN} yuc[t,i]); - -# time-dependent slack costs -var tot_slack_cost{s in SCEN,t in TESTTIME} = 1e+4*(slackp[s,t]+slackm[s,t]); - -# cost function -minimize cost: sum{t in TESTTIME} tot_start_cost[t] - + (1/(card(SCEN)))*sum{s in SCEN,t in TESTTIME} tot_gen_cost[s,t] - + (1/(card(SCEN)))*sum{s in SCEN,t in TESTTIME} tot_slack_cost[s,t]; Deleted: SwiftApps/swift-galaxy/swift/gftplist.swift =================================================================== --- SwiftApps/swift-galaxy/swift/gftplist.swift 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/gftplist.swift 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,18 +0,0 @@ -type file; - -app (file _status, file _stdout, file _stderr) stage (string _src, string _dst){ - stage _src _dst @_status stdout=@_stdout stderr=@_stderr; -} - -string src="gsiftp://trestles-dm2.sdsc.edu:2811/home/ketan/storage/"; -string dst="gsiftp://gridftp.stampede.tacc.utexas.edu:2811/scratch/01739/ketan/dest/"; - -string data[] = readData("datlist.txt"); - -foreach d, i in data{ - file status ; - file out ; - file err ; - //trace (@strcat(src, d)); - (status, out, err) = stage (@strcat(src, d), @strcat(dst, d)); -} Deleted: SwiftApps/swift-galaxy/swift/mysim.xml =================================================================== --- SwiftApps/swift-galaxy/swift/mysim.xml 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/mysim.xml 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,24 +0,0 @@ - - A Swift tool to run the mysim workflow - - - swift-mysim "$site" "$nsim" "$steps" "$range" "$vals" "$html_file.files_path" "$log_file" "$html_file" - - - - - - - - - - - - - - - - - - - Copied: SwiftApps/swift-galaxy/swift/old/2stage.xml (from rev 7862, SwiftApps/swift-galaxy/swift/2stage.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/2stage.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/2stage.xml 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,23 @@ + + A 2 stage Swift workflow + + + swift-2stage "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file" + + + + + + + + + + + + + + + + + + Copied: SwiftApps/swift-galaxy/swift/old/dogftp.sh (from rev 7862, SwiftApps/swift-galaxy/swift/dogftp.sh) =================================================================== --- SwiftApps/swift-galaxy/swift/old/dogftp.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/old/dogftp.sh 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,16 @@ +#!/bin/bash -x + +#stagein +#eg. globus-url-copy file:///work/01739/ketan/JSC_AdaptiveSchedule/workflow/sample.txt gsiftp://gridftp.stampede.tacc.utexas.edu:2811/scratch/01739/ketan/storage/ + +src=$1 +dest=$2 +token=$3 + +#echo $src > $token +#echo $dest >> $token + +globus-url-copy $src $dest +#ssh -i /home1/01739/ketan/.ssh/id_rsa maheshwari at swift.rcc.uchicago.edu globus-url-copy gsiftp://trestles-dm2.sdsc.edu:2811/home/ketan/storage/$src gsiftp://gridftp.stampede.tacc.utexas.edu:2811/work/01739/ketan/JSC_AdaptiveSchedule/workflow/dir/$dest + +echo $? > $token Copied: SwiftApps/swift-galaxy/swift/old/edrevised.mod (from rev 7862, SwiftApps/swift-galaxy/swift/edrevised.mod) =================================================================== --- SwiftApps/swift-galaxy/swift/old/edrevised.mod (rev 0) +++ SwiftApps/swift-galaxy/swift/old/edrevised.mod 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,115 @@ +# ampl model declaration +# V. M. Zavala - MCS/ANL 2012 + +# scenarios +param NS; # number of scenarios +set SCEN := 1..NS; # set of scenarios + +# dispatch horizon +param T; # horizon length +param TH; # simulation horizon (hours) +param dt; # length of time step [hr] +set HORIZON := 0..T-1; # horizon minus one +set TESTTIME:=0..T; # set of test times +param idx_data; # auxiliary parameter +param idx_state; # auxiliary parameter + +# lines +set LIN; # set of lines +param status{LIN}, symbolic; # line status +param snd_bus{LIN}; # sending bus +param rec_bus{LIN}; # receiving bus +param X{LIN}; # reactance - Ohm +param V{LIN}; # voltage drop - kV +param Pmax{LIN}; # maximum line capacity - MW +param install_cost{LIN}; # annualized installation cost $/yr + +# buses +set BUS; # set of buses +param lat_bus{BUS}; # latitude of bus +param lon_bus{BUS}; # longitude of bus +param ref_bus, symbolic; # reference bus + +# generators +set GEN; +param bus_gen{GEN}; # generator bus +param np_cap{GEN}; # name place capacity - MW +param sum_cap{GEN}; # summer capacity - MW +param win_cap{GEN}; # winter capacity - MW +param min_cap{GEN}; # minimum capacity - MW +param fuel{GEN}, symbolic; # type of fuel for generator +param min_hrate{GEN}; # heat rate at minimum capacity - BTU/kWh +param min_power{GEN}; # power at minimum capacity - MW +param max_ur{GEN}; # minimum up ramp - MW +param max_dr{GEN}; # maximum down ramp - MW +param lat_gen{GEN}; # latitude of generation bus +param lon_gen{GEN}; # longitude of generation bus +param gen_cost{GEN}; # generation cost $/MWhr +param Pgen_init{GEN}; # initial conditions generators MW +param clgen_cost{TESTTIME}; # closed-loop generation cost $/MWh + +# fuels +set FUEL; +param Description{FUEL}, symbolic; # fuel description +param HV{FUEL}; # heating value (Mbtu/unit) +param Unitprice{FUEL}; # unit price ($/unit) +param Units{FUEL}, symbolic; # units + +# loads +set TIME; # set of times for loads +set LOAD; # set of loads +param bus_load{LOAD}; # load bus +param loads{TIME,LOAD}; # loads in space and time - MW +param wind_totalmh{SCEN,TESTTIME}; # total wind power - MW + +# wind data +set WIND; +param wind_total{SCEN,TIME}; # total wind power - MW +param bus_wind{WIND}; # wind bus +param wind_share{WIND}; # wind share of total power MW/MW + +# variables +var Pgen{SCEN,TESTTIME,GEN}; # generation levels - MW +var Pload{SCEN,TESTTIME,LOAD}; # load levels - MW +var yuc{TESTTIME,j in GEN} >= 0, <= 1; # ON/OFF Status generators +var slackp{s in SCEN, t in TESTTIME}>=0; # slack - for modeling purposes only +var slackm{s in SCEN, t in TESTTIME}>=0; # slack - for modeling purposes only +var Pgen_initial{GEN}; # initial state for generators - MW + +# params for inference +param fix_x; + +# power flow equations +pfeq{s in SCEN, t in TESTTIME}: + +(sum{i in GEN} Pgen[s,t,i]) + -(sum{i in LOAD} Pload[s,t,i]) + + wind_totalmh[s,t] + = slackp[s,t]-slackm[s,t]; + +# upper bound and lower bounds for generation flows [MW] +Pgenequb{s in SCEN,t in TESTTIME,i in GEN}: Pgen[s,t,i] <= np_cap[i]*yuc[t,i]; +Pgeneqlb{s in SCEN,t in TESTTIME,i in GEN}: Pgen[s,t,i] >= 0; + +Pgenequbinit{i in GEN}: Pgen_initial[i] <= np_cap[i]*yuc[1,i]; +Pgeneqlbinit{i in GEN}: Pgen_initial[i] >= 0; + +# upper and lower bounds on ramps +upramp{s in SCEN,t in HORIZON,i in GEN}: Pgen[s,t+1,i] - Pgen[s,t,i] <= +max_ur[i]; +loramp{s in SCEN,t in HORIZON,i in GEN}: Pgen[s,t,i] - Pgen[s,t+1,i] >= -max_dr[i]; + +upramp0{s in SCEN,i in GEN}: Pgen[s,1,i] - Pgen_initial[i] <= +max_ur[i]; +loramp0{s in SCEN,i in GEN}: Pgen_initial[i] - Pgen[s,1,i] >= -max_dr[i]; + +# time-dependent generation costs +var tot_gen_cost{s in SCEN,t in TESTTIME} = 1e+0*sum{i in GEN} (gen_cost[i]*Pgen[s,t,i]); + +# time-dependent startup/shutdown costs +var tot_start_cost{t in TESTTIME} = 1e+3*(sum{i in GEN} yuc[t,i]); + +# time-dependent slack costs +var tot_slack_cost{s in SCEN,t in TESTTIME} = 1e+4*(slackp[s,t]+slackm[s,t]); + +# cost function +minimize cost: sum{t in TESTTIME} tot_start_cost[t] + + (1/(card(SCEN)))*sum{s in SCEN,t in TESTTIME} tot_gen_cost[s,t] + + (1/(card(SCEN)))*sum{s in SCEN,t in TESTTIME} tot_slack_cost[s,t]; Copied: SwiftApps/swift-galaxy/swift/old/gftplist.swift (from rev 7862, SwiftApps/swift-galaxy/swift/gftplist.swift) =================================================================== --- SwiftApps/swift-galaxy/swift/old/gftplist.swift (rev 0) +++ SwiftApps/swift-galaxy/swift/old/gftplist.swift 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,18 @@ +type file; + +app (file _status, file _stdout, file _stderr) stage (string _src, string _dst){ + stage _src _dst @_status stdout=@_stdout stderr=@_stderr; +} + +string src="gsiftp://trestles-dm2.sdsc.edu:2811/home/ketan/storage/"; +string dst="gsiftp://gridftp.stampede.tacc.utexas.edu:2811/scratch/01739/ketan/dest/"; + +string data[] = readData("datlist.txt"); + +foreach d, i in data{ + file status ; + file out ; + file err ; + //trace (@strcat(src, d)); + (status, out, err) = stage (@strcat(src, d), @strcat(dst, d)); +} Copied: SwiftApps/swift-galaxy/swift/old/mysim.xml (from rev 7862, SwiftApps/swift-galaxy/swift/mysim.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/mysim.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/mysim.xml 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,24 @@ + + A Swift tool to run the mysim workflow + + + swift-mysim "$site" "$nsim" "$steps" "$range" "$vals" "$html_file.files_path" "$log_file" "$html_file" + + + + + + + + + + + + + + + + + + + Copied: SwiftApps/swift-galaxy/swift/old/swift-2stage (from rev 7862, SwiftApps/swift-galaxy/swift/swift-2stage) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-2stage (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-2stage 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,319 @@ +#!/bin/bash + +site=$1 +data=$2 +n=$3 +outdir=$4 +logfile=$5 +htmlfile=$6 +mkdir -p $outdir +echo $@ >> /tmp/args.txt + +swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift +# use "here" document for configs + +cat << EOF > /tmp/2stage.swift +type file; +type script; + +app (file o) s (file i) { + cat @i stdout=@o; +} + +file out1[]; +file out2[]; + +string d=@arg("data"); + +foreach j in [1:@toInt(@arg("n","1"))] { + file data; + out1[j] = s(data); + out2[j] = s(out1[j]); +} +EOF + +case "$site" in +midway ) +cat << 'EOF' > /tmp/sites.xml + + + + sandyb + 16 + 02:00:00 + 7500 + 100 + 100 + 2 + 1 + 1 + .31 + 10000 + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +midway echo echo +midway cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +uc3 ) +cat << 'EOF' > /tmp/sites.xml + + + + + .09 + 1000 + 1 + 3600 + 1 + 100 + 100 + 1 + UidDomain == "osg-gk.mwt2.org" + 1 + "group_friends.{env.USER}" + nonshared + + + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +uc3 echo /bin/echo +uc3 cat /bin/cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +stampede ) +cat << 'EOF' > /tmp/sites.xml + + + + + + + 16 + 16 + TG-STA110005S + normal + ketancmaheshwari at gmail.com + ALL + 02:00:00 + 8000 + 100 + 100 + 1 + 1 + 1 + .15 + 10000 + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +stampede echo echo +stampede cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +localhost ) +cat << 'EOF' > /tmp/sites.xml + + + + + + /tmp/swift.work + 0.31 + + +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +* ) +echo "defaulting to localhost" + +cat << 'EOF' > /tmp/sites.xml + + + + + + /tmp/swift.work + 0.31 + + +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; +esac + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/2stage.swift -n=$n -data=$data 2> std.err +EXITCODE=$? + +swiftlog=`\ls -t *.log | head -1` + +#/tmp/plotswiftlogs $logfile 2>gnuplot.err + +cat << 'EOF' > /tmp/plotswiftlogs +#!/bin/bash + +#usage: ./extract4plots + +SWIFTLOGFILE=$1 +#TMPDIR=`mktemp -d plotlog.XXX` +grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out +SWIFTOUTFILE=swiftoutfile.out +#extract start time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` +START_TIME=`date +%s -d "$TMPDATE"` + +#extract end time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` +END_TIME=`date +%s -d "$TMPDATE"` + +#duration +DIFFTIME=$((END_TIME - START_TIME)) +#echo 0 > active.txt +#echo 0 > cumulative.txt +#extract active runs in a file +(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >> active.txt) + +#extract successful completions in a file +(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >> cumulative.txt) + +#prepare tics +activelines=`wc -l active.txt | awk '{print $1}'` +cumulines=`wc -l cumulative.txt | awk '{print $1}'` + +if [ $activelines -ne 0 ] +then + activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` +fi +seq 0 $activelinespertic $DIFFTIME > activetics.txt + +if [ $cumulines -ne 0 ] +then + cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` +fi + +seq 0 $cumulinespertic $DIFFTIME > cumultics.txt + +#final plot data +paste activetics.txt active.txt > plot_active.txt +paste cumultics.txt cumulative.txt > plot_cumulative.txt + +cat << EOF1 > plotit.gp +set terminal png enhanced +set output "activeplot.png" +set nokey +set xlabel "Time in sec" +set ylabel "number of active jobs" +set title "Active jobs" +plot "plot_active.txt" using 1:2 with line + +set output "cumulativeplot.png" +set ylabel "number of completed jobs" +set title "Cumulative jobs" +plot "plot_cumulative.txt" using 1:2 with lines +EOF1 + +gnuplot plotit.gp 2>/dev/null + +#cp *.png ../ +echo `pwd`>/tmp/curdir +echo `ls`>/tmp/dirlist +#cd .. +EOF + +sh /tmp/plotswiftlogs $swiftlog +cp *.png $outdir/ + +cat << EOF > $htmlfile + + + + + 2stage workflow output + + +

Swift run trace plots

+ +active plot +cumulative plot + +

Browse output of 2stage application

+`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` + + +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat std.err >&2 +fi +cp std.err $logfile +rm std.err +exit $EXITCODE Copied: SwiftApps/swift-galaxy/swift/old/swift-3stage (from rev 7862, SwiftApps/swift-galaxy/swift/swift-3stage) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-3stage (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-3stage 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,325 @@ +#!/bin/bash + +site=$1 +data=$2 +n=$3 +outdir=$4 +logfile=$5 +htmlfile=$6 +mkdir -p $outdir +echo $@ >> /tmp/args.txt + +swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift +# use "here" document for configs + +cat << EOF > /tmp/2stage.swift +type file; +type script; + +app (file o) s (file i) { + cat @i stdout=@o; +} + +file out1[]; +file out2[]; + +string d=@arg("data"); + +foreach j in [1:@toInt(@arg("n","1"))] { + file data; + out1[j] = s(data); + out2[j] = s(out1[j]); +} +EOF + +case "$site" in +midway ) +cat << 'EOF' > /tmp/sites.xml + + + + sandyb + 16 + 02:00:00 + 7500 + 100 + 100 + 2 + 1 + 1 + .31 + 10000 + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +midway echo echo +midway cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +uc3 ) +cat << 'EOF' > /tmp/sites.xml + + + + + .09 + 1000 + 1 + 3600 + 1 + 100 + 100 + 1 + UidDomain == "osg-gk.mwt2.org" + 1 + "group_friends.{env.USER}" + nonshared + + + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +uc3 echo /bin/echo +uc3 cat /bin/cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +stampede ) +cat << 'EOF' > /tmp/sites.xml + + + + + + + 16 + 16 + TG-STA110005S + normal + ketancmaheshwari at gmail.com + ALL + 02:00:00 + 8000 + 100 + 100 + 1 + 1 + 1 + .15 + 10000 + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +stampede echo echo +stampede cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +localhost ) +cat << 'EOF' > /tmp/sites.xml + + + + + + /tmp/swift.work + 0.31 + + +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +* ) +echo "defaulting to localhost" + +cat << 'EOF' > /tmp/sites.xml + + + + + + /tmp/swift.work + 0.31 + + +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; +esac + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/2stage.swift -n=$n -data=$data 2> std.err +EXITCODE=$? + +swiftlog=`\ls -t *.log | head -1` + +#/tmp/plotswiftlogs $logfile 2>gnuplot.err + +cat << 'EOF' > /tmp/plotswiftlogs +#!/bin/bash + +#usage: ./extract4plots + +SWIFTLOGFILE=$1 +#TMPDIR=`mktemp -d plotlog.XXX` +grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out +SWIFTOUTFILE=swiftoutfile.out +#extract start time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` +START_TIME=`date +%s -d "$TMPDATE"` + +#extract end time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` +END_TIME=`date +%s -d "$TMPDATE"` + +#duration +DIFFTIME=$((END_TIME - START_TIME)) +#echo 0 > active.txt +#echo 0 > cumulative.txt +#extract active runs in a file +(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt) + +#extract successful completions in a file +(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) + +#prepare tics +activelines=`wc -l active.txt | awk '{print $1}'` +cumulines=`wc -l cumulative.txt | awk '{print $1}'` + +if [ $activelines -ne 0 ] +then + activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` +fi +seq 0 $activelinespertic $DIFFTIME > activetics.txt + +if [ $cumulines -ne 0 ] +then + cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` +fi + +seq 0 $cumulinespertic $DIFFTIME > cumultics.txt + +#final plot data +paste activetics.txt active.txt > plot_active.txt +paste cumultics.txt cumulative.txt > plot_cumulative.txt + +cat << EOF1 > plotit.gp +set terminal png enhanced +set output "activeplot.png" +set nokey +set xlabel "Time in sec" +set ylabel "number of active jobs" +set title "Active jobs" +plot "plot_active.txt" using 1:2 with line + +set output "cumulativeplot.png" +set ylabel "number of completed jobs" +set title "Cumulative jobs" +plot "plot_cumulative.txt" using 1:2 with lines +EOF1 + +gnuplot plotit.gp 2>/dev/null + +#cp *.png ../ +echo `pwd`>/tmp/curdir +echo `ls`>/tmp/dirlist +#cd .. +EOF + +sh /tmp/plotswiftlogs $swiftlog +cp *.png $outdir/ + +#for i in `find $outdir` +#do +# lastfile=$i +#done +#bname=`basename $lastfile` + +cat << EOF > $htmlfile + + + + + 2stage workflow output + + +

Swift run trace plots

+ +active plot +cumulative plot + +

Browse output of 2stage application

+`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` + + +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat std.err >&2 +fi +cp std.err $logfile +rm std.err +exit $EXITCODE Copied: SwiftApps/swift-galaxy/swift/old/swift-skeleton.sh (from rev 7862, SwiftApps/swift-galaxy/swift/swift-skeleton.sh) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-skeleton.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-skeleton.sh 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,187 @@ +#!/bin/bash + +#set -x +set -e + +site=$1 +data=$2 +n=$3 +outdir=$4 +logfile=$5 +htmlfile=$6 +mkdir -p $outdir + +swift=`which swift` +# use "here" document for configs + +cat << EOF > /tmp/script.swift +### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< +# Place your Swift script here +# Make sure all data references are full-path +### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< +EOF + +case "$site" in +midway ) +cat << 'EOF' > /tmp/sites.xml + + + + + sandyb + 16 + 02:00:00 + 7500 + 100 + 100 + 2 + 1 + 1 + .31 + 10000 + /tmp/swift.work + + +EOF +;; + +uc3 ) +cat << 'EOF' > /tmp/sites.xml + + + + + + .09 + 1000 + 1 + 3600 + 1 + 100 + 100 + 1 + UidDomain == "osg-gk.mwt2.org" + 1 + "group_friends.{env.USER}" + nonshared + + + /tmp/swift.work + + +EOF +;; + +stampede ) +cat << 'EOF' > /tmp/sites.xml + + + + + + 16 + 16 + TG-STA110005S + normal + ketancmaheshwari at gmail.com + ALL + 02:00:00 + 8000 + 100 + 100 + 1 + 1 + 1 + .15 + 10000 + /tmp/swift.work + + +EOF +;; + +localhost ) +cat << 'EOF' > /tmp/sites.xml + + + + + + + /tmp/swift.work + 0.31 + + +EOF +;; + +* ) +echo "defaulting to localhost" + +cat << 'EOF' > /tmp/sites.xml + + + + + + + /tmp/swift.work + 0.31 + + +EOF +;; +esac + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +stampede echo echo +stampede cat cat +uc3 echo echo +uc3 cat cat +midway echo echo +midway cat cat +EOF + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> $logfile +EXITCODE=$? + +#for i in `find $outdir` +#do +# lastfile=$i +#done +#bname=`basename $lastfile` + +cat << EOF > $htmlfile + + + + + Catsn output + + +

Swift run trace plots

+ +active plot +cumulative plot + +

Browse output of catsn application

+`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` + + +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 +fi +exit $EXITCODE Copied: SwiftApps/swift-galaxy/swift/old/swift-skeleton.xml (from rev 7862, SwiftApps/swift-galaxy/swift/swift-skeleton.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-skeleton.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-skeleton.xml 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,22 @@ + + A description here + + + swift-skeleton.sh "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file" + + + + + + + + + + + + + + + + + Copied: SwiftApps/swift-galaxy/swift/old/tc (from rev 7862, SwiftApps/swift-galaxy/swift/tc) =================================================================== --- SwiftApps/swift-galaxy/swift/old/tc (rev 0) +++ SwiftApps/swift-galaxy/swift/old/tc 2014-05-12 19:44:03 UTC (rev 7865) @@ -0,0 +1,7 @@ +localhost echo echo +localhost cat cat +localhost gensample /home/maheshwari/powergridapps/bin/gensample.py +localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py +localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py +localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py +localhost stage /scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/dogftp.sh Deleted: SwiftApps/swift-galaxy/swift/swift-2stage =================================================================== --- SwiftApps/swift-galaxy/swift/swift-2stage 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/swift-2stage 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,319 +0,0 @@ -#!/bin/bash - -site=$1 -data=$2 -n=$3 -outdir=$4 -logfile=$5 -htmlfile=$6 -mkdir -p $outdir -echo $@ >> /tmp/args.txt - -swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift -# use "here" document for configs - -cat << EOF > /tmp/2stage.swift -type file; -type script; - -app (file o) s (file i) { - cat @i stdout=@o; -} - -file out1[]; -file out2[]; - -string d=@arg("data"); - -foreach j in [1:@toInt(@arg("n","1"))] { - file data; - out1[j] = s(data); - out2[j] = s(out1[j]); -} -EOF - -case "$site" in -midway ) -cat << 'EOF' > /tmp/sites.xml - - - - sandyb - 16 - 02:00:00 - 7500 - 100 - 100 - 2 - 1 - 1 - .31 - 10000 - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -midway echo echo -midway cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -uc3 ) -cat << 'EOF' > /tmp/sites.xml - - - - - .09 - 1000 - 1 - 3600 - 1 - 100 - 100 - 1 - UidDomain == "osg-gk.mwt2.org" - 1 - "group_friends.{env.USER}" - nonshared - - - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -uc3 echo /bin/echo -uc3 cat /bin/cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -stampede ) -cat << 'EOF' > /tmp/sites.xml - - - - - - - 16 - 16 - TG-STA110005S - normal - ketancmaheshwari at gmail.com - ALL - 02:00:00 - 8000 - 100 - 100 - 1 - 1 - 1 - .15 - 10000 - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -stampede echo echo -stampede cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -localhost ) -cat << 'EOF' > /tmp/sites.xml - - - - - - /tmp/swift.work - 0.31 - - -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -* ) -echo "defaulting to localhost" - -cat << 'EOF' > /tmp/sites.xml - - - - - - /tmp/swift.work - 0.31 - - -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; -esac - -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/2stage.swift -n=$n -data=$data 2> std.err -EXITCODE=$? - -swiftlog=`\ls -t *.log | head -1` - -#/tmp/plotswiftlogs $logfile 2>gnuplot.err - -cat << 'EOF' > /tmp/plotswiftlogs -#!/bin/bash - -#usage: ./extract4plots - -SWIFTLOGFILE=$1 -#TMPDIR=`mktemp -d plotlog.XXX` -grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out -SWIFTOUTFILE=swiftoutfile.out -#extract start time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` -START_TIME=`date +%s -d "$TMPDATE"` - -#extract end time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` -END_TIME=`date +%s -d "$TMPDATE"` - -#duration -DIFFTIME=$((END_TIME - START_TIME)) -#echo 0 > active.txt -#echo 0 > cumulative.txt -#extract active runs in a file -(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >> active.txt) - -#extract successful completions in a file -(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >> cumulative.txt) - -#prepare tics -activelines=`wc -l active.txt | awk '{print $1}'` -cumulines=`wc -l cumulative.txt | awk '{print $1}'` - -if [ $activelines -ne 0 ] -then - activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` -fi -seq 0 $activelinespertic $DIFFTIME > activetics.txt - -if [ $cumulines -ne 0 ] -then - cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` -fi - -seq 0 $cumulinespertic $DIFFTIME > cumultics.txt - -#final plot data -paste activetics.txt active.txt > plot_active.txt -paste cumultics.txt cumulative.txt > plot_cumulative.txt - -cat << EOF1 > plotit.gp -set terminal png enhanced -set output "activeplot.png" -set nokey -set xlabel "Time in sec" -set ylabel "number of active jobs" -set title "Active jobs" -plot "plot_active.txt" using 1:2 with line - -set output "cumulativeplot.png" -set ylabel "number of completed jobs" -set title "Cumulative jobs" -plot "plot_cumulative.txt" using 1:2 with lines -EOF1 - -gnuplot plotit.gp 2>/dev/null - -#cp *.png ../ -echo `pwd`>/tmp/curdir -echo `ls`>/tmp/dirlist -#cd .. -EOF - -sh /tmp/plotswiftlogs $swiftlog -cp *.png $outdir/ - -cat << EOF > $htmlfile - - - - - 2stage workflow output - - -

Swift run trace plots

- -active plot -cumulative plot - -

Browse output of 2stage application

-`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` - - -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat std.err >&2 -fi -cp std.err $logfile -rm std.err -exit $EXITCODE Deleted: SwiftApps/swift-galaxy/swift/swift-3stage =================================================================== --- SwiftApps/swift-galaxy/swift/swift-3stage 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/swift-3stage 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,325 +0,0 @@ -#!/bin/bash - -site=$1 -data=$2 -n=$3 -outdir=$4 -logfile=$5 -htmlfile=$6 -mkdir -p $outdir -echo $@ >> /tmp/args.txt - -swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift -# use "here" document for configs - -cat << EOF > /tmp/2stage.swift -type file; -type script; - -app (file o) s (file i) { - cat @i stdout=@o; -} - -file out1[]; -file out2[]; - -string d=@arg("data"); - -foreach j in [1:@toInt(@arg("n","1"))] { - file data; - out1[j] = s(data); - out2[j] = s(out1[j]); -} -EOF - -case "$site" in -midway ) -cat << 'EOF' > /tmp/sites.xml - - - - sandyb - 16 - 02:00:00 - 7500 - 100 - 100 - 2 - 1 - 1 - .31 - 10000 - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -midway echo echo -midway cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -uc3 ) -cat << 'EOF' > /tmp/sites.xml - - - - - .09 - 1000 - 1 - 3600 - 1 - 100 - 100 - 1 - UidDomain == "osg-gk.mwt2.org" - 1 - "group_friends.{env.USER}" - nonshared - - - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -uc3 echo /bin/echo -uc3 cat /bin/cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -stampede ) -cat << 'EOF' > /tmp/sites.xml - - - - - - - 16 - 16 - TG-STA110005S - normal - ketancmaheshwari at gmail.com - ALL - 02:00:00 - 8000 - 100 - 100 - 1 - 1 - 1 - .15 - 10000 - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -stampede echo echo -stampede cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -localhost ) -cat << 'EOF' > /tmp/sites.xml - - - - - - /tmp/swift.work - 0.31 - - -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -* ) -echo "defaulting to localhost" - -cat << 'EOF' > /tmp/sites.xml - - - - - - /tmp/swift.work - 0.31 - - -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; -esac - -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/2stage.swift -n=$n -data=$data 2> std.err -EXITCODE=$? - -swiftlog=`\ls -t *.log | head -1` - -#/tmp/plotswiftlogs $logfile 2>gnuplot.err - -cat << 'EOF' > /tmp/plotswiftlogs -#!/bin/bash - -#usage: ./extract4plots - -SWIFTLOGFILE=$1 -#TMPDIR=`mktemp -d plotlog.XXX` -grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out -SWIFTOUTFILE=swiftoutfile.out -#extract start time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` -START_TIME=`date +%s -d "$TMPDATE"` - -#extract end time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` -END_TIME=`date +%s -d "$TMPDATE"` - -#duration -DIFFTIME=$((END_TIME - START_TIME)) -#echo 0 > active.txt -#echo 0 > cumulative.txt -#extract active runs in a file -(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt) - -#extract successful completions in a file -(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) - -#prepare tics -activelines=`wc -l active.txt | awk '{print $1}'` -cumulines=`wc -l cumulative.txt | awk '{print $1}'` - -if [ $activelines -ne 0 ] -then - activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` -fi -seq 0 $activelinespertic $DIFFTIME > activetics.txt - -if [ $cumulines -ne 0 ] -then - cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` -fi - -seq 0 $cumulinespertic $DIFFTIME > cumultics.txt - -#final plot data -paste activetics.txt active.txt > plot_active.txt -paste cumultics.txt cumulative.txt > plot_cumulative.txt - -cat << EOF1 > plotit.gp -set terminal png enhanced -set output "activeplot.png" -set nokey -set xlabel "Time in sec" -set ylabel "number of active jobs" -set title "Active jobs" -plot "plot_active.txt" using 1:2 with line - -set output "cumulativeplot.png" -set ylabel "number of completed jobs" -set title "Cumulative jobs" -plot "plot_cumulative.txt" using 1:2 with lines -EOF1 - -gnuplot plotit.gp 2>/dev/null - -#cp *.png ../ -echo `pwd`>/tmp/curdir -echo `ls`>/tmp/dirlist -#cd .. -EOF - -sh /tmp/plotswiftlogs $swiftlog -cp *.png $outdir/ - -#for i in `find $outdir` -#do -# lastfile=$i -#done -#bname=`basename $lastfile` - -cat << EOF > $htmlfile - - - - - 2stage workflow output - - -

Swift run trace plots

- -active plot -cumulative plot - -

Browse output of 2stage application

-`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` - - -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat std.err >&2 -fi -cp std.err $logfile -rm std.err -exit $EXITCODE Deleted: SwiftApps/swift-galaxy/swift/swift-skeleton.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swift-skeleton.sh 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/swift-skeleton.sh 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,187 +0,0 @@ -#!/bin/bash - -#set -x -set -e - -site=$1 -data=$2 -n=$3 -outdir=$4 -logfile=$5 -htmlfile=$6 -mkdir -p $outdir - -swift=`which swift` -# use "here" document for configs - -cat << EOF > /tmp/script.swift -### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< -# Place your Swift script here -# Make sure all data references are full-path -### <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< -EOF - -case "$site" in -midway ) -cat << 'EOF' > /tmp/sites.xml - - - - - sandyb - 16 - 02:00:00 - 7500 - 100 - 100 - 2 - 1 - 1 - .31 - 10000 - /tmp/swift.work - - -EOF -;; - -uc3 ) -cat << 'EOF' > /tmp/sites.xml - - - - - - .09 - 1000 - 1 - 3600 - 1 - 100 - 100 - 1 - UidDomain == "osg-gk.mwt2.org" - 1 - "group_friends.{env.USER}" - nonshared - - - /tmp/swift.work - - -EOF -;; - -stampede ) -cat << 'EOF' > /tmp/sites.xml - - - - - - 16 - 16 - TG-STA110005S - normal - ketancmaheshwari at gmail.com - ALL - 02:00:00 - 8000 - 100 - 100 - 1 - 1 - 1 - .15 - 10000 - /tmp/swift.work - - -EOF -;; - -localhost ) -cat << 'EOF' > /tmp/sites.xml - - - - - - - /tmp/swift.work - 0.31 - - -EOF -;; - -* ) -echo "defaulting to localhost" - -cat << 'EOF' > /tmp/sites.xml - - - - - - - /tmp/swift.work - 0.31 - - -EOF -;; -esac - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -stampede echo echo -stampede cat cat -uc3 echo echo -uc3 cat cat -midway echo echo -midway cat cat -EOF - -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> $logfile -EXITCODE=$? - -#for i in `find $outdir` -#do -# lastfile=$i -#done -#bname=`basename $lastfile` - -cat << EOF > $htmlfile - - - - - Catsn output - - -

Swift run trace plots

- -active plot -cumulative plot - -

Browse output of catsn application

-`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` - - -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 -fi -exit $EXITCODE Deleted: SwiftApps/swift-galaxy/swift/swift-skeleton.xml =================================================================== --- SwiftApps/swift-galaxy/swift/swift-skeleton.xml 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/swift-skeleton.xml 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,22 +0,0 @@ - - A description here - - - swift-skeleton.sh "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file" - - - - - - - - - - - - - - - - - Deleted: SwiftApps/swift-galaxy/swift/tc =================================================================== --- SwiftApps/swift-galaxy/swift/tc 2014-05-12 19:42:49 UTC (rev 7864) +++ SwiftApps/swift-galaxy/swift/tc 2014-05-12 19:44:03 UTC (rev 7865) @@ -1,7 +0,0 @@ -localhost echo echo -localhost cat cat -localhost gensample /home/maheshwari/powergridapps/bin/gensample.py -localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py -localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py -localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py -localhost stage /scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/dogftp.sh From ketan at ci.uchicago.edu Mon May 12 14:45:47 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 14:45:47 -0500 (CDT) Subject: [Swift-commit] r7866 - in SwiftApps/swift-galaxy/swift: . old Message-ID: <20140512194547.071D19CCF4@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 14:45:46 -0500 (Mon, 12 May 2014) New Revision: 7866 Added: SwiftApps/swift-galaxy/swift/old/swiftforeach.sh SwiftApps/swift-galaxy/swift/old/swiftforeachlist.py SwiftApps/swift-galaxy/swift/old/swiftforeachrange.py SwiftApps/swift-galaxy/swift/old/swiftforeachrange.sh SwiftApps/swift-galaxy/swift/old/test_exec/ Removed: SwiftApps/swift-galaxy/swift/swiftforeach.sh SwiftApps/swift-galaxy/swift/swiftforeachlist.py SwiftApps/swift-galaxy/swift/swiftforeachrange.py SwiftApps/swift-galaxy/swift/swiftforeachrange.sh SwiftApps/swift-galaxy/swift/test_exec/ Log: clean Copied: SwiftApps/swift-galaxy/swift/old/swiftforeach.sh (from rev 7862, SwiftApps/swift-galaxy/swift/swiftforeach.sh) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swiftforeach.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swiftforeach.sh 2014-05-12 19:45:46 UTC (rev 7866) @@ -0,0 +1,282 @@ +#!/bin/bash +# originally list + +#echo $@ > /tmp/args.txt +#set workdir +wdir=$(mktemp -d /tmp/swift-gal.XXXX) + +#determine loop mode: over a list or a range +range_or_list=$1 +shift + +#determine invocation: Swift /K or /T +k_or_t=$1 +shift + +#build commandline arguments + +sites=$1 +shift +swiftargs=$1 +shift +interpreter=$1 +shift +executable=$1 +shift +stdin=$1 +shift + +if [ "$range_or_list" = "list" ] ; then + listfile=$1 + shift + n=$(wc -l < $listfile) +else + rstart=$1 + shift + rend=$1 + shift + stepsize=$1 + shift + n=$(( ($rend - $rstart) / $stepsize )) +fi + +outloc=$1 +shift +logfile=$1 +shift +outlistfile=$1 +shift +stringargs=$1 +shift + +swift=$(\which swift 2>/dev/null) +BASEDIR=$(dirname $0) + +#workout the file args +fileargs=() +while [ $# -gt 0 ] ; do + fileargs+=("\"$1\"") + shift +done + +#create outloc dir +mkdir -p $outloc + +#workout the stringargs +if [ "${stringargs}" ] ; then + stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; +else + stringarrayexpr='string stringargs[];' +fi + +#workout standard input file +if [ "${stdin}" ] ; then + if [ "${k_or_t}" = "swiftK" ] ; then + stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' + else + stdinfilexpr='file stdinfile = input('\"${stdin}\"');' + fi +else + if [ "${k_or_t}" = "swiftK" ] ; then + stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' + else + stdinfilexpr='file stdinfile = input('\"/dev/null\"');' + fi +fi + +#workout the array only if user adds args else make it blank +if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + filearrayexpr='file fileargs[] ;'; +else + filearrayexpr='file fileargs[];' +fi + +#build swift source + if [ "$range_or_list" == "list" ] ; then ## list + + cat << EOF > $wdir/script.swift + /* + * + * Swift source script is generated automatically via the Swift-Galaxy + * integration tool. Be careful before making any changes to it directly as + * it might be tied with other configuration and paramter values. + * + */ + type file; + + app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; + } + $stringarrayexpr + $filearrayexpr + $stdinfilexpr + + file exec<"$executable">; + file listfile<"$listfile">; + string list[] = readData(listfile); + foreach l, i in list{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); + } +EOF + +else ## range + +cat << EOF > $wdir/script.swift + /* + * + * Swift source script is generated automatically via the Swift-Galaxy + * integration tool. Be careful before making any changes to it directly as + * it might be tied with other configuration and paramter values. + * + */ +type file; + +app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; +} + +$stringarrayexpr +$filearrayexpr +$stdinfilexpr + +file exec<"$executable">; +foreach i in [$rstart:$rend:$stepsize]{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); +} +EOF + +fi + +touch None # Create a "None" file in case user does not specify any stdin file + + +############# End common setup ################ + + +if [ "$k_or_t" = "swiftK" ] ; then #### invoke K + +$BASEDIR/buildsite.sh $sites $wdir + +#build config +cat << EOF > $wdir/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF + +#build tc from sites and interpreter +IFS=',' read -ra S <<< "$sites" +for site in "${S[@]}" ; do +cat << EOF >> $wdir/tc +$site $interpreter $interpreter +EOF +done + +$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile +EXITCODE=$? + +else + +### Invoke T + + #workout the array only if user adds args else make it blank + if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + filearrayexpr='file fileargs[] ;'; + else + filearrayexpr='file fileargs[];' + fi + + cat << EOF > $wdir/script.swift + /* + * + * Swift source script is generated automatically via the Swift-Galaxy + * integration tool. Be careful before making any changes to it directly as + * it might be tied with other configuration and paramter values. + * + */ + + import files; + import string; + import io; + + app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + "$interpreter" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; + } + +main{ + $stringarrayexpr + $filearrayexpr + $stdinfilexpr + + file exec=input("$executable"); + $( if [ "$range_or_list" = "list" ] ; then + echo "file listfile=input("$listfile");" + echo "string list[] = split(read(listfile),"\n");" + echo "foreach l, i in list{" + echo " file out ;" + echo " file err ;" + else + echo "foreach i in [$rstart:$rend:$stepsize]{" + echo " file out ;" + echo " file err ;" + fi + ) + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); + } + printf("%s", "Done!"); + } +EOF + +EXITCODE=0 +case $sites in + "localhost" ) + #echo "in localhost" >> /tmp/status.txt + STC=$(\which stc) + #echo "got stc" >> /tmp/status.txt + turbine=$(\which turbine) + #echo "got turbine" >> /tmp/status.txt + $STC -j $(\which java) $wdir/script.swift 2>$logfile + #echo "done running stc" >> /tmp/status.txt + #echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt + $turbine -n $n $wdir/script.tcl "${swiftargs}" > /tmp/status.txt + EXITCODE=$? + #echo "exit code obtained" >> /tmp/status.txt + ;; +"Vesta" ) + TURBINE_HOME=/home/wozniak/Public/sfw/turbine + STC=/home/wozniak/Public/sfw/stc/bin/stc + $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile + $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl + EXITCODE=$? + ;; +"Mira" ) + TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine + STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc + $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile + $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl + EXITCODE=$? + ;; +esac + +fi + +cat << EOF > $outlistfile +$(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 +fi + +exit $EXITCODE + Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachlist.py (from rev 7862, SwiftApps/swift-galaxy/swift/swiftforeachlist.py) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swiftforeachlist.py (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swiftforeachlist.py 2014-05-12 19:45:46 UTC (rev 7866) @@ -0,0 +1,110 @@ +#!/usr/bin/env python + +import subprocess +import sys +#import rpy +import os + +def setwdir(): + return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) + +# numeric range +def main(): + print "%d" % (len(sys.argv)) + print "%s" % (sys.argv) + try: + sites=sys.argv[1] # comma separated list of sites + swiftargs=sys.argv[2] + interpreter=sys.argv[3] + executable=sys.argv[4] + rstart=sys.argv[5] + rend=sys.argv[6] + stepsize=sys.argv[7] + outloc=sys.argv[8] + outlistfile=sys.argv[9] + logfile=sys.argv[10] + stringargs=sys.argv[11] + except: + #rpy.stop_error('Wrong number or type of args') + sys.exit(1) + + print sites + print swiftargs + print interpreter + + #workout the array only if user adds args else make it blank + if stringargs: + stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n" + else: + stringarrayexpr="string stringargs[];\n" + + fileargs=list() + for files in sys.argv[12:]: + fileargs.append(files) + + #workout the array only if user adds file args else make it blank + if fileargs: + filearrayexpr="file fileargs[]="+str(fileargs)+";\n" + else: + filearrayexpr="file fileargs[];\n" + + wdir=setwdir().rstrip() + #Build Swift source code + f=open(wdir+"/script.swift", 'w'); + f.write("type file; \n") + + f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n") + f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n") + f.write("}\n") + + f.write(stringarrayexpr) + f.write(filearrayexpr) + + f.write("file exec<'"+executable+"'>;\n") + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n") + f.write("}\n") + f.close() + + swift=subprocess.check_output(["which", "swift"]).rstrip() + + #build site + if os.path.dirname(sys.argv[0]): + basedir=os.path.dirname(sys.argv[0]) + else: + basedir='.' + + print "%s %s %s" %(basedir, sites, wdir) + subprocess.call([basedir+"/buildsite.sh", sites, wdir]) + + #build config + f=open(wdir+"/cf", 'w'); + f.write('use.provider.staging=true') + f.write('wrapperlog.always.transfer=true') + f.write('execution.retries=0') + f.write('provider.staging.pin.swiftfiles=false') + f.write('sitedir.keep=true') + f.write('lazy.errors=false') + f.close() + + + #build tc + f=open(wdir+"/tc", 'w'); + for site in sites.split(): + f.write(site+' '+interpreter+' '+interpreter) + f.close() + + #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"]) + + #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` + #Populate output file + outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) + f=open(outlistfile) + f.write(outlist); + f.close() + #dum ditty dum ditty dum dum dum + +if __name__=='__main__': + main() Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachrange.py (from rev 7862, SwiftApps/swift-galaxy/swift/swiftforeachrange.py) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swiftforeachrange.py (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swiftforeachrange.py 2014-05-12 19:45:46 UTC (rev 7866) @@ -0,0 +1,110 @@ +#!/usr/bin/env python + +import subprocess +import sys +#import rpy +import os + +def setwdir(): + return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) + +# numeric range +def main(): + print "%d" % (len(sys.argv)) + print "%s" % (sys.argv) + try: + sites=sys.argv[1] # comma separated list of sites + swiftargs=sys.argv[2] + interpreter=sys.argv[3] + executable=sys.argv[4] + rstart=sys.argv[5] + rend=sys.argv[6] + stepsize=sys.argv[7] + outloc=sys.argv[8] + outlistfile=sys.argv[9] + logfile=sys.argv[10] + stringargs=sys.argv[11] + except: + #rpy.stop_error('Wrong number or type of args') + sys.exit(1) + + print sites + print swiftargs + print interpreter + + #workout the array only if user adds args else make it blank + if stringargs: + stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n" + else: + stringarrayexpr="string stringargs[];\n" + + fileargs=list() + for files in sys.argv[12:]: + fileargs.append(files) + + #workout the array only if user adds file args else make it blank + if fileargs: + filearrayexpr="file fileargs[]="+str(fileargs)+";\n" + else: + filearrayexpr="file fileargs[];\n" + + wdir=setwdir().rstrip() + #Build Swift source code + f=open(wdir+"/script.swift", 'w'); + f.write("type file; \n") + + f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n") + f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n") + f.write("}\n") + + f.write(stringarrayexpr) + f.write(filearrayexpr) + + f.write("file exec<'"+executable+"'>;\n") + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n") + f.write("}\n") + f.close() + + swift=subprocess.check_output(["which", "swift"]).rstrip() + + #build site + if os.path.dirname(sys.argv[0]): + basedir=os.path.dirname(sys.argv[0]) + else: + basedir='.' + + print "%s %s %s" %(basedir, sites, wdir) + subprocess.call([basedir+"/buildsite.sh", sites, wdir]) + + #build config + f=open(wdir+"/cf", 'w'); + f.write('use.provider.staging=true') + f.write('wrapperlog.always.transfer=true') + f.write('execution.retries=0') + f.write('provider.staging.pin.swiftfiles=false') + f.write('sitedir.keep=true') + f.write('lazy.errors=false') + f.close() + + + #build tc + f=open(wdir+"/tc", 'w'); + for site in sites.split(): + f.write(site+' '+interpreter+' '+interpreter) + f.close() + + #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"]) + + #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` + #Populate output file + outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) + f=open(outlistfile) + f.write(outlist); + f.close() + #dum ditty dum ditty dum dum dum + +if __name__=='__main__': + main() Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachrange.sh (from rev 7862, SwiftApps/swift-galaxy/swift/swiftforeachrange.sh) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swiftforeachrange.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swiftforeachrange.sh 2014-05-12 19:45:46 UTC (rev 7866) @@ -0,0 +1,138 @@ +#!/bin/bash + +echo $@ > /tmp/args.txt +wdir=$(mktemp -d /tmp/swift-gal.XXXX) + +k_or_t=$1 +shift +sites=$1 # comma separated list of sites +shift +swiftargs=$1 +shift +interpreter=$1 +shift +executable=$1 +shift +stdin=$1 +shift +rstart=$1 +shift +rend=$1 +shift +stepsize=$1 +shift +outloc=$1 +shift +logfile=$1 +shift +outlistfile=$1 +shift +stringargs=$1 +shift + +#workout the array only if user adds args else make it blank +if [ "${stringargs}" ] ; then + stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; +else + stringarrayexpr='string stringargs[];' +fi + +#workout standard input file +if [ "${stdin}" ] ; then + stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' +else + stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' +fi + +fileargs=() +while [ $# -gt 0 ] ; do + fileargs+=("\"$1\"") + shift +done + +#workout the array only if user adds file args else make it blank +if [ ${#fileargs[@]} -gt 0 ] ; then + tmp=$(printf -- '%s,' "${fileargs[@]}") + #file fileargs[] ; + filearrayexpr='file fileargs[] ;'; +else + filearrayexpr='file fileargs[];' +fi + +#Build Swift source code +cat << EOF > $wdir/script.swift + +/* +* +* Swift source script is generated automatically via the Swift-Galaxy +* integration tool. Be careful before making any changes to it directly as +* it might be tied with other configuration and paramter values. +* +*/ + +type file; + +app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ + $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; +} + +$stringarrayexpr +$filearrayexpr +$stdinfilexpr + +file exec<"$executable">; +foreach i in [$rstart:$rend:$stepsize]{ + file out ; + file err ; + (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); +} +EOF + +swift=$(which swift) + +#build site +BASEDIR=$(dirname $0) +$BASEDIR/buildsite.sh $sites $wdir + +#build config +cat << EOF > $wdir/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF + +#build tc +IFS=',' read -ra S <<< "$sites" +for site in "${S[@]}" ; do +cat << EOF >> $wdir/tc +$site $interpreter $interpreter +EOF +done + +touch None # Create a "None" file in case user does not specify any stdin file +if [ "$k_or_t" = "swiftK" ] ; then + $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile + EXITCODE=$? +else + echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt +fi + +#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` +#Populate output file +cat << EOF > $outlistfile +`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done` +EOF + +#cleanup +#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift + +if [ "$EXITCODE" -ne "0" ]; then + cat $logfile >&2 +fi + +exit $EXITCODE + +#dum ditty dum ditty dum dum dum Deleted: SwiftApps/swift-galaxy/swift/swiftforeach.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-12 19:44:03 UTC (rev 7865) +++ SwiftApps/swift-galaxy/swift/swiftforeach.sh 2014-05-12 19:45:46 UTC (rev 7866) @@ -1,282 +0,0 @@ -#!/bin/bash -# originally list - -#echo $@ > /tmp/args.txt -#set workdir -wdir=$(mktemp -d /tmp/swift-gal.XXXX) - -#determine loop mode: over a list or a range -range_or_list=$1 -shift - -#determine invocation: Swift /K or /T -k_or_t=$1 -shift - -#build commandline arguments - -sites=$1 -shift -swiftargs=$1 -shift -interpreter=$1 -shift -executable=$1 -shift -stdin=$1 -shift - -if [ "$range_or_list" = "list" ] ; then - listfile=$1 - shift - n=$(wc -l < $listfile) -else - rstart=$1 - shift - rend=$1 - shift - stepsize=$1 - shift - n=$(( ($rend - $rstart) / $stepsize )) -fi - -outloc=$1 -shift -logfile=$1 -shift -outlistfile=$1 -shift -stringargs=$1 -shift - -swift=$(\which swift 2>/dev/null) -BASEDIR=$(dirname $0) - -#workout the file args -fileargs=() -while [ $# -gt 0 ] ; do - fileargs+=("\"$1\"") - shift -done - -#create outloc dir -mkdir -p $outloc - -#workout the stringargs -if [ "${stringargs}" ] ; then - stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; -else - stringarrayexpr='string stringargs[];' -fi - -#workout standard input file -if [ "${stdin}" ] ; then - if [ "${k_or_t}" = "swiftK" ] ; then - stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' - else - stdinfilexpr='file stdinfile = input('\"${stdin}\"');' - fi -else - if [ "${k_or_t}" = "swiftK" ] ; then - stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' - else - stdinfilexpr='file stdinfile = input('\"/dev/null\"');' - fi -fi - -#workout the array only if user adds args else make it blank -if [ ${#fileargs[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${fileargs[@]}") - filearrayexpr='file fileargs[] ;'; -else - filearrayexpr='file fileargs[];' -fi - -#build swift source - if [ "$range_or_list" == "list" ] ; then ## list - - cat << EOF > $wdir/script.swift - /* - * - * Swift source script is generated automatically via the Swift-Galaxy - * integration tool. Be careful before making any changes to it directly as - * it might be tied with other configuration and paramter values. - * - */ - type file; - - app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ - $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; - } - $stringarrayexpr - $filearrayexpr - $stdinfilexpr - - file exec<"$executable">; - file listfile<"$listfile">; - string list[] = readData(listfile); - foreach l, i in list{ - file out ; - file err ; - (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); - } -EOF - -else ## range - -cat << EOF > $wdir/script.swift - /* - * - * Swift source script is generated automatically via the Swift-Galaxy - * integration tool. Be careful before making any changes to it directly as - * it might be tied with other configuration and paramter values. - * - */ -type file; - -app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ - $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; -} - -$stringarrayexpr -$filearrayexpr -$stdinfilexpr - -file exec<"$executable">; -foreach i in [$rstart:$rend:$stepsize]{ - file out ; - file err ; - (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); -} -EOF - -fi - -touch None # Create a "None" file in case user does not specify any stdin file - - -############# End common setup ################ - - -if [ "$k_or_t" = "swiftK" ] ; then #### invoke K - -$BASEDIR/buildsite.sh $sites $wdir - -#build config -cat << EOF > $wdir/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF - -#build tc from sites and interpreter -IFS=',' read -ra S <<< "$sites" -for site in "${S[@]}" ; do -cat << EOF >> $wdir/tc -$site $interpreter $interpreter -EOF -done - -$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile -EXITCODE=$? - -else - -### Invoke T - - #workout the array only if user adds args else make it blank - if [ ${#fileargs[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${fileargs[@]}") - filearrayexpr='file fileargs[] ;'; - else - filearrayexpr='file fileargs[];' - fi - - cat << EOF > $wdir/script.swift - /* - * - * Swift source script is generated automatically via the Swift-Galaxy - * integration tool. Be careful before making any changes to it directly as - * it might be tied with other configuration and paramter values. - * - */ - - import files; - import string; - import io; - - app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ - "$interpreter" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; - } - -main{ - $stringarrayexpr - $filearrayexpr - $stdinfilexpr - - file exec=input("$executable"); - $( if [ "$range_or_list" = "list" ] ; then - echo "file listfile=input("$listfile");" - echo "string list[] = split(read(listfile),"\n");" - echo "foreach l, i in list{" - echo " file out ;" - echo " file err ;" - else - echo "foreach i in [$rstart:$rend:$stepsize]{" - echo " file out ;" - echo " file err ;" - fi - ) - (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); - } - printf("%s", "Done!"); - } -EOF - -EXITCODE=0 -case $sites in - "localhost" ) - #echo "in localhost" >> /tmp/status.txt - STC=$(\which stc) - #echo "got stc" >> /tmp/status.txt - turbine=$(\which turbine) - #echo "got turbine" >> /tmp/status.txt - $STC -j $(\which java) $wdir/script.swift 2>$logfile - #echo "done running stc" >> /tmp/status.txt - #echo $turbine -n $n $wdir/script.tcl "${swiftargs}" >> /tmp/status.txt - $turbine -n $n $wdir/script.tcl "${swiftargs}" > /tmp/status.txt - EXITCODE=$? - #echo "exit code obtained" >> /tmp/status.txt - ;; -"Vesta" ) - TURBINE_HOME=/home/wozniak/Public/sfw/turbine - STC=/home/wozniak/Public/sfw/stc/bin/stc - $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile - $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl - EXITCODE=$? - ;; -"Mira" ) - TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine - STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc - $STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile - $TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl - EXITCODE=$? - ;; -esac - -fi - -cat << EOF > $outlistfile -$(for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done) -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 -fi - -exit $EXITCODE - Deleted: SwiftApps/swift-galaxy/swift/swiftforeachlist.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachlist.py 2014-05-12 19:44:03 UTC (rev 7865) +++ SwiftApps/swift-galaxy/swift/swiftforeachlist.py 2014-05-12 19:45:46 UTC (rev 7866) @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import sys -#import rpy -import os - -def setwdir(): - return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) - -# numeric range -def main(): - print "%d" % (len(sys.argv)) - print "%s" % (sys.argv) - try: - sites=sys.argv[1] # comma separated list of sites - swiftargs=sys.argv[2] - interpreter=sys.argv[3] - executable=sys.argv[4] - rstart=sys.argv[5] - rend=sys.argv[6] - stepsize=sys.argv[7] - outloc=sys.argv[8] - outlistfile=sys.argv[9] - logfile=sys.argv[10] - stringargs=sys.argv[11] - except: - #rpy.stop_error('Wrong number or type of args') - sys.exit(1) - - print sites - print swiftargs - print interpreter - - #workout the array only if user adds args else make it blank - if stringargs: - stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n" - else: - stringarrayexpr="string stringargs[];\n" - - fileargs=list() - for files in sys.argv[12:]: - fileargs.append(files) - - #workout the array only if user adds file args else make it blank - if fileargs: - filearrayexpr="file fileargs[]="+str(fileargs)+";\n" - else: - filearrayexpr="file fileargs[];\n" - - wdir=setwdir().rstrip() - #Build Swift source code - f=open(wdir+"/script.swift", 'w'); - f.write("type file; \n") - - f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n") - f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n") - f.write("}\n") - - f.write(stringarrayexpr) - f.write(filearrayexpr) - - f.write("file exec<'"+executable+"'>;\n") - f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n") - f.write("}\n") - f.close() - - swift=subprocess.check_output(["which", "swift"]).rstrip() - - #build site - if os.path.dirname(sys.argv[0]): - basedir=os.path.dirname(sys.argv[0]) - else: - basedir='.' - - print "%s %s %s" %(basedir, sites, wdir) - subprocess.call([basedir+"/buildsite.sh", sites, wdir]) - - #build config - f=open(wdir+"/cf", 'w'); - f.write('use.provider.staging=true') - f.write('wrapperlog.always.transfer=true') - f.write('execution.retries=0') - f.write('provider.staging.pin.swiftfiles=false') - f.write('sitedir.keep=true') - f.write('lazy.errors=false') - f.close() - - - #build tc - f=open(wdir+"/tc", 'w'); - for site in sites.split(): - f.write(site+' '+interpreter+' '+interpreter) - f.close() - - #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"]) - - #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` - #Populate output file - outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) - f=open(outlistfile) - f.write(outlist); - f.close() - #dum ditty dum ditty dum dum dum - -if __name__=='__main__': - main() Deleted: SwiftApps/swift-galaxy/swift/swiftforeachrange.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachrange.py 2014-05-12 19:44:03 UTC (rev 7865) +++ SwiftApps/swift-galaxy/swift/swiftforeachrange.py 2014-05-12 19:45:46 UTC (rev 7866) @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import sys -#import rpy -import os - -def setwdir(): - return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) - -# numeric range -def main(): - print "%d" % (len(sys.argv)) - print "%s" % (sys.argv) - try: - sites=sys.argv[1] # comma separated list of sites - swiftargs=sys.argv[2] - interpreter=sys.argv[3] - executable=sys.argv[4] - rstart=sys.argv[5] - rend=sys.argv[6] - stepsize=sys.argv[7] - outloc=sys.argv[8] - outlistfile=sys.argv[9] - logfile=sys.argv[10] - stringargs=sys.argv[11] - except: - #rpy.stop_error('Wrong number or type of args') - sys.exit(1) - - print sites - print swiftargs - print interpreter - - #workout the array only if user adds args else make it blank - if stringargs: - stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n" - else: - stringarrayexpr="string stringargs[];\n" - - fileargs=list() - for files in sys.argv[12:]: - fileargs.append(files) - - #workout the array only if user adds file args else make it blank - if fileargs: - filearrayexpr="file fileargs[]="+str(fileargs)+";\n" - else: - filearrayexpr="file fileargs[];\n" - - wdir=setwdir().rstrip() - #Build Swift source code - f=open(wdir+"/script.swift", 'w'); - f.write("type file; \n") - - f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n") - f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n") - f.write("}\n") - - f.write(stringarrayexpr) - f.write(filearrayexpr) - - f.write("file exec<'"+executable+"'>;\n") - f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n") - f.write("}\n") - f.close() - - swift=subprocess.check_output(["which", "swift"]).rstrip() - - #build site - if os.path.dirname(sys.argv[0]): - basedir=os.path.dirname(sys.argv[0]) - else: - basedir='.' - - print "%s %s %s" %(basedir, sites, wdir) - subprocess.call([basedir+"/buildsite.sh", sites, wdir]) - - #build config - f=open(wdir+"/cf", 'w'); - f.write('use.provider.staging=true') - f.write('wrapperlog.always.transfer=true') - f.write('execution.retries=0') - f.write('provider.staging.pin.swiftfiles=false') - f.write('sitedir.keep=true') - f.write('lazy.errors=false') - f.close() - - - #build tc - f=open(wdir+"/tc", 'w'); - for site in sites.split(): - f.write(site+' '+interpreter+' '+interpreter) - f.close() - - #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"]) - - #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` - #Populate output file - outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) - f=open(outlistfile) - f.write(outlist); - f.close() - #dum ditty dum ditty dum dum dum - -if __name__=='__main__': - main() Deleted: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-05-12 19:44:03 UTC (rev 7865) +++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh 2014-05-12 19:45:46 UTC (rev 7866) @@ -1,138 +0,0 @@ -#!/bin/bash - -echo $@ > /tmp/args.txt -wdir=$(mktemp -d /tmp/swift-gal.XXXX) - -k_or_t=$1 -shift -sites=$1 # comma separated list of sites -shift -swiftargs=$1 -shift -interpreter=$1 -shift -executable=$1 -shift -stdin=$1 -shift -rstart=$1 -shift -rend=$1 -shift -stepsize=$1 -shift -outloc=$1 -shift -logfile=$1 -shift -outlistfile=$1 -shift -stringargs=$1 -shift - -#workout the array only if user adds args else make it blank -if [ "${stringargs}" ] ; then - stringarrayexpr='string stringargs[]=['$(echo $stringargs | sed -e 's/ *$//g' -e 's/[^ ][^ ]*/"&"/g' -e 's/ /,/g')'];'; -else - stringarrayexpr='string stringargs[];' -fi - -#workout standard input file -if [ "${stdin}" ] ; then - stdinfilexpr='file stdinfile<'\"${stdin}\"'>;' -else - stdinfilexpr='file stdinfile<'\"/dev/null\"'>;' -fi - -fileargs=() -while [ $# -gt 0 ] ; do - fileargs+=("\"$1\"") - shift -done - -#workout the array only if user adds file args else make it blank -if [ ${#fileargs[@]} -gt 0 ] ; then - tmp=$(printf -- '%s,' "${fileargs[@]}") - #file fileargs[] ; - filearrayexpr='file fileargs[] ;'; -else - filearrayexpr='file fileargs[];' -fi - -#Build Swift source code -cat << EOF > $wdir/script.swift - -/* -* -* Swift source script is generated automatically via the Swift-Galaxy -* integration tool. Be careful before making any changes to it directly as -* it might be tied with other configuration and paramter values. -* -*/ - -type file; - -app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){ - $interpreter @_exec _i _stringargs @_fileargs stdin=@_stdin stdout=@_out stderr=@_err; -} - -$stringarrayexpr -$filearrayexpr -$stdinfilexpr - -file exec<"$executable">; -foreach i in [$rstart:$rend:$stepsize]{ - file out ; - file err ; - (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile); -} -EOF - -swift=$(which swift) - -#build site -BASEDIR=$(dirname $0) -$BASEDIR/buildsite.sh $sites $wdir - -#build config -cat << EOF > $wdir/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF - -#build tc -IFS=',' read -ra S <<< "$sites" -for site in "${S[@]}" ; do -cat << EOF >> $wdir/tc -$site $interpreter $interpreter -EOF -done - -touch None # Create a "None" file in case user does not specify any stdin file -if [ "$k_or_t" = "swiftK" ] ; then - $swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile - EXITCODE=$? -else - echo "stc + turbine; not implemented yet" > /tmp/swiftT.txt -fi - -#`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` -#Populate output file -cat << EOF > $outlistfile -`for i in $(find $outloc -type f -iname "*.out"); do echo "$i"; done` -EOF - -#cleanup -#rm -rf /tmp/sites.xml /tmp/tc /tmp/cf /tmp/script.swift - -if [ "$EXITCODE" -ne "0" ]; then - cat $logfile >&2 -fi - -exit $EXITCODE - -#dum ditty dum ditty dum dum dum From ketan at ci.uchicago.edu Mon May 12 16:00:07 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 16:00:07 -0500 (CDT) Subject: [Swift-commit] r7867 - SwiftApps/swift-galaxy/swift Message-ID: <20140512210007.0EE409CCF4@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 16:00:06 -0500 (Mon, 12 May 2014) New Revision: 7867 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py SwiftApps/swift-galaxy/swift/swiftforeach.xml Log: clean up in code and some notes Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 19:45:46 UTC (rev 7866) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 21:00:06 UTC (rev 7867) @@ -37,19 +37,12 @@ try: this=next(myargs) range_or_list = next(myargs) - #print "range or list=> "+ range_or_list k_or_t = next(myargs) - #print "k or t=> "+ k_or_t sites = next(myargs) - #print "sites=> " + sites swiftargs = next(myargs) - #print "swiftargs=> " + swiftargs interpreter = next(myargs) - #print "interpreter=> " + interpreter executable = next(myargs) - #print "executable=> " + executable stdin = next(myargs) - #print "stdin => " + stdin if range_or_list == "list": #list of items @@ -64,7 +57,6 @@ outloc = next(myargs) outlistfile = next(myargs) - #print "outlistfile=>"+outlistfile logfile = next(myargs) stringargs = next(myargs) except: @@ -105,10 +97,8 @@ #workout standard input file if stdin != "None": if k_or_t == "swiftK": - #print "stdinK" stdinfilexpr = "file stdinfile<\""+stdin+"\">;\n" else: - #print "stdinT" stdinfilexpr = "file stdinfile = input(\""+stdin+"\");\n" else: #print "no stdin" @@ -137,8 +127,8 @@ f.write("file listfile<\""+listfile+"\">;\n") f.write("string list[] = readData(listfile);\n"); f.write("foreach l, i in list{\n") - f.write("file out ;\n") - f.write("file err ;\n") + f.write(" file out ;\n") + f.write(" file err ;\n") else: @@ -157,15 +147,13 @@ #print "%s %s %s" %(basedir, sites, wdir) subprocess.call([basedir+"/buildsite.sh", sites, wdir]) - #build config + #build config file buildcfg(wdir) - #build tc - tc=open(wdir+"/tc", 'w'); - for site in sites.split(): - tc.write(site+' '+interpreter+' '+interpreter) - - tc.close() + #build tc file + with open(wdir+"/tc", 'w') as tc: + for site in sites.split(): + tc.write(site+' '+interpreter+' '+interpreter) else: # T @@ -174,7 +162,7 @@ f.write("import io;\n"); f.write("\n"); f.write(" app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[], file _stdin){\n"); - f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n"); + f.write(" \"" +interpreter+"\" _exec _i _stringargs _fileargs @stdin=_stdin @stdout=_out @stderr=_err; \n"); f.write("}\n"); f.write("main{\n"); f.write(stringarrayexpr + " \n"); @@ -203,12 +191,10 @@ f.write("}\n"); f.close() - #Invocation if k_or_t == "swiftK": - #$swift -sites.file $wdir/sites.xml -tc.file $wdir/tc -config $wdir/cf $wdir/script.swift "${swiftargs}" 2>$logfile with open(logfile,"wb") as errfile: exitcode=subprocess.check_call([swift, "-sites.file", wdir+"/sites.xml", "-tc.file", wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs],stderr=errfile) print exitcode @@ -216,7 +202,6 @@ else: #print "Invoking T" - #EXITCODE=0 stc=distutils.spawn.find_executable("stc") java=distutils.spawn.find_executable("java") turbine= distutils.spawn.find_executable("turbine") @@ -246,24 +231,19 @@ elif sites == "Mira": print "mira" - + pwd=os.path.getcwd() + turbine_home="/home/wozniak/Public/sfw/ppc64/turbine" + stc="/home/wozniak/Public/sfw/ppc64/stc/bin/stc" + with open(logfile, "wb") as errfile: + exitcode=subprocess.check_call([stc, "-t", "checkpointing", "-r", pwd, "-j", java, wdir+"/script.swift"], stderr=errfile) + + if exitcode == 0: + with open(logfile, "wb") as errfile, open("/dev/null", "r") as infile: + exitcode=subprocess.check_call([turbine_home+"/scripts/submit/cobalt/turbine-cobalt-run.zsh", "-n", str(n), wdir+"/script.tcl"], stdin=infile, stderr=errfile) + else: - print "site not recognized" - #"Vesta" ) - #STC=/home/wozniak/Public/sfw/stc/bin/stc - #$STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile - #$TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -s $HOME/cf $wdir/script.tcl - #EXITCODE=$? - #;; - #"Mira" ) - #TURBINE_HOME=/home/wozniak/Public/sfw/ppc64/turbine - #STC=/home/wozniak/Public/sfw/ppc64/stc/bin/stc - #$STC -j $(\which java) -t checkpointing -r ${PWD} $wdir/script.swift 2>$logfile - #$TURBINE_HOME/scripts/submit/cobalt/turbine-cobalt-run.zsh -n $n -i $HOME/prerun.sh -s $HOME/cf $wdir/script.tcl - #EXITCODE=$? - #;; - #esac + print "Site not recognized." #Populate output file outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) Modified: SwiftApps/swift-galaxy/swift/swiftforeach.xml =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-12 19:45:46 UTC (rev 7866) +++ SwiftApps/swift-galaxy/swift/swiftforeach.xml 2014-05-12 21:00:06 UTC (rev 7867) @@ -82,7 +82,7 @@ - + @@ -130,26 +130,7 @@ - + From ketan at ci.uchicago.edu Mon May 12 17:06:51 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 12 May 2014 17:06:51 -0500 (CDT) Subject: [Swift-commit] r7868 - SwiftApps/swift-galaxy/swift Message-ID: <20140512220651.DDD4A9D587@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-12 17:06:49 -0500 (Mon, 12 May 2014) New Revision: 7868 Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py Log: small change Modified: SwiftApps/swift-galaxy/swift/swiftforeach.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 21:00:06 UTC (rev 7867) +++ SwiftApps/swift-galaxy/swift/swiftforeach.py 2014-05-12 22:06:49 UTC (rev 7868) @@ -109,7 +109,7 @@ #Build Swift source code - f = open(wdir+"/script.swift", 'w'); + f = open(wdir+"/script.swift", "w"); if k_or_t == "swiftK": @@ -182,10 +182,10 @@ else: f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{ \n"); - f.write(" file out ; \n"); - f.write(" file err ; \n"); + f.write(" file out ; \n"); + f.write(" file err ; \n"); - f.write("(out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") + f.write(" (out,err) = anapp(exec, i, stringargs, fileargs, stdinfile);\n") f.write("}\n"); f.write("printf(\"%s\", \"Done!\");\n"); f.write("}\n"); From ketan at ci.uchicago.edu Tue May 13 09:33:33 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 13 May 2014 09:33:33 -0500 (CDT) Subject: [Swift-commit] r7869 - in SwiftApps/swift-galaxy/swift: . old Message-ID: <20140513143333.273759D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-13 09:33:31 -0500 (Tue, 13 May 2014) New Revision: 7869 Added: SwiftApps/swift-galaxy/swift/old/catsn.swift SwiftApps/swift-galaxy/swift/old/dssat.xml SwiftApps/swift-galaxy/swift/old/genswift.xml SwiftApps/swift-galaxy/swift/old/postprocess.py SwiftApps/swift-galaxy/swift/old/powergrid.xml SwiftApps/swift-galaxy/swift/old/sites.xml SwiftApps/swift-galaxy/swift/old/statswift.xml SwiftApps/swift-galaxy/swift/old/swift-catsn.sh SwiftApps/swift-galaxy/swift/old/swift-catsn.xml SwiftApps/swift-galaxy/swift/old/swift-powergrid SwiftApps/swift-galaxy/swift/old/swift-stats SwiftApps/swift-galaxy/swift/old/swift-try SwiftApps/swift-galaxy/swift/old/swiftforeachpy.xml SwiftApps/swift-galaxy/swift/old/swiftforeachtest.py SwiftApps/swift-galaxy/swift/old/testrun SwiftApps/swift-galaxy/swift/old/try.xml Removed: SwiftApps/swift-galaxy/swift/catsn.swift SwiftApps/swift-galaxy/swift/dssat.xml SwiftApps/swift-galaxy/swift/genswift.xml SwiftApps/swift-galaxy/swift/postprocess.py SwiftApps/swift-galaxy/swift/powergrid.xml SwiftApps/swift-galaxy/swift/sites.xml SwiftApps/swift-galaxy/swift/statswift.xml SwiftApps/swift-galaxy/swift/swift-catsn.sh SwiftApps/swift-galaxy/swift/swift-catsn.xml SwiftApps/swift-galaxy/swift/swift-powergrid SwiftApps/swift-galaxy/swift/swift-stats SwiftApps/swift-galaxy/swift/swift-try SwiftApps/swift-galaxy/swift/swiftforeachpy.xml SwiftApps/swift-galaxy/swift/swiftforeachtest.py SwiftApps/swift-galaxy/swift/testrun SwiftApps/swift-galaxy/swift/try.xml Log: move remaining old stuff Deleted: SwiftApps/swift-galaxy/swift/catsn.swift =================================================================== --- SwiftApps/swift-galaxy/swift/catsn.swift 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/catsn.swift 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,11 +0,0 @@ -type file; - -app (file o) cat (file i) { - cat @i stdout=@o; -} -file out[]; - -foreach j in [1:10] { - file data; - out[j] = cat(data); -} Deleted: SwiftApps/swift-galaxy/swift/dssat.xml =================================================================== --- SwiftApps/swift-galaxy/swift/dssat.xml 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/dssat.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,27 +0,0 @@ - - Swift - DSSAT - - - /scratch/midway/maheshwari/DSSAT.cloud/DSSAT/dssat-galaxy.sh $njobs $template - - - - - - - - - - - - - - - - - - - - - - Deleted: SwiftApps/swift-galaxy/swift/genswift.xml =================================================================== --- SwiftApps/swift-galaxy/swift/genswift.xml 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/genswift.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,170 +0,0 @@ - - - A generic tool to run executable via Swift foreach parallel construct - - - - #if $rangeorlist.rl == "rng" - swiftforeachrange.sh - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${rangeorlist.rngstart}" - "${rangeorlist.rend}" - "${rangeorlist.stepsize}" - "${outloc}" - "${logfile}" - "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #elif $rangeorlist.rl == "lst" - swiftforeachlist.sh - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${rangeorlist.listfile}" - "${outloc}" - "${logfile}" - "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #else - swiftforeachtest.sh - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${rangeorlist.rngstart}" - "${rangeorlist.rend}" - "${rangeorlist.stepsize}" - "${rangeorlist.listfile}" - "${outloc}" - "${logfile}" - "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #end if - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -.. class:: warningmark - -**TIP**. Add args to provide additional arguments to your executable. - ------ - -**What it does** - -This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file. - -Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user. - -Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces. - ------ - -**Example** - -Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello". - -If an executable is echo.sh with following line: - -echo $@ - -The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8: - -echo n hello - - - Copied: SwiftApps/swift-galaxy/swift/old/catsn.swift (from rev 7868, SwiftApps/swift-galaxy/swift/catsn.swift) =================================================================== --- SwiftApps/swift-galaxy/swift/old/catsn.swift (rev 0) +++ SwiftApps/swift-galaxy/swift/old/catsn.swift 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,11 @@ +type file; + +app (file o) cat (file i) { + cat @i stdout=@o; +} +file out[]; + +foreach j in [1:10] { + file data; + out[j] = cat(data); +} Copied: SwiftApps/swift-galaxy/swift/old/dssat.xml (from rev 7868, SwiftApps/swift-galaxy/swift/dssat.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/dssat.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/dssat.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,27 @@ + + Swift - DSSAT + + + /scratch/midway/maheshwari/DSSAT.cloud/DSSAT/dssat-galaxy.sh $njobs $template + + + + + + + + + + + + + + + + + + + + + + Copied: SwiftApps/swift-galaxy/swift/old/genswift.xml (from rev 7868, SwiftApps/swift-galaxy/swift/genswift.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/genswift.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/genswift.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,170 @@ + + + A generic tool to run executable via Swift foreach parallel construct + + + + #if $rangeorlist.rl == "rng" + swiftforeachrange.sh + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${rangeorlist.rngstart}" + "${rangeorlist.rend}" + "${rangeorlist.stepsize}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #elif $rangeorlist.rl == "lst" + swiftforeachlist.sh + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${rangeorlist.listfile}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #else + swiftforeachtest.sh + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${rangeorlist.rngstart}" + "${rangeorlist.rend}" + "${rangeorlist.stepsize}" + "${rangeorlist.listfile}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #end if + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +.. class:: warningmark + +**TIP**. Add args to provide additional arguments to your executable. + +----- + +**What it does** + +This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file. + +Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user. + +Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces. + +----- + +**Example** + +Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello". + +If an executable is echo.sh with following line: + +echo $@ + +The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8: + +echo n hello + + + Copied: SwiftApps/swift-galaxy/swift/old/postprocess.py (from rev 7868, SwiftApps/swift-galaxy/swift/postprocess.py) =================================================================== --- SwiftApps/swift-galaxy/swift/old/postprocess.py (rev 0) +++ SwiftApps/swift-galaxy/swift/old/postprocess.py 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,65 @@ +#post processing, set build for data and add additional data to history^M +from galaxy import datatypes, config, jobs, tools +from galaxy.model import HistoryDatasetAssociation +import operator +import shutil +import os +import glob +from os.path import basename + +def createDsRef(outDataItems, param_dict, outputName, outputFormat, outputDsRefName, nameSpace): + dsrefXML="\n" + dsrefXML=dsrefXML+"\n" + + dsrefFileName="" + + for outDataName, outDataItem in outDataItems: + if outDataName==outputDsRefName: + dsrefFileName=outDataItem.get_file_name() + else: + if outDataName==outputName: + dsrefXML=dsrefXML+"\n" % (outDataItem.id,outDataItem.name,outDataItem.get_file_name(),outputFormat) + + print "dsrefFileName:"+dsrefFileName + + primaryCollectedDatasets=param_dict["__collected_datasets__"]["primary"] + + for primaryCollectedDataset in primaryCollectedDatasets: + primaryCollectedDatasetItems=primaryCollectedDatasets[primaryCollectedDataset] + for primaryCollectedDatasetItem in primaryCollectedDatasetItems: + outDataItem=primaryCollectedDatasetItems[primaryCollectedDatasetItem] + if outputFormat in outDataItem.extension: + + dsrefXML=dsrefXML+"\n" % (outDataItem.id,outDataItem.name,outDataItem.get_file_name(),outDataItem.extension) + + dsrefXML=dsrefXML+"\n" + + return dsrefFileName, dsrefXML + +def writeDsRef(dsrefFileName, dsrefXML): + dsrefFile = open(dsrefFileName, "w", 0) + dsrefFile.write(dsrefXML) + dsrefFile.flush() + dsrefFile.close() + +def setSizeAndPeak(outDataItems,outputDsRefName): + for outDataName, outDataItem in outDataItems: + if outDataName==outputDsRefName: + outDataItem.dataset.file_size=None + outDataItem.set_size() + outDataItem.set_peek() + +def writeoutlist(app, inp_data, out_data, param_dict, tool, stdout, stderr): + + outDataItems = out_data.items() + outDataItems = sorted(outDataItems, key=operator.itemgetter(0)) + + dsrefAcmoFileName, dsrefAcmoXML = createDsRef(outDataItems,param_dict,"acmo","acmo.csv","acmoDsRef","facit/acmo") + + writeDsRef(dsrefAcmoFileName, dsrefAcmoXML) + + setSizeAndPeak(outDataItems,"acmoDsRef") + + + app.model.context.flush() + Copied: SwiftApps/swift-galaxy/swift/old/powergrid.xml (from rev 7868, SwiftApps/swift-galaxy/swift/powergrid.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/powergrid.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/powergrid.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,32 @@ + + Swift - Power Grid Inference for Stochastic Optimization + + + swift-powergrid $site $batch $log_file + + + + + + + + + + + + + + + + + + + + Copied: SwiftApps/swift-galaxy/swift/old/sites.xml (from rev 7868, SwiftApps/swift-galaxy/swift/sites.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/sites.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/sites.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,139 @@ + + + + + + + + + /tmp/swift.workdir + 0 + + + + + + + + + + + + + + + + + + + + + Copied: SwiftApps/swift-galaxy/swift/old/statswift.xml (from rev 7868, SwiftApps/swift-galaxy/swift/statswift.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/statswift.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/statswift.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,47 @@ + + A Swift tool to run stats on compute nodes + + + swift-stats "$site.site_select" "$n" "$html_file.files_path" "$log_file" "$html_file" "$site.walltime" "$site.slots" "$site.queue" "$site.nodes" "$site.throttle" "$site.workdir" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Copied: SwiftApps/swift-galaxy/swift/old/swift-catsn.sh (from rev 7868, SwiftApps/swift-galaxy/swift/swift-catsn.sh) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-catsn.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-catsn.sh 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,276 @@ +#!/bin/bash + +#set -x +set -e + +site=$1 +data=$2 +n=$3 +outdir=$4 +logfile=$5 +htmlfile=$6 +mkdir -p $outdir + +swift=`which swift` +# use "here" document for configs + +cat << EOF > /tmp/catsn.swift +type file; + +app (file o) cat (file i) { + cat @i stdout=@o; +} +#file out[]; +file out[]; + +string d=@arg("data"); + +foreach j in [1:@toInt(@arg("n","1"))] { + file data; + out[j] = cat(data); +} +EOF + +case "$site" in +midway ) +cat << 'EOF' > /tmp/sites.xml + + + + + sandyb + 16 + 02:00:00 + 7500 + 100 + 100 + 2 + 1 + 1 + .31 + 10000 + /tmp/swift.work + + +EOF +;; + +uc3 ) +cat << 'EOF' > /tmp/sites.xml + + + + + + .09 + 1000 + 1 + 3600 + 1 + 100 + 100 + 1 + UidDomain == "osg-gk.mwt2.org" + 1 + "group_friends.{env.USER}" + nonshared + + + /tmp/swift.work + + +EOF +;; + +stampede ) +cat << 'EOF' > /tmp/sites.xml + + + + + + 16 + 16 + TG-STA110005S + normal + ketancmaheshwari at gmail.com + ALL + 02:00:00 + 8000 + 100 + 100 + 1 + 1 + 1 + .15 + 10000 + /tmp/swift.work + + +EOF +;; + +localhost ) +cat << 'EOF' > /tmp/sites.xml + + + + + + + /tmp/swift.work + 0.31 + + +EOF +;; + +* ) +echo "defaulting to localhost" + +cat << 'EOF' > /tmp/sites.xml + + + + + + + /tmp/swift.work + 0.31 + + +EOF +;; +esac + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +stampede echo echo +stampede cat cat +uc3 echo /bin/echo +uc3 cat /bin/cat +midway echo echo +midway cat cat +EOF + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> std.err +EXITCODE=$? + +swiftlog=`\ls -t *.log | head -1` + +#/tmp/plotswiftlogs $logfile 2>gnuplot.err +cat << 'EOF' > /tmp/plotswiftlogs +#!/bin/bash + +#usage: ./extract4plots + +SWIFTLOGFILE=$1 +#TMPDIR=`mktemp -d plotlog.XXX` +grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out +SWIFTOUTFILE=swiftoutfile.out +#extract start time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` +START_TIME=`date +%s -d "$TMPDATE"` + +#extract end time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` +END_TIME=`date +%s -d "$TMPDATE"` + +#duration +DIFFTIME=$((END_TIME - START_TIME)) +#echo 0 > active.txt +#echo 0 > cumulative.txt +#extract active runs in a file +(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt) + +#extract successful completions in a file +(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) + +#prepare tics +activelines=`wc -l active.txt | awk '{print $1}'` +cumulines=`wc -l cumulative.txt | awk '{print $1}'` + +if [ $activelines -ne 0 ] +then + activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` +fi +seq 0 $activelinespertic $DIFFTIME > activetics.txt + +if [ $cumulines -ne 0 ] +then + cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` +fi + +seq 0 $cumulinespertic $DIFFTIME > cumultics.txt + +#final plot data +paste activetics.txt active.txt > plot_active.txt +paste cumultics.txt cumulative.txt > plot_cumulative.txt + +cat << EOF1 > plotit.gp +set terminal png enhanced +set output "activeplot.png" +set nokey +set xlabel "Time in sec" +set ylabel "number of active jobs" +set title "Active jobs" +plot "plot_active.txt" using 1:2 with line + +set output "cumulativeplot.png" +set ylabel "number of completed jobs" +set title "Cumulative jobs" +plot "plot_cumulative.txt" using 1:2 with lines +EOF1 + +gnuplot plotit.gp 2>/dev/null + +#cp *.png ../ +echo `pwd`>/tmp/curdir +echo `ls`>/tmp/dirlist +#cd .. +EOF + +sh /tmp/plotswiftlogs $swiftlog +cp *.png $outdir/ + +#for i in `find $outdir` +#do +# lastfile=$i +#done +#bname=`basename $lastfile` + +cat << EOF > $htmlfile + + + + + Catsn output + + +

Swift run trace plots

+ +active plot +cumulative plot + +

Browse output of catsn application

+`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` + + +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat std.err >&2 +fi +cp std.err $logfile +rm std.err +exit $EXITCODE Copied: SwiftApps/swift-galaxy/swift/old/swift-catsn.xml (from rev 7868, SwiftApps/swift-galaxy/swift/swift-catsn.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-catsn.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-catsn.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,22 @@ + + A Swift tool to run the catsn workflow + + + swift-catsn "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file" + + + + + + + + + + + + + + + + + Copied: SwiftApps/swift-galaxy/swift/old/swift-powergrid (from rev 7868, SwiftApps/swift-galaxy/swift/swift-powergrid) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-powergrid (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-powergrid 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,321 @@ +#!/bin/bash + +site=$1 +batch=$2 +#stochmodel=$3 +#edrevised=$4 +#asciidata=$5 +#printres=$6 +#illinois=$7 +#wind_data=$8 +logfile=$3 + +#cp $stochmodel stochmodel.run +#cp $edrevised edrevised.mod +#cp $asciidata set_data_ascii.inc +#cp $printres print_results_revised.inc +#cp $illinois Illinois.tgz +#cp $wind_data wind_data.dat +echo $@>>/tmp/tmp.txt +#swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift +swift=/home/maheshwari/swift-devel/cog/modules/swift/dist/swift-svn/bin/swift +# use "here" document for configs + +cat << 'EOF' > /tmp/inference4galaxy.swift +type file; + +app ( file _obj_out, file _xhat_out1, file _xhat_out2, file _ampl_res, file _out) + ampl_app ( int _s, + int _fix_x, + file _stoch_model, + file _illinois, + file _edrevised_model, + file _ascii_data, + file _print_results, + file _wind_data + ) { + runampl _s _fix_x @_stoch_model @_xhat_out1 @_xhat_out2 @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_res @_wind_data stdout=@_out; +} + +app (file _obj_out, file _ampl_l_res, file _out) ampl_app_L ( + int _s, + int _fix_x, + file _stoch_model, + file _illinois, + file _edrevised_model, + file _ascii_data, + file _print_results + ) { + + runampl_l _s _fix_x @_stoch_model @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_l_res stdout=@_out; +} + +app (file _obj_out, file _ampl_u_res, file _out) ampl_app_U ( + int _s, + int _fix_x, + file _stoch_model, + file _xhat, + file _illinois, + file _edrevised_model, + file _ascii_data, + file _print_results, + file _wind_data + ) { + + runampl_u _s _fix_x @_stoch_model @_xhat @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_u_res @_wind_data stdout=@_out; +} +file stoch_model<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/stochmodel.run">; +file edrevised_model<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/edrevised.mod">; +file ascii_data<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/set_data_ascii.inc">; +file print_results<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/print_results_revised.inc">; +file illinois<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/Illinois.tgz">; +file wind_data<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/wind_data.dat">; + +file out[] ; +file obj_out[] ; +file ampl_res[] ; + +file xhat_out1[] ; +file xhat_out2[] ; + +file out_l[][][] ; +file obj_out_l[][][] ; +file ampl_l_res[][][] ; +file xhat_out_l1[][][] ; +file xhat_out_l2[][][] ; + +file out_u[][][] ; +file obj_out_u[][][] ; +file ampl_u_res[][][] ; +#file upper_out[][][] ; +#file lower_out[][][] ; + +int nS[] = [10]; //number of scenario sweeps +int Batch = @toInt(@arg("batch", "20")); + +foreach S, idxs in nS{ +(obj_out[idxs], xhat_out1[idxs], xhat_out2[idxs], ampl_res[idxs], out[idxs]) = ampl_app (S, 0, stoch_model, illinois, edrevised_model, ascii_data, print_results, wind_data); + + +foreach B, idxb in [10:Batch:10]{ + foreach k in [0:B]{ + (obj_out_l[idxs][idxb][k], xhat_out_l1[idxs][idxb][k], xhat_out_l2[idxs][idxb][k], ampl_l_res[idxs][idxb][k], out_l[idxs][idxb][k]) = ampl_app (S, 0, stoch_model, illinois, edrevised_model, ascii_data, print_results, wind_data); + + (obj_out_u[idxs][idxb][k], ampl_u_res[idxs][idxb][k], out_u[idxs][idxb][k]) = ampl_app_U(S, 1, stoch_model, xhat_out1[idxs], illinois, edrevised_model, ascii_data, print_results, wind_data); + +} +} +} +EOF + +case "$site" in +midway ) +cat << 'EOF' > /tmp/sites.xml + + + + + sandyb + 16 + 02:00:00 + 7500 + 100 + 100 + 2 + 1 + 1 + .31 + 10000 + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +midway echo echo +midway cat cat +midway gensample /home/maheshwari/powergridapps/bin/gensample.py +midway runampl /home/maheshwari/powergridapps/bin/run_ampl.py +midway runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py +midway runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +use.wrapper.staging=false +EOF +;; + +uc3 ) +cat << 'EOF' > /tmp/sites.xml + + + + + .09 + 1000 + 1 + 3600 + 1 + 100 + 100 + 1 + UidDomain == "osg-gk.mwt2.org" + 1 + "group_friends.{env.USER}" + nonshared + + + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +uc3 echo /bin/echo +uc3 cat /bin/cat +uc3 runampl run_ampl.py +uc3 runampl_l run_ampl_l.py +uc3 runampl_u run_ampl_u.py +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +stampede ) +cat << 'EOF' > /tmp/sites.xml + + + + + + 16 + 16 + TG-EAR130015 + normal + ketancmaheshwari at gmail.com + ALL + 02:00:00 + 8000 + 100 + 100 + 1 + 1 + 1 + .15 + 10000 + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +stampede echo echo +stampede cat cat +stampede gensample gensample.py +stampede runampl run_ampl.py +stampede runampl_l run_ampl_l.py +stampede runampl_u run_ampl_u.py +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +use.wrapper.staging=false +EOF +;; + +localhost ) +cat << 'EOF' > /tmp/sites.xml + + + + + + /tmp/swift.work + 0.2 + + +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +localhost gensample /home/maheshwari/powergridapps/bin/gensample.py +localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py +localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py +localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +use.wrapper.staging=false +EOF +;; + +* ) +echo "defaulting to localhost" +cat << 'EOF' > /tmp/sites.xml + + + + + + /tmp/swift.work + 0.31 + + +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +localhost gensample /home/maheshwari/powergridapps/bin/gensample.py +localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py +localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py +localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; +esac + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/inference4galaxy.swift -batch=$batch 2> std.err +#$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/inference4galaxy.swift -batch=$batch -stochmodel=stochmodel.run -edrevised=edrevised.mod -asciidata=set_data_ascii.inc -printres=print_results_revised.inc -illinois=Illinois.tgz -wind_data=wind_data.dat 2> std.err + +EXITCODE=$? + +if [ "$EXITCODE" -ne "0" ]; then + cat std.err >&2 +fi +cp std.err $logfile +rm std.err +exit $EXITCODE Copied: SwiftApps/swift-galaxy/swift/old/swift-stats (from rev 7868, SwiftApps/swift-galaxy/swift/swift-stats) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-stats (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-stats 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,205 @@ +#!/bin/bash +# swift-stats "$site.site_select" "$n" "$html_file.files_path" "$log_file" "$html_file" "$site.walltime" "$site.slots" "$site.queue" "$site.nodes" "$site.throttle" "$site.workdir" +# 1 2 3 4 5 6 7 8 9 10 11 + +site=$1 +n=$2 +outdir=$3 +logfile=$4 +htmlfile=$5 +walltime=$6 +slots=$7 +queue=$8 +nodes=$9 +conc=${10} +workdir=${11} +throttle=$(echo "scale=5;($conc-1)/100" |bc) + +mkdir -p $outdir + +echo $6 > /tmp/args.txt + +swift=`\which swift` + +# use "here" document for configs +cat << EOF > /tmp/stats.swift +type file; + +app (file o) statsapp () { + stats stdout=@o; +} + +file out[]; + +foreach j in [1:@toInt(@arg("n","2"))] { + out[j] = statsapp (); +} +EOF + +case "$site" in +clusterPBS ) +cat << EOF > /tmp/sites.xml + + + + + default + 24 + ${walltime} + pbs.aprun;pbs.mpp;depth=24 + 100 + 100 + ${slots} + ${nodes} + ${nodes} + ${throttle} + 10000 + ${workdir} + + +EOF + +cat << EOF > /tmp/tc +clusterpbs echo echo +clusterpbs cat cat +clusterpbs stats stats.sh +EOF + +cat << EOF > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +clusterSLURM ) +cat << EOF > /tmp/sites.xml + + + + + ${queue} + 8 + 8 + ${walltime} + 100 + 100 + ${slots} + ${nodes} + ${nodes} + ${throttle} + 10000 + ${workdir} + + +EOF + +cat << EOF > /tmp/tc +clusterslurm echo echo +clusterslurm cat cat +clusterslurm stats stats.sh +EOF + +cat << EOF > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +localhost ) +cat << EOF > /tmp/sites.xml + + + + + 0.02 + 10000 + + /tmp/swift.work + local + + +EOF + +cat << EOF > /tmp/tc +localhost echo echo +localhost cat cat +localhost stats stats.sh +EOF + +cat << EOF > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +* ) +echo "defaulting to localhost" + +cat << EOF > /tmp/sites.xml + + + + + 0.02 + 10000 + + /tmp/swift.work + local + + +EOF + +cat << EOF > /tmp/tc +localhost echo echo +localhost cat cat +localhost stats stats.sh +EOF + +cat << EOF > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=0 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; +esac + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/stats.swift -n=$n -data=$data 2> std.err +EXITCODE=$? + +swiftlog=`\ls -t *.log | head -1` + +cat << EOF > $htmlfile + + + + + Hello output + + +

Browse output of hello application

+`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` + + +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat std.err >&2 +fi +cp std.err $logfile +rm std.err +exit $EXITCODE Copied: SwiftApps/swift-galaxy/swift/old/swift-try (from rev 7868, SwiftApps/swift-galaxy/swift/swift-try) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swift-try (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swift-try 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,341 @@ +#!/bin/bash + +site=$1 +nsim=$2 +steps=$3 +range=$4 +values=$5 +outdir=$6 +logfile=$7 +htmlfile=$8 + +mkdir -p $outdir + +swift=`which swift` +# use "here" document for configs + +cat << EOF > /tmp/mysim.swift +type file; + +app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values) +{ + simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log; +} + +app (file out, file log) analyze (file s[]) +{ + stats @filenames(s) stdout=@out stderr=@log; +} + +int nsim = @toInt(@arg("nsim", "10")); +int steps = @toInt(@arg("steps", "1")); +int range = @toInt(@arg("range", "100")); +int values = @toInt(@arg("values", "5")); + +file sims[]; + +foreach i in [0:nsim-1] { + file simout ; + file simlog ; + (simout,simlog) = simulation(steps,range,values); + sims[i] = simout; +} + +file stats_out<"output/average.out">; +file stats_log<"output/average.log">; +(stats_out, stats_log) = analyze(sims); + +EOF + +case "$site" in +midway ) +cat << 'EOF' > /tmp/sites.xml + + + + + sandyb + 16 + 02:00:00 + 7500 + 100 + 100 + 4 + 2 + 2 + 1.27 + 10000 + /scratch/midway/maheshwari/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +midway echo echo +midway cat cat +midway simulate simulate.sh +midway stats stats.sh +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +uc3 ) +cat << 'EOF' > /tmp/sites.xml + + + + + + .09 + 1000 + 1 + 3600 + 1 + 100 + 100 + 1 + UidDomain == "osg-gk.mwt2.org" + 1 + "group_friends.{env.USER}" + nonshared + + + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +uc3 echo /bin/echo +uc3 cat /bin/cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +stampede ) +cat << 'EOF' > /tmp/sites.xml + + + + + + + + 16 + 16 + TG-STA110005S + normal + ketancmaheshwari at gmail.com + ALL + 02:00:00 + 8000 + 100 + 100 + 1 + 1 + 1 + .15 + 10000 + /tmp/swift.work + + +EOF + +cat << 'EOF' > /tmp/tc +stampede echo echo +stampede cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=true +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +localhost ) +cat << 'EOF' > /tmp/sites.xml + + + + + + + /tmp/swift.work + 0.31 + + +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; + +* ) +echo "defaulting to localhost" + +cat << 'EOF' > /tmp/sites.xml + + + + + + + /tmp/swift.work + 0.31 + + +EOF + +cat << 'EOF' > /tmp/tc +localhost echo echo +localhost cat cat +EOF + +cat << 'EOF' > /tmp/cf +use.provider.staging=false +wrapperlog.always.transfer=true +execution.retries=1 +provider.staging.pin.swiftfiles=false +sitedir.keep=true +lazy.errors=false +EOF +;; +esac + +$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/mysim.swift -nsim=$nsim -steps=$steps -range=$range -values=$values 2> std.err +EXITCODE=$? + +swiftlog=`\ls -t *.log | head -1` + +#/tmp/plotswiftlogs $logfile 2>gnuplot.err +cat << 'EOF' > /tmp/plotswiftlogs +#!/bin/bash + +#usage: ./plotswiftlogs.ketan + +SWIFTLOGFILE=$1 + +#TMPDIR=`mktemp -d plotlog.XXX` + +grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out + +SWIFTOUTFILE=swiftoutfile.out + +#extract start time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` +START_TIME=`date +%s -d "$TMPDATE"` + +#extract end time +TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` +END_TIME=`date +%s -d "$TMPDATE"` + +#duration +DIFFTIME=$((END_TIME - START_TIME)) + +#extract active runs in a file +(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt) + +#extract successful completions in a file +(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) + +#prepare tics +activelines=`wc -l active.txt | awk '{print $1}'` +cumulines=`wc -l cumulative.txt | awk '{print $1}'` + +if [ $activelines -ne 0 ] +then + activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` +fi + +seq 0 $activelinespertic $DIFFTIME > activetics.txt + +if [ $cumulines -ne 0 ] +then + cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` +fi + +seq 0 $cumulinespertic $DIFFTIME > cumultics.txt + +#final plot data +paste activetics.txt active.txt > plot_active.txt +paste cumultics.txt cumulative.txt > plot_cumulative.txt + +cat << EOF1 > plotit.gp +set terminal png enhanced +set nokey +set output "cumulativeplot.png" +set xlabel "Time in sec" +set ylabel "number of completed apps" +set title "Cumulative Apps Completed" +plot "plot_cumulative.txt" using 1:2 with lines +set output "activeplot.png" +set xlabel "Time in sec" +set ylabel "number of active apps" +set title "Active Apps" +plot "plot_active.txt" using 1:2 with line +EOF1 + +gnuplot plotit.gp 2>/dev/null +rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt + +EOF + +sh /tmp/plotswiftlogs $swiftlog +cp *.png $outdir/ + +cat << EOF > $htmlfile + + + + + Application output + + +

Swift run trace plots

+ +active plot +cumulative plot + +

Browse output of application

+`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` + + +EOF + +if [ "$EXITCODE" -ne "0" ]; then + cat std.err >&2 +fi +cp std.err $logfile +rm std.err +exit $EXITCODE Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachpy.xml (from rev 7868, SwiftApps/swift-galaxy/swift/swiftforeachpy.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swiftforeachpy.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swiftforeachpy.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,170 @@ + + + A generic tool to run executable via Swift foreach parallel construct + + + + #if $rangeorlist.rl == "rng" + swiftforeachrange.py + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${rangeorlist.rngstart}" + "${rangeorlist.rend}" + "${rangeorlist.stepsize}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #elif $rangeorlist.rl == "lst" + swiftforeachlist.py + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${rangeorlist.listfile}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #else + swiftforeachtest.py + "${site}" + "${swiftargs}" + "${interpret}" + "${exec}" + "${rangeorlist.rngstart}" + "${rangeorlist.rend}" + "${rangeorlist.stepsize}" + "${rangeorlist.listfile}" + "${outloc}" + "${logfile}" + "${outlist}" + "${stringargs}" + #for $a in $filearg + ${a.fileargname} + #end for + #end if + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +.. class:: warningmark + +**TIP**. Add args to provide additional arguments to your executable. + +----- + +**What it does** + +This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file. + +Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user. + +Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces. + +----- + +**Example** + +Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello". + +If an executable is echo.sh with following line: + +echo $@ + +The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8: + +echo n hello + + + Copied: SwiftApps/swift-galaxy/swift/old/swiftforeachtest.py (from rev 7868, SwiftApps/swift-galaxy/swift/swiftforeachtest.py) =================================================================== --- SwiftApps/swift-galaxy/swift/old/swiftforeachtest.py (rev 0) +++ SwiftApps/swift-galaxy/swift/old/swiftforeachtest.py 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,110 @@ +#!/usr/bin/env python + +import subprocess +import sys +#import rpy +import os + +def setwdir(): + return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) + +# numeric range +def main(): + print "%d" % (len(sys.argv)) + print "%s" % (sys.argv) + try: + sites=sys.argv[1] # comma separated list of sites + swiftargs=sys.argv[2] + interpreter=sys.argv[3] + executable=sys.argv[4] + rstart=sys.argv[5] + rend=sys.argv[6] + stepsize=sys.argv[7] + outloc=sys.argv[8] + outlistfile=sys.argv[9] + logfile=sys.argv[10] + stringargs=sys.argv[11] + except: + #rpy.stop_error('Wrong number or type of args') + sys.exit(1) + + print sites + print swiftargs + print interpreter + + #workout the array only if user adds args else make it blank + if stringargs: + stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n" + else: + stringarrayexpr="string stringargs[];\n" + + fileargs=list() + for files in sys.argv[12:]: + fileargs.append(files) + + #workout the array only if user adds file args else make it blank + if fileargs: + filearrayexpr="file fileargs[]="+str(fileargs)+";\n" + else: + filearrayexpr="file fileargs[];\n" + + wdir=setwdir().rstrip() + #Build Swift source code + f=open(wdir+"/script.swift", 'w'); + f.write("type file; \n") + + f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n") + f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n") + f.write("}\n") + + f.write(stringarrayexpr) + f.write(filearrayexpr) + + f.write("file exec<'"+executable+"'>;\n") + f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") + f.write(" file out ;\n") + f.write(" file err ;\n") + f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n") + f.write("}\n") + f.close() + + swift=subprocess.check_output(["which", "swift"]).rstrip() + + #build site + if os.path.dirname(sys.argv[0]): + basedir=os.path.dirname(sys.argv[0]) + else: + basedir='.' + + print "%s %s %s" %(basedir, sites, wdir) + subprocess.call([basedir+"/buildsite.sh", sites, wdir]) + + #build config + f=open(wdir+"/cf", 'w'); + f.write('use.provider.staging=true') + f.write('wrapperlog.always.transfer=true') + f.write('execution.retries=0') + f.write('provider.staging.pin.swiftfiles=false') + f.write('sitedir.keep=true') + f.write('lazy.errors=false') + f.close() + + + #build tc + f=open(wdir+"/tc", 'w'); + for site in sites.split(): + f.write(site+' '+interpreter+' '+interpreter) + f.close() + + #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"]) + + #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` + #Populate output file + outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) + f=open(outlistfile) + f.write(outlist); + f.close() + #dum ditty dum ditty dum dum dum + +if __name__=='__main__': + main() Copied: SwiftApps/swift-galaxy/swift/old/testrun (from rev 7868, SwiftApps/swift-galaxy/swift/testrun) =================================================================== --- SwiftApps/swift-galaxy/swift/old/testrun (rev 0) +++ SwiftApps/swift-galaxy/swift/old/testrun 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,2 @@ +python swiftforeachrange.py localhost someargs java /Users/ketan/galaxy-dist/database/files/000/dataset_248.dat 0 9 1 /Users/ketan/galaxy-dist/database/files/000/dataset_306.dat /Users/ketan/swift-sandbox /Users/ketan/galaxy-dist/database/files/000/dataset_307.dat /Users/ketan/galaxy-dist/database/files/000/dataset_308.dat hello /Users/ketan/galaxy-dist/database/files/000/dataset_302.dat + Copied: SwiftApps/swift-galaxy/swift/old/try.xml (from rev 7868, SwiftApps/swift-galaxy/swift/try.xml) =================================================================== --- SwiftApps/swift-galaxy/swift/old/try.xml (rev 0) +++ SwiftApps/swift-galaxy/swift/old/try.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -0,0 +1,31 @@ + + Try Swift + + + swift-try $site $swiftscript $log_file + + + + + + + + + + + + + + + + + + + Deleted: SwiftApps/swift-galaxy/swift/postprocess.py =================================================================== --- SwiftApps/swift-galaxy/swift/postprocess.py 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/postprocess.py 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,65 +0,0 @@ -#post processing, set build for data and add additional data to history^M -from galaxy import datatypes, config, jobs, tools -from galaxy.model import HistoryDatasetAssociation -import operator -import shutil -import os -import glob -from os.path import basename - -def createDsRef(outDataItems, param_dict, outputName, outputFormat, outputDsRefName, nameSpace): - dsrefXML="\n" - dsrefXML=dsrefXML+"\n" - - dsrefFileName="" - - for outDataName, outDataItem in outDataItems: - if outDataName==outputDsRefName: - dsrefFileName=outDataItem.get_file_name() - else: - if outDataName==outputName: - dsrefXML=dsrefXML+"\n" % (outDataItem.id,outDataItem.name,outDataItem.get_file_name(),outputFormat) - - print "dsrefFileName:"+dsrefFileName - - primaryCollectedDatasets=param_dict["__collected_datasets__"]["primary"] - - for primaryCollectedDataset in primaryCollectedDatasets: - primaryCollectedDatasetItems=primaryCollectedDatasets[primaryCollectedDataset] - for primaryCollectedDatasetItem in primaryCollectedDatasetItems: - outDataItem=primaryCollectedDatasetItems[primaryCollectedDatasetItem] - if outputFormat in outDataItem.extension: - - dsrefXML=dsrefXML+"\n" % (outDataItem.id,outDataItem.name,outDataItem.get_file_name(),outDataItem.extension) - - dsrefXML=dsrefXML+"\n" - - return dsrefFileName, dsrefXML - -def writeDsRef(dsrefFileName, dsrefXML): - dsrefFile = open(dsrefFileName, "w", 0) - dsrefFile.write(dsrefXML) - dsrefFile.flush() - dsrefFile.close() - -def setSizeAndPeak(outDataItems,outputDsRefName): - for outDataName, outDataItem in outDataItems: - if outDataName==outputDsRefName: - outDataItem.dataset.file_size=None - outDataItem.set_size() - outDataItem.set_peek() - -def writeoutlist(app, inp_data, out_data, param_dict, tool, stdout, stderr): - - outDataItems = out_data.items() - outDataItems = sorted(outDataItems, key=operator.itemgetter(0)) - - dsrefAcmoFileName, dsrefAcmoXML = createDsRef(outDataItems,param_dict,"acmo","acmo.csv","acmoDsRef","facit/acmo") - - writeDsRef(dsrefAcmoFileName, dsrefAcmoXML) - - setSizeAndPeak(outDataItems,"acmoDsRef") - - - app.model.context.flush() - Deleted: SwiftApps/swift-galaxy/swift/powergrid.xml =================================================================== --- SwiftApps/swift-galaxy/swift/powergrid.xml 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/powergrid.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,32 +0,0 @@ - - Swift - Power Grid Inference for Stochastic Optimization - - - swift-powergrid $site $batch $log_file - - - - - - - - - - - - - - - - - - - - Deleted: SwiftApps/swift-galaxy/swift/sites.xml =================================================================== --- SwiftApps/swift-galaxy/swift/sites.xml 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/sites.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,139 +0,0 @@ - - - - - - - - - /tmp/swift.workdir - 0 - - - - - - - - - - - - - - - - - - - - - Deleted: SwiftApps/swift-galaxy/swift/statswift.xml =================================================================== --- SwiftApps/swift-galaxy/swift/statswift.xml 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/statswift.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,47 +0,0 @@ - - A Swift tool to run stats on compute nodes - - - swift-stats "$site.site_select" "$n" "$html_file.files_path" "$log_file" "$html_file" "$site.walltime" "$site.slots" "$site.queue" "$site.nodes" "$site.throttle" "$site.workdir" - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Deleted: SwiftApps/swift-galaxy/swift/swift-catsn.sh =================================================================== --- SwiftApps/swift-galaxy/swift/swift-catsn.sh 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/swift-catsn.sh 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,276 +0,0 @@ -#!/bin/bash - -#set -x -set -e - -site=$1 -data=$2 -n=$3 -outdir=$4 -logfile=$5 -htmlfile=$6 -mkdir -p $outdir - -swift=`which swift` -# use "here" document for configs - -cat << EOF > /tmp/catsn.swift -type file; - -app (file o) cat (file i) { - cat @i stdout=@o; -} -#file out[]; -file out[]; - -string d=@arg("data"); - -foreach j in [1:@toInt(@arg("n","1"))] { - file data; - out[j] = cat(data); -} -EOF - -case "$site" in -midway ) -cat << 'EOF' > /tmp/sites.xml - - - - - sandyb - 16 - 02:00:00 - 7500 - 100 - 100 - 2 - 1 - 1 - .31 - 10000 - /tmp/swift.work - - -EOF -;; - -uc3 ) -cat << 'EOF' > /tmp/sites.xml - - - - - - .09 - 1000 - 1 - 3600 - 1 - 100 - 100 - 1 - UidDomain == "osg-gk.mwt2.org" - 1 - "group_friends.{env.USER}" - nonshared - - - /tmp/swift.work - - -EOF -;; - -stampede ) -cat << 'EOF' > /tmp/sites.xml - - - - - - 16 - 16 - TG-STA110005S - normal - ketancmaheshwari at gmail.com - ALL - 02:00:00 - 8000 - 100 - 100 - 1 - 1 - 1 - .15 - 10000 - /tmp/swift.work - - -EOF -;; - -localhost ) -cat << 'EOF' > /tmp/sites.xml - - - - - - - /tmp/swift.work - 0.31 - - -EOF -;; - -* ) -echo "defaulting to localhost" - -cat << 'EOF' > /tmp/sites.xml - - - - - - - /tmp/swift.work - 0.31 - - -EOF -;; -esac - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -stampede echo echo -stampede cat cat -uc3 echo /bin/echo -uc3 cat /bin/cat -midway echo echo -midway cat cat -EOF - -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/catsn.swift -n=$n -data=$data 2> std.err -EXITCODE=$? - -swiftlog=`\ls -t *.log | head -1` - -#/tmp/plotswiftlogs $logfile 2>gnuplot.err -cat << 'EOF' > /tmp/plotswiftlogs -#!/bin/bash - -#usage: ./extract4plots - -SWIFTLOGFILE=$1 -#TMPDIR=`mktemp -d plotlog.XXX` -grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out -SWIFTOUTFILE=swiftoutfile.out -#extract start time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` -START_TIME=`date +%s -d "$TMPDATE"` - -#extract end time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` -END_TIME=`date +%s -d "$TMPDATE"` - -#duration -DIFFTIME=$((END_TIME - START_TIME)) -#echo 0 > active.txt -#echo 0 > cumulative.txt -#extract active runs in a file -(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > active.txt) - -#extract successful completions in a file -(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) - -#prepare tics -activelines=`wc -l active.txt | awk '{print $1}'` -cumulines=`wc -l cumulative.txt | awk '{print $1}'` - -if [ $activelines -ne 0 ] -then - activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` -fi -seq 0 $activelinespertic $DIFFTIME > activetics.txt - -if [ $cumulines -ne 0 ] -then - cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` -fi - -seq 0 $cumulinespertic $DIFFTIME > cumultics.txt - -#final plot data -paste activetics.txt active.txt > plot_active.txt -paste cumultics.txt cumulative.txt > plot_cumulative.txt - -cat << EOF1 > plotit.gp -set terminal png enhanced -set output "activeplot.png" -set nokey -set xlabel "Time in sec" -set ylabel "number of active jobs" -set title "Active jobs" -plot "plot_active.txt" using 1:2 with line - -set output "cumulativeplot.png" -set ylabel "number of completed jobs" -set title "Cumulative jobs" -plot "plot_cumulative.txt" using 1:2 with lines -EOF1 - -gnuplot plotit.gp 2>/dev/null - -#cp *.png ../ -echo `pwd`>/tmp/curdir -echo `ls`>/tmp/dirlist -#cd .. -EOF - -sh /tmp/plotswiftlogs $swiftlog -cp *.png $outdir/ - -#for i in `find $outdir` -#do -# lastfile=$i -#done -#bname=`basename $lastfile` - -cat << EOF > $htmlfile - - - - - Catsn output - - -

Swift run trace plots

- -active plot -cumulative plot - -

Browse output of catsn application

-`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` - - -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat std.err >&2 -fi -cp std.err $logfile -rm std.err -exit $EXITCODE Deleted: SwiftApps/swift-galaxy/swift/swift-catsn.xml =================================================================== --- SwiftApps/swift-galaxy/swift/swift-catsn.xml 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/swift-catsn.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,22 +0,0 @@ - - A Swift tool to run the catsn workflow - - - swift-catsn "$site" "$data" "$n" "$html_file.files_path" "$log_file" "$html_file" - - - - - - - - - - - - - - - - - Deleted: SwiftApps/swift-galaxy/swift/swift-powergrid =================================================================== --- SwiftApps/swift-galaxy/swift/swift-powergrid 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/swift-powergrid 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,321 +0,0 @@ -#!/bin/bash - -site=$1 -batch=$2 -#stochmodel=$3 -#edrevised=$4 -#asciidata=$5 -#printres=$6 -#illinois=$7 -#wind_data=$8 -logfile=$3 - -#cp $stochmodel stochmodel.run -#cp $edrevised edrevised.mod -#cp $asciidata set_data_ascii.inc -#cp $printres print_results_revised.inc -#cp $illinois Illinois.tgz -#cp $wind_data wind_data.dat -echo $@>>/tmp/tmp.txt -#swift=/home/maheshwari/swift-0.94/cog/modules/swift/dist/swift-svn/bin/swift -swift=/home/maheshwari/swift-devel/cog/modules/swift/dist/swift-svn/bin/swift -# use "here" document for configs - -cat << 'EOF' > /tmp/inference4galaxy.swift -type file; - -app ( file _obj_out, file _xhat_out1, file _xhat_out2, file _ampl_res, file _out) - ampl_app ( int _s, - int _fix_x, - file _stoch_model, - file _illinois, - file _edrevised_model, - file _ascii_data, - file _print_results, - file _wind_data - ) { - runampl _s _fix_x @_stoch_model @_xhat_out1 @_xhat_out2 @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_res @_wind_data stdout=@_out; -} - -app (file _obj_out, file _ampl_l_res, file _out) ampl_app_L ( - int _s, - int _fix_x, - file _stoch_model, - file _illinois, - file _edrevised_model, - file _ascii_data, - file _print_results - ) { - - runampl_l _s _fix_x @_stoch_model @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_l_res stdout=@_out; -} - -app (file _obj_out, file _ampl_u_res, file _out) ampl_app_U ( - int _s, - int _fix_x, - file _stoch_model, - file _xhat, - file _illinois, - file _edrevised_model, - file _ascii_data, - file _print_results, - file _wind_data - ) { - - runampl_u _s _fix_x @_stoch_model @_xhat @_obj_out @_illinois @_edrevised_model @_ascii_data @_print_results @_ampl_u_res @_wind_data stdout=@_out; -} -file stoch_model<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/stochmodel.run">; -file edrevised_model<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/edrevised.mod">; -file ascii_data<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/set_data_ascii.inc">; -file print_results<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/print_results_revised.inc">; -file illinois<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/Illinois.tgz">; -file wind_data<"/scratch/midway/maheshwari/globusonline-galaxy-globus-738fb324c285/tools/swift/wind_data.dat">; - -file out[] ; -file obj_out[] ; -file ampl_res[] ; - -file xhat_out1[] ; -file xhat_out2[] ; - -file out_l[][][] ; -file obj_out_l[][][] ; -file ampl_l_res[][][] ; -file xhat_out_l1[][][] ; -file xhat_out_l2[][][] ; - -file out_u[][][] ; -file obj_out_u[][][] ; -file ampl_u_res[][][] ; -#file upper_out[][][] ; -#file lower_out[][][] ; - -int nS[] = [10]; //number of scenario sweeps -int Batch = @toInt(@arg("batch", "20")); - -foreach S, idxs in nS{ -(obj_out[idxs], xhat_out1[idxs], xhat_out2[idxs], ampl_res[idxs], out[idxs]) = ampl_app (S, 0, stoch_model, illinois, edrevised_model, ascii_data, print_results, wind_data); - - -foreach B, idxb in [10:Batch:10]{ - foreach k in [0:B]{ - (obj_out_l[idxs][idxb][k], xhat_out_l1[idxs][idxb][k], xhat_out_l2[idxs][idxb][k], ampl_l_res[idxs][idxb][k], out_l[idxs][idxb][k]) = ampl_app (S, 0, stoch_model, illinois, edrevised_model, ascii_data, print_results, wind_data); - - (obj_out_u[idxs][idxb][k], ampl_u_res[idxs][idxb][k], out_u[idxs][idxb][k]) = ampl_app_U(S, 1, stoch_model, xhat_out1[idxs], illinois, edrevised_model, ascii_data, print_results, wind_data); - -} -} -} -EOF - -case "$site" in -midway ) -cat << 'EOF' > /tmp/sites.xml - - - - - sandyb - 16 - 02:00:00 - 7500 - 100 - 100 - 2 - 1 - 1 - .31 - 10000 - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -midway echo echo -midway cat cat -midway gensample /home/maheshwari/powergridapps/bin/gensample.py -midway runampl /home/maheshwari/powergridapps/bin/run_ampl.py -midway runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py -midway runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -use.wrapper.staging=false -EOF -;; - -uc3 ) -cat << 'EOF' > /tmp/sites.xml - - - - - .09 - 1000 - 1 - 3600 - 1 - 100 - 100 - 1 - UidDomain == "osg-gk.mwt2.org" - 1 - "group_friends.{env.USER}" - nonshared - - - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -uc3 echo /bin/echo -uc3 cat /bin/cat -uc3 runampl run_ampl.py -uc3 runampl_l run_ampl_l.py -uc3 runampl_u run_ampl_u.py -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -stampede ) -cat << 'EOF' > /tmp/sites.xml - - - - - - 16 - 16 - TG-EAR130015 - normal - ketancmaheshwari at gmail.com - ALL - 02:00:00 - 8000 - 100 - 100 - 1 - 1 - 1 - .15 - 10000 - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -stampede echo echo -stampede cat cat -stampede gensample gensample.py -stampede runampl run_ampl.py -stampede runampl_l run_ampl_l.py -stampede runampl_u run_ampl_u.py -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -use.wrapper.staging=false -EOF -;; - -localhost ) -cat << 'EOF' > /tmp/sites.xml - - - - - - /tmp/swift.work - 0.2 - - -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -localhost gensample /home/maheshwari/powergridapps/bin/gensample.py -localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py -localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py -localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -use.wrapper.staging=false -EOF -;; - -* ) -echo "defaulting to localhost" -cat << 'EOF' > /tmp/sites.xml - - - - - - /tmp/swift.work - 0.31 - - -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -localhost gensample /home/maheshwari/powergridapps/bin/gensample.py -localhost runampl /home/maheshwari/powergridapps/bin/run_ampl.py -localhost runampl_l /home/maheshwari/powergridapps/bin/run_ampl_l.py -localhost runampl_u /home/maheshwari/powergridapps/bin/run_ampl_u.py -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; -esac - -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/inference4galaxy.swift -batch=$batch 2> std.err -#$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/inference4galaxy.swift -batch=$batch -stochmodel=stochmodel.run -edrevised=edrevised.mod -asciidata=set_data_ascii.inc -printres=print_results_revised.inc -illinois=Illinois.tgz -wind_data=wind_data.dat 2> std.err - -EXITCODE=$? - -if [ "$EXITCODE" -ne "0" ]; then - cat std.err >&2 -fi -cp std.err $logfile -rm std.err -exit $EXITCODE Deleted: SwiftApps/swift-galaxy/swift/swift-stats =================================================================== --- SwiftApps/swift-galaxy/swift/swift-stats 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/swift-stats 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,205 +0,0 @@ -#!/bin/bash -# swift-stats "$site.site_select" "$n" "$html_file.files_path" "$log_file" "$html_file" "$site.walltime" "$site.slots" "$site.queue" "$site.nodes" "$site.throttle" "$site.workdir" -# 1 2 3 4 5 6 7 8 9 10 11 - -site=$1 -n=$2 -outdir=$3 -logfile=$4 -htmlfile=$5 -walltime=$6 -slots=$7 -queue=$8 -nodes=$9 -conc=${10} -workdir=${11} -throttle=$(echo "scale=5;($conc-1)/100" |bc) - -mkdir -p $outdir - -echo $6 > /tmp/args.txt - -swift=`\which swift` - -# use "here" document for configs -cat << EOF > /tmp/stats.swift -type file; - -app (file o) statsapp () { - stats stdout=@o; -} - -file out[]; - -foreach j in [1:@toInt(@arg("n","2"))] { - out[j] = statsapp (); -} -EOF - -case "$site" in -clusterPBS ) -cat << EOF > /tmp/sites.xml - - - - - default - 24 - ${walltime} - pbs.aprun;pbs.mpp;depth=24 - 100 - 100 - ${slots} - ${nodes} - ${nodes} - ${throttle} - 10000 - ${workdir} - - -EOF - -cat << EOF > /tmp/tc -clusterpbs echo echo -clusterpbs cat cat -clusterpbs stats stats.sh -EOF - -cat << EOF > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -clusterSLURM ) -cat << EOF > /tmp/sites.xml - - - - - ${queue} - 8 - 8 - ${walltime} - 100 - 100 - ${slots} - ${nodes} - ${nodes} - ${throttle} - 10000 - ${workdir} - - -EOF - -cat << EOF > /tmp/tc -clusterslurm echo echo -clusterslurm cat cat -clusterslurm stats stats.sh -EOF - -cat << EOF > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -localhost ) -cat << EOF > /tmp/sites.xml - - - - - 0.02 - 10000 - - /tmp/swift.work - local - - -EOF - -cat << EOF > /tmp/tc -localhost echo echo -localhost cat cat -localhost stats stats.sh -EOF - -cat << EOF > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -* ) -echo "defaulting to localhost" - -cat << EOF > /tmp/sites.xml - - - - - 0.02 - 10000 - - /tmp/swift.work - local - - -EOF - -cat << EOF > /tmp/tc -localhost echo echo -localhost cat cat -localhost stats stats.sh -EOF - -cat << EOF > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=0 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; -esac - -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/stats.swift -n=$n -data=$data 2> std.err -EXITCODE=$? - -swiftlog=`\ls -t *.log | head -1` - -cat << EOF > $htmlfile - - - - - Hello output - - -

Browse output of hello application

-`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` - - -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat std.err >&2 -fi -cp std.err $logfile -rm std.err -exit $EXITCODE Deleted: SwiftApps/swift-galaxy/swift/swift-try =================================================================== --- SwiftApps/swift-galaxy/swift/swift-try 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/swift-try 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,341 +0,0 @@ -#!/bin/bash - -site=$1 -nsim=$2 -steps=$3 -range=$4 -values=$5 -outdir=$6 -logfile=$7 -htmlfile=$8 - -mkdir -p $outdir - -swift=`which swift` -# use "here" document for configs - -cat << EOF > /tmp/mysim.swift -type file; - -app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values) -{ - simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log; -} - -app (file out, file log) analyze (file s[]) -{ - stats @filenames(s) stdout=@out stderr=@log; -} - -int nsim = @toInt(@arg("nsim", "10")); -int steps = @toInt(@arg("steps", "1")); -int range = @toInt(@arg("range", "100")); -int values = @toInt(@arg("values", "5")); - -file sims[]; - -foreach i in [0:nsim-1] { - file simout ; - file simlog ; - (simout,simlog) = simulation(steps,range,values); - sims[i] = simout; -} - -file stats_out<"output/average.out">; -file stats_log<"output/average.log">; -(stats_out, stats_log) = analyze(sims); - -EOF - -case "$site" in -midway ) -cat << 'EOF' > /tmp/sites.xml - - - - - sandyb - 16 - 02:00:00 - 7500 - 100 - 100 - 4 - 2 - 2 - 1.27 - 10000 - /scratch/midway/maheshwari/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -midway echo echo -midway cat cat -midway simulate simulate.sh -midway stats stats.sh -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -uc3 ) -cat << 'EOF' > /tmp/sites.xml - - - - - - .09 - 1000 - 1 - 3600 - 1 - 100 - 100 - 1 - UidDomain == "osg-gk.mwt2.org" - 1 - "group_friends.{env.USER}" - nonshared - - - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -uc3 echo /bin/echo -uc3 cat /bin/cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -stampede ) -cat << 'EOF' > /tmp/sites.xml - - - - - - - - 16 - 16 - TG-STA110005S - normal - ketancmaheshwari at gmail.com - ALL - 02:00:00 - 8000 - 100 - 100 - 1 - 1 - 1 - .15 - 10000 - /tmp/swift.work - - -EOF - -cat << 'EOF' > /tmp/tc -stampede echo echo -stampede cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=true -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -localhost ) -cat << 'EOF' > /tmp/sites.xml - - - - - - - /tmp/swift.work - 0.31 - - -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; - -* ) -echo "defaulting to localhost" - -cat << 'EOF' > /tmp/sites.xml - - - - - - - /tmp/swift.work - 0.31 - - -EOF - -cat << 'EOF' > /tmp/tc -localhost echo echo -localhost cat cat -EOF - -cat << 'EOF' > /tmp/cf -use.provider.staging=false -wrapperlog.always.transfer=true -execution.retries=1 -provider.staging.pin.swiftfiles=false -sitedir.keep=true -lazy.errors=false -EOF -;; -esac - -$swift -sites.file /tmp/sites.xml -tc.file /tmp/tc -config /tmp/cf /tmp/mysim.swift -nsim=$nsim -steps=$steps -range=$range -values=$values 2> std.err -EXITCODE=$? - -swiftlog=`\ls -t *.log | head -1` - -#/tmp/plotswiftlogs $logfile 2>gnuplot.err -cat << 'EOF' > /tmp/plotswiftlogs -#!/bin/bash - -#usage: ./plotswiftlogs.ketan - -SWIFTLOGFILE=$1 - -#TMPDIR=`mktemp -d plotlog.XXX` - -grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out - -SWIFTOUTFILE=swiftoutfile.out - -#extract start time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '` -START_TIME=`date +%s -d "$TMPDATE"` - -#extract end time -TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '` -END_TIME=`date +%s -d "$TMPDATE"` - -#duration -DIFFTIME=$((END_TIME - START_TIME)) - -#extract active runs in a file -(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt) - -#extract successful completions in a file -(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt) - -#prepare tics -activelines=`wc -l active.txt | awk '{print $1}'` -cumulines=`wc -l cumulative.txt | awk '{print $1}'` - -if [ $activelines -ne 0 ] -then - activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc` -fi - -seq 0 $activelinespertic $DIFFTIME > activetics.txt - -if [ $cumulines -ne 0 ] -then - cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc` -fi - -seq 0 $cumulinespertic $DIFFTIME > cumultics.txt - -#final plot data -paste activetics.txt active.txt > plot_active.txt -paste cumultics.txt cumulative.txt > plot_cumulative.txt - -cat << EOF1 > plotit.gp -set terminal png enhanced -set nokey -set output "cumulativeplot.png" -set xlabel "Time in sec" -set ylabel "number of completed apps" -set title "Cumulative Apps Completed" -plot "plot_cumulative.txt" using 1:2 with lines -set output "activeplot.png" -set xlabel "Time in sec" -set ylabel "number of active apps" -set title "Active Apps" -plot "plot_active.txt" using 1:2 with line -EOF1 - -gnuplot plotit.gp 2>/dev/null -rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt - -EOF - -sh /tmp/plotswiftlogs $swiftlog -cp *.png $outdir/ - -cat << EOF > $htmlfile - - - - - Application output - - -

Swift run trace plots

- -active plot -cumulative plot - -

Browse output of application

-`for i in \`find $outdir -type f\`; do echo "Output: \`basename $i\`
"; done` - - -EOF - -if [ "$EXITCODE" -ne "0" ]; then - cat std.err >&2 -fi -cp std.err $logfile -rm std.err -exit $EXITCODE Deleted: SwiftApps/swift-galaxy/swift/swiftforeachpy.xml =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachpy.xml 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/swiftforeachpy.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,170 +0,0 @@ - - - A generic tool to run executable via Swift foreach parallel construct - - - - #if $rangeorlist.rl == "rng" - swiftforeachrange.py - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${rangeorlist.rngstart}" - "${rangeorlist.rend}" - "${rangeorlist.stepsize}" - "${outloc}" - "${logfile}" - "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #elif $rangeorlist.rl == "lst" - swiftforeachlist.py - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${rangeorlist.listfile}" - "${outloc}" - "${logfile}" - "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #else - swiftforeachtest.py - "${site}" - "${swiftargs}" - "${interpret}" - "${exec}" - "${rangeorlist.rngstart}" - "${rangeorlist.rend}" - "${rangeorlist.stepsize}" - "${rangeorlist.listfile}" - "${outloc}" - "${logfile}" - "${outlist}" - "${stringargs}" - #for $a in $filearg - ${a.fileargname} - #end for - #end if - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -.. class:: warningmark - -**TIP**. Add args to provide additional arguments to your executable. - ------ - -**What it does** - -This is a generic Swift tool that runs an executable in parallel over a range of numbers or a list of items in a file. - -Execution location allows user to declare where to run the tool. Executable can be any arbitrary executable of type "binexec" uploaded by user. - -Start, end and stepsize are integer values. Note that the stepsize cannot be less than 1. Optionally, sser can add new arguments to be provided to the tool executable after the numeric argument. Each args box must contain exactly one argument without spaces. - ------ - -**Example** - -Pick up an executable from the Executable list. Enter the start, end and the step size values. Enter an additional argument "hello". - -If an executable is echo.sh with following line: - -echo $@ - -The start, end and step is 0, 9 and 2, the executable will be invoked in parallel 5 times for n where n is 0 2 4 6 8: - -echo n hello - - - Deleted: SwiftApps/swift-galaxy/swift/swiftforeachtest.py =================================================================== --- SwiftApps/swift-galaxy/swift/swiftforeachtest.py 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/swiftforeachtest.py 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,110 +0,0 @@ -#!/usr/bin/env python - -import subprocess -import sys -#import rpy -import os - -def setwdir(): - return subprocess.check_output(["mktemp", "-d", "/tmp/swift-gal.XXXX"]) - -# numeric range -def main(): - print "%d" % (len(sys.argv)) - print "%s" % (sys.argv) - try: - sites=sys.argv[1] # comma separated list of sites - swiftargs=sys.argv[2] - interpreter=sys.argv[3] - executable=sys.argv[4] - rstart=sys.argv[5] - rend=sys.argv[6] - stepsize=sys.argv[7] - outloc=sys.argv[8] - outlistfile=sys.argv[9] - logfile=sys.argv[10] - stringargs=sys.argv[11] - except: - #rpy.stop_error('Wrong number or type of args') - sys.exit(1) - - print sites - print swiftargs - print interpreter - - #workout the array only if user adds args else make it blank - if stringargs: - stringarrayexpr="string stringargs[]="+str(stringargs.split())+";\n" - else: - stringarrayexpr="string stringargs[];\n" - - fileargs=list() - for files in sys.argv[12:]: - fileargs.append(files) - - #workout the array only if user adds file args else make it blank - if fileargs: - filearrayexpr="file fileargs[]="+str(fileargs)+";\n" - else: - filearrayexpr="file fileargs[];\n" - - wdir=setwdir().rstrip() - #Build Swift source code - f=open(wdir+"/script.swift", 'w'); - f.write("type file; \n") - - f.write("app (file _out, file _err) anapp(file _exec, int _i, string _stringargs[], file _fileargs[]){ \n") - f.write(" "+interpreter+" @_exec _i _stringargs @_fileargs stdout=@_out stderr=@_err; \n") - f.write("}\n") - - f.write(stringarrayexpr) - f.write(filearrayexpr) - - f.write("file exec<'"+executable+"'>;\n") - f.write("foreach i in ["+rstart+":"+rend+":"+stepsize+"]{\n") - f.write(" file out ;\n") - f.write(" file err ;\n") - f.write(" (out,err) = anapp(exec, i, "+stringargs+", "+",".join(fileargs)+");\n") - f.write("}\n") - f.close() - - swift=subprocess.check_output(["which", "swift"]).rstrip() - - #build site - if os.path.dirname(sys.argv[0]): - basedir=os.path.dirname(sys.argv[0]) - else: - basedir='.' - - print "%s %s %s" %(basedir, sites, wdir) - subprocess.call([basedir+"/buildsite.sh", sites, wdir]) - - #build config - f=open(wdir+"/cf", 'w'); - f.write('use.provider.staging=true') - f.write('wrapperlog.always.transfer=true') - f.write('execution.retries=0') - f.write('provider.staging.pin.swiftfiles=false') - f.write('sitedir.keep=true') - f.write('lazy.errors=false') - f.close() - - - #build tc - f=open(wdir+"/tc", 'w'); - for site in sites.split(): - f.write(site+' '+interpreter+' '+interpreter) - f.close() - - #exitcode=subprocess.call([swift, '-sites.file', wdir+"/sites.xml", '-tc.file', wdir+"/tc", "-config", wdir+"/cf", wdir+"/script.swift", swiftargs, "2>logfile"]) - - #`for i in \`find $HOME/swift-sandbox -type f\`; do echo "\`basename $i\` $i"; done` - #Populate output file - outlist=subprocess.check_output(["find", outloc, "-type", "f", "-iname", "*.out"]) - f=open(outlistfile) - f.write(outlist); - f.close() - #dum ditty dum ditty dum dum dum - -if __name__=='__main__': - main() Deleted: SwiftApps/swift-galaxy/swift/testrun =================================================================== --- SwiftApps/swift-galaxy/swift/testrun 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/testrun 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,2 +0,0 @@ -python swiftforeachrange.py localhost someargs java /Users/ketan/galaxy-dist/database/files/000/dataset_248.dat 0 9 1 /Users/ketan/galaxy-dist/database/files/000/dataset_306.dat /Users/ketan/swift-sandbox /Users/ketan/galaxy-dist/database/files/000/dataset_307.dat /Users/ketan/galaxy-dist/database/files/000/dataset_308.dat hello /Users/ketan/galaxy-dist/database/files/000/dataset_302.dat - Deleted: SwiftApps/swift-galaxy/swift/try.xml =================================================================== --- SwiftApps/swift-galaxy/swift/try.xml 2014-05-12 22:06:49 UTC (rev 7868) +++ SwiftApps/swift-galaxy/swift/try.xml 2014-05-13 14:33:31 UTC (rev 7869) @@ -1,31 +0,0 @@ - - Try Swift - - - swift-try $site $swiftscript $log_file - - - - - - - - - - - - - - - - - - - From ketan at ci.uchicago.edu Tue May 13 11:29:19 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 13 May 2014 11:29:19 -0500 (CDT) Subject: [Swift-commit] r7870 - in SwiftApps/swift-galaxy: . swift swift/old Message-ID: <20140513162919.806C29D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-13 11:29:19 -0500 (Tue, 13 May 2014) New Revision: 7870 Added: SwiftApps/swift-galaxy/setup.py SwiftApps/swift-galaxy/swift/old/test.sh Removed: SwiftApps/swift-galaxy/swift/test.sh Modified: SwiftApps/swift-galaxy/swift/testpy.sh Log: adding setup for installation Added: SwiftApps/swift-galaxy/setup.py =================================================================== --- SwiftApps/swift-galaxy/setup.py (rev 0) +++ SwiftApps/swift-galaxy/setup.py 2014-05-13 16:29:19 UTC (rev 7870) @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +""" +Setup script to install swift-galaxy framework to your Galaxy installation. +""" + +import subprocess +import sys +import os +import distutils.spawn +import traceback +import fnmatch + +galaxyloc=sys.argv[1] + +#step 1. Create symlink to the swift directory from the tools directory of galaxyloc if not already present +os.symlink (os.getcwd()+"/screenshots", galaxyloc+"/tools/screenshots"); + +#step 2. update the tool_conf.xml + +#
+# +# +# +#
+ +#step 3. Print message + + Property changes on: SwiftApps/swift-galaxy/setup.py ___________________________________________________________________ Added: svn:executable + * Copied: SwiftApps/swift-galaxy/swift/old/test.sh (from rev 7869, SwiftApps/swift-galaxy/swift/test.sh) =================================================================== --- SwiftApps/swift-galaxy/swift/old/test.sh (rev 0) +++ SwiftApps/swift-galaxy/swift/old/test.sh 2014-05-13 16:29:19 UTC (rev 7870) @@ -0,0 +1,41 @@ +#!/bin/bash + +#!/bin/bash + +cat << EOF > list.txt +a +b +c +EOF + +cat << EOF > x.sh +#!/bin/bash + +echo \$\@ +EOF + +chmod 755 x.sh + +echo "List, no input parameters and files" +./swiftforeach.sh list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" + +echo "Range, no input parameters and files" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" + +echo "List, string input parameters" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there list!" + +echo "Range, string input parameters" +./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hi there range!" + +echo "List, no input parameters and files stdin /dev/null" +./swiftforeach.sh list swiftK localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" + +echo "T list" +./swiftforeach.sh list swiftT localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" + +echo "T range" +./swiftforeach.sh range swiftT localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" + +echo "Call from Galaxy" +bash /nfs2/ketan/galaxy-dist/tools/swift/swiftforeach.sh "range" "swiftT" "localhost" "" "sh" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_1.dat" "None" "0" "9" "1" "$HOME/swift-sandbox" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_12.dat" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_13.dat" "hello how are you" Deleted: SwiftApps/swift-galaxy/swift/test.sh =================================================================== --- SwiftApps/swift-galaxy/swift/test.sh 2014-05-13 14:33:31 UTC (rev 7869) +++ SwiftApps/swift-galaxy/swift/test.sh 2014-05-13 16:29:19 UTC (rev 7870) @@ -1,41 +0,0 @@ -#!/bin/bash - -#!/bin/bash - -cat << EOF > list.txt -a -b -c -EOF - -cat << EOF > x.sh -#!/bin/bash - -echo \$\@ -EOF - -chmod 755 x.sh - -echo "List, no input parameters and files" -./swiftforeach.sh list swiftK localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" - -echo "Range, no input parameters and files" -./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "" - -echo "List, string input parameters" -./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hello there list!" - -echo "Range, string input parameters" -./swiftforeach.sh range swiftK localhost "" bash x.sh "" 0 9 1 tmpoutrange b.log list2.out "hi there range!" - -echo "List, no input parameters and files stdin /dev/null" -./swiftforeach.sh list swiftK localhost "" bash x.sh "/dev/null" list.txt tmpoutlist a.log list1.out "" - -echo "T list" -./swiftforeach.sh list swiftT localhost "" bash x.sh "" list.txt tmpoutlist a.log list1.out "" - -echo "T range" -./swiftforeach.sh range swiftT localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" - -echo "Call from Galaxy" -bash /nfs2/ketan/galaxy-dist/tools/swift/swiftforeach.sh "range" "swiftT" "localhost" "" "sh" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_1.dat" "None" "0" "9" "1" "$HOME/swift-sandbox" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_12.dat" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_13.dat" "hello how are you" Modified: SwiftApps/swift-galaxy/swift/testpy.sh =================================================================== --- SwiftApps/swift-galaxy/swift/testpy.sh 2014-05-13 14:33:31 UTC (rev 7869) +++ SwiftApps/swift-galaxy/swift/testpy.sh 2014-05-13 16:29:19 UTC (rev 7870) @@ -37,5 +37,8 @@ echo "T range" ./swiftforeach.py range swiftT localhost "" bash x.sh "" 0 9 1 tmpoutlist a.log list1.out "hello there T" -echo "Call from Galaxy" +echo "Call K from Galaxy" +python /Users/ketan/galaxy-dist/tools/swift/swiftforeach.py "range" "swiftK" "localhost" "swiftarg" "sh" "/Users/ketan/galaxy-dist/database/files/000/dataset_348.dat" "/dev/null" "0" "9" "1" "$HOME/sandbox2" "/Users/ketan/galaxy-dist/database/files/000/dataset_353.dat" "/Users/ketan/galaxy-dist/database/files/000/dataset_354.dat" "hello how are you" + +echo "Call T from Galaxy" python /nfs2/ketan/galaxy-dist/tools/swift/swiftforeach.py "range" "swiftT" "localhost" "" "sh" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_1.dat" "None" "0" "9" "1" "$HOME/swift-sandbox" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_12.dat" "/nfs2/ketan/galaxy-dist/database/files/000/dataset_13.dat" "hello how are you" From wilde at ci.uchicago.edu Thu May 15 14:19:58 2014 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 15 May 2014 14:19:58 -0500 (CDT) Subject: [Swift-commit] r7871 - in SwiftTutorials/swift-cloud-tutorial: . part07 Message-ID: <20140515191958.D032F9CCF4@svn.ci.uchicago.edu> Author: wilde Date: 2014-05-15 14:19:57 -0500 (Thu, 15 May 2014) New Revision: 7871 Added: SwiftTutorials/swift-cloud-tutorial/part07/ SwiftTutorials/swift-cloud-tutorial/part07/apps SwiftTutorials/swift-cloud-tutorial/part07/p7.swift SwiftTutorials/swift-cloud-tutorial/part07/simulate.sh SwiftTutorials/swift-cloud-tutorial/part07/sites.xml SwiftTutorials/swift-cloud-tutorial/part07/stats.sh Modified: SwiftTutorials/swift-cloud-tutorial/setup.sh Log: initial part07: genetic algorithm framework example. Added: SwiftTutorials/swift-cloud-tutorial/part07/apps =================================================================== --- SwiftTutorials/swift-cloud-tutorial/part07/apps (rev 0) +++ SwiftTutorials/swift-cloud-tutorial/part07/apps 2014-05-15 19:19:57 UTC (rev 7871) @@ -0,0 +1 @@ +localhost sh /bin/bash Added: SwiftTutorials/swift-cloud-tutorial/part07/p7.swift =================================================================== --- SwiftTutorials/swift-cloud-tutorial/part07/p7.swift (rev 0) +++ SwiftTutorials/swift-cloud-tutorial/part07/p7.swift 2014-05-15 19:19:57 UTC (rev 7871) @@ -0,0 +1,72 @@ +type file; + +# app() functions for application programs to be called: + +app (file out) genseed (int nseeds, file seed_script) +{ + sh "simulate.sh" "-r" 2000000 "-n" nseeds stdout=@out; +} + +app (file out) genbias (int bias_range, int nvalues, file bias_script) +{ + sh "simulate.sh" "-r" bias_range "-n" nvalues stdout=@out; +} + +app (file out, file log) simulation (int timesteps, int sim_range, + file bias_file, int scale, int sim_count, + file sim_script, file seed_file) +{ + sh "simulate.sh" "-t" timesteps "-r" sim_range "-B" @bias_file "-x" scale + "-n" sim_count "-S" @seed_file stdout=@out stderr=@log; +} + +app (file out, file log) analyze (file s[], file stat_script) +{ + sh "stats.sh" @filenames(s) stdout=@out stderr=@log; +} + +(file simout, file simlog) simulate (int biasrange, int biascount, int timesteps, int sim_range, + int scale, int sim_count, file sim_script, file seed_file) +{ + file biasfile = genbias(biasrange, biascount, sim_script); + (simout,simlog) = simulation(timesteps, sim_range, biasfile, 1000000, sim_count, sim_script, seed_file); +} + +generation(file seedfile) +{ + # Command line arguments + + int nsim = @toInt(@arg("nsim", "10")); # number of simulation programs to run + int steps = @toInt(@arg("steps", "1")); # number of timesteps (seconds) per simulation + int range = @toInt(@arg("range", "100")); # range of the generated random numbers + int values = @toInt(@arg("values", "10")); # number of values generated per simulation + + # Main script and data + + file stats_script <"stats.sh">; + + tracef("\n*** Script parameters: nsim=%i range=%i num values=%i\n\n", nsim, range, values); + + file sims[]; # Array of files to hold each simulation output + + foreach i in [0:nsim-1] { + # file biasfile ; + file simout ; + file simlog ; + (simout,simlog) = simulate(1000, 20, steps, range, 1000000, values, simulate_script, seedfile); + sims[i] = simout; + } + + file stats_out<"output/average.out">; + file stats_log<"output/average.log">; + (stats_out,stats_log) = analyze(sims, stats_script); +} + +file seedfile <"output/seed.dat">; # Dynamically generated bias for simulation ensemble + +global file simulate_script <"simulate.sh">; + +seedfile = genseed(1,simulate_script); + + +generation(seedfile); Added: SwiftTutorials/swift-cloud-tutorial/part07/simulate.sh =================================================================== --- SwiftTutorials/swift-cloud-tutorial/part07/simulate.sh (rev 0) +++ SwiftTutorials/swift-cloud-tutorial/part07/simulate.sh 2014-05-15 19:19:57 UTC (rev 7871) @@ -0,0 +1,135 @@ +#! /bin/bash + +printparams() +{ + printf "\nSimulation parameters:\n\n" + echo bias=$bias + echo biasfile=$biasfile + echo initseed=$initseed + echo log=$log + echo paramfile=$paramfile + echo range=$range + echo scale=$scale + echo seedfile=$seedfile + echo timesteps=$timesteps + echo output width=$width +} + +log() { + printf "\nCalled as: $0: $cmdargs\n\n" + printf "Start time: "; /bin/date + printf "Running as user: "; /usr/bin/id + printf "Running on node: "; /bin/hostname + printf "Node IP address: "; /bin/hostname -I + printparams + printf "\nEnvironment:\n\n" + printenv | sort +} + +addsims() { + while read f1 ; do + read -u 3 f2 + if [ _$f1 = _ ]; then f1=$lastf1; fi + if [ _$f2 = _ ]; then f2=$lastf2; fi + printf "%${width}d\n" $(($f1+$f2)) + lastf1=$f1 + lastf2=$f2 + done <$1 3<$2 +} + +# set defaults + +bias=0 +biasfile=none +initseed=none +log=yes +paramfile=none +range=100 +scale=1 +seedfile=none +timesteps=0 +nvalues=1 +width=8 +cmdargs="$*" + +usage() +{ + echo $0: usage: + cat <$simout + +# process file of biases + +if [ $biasfile != none ]; then + addsims $simout $biasfile +else + cat $simout +fi +rm $simout + +# log environmental data + +if [ $log != off ]; then + log 1>&2 +fi Property changes on: SwiftTutorials/swift-cloud-tutorial/part07/simulate.sh ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/swift-cloud-tutorial/part07/sites.xml =================================================================== --- SwiftTutorials/swift-cloud-tutorial/part07/sites.xml (rev 0) +++ SwiftTutorials/swift-cloud-tutorial/part07/sites.xml 2014-05-15 19:19:57 UTC (rev 7871) @@ -0,0 +1,11 @@ + + + + + .10 + 10000 + + .swift/tmp + local + + Added: SwiftTutorials/swift-cloud-tutorial/part07/stats.sh =================================================================== --- SwiftTutorials/swift-cloud-tutorial/part07/stats.sh (rev 0) +++ SwiftTutorials/swift-cloud-tutorial/part07/stats.sh 2014-05-15 19:19:57 UTC (rev 7871) @@ -0,0 +1,19 @@ +#! /bin/sh + +log() { + printf "\nCalled as: $0: $cmdargs\n\n" + printf "Start time: "; /bin/date + printf "Running as user: "; /usr/bin/id + printf "Running on node: "; /bin/hostname + printf "Node IP address: "; /bin/hostname -I + printf "\nEnvironment:\n\n" + printenv | sort +} + +awk ' + +{ sum += $1} + +END { printf("%d\n",sum/NR) } +' $* +log 1>&2 Property changes on: SwiftTutorials/swift-cloud-tutorial/part07/stats.sh ___________________________________________________________________ Added: svn:executable + * Modified: SwiftTutorials/swift-cloud-tutorial/setup.sh =================================================================== --- SwiftTutorials/swift-cloud-tutorial/setup.sh 2014-05-13 16:29:19 UTC (rev 7870) +++ SwiftTutorials/swift-cloud-tutorial/setup.sh 2014-05-15 19:19:57 UTC (rev 7871) @@ -1,16 +1,13 @@ # ensure that this script is being sourced if [[ $0 =~ .*bash ]] ; then -# Bash: if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" = "${0}" ] ; then echo ERROR: script ${BASH_SOURCE[0]} must be executed as: source ${BASH_SOURCE[0]} exit 1 fi # Lookup tutorial directory based on PATH entry for this script TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -fi -if [[ $ZSH_VERSION != "" ]] ; then - # ZSH: +elif [[ $ZSH_VERSION != "" ]] ; then if [[ ! $- =~ .*i.* ]] ; then echo ERROR: script $0 must be executed as: source $0 return 1 @@ -59,6 +56,8 @@ END +return # TEMP: for localhost execuiton on Mac + # Start coaster-service and generate sites.xml cd scs start-coaster-service From hategan at ci.uchicago.edu Sat May 17 21:08:48 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 17 May 2014 21:08:48 -0500 (CDT) Subject: [Swift-commit] r7872 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20140518020848.6D3399D09F@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-17 21:08:47 -0500 (Sat, 17 May 2014) New Revision: 7872 Modified: trunk/src/org/griphyn/vdl/mapping/file/ExternalMapper.java Log: read output of external mapper progressively instead of buffering all output and then processing it Modified: trunk/src/org/griphyn/vdl/mapping/file/ExternalMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/ExternalMapper.java 2014-05-15 19:19:57 UTC (rev 7871) +++ trunk/src/org/griphyn/vdl/mapping/file/ExternalMapper.java 2014-05-18 02:08:47 UTC (rev 7872) @@ -81,7 +81,8 @@ if (logger.isDebugEnabled()) { logger.debug("invoking external mapper: " + cmd); } - Process p = Runtime.getRuntime().exec(cmd.toArray(STRING_ARRAY)); + Process p = Runtime.getRuntime().exec(cmd.toArray(STRING_ARRAY)); + process(p.getInputStream()); List lines = fetchOutput(p.getInputStream()); if (logger.isDebugEnabled()) { logger.debug("external mapper output: " + lines); @@ -91,7 +92,6 @@ throw new RuntimeException("External executable failed. Exit code: " + ec + "\n\t" + join(lines) + "\n\t" + join(fetchOutput(p.getErrorStream()))); } - processLines(cp, lines); } catch (IOException e) { throw new RuntimeException(e); @@ -109,6 +109,15 @@ sb.append('\t'); } return sb.toString(); + } + + private void process(InputStream is) throws IOException { + BufferedReader br = new BufferedReader(new InputStreamReader(is)); + String line = br.readLine(); + while (line != null) { + processLine(line); + line = br.readLine(); + } } private List fetchOutput(InputStream is) throws IOException { @@ -122,20 +131,18 @@ return lines; } - private void processLines(ExternalMapperParams cp, List lines) { - for (String line : lines) { - int s = line.indexOf(' '); - int t = line.indexOf('\t'); - int m = Math.min(s == -1 ? t : s, t == -1 ? s : t); - if (m == -1) { - throw new RuntimeException("Invalid line in mapper script output: " + line); - } - String spath = line.substring(0, m); - Path p = Path.parse(spath); - AbsFile f = new AbsFile(line.substring(m + 1).trim()); - map.put(p, f); - rmap.put(spath, p); + private void processLine(String line) { + int s = line.indexOf(' '); + int t = line.indexOf('\t'); + int m = Math.min(s == -1 ? t : s, t == -1 ? s : t); + if (m == -1) { + throw new RuntimeException("Invalid line in mapper script output: " + line); } + String spath = line.substring(0, m); + Path p = Path.parse(spath); + AbsFile f = new AbsFile(line.substring(m + 1).trim()); + map.put(p, f); + rmap.put(spath, p); } @Override From hategan at ci.uchicago.edu Sat May 17 21:09:43 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 17 May 2014 21:09:43 -0500 (CDT) Subject: [Swift-commit] r7873 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20140518020943.AF4C49D09F@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-17 21:09:43 -0500 (Sat, 17 May 2014) New Revision: 7873 Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java Log: make structured regexp mapper work with string array as source (bug 1271) Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-05-18 02:08:47 UTC (rev 7872) +++ trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-05-18 02:09:43 UTC (rev 7873) @@ -35,6 +35,8 @@ import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.type.Type; +import org.griphyn.vdl.type.Types; public class StructuredRegularExpressionMapper extends AbstractMapper { @@ -74,9 +76,10 @@ catch (InvalidPathException ipe) { return Collections.emptyList(); } - catch (HandleOpenException hoe) { - throw new RuntimeException( - "Handle open. Throwing this exception may not be the right thing to do. TODO"); + catch (HandleOpenException hoe) { + // should not be happening since the mapper is only initialized after + // all parameters have been closed + throw new RuntimeException("Handle open in mapper: " + sourceHandle); } Iterator i = sourceFields.iterator(); while (i.hasNext()) { @@ -109,15 +112,26 @@ + " from source data structure"); } - PhysicalFormat source = hereHandle.getMapper().map(hereHandle.getPathFromRoot()); - if (!source.getType().equals("file")) { - throw new RuntimeException( - "Cannot use the regular expression mapper with a source that has a mapper that is not file-based"); + String sourceStr; + Type sourceType = hereHandle.getType(); + if (sourceType.equals(Types.STRING)) { + sourceStr = (String) hereHandle.getValue(); + } + else if (!sourceType.isComposite() && !sourceType.isPrimitive()) { + PhysicalFormat source = hereHandle.getMapper().map(hereHandle.getPathFromRoot()); + if (!source.getType().equals("file")) { + throw new RuntimeException( + "Cannot use the regular expression mapper with a source that has a mapper that is not file-based"); + } + sourceStr = ((AbsFile) source).getPath(); + } + else { + throw new RuntimeException("Source must be an array of mapped objects or strings. Found " + sourceType); } Pattern p = Pattern.compile(cp.getMatch()); - Matcher m = p.matcher(((AbsFile) source).getPath()); + Matcher m = p.matcher(sourceStr); if (!m.find()) { - throw new RuntimeException("No match found! source='" + source + "' match = '" + cp.getMatch() + throw new RuntimeException("No match found! source='" + sourceStr + "' match = '" + cp.getMatch() + "'"); } // find group number to replace From hategan at ci.uchicago.edu Sat May 17 21:25:11 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 17 May 2014 21:25:11 -0500 (CDT) Subject: [Swift-commit] r7874 - branches/release-0.95/src/org/griphyn/vdl/mapping/file Message-ID: <20140518022511.4BFA99D09F@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-17 21:25:11 -0500 (Sat, 17 May 2014) New Revision: 7874 Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java Log: make structured regexp mapper work with string array as source (bug 1271) Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java =================================================================== --- branches/release-0.95/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-05-18 02:09:43 UTC (rev 7873) +++ branches/release-0.95/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-05-18 02:25:11 UTC (rev 7874) @@ -34,6 +34,8 @@ import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.type.Type; +import org.griphyn.vdl.type.Types; public class StructuredRegularExpressionMapper extends AbstractMapper { @@ -68,14 +70,15 @@ Collection output = new ArrayList(); Collection sourceFields; try { - sourceFields = sourceHandle.getFields(Path.CHILDREN); + sourceFields = sourceHandle.getAllFields(); } catch (InvalidPathException ipe) { return Collections.emptyList(); } catch (HandleOpenException hoe) { - throw new RuntimeException( - "Handle open. Throwing this exception may not be the right thing to do. TODO"); + // should not be happening since the mapper is only initialized after + // all parameters have been closed + throw new RuntimeException("Handle open in mapper: " + sourceHandle); } Iterator i = sourceFields.iterator(); while (i.hasNext()) { @@ -103,15 +106,26 @@ + " from source data structure"); } - PhysicalFormat source = hereHandle.getMapper().map(hereHandle.getPathFromRoot()); - if (!source.getType().equals("file")) { - throw new RuntimeException( - "Cannot use the regular expression mapper with a source that has a mapper that is not file-based"); + String sourceStr; + Type sourceType = hereHandle.getType(); + if (sourceType.equals(Types.STRING)) { + sourceStr = (String) hereHandle.getValue(); } + else if (!sourceType.isComposite() && !sourceType.isPrimitive()) { + PhysicalFormat source = hereHandle.getMapper().map(hereHandle.getPathFromRoot()); + if (!source.getType().equals("file")) { + throw new RuntimeException( + "Cannot use the regular expression mapper with a source that has a mapper that is not file-based"); + } + sourceStr = ((AbsFile) source).getPath(); + } + else { + throw new RuntimeException("Source must be an array of mapped objects or strings. Found " + sourceType); + } Pattern p = Pattern.compile(cp.getMatch()); - Matcher m = p.matcher(((AbsFile) source).getPath()); + Matcher m = p.matcher(sourceStr); if (!m.find()) { - throw new RuntimeException("No match found! source='" + source + "' match = '" + cp.getMatch() + throw new RuntimeException("No match found! source='" + sourceStr + "' match = '" + cp.getMatch() + "'"); } // find group number to replace From hategan at ci.uchicago.edu Sat May 17 21:29:15 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 17 May 2014 21:29:15 -0500 (CDT) Subject: [Swift-commit] r7875 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20140518022915.4E8839D09F@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-17 21:29:15 -0500 (Sat, 17 May 2014) New Revision: 7875 Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java Log: somehow new lines were in windows style Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-05-18 02:25:11 UTC (rev 7874) +++ trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-05-18 02:29:15 UTC (rev 7875) @@ -15,8 +15,8 @@ */ -package org.griphyn.vdl.mapping.file; - +package org.griphyn.vdl.mapping.file; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -37,21 +37,21 @@ import org.griphyn.vdl.mapping.PhysicalFormat; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; - -public class StructuredRegularExpressionMapper extends AbstractMapper { - - public static final Logger logger = - Logger.getLogger(StructuredRegularExpressionMapper.class); + +public class StructuredRegularExpressionMapper extends AbstractMapper { + + public static final Logger logger = + Logger.getLogger(StructuredRegularExpressionMapper.class); @Override protected void getValidMappingParams(Set s) { s.addAll(StructuredRegularExpressionMapperParams.NAMES); super.getValidMappingParams(s); } - - public StructuredRegularExpressionMapper() { - } + public StructuredRegularExpressionMapper() { + } + @Override public String getName() { return "StructuredRegexpMapper"; @@ -62,55 +62,55 @@ return new StructuredRegularExpressionMapperParams(); } - @Override + @Override public Collection existing() { - StructuredRegularExpressionMapperParams cp = getParams(); - - DSHandle sourceHandle = cp.getSource(); - - Collection output = new ArrayList(); - Collection sourceFields; - try { - sourceFields = sourceHandle.getAllFields(); - } - catch (InvalidPathException ipe) { - return Collections.emptyList(); - } + StructuredRegularExpressionMapperParams cp = getParams(); + + DSHandle sourceHandle = cp.getSource(); + + Collection output = new ArrayList(); + Collection sourceFields; + try { + sourceFields = sourceHandle.getAllFields(); + } + catch (InvalidPathException ipe) { + return Collections.emptyList(); + } catch (HandleOpenException hoe) { // should not be happening since the mapper is only initialized after - // all parameters have been closed - throw new RuntimeException("Handle open in mapper: " + sourceHandle); - } - Iterator i = sourceFields.iterator(); - while (i.hasNext()) { - DSHandle f = i.next(); - output.add(f.getPathFromRoot()); - } - - return output; + // all parameters have been closed + throw new RuntimeException("Handle open in mapper: " + sourceHandle); + } + Iterator i = sourceFields.iterator(); + while (i.hasNext()) { + DSHandle f = i.next(); + output.add(f.getPathFromRoot()); + } + + return output; } @Override public Collection existing(FileSystemLister l) { throw new UnsupportedOperationException(); - } + } - @Override + @Override public PhysicalFormat map(Path path) { - if (logger.isDebugEnabled()) { + if (logger.isDebugEnabled()) { logger.debug("map(): path: " + path); } - StructuredRegularExpressionMapperParams cp = getParams(); - - DSHandle sourceHandle = cp.getSource(); - DSHandle hereHandle; - try { - hereHandle = sourceHandle.getField(path); - } - catch (InvalidPathException ipe) { - throw new RuntimeException("Cannot get requested path " + path - + " from source data structure"); + StructuredRegularExpressionMapperParams cp = getParams(); + + DSHandle sourceHandle = cp.getSource(); + DSHandle hereHandle; + try { + hereHandle = sourceHandle.getField(path); } + catch (InvalidPathException ipe) { + throw new RuntimeException("Cannot get requested path " + path + + " from source data structure"); + } String sourceStr; Type sourceType = hereHandle.getType(); @@ -118,41 +118,41 @@ sourceStr = (String) hereHandle.getValue(); } else if (!sourceType.isComposite() && !sourceType.isPrimitive()) { - PhysicalFormat source = hereHandle.getMapper().map(hereHandle.getPathFromRoot()); - if (!source.getType().equals("file")) { - throw new RuntimeException( - "Cannot use the regular expression mapper with a source that has a mapper that is not file-based"); + PhysicalFormat source = hereHandle.getMapper().map(hereHandle.getPathFromRoot()); + if (!source.getType().equals("file")) { + throw new RuntimeException( + "Cannot use the regular expression mapper with a source that has a mapper that is not file-based"); } sourceStr = ((AbsFile) source).getPath(); } else { throw new RuntimeException("Source must be an array of mapped objects or strings. Found " + sourceType); - } - Pattern p = Pattern.compile(cp.getMatch()); - Matcher m = p.matcher(sourceStr); - if (!m.find()) { - throw new RuntimeException("No match found! source='" + sourceStr + "' match = '" + cp.getMatch() - + "'"); - } - // find group number to replace - Pattern p2 = Pattern.compile("(\\\\\\d)"); - Matcher m2 = p2.matcher(cp.getTransform()); - StringBuffer sb = new StringBuffer(); - while (m2.find()) { - String group = m2.group(1); - int g = Integer.parseInt(group.substring(1)); - m2.appendReplacement(sb, m.group(g)); - } - m2.appendTail(sb); - return new AbsFile(sb.toString()); - } - - public Path rmap(String name) { - return Path.EMPTY_PATH; - } - - public boolean isStatic() { - return true; - } - -} + } + Pattern p = Pattern.compile(cp.getMatch()); + Matcher m = p.matcher(sourceStr); + if (!m.find()) { + throw new RuntimeException("No match found! source='" + sourceStr + "' match = '" + cp.getMatch() + + "'"); + } + // find group number to replace + Pattern p2 = Pattern.compile("(\\\\\\d)"); + Matcher m2 = p2.matcher(cp.getTransform()); + StringBuffer sb = new StringBuffer(); + while (m2.find()) { + String group = m2.group(1); + int g = Integer.parseInt(group.substring(1)); + m2.appendReplacement(sb, m.group(g)); + } + m2.appendTail(sb); + return new AbsFile(sb.toString()); + } + + public Path rmap(String name) { + return Path.EMPTY_PATH; + } + + public boolean isStatic() { + return true; + } + +} From hategan at ci.uchicago.edu Sun May 18 14:43:52 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 18 May 2014 14:43:52 -0500 (CDT) Subject: [Swift-commit] r7876 - branches/release-0.95/src/org/griphyn/vdl/karajan/lib Message-ID: <20140518194352.B72429D04B@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-18 14:43:51 -0500 (Sun, 18 May 2014) New Revision: 7876 Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/AppendArray.java Log: fixed NPE due to missing _traceline param Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/AppendArray.java =================================================================== --- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/AppendArray.java 2014-05-18 02:29:15 UTC (rev 7875) +++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/AppendArray.java 2014-05-18 19:43:51 UTC (rev 7876) @@ -28,7 +28,7 @@ @Override protected Signature getSignature() { - return new Signature(params("var", "value")); + return new Signature(params("var", "value", optional("_traceline", null))); } @Override From ketan at ci.uchicago.edu Sun May 18 16:41:00 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Sun, 18 May 2014 16:41:00 -0500 (CDT) Subject: [Swift-commit] r7877 - SwiftApps/swift-galaxy Message-ID: <20140518214100.959359CCF4@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-18 16:41:00 -0500 (Sun, 18 May 2014) New Revision: 7877 Modified: SwiftApps/swift-galaxy/setup.py Log: fix symlink Modified: SwiftApps/swift-galaxy/setup.py =================================================================== --- SwiftApps/swift-galaxy/setup.py 2014-05-18 19:43:51 UTC (rev 7876) +++ SwiftApps/swift-galaxy/setup.py 2014-05-18 21:41:00 UTC (rev 7877) @@ -14,7 +14,7 @@ galaxyloc=sys.argv[1] #step 1. Create symlink to the swift directory from the tools directory of galaxyloc if not already present -os.symlink (os.getcwd()+"/screenshots", galaxyloc+"/tools/screenshots"); +os.symlink (os.getcwd()+"/swift", galaxyloc+"/tools/swift"); #step 2. update the tool_conf.xml From ketan at ci.uchicago.edu Sun May 18 18:08:20 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Sun, 18 May 2014 18:08:20 -0500 (CDT) Subject: [Swift-commit] r7878 - SwiftApps/swift-galaxy Message-ID: <20140518230820.400EC9D04B@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-18 18:08:20 -0500 (Sun, 18 May 2014) New Revision: 7878 Modified: SwiftApps/swift-galaxy/setup.py Log: setup changes Modified: SwiftApps/swift-galaxy/setup.py =================================================================== --- SwiftApps/swift-galaxy/setup.py 2014-05-18 21:41:00 UTC (rev 7877) +++ SwiftApps/swift-galaxy/setup.py 2014-05-18 23:08:20 UTC (rev 7878) @@ -1,5 +1,7 @@ #!/usr/bin/env python +import re + """ Setup script to install swift-galaxy framework to your Galaxy installation. """ @@ -18,6 +20,10 @@ #step 2. update the tool_conf.xml +lines = open(galaxyloc+"/tool_conf.xml").readlines() +#i = lines.index("") +#lines.insert(i, 'Random text to insert') + #
# # From swift at ci.uchicago.edu Sun May 18 19:15:04 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Sun, 18 May 2014 19:15:04 -0500 (CDT) Subject: [Swift-commit] cog r3906 Message-ID: <20140519001504.C52978D000FE@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3906 | hategan | 2014-05-18 19:12:07 -0500 (Sun, 18 May 2014) | 1 line fix for bug #1197 (port from 4.1.10) ------------------------------------------------------------------------ Index: modules/provider-ssh/src/org/globus/cog/abstraction/impl/sshcl/execution/JobSubmissionTaskHandler.java =================================================================== --- modules/provider-ssh/src/org/globus/cog/abstraction/impl/sshcl/execution/JobSubmissionTaskHandler.java (revision 3905) +++ modules/provider-ssh/src/org/globus/cog/abstraction/impl/sshcl/execution/JobSubmissionTaskHandler.java (working copy) @@ -90,6 +90,9 @@ */ cmdarray.add("-s"); + if (logger.isInfoEnabled()) { + logger.info("SSH-CL cmd-array: " + new ArrayList(cmdarray)); + } return cmdarray; } @@ -119,6 +122,17 @@ ps.println(escape(spec.getEnvironmentVariable(env))); } + /** + * Fix for bug #1197 which is really just some messed up logic + * in a particular piece of software that does what it shouldn't do + * in a profile script. + * + * This fix is in the wrong place, in the sense that, if anything, + * it should be in the coaster code. Unfortunately there's no + * nice place to fit it in the coaster code. + */ + ps.println("export SHLVL=1"); + ps.print(escape(spec.getExecutable())); for (String arg : spec.getArgumentsAsList()) { ps.print(" "); @@ -129,7 +143,7 @@ ps.flush(); super.processIN(in, dir, os); } - + /* * Escape everything that bash cares about (I hope this is everything) */ From ketan at ci.uchicago.edu Mon May 19 09:49:12 2014 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 19 May 2014 09:49:12 -0500 (CDT) Subject: [Swift-commit] r7879 - SwiftApps/swift-galaxy Message-ID: <20140519144913.6874B9D66C@svn.ci.uchicago.edu> Author: ketan Date: 2014-05-19 09:49:05 -0500 (Mon, 19 May 2014) New Revision: 7879 Modified: SwiftApps/swift-galaxy/setup.py Log: setup mods to update tool_conf xml Modified: SwiftApps/swift-galaxy/setup.py =================================================================== --- SwiftApps/swift-galaxy/setup.py 2014-05-18 23:08:20 UTC (rev 7878) +++ SwiftApps/swift-galaxy/setup.py 2014-05-19 14:49:05 UTC (rev 7879) @@ -1,6 +1,6 @@ #!/usr/bin/env python -import re +import xml.etree.ElementTree as ET """ Setup script to install swift-galaxy framework to your Galaxy installation. @@ -16,11 +16,18 @@ galaxyloc=sys.argv[1] #step 1. Create symlink to the swift directory from the tools directory of galaxyloc if not already present -os.symlink (os.getcwd()+"/swift", galaxyloc+"/tools/swift"); +#os.symlink (os.getcwd()+"/swift", galaxyloc+"/tools/swift"); #step 2. update the tool_conf.xml -lines = open(galaxyloc+"/tool_conf.xml").readlines() +tree = ET.parse(galaxyloc+"/tool_conf.xml") +root = tree.getroot() + +if 'swift' not in root.attrib: + root.append((ET.fromstring('
\n \n \n \n
\n '))) + +tree.write (galaxyloc+"/tool_conf.xml") + #i = lines.index("") #lines.insert(i, 'Random text to insert') @@ -31,5 +38,5 @@ #
#step 3. Print message +print "Swift/Galaxy Installed." - From swift at ci.uchicago.edu Wed May 28 23:15:08 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 28 May 2014 23:15:08 -0500 (CDT) Subject: [Swift-commit] cog r3907 Message-ID: <20140529041508.629708D000A3@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3907 | hategan | 2014-05-28 23:14:07 -0500 (Wed, 28 May 2014) | 1 line fixed bug where multiple stageout notifications were sent per job; abort stageouts and properly send job status if a stageout encounters an I/O error ------------------------------------------------------------------------ Index: modules/provider-coaster/resources/worker.pl =================================================================== --- modules/provider-coaster/resources/worker.pl (revision 3906) +++ modules/provider-coaster/resources/worker.pl (working copy) @@ -81,7 +81,7 @@ ERROR_STAGEIN_FILE_WRITE => 522, ERROR_STAGEIN_COPY => 524, ERROR_STAGEIN_REQUEST => 525, - ERROR_STAGEOUT_COPY => 528, + ERROR_STAGEOUT_IO => 528, ERROR_STAGEOUT_SEND => 515, ERROR_STAGEOUT_TIMEOUT => 516, ERROR_PROCESS_FORK => 512, @@ -667,12 +667,20 @@ my $buffer; my $sz = read($handle, $buffer, IOBUFSZ); if (!defined $sz) { - wlog INFO, "$tag Failed to read data from file: $!\n"; - return (FINAL_FLAG + ERROR_FLAG, "$!", CONTINUE); + my $err = "Failed to read data from file: $!"; + my $jobid = $$state{"jobid"}; + wlog INFO, "$tag $err\n"; + abortStageouts($jobid); + queueJobStatusCmd($jobid, FAILED, ERROR_STAGEOUT_IO, $err); + return (FINAL_FLAG + ERROR_FLAG, "$err", CONTINUE); } elsif ($sz == 0 && $$state{"sent"} < $$state{"size"}) { - wlog INFO, "$tag File size mismatch. $$state{'size'} vs. $$state{'sent'}\n"; - return (FINAL_FLAG + ERROR_FLAG, "File size mismatch. Expected $$state{'size'}, got $$state{'sent'}", CONTINUE); + my $err = "File size mismatch. Expected $$state{'size'}, got $$state{'sent'}"; + my $jobid = $$state{"jobid"}; + wlog INFO, "$tag $err\n"; + abortStageouts($jobid); + queueJobStatusCmd($jobid, FAILED, ERROR_STAGEOUT_IO, $err); + return (FINAL_FLAG + ERROR_FLAG, $err, CONTINUE); } $$state{"sent"} += $sz; wlog DEBUG, "$tag size: $$state{'size'}, sent: $$state{'sent'}\n"; @@ -691,7 +699,7 @@ } sub fileData { - my ($cmd, $lname, $rname) = @_; + my ($cmd, $jobid, $lname, $rname) = @_; my $desc; if (!open($desc, "<", "$lname")) { @@ -700,6 +708,7 @@ } return { "cmd" => $cmd, + "jobid" => $jobid, "state" => 0, "handle" => $desc, "nextData" => \&nextFileData, @@ -1519,6 +1528,13 @@ push(@$waiting, $jobid); } +sub abortStageouts { + my ($jobid) = @_; + + # something larger than the number of actual stageouts + $JOBDATA{$jobid}{"stageindex"} = 1000000; +} + sub stageout { my ($jobid) = @_; @@ -1558,7 +1574,7 @@ if (!defined($JOBDATA{$jobid}{"stagoutStatusSent"})) { wlog DEBUG, "$jobid Sending STAGEOUT status\n"; queueJobStatusCmd($jobid, STAGEOUT, 0, "workerid=$ID"); - $JOBDATA{$jobid}{"jobStatusSent"} = 1; + $JOBDATA{$jobid}{"stageoutStatusSent"} = 1; } my $rfile = $$STAGED[$STAGEINDEX]; $JOBDATA{$jobid}{"stageindex"} = $STAGEINDEX + 1; @@ -1572,13 +1588,13 @@ $JOBDATA{$jobid}{"stageoutCount"} += 1; wlog DEBUG, "$jobid Stagecount is $JOBDATA{$jobid}{stageoutCount}\n"; - queueCmdCustomDataHandling(putFileCB($jobid), fileData("PUT", $lfile, $rfile)); + queueCmdCustomDataHandling(putFileCB($jobid), fileData("PUT", $jobid, $lfile, $rfile)); } elsif ($protocol eq "sfs") { mkfdir($jobid, $path); if (!copy($lfile, $path)) { wlog DEBUG, "$jobid Error staging out $lfile to $path: $!\n"; - queueJobStatusCmd($jobid, FAILED, ERROR_STAGEOUT_COPY, "$!"); + queueJobStatusCmd($jobid, FAILED, ERROR_STAGEOUT_IO, "$!"); return; } else { From hategan at ci.uchicago.edu Wed May 28 23:16:22 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 28 May 2014 23:16:22 -0500 (CDT) Subject: [Swift-commit] r7880 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20140529041622.22E659D5A5@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-28 23:16:21 -0500 (Wed, 28 May 2014) New Revision: 7880 Modified: trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java Log: coalesce multiple separators in fixed array mapper (i.e. "a, b" is now properly split into "a", "b" instead of "a", "", "b") Modified: trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java 2014-05-19 14:49:05 UTC (rev 7879) +++ trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java 2014-05-29 04:16:21 UTC (rev 7880) @@ -71,7 +71,7 @@ } else if (Types.STRING.equals(files.getType())) { String v = (String) files.getValue(); - this.files = v.split("[\\s,;]"); + this.files = v.split("[\\s,;]+"); } } From hategan at ci.uchicago.edu Thu May 29 01:38:04 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 29 May 2014 01:38:04 -0500 (CDT) Subject: [Swift-commit] r7881 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20140529063804.4175B9D047@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-29 01:38:03 -0500 (Thu, 29 May 2014) New Revision: 7881 Modified: trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java trunk/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java trunk/src/org/griphyn/vdl/mapping/file/ROIMapper.java Log: fixed recursive globbing in the filesys mapper Modified: trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2014-05-29 04:16:21 UTC (rev 7880) +++ trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2014-05-29 06:38:03 UTC (rev 7881) @@ -240,7 +240,7 @@ if (logger.isDebugEnabled()) { logger.debug("Processing existing file " + file.getName()); } - Path p = rmap(cp, file.getName()); + Path p = rmap(cp, file); if (p != null) { if (logger.isDebugEnabled()) { logger.debug("reverse-mapped to path " + p); @@ -351,7 +351,8 @@ * @param name the filename to map to a path * @return a Path to the supplied filename, null on failure */ - public Path rmap(AbstractFileMapperParams cp, String name) { + protected Path rmap(AbstractFileMapperParams cp, AbsFile file) { + String name = file.getName(); logger.debug("rmap "+name); String prefix = cp.getPrefix(); Modified: trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java 2014-05-29 04:16:21 UTC (rev 7880) +++ trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java 2014-05-29 06:38:03 UTC (rev 7881) @@ -28,6 +28,7 @@ import k.thr.LWThread; +import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; @@ -84,7 +85,7 @@ } @Override - public Path rmap(AbstractFileMapperParams cp, String name) { + protected Path rmap(AbstractFileMapperParams cp, AbsFile file) { throw new UnsupportedOperationException(); } Modified: trunk/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java 2014-05-29 04:16:21 UTC (rev 7880) +++ trunk/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java 2014-05-29 06:38:03 UTC (rev 7881) @@ -28,7 +28,7 @@ import org.griphyn.vdl.type.Type; public class FileSystemArrayMapper extends AbstractFileMapper { - private Map filenames = new HashMap(); + private Map, String> filenames = new HashMap, String>(); private int count = 0; @Override @@ -36,15 +36,15 @@ return "FilesysMapper"; } - @Override - public Path rmap(AbstractFileMapperParams cp, String name) { - if (name == null || name.equals("")) { + @Override + protected Path rmap(AbstractFileMapperParams cp, AbsFile file) { + if (file == null) { + // This test is fishy return null; - } - filenames.put(count, name); + } Path p = Path.EMPTY_PATH; - p = p.addFirst(count, true); - ++count; + p = p.addFirst(count++, true); + filenames.put(p.getFirst(), file.getPath()); return p; } @@ -63,13 +63,6 @@ if (filename == null) { return null; } - if (location != null) { - if (!location.endsWith("/")) { - filename = location + '/' + filename; - } else { - filename = location + filename; - } - } return new AbsFile(filename); } Modified: trunk/src/org/griphyn/vdl/mapping/file/ROIMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/ROIMapper.java 2014-05-29 04:16:21 UTC (rev 7880) +++ trunk/src/org/griphyn/vdl/mapping/file/ROIMapper.java 2014-05-29 06:38:03 UTC (rev 7881) @@ -17,14 +17,18 @@ package org.griphyn.vdl.mapping.file; -import java.util.HashMap; -import java.util.Map; -import java.util.StringTokenizer; - -import org.griphyn.vdl.mapping.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.StringTokenizer; +import org.apache.log4j.Logger; +import org.griphyn.vdl.mapping.AbsFile; +import org.griphyn.vdl.mapping.Path; + //TODO Is this still being used? -public class ROIMapper extends AbstractFileMapper { +public class ROIMapper extends AbstractFileMapper { + public static final Logger logger = Logger.getLogger(ROIMapper.class); + private int count; private Map names; @@ -40,7 +44,8 @@ } @Override - public Path rmap(AbstractFileMapperParams cp, String name) { + protected Path rmap(AbstractFileMapperParams cp, AbsFile file) { + String name = file.getName(); if (name.indexOf(".ROI") == -1) { return null; } @@ -91,8 +96,10 @@ } else { return null; } + } + if (logger.isInfoEnabled()) { + logger.info(name + " parsed into " + path); } - System.out.println(name + " parsed into " + path); return path; } } From hategan at ci.uchicago.edu Thu May 29 01:51:17 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 29 May 2014 01:51:17 -0500 (CDT) Subject: [Swift-commit] r7882 - in trunk/src/org/griphyn/vdl: karajan/lib mapping/nodes Message-ID: <20140529065117.F34489D047@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-29 01:51:16 -0500 (Thu, 29 May 2014) New Revision: 7882 Modified: trunk/src/org/griphyn/vdl/karajan/lib/Mark.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java Log: properly set exceptions on dynamic arrays when an app fails Modified: trunk/src/org/griphyn/vdl/karajan/lib/Mark.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Mark.java 2014-05-29 06:38:03 UTC (rev 7881) +++ trunk/src/org/griphyn/vdl/karajan/lib/Mark.java 2014-05-29 06:51:16 UTC (rev 7882) @@ -32,6 +32,7 @@ import org.griphyn.vdl.mapping.DataDependentException; import org.griphyn.vdl.mapping.MappingDependentException; import org.griphyn.vdl.mapping.nodes.AbstractDataNode; +import org.griphyn.vdl.mapping.nodes.AbstractFutureArrayDataNode; public class Mark extends SwiftFunction { private ArgRef> restarts; @@ -48,18 +49,31 @@ try { if (err.getValue(stack)) { boolean mapping = this.mapping.getValue(stack); + Channel files = this.restarts.getValue(stack); for (AbstractDataNode dn : files) { - Collection leaves = dn.getLeaves(); - for (DSHandle leaf : leaves) { - synchronized (leaf) { - if (mapping) { - leaf.setValue(new MappingDependentException(leaf, null)); + if (dn.getType().isArray() && !dn.isClosed()) { + // a dynamic array + AbstractFutureArrayDataNode fdn = (AbstractFutureArrayDataNode) dn; + if (mapping) { + fdn.setException(new MappingDependentException(dn, null)); + } + else { + fdn.setException(new DataDependentException(dn, null)); + } + } + else { + Collection leaves = dn.getLeaves(); + for (DSHandle leaf : leaves) { + synchronized (leaf) { + if (mapping) { + leaf.setValue(new MappingDependentException(leaf, null)); + } + else { + leaf.setValue(new DataDependentException(leaf, null)); + } + leaf.closeShallow(); } - else { - leaf.setValue(new DataDependentException(leaf, null)); - } - leaf.closeShallow(); } } } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java 2014-05-29 06:38:03 UTC (rev 7881) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java 2014-05-29 06:51:16 UTC (rev 7882) @@ -229,4 +229,9 @@ public int arraySize() { return handles.size(); } + + public synchronized void setException(RuntimeException e) { + this.exception = e; + closeShallow(); + } } From hategan at ci.uchicago.edu Thu May 29 04:18:29 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 29 May 2014 04:18:29 -0500 (CDT) Subject: [Swift-commit] r7883 - trunk/libexec Message-ID: <20140529091829.53D789D595@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-29 04:18:27 -0500 (Thu, 29 May 2014) New Revision: 7883 Modified: trunk/libexec/swift-int.k Log: fixed wrapper.log.always.transfer Modified: trunk/libexec/swift-int.k =================================================================== --- trunk/libexec/swift-int.k 2014-05-29 06:51:16 UTC (rev 7882) +++ trunk/libexec/swift-int.k 2014-05-29 09:18:27 UTC (rev 7883) @@ -357,7 +357,7 @@ stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout)) stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)), - + (fileDirs, inFiles, outFiles, outCollect) := getStagingInfo(stagein, stageout) os := siteProfile(rhost, "SYSINFO:OS") @@ -471,7 +471,7 @@ } if (configProperty("wrapperlog.always.transfer") == "true") { - discard(transferWrapperLog(rhost, RUN_DIR, jobid, jobdir)) + discard(transferWrapperLog(rhost, jobid, jobdir)) } cacheUnlockFiles(inFiles, SHARED_DIR, rhost) { From hategan at ci.uchicago.edu Thu May 29 04:21:56 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 29 May 2014 04:21:56 -0500 (CDT) Subject: [Swift-commit] r7884 - trunk/libexec Message-ID: <20140529092156.9D7679D595@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-29 04:21:53 -0500 (Thu, 29 May 2014) New Revision: 7884 Modified: trunk/libexec/_swiftwrap Log: Don't overwrite the DIR variable; only do the collect if MPI rank 0 Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2014-05-29 09:18:27 UTC (rev 7883) +++ trunk/libexec/_swiftwrap 2014-05-29 09:21:53 UTC (rev 7884) @@ -547,19 +547,21 @@ fi fi -if [ "$COLLECT" != "" ]; then - logstate "COLLECT" - log "Collect list is $COLLECT" - exec 4<> "$WFDIR/shared/${ID}-collect" - for O in "$COLLECT" ; do - log "Collecting $O" - ls -1 $O >&4 - for OO in $O ; do - DIR=`dirname "$OO"` - mv $OO "$WFDIR/shared/$DIR" 2>&1 >& "$INFO" +if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then + if [ "$COLLECT" != "" ]; then + logstate "COLLECT" + log "Collect list is $COLLECT" + exec 4<> "$WFDIR/shared/${ID}-collect" + for O in "$COLLECT" ; do + log "Collecting $O" + ls -1 $O >&4 + for OO in $O ; do + OODIR=`dirname "$OO"` + mv $OO "$WFDIR/shared/$OODIR" 2>&1 >& "$INFO" + done done - done - exec 4>&- + exec 4>&- + fi fi log "Moving back to workflow directory $WFDIR" From davidk at ci.uchicago.edu Thu May 29 11:41:17 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 29 May 2014 11:41:17 -0500 (CDT) Subject: [Swift-commit] r7885 - in trunk: bin docs/userguide Message-ID: <20140529164117.4A3C39D047@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-29 11:41:17 -0500 (Thu, 29 May 2014) New Revision: 7885 Modified: trunk/bin/swiftrun trunk/docs/userguide/configuration Log: Document and allow property file.gc.enabled Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2014-05-29 09:21:53 UTC (rev 7884) +++ trunk/bin/swiftrun 2014-05-29 16:41:17 UTC (rev 7885) @@ -71,6 +71,7 @@ 'clustering.queue.delay' => 'clustering.queue.delay', 'config.rundirs' => 'config.rundirs', 'execution.retries' => 'execution.retries', + 'file.gc.enabled' => 'file.gc.enabled', 'foreach.max.threads' => 'foreach.max.threads', 'kickstart.enabled' => 'kickstart.enabled', 'kickstart.always.transfer' => 'kickstart.always.transfer', Modified: trunk/docs/userguide/configuration =================================================================== --- trunk/docs/userguide/configuration 2014-05-29 09:21:53 UTC (rev 7884) +++ trunk/docs/userguide/configuration 2014-05-29 16:41:17 UTC (rev 7885) @@ -343,6 +343,14 @@ |The number of time a job will be retried if it fails (giving a maximum of 1 + execution.retries attempts at execution) +|file.gc.enabled +|true, false +|true +|Files mapped by the concurrent mapper (i.e. when you don't + explicitly specify a mapper) are deleted when they are not + in use any more. This property can be used to prevent + files mapped by the concurrent mapper from being deleted. + |foreach.max.threads |Positive integer |1024 From davidk at ci.uchicago.edu Thu May 29 11:49:40 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 29 May 2014 11:49:40 -0500 (CDT) Subject: [Swift-commit] r7886 - branches/release-0.95/bin Message-ID: <20140529164940.299569D047@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-29 11:49:40 -0500 (Thu, 29 May 2014) New Revision: 7886 Modified: branches/release-0.95/bin/swiftrun Log: Add file.gc.enabled property to config Modified: branches/release-0.95/bin/swiftrun =================================================================== --- branches/release-0.95/bin/swiftrun 2014-05-29 16:41:17 UTC (rev 7885) +++ branches/release-0.95/bin/swiftrun 2014-05-29 16:49:40 UTC (rev 7886) @@ -71,6 +71,7 @@ 'clustering.queue.delay' => 'clustering.queue.delay', 'config.rundirs' => 'config.rundirs', 'execution.retries' => 'execution.retries', + 'file.gc.enabled' => 'file.gc.enabled', 'foreach.max.threads' => 'foreach.max.threads', 'kickstart.enabled' => 'kickstart.enabled', 'kickstart.always.transfer' => 'kickstart.always.transfer', From hategan at ci.uchicago.edu Thu May 29 15:51:00 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 29 May 2014 15:51:00 -0500 (CDT) Subject: [Swift-commit] r7887 - trunk/src/org/griphyn/vdl/mapping/nodes Message-ID: <20140529205100.D2CA29D047@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-29 15:51:00 -0500 (Thu, 29 May 2014) New Revision: 7887 Modified: trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java Log: run non-composite mapped data through the DMC Modified: trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java 2014-05-29 16:49:40 UTC (rev 7886) +++ trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java 2014-05-29 20:51:00 UTC (rev 7887) @@ -186,6 +186,7 @@ return; } if (!root.getType().isComposite()) { + checkConsistency(root, root, dmc); return; } // Static mappers are (array) mappers which know the size of From hategan at ci.uchicago.edu Thu May 29 23:18:38 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 29 May 2014 23:18:38 -0500 (CDT) Subject: [Swift-commit] r7888 - trunk/src/org/griphyn/vdl/mapping/nodes Message-ID: <20140530041838.BB2579D595@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-29 23:18:38 -0500 (Thu, 29 May 2014) New Revision: 7888 Modified: trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedPrimitiveDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java Log: avoid stack overflow when getMapper is called on a primitive node Modified: trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedPrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedPrimitiveDataNode.java 2014-05-29 20:51:00 UTC (rev 7887) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootClosedPrimitiveDataNode.java 2014-05-30 04:18:38 UTC (rev 7888) @@ -102,6 +102,11 @@ } @Override + public Mapper getMapper() { + throw new UnsupportedOperationException("Primitive types do not have a mapper"); + } + + @Override public void closeArraySizes() { // does not apply } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java 2014-05-29 20:51:00 UTC (rev 7887) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java 2014-05-30 04:18:38 UTC (rev 7888) @@ -94,6 +94,11 @@ } @Override + public Mapper getMapper() { + throw new UnsupportedOperationException("Primitive types do not have a mapper"); + } + + @Override public void closeArraySizes() { // does not apply } From hategan at ci.uchicago.edu Thu May 29 23:19:02 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 29 May 2014 23:19:02 -0500 (CDT) Subject: [Swift-commit] r7889 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20140530041902.97DC89D595@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-29 23:19:02 -0500 (Thu, 29 May 2014) New Revision: 7889 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java Log: throw a nice exception when filename() is called with primitive data as an argument Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2014-05-30 04:18:38 UTC (rev 7888) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2014-05-30 04:19:02 UTC (rev 7889) @@ -17,6 +17,7 @@ package org.griphyn.vdl.karajan.lib.swiftscript; +import k.rt.ExecutionException; import k.rt.Stack; import org.globus.cog.karajan.analyzer.ArgRef; @@ -37,7 +38,10 @@ @Override public Object function(Stack stack) { - AbstractDataNode var = this.var.getValue(stack); + AbstractDataNode var = this.var.getValue(stack); + if (var.getType().isPrimitive()) { + throw new ExecutionException(this, "Cannot invoke filename() on a primitive value (" + var + ")"); + } String s = argList(filename(var), true); DSHandle result = NodeFactory.newRoot(Field.GENERIC_STRING, s); if (PROVENANCE_ENABLED) { From hategan at ci.uchicago.edu Fri May 30 18:25:27 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 30 May 2014 18:25:27 -0500 (CDT) Subject: [Swift-commit] r7890 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20140530232527.730E19D047@svn.ci.uchicago.edu> Author: hategan Date: 2014-05-30 18:25:27 -0500 (Fri, 30 May 2014) New Revision: 7890 Modified: trunk/src/org/griphyn/vdl/karajan/lib/While.java Log: fixed thread id issue in iterate() which can cause duplicate mapping errors Modified: trunk/src/org/griphyn/vdl/karajan/lib/While.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/While.java 2014-05-30 04:19:02 UTC (rev 7889) +++ trunk/src/org/griphyn/vdl/karajan/lib/While.java 2014-05-30 23:25:27 UTC (rev 7890) @@ -12,6 +12,7 @@ import java.util.LinkedList; import java.util.List; +import k.rt.KRunnable; import k.rt.SingleValueChannel; import k.rt.Stack; import k.thr.LWThread; @@ -80,42 +81,55 @@ int i = thr.checkSliceAndPopState(); SingleValueChannel next = (SingleValueChannel) thr.popState(); List drefs = (List) thr.popState(); + LWThread thr2 = (LWThread) thr.popState(); Stack stack = thr.getStack(); try { - switch(i) { - case 0: - drefs = RefCount.build(stack, srefs); - var.setValue(stack, NodeFactory.newRoot(Field.GENERIC_INT, 0)); - c_next.create(stack); - RefCount.incRefs(drefs); - next = (SingleValueChannel) c_next.get(stack); - if (tracer.isEnabled()) { - tracer.trace(thr, unwrap(next)); - } - i++; - case 1: - while (true) { - body.run(thr); - if (next.isEmpty()) { - // must do this twice since the closeDataSet calls - // inside the iterate won't be called if the iterate - // condition is true - RefCount.decRefs(drefs); - RefCount.decRefs(drefs); - break; - } - else { + out: + while(true) { + switch(i) { + case 0: + drefs = RefCount.build(stack, srefs); + var.setValue(stack, NodeFactory.newRoot(Field.GENERIC_INT, 0)); + c_next.create(stack); RefCount.incRefs(drefs); - } - Object val = next.removeFirst(); - if (tracer.isEnabled()) { - tracer.trace(thr, unwrap(next)); - } - var.setValue(stack, val); + next = (SingleValueChannel) c_next.get(stack); + if (tracer.isEnabled()) { + tracer.trace(thr, unwrap(next)); + } + i++; + case 1: + thr2 = thr.fork(new KRunnable() { + @Override + public void run(LWThread thr) { + body.run(thr); + } + }); + i++; + thr2.start(); + case 2: + thr2.waitFor(); + if (next.isEmpty()) { + // must do this twice since the closeDataSet calls + // inside the iterate won't be called if the iterate + // condition is true + RefCount.decRefs(drefs); + RefCount.decRefs(drefs); + break out; + } + else { + RefCount.incRefs(drefs); + } + Object val = next.removeFirst(); + if (tracer.isEnabled()) { + tracer.trace(thr, unwrap(next)); + } + var.setValue(stack, val); + i = 1; } - } + } } catch (Yield y) { + y.getState().push(thr2); y.getState().push(drefs); y.getState().push(next); y.getState().push(i); From wilde at ci.uchicago.edu Fri May 30 20:06:55 2014 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Fri, 30 May 2014 20:06:55 -0500 (CDT) Subject: [Swift-commit] r7891 - www Message-ID: <20140531010655.5A4AD9D5A5@svn.ci.uchicago.edu> Author: wilde Date: 2014-05-30 20:06:55 -0500 (Fri, 30 May 2014) New Revision: 7891 Modified: www/index.html Log: Remove visible redirect message from Swift home page. Modified: www/index.html =================================================================== --- www/index.html 2014-05-30 23:25:27 UTC (rev 7890) +++ www/index.html 2014-05-31 01:06:55 UTC (rev 7891) @@ -1,9 +1,7 @@ - Redirecting to Swift Home Page - Redirecting to Swift Home Page From davidk at ci.uchicago.edu Fri May 30 20:51:29 2014 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Fri, 30 May 2014 20:51:29 -0500 (CDT) Subject: [Swift-commit] r7892 - in SwiftApps/tryswift: . scripts Message-ID: <20140531015129.AF5619D047@svn.ci.uchicago.edu> Author: davidk Date: 2014-05-30 20:51:29 -0500 (Fri, 30 May 2014) New Revision: 7892 Modified: SwiftApps/tryswift/index.php SwiftApps/tryswift/scripts/001-introduction.swift Log: Better intro text Fix some complaints that firefox was giving related to charset and a bug in jquery Modified: SwiftApps/tryswift/index.php =================================================================== --- SwiftApps/tryswift/index.php 2014-05-31 01:06:55 UTC (rev 7891) +++ SwiftApps/tryswift/index.php 2014-05-31 01:51:29 UTC (rev 7892) @@ -1,6 +1,7 @@ + Try Swift @@ -65,7 +66,7 @@ } ?> ]; - +