From noreply at svn.ci.uchicago.edu Fri Oct 1 00:30:16 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 1 Oct 2010 00:30:16 -0500 (CDT) Subject: [Swift-commit] r3652 - SwiftApps/Montage/scripts Message-ID: <20101001053016.779489CC8F@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-10-01 00:30:16 -0500 (Fri, 01 Oct 2010) New Revision: 3652 Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift Log: o SwiftApps/Montage/scripts/Swift_Montage_Apps.swift -- Added app invocation for mMakeHdr. Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-09-30 20:54:24 UTC (rev 3651) +++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-10-01 05:30:16 UTC (rev 3652) @@ -18,7 +18,8 @@ app ( Table fits_tbl ) mConcatFit( Table status_tbl, Status stats[] ) { - mConcatFit @status_tbl @fits_tbl @dirname( stats[0] ); +// mConcatFit @status_tbl @fits_tbl @dirname( stats[0] ); + mConcatFit_wrap @status_tbl @fits_tbl @dirname( stats[0] ); } @@ -42,6 +43,12 @@ } +app ( MosaicData hdr ) mMakeHdr( Table img_tbl ) +{ + mMakeHdr @img_tbl @hdr; +} + + app ( JPEG mos_img_jpg ) mJPEG( Image mos_img ) { mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg; @@ -72,6 +79,7 @@ /* Util Scripts */ /* */ /*----------------------------------------------------------------------------------*/ + app ( Table back_tbl ) Background_list( Table imgs_tbl, Table corrs_tbl ) { Background_list @imgs_tbl @corrs_tbl @back_tbl; From noreply at svn.ci.uchicago.edu Fri Oct 1 15:35:58 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 1 Oct 2010 15:35:58 -0500 (CDT) Subject: [Swift-commit] r3653 - trunk/libexec Message-ID: <20101001203558.AA05C9D8E5@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-01 15:35:58 -0500 (Fri, 01 Oct 2010) New Revision: 3653 Modified: trunk/libexec/cdm.pl Log: Bug fix to CDM lookup Modified: trunk/libexec/cdm.pl =================================================================== --- trunk/libexec/cdm.pl 2010-10-01 05:30:16 UTC (rev 3652) +++ trunk/libexec/cdm.pl 2010-10-01 20:35:58 UTC (rev 3653) @@ -73,7 +73,10 @@ $key = shift(@tokens); $rest = join(' ', @tokens); @keys = (@keys, $key); - $map{$key} = $rest; + if (! defined $map{$key}) + { + $map{$key} = $rest; + } } elsif ($type eq "property") { From noreply at svn.ci.uchicago.edu Fri Oct 1 15:48:46 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 1 Oct 2010 15:48:46 -0500 (CDT) Subject: [Swift-commit] r3654 - trunk/libexec Message-ID: <20101001204846.462089D17F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-01 15:48:46 -0500 (Fri, 01 Oct 2010) New Revision: 3654 Modified: trunk/libexec/cdm.pl Log: Additional minor fixes to CDM lookup Modified: trunk/libexec/cdm.pl =================================================================== --- trunk/libexec/cdm.pl 2010-10-01 20:35:58 UTC (rev 3653) +++ trunk/libexec/cdm.pl 2010-10-01 20:48:46 UTC (rev 3654) @@ -70,11 +70,11 @@ $type = shift(@tokens); if ($type eq "rule") { - $key = shift(@tokens); - $rest = join(' ', @tokens); - @keys = (@keys, $key); + $key = shift(@tokens); if (! defined $map{$key}) { + $rest = join(' ', @tokens); + @keys = (@keys, $key); $map{$key} = $rest; } } From noreply at svn.ci.uchicago.edu Fri Oct 1 19:18:26 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 1 Oct 2010 19:18:26 -0500 (CDT) Subject: [Swift-commit] r3655 - in SwiftApps/SwiftR: . Swift/R Swift/exec Swift/tests Message-ID: <20101002001826.654869D8E5@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-01 19:18:26 -0500 (Fri, 01 Oct 2010) New Revision: 3655 Added: SwiftApps/SwiftR/Swift/exec/configure-site-local SwiftApps/SwiftR/Swift/exec/configure-site-pbs SwiftApps/SwiftR/Swift/exec/start-swift Modified: SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/rserver.swift SwiftApps/SwiftR/Swift/tests/TestSwift.R SwiftApps/SwiftR/TODO Log: Start of changes to split out sites into separate servers. Added mutex to EvalRBatchPersistent for local provider throttling problems. Simplified reserver.swift back down to a single site. configure-pbs is preliminary version, not yet working. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-01 20:48:46 UTC (rev 3654) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-02 00:18:26 UTC (rev 3655) @@ -1,11 +1,11 @@ -swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL, runmode=NULL, initialize=NULL, workerhosts=NULL, keepwork=NULL ) +swiftapply <- function( func, arglists, swiftserver=NULL, callsperbatch=NULL, runmode=NULL, initialize=NULL, workerhosts=NULL, keepwork=NULL ) { # Set Swift default options if not passed as keyword paramets or pre-set by user - if(is.null(site)) - site <- getOption("swift.site") - if(is.null(site)) - site<-"local" + if(is.null(swiftserver)) + site <- getOption("swift.server") + if(is.null(swiftserver)) + swiftserver<-"local" if(is.null(callsperbatch)) callsperbatch <- getOption("swift.callsperbatch") @@ -36,7 +36,7 @@ keepwork <- FALSE; cat("\nSwift properties:\n") - cat(" site =",site,"\n"); + cat(" swiftserver =", swiftserver,"\n"); cat(" callsperbatch =", callsperbatch,"\n") cat(" runmode =", runmode,"\n") cat(" initialize =", initialize,"\n") @@ -45,15 +45,32 @@ user <- Sys.info()[["user"]] + # Initialize globals if first call in this workspace + + requestdirbase = mget("swift.requestdirbase",envir=globalenv(),ifnotfound=list(NULL))[[1]] # FIXME: state more elegantly? + if(is.null(requestdirbase)) { + requestdirbase = sprintf("/tmp/%s/SwiftR/requests.P%.5d",user,Sys.getpid()) + dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE) + assign("swift.requestdirbase",requestdirbase,globalenv()) + requestid = 0; + assign("swift.requestid",0,globalenv()) + } + else { + requestid = get("swift.requestid",envir=globalenv()) + 1; + assign("swift.requestid",requestid,globalenv()) + } + # Execute the calls in batches # rundir <- system("mktemp -d SwiftR.run.XXXX",intern=TRUE) # FIXME: put this in /tmp/$USER/SwiftR/requests - basedir = paste("/tmp/",user,"/SwiftR/requests",sep="") - dir.create(basedir,recursive=TRUE,showWarnings=FALSE) - reqdir <- system(paste("mktemp -d ", basedir, "/SwiftR.run.XXXX",sep=""),intern=TRUE) +# basedir = paste("/tmp/",user,"/SwiftR/requests",sep="") +# reqdir <- system(paste("mktemp -d ", basedir, "/SwiftR.run.XXXX",sep=""),intern=TRUE) + reqdir = sprintf("%s/R%.7d",requestdirbase,requestid) + dir.create(reqdir,showWarnings=FALSE) cat("Swift request is in",reqdir,"\n") - cat(initialize,file=paste(reqdir,"/initialize.R",sep="")) # FIXME: honor this??? + cat(initialize,file=paste(reqdir,"/initialize.R",sep="")) # FIXME: honor this on calls after the first in an R server? + # FIXME: eliminate this oevrhead on each call to swiftapply narglists <- length(arglists) # number of arglists to process batch <- 1 # Next arglist batch number to fill @@ -81,7 +98,7 @@ if( runmode == "manual" ) { # Prompt for return (empty line) to continue; assumes user ran a manual R to process the call. cat("Manual Swift Run:\n run dir: ", getwd(), "/", reqdir,"\n\n") cat(" swift script: ", RunSwiftScript, "\n") - cat(" site: ", site,"\n") + cat(" swiftserver: ", swiftserver,"\n") cat(" swiftapplyScript: ", swiftapplyScript,"\n") cat(" Use RunAllR.sh to process and press return when complete:") system(paste("cp ", system.file(package="Swift","exec/RunAllR.sh"), reqdir)) @@ -89,13 +106,13 @@ } else if (runmode == "script") { RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh") - system(paste(RunSwiftScript,reqdir,site,swiftapplyScript,"\"",workerhosts,"\"")) + system(paste(RunSwiftScript,reqdir,swiftserver,swiftapplyScript,"\"",workerhosts,"\"")) } else { # runmode == "service" # FIXME: check and post error if not "service" # Send request to service - swiftServerDir = paste("/tmp/",user,"/SwiftR/swiftserver",sep="") + swiftServerDir = paste("/tmp/",user,"/SwiftR/swift.",swiftserver,sep="") cat(initialize,file=paste(swiftServerDir,"/initialize.R",sep="")) # FIXME Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-01 20:48:46 UTC (rev 3654) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-02 00:18:26 UTC (rev 3655) @@ -95,6 +95,15 @@ # Ready to talk to the server: send request and read response +while true; do + mkdir $SLOTDIR/mutex + if [ $? != 0 ]; then + sleep 1; + else + break; + fi +done + echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo touch $SLOTDIR/lastwrite @@ -102,4 +111,6 @@ head -3 < $SLOTDIR/fromR.fifo # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit") +rmdir $SLOTDIR/mutex + # Fixme: how to get exceptions and stdout/stderr text from R server ??? \ No newline at end of file Added: SwiftApps/SwiftR/Swift/exec/configure-site-local =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-site-local (rev 0) +++ SwiftApps/SwiftR/Swift/exec/configure-site-local 2010-10-02 00:18:26 UTC (rev 3655) @@ -0,0 +1,61 @@ +#! /bin/bash + +throttleOneCore="-0.001" +localcores=5 # FIXME: parameterize: localthreads=N + +#### DBDBDBDB vvvvvv +#localcores=1 + +cat >tc <sites.xml < + + + 0.15 + 10000 + + $(pwd)/swiftwork + +END + +for i in `seq -w 0 $((localcores-1))`; do + +# FIXME: how to get the right PATH for R in here? +# use /bin/sh and count on users PATH? +# then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename) + +cat >>tc <>sites.xml < + + $throttleOneCore + 10000 + + $(pwd)/swiftwork + +END + +done + +# 1 + +echo '' >>sites.xml + +cat >cf <tc <sites.xml < + + + 0.15 + 10000 + + $(pwd)/swiftwork + + + + + fast + 00:59:00 + $throttlePBS + 10000 + + $HOME/swiftwork + + + + + + fast + 3500 + 00:00:01 + 1 + 1 + 1 + 1 + 2.55 + 10000 + + + /home/wilde/swiftwork + + +END + +cat >cf </dev/tty; echo done > /tmp/SwiftR/swiftserver/resultpipe"; @@ -47,22 +27,8 @@ file initScript <"initialize.R">; file rsScript <"SwiftRServer.sh">; - string location=@arg("location","local"); # FIXME: pass the execution site in as an argument (selects tc entry) - foreach c, i in rcalls { - # FIXME: swift needs a better way to express the dynamic function selection below - if(location=="default") { - (e[i], results[i],stout[i], sterr[i]) = runRdefault(runRscript,initScript,rsScript,c); - } - if(location=="local") { - (e[i], results[i],stout[i], sterr[i]) = runRlocal(runRscript,initScript,rsScript,c); - } - if(location=="service") { - (e[i], results[i],stout[i], sterr[i]) = runRservice(runRscript,initScript,rsScript,c); - } - if(location=="pbs") { - (e[i], results[i],stout[i], sterr[i]) = runRpbs(runRscript,initScript,rsScript,c); - } + (e[i], results[i],stout[i], sterr[i]) = runR(runRscript,initScript,rsScript,c); } } @@ -74,7 +40,7 @@ boolean done; string dir; -# FIXME: read swiftserver dir via @args + # FIXME: read swiftserver dir via @args dir = readData(requestPipeName); # Reads direct from this local pipe. Assumes Swift started in right dir. trace("rserver: got dir", dir); @@ -84,10 +50,11 @@ if (dir=="done") { done=true; } else { done=false;} -# file f<"/tmp/SwiftR/swiftserver/resultpipe">; -# f = writeData("completed\n"); # Doesnt work: runs as soon as the block is entered. -# want: tracef("%k completed\n", "my/responsepipe.fifo", wait); -# %k waits for wait to be fully closed; then sends formatted string to specified file(doing open, write, close). + # file f<"/tmp/SwiftR/swiftserver/resultpipe">; + # f = writeData("completed\n"); # Doesnt work: runs as soon as the block is entered. + # want: tracef("%k completed\n", "my/responsepipe.fifo", wait); + # %k waits for wait to be fully closed; then sends formatted string to specified file(doing open, write, close). + ack(wait); } until (done); Added: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift (rev 0) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-02 00:18:26 UTC (rev 3655) @@ -0,0 +1,45 @@ +#! /bin/bash + +site=$1 + +# FIXME: check args and use better arg parsing + +tmp=/tmp # FIXME: allow this to change eg for sites with main tmp dir elsewhere +throttleOneCore="-0.001" +localcores=5 # FIXME: parameterize: localthreads=N + +SWIFTRBIN=$(cd $(dirname $0); pwd) +rundir=$tmp/$USER/SwiftR/swift.local # rundir prefix # FIXME: handle multiple concurent independent swift servers per user +mkdir -p $(dirname $rundir) +trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success +rm -rf $rundir +ln -s $trundir $rundir +cd $rundir + +script=$SWIFTRBIN/rserver.swift +cp $script $SWIFTRBIN/passive-coaster-swift $SWIFTRBIN/swift.properties $rundir +script=$(basename $script) +cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.sh} . + +# FIXME: rework this script to transfer all shells and rscripts +# needed, and to copy in the R prelude for the R server processes (to +# include for example the OpenMx library) NOTE: Both were done in older version of this script. + +# rm -f requestpipe resultpipe +mkfifo requestpipe resultpipe + +#FIXME JUNK app=/bin/bash +# FIXME: remove these fossils: +#serviceport=1985 +#site=local +#location=$1 +#stagingmethod=$2 + +source $SWIFTRBIN/configure-site-$1 + +$SWIFTRBIN/../swift/bin/swift -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& swift.stdouterr Author: jonmon Date: 2010-10-01 20:51:49 -0500 (Fri, 01 Oct 2010) New Revision: 3656 Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift Log: o SwiftApps/Montage/scripts/Swift_Montage_Apps.swif -- Minor change to mConcatFit app Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-10-02 00:18:26 UTC (rev 3655) +++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-10-02 01:51:49 UTC (rev 3656) @@ -18,8 +18,7 @@ app ( Table fits_tbl ) mConcatFit( Table status_tbl, Status stats[] ) { -// mConcatFit @status_tbl @fits_tbl @dirname( stats[0] ); - mConcatFit_wrap @status_tbl @fits_tbl @dirname( stats[0] ); + mConcatFit @status_tbl @fits_tbl @dirname( stats[0] ); } From noreply at svn.ci.uchicago.edu Mon Oct 4 19:10:54 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 4 Oct 2010 19:10:54 -0500 (CDT) Subject: [Swift-commit] r3657 - in SwiftApps/SwiftR: . Swift/R Swift/exec Swift/tests Message-ID: <20101005001054.E21089CCC5@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-04 19:10:54 -0500 (Mon, 04 Oct 2010) New Revision: 3657 Added: SwiftApps/SwiftR/install.sh Modified: SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/configure-site-local SwiftApps/SwiftR/Swift/exec/configure-site-pbs SwiftApps/SwiftR/Swift/exec/rserver.swift SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/Swift/tests/TestSwift.R SwiftApps/SwiftR/TODO Log: Added SWIFTR_TMP to allow tmp dir to be set for different systems; initial working code for configure-ite-pbs. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-02 01:51:49 UTC (rev 3656) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-05 00:10:54 UTC (rev 3657) @@ -1,4 +1,4 @@ -swiftapply <- function( func, arglists, swiftserver=NULL, callsperbatch=NULL, runmode=NULL, initialize=NULL, workerhosts=NULL, keepwork=NULL ) +swiftapply <- function( func, arglists, swiftserver=NULL, callsperbatch=NULL, runmode=NULL, initialize=NULL, workerhosts=NULL, keepwork=NULL, tmpdir=NULL ) { # Set Swift default options if not passed as keyword paramets or pre-set by user @@ -35,6 +35,13 @@ if(is.null(keepwork)) keepwork <- FALSE; + if(is.null(tmpdir)) + tmpdir <- getOption("swift.tmpdir") + if(is.null(tmpdir)) + tmpdir <- Sys.getenv("SWIFTR_TMP"); + if(tmpdir=="") + tmpdir <- "/tmp"; + cat("\nSwift properties:\n") cat(" swiftserver =", swiftserver,"\n"); cat(" callsperbatch =", callsperbatch,"\n") @@ -49,7 +56,7 @@ requestdirbase = mget("swift.requestdirbase",envir=globalenv(),ifnotfound=list(NULL))[[1]] # FIXME: state more elegantly? if(is.null(requestdirbase)) { - requestdirbase = sprintf("/tmp/%s/SwiftR/requests.P%.5d",user,Sys.getpid()) + requestdirbase = sprintf("%s/%s/SwiftR/requests.P%.5d",tmpdir,user,Sys.getpid()) dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE) assign("swift.requestdirbase",requestdirbase,globalenv()) requestid = 0; @@ -112,7 +119,7 @@ # Send request to service - swiftServerDir = paste("/tmp/",user,"/SwiftR/swift.",swiftserver,sep="") + swiftServerDir = paste(tmpdir,"/",user,"/SwiftR/swift.",swiftserver,sep="") cat(initialize,file=paste(swiftServerDir,"/initialize.R",sep="")) # FIXME Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-02 01:51:49 UTC (rev 3656) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-05 00:10:54 UTC (rev 3657) @@ -8,6 +8,10 @@ exit 1 fi +# tmp=/tmp # FIXME: allow this to change eg for sites with main tmp dir elsewhere +# tmp=/scratch/local # FIXME: allow this to change eg for sites with main tmp dir elsewhere +tmp=${SWIFTR_TMP:-/tmp} + initRScript=$1 RServerScript=$2 callFile=$3 @@ -60,7 +64,7 @@ # Ensure that the dir for this slot exists. -BASEDIR=/tmp/$(id -nu)/SwiftR/Rworkers +BASEDIR=$tmp/$(id -nu)/SwiftR/Rworkers WORKERDIR=$BASEDIR/worker.$SWIFT_WORKER_PID SLOTDIR=$WORKERDIR/${SWIFT_JOB_SLOT} Modified: SwiftApps/SwiftR/Swift/exec/configure-site-local =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-site-local 2010-10-02 01:51:49 UTC (rev 3656) +++ SwiftApps/SwiftR/Swift/exec/configure-site-local 2010-10-05 00:10:54 UTC (rev 3657) @@ -1,6 +1,7 @@ #! /bin/bash throttleOneCore="-0.001" +throttleOneCore="0.00" # FIXME: test if new swft fix makes zero OK rather than -0.001 localcores=5 # FIXME: parameterize: localthreads=N #### DBDBDBDB vvvvvv Modified: SwiftApps/SwiftR/Swift/exec/configure-site-pbs =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-site-pbs 2010-10-02 01:51:49 UTC (rev 3656) +++ SwiftApps/SwiftR/Swift/exec/configure-site-pbs 2010-10-05 00:10:54 UTC (rev 3657) @@ -4,7 +4,7 @@ cat >tc <sites.xml < - - fast - 3500 - 00:00:01 - 1 + short + 12000 + 32 1 1 1 @@ -41,11 +39,14 @@ 10000 - /home/wilde/swiftwork + $HOME/swiftwork END +# 00:00:01 +# fast + cat >cf </dev/tty; echo done > /tmp/SwiftR/swiftserver/resultpipe"; - bash "-c" @strcat("echo done > ",resultPipeName); + bashlocal "-c" @strcat("echo done > ",resultPipeName); } (external e[]) apply (string runDir) Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-02 01:51:49 UTC (rev 3656) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-05 00:10:54 UTC (rev 3657) @@ -5,6 +5,10 @@ # FIXME: check args and use better arg parsing tmp=/tmp # FIXME: allow this to change eg for sites with main tmp dir elsewhere +tmp=/scratch/local # FIXME: allow this to change eg for sites with main tmp dir elsewhere + +tmp=${SWIFTR_TMP:-/tmp} + throttleOneCore="-0.001" localcores=5 # FIXME: parameterize: localthreads=N Modified: SwiftApps/SwiftR/Swift/tests/TestSwift.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-10-02 01:51:49 UTC (rev 3656) +++ SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-10-05 00:10:54 UTC (rev 3657) @@ -200,7 +200,8 @@ swiftres = swiftapply(matfunc,arglist,callsperbatch=9) diffs <- 0 -for(i in 1:length(swiftres) ) { +#for(i in 1:length(swiftres) ) { +for(i in c(seq(1,100,10),100)) { if( !all.equal(swiftres[[i]],localres) ) { diffs <- diffs + 1 if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) @@ -230,7 +231,9 @@ swiftres = swiftapply(matfunc,arglist,callsperbatch=7) diffs <- 0 -for(i in 1:length(swiftres) ) { +#for(i in 1:length(swiftres) ) { +for(i in c(seq(1,length(swiftres),10),length(swiftres))) { + if( !all.equal(swiftres[[i]],localres) ) { diffs <- diffs + 1 if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) Modified: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO 2010-10-02 01:51:49 UTC (rev 3656) +++ SwiftApps/SwiftR/TODO 2010-10-05 00:10:54 UTC (rev 3657) @@ -1,15 +1,22 @@ *** TO DO LIST: MAIN + +- complete change for envvars like SWIFTR_TMP +- add sourcing of $HOME/.SwiftR.init: pick up variables for the configure_ scripts from here; maybe one file in this dir for each site supported? maybe configure scripts go here? + +- try coasters to communicado-bridled: use simple script for passive coasters: ala swift/lab/mcsswift.sh? +- try tests of coasters provider staging + - coaster provider data mover - tracef to replace job - interrupt a pipe read in R - develop the common config-site-$SITE scripts - finalize() method to clean up request dir? - test with swift logging turned way down +- make tmpdir a param; for start-nnn scripts use SWIFTR_TMP, ~/.SwiftR +- make ports flexible and non-conflicting - - See if we can get Swift to just shut down the channel and start it again? - no extended idle timer - start new swift worker Added: SwiftApps/SwiftR/install.sh =================================================================== --- SwiftApps/SwiftR/install.sh (rev 0) +++ SwiftApps/SwiftR/install.sh 2010-10-05 00:10:54 UTC (rev 3657) @@ -0,0 +1,4 @@ +ver=0.1 +R CMD build Swift +R CMD INSTALL Swift_${ver}.tar.gz +cp Swift_${ver}.tar.gz ~/public_html Property changes on: SwiftApps/SwiftR/install.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Tue Oct 5 16:44:04 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 5 Oct 2010 16:44:04 -0500 (CDT) Subject: [Swift-commit] r3658 - SwiftApps/SwiftR/Swift/exec Message-ID: <20101005214404.E9B489CCC5@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-05 16:44:04 -0500 (Tue, 05 Oct 2010) New Revision: 3658 Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift SwiftApps/SwiftR/Swift/exec/start-swift Log: Add support for start-swift ssh case; make rserver.swift run a dummy job on startup to force the coaster provider into passive state and announce its worker connection port. Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/rserver.swift 2010-10-05 00:10:54 UTC (rev 3657) +++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2010-10-05 21:44:04 UTC (rev 3658) @@ -16,6 +16,11 @@ bashlocal "-c" @strcat("echo done > ",resultPipeName); } +app passivate () +{ + bash "-c" "echo dummy swift job;"; +} + (external e[]) apply (string runDir) { RData rcalls[] ; @@ -32,6 +37,8 @@ } } +passivate(); + string pipedir = @arg("pipedir"); global string requestPipeName = @strcat(pipedir,"/requestpipe"); global string resultPipeName = @strcat(pipedir,"/resultpipe"); Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-05 00:10:54 UTC (rev 3657) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-05 21:44:04 UTC (rev 3658) @@ -1,18 +1,72 @@ #! /bin/bash -site=$1 +# Define internal functions +function wait-and-start-workers +{ + # Look for: + # Passive queue processor initialized. Callback URI is http://140.221.8.62:55379 + + for try in $(seq 1 20); do + uriline=$(grep "Passive queue processor initialized. Callback URI is" $out 2> /dev/null) + if [ "_$uriline" = _ ]; then + sleep 1 + else + break; + fi + done + + if [ "_$uriline" = _ ]; then + echo "$0: No passive state message from Swift - exiting." + exit 1 + fi + + CONTACT=$(echo $uriline | sed -e 's/^.*http:/http:/') + echo Coaster service contact URI: $CONTACT + + LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env? + LOGDIR=/tmp/$USER/SwiftR/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way? + + # mkdir -p $LOGDIR # is done with the ssh command, below + + IDLETIMEOUT=$((60*60*240)) # 10 days: FIXME: make this a command line arg + + rm -rf remotepid.* # FIXME: should not be needed if we start in a new dir each time + for host in $(echo $COMPUTEHOSTS); do + timestamp=$(date "+%Y.%m%d.%H%M%S") + random=$(awk "BEGIN {printf \"%0.5d\", $RANDOM}") + ID=$timestamp.$random + # FIXME: make logging an argument; set false by default + # fixme:send worker.pl to remote host via stdin or scp. + ssh $host /bin/sh -c \'"mkdir -p $LOGDIR"\' + scp $SWIFTBIN/worker.pl $host:$LOGDIR + ssh $host '/bin/sh -c '\'"WORKER_LOGGING_ENABLED=true $LOGDIR/worker.pl $CONTACT $ID $LOGDIR $IDLETIMEOUT 2>&1 & echo PID=\$!"\' >remotepid.$host $sshpidfile +} + +# main script + +site=$1 # local, ssh, ... + # FIXME: check args and use better arg parsing -tmp=/tmp # FIXME: allow this to change eg for sites with main tmp dir elsewhere -tmp=/scratch/local # FIXME: allow this to change eg for sites with main tmp dir elsewhere - tmp=${SWIFTR_TMP:-/tmp} +echo DB $0: site=$site tmp=$tmp + throttleOneCore="-0.001" +throttleOneCore="0.00" localcores=5 # FIXME: parameterize: localthreads=N SWIFTRBIN=$(cd $(dirname $0); pwd) +SWIFTBIN=$SWIFTRBIN/../swift/bin # This depends on ~/SwiftR/Swift/swift being a symlink to swift in RLibrary/Swift + +echo DB $0: SWIFTRBIN=$SWIFTRBIN SWIFTBIN=$SWIFTBIN + rundir=$tmp/$USER/SwiftR/swift.local # rundir prefix # FIXME: handle multiple concurent independent swift servers per user mkdir -p $(dirname $rundir) trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success @@ -20,30 +74,59 @@ ln -s $trundir $rundir cd $rundir +echo DB $0: rundir=$(pwd) SWIFTRBIN=$SWIFTRBIN SWIFTBIN=$SWIFTBIN + script=$SWIFTRBIN/rserver.swift -cp $script $SWIFTRBIN/passive-coaster-swift $SWIFTRBIN/swift.properties $rundir +#cp $script $SWIFTRBIN/passive-coaster-swift $SWIFTRBIN/swift.properties $rundir +cp $script . script=$(basename $script) cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.sh} . -# FIXME: rework this script to transfer all shells and rscripts +# DONE: FIXME: rework this script to transfer all shells and rscripts # needed, and to copy in the R prelude for the R server processes (to # include for example the OpenMx library) NOTE: Both were done in older version of this script. # rm -f requestpipe resultpipe mkfifo requestpipe resultpipe -#FIXME JUNK app=/bin/bash -# FIXME: remove these fossils: -#serviceport=1985 -#site=local -#location=$1 -#stagingmethod=$2 - source $SWIFTRBIN/configure-site-$1 -$SWIFTRBIN/../swift/bin/swift -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& swift.stdouterr & /dev/null + fi + kill 0 # Kill all procs in current process group # FIXME: what was this for???? + } + + trap onexit $TRAPS + + wait-and-start-workers & + starterpid=$! + +fi + +$SWIFTRBIN/../swift/bin/swift -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& $out Author: wilde Date: 2010-10-05 18:06:00 -0500 (Tue, 05 Oct 2010) New Revision: 3659 Modified: SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/TODO SwiftApps/SwiftR/UserGuide Log: Small fix to start-swift for multiple concurrent swift servers; correct swift.server option handling and some costmetic improvements in Swift.R Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-05 21:44:04 UTC (rev 3658) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-05 23:06:00 UTC (rev 3659) @@ -1,9 +1,16 @@ -swiftapply <- function( func, arglists, swiftserver=NULL, callsperbatch=NULL, runmode=NULL, initialize=NULL, workerhosts=NULL, keepwork=NULL, tmpdir=NULL ) +swiftapply <- function( func, arglists, + swiftserver=NULL, + callsperbatch=NULL, + runmode=NULL, + initialize=NULL, + workerhosts=NULL, + keepwork=NULL, + tmpdir=NULL ) { - # Set Swift default options if not passed as keyword paramets or pre-set by user + # Set Swift default options if not passed as keywords or pre-set by user if(is.null(swiftserver)) - site <- getOption("swift.server") + swiftserver <- getOption("swift.server") if(is.null(swiftserver)) swiftserver<-"local" @@ -17,7 +24,7 @@ if(is.null(runmode)) runmode <- "service" # service: send requests to Swift service loop via fifo - # script: run swift for each request, via RunSwiftScript.sh + # script: run swift for each request, via RunSwiftScript.sh (currently broken) # manual: for testing, let user run remote R server manually if(is.null(initialize)) @@ -46,9 +53,9 @@ cat(" swiftserver =", swiftserver,"\n"); cat(" callsperbatch =", callsperbatch,"\n") cat(" runmode =", runmode,"\n") - cat(" initialize =", initialize,"\n") + cat(" tmpdir =", tmpdir,"\n") cat(" workerhosts =", workerhosts,"\n") - cat("\nCurrent dir: ",getwd(),"\n"); + cat(" initialize =", initialize,"\n\n") user <- Sys.info()[["user"]] Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-05 21:44:04 UTC (rev 3658) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-05 23:06:00 UTC (rev 3659) @@ -67,7 +67,7 @@ echo DB $0: SWIFTRBIN=$SWIFTRBIN SWIFTBIN=$SWIFTBIN -rundir=$tmp/$USER/SwiftR/swift.local # rundir prefix # FIXME: handle multiple concurent independent swift servers per user +rundir=$tmp/$USER/SwiftR/swift.$site # rundir prefix # FIXME: handle multiple concurent independent swift servers per user mkdir -p $(dirname $rundir) trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success rm -rf $rundir Modified: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO 2010-10-05 21:44:04 UTC (rev 3658) +++ SwiftApps/SwiftR/TODO 2010-10-05 23:06:00 UTC (rev 3659) @@ -24,6 +24,7 @@ test logic for creating trundir in start-swift-Rserver +test for suitable java in start-swift command. test using persistent server and ssh to communcado; then mcs. @@ -31,9 +32,11 @@ start-swift-workers: if hostis localhost, launch workers directly, not with ssh - debug why workers/coasters hang after several minutes of idle time +fixed: ?? debug why workers/coasters hang after several minutes of idle time +- fix R hanging on fifo read. be able to interrupt it and re-sync. maybe change to read and file not fifo? + ----- find why TestSwift is faster with local internal coaster service (13 secs) than with external service (19 seconds) Modified: SwiftApps/SwiftR/UserGuide =================================================================== --- SwiftApps/SwiftR/UserGuide 2010-10-05 21:44:04 UTC (rev 3658) +++ SwiftApps/SwiftR/UserGuide 2010-10-05 23:06:00 UTC (rev 3659) @@ -20,8 +20,17 @@ CAVEATS -Only one Swift server running per user; can only be used by one R client workspace at a time. +# fixed: Only one Swift server running per user; can only be used by one R client workspace at a time. +when fifos get hung, need to use kill or Quit to break out of R; will fix. + +no retry if swift dies in server loop. + +Swift returns Error object when remote side fails. + +only lapply is implemented (also SwiftApply) - need to see if we can cut down arg passing overhead for many of the apply() cases + + INSTALL cd ??? From noreply at svn.ci.uchicago.edu Wed Oct 6 15:19:47 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 6 Oct 2010 15:19:47 -0500 (CDT) Subject: [Swift-commit] r3660 - in SwiftApps/SwiftR: . Swift/exec Message-ID: <20101006201948.007989CC9E@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-06 15:19:47 -0500 (Wed, 06 Oct 2010) New Revision: 3660 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/configure-site-local SwiftApps/SwiftR/Swift/exec/configure-site-pbs SwiftApps/SwiftR/TODO SwiftApps/SwiftR/UserGuide Log: Make local configution determine number of cores dynamically. Enable EvalRBatchPersistent.sh to log what signal terminated it, and add some temporary looking to debug a mysterious process exit incident. A few cosmetic comment changes. Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-05 23:06:00 UTC (rev 3659) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-06 20:19:47 UTC (rev 3660) @@ -37,15 +37,19 @@ TRAPS="EXIT 1 2 3 15" -function onexit { # FIXME: move this logic into worker.pl, or try to use R timeout option if it works on fifos - need to test. +function onexit() { # FIXME: move this logic into worker.pl, or try to use R timeout option if it works on fifos - need to test. trap - $TRAPS - echo "Terminating R processes $RPIDS" + echo "EvalRBatchPersistent.sh: onexit trap: SIGNAL=$1 RPIDS=$RPIDS" if [ "_$RPIDS" != _ ]; then kill $RPIDS >& /dev/null fi } -trap onexit $TRAPS +trap "onexit 1" 1 +trap "onexit 2" 2 +trap "onexit 3" 3 +trap "onexit 15" 15 +trap "onexit EXIT" EXIT function idletimer { cd $1 @@ -107,14 +111,21 @@ break; fi done +echo DB: Obtained $SLOTDIR/mutex echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo touch $SLOTDIR/lastwrite +echo DB: Sent request + echo dummy stderr response 1>&2 # FIXME - testing if this is the provider staging problem (not xfering zero len stderr) head -3 < $SLOTDIR/fromR.fifo # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit") +echo DB: Got response + rmdir $SLOTDIR/mutex +echo DB: Freed $SLOTDIR/mutex + # Fixme: how to get exceptions and stdout/stderr text from R server ??? \ No newline at end of file Modified: SwiftApps/SwiftR/Swift/exec/configure-site-local =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-site-local 2010-10-05 23:06:00 UTC (rev 3659) +++ SwiftApps/SwiftR/Swift/exec/configure-site-local 2010-10-06 20:19:47 UTC (rev 3660) @@ -2,8 +2,13 @@ throttleOneCore="-0.001" throttleOneCore="0.00" # FIXME: test if new swft fix makes zero OK rather than -0.001 -localcores=5 # FIXME: parameterize: localthreads=N +if [ -r /proc/cpuinfo ]; then + localcores=$(grep '^processor' /proc/cpuinfo | wc -l) +else + localcores=4 +fi + #### DBDBDBDB vvvvvv #localcores=1 @@ -29,7 +34,7 @@ # then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename) cat >>tc <>sites.xml <tc </Swift +SWIFT=/Swift/ $SWIFT/exec/start-swift-workers hostname $SWIFT/exec/start-swift-server +local and ssh servers can be started and left running, across R runs +found via: + HELLO WORLD TEST RUN FULL TEST @@ -57,11 +71,24 @@ or R CMD TEST etc? +USAGE + +Swift returns Error object when remote side fails. + +options: + swift.server: matched server name on start-swift + swift.callsperbatch + initialize: + +less likely to touch: + remove temp reqs (sp???) FIXME + mode (service, manual, ???) + OPENMX EXAMPLES DEBUGGING -8 manual mode +* manual mode * logs to look at @@ -69,3 +96,4 @@ (FIXME: need swiftsnapshot script) + From noreply at svn.ci.uchicago.edu Thu Oct 7 12:28:57 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 12:28:57 -0500 (CDT) Subject: [Swift-commit] r3661 - in trunk/tests/cdm: . star Message-ID: <20101007172857.B46D59CC9E@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 12:28:57 -0500 (Thu, 07 Oct 2010) New Revision: 3661 Added: trunk/tests/cdm/star/ trunk/tests/cdm/star/202-direct-star.check.sh trunk/tests/cdm/star/202-direct-star.clean.sh trunk/tests/cdm/star/202-direct-star.setup.sh trunk/tests/cdm/star/202-direct-star.swift trunk/tests/cdm/star/README.txt trunk/tests/cdm/star/fs.template.data trunk/tests/cdm/star/title.txt Log: Test for CDM with simple .* pattern Added: trunk/tests/cdm/star/202-direct-star.check.sh =================================================================== --- trunk/tests/cdm/star/202-direct-star.check.sh (rev 0) +++ trunk/tests/cdm/star/202-direct-star.check.sh 2010-10-07 17:28:57 UTC (rev 3661) @@ -0,0 +1,8 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 201/201-input.txt || exit 1 +grep $( uname -m ) 201/201-output.txt || exit 1 + +exit 0 Property changes on: trunk/tests/cdm/star/202-direct-star.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/star/202-direct-star.clean.sh =================================================================== --- trunk/tests/cdm/star/202-direct-star.clean.sh (rev 0) +++ trunk/tests/cdm/star/202-direct-star.clean.sh 2010-10-07 17:28:57 UTC (rev 3661) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -rv 201 || exit 1 + +exit 0 Property changes on: trunk/tests/cdm/star/202-direct-star.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/star/202-direct-star.setup.sh =================================================================== --- trunk/tests/cdm/star/202-direct-star.setup.sh (rev 0) +++ trunk/tests/cdm/star/202-direct-star.setup.sh 2010-10-07 17:28:57 UTC (rev 3661) @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +mkdir 201 + +{ + uname -a + date +} > 201/201-input.txt + +exit 0 Property changes on: trunk/tests/cdm/star/202-direct-star.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/star/202-direct-star.swift =================================================================== --- trunk/tests/cdm/star/202-direct-star.swift (rev 0) +++ trunk/tests/cdm/star/202-direct-star.swift 2010-10-07 17:28:57 UTC (rev 3661) @@ -0,0 +1,15 @@ + +type file; + +app (file o) copy (file i) +{ + cp @i @o; +} + +// Will be transformed by CDM to 201/201-input.txt: +file f1<"201-input.txt">; + +// Will be transformed by CDM to 201/201-output.txt: +file f2<"201-output.txt">; + +f2 = copy(f1); Added: trunk/tests/cdm/star/README.txt =================================================================== --- trunk/tests/cdm/star/README.txt (rev 0) +++ trunk/tests/cdm/star/README.txt 2010-10-07 17:28:57 UTC (rev 3661) @@ -0,0 +1 @@ +These tests try to use CDM to match everything (.*) Added: trunk/tests/cdm/star/fs.template.data =================================================================== --- trunk/tests/cdm/star/fs.template.data (rev 0) +++ trunk/tests/cdm/star/fs.template.data 2010-10-07 17:28:57 UTC (rev 3661) @@ -0,0 +1 @@ +rule .* DIRECT _PWD_/201 Added: trunk/tests/cdm/star/title.txt =================================================================== --- trunk/tests/cdm/star/title.txt (rev 0) +++ trunk/tests/cdm/star/title.txt 2010-10-07 17:28:57 UTC (rev 3661) @@ -0,0 +1 @@ +CDM tests w/ "star" From noreply at svn.ci.uchicago.edu Thu Oct 7 12:30:11 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 12:30:11 -0500 (CDT) Subject: [Swift-commit] r3662 - trunk/tests Message-ID: <20101007173011.6549D9CC9E@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 12:30:11 -0500 (Thu, 07 Oct 2010) New Revision: 3662 Modified: trunk/tests/nightly.sh Log: HTML improvements Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-10-07 17:28:57 UTC (rev 3661) +++ trunk/tests/nightly.sh 2010-10-07 17:30:11 UTC (rev 3662) @@ -216,7 +216,19 @@ } html_table() { - html "" + OPTS="" + while [[ ${#*} > 0 ]] + do + case $1 in + cellpadding) + OPTS+="cellpadding=\"$2\" " + shift 2;; + border) + OPTS+="border=\"$2\" " + shift 2;; + esac + done + html "
" } html_~table() { @@ -250,12 +262,25 @@ } html_td() { - ALIGN=$1 - if [ -n $ALIGN ]; then - html "
" - else - html "" - fi + OPTS="" + while [[ ${#*} > 0 ]] + do + case $1 in + align) + OPTS+="align=\"$2\" " + shift 2;; + class) + OPTS+="class=\"$2\" " + shift 2;; + title) + OPTS+="title=\"$2\" " + shift 2;; + width) + OPTS+="width=\"$2\" " + shift 2;; + esac + done + html "" } html_~td() { @@ -358,14 +383,16 @@ CMD=$3 # Command issued (td title) RESULT=$4 # Passed or Failed - WIDTH=$( width "$LABEL" ) + # WIDTH=$( width "$LABEL" ) if [ "$RESULT" == "Passed" ]; then - html "" + # html "" + html_td class "success" width 25 title "$CMD" html_a_href $TEST_LOG "$LABEL" else echo "FAILED" cat $RUNDIR/$TEST_LOG < /dev/null - html "" + # html "" + html_td class "failure" width 25 title "$CMD" html_a_href $TEST_LOG $LABEL fi html_~td @@ -381,11 +408,14 @@ html_h1 "Test results" html_a_name "tests" html_a_href "tests.log" "Output log from tests" - html_table + html_table border 0 cellpadding 1 } start_part() { PART=$1 + echo + echo $PART + echo html_tr part html_th 3 html "$PART" @@ -395,15 +425,18 @@ start_row() { html_tr testline - html_td right + html_td align right width 50 html "$SWIFTCOUNT" + html " " html_~td - html_td right + html_td align right + html " " if [[ -n $TESTLINK ]]; then html_a_href $TESTLINK $TESTNAME else html $TESTNAME fi + html " " html_~td html_td html_table @@ -511,7 +544,7 @@ EXEC_PID=$1 echo "process_exec_trap()" kill -KILL $EXEC_PID - killall -9 java + # killall -9 java } # Execute as part of test set @@ -844,15 +877,16 @@ SKIP_COUNTER=0 -GROUPLIST=( $TESTDIR/functions $TESTDIR/local $TESTDIR/cdm $TESTDIR/cdm/ps $TESTDIR/cdm/ps/pinned ) +# GROUPLIST=( $TESTDIR/functions $TESTDIR/local $TESTDIR/cdm $TESTDIR/cdm/star ) +# $TESTDIR/cdm/ps/pinned -# GROUPLIST=( $TESTDIR/language-behaviour -# $TESTDIR/language/working \ -# $TESTDIR/local \ -# $TESTDIR/language/should-not-work \ -# $TESTDIR/cdm \ -# $TESTDIR/cdm/ps \ -# $TESTDIR/cdm/ps/pinned ) +GROUPLIST=( $TESTDIR/language-behaviour + $TESTDIR/language/working \ + $TESTDIR/local \ + $TESTDIR/language/should-not-work \ + $TESTDIR/cdm \ + $TESTDIR/cdm/ps \ + $TESTDIR/cdm/ps/pinned ) GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do From noreply at svn.ci.uchicago.edu Thu Oct 7 12:49:18 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 12:49:18 -0500 (CDT) Subject: [Swift-commit] r3663 - trunk/tests/language/should-not-work Message-ID: <20101007174918.2474C9CCAB@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 12:49:18 -0500 (Thu, 07 Oct 2010) New Revision: 3663 Modified: trunk/tests/language/should-not-work/1151-bad-comment.swift trunk/tests/language/should-not-work/116-keyword-misuse-in.swift trunk/tests/language/should-not-work/117-empty-program.swift trunk/tests/language/should-not-work/case-broken2.swift trunk/tests/language/should-not-work/closes.swift Log: Add THIS-SCRIPT-SHOULD-FAIL Modified: trunk/tests/language/should-not-work/1151-bad-comment.swift =================================================================== --- trunk/tests/language/should-not-work/1151-bad-comment.swift 2010-10-07 17:30:11 UTC (rev 3662) +++ trunk/tests/language/should-not-work/1151-bad-comment.swift 2010-10-07 17:49:18 UTC (rev 3663) @@ -1 +1,4 @@ + +// THIS-SCRIPT-SHOULD-FAIL + /namedOutputs = waveletTransf(waveletScript, 101, "FB"); Modified: trunk/tests/language/should-not-work/116-keyword-misuse-in.swift =================================================================== --- trunk/tests/language/should-not-work/116-keyword-misuse-in.swift 2010-10-07 17:30:11 UTC (rev 3662) +++ trunk/tests/language/should-not-work/116-keyword-misuse-in.swift 2010-10-07 17:49:18 UTC (rev 3663) @@ -1 +1,4 @@ + +// THIS-SCRIPT-SHOULD-FAIL + in a; Modified: trunk/tests/language/should-not-work/117-empty-program.swift =================================================================== --- trunk/tests/language/should-not-work/117-empty-program.swift 2010-10-07 17:30:11 UTC (rev 3662) +++ trunk/tests/language/should-not-work/117-empty-program.swift 2010-10-07 17:49:18 UTC (rev 3663) @@ -1 +1,4 @@ + +// THIS-SCRIPT-SHOULD-FAIL + ; Modified: trunk/tests/language/should-not-work/case-broken2.swift =================================================================== --- trunk/tests/language/should-not-work/case-broken2.swift 2010-10-07 17:30:11 UTC (rev 3662) +++ trunk/tests/language/should-not-work/case-broken2.swift 2010-10-07 17:49:18 UTC (rev 3663) @@ -1,4 +1,6 @@ +// THIS-SCRIPT-SHOULD-FAIL + switch(d) { case 1: Modified: trunk/tests/language/should-not-work/closes.swift =================================================================== --- trunk/tests/language/should-not-work/closes.swift 2010-10-07 17:30:11 UTC (rev 3662) +++ trunk/tests/language/should-not-work/closes.swift 2010-10-07 17:49:18 UTC (rev 3663) @@ -1,2 +1,5 @@ + +// THIS-SCRIPT-SHOULD-FAIL + }}}} From noreply at svn.ci.uchicago.edu Thu Oct 7 15:22:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 15:22:24 -0500 (CDT) Subject: [Swift-commit] r3664 - trunk/tests Message-ID: <20101007202224.1A2919CCAB@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 15:22:23 -0500 (Thu, 07 Oct 2010) New Revision: 3664 Modified: trunk/tests/nightly.sh Log: Better process mgmt Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-10-07 17:49:18 UTC (rev 3663) +++ trunk/tests/nightly.sh 2010-10-07 20:22:23 UTC (rev 3664) @@ -42,8 +42,6 @@ # stdout.txt retains stdout from the previous test (for *.clean.sh) # output_*.txt is the HTML-linked permanent output from a test -# WARNING: On timeout, this script will call killall on java and sleep - # All timeouts in this script are in seconds # PID TREE: @@ -544,9 +542,24 @@ EXEC_PID=$1 echo "process_exec_trap()" kill -KILL $EXEC_PID - # killall -9 java + killall_swift } +# Kill all subordinate swift/java processes +killall_swift() { + echo "killing all swifts..." + set -x + echo $$ + ps -f + kill_this $( ps -f | grep $$'.*'java | grep -v grep ) + set +x +} + +# Kill a process given its line output from "ps -f" +kill_this() { + [ -n $2 ] && /bin/kill -KILL $2 +} + # Execute as part of test set # Equivalent to monitored_exec() (but w/o monitoring) test_exec() { @@ -585,11 +598,15 @@ wait $SLEEP_PID [ $? != 0 ] && verbose "monitor($V) cancelled" && return 0 - echo "monitor($V): killing test process..." - /bin/kill -TERM $PID - KILLCODE=$? - if [ $KILLCODE == 0 ]; then - echo "monitor($V): killed process_exec (TERM)" + if ps | grep $PID + then + echo "monitor($V): killing test process..." + touch killed_test + /bin/kill -TERM $PID + KILLCODE=$? + if [ $KILLCODE == 0 ]; then + echo "monitor($V): killed process_exec (TERM)" + fi fi sleep 1 @@ -602,7 +619,8 @@ monitor_trap() { SLEEP_PID=$1 V=$2 - /bin/kill -9 $SLEEP_PID + verbose "monitor_trap(): kill sleep" + /bin/kill -KILL $SLEEP_PID } # Execute given command line in background with monitor @@ -629,8 +647,8 @@ STOP=$( date +%s ) - # If EXITCODE != 0, monitor() may have work to do - (( $EXITCODE != 0 )) && sleep 5 + # If the test was killed, monitor() may have work to do + rm killed_test && sleep 5 verbose "Killing monitor..." /bin/kill -TERM $MONITOR_PID @@ -877,9 +895,6 @@ SKIP_COUNTER=0 -# GROUPLIST=( $TESTDIR/functions $TESTDIR/local $TESTDIR/cdm $TESTDIR/cdm/star ) -# $TESTDIR/cdm/ps/pinned - GROUPLIST=( $TESTDIR/language-behaviour $TESTDIR/language/working \ $TESTDIR/local \ From noreply at svn.ci.uchicago.edu Thu Oct 7 15:38:28 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 15:38:28 -0500 (CDT) Subject: [Swift-commit] r3665 - trunk/tests Message-ID: <20101007203828.C278A9CCAB@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 15:38:28 -0500 (Thu, 07 Oct 2010) New Revision: 3665 Modified: trunk/tests/nightly.sh Log: Add -n option Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-10-07 20:22:23 UTC (rev 3664) +++ trunk/tests/nightly.sh 2010-10-07 20:38:28 UTC (rev 3665) @@ -11,7 +11,7 @@ # useful HTML output and tests.log # Run nightly.sh -h for quick help -# When something goes wrong, find and check tests.log +# When something goes wrong, find and check tests.log or use -v # Code is checked out into TOPDIR # Swift is installed in its source tree # The run is executed in RUNDIR (TOPDIR/RUNDIRBASE) @@ -37,6 +37,7 @@ # 4) optionally a swift.properties # 5) optionally a title.txt # 6) preferably a README.txt +# Edit GROUPLIST at the end of this script before running # OUTPUT is the stdout of the current test # stdout.txt retains stdout from the previous test (for *.clean.sh) @@ -69,6 +70,7 @@ printf "\t -g Do not run grid tests \n" printf "\t -h This message \n" printf "\t -k Skip first N tests \n" + printf "\t -n Run N tests and quit \n" printf "\t -p Do not build the package \n" printf "\t -s Do not do a fresh svn checkout \n" printf "\t -x Do not continue after a failure \n" @@ -106,6 +108,9 @@ -k) SKIP_TESTS=$2 shift 2;; + -n) + NUMBER_OF_TESTS=$2 + shift 2;; -p) BUILD_PACKAGE=0 shift;; @@ -833,6 +838,7 @@ start_row for ((i=0; $i<$ITERS_LOCAL; i=$i+1)); do swift_test_case $TESTNAME + (( $SWIFTCOUNT >= $NUMBER_OF_TESTS )) && return done end_row done @@ -895,14 +901,17 @@ SKIP_COUNTER=0 -GROUPLIST=( $TESTDIR/language-behaviour - $TESTDIR/language/working \ - $TESTDIR/local \ - $TESTDIR/language/should-not-work \ - $TESTDIR/cdm \ - $TESTDIR/cdm/ps \ - $TESTDIR/cdm/ps/pinned ) +GROUPLIST=( $TESTDIR/language/should-not-work $TESTDIR/functions $TESTDIR/local $TESTDIR/cdm $TESTDIR/cdm/star ) +# $TESTDIR/cdm/ps/pinned +# GROUPLIST=( $TESTDIR/language-behaviour \ +# $TESTDIR/language/working \ +# $TESTDIR/local \ +# $TESTDIR/language/should-not-work \ +# $TESTDIR/cdm \ +# $TESTDIR/cdm/ps \ +# $TESTDIR/cdm/ps/pinned ) + GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do export GROUP=$G @@ -910,6 +919,7 @@ start_part "Part $GROUPCOUNT: $TITLE" test_group (( GROUPCOUNT++ )) + (( $SWIFTCOUNT >= $NUMBER_OF_TESTS )) && break done if [ $GRID_TESTS == "0" ]; then From noreply at svn.ci.uchicago.edu Thu Oct 7 15:40:02 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 15:40:02 -0500 (CDT) Subject: [Swift-commit] r3666 - trunk/tests Message-ID: <20101007204002.043F49CCAB@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 15:40:01 -0500 (Thu, 07 Oct 2010) New Revision: 3666 Modified: trunk/tests/nightly.sh Log: Undo damage to GROUPLIST Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-10-07 20:38:28 UTC (rev 3665) +++ trunk/tests/nightly.sh 2010-10-07 20:40:01 UTC (rev 3666) @@ -901,17 +901,14 @@ SKIP_COUNTER=0 -GROUPLIST=( $TESTDIR/language/should-not-work $TESTDIR/functions $TESTDIR/local $TESTDIR/cdm $TESTDIR/cdm/star ) -# $TESTDIR/cdm/ps/pinned +GROUPLIST=( $TESTDIR/language-behaviour \ + $TESTDIR/language/working \ + $TESTDIR/local \ + $TESTDIR/language/should-not-work \ + $TESTDIR/cdm \ + $TESTDIR/cdm/ps \ + $TESTDIR/cdm/ps/pinned ) -# GROUPLIST=( $TESTDIR/language-behaviour \ -# $TESTDIR/language/working \ -# $TESTDIR/local \ -# $TESTDIR/language/should-not-work \ -# $TESTDIR/cdm \ -# $TESTDIR/cdm/ps \ -# $TESTDIR/cdm/ps/pinned ) - GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do export GROUP=$G From noreply at svn.ci.uchicago.edu Thu Oct 7 16:25:29 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 16:25:29 -0500 (CDT) Subject: [Swift-commit] r3667 - in trunk/src/org/globus/swift/data: . policy util Message-ID: <20101007212529.6C15F9CC9E@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 16:25:29 -0500 (Thu, 07 Oct 2010) New Revision: 3667 Modified: trunk/src/org/globus/swift/data/Director.java trunk/src/org/globus/swift/data/Query.java trunk/src/org/globus/swift/data/policy/Broadcast.java trunk/src/org/globus/swift/data/util/LineReader.java Log: Minor refactoring Modified: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java 2010-10-07 20:40:01 UTC (rev 3666) +++ trunk/src/org/globus/swift/data/Director.java 2010-10-07 21:25:29 UTC (rev 3667) @@ -7,12 +7,10 @@ import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -21,7 +19,6 @@ import org.globus.swift.data.policy.Policy; import org.globus.swift.data.policy.Broadcast; import org.globus.swift.data.util.LineReader; -import org.griphyn.vdl.karajan.Loader; /** * Manages CDM policies for files based on pattern matching. @@ -29,7 +26,8 @@ * */ public class Director { - private static final Logger logger = Logger.getLogger(Director.class); + private static final Logger logger = + Logger.getLogger(Director.class); /** Has a CDM policy file been provided? @@ -44,12 +42,14 @@ /** Maps from Patterns to Policies for fs.data rules */ - static Map map = new LinkedHashMap(); + static Map map = + new LinkedHashMap(); /** Maps from String names to String values for fs.data properties */ - static Map properties = new HashMap(); + static Map properties = + new HashMap(); /** Remember the files we have broadcasted. @@ -82,12 +82,9 @@ logger.info("CDM file: " + policyFile); enabled = true; Director.policyFile = policyFile; - LineReader lines = new LineReader(); - List list = lines.read(policyFile); - for (Iterator it = list.iterator(); it.hasNext(); ) { - String s = (String) it.next(); + List list = LineReader.read(policyFile); + for (String s : list) addLine(s); - } } /** @@ -168,7 +165,8 @@ /** Add a file to the list of files to be broadcasted. */ - public static synchronized void addBroadcast(String srcdir, String srcfile) { + public static synchronized void addBroadcast(String srcdir, + String srcfile) { logger.debug("addBroadcast(): " + srcdir + " " + srcfile); String path = srcdir+"/"+srcfile; broadcastWork.add(path); @@ -266,7 +264,6 @@ } try { - String name = args[0]; File policyFile = new File(args[1]); if (! policyFile.exists()) { Modified: trunk/src/org/globus/swift/data/Query.java =================================================================== --- trunk/src/org/globus/swift/data/Query.java 2010-10-07 20:40:01 UTC (rev 3666) +++ trunk/src/org/globus/swift/data/Query.java 2010-10-07 21:25:29 UTC (rev 3667) @@ -1,14 +1,9 @@ package org.globus.swift.data; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; - import org.apache.log4j.Logger; import org.globus.cog.karajan.arguments.Arg; import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.util.TypeUtil; import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; Modified: trunk/src/org/globus/swift/data/policy/Broadcast.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Broadcast.java 2010-10-07 20:40:01 UTC (rev 3666) +++ trunk/src/org/globus/swift/data/policy/Broadcast.java 2010-10-07 21:25:29 UTC (rev 3667) @@ -4,9 +4,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Map; -import java.util.Map.Entry; - import org.globus.swift.data.Director; public class Broadcast extends Policy { Modified: trunk/src/org/globus/swift/data/util/LineReader.java =================================================================== --- trunk/src/org/globus/swift/data/util/LineReader.java 2010-10-07 20:40:01 UTC (rev 3666) +++ trunk/src/org/globus/swift/data/util/LineReader.java 2010-10-07 21:25:29 UTC (rev 3667) @@ -10,27 +10,26 @@ import java.util.ArrayList; import java.util.List; -import org.globus.cog.karajan.arguments.Arg; - public class LineReader { public LineReader() {} - public static List read(File file) throws FileNotFoundException { + public static List read(File file) + throws FileNotFoundException { BufferedReader reader = new BufferedReader(new FileReader(file)); return read(reader); } - public static List read(String s) { + public static List read(String s) { BufferedReader reader = new BufferedReader(new StringReader(s)); return read(reader); } - public static List read(BufferedReader reader) { - List result = new ArrayList(); + public static List read(BufferedReader reader) { + List result = new ArrayList(); try { String prevline = ""; @@ -65,7 +64,7 @@ public static String[] tokenize(String line) { if (line == null) return null; - List words = new ArrayList(); + List words = new ArrayList(); String[] ws = line.split("\\s"); for (int i = 0; i < ws.length; i++) if (ws[i].length() > 0) From noreply at svn.ci.uchicago.edu Thu Oct 7 16:28:01 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 16:28:01 -0500 (CDT) Subject: [Swift-commit] r3668 - in trunk: libexec src/org/globus/swift/data src/org/globus/swift/data/policy tests/cdm Message-ID: <20101007212801.BEFAE9CC9E@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 16:28:01 -0500 (Thu, 07 Oct 2010) New Revision: 3668 Added: trunk/src/org/globus/swift/data/policy/External.java trunk/tests/cdm/210-external.check.sh trunk/tests/cdm/210-external.clean.sh trunk/tests/cdm/210-external.setup.sh trunk/tests/cdm/210-external.swift Modified: trunk/libexec/_swiftwrap trunk/libexec/vdl-int.k trunk/libexec/vdl-lib.xml trunk/src/org/globus/swift/data/Action.java trunk/src/org/globus/swift/data/policy/Policy.java trunk/tests/cdm/fs.template.data Log: New CDM EXTERNAL method and test Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-10-07 21:25:29 UTC (rev 3667) +++ trunk/libexec/_swiftwrap 2010-10-07 21:28:01 UTC (rev 3668) @@ -352,7 +352,8 @@ if [ ! -z $CDM_FILE ]; then CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L ) fi - if [ $CDM_POLICY != "DEFAULT" ]; then + if [ $CDM_POLICY != "DEFAULT" && + $CDM_POLICY != "EXTERNAL" ]; then log "CDM_POLICY: $L -> $CDM_POLICY" eval cdm_action $DIR "INPUT" $L $CDM_POLICY continue Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-10-07 21:25:29 UTC (rev 3667) +++ trunk/libexec/vdl-int.k 2010-10-07 21:28:01 UTC (rev 3668) @@ -23,15 +23,15 @@ ) dir:remove(dir, host=host) ) - + element(createdirs, [path, dir, host] dc := dircat(dir, path) log(LOG:INFO, "START path={path} dir={dir} - Creating directory structure") dir:make(dc, host=host) ) - - + + element(checkJobStatus, [rhost, wfdir, jobid, tr, jobdir] log(LOG:DEBUG, "START jobid={jobid}") try( @@ -40,9 +40,9 @@ * This is a bit of optimization, but I'm not completely * sure of its correctness. The goal is to both detect * the presence of the success file and remove it, all - * in one operation. It relies on file:remove() throwing - * an exception if the file is not there. - */ + * in one operation. It relies on file:remove() throwing + * an exception if the file is not there. + */ file:remove("{wfdir}/status/{jobdir}/{jobid}-success", host=rhost) log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") ) @@ -59,14 +59,14 @@ ) throw(error) ) - else ( + else ( log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing") - throw("No status file was found. Check the shared filesystem on {rhost}") + throw("No status file was found. Check the shared filesystem on {rhost}") ) ) ) ) - + element(initSharedDir, [rhost] once(list(rhost, "shared") vdl:setprogress("Initializing site shared directory") @@ -97,7 +97,7 @@ log(LOG:INFO, "END host={rhost} - Done initializing shared directory") ) ) - + element(initDDir, [] ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d" once(ddir @@ -107,11 +107,11 @@ ) ddir ) - + element(inFiles, [stageins] pathnames(stageins) ) - + element(fileDirs, [stageins, stageouts] list( unique( @@ -120,7 +120,7 @@ ) ) ) - + element(createDirSet, [jobid, destdir, host, dirs] /* * Ideally this would be done by creating a tree of the directories @@ -134,23 +134,23 @@ ) log(LOG:INFO, "END jobid={jobid} - Done initializing directory structure") ) - + element(cleanup, [dir, host] log(LOG:INFO, "START dir={dir} host={host}") cdmfile := cdm:file() log(LOG:INFO, "cdmfile {cdmfile}") if(cdmfile != "" then( log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}") - task:transfer(srcfile="cdm_cleanup.sh", + task:transfer(srcfile="cdm_cleanup.sh", srcdir="{swift.home}/libexec", desthost=host, destdir=dir) - task:transfer(srcfile="cdm_lib.sh", + task:transfer(srcfile="cdm_lib.sh", srcdir="{swift.home}/libexec", desthost=host, destdir=dir) log(LOG:INFO, "execute: cdm_cleanup.sh") task:execute( - executable="/bin/bash", - arguments=list("{dir}/cdm_cleanup.sh", + executable="/bin/bash", + arguments=list("{dir}/cdm_cleanup.sh", cdm:get("GATHER_DIR"), cdm:get("GATHER_TARGET") sys:uid() ) host=host, batch=true, tcprofile(host)) @@ -158,7 +158,7 @@ ) if(vdl:configProperty("sitedir.keep") == "false" task:execute( - vdl:siteprofile(host, "swift:cleanupCommand"), + vdl:siteprofile(host, "swift:cleanupCommand"), arguments=list( vdl:siteprofile(host, "swift:cleanupCommandOptions"), dir @@ -167,7 +167,7 @@ ) log(LOG:INFO, "END dir={dir} host={host}") ) - + element(cleanups, [cleanup] log(LOG:INFO, "START cleanups={cleanup}") parallelFor(i, cleanup @@ -182,7 +182,7 @@ ) log(LOG:INFO, "END cleanups={cleanup}") ) - + element(cleanupFiles, [files, host] uParallelFor(r, files log(LOG:INFO, "Purging ", r, " on ", host) @@ -190,8 +190,8 @@ vdl:cacheFileRemoved(r, host) ) ) - + element(stageWrapperParams, [jobid, jobdir, wrapfile, dir, host] log(LOG:INFO, "START jobid={jobid} - staging in wrapper params") provider := provider(wrapfile) @@ -216,14 +216,14 @@ libexec := "{swift.home}/libexec" if (cdmfile != "" then( - doStageinFile(provider="file", srchost="localhost", srcfile=basename(cdmfile), - srcdir=vdl:dirname(cdmfile), desthost=host, destdir=dir, + doStageinFile(provider="file", srchost="localhost", srcfile=basename(cdmfile), + srcdir=vdl:dirname(cdmfile), desthost=host, destdir=dir, size=file:size(cdmfile), policy="DEFAULT") - doStageinFile(provider="file", srchost="localhost", srcfile="cdm.pl", - srcdir=libexec, desthost=host, destdir=dir, + doStageinFile(provider="file", srchost="localhost", srcfile="cdm.pl", + srcdir=libexec, desthost=host, destdir=dir, size=file:size("{libexec}/cdm.pl}"), policy="DEFAULT") - doStageinFile(provider="file", srchost="localhost", srcfile="cdm_lib.sh", - srcdir=libexec, desthost=host, destdir=dir, + doStageinFile(provider="file", srchost="localhost", srcfile="cdm_lib.sh", + srcdir=libexec, desthost=host, destdir=dir, size=file:size("{libexec}/cdm_lib.sh}"), policy="DEFAULT") )) @@ -238,9 +238,9 @@ policy := cdm:query(query=file) log(LOG:DEBUG, "CDM: {file} : {policy}") - doStageinFile(provider=provider, srchost=srchost, srcfile=filename, + doStageinFile(provider=provider, srchost=srchost, srcfile=filename, srcdir=srcdir, desthost=host, destdir=destdir, size=size, policy=policy) - ) + ) log(LOG:INFO, "END jobid={jobid} - Staging in finished") ) @@ -250,17 +250,21 @@ log(LOG:DEBUG, "FILE_STAGE_IN_START file={srcfile} ", "srchost={srchost} srcdir={srcdir} srcname={srcfile} ", - "desthost={desthost} destdir={destdir} provider={provider} ", + "desthost={desthost} destdir={destdir} provider={provider} ", "policy={policy}") - if (policy == "DEFAULT" then( + if (policy == "DEFAULT" then( restartOnError(".*", 2 - task:transfer(srcprovider=provider, srchost=srchost, srcfile=srcfile, - srcdir=srcdir, desthost=desthost, destdir=destdir))) + task:transfer(srcprovider=provider, srchost=srchost, srcfile=srcfile, + srcdir=srcdir, desthost=desthost, destdir=destdir))) policy == "BROADCAST" then( log(LOG:DEBUG, "FILE_STAGE_IN_BROADCAST file={srcfile} policy={policy}") cdm:broadcast(srcfile=srcfile, srcdir=srcdir)) + policy == "EXTERNAL" then( + log(LOG:DEBUG, "FILE_STAGE_IN_EXTERNAL file={srcfile} policy={policy}") + cdm:external(srcfile=srcfile, srcdir=srcdir, + desthost=desthost, destdir=destdir)) else(log(LOG:DEBUG, "FILE_STAGE_IN_SKIP file={srcfile} policy={policy}"))) - log(LOG:DEBUG, "FILE_STAGE_IN_END file={srcfile} ", + log(LOG:DEBUG, "FILE_STAGE_IN_END file={srcfile} ", "srchost={srchost} srcdir={srcdir} srcname={srcfile} ", "desthost={desthost} destdir={destdir} provider={provider}") ) @@ -269,6 +273,7 @@ element(doStageout, [jobid, stageouts, dir, host] log(LOG:INFO, "START jobid={jobid} - Staging out files") + log(LOG:DEBUG, "stageouts: {stageouts}") done := list( uParallelFor(pv, stageouts [path, var] := each(pv) @@ -280,7 +285,7 @@ ldir := vdl:dirname(file) fullLocal := dircat(ldir, bname) fullRemote := dircat(rdir, bname) - + log(LOG:DEBUG, "FILE_STAGE_OUT_START srcname={bname} srcdir={rdir} srchost={host} ", "destdir={ldir} desthost={dhost} provider={provider}") //make sure we do have the directory on the client side @@ -289,13 +294,13 @@ if (sys:or(policy == "DEFAULT", policy == "BROADCAST") then( restartOnError(".*", 2 - task:transfer(srchost=host, srcfile=bname,srcdir=rdir, + task:transfer(srchost=host, srcfile=bname,srcdir=rdir, destdir=ldir, desthost=dhost, destprovider=provider))) else(log(LOG:DEBUG, "FILE_STAGE_OUT_SKIP srcname={bname}")) ) log(LOG:DEBUG, "FILE_STAGE_OUT_END srcname={bname} srcdir={rdir} srchost={host} ", "destdir={ldir} desthost={dhost} provider={provider}") - + list(bname, rdir, host, file:size(fullLocal)) ) ) @@ -307,13 +312,13 @@ ) log(LOG:INFO, "END jobid={jobid} - Staging out finished") ) - + element(graphStuff, [tr, stagein, stageout, err, optional(args)] if( vdl:configProperty("pgraph") != "false" then( errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1") tp := vdl:threadPrefix() - to(graph, + to(graph, concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") ) for(si, stagein @@ -329,7 +334,7 @@ file := basename(file) label := vdl:niceName(var, path = path) to(graph - concat(str:quote(file), " [shape=parallelogram,label=", + concat(str:quote(file), " [shape=parallelogram,label=", str:quote(label), "]") concat(str:quote(tp), " -> ", str:quote(file)) ) @@ -337,16 +342,16 @@ ) ) ) - + element(fileSizes, [files] math:sum( for(f, files, file:size(file)) ) ) - + element(transferStandardFiles, [rhost, tmpdir, jobid, stdout, stderr] concat( - for(f, list(list("stderr.txt", stderr), list("stdout.txt", stdout)) + for(f, list(list("stderr.txt", stderr), list("stdout.txt", stdout)) [name, file] := each(f) destfile := "{jobid}-{file}" nl() @@ -364,7 +369,7 @@ ) ) ) - + element(transferKickstartRec, [rhost, wfdir, jobid, jobdir] recfile := "{jobid}-kickstart.xml" srcdir := dircat("{wfdir}/kickstart/", jobdir) @@ -397,17 +402,17 @@ stagein := list(unique(each(stagein))) stageout := list(unique(each(stageout))) allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein) - + ddir := initDDir() [wfdir, sharedDir] := try( initSharedDir(rhost) throw(exception("Could not initialize shared directory on {rhost}", exception)) ) - + uid := uid() jobdir := substring(uid, from=0, to=1) jobid := "{tr}-{uid}" - + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") statusMode := configProperty("status.mode",host=rhost) @@ -419,7 +424,7 @@ stderr := try(stderr, "stderr.txt") kickstart := vdl:kickstart(rhost) fileDirs := fileDirs(stagein, stageout) - os := vdl:siteprofile(rhost, "SYSINFO:OS") + os := vdl:siteprofile(rhost, "SYSINFO:OS") if(wrapperMode == "files" sequential( @@ -443,12 +448,12 @@ vdl:setprogress("Stage in") tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) - - - + + + try( sequential( - + createDirSet(jobid, sharedDir, rhost, fileDirs) doStagein(jobid, stagein, sharedDir, rhost) if(wrapperMode == "files" @@ -457,7 +462,7 @@ log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") - + vdl:setprogress("Submitting") if(wrapperMode == "files" @@ -465,7 +470,7 @@ vdl:siteprofile(rhost, "swift:wrapperInterpreter"), list( vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"), - dircat("shared", vdl:siteprofile(rhost, "swift:wrapperScript"), os=os), + dircat("shared", vdl:siteprofile(rhost, "swift:wrapperScript"), os=os), jobid, "-p", jobdir ) directory=wfdir @@ -486,12 +491,12 @@ jobid, "-jobdir", jobdir, "-scratch", try(vdl:siteprofile(rhost, "scratch"), "") - "-e", vdl:executable(tr, rhost), - "-out", stdout, - "-err", stderr, + "-e", vdl:executable(tr, rhost), + "-out", stdout, + "-err", stderr, "-i", maybe(stdin), "-d", flatten(each(fileDirs)), - "-if", flatten(infiles(stagein)), + "-if", flatten(infiles(stagein)), "-of", flatten(outfiles(stageout)), "-k", kickstart, "-cdmfile", cdm:file(), @@ -519,11 +524,11 @@ log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") - + /* need to stage the files to upper scratch area in case they are not transfered to another site before all the files get cleaned out */ - - + + vdl:setprogress("Stage out") doStageout(jobid, stageout, sharedDir, rhost) doRestartlog(restartout) @@ -555,9 +560,9 @@ vdl:cacheUnlockFiles(stagein, sharedDir, rhost, force=false cleanupFiles(cacheFilesToRemove, rhost) ) - + outs := transferStandardFiles(rhost, tmpdir, jobid, stdout, stderr) - + discard(maybe(transferWrapperLog(rhost, wfdir, jobid, jobdir))) kickstartRec := if( @@ -572,7 +577,7 @@ ) ) ) - + throw( exception( concat( @@ -591,7 +596,7 @@ ) ) ) - + element(generateProvenanceGraph, [gdata] pgraph := vdl:configProperty("pgraph") gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph) @@ -599,7 +604,7 @@ "digraph SwiftProvenance {{", nl() " graph [", vdl:configProperty("pgraph.graph.options"), "];", nl() " node [", vdl:configProperty("pgraph.node.options"), "];", nl() - + for(i, gdata " ", i, nl() ) Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-10-07 21:25:29 UTC (rev 3667) +++ trunk/libexec/vdl-lib.xml 2010-10-07 21:28:01 UTC (rev 3668) @@ -126,6 +126,7 @@ + Modified: trunk/src/org/globus/swift/data/Action.java =================================================================== --- trunk/src/org/globus/swift/data/Action.java 2010-10-07 21:25:29 UTC (rev 3667) +++ trunk/src/org/globus/swift/data/Action.java 2010-10-07 21:28:01 UTC (rev 3668) @@ -6,22 +6,36 @@ import org.globus.cog.karajan.arguments.Arg; import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; import org.globus.swift.data.policy.Broadcast; +import org.globus.swift.data.policy.External; import org.globus.swift.data.policy.Policy; /** * Karajan-accessible CDM functions that change something. * */ public class Action extends FunctionsCollection { - private static final Logger logger = Logger.getLogger(Action.class); + private static final Logger logger = + Logger.getLogger(Action.class); - public static final Arg PA_FILE = new Arg.Positional("srcfile"); - public static final Arg PA_DIR = new Arg.Positional("srcdir"); + public static final Arg PA_SRCFILE = + new Arg.Positional("srcfile"); + public static final Arg PA_SRCDIR = + new Arg.Positional("srcdir"); + public static final Arg PA_DESTHOST = + new Arg.Positional("desthost"); + public static final Arg PA_DESTDIR = + new Arg.Positional("destdir"); static { - setArguments("cdm_broadcast", new Arg[]{ PA_FILE, PA_DIR }); + setArguments("cdm_broadcast", new Arg[]{ PA_SRCFILE, + PA_SRCDIR }); + setArguments("cdm_external", new Arg[]{ PA_SRCFILE, + PA_SRCDIR, + PA_DESTHOST, + PA_DESTDIR }); setArguments("cdm_wait", new Arg[]{}); } @@ -29,9 +43,10 @@ Register a file for broadcast by CDM. The actual broadcast is triggered by {@link cdm_wait}. */ - public void cdm_broadcast(VariableStack stack) throws ExecutionException { - String srcfile = (String) PA_FILE.getValue(stack); - String srcdir = (String) PA_DIR.getValue(stack); + public void cdm_broadcast(VariableStack stack) + throws ExecutionException { + String srcfile = (String) PA_SRCFILE.getValue(stack); + String srcdir = (String) PA_SRCDIR.getValue(stack); logger.debug("cdm_broadcast()"); @@ -39,8 +54,8 @@ if (!(policy instanceof Broadcast)) { throw new RuntimeException - ("Attempting to BROADCAST the wrong file: directory: `" + - srcdir + "' `" + srcfile + "' -> " + policy); + ("Attempting to BROADCAST the wrong file: " + + srcdir + " " + srcfile + " -> " + policy); } if (srcdir == "") { @@ -50,10 +65,28 @@ Director.addBroadcast(srcdir, srcfile); } + public void cdm_external(VariableStack stack) + throws ExecutionException + { + String srcfile = (String) PA_SRCFILE.getValue(stack); + String srcdir = (String) PA_SRCDIR.getValue(stack); + BoundContact bc = (BoundContact) PA_DESTHOST.getValue(stack); + String destdir = (String) PA_DESTDIR.getValue(stack); + + if (srcdir.length() == 0) + srcdir = "."; + String desthost = bc.getHost(); + String workdir = (String) bc.getProperty("workdir"); + + External.doExternal(srcfile, srcdir, + desthost, workdir+"/"+destdir); + } + /** Wait until CDM has ensured that all data has been propagated. */ - public void cdm_wait(VariableStack stack) throws ExecutionException { + public void cdm_wait(VariableStack stack) + throws ExecutionException { logger.debug("cdm_wait()"); Director.doBroadcast(); } Added: trunk/src/org/globus/swift/data/policy/External.java =================================================================== --- trunk/src/org/globus/swift/data/policy/External.java (rev 0) +++ trunk/src/org/globus/swift/data/policy/External.java 2010-10-07 21:28:01 UTC (rev 3668) @@ -0,0 +1,68 @@ +package org.globus.swift.data.policy; + +import java.util.List; + +import org.apache.log4j.Logger; +import org.globus.swift.data.Director; + +public class External +extends Policy { + + private static final Logger logger = + Logger.getLogger(External.class); + + /** + The name of the external program + */ + String progname = null; + + public static void doExternal(String srcfile, String srcdir, + String desthost, String destdir) { + logger.debug("doExternal: " + + "srcfile: " + srcfile + + "srcdir: " + srcdir + + "desthost: " + desthost + + "destdir: " + destdir); + + Policy policy = Director.lookup(srcdir + "/" + srcfile); + + if (!(policy instanceof External)) + throw new RuntimeException + ("doExternal called on non-EXTERNAL file!"); + + External external = (External) policy; + external.callProgram(srcdir, srcfile, desthost, destdir); + } + + void callProgram(String srcdir, String srcfile, + String desthost, String destdir) + { + String[] cmdline = new String[] { progname, srcdir, srcfile, + desthost, destdir }; + Process process = null; + try { + process = Runtime.getRuntime().exec(cmdline); + process.waitFor(); + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException + ("Could not launch external broadcast"); + } + } + + @Override + public void settings(List tokens) { + try { + progname = tokens.get(0); + } + catch (Exception e) { + throw new RuntimeException + ("Incorrect settings for EXTERNAL"); + } + } + + public String toString() { + return "EXTERNAL"; + } +} Modified: trunk/src/org/globus/swift/data/policy/Policy.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Policy.java 2010-10-07 21:25:29 UTC (rev 3667) +++ trunk/src/org/globus/swift/data/policy/Policy.java 2010-10-07 21:28:01 UTC (rev 3668) @@ -23,6 +23,8 @@ return new Broadcast(); else if (token.compareToIgnoreCase("gather") == 0) return new Gather(); + else if (token.compareToIgnoreCase("external") == 0) + return new External(); return null; } } Added: trunk/tests/cdm/210-external.check.sh =================================================================== --- trunk/tests/cdm/210-external.check.sh (rev 0) +++ trunk/tests/cdm/210-external.check.sh 2010-10-07 21:28:01 UTC (rev 3668) @@ -0,0 +1,11 @@ +#!/bin/sh + +set -x + +cat external.out || exit 1 +rm -v external.out || exit 1 + +grep $( uname -m ) 210-input.txt || exit 1 +grep $( uname -m ) 210-output.txt || exit 1 + +exit 0 Property changes on: trunk/tests/cdm/210-external.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/210-external.clean.sh =================================================================== --- trunk/tests/cdm/210-external.clean.sh (rev 0) +++ trunk/tests/cdm/210-external.clean.sh 2010-10-07 21:28:01 UTC (rev 3668) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -v 210-input.txt 210-output.txt + +exit 0 Property changes on: trunk/tests/cdm/210-external.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/210-external.setup.sh =================================================================== --- trunk/tests/cdm/210-external.setup.sh (rev 0) +++ trunk/tests/cdm/210-external.setup.sh 2010-10-07 21:28:01 UTC (rev 3668) @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +{ + uname -a + date +} > 210-input.txt + +cp -v $GROUP/external.sh . + +exit 0 Property changes on: trunk/tests/cdm/210-external.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/210-external.swift =================================================================== --- trunk/tests/cdm/210-external.swift (rev 0) +++ trunk/tests/cdm/210-external.swift 2010-10-07 21:28:01 UTC (rev 3668) @@ -0,0 +1,11 @@ + +type file; + +app (file o) copy (file i) +{ + cp @i @o; +} + +file f1<"210-input.txt">; +file f2<"210-output.txt">; +f2 = copy(f1); Modified: trunk/tests/cdm/fs.template.data =================================================================== --- trunk/tests/cdm/fs.template.data 2010-10-07 21:25:29 UTC (rev 3667) +++ trunk/tests/cdm/fs.template.data 2010-10-07 21:28:01 UTC (rev 3668) @@ -1 +1,2 @@ -rule .*output.txt DIRECT _PWD_/201 +rule .*201-output.txt DIRECT _PWD_/201 +rule .*210-input.txt EXTERNAL _PWD_/external.sh From noreply at svn.ci.uchicago.edu Thu Oct 7 16:28:39 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 Oct 2010 16:28:39 -0500 (CDT) Subject: [Swift-commit] r3669 - trunk/tests/language/should-not-work Message-ID: <20101007212839.B98409CC9E@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-07 16:28:39 -0500 (Thu, 07 Oct 2010) New Revision: 3669 Added: trunk/tests/language/should-not-work/201-hang.swift Log: Test that causes Swift to hang Added: trunk/tests/language/should-not-work/201-hang.swift =================================================================== --- trunk/tests/language/should-not-work/201-hang.swift (rev 0) +++ trunk/tests/language/should-not-work/201-hang.swift 2010-10-07 21:28:39 UTC (rev 3669) @@ -0,0 +1,8 @@ + +// THIS-SCRIPT-SHOULD-FAIL + +int i; +int j; + +j = i; + From noreply at svn.ci.uchicago.edu Fri Oct 8 11:58:26 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 8 Oct 2010 11:58:26 -0500 (CDT) Subject: [Swift-commit] r3670 - trunk/libexec Message-ID: <20101008165826.84A559CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-08 11:58:26 -0500 (Fri, 08 Oct 2010) New Revision: 3670 Modified: trunk/libexec/_swiftwrap Log: Bug fix to CDM handling in _swiftwrap Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-10-07 21:28:39 UTC (rev 3669) +++ trunk/libexec/_swiftwrap 2010-10-08 16:58:26 UTC (rev 3670) @@ -349,13 +349,12 @@ logstate "LINK_INPUTS" for L in $INF ; do CDM_POLICY="DEFAULT" - if [ ! -z $CDM_FILE ]; then + if [[ $CDM_FILE != "" ]]; then CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L ) fi - if [ $CDM_POLICY != "DEFAULT" && - $CDM_POLICY != "EXTERNAL" ]; then + if [[ $CDM_POLICY != "DEFAULT" && $CDM_POLICY != "EXTERNAL"* ]]; then log "CDM_POLICY: $L -> $CDM_POLICY" - eval cdm_action $DIR "INPUT" $L $CDM_POLICY + eval cdm_action $DIR "INPUT" $L $CDM_POLICY continue fi if [ $COPYNOTLINK = 1 ]; then From noreply at svn.ci.uchicago.edu Fri Oct 8 11:59:09 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 8 Oct 2010 11:59:09 -0500 (CDT) Subject: [Swift-commit] r3671 - trunk/tests/cdm Message-ID: <20101008165909.8BF069CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-08 11:59:09 -0500 (Fri, 08 Oct 2010) New Revision: 3671 Added: trunk/tests/cdm/external.sh Log: Should have been added with tests/cdm/210 Added: trunk/tests/cdm/external.sh =================================================================== --- trunk/tests/cdm/external.sh (rev 0) +++ trunk/tests/cdm/external.sh 2010-10-08 16:59:09 UTC (rev 3671) @@ -0,0 +1,15 @@ +#!/bin/bash + +set -x + +{ + SRCDIR=$1 + SRCFILE=$2 + DESTHOST=$3 + DESTDIR=$4 + + cp -v $SRCDIR/$SRCFILE $DESTDIR || exit 1 + +} > external.out 2>&1 + +exit 0 Property changes on: trunk/tests/cdm/external.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Fri Oct 8 12:00:17 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 8 Oct 2010 12:00:17 -0500 (CDT) Subject: [Swift-commit] r3672 - in trunk/src/org: globus/swift/catalog globus/swift/catalog/util griphyn/vdl/karajan/lib Message-ID: <20101008170017.D98F39CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-08 12:00:17 -0500 (Fri, 08 Oct 2010) New Revision: 3672 Modified: trunk/src/org/globus/swift/catalog/TransformationCatalogEntry.java trunk/src/org/globus/swift/catalog/util/Profile.java trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java Log: Minor changes: cleanups and generics Modified: trunk/src/org/globus/swift/catalog/TransformationCatalogEntry.java =================================================================== --- trunk/src/org/globus/swift/catalog/TransformationCatalogEntry.java 2010-10-08 16:59:09 UTC (rev 3671) +++ trunk/src/org/globus/swift/catalog/TransformationCatalogEntry.java 2010-10-08 17:00:17 UTC (rev 3672) @@ -68,7 +68,7 @@ /** * The profiles associated with the transformation; */ - private List profiles; + private List profiles; /** * The System Info for the transformation. @@ -121,7 +121,7 @@ public TransformationCatalogEntry( String namespace, String name, String version, String resourceid, String physicalname, TCType type, - List profiles, + List profiles, SysInfo sysinfo ) { this.namespace = namespace; this.version = version; @@ -162,13 +162,10 @@ "\n SysInfo : " + ((this.sysinfo == null) ? "" : this.sysinfo.toString()) + "\n TYPE : " + ((this.type == null) ? "" : type.toString()); if(profiles != null){ - for (Iterator i = profiles.listIterator(); i.hasNext(); ) { - st = st + - "\n Profile : " + ( (Profile) i.next()).toString(); - } + for (Profile p : profiles) + st = st + "\n Profile : " + p.toString(); } return st; - } /** @@ -201,10 +198,8 @@ + " sysinfo=\"" + this.getSysInfo() + "\""; if ( this.profiles != null ) { xml += " >\n"; - for ( Iterator iter = this.profiles.iterator(); iter.hasNext(); ) { - Profile profile = ( Profile ) iter.next(); - xml += "\t\t\t" + profile.toXML() + "\n"; - } + for (Profile p : profiles) + xml += "\t\t\t" + p.toXML() + "\n"; xml += "\t\t\n"; } else { xml += " />\n"; @@ -301,7 +296,7 @@ public void setProfile( Profile profile ) { if ( profile != null ) { if ( this.profiles == null ) { - this.profiles = new ArrayList( 5 ); + this.profiles = new ArrayList( 5 ); } this.profiles.add( profile ); } @@ -311,10 +306,11 @@ * Allows you to add multiple profiles to the transformation. * @param profiles List of Profile objects containing the profile information. */ - public void setProfiles( List profiles ) { + public void setProfiles( List profiles ) { if ( profiles != null ) { if ( this.profiles == null ) { - this.profiles = new ArrayList( profiles.size() ); + this.profiles = + new ArrayList( profiles.size() ); } this.profiles.addAll( profiles ); } @@ -388,7 +384,7 @@ * Returns the list of profiles associated with the transformation. * @return List Returns null if no profiles associated. */ - public List getProfiles() { + public List getProfiles() { return this.profiles; } @@ -397,14 +393,13 @@ * @param namespace String The namespace of the profile * @return List List of Profile objects. returns null if none are found. */ - public List getProfiles( String namespace ) { - List results = null; + public List getProfiles( String namespace ) { + List results = null; if ( profiles != null ) { - for ( Iterator i = profiles.iterator(); i.hasNext(); ) { - Profile p = ( Profile ) i.next(); + for ( Profile p : profiles ) { if ( p.getProfileNamespace().equalsIgnoreCase( namespace ) ) { if ( results == null ) { - results = new ArrayList(); + results = new ArrayList(); } results.add( p ); } Modified: trunk/src/org/globus/swift/catalog/util/Profile.java =================================================================== --- trunk/src/org/globus/swift/catalog/util/Profile.java 2010-10-08 16:59:09 UTC (rev 3671) +++ trunk/src/org/globus/swift/catalog/util/Profile.java 2010-10-08 17:00:17 UTC (rev 3672) @@ -122,8 +122,8 @@ * Returns the Profile (namespace, value and key); * @return ArrayList */ - public List getProfile() { - ArrayList m_profile = new ArrayList( 3 ); + public List getProfile() { + List m_profile = new ArrayList( 3 ); m_profile.add( mNamespace ); m_profile.add( mKey ); m_profile.add( mValue ); Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2010-10-08 16:59:09 UTC (rev 3671) +++ trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2010-10-08 17:00:17 UTC (rev 3672) @@ -5,7 +5,6 @@ import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -21,7 +20,6 @@ import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.nodes.grid.GridExec; import org.globus.swift.catalog.TransformationCatalogEntry; -import org.globus.swift.catalog.types.Os; import org.globus.swift.catalog.util.Profile; import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.util.FQN; @@ -36,10 +34,10 @@ setArguments(TCProfile.class, new Arg[] { PA_HOST, OA_TR }); } - private static Map PROFILE_T; + private static Map PROFILE_T; static { - PROFILE_T = new HashMap(); + PROFILE_T = new HashMap(); PROFILE_T.put("count", GridExec.A_COUNT); PROFILE_T.put("jobtype", GridExec.A_JOBTYPE); PROFILE_T.put("maxcputime", GridExec.A_MAXCPUTIME); @@ -60,7 +58,7 @@ BoundContact bc = (BoundContact) PA_HOST.getValue(stack); NamedArguments named = ArgUtil.getNamedReturn(stack); - Map attrs = null; + Map attrs = null; attrs = attributesFromHost(bc, attrs, named); @@ -69,7 +67,7 @@ tce = getTCE(tc, new FQN(tr), bc); } - Map env = new HashMap(); + Map env = new HashMap(); if (tce != null) { addEnvironment(env, tce); addEnvironment(env, bc); @@ -104,7 +102,8 @@ } } - private static final Set warnedAboutWalltime = new HashSet(); + private static final Set warnedAboutWalltime = + new HashSet(); private void warn(String tr, String message) { synchronized (warnedAboutWalltime) { @@ -114,24 +113,23 @@ } } - private void addEnvironment(Map m, TransformationCatalogEntry tce) { - List l = tce.getProfiles(Profile.ENV); - if (l != null) { - Iterator i = l.iterator(); - while (i.hasNext()) { - Profile p = (Profile) i.next(); + private void addEnvironment(Map m, + TransformationCatalogEntry tce) { + List list = tce.getProfiles(Profile.ENV); + if (list != null) { + for (Profile p : list) { m.put(p.getProfileKey(), p.getProfileValue()); } } } - public static final String PROFILE_GLOBUS_PREFIX = (Profile.GLOBUS + "::").toLowerCase(); + public static final String PROFILE_GLOBUS_PREFIX = + (Profile.GLOBUS + "::").toLowerCase(); - private void addEnvironment(Map m, BoundContact bc) { - Map props = bc.getProperties(); - Iterator i = props.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); + private void addEnvironment(Map m, + BoundContact bc) { + Map props = bc.getProperties(); + for (Map.Entry e : props.entrySet()) { String name = (String) e.getKey(); FQN fqn = new FQN(name); String value = (String) e.getValue(); @@ -141,7 +139,8 @@ } } - private void addAttributes(NamedArguments named, Map attrs) { + private void addAttributes(NamedArguments named, + Map attrs) { if (logger.isDebugEnabled()) { logger.debug("Attributes: " + attrs); } @@ -151,16 +150,17 @@ named.add(GridExec.A_ATTRIBUTES, attrs); } - private Map attributesFromTC(TransformationCatalogEntry tce, Map attrs, NamedArguments named) { - List l = tce.getProfiles(Profile.GLOBUS); - if (l != null) { - Iterator i = l.iterator(); - while (i.hasNext()) { - Profile p = (Profile) i.next(); - Arg a = (Arg) PROFILE_T.get(p.getProfileKey()); + private Map + attributesFromTC(TransformationCatalogEntry tce, + Map attrs, + NamedArguments named) { + List list = tce.getProfiles(Profile.GLOBUS); + if (list != null) { + for (Profile p : list) { + Arg a = PROFILE_T.get(p.getProfileKey()); if (a == null) { if (attrs == null) { - attrs = new HashMap(); + attrs = new HashMap(); } attrs.put(p.getProfileKey(), p.getProfileValue()); } @@ -172,18 +172,19 @@ return attrs; } - private Map attributesFromHost(BoundContact bc, Map attrs, NamedArguments named) { - Map props = bc.getProperties(); + private Map + attributesFromHost(BoundContact bc, + Map attrs, + NamedArguments named) { + Map props = bc.getProperties(); if (props != null) { - Iterator i = props.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); + for (Map.Entry e : props.entrySet()) { FQN fqn = new FQN((String) e.getKey()); if (Profile.GLOBUS.equalsIgnoreCase(fqn.getNamespace())) { Arg a = (Arg) PROFILE_T.get(fqn.getName()); if (a == null) { if (attrs == null) { - attrs = new HashMap(); + attrs = new HashMap(); } attrs.put(fqn.getName(), e.getValue()); } From noreply at svn.ci.uchicago.edu Fri Oct 8 14:15:02 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 8 Oct 2010 14:15:02 -0500 (CDT) Subject: [Swift-commit] r3673 - trunk/tests Message-ID: <20101008191502.61E449CCAB@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-08 14:15:02 -0500 (Fri, 08 Oct 2010) New Revision: 3673 Modified: trunk/tests/nightly.sh Log: Minor changes + suppressing cdm/ps/pinned for now Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-10-08 17:00:17 UTC (rev 3672) +++ trunk/tests/nightly.sh 2010-10-08 19:15:02 UTC (rev 3673) @@ -2,14 +2,10 @@ # USAGE NOTES: -# Run this script from a working directory in which you -# are willing to check out the whole Swift source and -# generate many small test files. +# The script will (optionally) checkout Swift, run several tests in a +# subdirectory called run-DATE, and generate useful HTML output and +# tests.log -# The script will checkout Swift, run several tests -# in a subdirectory called run-DATE, and generate -# useful HTML output and tests.log - # Run nightly.sh -h for quick help # When something goes wrong, find and check tests.log or use -v # Code is checked out into TOPDIR @@ -83,6 +79,7 @@ RUN_ANT=1 CLEAN=1 SKIP_TESTS=0 +NUMBER_OF_TESTS=1000000 # Run all tests by default BUILD_PACKAGE=1 GRID_TESTS=1 SKIP_CHECKOUT=0 @@ -653,7 +650,7 @@ STOP=$( date +%s ) # If the test was killed, monitor() may have work to do - rm killed_test && sleep 5 + rm killed_test > /dev/null 2>&1 && sleep 5 verbose "Killing monitor..." /bin/kill -TERM $MONITOR_PID @@ -907,8 +904,10 @@ $TESTDIR/language/should-not-work \ $TESTDIR/cdm \ $TESTDIR/cdm/ps \ - $TESTDIR/cdm/ps/pinned ) + $TESTDIR/cdm/star ) +# $TESTDIR/cdm/ps/pinned \ # Currently broken + GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do export GROUP=$G From noreply at svn.ci.uchicago.edu Fri Oct 8 14:35:09 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 8 Oct 2010 14:35:09 -0500 (CDT) Subject: [Swift-commit] r3674 - in SwiftApps/SwiftR: . Swift/man Message-ID: <20101008193509.EE76B9CCAB@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-08 14:35:09 -0500 (Fri, 08 Oct 2010) New Revision: 3674 Removed: SwiftApps/SwiftR/UserGuide Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd SwiftApps/SwiftR/TODO Log: Move prelim User Gude text to Swift R package doc file Swift/man/Swift-package.Rd. Some updates to TODO list. Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-10-08 19:15:02 UTC (rev 3673) +++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-10-08 19:35:09 UTC (rev 3674) @@ -29,11 +29,111 @@ options(swift.site=sitename) # sitename = "local" to run on the current host and "pbs" to submit to a local PBS cluster. +PREREQS + +Sun Java (pref 1.6 - will it work below?) +How to get Java if needed + +R v2.11 or higher in your PATH (on client and server machines) + +Ability to ssh to server machines (without password: agents, master control channel, etc) +? Needed? Passwords or ssh key passphrases OK for some scenarios. + +ssh from Mac +ssh -A when jumping to a new host (to forward the ssh agent) + +or set up ssh agents manually + +(document ssh tricks here for pw-less access) + + +CAVEATS + +# fixed: Only one Swift server running per user; can only be used by one R client workspace at a time. + +when fifos get hung, need to use kill or Quit to break out of R; will fix. + +no auto-restart yet if swift dies in server loop. + +only lapply is implemented (also SwiftApply) - need to see if we can cut down arg passing overhead for many of the apply() cases + +log records build up fast; these will be reduced as we get more confidence withthe code and shake out bugs + +initVar only affects first calls on a server - if you change these you need to start a new server (FIXME)! + + + +INSTALL + +cd ??? +wget http://www.ci.uchicago.edu/~wilde/Swift_0.1.tar.gz +R CMS INSTALL Swift_0.1.tar.gz + +SVN + +svn checkout https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SwiftR + +CONFIGURE SERVERS + +edit configure-site-NAME in exec/ + +can put local cores into an ssh pool + + +START SERVERS + +# do this outside of R + +SWIFT=/Swift/ +$SWIFT/exec/start-swift-workers hostname +$SWIFT/exec/start-swift-server + +local and ssh servers can be started and left running, across R runs +found via: + +HELLO WORLD TEST + +RUN FULL TEST + +source("Swift/tests/TestSwift.R") + + or R CMD TEST etc? + +USAGE + +Swift returns Error object when remote side fails. + +options: + swift.server: matched server name on start-swift + swift.callsperbatch + initialize: + +less likely to touch: + remove temp reqs (sp???) FIXME + mode (service, manual, ???) + +OPENMX EXAMPLES + +DEBUGGING + +* manual mode + +* logs to look at + +* reporting bugs: what to send (FIXME: need swiftsnapshot script) + +* setting Swift worker logging with $HOME/.globus/coasters/loglevel file. This file should contain a single text integer: 0=most detailed, 4=least detaild, 5=off. This is an interim log control mechanism and may be deprecated in the future. + } + \author{ -Michael Wilde +Swift R package developed by Michael Wilde + +Swift was developed by: Mihael Hategan, Ben Clifford, Justin Wozniak, Yong Zhao, Ian Foster, and Michael Wilde with contributions from Sarah Kenny, Ioan Raicu, Luiz Gadelha, Allan Espinosa, Zhao Zhang, David Kelly, Jon Monette, Glen Hocky, Tom Uram, Wenjun Wu, and other users. + Maintainer: Michael Wilde + } \references{ http://www.ci/uchicago.edu/swift Modified: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO 2010-10-08 19:15:02 UTC (rev 3673) +++ SwiftApps/SwiftR/TODO 2010-10-08 19:35:09 UTC (rev 3674) @@ -1,5 +1,8 @@ *** TO DO LIST: +BUG: Passivate failing on swift pbs config on pads. +^^^^^^^^^^^^^^^^^^ + MAIN - still seeing sleep 1 mutex? @@ -16,6 +19,7 @@ - make start-swift restartable within a single R swiftapply() +- only lapply is implemented (also SwiftApply) - need to see if we can cut down arg passing overhead for many of the apply() cases x complete change for envvars like SWIFTR_TMP - add sourcing of $HOME/.SwiftR.init: pick up variables for the configure_ scripts from here; maybe one file in this dir for each site supported? maybe configure scripts go here? Deleted: SwiftApps/SwiftR/UserGuide =================================================================== --- SwiftApps/SwiftR/UserGuide 2010-10-08 19:15:02 UTC (rev 3673) +++ SwiftApps/SwiftR/UserGuide 2010-10-08 19:35:09 UTC (rev 3674) @@ -1,99 +0,0 @@ -Prelim Swift R User Guide -------------------------- - - -PREREQS - -Sun Java (pref 1.6 - will it work below?) -How to get Java if needed - -R v2.11 or higher in your PATH (on client and server machines) - -Ability to ssh to server machines (without password: agents, master control channel, etc) -? Needed? Passwords or ssh key passphrases OK for some scenarios. - -ssh from Mac -ssh -A when jumping to a new host (to forward the ssh agent) - -or set up ssh agents manually - -(document ssh tricks here for pw-less access) - - -CAVEATS - -# fixed: Only one Swift server running per user; can only be used by one R client workspace at a time. - -when fifos get hung, need to use kill or Quit to break out of R; will fix. - -no auto-restart yet if swift dies in server loop. - - -only lapply is implemented (also SwiftApply) - need to see if we can cut down arg passing overhead for many of the apply() cases - -log records build up fast; these will be reduced as we get more confidence withthe code and shake out bugs - -initVar only affects first calls on a server - if you change these you need to start a new server (FIXME)! - -INSTALL - -cd ??? -wget http://www.ci.uchicago.edu/~wilde/Swift_0.1.tar.gz -R CMS INSTALL Swift_0.1.tar.gz - -SVN - -svn checkout https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SwiftR - -CONFIGURE SERVERS - -edit configure-site-NAME in exec/ - -can put local cores into an ssh pool - - -START SERVERS - -# do this outside of R - -SWIFT=/Swift/ -$SWIFT/exec/start-swift-workers hostname -$SWIFT/exec/start-swift-server - -local and ssh servers can be started and left running, across R runs -found via: - -HELLO WORLD TEST - -RUN FULL TEST - -source("Swift/tests/TestSwift.R") - - or R CMD TEST etc? - -USAGE - -Swift returns Error object when remote side fails. - -options: - swift.server: matched server name on start-swift - swift.callsperbatch - initialize: - -less likely to touch: - remove temp reqs (sp???) FIXME - mode (service, manual, ???) - -OPENMX EXAMPLES - -DEBUGGING - -* manual mode - -* logs to look at - -* reporting bugs: what to send - (FIXME: need swiftsnapshot script) - - - From noreply at svn.ci.uchicago.edu Sat Oct 9 09:27:21 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 9 Oct 2010 09:27:21 -0500 (CDT) Subject: [Swift-commit] r3675 - in SwiftApps/SwiftR: . Swift/R Swift/exec Swift/man Swift/tests Message-ID: <20101009142721.7011A9CC90@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-09 09:27:20 -0500 (Sat, 09 Oct 2010) New Revision: 3675 Modified: SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh SwiftApps/SwiftR/Swift/exec/rserver.swift SwiftApps/SwiftR/Swift/man/Swift-package.Rd SwiftApps/SwiftR/Swift/tests/TestSwift.R SwiftApps/SwiftR/TODO Log: Enhanced user-specified initializtion script to run in any server and whenever script is changed. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-08 19:35:09 UTC (rev 3674) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-09 14:27:20 UTC (rev 3675) @@ -83,9 +83,6 @@ dir.create(reqdir,showWarnings=FALSE) cat("Swift request is in",reqdir,"\n") - cat(initialize,file=paste(reqdir,"/initialize.R",sep="")) # FIXME: honor this on calls after the first in an R server? - # FIXME: eliminate this oevrhead on each call to swiftapply - narglists <- length(arglists) # number of arglists to process batch <- 1 # Next arglist batch number to fill arglist <- 1 # Next arglist number to insert @@ -102,7 +99,7 @@ arglistbatch[[i]] <- arglists[[arglist]] arglist <- arglist +1 } - rcall <- list(func=func,arglistbatch=arglistbatch) + rcall <- list(initializer=initialize,func=func,arglistbatch=arglistbatch) save(rcall,file=paste(reqdir,"/cbatch.",as.character(batch),".Rdata",sep="")) batch <- batch + 1; } @@ -128,8 +125,6 @@ swiftServerDir = paste(tmpdir,"/",user,"/SwiftR/swift.",swiftserver,sep="") -cat(initialize,file=paste(swiftServerDir,"/initialize.R",sep="")) # FIXME - requestPipeName=paste(swiftServerDir,"/requestpipe",sep="") resultPipeName=paste(swiftServerDir,"/resultpipe",sep="") @@ -185,3 +180,416 @@ #swiftapply: no visible binding for '<<-' assignment to 'swiftprops' #swiftapply: no visible binding for global variable 'swiftprops' #swiftapply: no visible binding for global variable 'result' + + +#------------------ Tests moved here from test dir: + + +initSwiftTestOptions <- function() +{ + options(swift.site="service") + options(swift.keepwork=TRUE) + initcmds <- "initVar1 <<- 19; initVar2 <<- sqrt(400)+3" + options(swift.initialize=initcmds) # Set here; used in test group 4 +} + +swiftTest_1.1 <- function() +{ + + initSwiftTestOptions() + + cat("\n*** Starting test 1.1 ***\n\n") + + sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } + data(cars) + data(trees) + + args=list(trees,cars) + arglist = rep(list(args),1) + + cat("Test of local do.call(sumstuff)\n") + localres = do.call(sumstuff,args) + cat("local result=\n") + print(localres) + + cat("\nTest of swiftapply(sumstuff,arglist)\n") + swiftres = swiftapply(sumstuff,arglist) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres[[1]])) { + cat("\n==> test 1.1 passed\n") + } else { + cat("\n==> test 1.1 FAILED !!!!!\n") + } +} + +basicSwiftTest <- function() { swiftTest_1.1() } + +# .... more tests from below to move here + +swiftTest_4.1 <- function() +{ + sumivars <- function() { initVar1+initVar2 } + + args=list() + arglist = rep(list(args),1) + + localres = 42 + + cat("\nTest of swiftapply(sumivars,arglist)\n") + swiftres = swiftapply(sumivars,arglist) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres[[1]])) { + cat("\n==> test 4.1 passed\n") + } else { + cat("\n==> test 4.1 FAILED !!!!!\n") + } +} + +swiftTest_4.2 <- function() +{ + + options(swift.initialize="initVar3 <<- 123; initVar4 <<- 100"); + + mulivars <- function() { initVar3*initVar4 } + + args=list() + arglist = rep(list(args),1) + + localres = 12300; + + cat("\nTest of swiftapply(mulivars,arglist)\n") + swiftres = swiftapply(mulivars,arglist) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres[[1]])) { + cat("\n==> test 4.2 passed\n") + } else { + cat("\n==> test 4.2 FAILED !!!!!\n") + } +} + +runAllSwiftTests <- function() +{ + +### FIXME: Save prior options here: restore them when tests are done. Recovery if interrrupted? + +failures=0 + +startTime = proc.time()[["elapsed"]] + +cat("\n*** Starting test group 1 - functions on simple data structures ***\n\n") + +swiftTest_1.1() + +##### Test 1.2 + +# test 10 remote calls + +sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } +data(cars) +data(trees) + +args=list(trees,cars) +arglist <- rep(list(args),10) + +localres = do.call(sumstuff,args) + +cat("\n*** Test 1.2.1: 10 calls to substuff()\n") +swiftres <- swiftapply(sumstuff,arglist) +cat("Swift result:\n") +format(swiftres) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format( swiftres[[i]] ))) + } +} + +if(diffs == 0) { + cat("\n==> test 1.2.1 passed\n") +} else { + cat("\n!!!==> test 1.2.1 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +cat("\n*** Test 1.2.2: 10 calls to substuff() - callsperbatch=10\n") +swiftres = swiftapply(sumstuff,arglist,callsperbatch=10) +cat("Swift result:\n") +format(swiftres) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format( swiftres[[i]] ))) + } +} + +if(diffs == 0) { + cat("\n==> test 1.2.2 passed\n") +} else { + cat("\n!!!==> test 1.2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +cat("\n*** Test 1.2.3: 10 calls to substuff() - callsperbatch=2\n") +swiftres = swiftapply(sumstuff,arglist,callsperbatch=2) +cat("Swift result:\n") +format(swiftres) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i, format( swiftres[[i]] ))) + } +} + +if(diffs == 0) { + cat("\n==> test 1.2.3 passed\n") +} else { + cat("\n!!!==> test 1.2.3 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +cat("\n*** Test 1.2.4: 10 calls to substuff() - callsperbatch=3\n") +swiftres = swiftapply(sumstuff,arglist,callsperbatch=3) +swiftres <- swiftapply(sumstuff,arglist) +cat("Swift result:\n") +format(swiftres) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i, format( swiftres[[i]] ))) + } +} + +if(diffs == 0) { + cat("\n==> test 1.2.4 passed\n") +} else { + cat("\n!!!==> test 1.2.4 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +# swiftres = swiftapply(sumstuff,arglist,callsperbatch=2,site="pbs") +# test variations on local vs ssh vs pbs; coasters vs non; etc. + + + +##### Test Group 2 + +cat("\n*** Starting test group 2 - test matrix passing***\n") + +matfunc <- function( m1, m2 ) +{ + (1/m1) %*% m2 +} + +n <- 5 +m1 <- array(sin(1:n**2), dim=c(n,n)) +m2 <- t(m1) + +localres = matfunc(m1,m2) + +cat("\n*** Test 2.1: 100 calls to matfunc(dim=5x5) - callsperbatch=9\n") + +args=list(m1,m2) +arglist <- rep(list(args),100) + +swiftres = swiftapply(matfunc,arglist,callsperbatch=9) + +diffs <- 0 +#for(i in 1:length(swiftres) ) { +for(i in c(seq(1,100,10),100)) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 2.1 passed\n") +} else { + cat("\n!!!==> test 2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +n <- 237 +n <- 50 +m1 <- array(sin(1:n**2), dim=c(n,n)) +m2 <- t(m1) + +localres = matfunc(m1,m2) + +cat("\n*** Test 2.2: 123 calls to matfunc(dim=bigger) - callsperbatch=7\n") # FIXME make n easy to adjust and print actual value + +args=list(m1,m2) +arglist <- rep(list(args),123) + +swiftres = swiftapply(matfunc,arglist,callsperbatch=7) + +diffs <- 0 +#for(i in 1:length(swiftres) ) { +for(i in c(seq(1,length(swiftres),10),length(swiftres))) { + + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 2.2 passed\n") +} else { + cat("\n!!!==> test 2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + + +##### Test Group 3 + +cat("\n*** Starting test group 3 - test list element and name passing***\n") + +# Test if list element names are being sent and returned correctly + +n <- 5 +m1 <- array(sin(1:n**2), dim=c(n,n)) +m2 <- t(m1) + +inlist = list() +inlist[[1]]=123 +inlist[[2]]=456 +inlist$name1=789 +inlist$name2=987 +inlist$mat1 = m1 +inlist[[99]] = m2 + +listfunc <- function(ilist) +{ + olist = ilist + olist$sum = ilist[[1]] + ilist[[2]] + ilist$name1 + ilist$name2 + olist$names = names(ilist) + olist$mprod = ilist$mat1 %*% ilist[[99]] + return(olist) +} +localres = listfunc(inlist) + +cat("\n*** Starting test 3.1 - 4 calls in one batch of 5 ***\n") + +args=list(inlist) +arglist <- rep(list(args),4) + +swiftres = swiftapply(listfunc,arglist,callsperbatch=5) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d=%s\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 3.1 passed\n") +} else { + cat("\n!!!==> test 3.1 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +cat("\n*** Starting test 3.2 - 99 calls in batches of 11 ***\n") + +args=list(inlist) +arglist <- rep(list(args),99) + +swiftres = swiftapply(listfunc,arglist,callsperbatch=11) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 3.2 passed\n") +} else { + cat("\n!!!==> test 3.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +##### Test Group 4 # test initializer string + +cat("\n*** Starting test group 4 - test remote R service initialization string ***\n") + +swiftTest_4.1() +swiftTest_4.2() + + + + +##### Test Group 5 # test error handling + +cat("\n*** Starting test group 5 - test remote R service error ***\n") + +arglist = list(list(1.0),list(2.0),list("3.0"),list(4.0),list(5.0)) + +cat("\nTest of swiftapply(sumivars,arglist)\n") +swiftres = swiftapply(log,arglist) +cat("Swift result:\n") +print(swiftres) + +goodres = c("numeric","numeric","try-error","numeric","numeric") + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( class(swiftres[[i]]) != goodres[i] ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 5.1 passed\n") +} else { + cat("\n!!!==> test 5.1 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +endTime <- proc.time()[["elapsed"]] +runTime <- endTime - startTime + +cat("\n\n ===> Total elapsed test time = ",runTime," seconds.\n\n") + +} # end function runAllTests + +#options(swift.site="local") +#runAllTests() + +testloop <- function(npass) +{ + for(i in 1:npass) { + cat("\n\n\n ***** Starting test pass ", i, " ***** \n\n\n"); + runAllSwiftTests() + cat("\n\n\n ***** Completed test pass ", i, " ***** \n\n\n"); + system("sleep 3") + } +} Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-08 19:35:09 UTC (rev 3674) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-09 14:27:20 UTC (rev 3675) @@ -1,9 +1,10 @@ #! /bin/bash # Arguments: inputBatchSaveFile outputBatchSaveFile -# bash @shellscript @initRScript @RServerScript @rcall @result stdout=@stout stderr=@sterr; +# bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; -if [ $# != 4 ]; then +#if [ $# != 4 ]; then +if [ $# != 3 ]; then echo $0: expecting 4 arguments, got $#: $* 1>&2 exit 1 fi @@ -12,10 +13,9 @@ # tmp=/scratch/local # FIXME: allow this to change eg for sites with main tmp dir elsewhere tmp=${SWIFTR_TMP:-/tmp} -initRScript=$1 -RServerScript=$2 -callFile=$3 -resultFile=$4 +RServerScript=$1 +callFile=$2 +resultFile=$3 # Find our bin dir (to use for running utility scripts) @@ -86,7 +86,7 @@ mkfifo $SLOTDIR/toR.fifo mkfifo $SLOTDIR/fromR.fifo chmod +x $RServerScript - PATH=.:$PATH $RServerScript $SLOTDIR $initRScript >& $SLOTDIR/R.log & # launch R server + PATH=.:$PATH $RServerScript $SLOTDIR >& $SLOTDIR/R.log & # launch R server # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" else Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh 2010-10-08 19:35:09 UTC (rev 3674) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh 2010-10-09 14:27:20 UTC (rev 3675) @@ -1,17 +1,12 @@ #! /usr/bin/env Rscript -# require(OpenMx) # FIXME: make this a variable set of statements to exec or source - argv = commandArgs(TRUE) fifoDir = argv[1]; # FIXME: test for valid arguments -initScript = argv[2]; cat(Sys.getpid(),file=paste(fifoDir,"/R.pid",sep="")) -source(initScript) -cat("After sourcing initscript: ", initScript, "\n") -ls() +latestInitializer <- ""; SwiftRFifoServer <- function( fifoBasename ) { @@ -53,7 +48,13 @@ { load(callBatchFileName); - result=list() + result <- list() + initializer <- rcall$initializer; + if( initializer != latestInitializer) { + initialExpr <- parse(text=initializer) + eval(initialExpr) + latestInitializer <<- initializer + } for(c in 1:length(rcall$arglistbatch)) { # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) result[[c]] = try(do.call( rcall$func, rcall$arglistbatch[[c]] )) Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/rserver.swift 2010-10-08 19:35:09 UTC (rev 3674) +++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2010-10-09 14:27:20 UTC (rev 3675) @@ -4,10 +4,13 @@ # fixme: extend to enable coaster and non-coaster execution # needs to select between EvalRPersistent and simple RunR # enable persistent for local mode (not sure how) +# condense stdout/err to 1 file +# replace ack with ftracef to result pipe +# condense shellscript and rserverscript to one? -app (external e, RData result, file stout, file sterr) runR (file shellscript, file initRScript, file RServerScript, RData rcall) +app (external e, RData result, file stout, file sterr) runR (file shellscript, file RServerScript, RData rcall) { - bash @shellscript @initRScript @RServerScript @rcall @result stdout=@stout stderr=@sterr; + bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; } app ack (external e[]) @@ -29,11 +32,10 @@ file sterr[] ; file runRscript <"EvalRBatchPersistent.sh">; - file initScript <"initialize.R">; file rsScript <"SwiftRServer.sh">; foreach c, i in rcalls { - (e[i], results[i],stout[i], sterr[i]) = runR(runRscript,initScript,rsScript,c); + (e[i], results[i],stout[i], sterr[i]) = runR(runRscript,rsScript,c); } } Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-10-08 19:35:09 UTC (rev 3674) +++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-10-09 14:27:20 UTC (rev 3675) @@ -8,9 +8,13 @@ R interface to Swift parallel scripting language } \description{ -Description: Routines to invoke R functions and Swift scripts on remote resources through Swift. -R functions can be remotely executed in parallel in a manner similar to Snow using a list of argument lists. -Eventually more general Swift functions can be embedded and invked remotely as well. + +Description: Routines to invoke R functions and Swift scripts on +remote resources through Swift. R functions can be remotely executed +in parallel in a manner similar to Snow using a list of argument +lists. Eventually more general Swift functions can be embedded and +invked remotely as well. + } \details{ \tabular{ll}{ @@ -18,51 +22,44 @@ Type: \tab Package\cr Version: \tab 1.0\cr Date: \tab 2010-02-25\cr -License: \tab Globus Toolkit Public License v3 (based on Apache License 2.0): http://www.globus.org/toolkit/legal/4.0/license-v3.html \cr +License: \tab Globus Toolkit Public License v3 (based on Apache License 2.0): +http://www.globus.org/toolkit/legal/4.0/license-v3.html \cr LazyLoad: \tab yes\cr } -To use this package, create a list of argument lists, and then invoke: swiftapply(function,arglists). -As a preliminary interface, you can set R options() to control Swift's operation: +To use this package, create a list of argument lists, and then invoke: +swiftapply(function,arglists). -options(swift.callsperbatch=n) # n = number of R calls to perform in each Swift job. +As a preliminary interface, you can set R options() to control Swift's +operation: -options(swift.site=sitename) # sitename = "local" to run on the current host and "pbs" to submit to a local PBS cluster. +options(swift.callsperbatch=n) # n = number of R calls to perform in +each Swift job. + +options(swift.site=sitename) # sitename = "local" to run on the +current host and "pbs" to submit to a local PBS cluster. -PREREQS +PREREQUISITES Sun Java (pref 1.6 - will it work below?) -How to get Java if needed +FIXME: How to install Java if needed + R v2.11 or higher in your PATH (on client and server machines) -Ability to ssh to server machines (without password: agents, master control channel, etc) -? Needed? Passwords or ssh key passphrases OK for some scenarios. +Ability to ssh to server machines (without password: agents, master +control channel, etc) (FIXME: Are these limitations necessary?) +Passwords or ssh key passphrases OK for some scenarios. ssh from Mac + ssh -A when jumping to a new host (to forward the ssh agent) -or set up ssh agents manually +(or set up ssh agents manually) (document ssh tricks here for pw-less access) -CAVEATS - -# fixed: Only one Swift server running per user; can only be used by one R client workspace at a time. - -when fifos get hung, need to use kill or Quit to break out of R; will fix. - -no auto-restart yet if swift dies in server loop. - -only lapply is implemented (also SwiftApply) - need to see if we can cut down arg passing overhead for many of the apply() cases - -log records build up fast; these will be reduced as we get more confidence withthe code and shake out bugs - -initVar only affects first calls on a server - if you change these you need to start a new server (FIXME)! - - - INSTALL cd ??? @@ -79,22 +76,47 @@ can put local cores into an ssh pool +swift $HOME/.ssh/auth-defaults file for additional shs servers +access remote systems via ssh + +Export SWIFTR_TMP in your environment + START SERVERS -# do this outside of R +# do this outside of R - BEFORE trying to run R Swift functions SWIFT=/Swift/ $SWIFT/exec/start-swift-workers hostname $SWIFT/exec/start-swift-server local and ssh servers can be started and left running, across R runs + found via: HELLO WORLD TEST +# Start swift local server as above + +require(Swift) +basicSwiftTest() + RUN FULL TEST +As a regular user: + +require(Swift) +fullSwiftTest() + +# Then + +n=10 # 1o times through full test loop + +swiftTestLoop(n) + + +In source tree: + source("Swift/tests/TestSwift.R") or R CMD TEST etc? @@ -114,23 +136,70 @@ OPENMX EXAMPLES -DEBUGGING +This section is specific to users of the OpenMX R package for +structural equation modeling. +DIRECTORY STRUCTURE USED FOR SWIFT RUNTIME + +PROCEESS STRUCTURE USE FOR SWIFT RUNTIME + +DEBUGGING AND TROUBLESHOOTING + * manual mode * logs to look at +* is my swift server responding? + +tail -f $TMP/ + * reporting bugs: what to send (FIXME: need swiftsnapshot script) -* setting Swift worker logging with $HOME/.globus/coasters/loglevel file. This file should contain a single text integer: 0=most detailed, 4=least detaild, 5=off. This is an interim log control mechanism and may be deprecated in the future. +* setting Swift worker logging with $HOME/.globus/coasters/loglevel +file. This file should contain a single text integer: 0=most detailed, +4=least detaild, 5=off. This is an interim log control mechanism and +may be deprecated in the future. +CAVEATS + +You MUST start the Swift server before running a swiftapply() call +from R. Otherwise R hangs and must be killed and restarted. + +When Swift fifos (named pipes) get hung, you need to use kill or Quit +to break out of R. FIXME + +There is no automatic restart yet if swift dies in its server +loop. FIXME + +Variables set in the initialze script must typically be set in global +environment ( var <<- value); + +Only lapply is implemented (also SwiftApply) - need to see if we can +cut down arg passing overhead for many of the apply() cases + +Log records build up fast; these will be reduced as we get more +confidence with the code and shake out bugs + +There is no easy way yet to alter Swift configuration file variables +such as number of cores to use, etc. Do this for now by editing an +existing configuration under Swift/exec/conigure-swift-NNN where NNN +is the Swift server name. + +Each swiftapply() is pretty noisy - it echos its options etc. This +will quiet down. + + + } \author{ Swift R package developed by Michael Wilde -Swift was developed by: Mihael Hategan, Ben Clifford, Justin Wozniak, Yong Zhao, Ian Foster, and Michael Wilde with contributions from Sarah Kenny, Ioan Raicu, Luiz Gadelha, Allan Espinosa, Zhao Zhang, David Kelly, Jon Monette, Glen Hocky, Tom Uram, Wenjun Wu, and other users. +Swift was developed by: Mihael Hategan, Ben Clifford, Justin Wozniak, +Yong Zhao, Ian Foster, and Michael Wilde with contributions from Sarah +Kenny, Ioan Raicu, Luiz Gadelha, Allan Espinosa, Zhao Zhang, David +Kelly, Jon Monette, Glen Hocky, Tom Uram, Wenjun Wu, and other users. Maintainer: Michael Wilde Modified: SwiftApps/SwiftR/Swift/tests/TestSwift.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-10-08 19:35:09 UTC (rev 3674) +++ SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-10-09 14:27:20 UTC (rev 3675) @@ -1,17 +1,6 @@ ##### Setup -require(Swift) - -initcmds <- " -initVar1 <- 19 -initVar2 <- sqrt(400)+3 -" - -failures=0 - -options(swift.initialize=initcmds) # Set here; used in test group 4 - # To paste for quick testing: ########### if (FALSE) { @@ -39,366 +28,12 @@ } #################################### -# NOTE: swift.initialize statements must be set before any swiftapply() calls when running in "service" mode. +require(Swift) -runAllTests <- function() -{ +runAllSwiftTests() -startTime = proc.time()[["elapsed"]] -cat("\n*** Starting test group 1 - functions on simple data structures ***\n\n") -##### Test 1.1 - -cat("\n*** Starting test 1.1 ***\n\n") - -sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } -data(cars) -data(trees) - -args=list(trees,cars) -arglist = rep(list(args),1) - -cat("Test of local do.call(sumstuff)\n") -localres = do.call(sumstuff,args) -cat("local result=\n") -print(localres) - -cat("\nTest of swiftapply(sumstuff,arglist)\n") -swiftres = swiftapply(sumstuff,arglist) -cat("Swift result:\n") -print(swiftres) - -if(identical(localres,swiftres[[1]])) { - cat("\n==> test 1.1 passed\n") -} else { - cat("\n==> test 1.1 FAILED !!!!!\n") -} - - -##### Test 1.2 - -# test 10 remote calls - -sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } -data(cars) -data(trees) - -args=list(trees,cars) -arglist <- rep(list(args),10) - -cat("\n*** Test 1.2.1: 10 calls to substuff()\n") -swiftres <- swiftapply(sumstuff,arglist) -cat("Swift result:\n") -format(swiftres) - -diffs <- 0 -for(i in 1:length(swiftres) ) { - if( !identical(swiftres[[i]],localres) ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format( swiftres[[i]] ))) - } -} - -if(diffs == 0) { - cat("\n==> test 1.2.1 passed\n") -} else { - cat("\n!!!==> test 1.2.1 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - -cat("\n*** Test 1.2.2: 10 calls to substuff() - callsperbatch=10\n") -swiftres = swiftapply(sumstuff,arglist,callsperbatch=10) -cat("Swift result:\n") -format(swiftres) - -diffs <- 0 -for(i in 1:length(swiftres) ) { - if( !identical(swiftres[[i]],localres) ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format( swiftres[[i]] ))) - } -} - -if(diffs == 0) { - cat("\n==> test 1.2.2 passed\n") -} else { - cat("\n!!!==> test 1.2.2 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - -cat("\n*** Test 1.2.3: 10 calls to substuff() - callsperbatch=2\n") -swiftres = swiftapply(sumstuff,arglist,callsperbatch=2) -cat("Swift result:\n") -format(swiftres) - -diffs <- 0 -for(i in 1:length(swiftres) ) { - if( !identical(swiftres[[i]],localres) ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i, format( swiftres[[i]] ))) - } -} - -if(diffs == 0) { - cat("\n==> test 1.2.3 passed\n") -} else { - cat("\n!!!==> test 1.2.3 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - -cat("\n*** Test 1.2.4: 10 calls to substuff() - callsperbatch=3\n") -swiftres = swiftapply(sumstuff,arglist,callsperbatch=3) -swiftres <- swiftapply(sumstuff,arglist) -cat("Swift result:\n") -format(swiftres) - -diffs <- 0 -for(i in 1:length(swiftres) ) { - if( !identical(swiftres[[i]],localres) ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i, format( swiftres[[i]] ))) - } -} - -if(diffs == 0) { - cat("\n==> test 1.2.4 passed\n") -} else { - cat("\n!!!==> test 1.2.4 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - -# swiftres = swiftapply(sumstuff,arglist,callsperbatch=2,site="pbs") -# test variations on local vs ssh vs pbs; coasters vs non; etc. - - - -##### Test Group 2 - -cat("\n*** Starting test group 2 - test matrix passing***\n") - -matfunc <- function( m1, m2 ) -{ - (1/m1) %*% m2 -} - -n <- 5 -m1 <- array(sin(1:n**2), dim=c(n,n)) -m2 <- t(m1) - -localres = matfunc(m1,m2) - -cat("\n*** Test 2.1: 100 calls to matfunc(dim=5x5) - callsperbatch=9\n") - -args=list(m1,m2) -arglist <- rep(list(args),100) - -swiftres = swiftapply(matfunc,arglist,callsperbatch=9) - -diffs <- 0 -#for(i in 1:length(swiftres) ) { -for(i in c(seq(1,100,10),100)) { - if( !all.equal(swiftres[[i]],localres) ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) - } -} - -if(diffs == 0) { - cat("\n==> test 2.1 passed\n") -} else { - cat("\n!!!==> test 2.2 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - -n <- 237 -n <- 50 -m1 <- array(sin(1:n**2), dim=c(n,n)) -m2 <- t(m1) - -localres = matfunc(m1,m2) - -cat("\n*** Test 2.2: 123 calls to matfunc(dim=bigger) - callsperbatch=7\n") # FIXME make n easy to adjust and print actual value - -args=list(m1,m2) -arglist <- rep(list(args),123) - -swiftres = swiftapply(matfunc,arglist,callsperbatch=7) - -diffs <- 0 -#for(i in 1:length(swiftres) ) { -for(i in c(seq(1,length(swiftres),10),length(swiftres))) { - - if( !all.equal(swiftres[[i]],localres) ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) - } -} - -if(diffs == 0) { - cat("\n==> test 2.2 passed\n") -} else { - cat("\n!!!==> test 2.2 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - - -##### Test Group 3 - -cat("\n*** Starting test group 3 - test list element and name passing***\n") - -# Test if list element names are being sent and returned correctly - -n <- 5 -m1 <- array(sin(1:n**2), dim=c(n,n)) -m2 <- t(m1) - -inlist = list() -inlist[[1]]=123 -inlist[[2]]=456 -inlist$name1=789 -inlist$name2=987 -inlist$mat1 = m1 -inlist[[99]] = m2 - -listfunc <- function(ilist) -{ - olist = ilist - olist$sum = ilist[[1]] + ilist[[2]] + ilist$name1 + ilist$name2 - olist$names = names(ilist) - olist$mprod = ilist$mat1 %*% ilist[[99]] - return(olist) -} -localres = listfunc(inlist) - -cat("\n*** Starting test 3.1 - 4 calls in one batch of 5 ***\n") - -args=list(inlist) -arglist <- rep(list(args),4) - -swiftres = swiftapply(listfunc,arglist,callsperbatch=5) - -diffs <- 0 -for(i in 1:length(swiftres) ) { - if( !all.equal(swiftres[[i]],localres) ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d=%s\n",i,format(swiftres[[i]]))) - } -} - -if(diffs == 0) { - cat("\n==> test 3.1 passed\n") -} else { - cat("\n!!!==> test 3.1 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - -cat("\n*** Starting test 3.2 - 99 calls in batches of 11 ***\n") - -args=list(inlist) -arglist <- rep(list(args),99) - -swiftres = swiftapply(listfunc,arglist,callsperbatch=11) - -diffs <- 0 -for(i in 1:length(swiftres) ) { - if( !all.equal(swiftres[[i]],localres) ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) - } -} - -if(diffs == 0) { - cat("\n==> test 3.2 passed\n") -} else { - cat("\n!!!==> test 3.2 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - -##### Test Group 4 # test initializer string - -cat("\n*** Starting test group 4 - test remote R service initialization string ***\n") - -sumivars <- function() { initVar1+initVar2 } - -args=list() -arglist = rep(list(args),1) - -localres = 42 - -cat("\nTest of swiftapply(sumivars,arglist)\n") -swiftres = swiftapply(sumivars,arglist) -cat("Swift result:\n") -print(swiftres) - -if(identical(localres,swiftres[[1]])) { - cat("\n==> test 4.1 passed\n") -} else { - cat("\n==> test 4.1 FAILED !!!!!\n") -} - -##### Test Group 5 # test error handling - -cat("\n*** Starting test group 5 - test remote R service error ***\n") - -arglist = list(list(1.0),list(2.0),list("3.0"),list(4.0),list(5.0)) - -cat("\nTest of swiftapply(sumivars,arglist)\n") -swiftres = swiftapply(log,arglist) -cat("Swift result:\n") -print(swiftres) - -goodres = c("numeric","numeric","try-error","numeric","numeric") - -diffs <- 0 -for(i in 1:length(swiftres) ) { - if( class(swiftres[[i]]) != goodres[i] ) { - diffs <- diffs + 1 - if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) - } -} - -if(diffs == 0) { - cat("\n==> test 5.1 passed\n") -} else { - cat("\n!!!==> test 5.1 failed.\n") - cat(sprintf(" %d result elements failed to match.\n",diffs)); - failures=failures+1 -} - -endTime <- proc.time()[["elapsed"]] -runTime <- endTime - startTime - -cat("\n\n ===> Total elapsed test time = ",runTime," seconds.\n\n") - -} # end function runAllTests - -#options(swift.site="local") -#runAllTests() - -testloop <- function(npass) -{ - for(i in 1:npass) { - cat("\n\n\n ***** Starting test pass ", i, " ***** \n\n\n"); - runAllTests() - cat("\n\n\n ***** Completed test pass ", i, " ***** \n\n\n"); - system("sleep 3") - } -} - -options(swift.site="service") -options(swift.keepwork=TRUE) -runAllTests() - - if(FALSE) { # Test various batch sizes cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") Modified: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO 2010-10-08 19:35:09 UTC (rev 3674) +++ SwiftApps/SwiftR/TODO 2010-10-09 14:27:20 UTC (rev 3675) @@ -36,14 +36,27 @@ - make tmpdir a param; for start-nnn scripts use SWIFTR_TMP, ~/.SwiftR - make ports flexible and non-conflicting -- initVar only affects first calls on a server - if you change these you need to start a new server (FIXME)! -- better handlig of initvars: detet if it changes; work corrcetly if its empty or missing. +X initVar only affects first calls on a server - if you change these you need to start a new server (FIXME)! +X better handlig of initvars: detet if it changes; work corrcetly if its empty or missing. See if we can get Swift to just shut down the channel and start it again? - no extended idle timer - start new swift worker - kill old R workers? +- replace all cat() calls with printf() + +- implement swiftping(serverName): access fifo non-blocking and poll for results with 1-sec sleeps. Try N times, echoing response or lack of. options: rualive, do-simple-R-eval. + +- implement swiftserver(configName, hosts=c("h1",...,"hN"), cpus=N, ... ) + + swiftserver(start,stop,ping,status) + + +- dont put full Swift release in package; download automatically at install time, and build from source. This is needed to make SwiftR CRAN-eligible. + +- streamline RPC calling path by minimizing file creation and any other costly operations. + test logic for creating trundir in start-swift-Rserver test for suitable java in start-swift command. @@ -233,7 +246,7 @@ specify swift scripts specify data files to be passed to remote side setup the R envs (???) - specify initial R code to run on remote side: (if !initialized_swift) { initialRCodeHere } +X specify initial R code to run on remote side: (if !initialized_swift) { initialRCodeHere } run async and grab status (track 'runs' in R) increm result collect specifiy unique swift scritps ala Dirk's tools From noreply at svn.ci.uchicago.edu Sat Oct 9 15:00:42 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 9 Oct 2010 15:00:42 -0500 (CDT) Subject: [Swift-commit] r3676 - trunk/libexec Message-ID: <20101009200042.6C25D9CC90@svn.ci.uchicago.edu> Author: hategan Date: 2010-10-09 15:00:42 -0500 (Sat, 09 Oct 2010) New Revision: 3676 Modified: trunk/libexec/vdl-int.k Log: fixed -scratch parameter in parameter files Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-10-09 14:27:20 UTC (rev 3675) +++ trunk/libexec/vdl-int.k 2010-10-09 20:00:42 UTC (rev 3676) @@ -401,6 +401,7 @@ replicationGroup, replicationChannel] stagein := list(unique(each(stagein))) stageout := list(unique(each(stageout))) + allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein) ddir := initDDir() @@ -429,7 +430,7 @@ if(wrapperMode == "files" sequential( sys:file:write(wrapfile, - "-scratch", try(vdl:siteprofile(rhost, "scratch"), "") + "-scratch", try(vdl:siteprofile(rhost, "scratch"), ""), nl(), "-e ",vdl:executable(tr, rhost), nl(), "-out ",stdout,nl(), "-err ",stderr,nl(), From noreply at svn.ci.uchicago.edu Sun Oct 10 17:11:16 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 10 Oct 2010 17:11:16 -0500 (CDT) Subject: [Swift-commit] r3677 - trunk/libexec Message-ID: <20101010221116.C3EC89CC84@svn.ci.uchicago.edu> Author: hategan Date: 2010-10-10 17:11:16 -0500 (Sun, 10 Oct 2010) New Revision: 3677 Modified: trunk/libexec/vdl-int.k Log: grr: needs a space after the name since _swiftwrap greps for "^-scratch " Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-10-09 20:00:42 UTC (rev 3676) +++ trunk/libexec/vdl-int.k 2010-10-10 22:11:16 UTC (rev 3677) @@ -430,7 +430,7 @@ if(wrapperMode == "files" sequential( sys:file:write(wrapfile, - "-scratch", try(vdl:siteprofile(rhost, "scratch"), ""), nl(), + "-scratch ", try(vdl:siteprofile(rhost, "scratch"), ""), nl(), "-e ",vdl:executable(tr, rhost), nl(), "-out ",stdout,nl(), "-err ",stderr,nl(), From noreply at svn.ci.uchicago.edu Sun Oct 10 22:40:28 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 10 Oct 2010 22:40:28 -0500 (CDT) Subject: [Swift-commit] r3678 - SwiftApps/Montage Message-ID: <20101011034028.36E109CC9B@svn.ci.uchicago.edu> Author: jonmon Date: 2010-10-10 22:40:28 -0500 (Sun, 10 Oct 2010) New Revision: 3678 Added: SwiftApps/Montage/swift_montage.properties Log: o SwiftApps/Montage/swift_montage.properties -- swift.properties file with entries to have the Swift Montage wrappers work. Added: SwiftApps/Montage/swift_montage.properties =================================================================== --- SwiftApps/Montage/swift_montage.properties (rev 0) +++ SwiftApps/Montage/swift_montage.properties 2010-10-11 03:40:28 UTC (rev 3678) @@ -0,0 +1,7 @@ +execution.retries=0 +sitedir.keep=true +status.mode=provider +wrapper.log.always.transfer=true +foreach.maxthreads=1024 +wrapper.parameter.mode=files + From noreply at svn.ci.uchicago.edu Mon Oct 11 10:27:09 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 11 Oct 2010 10:27:09 -0500 (CDT) Subject: [Swift-commit] r3679 - trunk/libexec Message-ID: <20101011152709.BF49F9CC90@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-11 10:27:08 -0500 (Mon, 11 Oct 2010) New Revision: 3679 Modified: trunk/libexec/vdl-int.k Log: Only attempt a CDM cleanup if CDM GATHER did something Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-10-11 03:40:28 UTC (rev 3678) +++ trunk/libexec/vdl-int.k 2010-10-11 15:27:08 UTC (rev 3679) @@ -138,8 +138,9 @@ element(cleanup, [dir, host] log(LOG:INFO, "START dir={dir} host={host}") cdmfile := cdm:file() - log(LOG:INFO, "cdmfile {cdmfile}") - if(cdmfile != "" then( + log(LOG:DEBUG, "cdmfile {cdmfile}") + if(cdmfile != "" & + cdm:get("GATHER_DIR") != "UNSET" then( log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}") task:transfer(srcfile="cdm_cleanup.sh", srcdir="{swift.home}/libexec", From noreply at svn.ci.uchicago.edu Mon Oct 11 22:34:27 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 11 Oct 2010 22:34:27 -0500 (CDT) Subject: [Swift-commit] r3680 - in SwiftApps/SwiftR/Swift: exec man Message-ID: <20101012033427.E28699CC90@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-11 22:34:27 -0500 (Mon, 11 Oct 2010) New Revision: 3680 Modified: SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/Swift/man/Swift-package.Rd Log: Added pbsman as an option to start-swift for manual pbs coaster start. Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-11 15:27:08 UTC (rev 3679) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-12 03:34:27 UTC (rev 3680) @@ -2,7 +2,7 @@ # Define internal functions -function wait-and-start-workers +get-contact() { # Look for: # Passive queue processor initialized. Callback URI is http://140.221.8.62:55379 @@ -23,7 +23,11 @@ CONTACT=$(echo $uriline | sed -e 's/^.*http:/http:/') echo Coaster service contact URI: $CONTACT +} +function wait-and-start-ssh-workers +{ + get-contact LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env? LOGDIR=/tmp/$USER/SwiftR/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way? @@ -44,20 +48,51 @@ sshpids="$sshpids $!" done - echo Started workers from these ssh processes: $sshpids + echo Started workers from ssh processes $sshpids echo $sshpids > $sshpidfile } +make-pbs-submit-file() +{ +cat >pbs.sub <$pbsjobidfile + + echo Started workers from PBS job $(cat $pbsjobidfile) +} + # main script -site=$1 # local, ssh, ... +site=$1 # local, ssh, pbsauto, pbsman ... # FIXME: check args and use better arg parsing tmp=${SWIFTR_TMP:-/tmp} -echo DB $0: site=$site tmp=$tmp - throttleOneCore="-0.001" throttleOneCore="0.00" localcores=5 # FIXME: parameterize: localthreads=N @@ -65,8 +100,6 @@ SWIFTRBIN=$(cd $(dirname $0); pwd) SWIFTBIN=$SWIFTRBIN/../swift/bin # This depends on ~/SwiftR/Swift/swift being a symlink to swift in RLibrary/Swift -echo DB $0: SWIFTRBIN=$SWIFTRBIN SWIFTBIN=$SWIFTBIN - rundir=$tmp/$USER/SwiftR/swift.$site # rundir prefix # FIXME: handle multiple concurent independent swift servers per user mkdir -p $(dirname $rundir) trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success @@ -74,7 +107,7 @@ ln -s $trundir $rundir cd $rundir -echo DB $0: rundir=$(pwd) SWIFTRBIN=$SWIFTRBIN SWIFTBIN=$SWIFTBIN +echo Running in $trundir "(linked to $rundir)" script=$SWIFTRBIN/rserver.swift #cp $script $SWIFTRBIN/passive-coaster-swift $SWIFTRBIN/swift.properties $rundir @@ -101,8 +134,6 @@ sshpidfile=${out/stdouterr/workerpids} - echo swift output is in: $out, pids in $sshpidfile - TRAPS="EXIT 1 2 3 15" # Signals and conditions to trap function onexit { @@ -124,9 +155,40 @@ trap onexit $TRAPS - wait-and-start-workers & + wait-and-start-ssh-workers & starterpid=$! +elif [ $site = pbsman ]; then + + # FIXME: Parameterize: + + walltime="01:00:00" + nodes=1 + queue=short + + pbsjobidfile=${out/stdouterr/pbsjobid} + + TRAPS="EXIT 1 2 3 15" # Signals and conditions to trap + + function onexit { + coasterservicepid="" # null: saved in case we go back to using coaster servers + trap - $TRAPS + pbsjobid=$(cat $pbsjobidfile) + echo Terminating worker processes starter $starterpid and PBS job $pbsjobid + if [ "_$starterpid != _ ]; then + kill $starterpid + fi + if [ "_$pbsjobid != _ ]; then + qdel $pbsjobid + fi + kill 0 # Kill all procs in current process group # FIXME: what was this for???? + } + + trap onexit $TRAPS + + wait-and-start-pbs-workers & + starterpid=$! + fi $SWIFTRBIN/../swift/bin/swift -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& $out require(Swift) +>basicSwiftTest() +>runAllSwiftTests() + +These will produce output similar to: + +\\\\\\VERBATIM + +> require(Swift) +Loading required package: Swift +> basicSwiftTest() + +*** Starting test 1.1 *** + +Test of local do.call(sumstuff) +local result= +[1] 4505 + +Test of swiftapply(sumstuff,arglist) + +Swift properties: + swiftserver = local + callsperbatch = 1 + runmode = service + tmpdir = /tmp + workerhosts = localhost + initialize = initVar1 <<- 19; initVar2 <<- sqrt(400)+3 + +Swift request is in /tmp/wilde/SwiftR/requests.P04114/R0000000 +Swift result: +[[1]] +[1] 4505 + + +==> test 1.1 passed +> +////////////// VERBATIM + CONFIGURE SERVERS edit configure-site-NAME in exec/ @@ -87,13 +149,21 @@ # do this outside of R - BEFORE trying to run R Swift functions SWIFT=/Swift/ -$SWIFT/exec/start-swift-workers hostname -$SWIFT/exec/start-swift-server -local and ssh servers can be started and left running, across R runs +$SWIFT/exec/start-swift local -found via: +or +$SWIFT/exec/start-swift pbsman + +or + +$SWIFT/exec/start-swift ssh host1 ... hostN + +These Swift servers can be started and left running, across R runs + +options(swift.server="local") # or "pbsman" or "ssh" + HELLO WORLD TEST # Start swift local server as above @@ -115,34 +185,124 @@ swiftTestLoop(n) -In source tree: +Testing from the source tree: source("Swift/tests/TestSwift.R") - or R CMD TEST etc? +or R CMD TEST etc? FIXME + +STOPPING SWIFT SERVERS + +The following ps command is useful for displaying the many background +swift processes. I keep this aliased as "mp" (my processes): + + alias mp='ps -fjH -u $USER' + +Local (swift-start local): + +$ jobs +$ kill %1 + +Remote (swift-start ssh): + +$ jobs +$ kill %1 # This tries to track down the remote processes and kill them + +Cluster (swift-start pbs): + +$ jobs +$ kill %1 # Swift should terminate its queued and/or running cluster jobs + + +Occaasionally a killall R and/or killall java is required + USAGE Swift returns Error object when remote side fails. +swiftapply( ) + + options: + swift.server: matched server name on start-swift + swift.callsperbatch + initialize: less likely to touch: remove temp reqs (sp???) FIXME mode (service, manual, ???) +Other Swift functions (compatible with Snow/Snowfall packages): + +swiftLapply + +To be developed: swiftSapply, ... + +TESTS AND EXAMPLES + +basicSwiftTest() + +runAllSwiftTests() + +testloop(n) + + OPENMX EXAMPLES This section is specific to users of the OpenMX R package for structural equation modeling. +USING ADDITIONAL PARALLEL ENVIRONMENTS + +3) ssh confiured for password-free login (to run on remote worker nodes) + +Ability to ssh to server machines (without password: agents, master +control channel, etc) (FIXME: Are these limitations necessary?) +Passwords or ssh key passphrases OK for some scenarios. + +ssh from Mac + +ssh -A when jumping to a new host (to forward the ssh agent) + +(or set up ssh agents manually) + +(document ssh tricks here for pw-less access) + + + DIRECTORY STRUCTURE USED FOR SWIFT RUNTIME PROCEESS STRUCTURE USE FOR SWIFT RUNTIME +vvvv VERBATIM + +vanquish$ mp +UID PID PPID PGID SID C STIME TTY TIME CMD +wilde 3621 3553 3553 3553 0 19:17 ? 00:00:00 sshd: wilde at pts/1 +wilde 3622 3621 3622 3622 0 19:17 pts/1 00:00:00 -bash +wilde 3726 3622 3726 3622 0 19:20 pts/1 00:00:00 /bin/bash ./start-swift local +wilde 3775 3726 3726 3622 0 19:20 pts/1 00:00:00 /bin/sh /homes/wilde/RLibrary/Swift/exec/../swift/bin/swift -config +wilde 3835 3775 3726 3622 0 19:20 pts/1 00:00:11 java -Xmx256M -Djava.endorsed.dirs=/homes/wilde/RLibrary/Swift/exe +wilde 8664 3622 8664 3622 0 20:47 pts/1 00:00:00 ps -fjH -u wilde +wilde 3441 3366 3366 3366 0 19:16 ? 00:00:00 sshd: wilde at pts/0 +wilde 3442 3441 3442 3442 0 19:16 pts/0 00:00:00 -bash +wilde 4114 3442 4114 3442 0 19:35 pts/0 00:00:05 /usr/lib64/R/bin/exec/R +wilde 4667 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar +wilde 4611 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar +wilde 4569 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar +wilde 4562 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar +wilde 4522 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar +wilde 4455 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar +wilde 4270 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar +wilde 4160 1 3726 3622 0 19:36 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar +vanquish$ + +^^^^^ VERBATIM + DEBUGGING AND TROUBLESHOOTING * manual mode @@ -151,8 +311,17 @@ * is my swift server responding? -tail -f $TMP/ +tail -f $TMP/$USER/SwiftR/swift.local/swift.stdouterr +You should see periodic status update lines such as the following: + +vvvvvvvvv VERBATIM + + + + +^^^^^^^^^ VERBATIM + * reporting bugs: what to send (FIXME: need swiftsnapshot script) * setting Swift worker logging with $HOME/.globus/coasters/loglevel @@ -160,36 +329,58 @@ 4=least detaild, 5=off. This is an interim log control mechanism and may be deprecated in the future. +CHECKOUT AND BUILD SWIFT R PACKAGE FROM SVN +WITH COMPILED SWIFT BINARY RELEASE (TRUNK) FROM SVN + +cd ~ +svn checkout https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SwiftR +cd SwiftR/Swift +mkdir swift +cd swift +wget http://www.ci.uchicago.edu/~wilde/swift.rNNNN.cog.rNNNN.tar.gz + +cd ~/SwiftR +./install.sh # generates a .gz package in ~/public_html/*.gz + CAVEATS +Swift requires Sun Java 1.4 or above; preferably 1.6. It will not run +under gcj (GNU Java) although it is getting closer to being able to +and may work - to some extent in in some settings. You need to ensure +that a suitable Sun Java in in your PATH. + +In addition, the environment variable CLASSPATH should not be set. + +Variables set in the initialze script must typically be set in global +environment ( var <<- value ); + +The following caveats are high prioiry on the FIXME list: + You MUST start the Swift server before running a swiftapply() call from R. Otherwise R hangs and must be killed and restarted. -When Swift fifos (named pipes) get hung, you need to use kill or Quit -to break out of R. FIXME +When the FIFOs (named pipes) which are used to communicate from R to +Swift get hung, you need to use kill or Quit to break out of R. There is no automatic restart yet if swift dies in its server -loop. FIXME +loop. -Variables set in the initialze script must typically be set in global -environment ( var <<- value); - Only lapply is implemented (also SwiftApply) - need to see if we can -cut down arg passing overhead for many of the apply() cases +cut down arg passing overhead for many of the apply() cases. Log records build up fast; these will be reduced as we get more confidence with the code and shake out bugs +Lower priority issues are: + There is no easy way yet to alter Swift configuration file variables such as number of cores to use, etc. Do this for now by editing an existing configuration under Swift/exec/conigure-swift-NNN where NNN is the Swift server name. -Each swiftapply() is pretty noisy - it echos its options etc. This -will quiet down. +Each swiftapply() is pretty noisy - it echos its options, etc. This +will quiet down and "verbose" will be made an option. - - } \author{ From noreply at svn.ci.uchicago.edu Thu Oct 14 08:47:37 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 Oct 2010 08:47:37 -0500 (CDT) Subject: [Swift-commit] r3681 - in SwiftApps/SwiftR/Swift: R exec man Message-ID: <20101014134737.6C6839CC84@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-14 08:47:37 -0500 (Thu, 14 Oct 2010) New Revision: 3681 Added: SwiftApps/SwiftR/Swift/exec/configure-server-local SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto SwiftApps/SwiftR/Swift/exec/configure-server-ssh SwiftApps/SwiftR/Swift/exec/start-merlot Removed: SwiftApps/SwiftR/Swift/exec/configure-site-local SwiftApps/SwiftR/Swift/exec/configure-site-pbs Modified: SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/Swift/man/Swift-package.Rd Log: Added ability to connect pbs workers through firewall; temp version of this in start-merlot integrated into start-swift; added sections to Swift doc page. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-12 03:34:27 UTC (rev 3680) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-14 13:47:37 UTC (rev 3681) @@ -2,7 +2,7 @@ swiftserver=NULL, callsperbatch=NULL, runmode=NULL, - initialize=NULL, + initialexpr=NULL, workerhosts=NULL, keepwork=NULL, tmpdir=NULL ) @@ -27,10 +27,10 @@ # script: run swift for each request, via RunSwiftScript.sh (currently broken) # manual: for testing, let user run remote R server manually - if(is.null(initialize)) - initialize <- getOption("swift.initialize") - if(is.null(initialize)) - initialize <- ""; + if(is.null(initialexpr)) + initialexpr <- getOption("swift.initialexpr") + if(is.null(initialexpr)) + initialexpr <- ""; if(is.null(workerhosts)) workerhosts <- getOption("swift.workerhosts") @@ -55,7 +55,7 @@ cat(" runmode =", runmode,"\n") cat(" tmpdir =", tmpdir,"\n") cat(" workerhosts =", workerhosts,"\n") - cat(" initialize =", initialize,"\n\n") + cat(" initialexpr =", initialexpr,"\n\n") user <- Sys.info()[["user"]] @@ -99,7 +99,7 @@ arglistbatch[[i]] <- arglists[[arglist]] arglist <- arglist +1 } - rcall <- list(initializer=initialize,func=func,arglistbatch=arglistbatch) + rcall <- list(initializer=initialexpr,func=func,arglistbatch=arglistbatch) save(rcall,file=paste(reqdir,"/cbatch.",as.character(batch),".Rdata",sep="")) batch <- batch + 1; } @@ -190,7 +190,7 @@ options(swift.site="service") options(swift.keepwork=TRUE) initcmds <- "initVar1 <<- 19; initVar2 <<- sqrt(400)+3" - options(swift.initialize=initcmds) # Set here; used in test group 4 + options(swift.initialexpr=initcmds) # Set here; used in test group 4 } swiftTest_1.1 <- function() @@ -252,7 +252,7 @@ swiftTest_4.2 <- function() { - options(swift.initialize="initVar3 <<- 123; initVar4 <<- 100"); + options(swift.initialexpr="initVar3 <<- 123; initVar4 <<- 100"); mulivars <- function() { initVar3*initVar4 } @@ -535,7 +535,7 @@ failures=failures+1 } -##### Test Group 4 # test initializer string +##### Test Group 4 # test initialexpr string cat("\n*** Starting test group 4 - test remote R service initialization string ***\n") Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh 2010-10-12 03:34:27 UTC (rev 3680) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh 2010-10-14 13:47:37 UTC (rev 3681) @@ -1,5 +1,6 @@ #! /usr/bin/env Rscript +require(methods) argv = commandArgs(TRUE) fifoDir = argv[1]; # FIXME: test for valid arguments @@ -50,10 +51,12 @@ result <- list() initializer <- rcall$initializer; + # print(sprintf("received initializer=%s latestInitializer=%s\n",initializer, latestInitializer)); if( initializer != latestInitializer) { initialExpr <- parse(text=initializer) eval(initialExpr) latestInitializer <<- initializer + # print(sprintf("after eval: latestInitializer=%s\n", latestInitializer)); } for(c in 1:length(rcall$arglistbatch)) { # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) Copied: SwiftApps/SwiftR/Swift/exec/configure-server-local (from rev 3660, SwiftApps/SwiftR/Swift/exec/configure-site-local) =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-local (rev 0) +++ SwiftApps/SwiftR/Swift/exec/configure-server-local 2010-10-14 13:47:37 UTC (rev 3681) @@ -0,0 +1,69 @@ +#! /bin/bash + +throttleOneCore="0.00" # FIXME: test if new swft fix makes zero OK rather than -0.001 + +cores=$1 + +if [ -r /proc/cpuinfo ]; then + localcores=$(grep '^processor' /proc/cpuinfo | wc -l) +else + localcores=1 +fi + +if [ $cores -eq 0 ]; then + cores=$localcores +fi + +cat >tc <sites.xml < + + + 0.15 + 10000 + + $(pwd)/swiftwork + +END + +for i in `seq -w 0 $((cores-1))`; do + +# FIXME: how to get the right PATH for R in here? +# use /bin/sh and count on users PATH? +# then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename) + +cat >>tc <>sites.xml < + + $throttleOneCore + 10000 + + $(pwd)/swiftwork + +END + +done + +# 1 + +echo '' >>sites.xml + +cat >cf <tc <sites.xml < + + + 0.15 + 10000 + + $(pwd)/swiftwork + + + + + fast + 00:59:00 + $throttlePBS + 10000 + + $HOME/swiftwork + + + + + short + 12000 + 32 + 1 + 1 + 1 + 2.55 + 10000 + + + $HOME/swiftwork + + +END + +# 00:00:01 +# fast + +cat >cf <tc <sites.xml < + + + 0.15 + 10000 + + $(pwd)/swiftwork + + + + + passive + $cores + 2.55 + 10000 + + $HOME/swiftwork + + + +END + +cat >cf <00:00:01 +# fast + + + + + passive + 4 + .03 + 10000 + + $HOME/swiftwork + +END Deleted: SwiftApps/SwiftR/Swift/exec/configure-site-local =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-site-local 2010-10-12 03:34:27 UTC (rev 3680) +++ SwiftApps/SwiftR/Swift/exec/configure-site-local 2010-10-14 13:47:37 UTC (rev 3681) @@ -1,67 +0,0 @@ -#! /bin/bash - -throttleOneCore="-0.001" -throttleOneCore="0.00" # FIXME: test if new swft fix makes zero OK rather than -0.001 - -if [ -r /proc/cpuinfo ]; then - localcores=$(grep '^processor' /proc/cpuinfo | wc -l) -else - localcores=4 -fi - -#### DBDBDBDB vvvvvv -#localcores=1 - -cat >tc <sites.xml < - - - 0.15 - 10000 - - $(pwd)/swiftwork - -END - -for i in `seq -w 0 $((localcores-1))`; do - -# FIXME: how to get the right PATH for R in here? -# use /bin/sh and count on users PATH? -# then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename) - -cat >>tc <>sites.xml < - - $throttleOneCore - 10000 - - $(pwd)/swiftwork - -END - -done - -# 1 - -echo '' >>sites.xml - -cat >cf <tc <sites.xml < - - - 0.15 - 10000 - - $(pwd)/swiftwork - - - - - fast - 00:59:00 - $throttlePBS - 10000 - - $HOME/swiftwork - - - - - short - 12000 - 32 - 1 - 1 - 1 - 2.55 - 10000 - - - $HOME/swiftwork - - -END - -# 00:00:01 -# fast - -cat >cf < /dev/null) + if [ "_$uriline" = _ ]; then + sleep 1 + else + break; + fi + done + + if [ "_$uriline" = _ ]; then + echo "$0: No passive state message from Swift - exiting." + exit 1 + fi + + CONTACT=$(echo $uriline | sed -e 's/^.*http:/http:/') + echo Coaster service contact URI: $CONTACT +} + +function wait-and-start-ssh-workers +{ + get-contact + LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env? + LOGDIR=/tmp/$USER/SwiftR/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way? + + # mkdir -p $LOGDIR # is done with the ssh command, below + + IDLETIMEOUT=$((60*60*240)) # 10 days: FIXME: make this a command line arg + + rm -rf remotepid.* # FIXME: should not be needed if we start in a new dir each time + for host in $(echo $hosts); do + timestamp=$(date "+%Y.%m%d.%H%M%S") + random=$(awk "BEGIN {printf \"%0.5d\", $RANDOM}") + ID=$timestamp.$random + # FIXME: make logging an argument; set false by default + # fixme:send worker.pl to remote host via stdin or scp. + ssh $host /bin/sh -c \'"mkdir -p $LOGDIR"\' + scp $SWIFTBIN/worker.pl $host:$LOGDIR + ssh $host '/bin/sh -c '\'"WORKER_LOGGING_ENABLED=true $LOGDIR/worker.pl $CONTACT $ID $LOGDIR $IDLETIMEOUT 2>&1 & echo PID=\$!"\' >remotepid.$host $sshpidfile + wait +} + +make-pbs-submit-file() +{ + if [ $queue != default ]; then + queueDirective="#PBS -q $queue" + else + queueDirective="" + fi +cat >pbs.sub <$pbsjobidfile + + echo Started workers from PBS job $(cat $pbsjobidfile) +} + +usage() +{ + echo >&2 "usage: $0 -c cores -h 'host1 ... hostN' -n nodes -q queue -s server -t throttle" + echo >&2 " valid servers: local, ssh, pbs" + echo >&2 " defaults: cores=2 nodes=1 queue=none server=local throttle=10" +} + +verify-is-one-of() +{ + argname=$1 + arg=$2 + shift 2 + for v in $*; do + if [ $arg = $v ]; then + return 0 + fi + done + echo $0: value for argument $argname was $arg - must be one of: $* + usage + exit 1 +} + +verify-is-numeric() +{ + argname=$1; shift + if test $1 -ge 0 2>/dev/null; then + return 0 + else + echo $0: value for $argname must be a positive integer, was: $1 + usage + exit 1 + fi +} + +verify-not-null() +{ + argname=$1; shift + if [ _$1 != _ ]; then + return 0 + else + echo $0: value for $argname can not be null + usage + exit 1 + fi +} + +# main script + +tmp=${SWIFTR_TMP:-/tmp} + +# Process command line args + +server=local +time="00:30:00" +nodes=1 +queue=short +#throttleOneCore="-0.001" FIXME: Remove +#throttleOneCore="0.00" FIXME: Remove +localcores=5 +cores=0 +defaultCores=4 +throttle=10 +hosts=no-hosts-specified +queue=default + +while [ $# -gt 0 ] +do + case "$1" in + -c) cores=$2; verify-is-numeric cores $cores; shift ;; + -h) hosts=$2; verify-not-null hosts $hosts; shift ;; + -n) nodes=$2; verify-is-numeric nodes $nodes; shift ;; + -p) throttle=$2; verify-is-numeric throttle $throttle; shift ;; + -q) queue=$2; verify-not-null queue $queue; shift ;; + -s) server=$2; verify-is-one-of server $server local ssh pbs; shift ;; + -t) time=$2; verify-is-not-null time $time; shift ;; + *) usage; exit 1 ;; + esac + shift +done + +echo cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle + +SWIFTRBIN=$(cd $(dirname $0); pwd) +SWIFTBIN=$SWIFTRBIN/../swift/bin # This depends on ~/SwiftR/Swift/swift being a symlink to swift in RLibrary/Swift + +rundir=$tmp/$USER/SwiftR/swift.$server # rundir prefix # FIXME: handle multiple concurent independent swift servers per user +mkdir -p $(dirname $rundir) +trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success +rm -rf $rundir +ln -s $trundir $rundir +cd $rundir + +echo Running in $trundir "(linked to $rundir)" + +script=$SWIFTRBIN/rserver.swift +#cp $script $SWIFTRBIN/passive-coaster-swift $SWIFTRBIN/swift.properties $rundir +cp $script . +script=$(basename $script) +cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.sh} . + +# DONE: FIXME: rework this script to transfer all shells and rscripts +# needed, and to copy in the R prelude for the R server processes (to +# include for example the OpenMx library) NOTE: Both were done in older version of this script. + +# rm -f requestpipe resultpipe +mkfifo requestpipe resultpipe + +out=swift.stdouterr +touch $out + +if [ $server = local ]; then + + source $SWIFTRBIN/configure-server-local $cores + +elif [ $server = ssh ]; then + + if [ $cores -eq 0 ]; then + cores = $defaultRemoteCores + fi + + source $SWIFTRBIN/configure-server-ssh $cores $time + + sshpidfile=${out/stdouterr/workerpids} + + function onexit { + coasterservicepid="" # null: saved in case we go back to using coaster servers + trap - $TRAPEVENTS + sshpids=$(cat $sshpidfile) + echo Terminating worker processes $sshpids, starter $starterpid + for rpfile in $(ls -1 remotepid.*); do + rpid=$(grep PID= $rpfile | sed -e 's/PID=//') + rhost=$(echo $rpfile | sed -e 's/remotepid.//') + echo Based on $rpfile: terminating process group of process $rpid on $rhost + ssh $rhost sh -c \''PGID=$(ps -p '$rpid' -o pgid --no-headers|sed -e "s/ //g"); kill -s TERM -- -$PGID'\' + done + if [ "_$sshpids$starterpid$coasterservicepid" != _ ]; then + echo kill $sshpids $starterpid $coasterservicepid >& /dev/null + fi + kill 0 # Kill all procs in current process group # FIXME: what was this for???? + } + + trap onexit $TRAPEVENTS + + wait-and-start-ssh-workers & + starterpid=$! + +elif [ $server = pbs ]; then + + source $SWIFTRBIN/configure-server-pbs $cores + + pbsjobidfile=${out/stdouterr/pbsjobid} + + function onexit { + coasterservicepid="" # null: saved in case we go back to using coaster servers + trap - $TRAPEVENTS + pbsjobid=$(cat $pbsjobidfile) + echo Terminating worker processes starter $starterpid and PBS job $pbsjobid + if [ "_$starterpid != _ ]; then + kill $starterpid + fi + if [ "_$pbsjobid != _ ]; then + qdel $pbsjobid + fi + kill 0 # Kill all procs in current process group # FIXME: what was this for???? + } + + trap onexit $TRAPEVENTS + + wait-and-start-pbs-workers & + starterpid=$! + +fi + +$SWIFTRBIN/../swift/bin/swift -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& $out pbs.sub <pbs.sub <$pbsjobidfile echo Started workers from PBS job $(cat $pbsjobidfile) } -# main script +usage() +{ + echo >&2 "usage: $0 -c cores -h 'host1 ... hostN' -n nodes -q queue -s server -t throttle" + echo >&2 " valid servers: local, ssh, pbs, pbsf (pbs with firewalled workers)" + echo >&2 " defaults: cores=2 nodes=1 queue=none server=local throttle=10" +} -site=$1 # local, ssh, pbsauto, pbsman ... +verify-is-one-of() +{ + argname=$1 + arg=$2 + shift 2 + for v in $*; do + if [ $arg = $v ]; then + return 0 + fi + done + echo $0: value for argument $argname was $arg - must be one of: $* + usage + exit 1 +} -# FIXME: check args and use better arg parsing +verify-is-numeric() +{ + argname=$1; shift + if test $1 -ge 0 2>/dev/null; then + return 0 + else + echo $0: value for $argname must be a positive integer, was: $1 + usage + exit 1 + fi +} +verify-not-null() +{ + argname=$1; shift + if [ _$1 != _ ]; then + return 0 + else + echo $0: value for $argname can not be null + usage + exit 1 + fi +} + +# main script + tmp=${SWIFTR_TMP:-/tmp} -throttleOneCore="-0.001" -throttleOneCore="0.00" -localcores=5 # FIXME: parameterize: localthreads=N +# Process command line args +server=local +time="00:30:00" +nodes=1 +queue=short +#throttleOneCore="-0.001" FIXME: Remove +#throttleOneCore="0.00" FIXME: Remove +localcores=5 +cores=0 +defaultCores=4 +throttle=10 +hosts=no-hosts-specified +queue=default + +while [ $# -gt 0 ] +do + case "$1" in + -c) cores=$2; verify-is-numeric cores $cores; shift ;; + -h) hosts=$2; verify-not-null hosts $hosts; shift ;; + -n) nodes=$2; verify-is-numeric nodes $nodes; shift ;; + -p) throttle=$2; verify-is-numeric throttle $throttle; shift ;; + -q) queue=$2; verify-not-null queue $queue; shift ;; + -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf; shift ;; + -t) time=$2; verify-not-null time $time; shift ;; + *) usage; exit 1 ;; + esac + shift +done + +echo cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle + SWIFTRBIN=$(cd $(dirname $0); pwd) SWIFTBIN=$SWIFTRBIN/../swift/bin # This depends on ~/SwiftR/Swift/swift being a symlink to swift in RLibrary/Swift -rundir=$tmp/$USER/SwiftR/swift.$site # rundir prefix # FIXME: handle multiple concurent independent swift servers per user +rundir=$tmp/$USER/SwiftR/swift.$server # rundir prefix # FIXME: handle multiple concurent independent swift servers per user mkdir -p $(dirname $rundir) trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success rm -rf $rundir @@ -122,23 +249,28 @@ # rm -f requestpipe resultpipe mkfifo requestpipe resultpipe -source $SWIFTRBIN/configure-site-$1 - out=swift.stdouterr touch $out -if [ $site = ssh ]; then +if [ $server = local ]; then - shift - COMPUTEHOSTS=$* + source $SWIFTRBIN/configure-server-local $cores +elif [ $server = ssh ]; then + + if [ $cores -eq 0 ]; then + cores = $defaultRemoteCores + fi + + source $SWIFTRBIN/configure-server-ssh $cores $time + sshpidfile=${out/stdouterr/workerpids} - TRAPS="EXIT 1 2 3 15" # Signals and conditions to trap + TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap function onexit { coasterservicepid="" # null: saved in case we go back to using coaster servers - trap - $TRAPS + trap - $TRAPEVENTS sshpids=$(cat $sshpidfile) echo Terminating worker processes $sshpids, starter $starterpid for rpfile in $(ls -1 remotepid.*); do @@ -153,26 +285,22 @@ kill 0 # Kill all procs in current process group # FIXME: what was this for???? } - trap onexit $TRAPS + trap onexit $TRAPEVENTS wait-and-start-ssh-workers & starterpid=$! -elif [ $site = pbsman ]; then +elif [ \( $server = pbs \) -o \( $server = pbsf \) ]; then - # FIXME: Parameterize: + source $SWIFTRBIN/configure-server-pbs $cores - walltime="01:00:00" - nodes=1 - queue=short - pbsjobidfile=${out/stdouterr/pbsjobid} - TRAPS="EXIT 1 2 3 15" # Signals and conditions to trap + TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap function onexit { coasterservicepid="" # null: saved in case we go back to using coaster servers - trap - $TRAPS + trap - $TRAPEVENTS pbsjobid=$(cat $pbsjobidfile) echo Terminating worker processes starter $starterpid and PBS job $pbsjobid if [ "_$starterpid != _ ]; then @@ -184,7 +312,7 @@ kill 0 # Kill all procs in current process group # FIXME: what was this for???? } - trap onexit $TRAPS + trap onexit $TRAPEVENTS wait-and-start-pbs-workers & starterpid=$! Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-10-12 03:34:27 UTC (rev 3680) +++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-10-14 13:47:37 UTC (rev 3681) @@ -39,8 +39,10 @@ options(swift.site=sitename) # sitename = "local" to run on the current host and "pbs" to submit to a local PBS cluster. -PREREQUISITES +} +\section{PREREQUISITES}{ + 1) Sun Java 1.4 or higher (preferably 1.6) installed and in your PATH Download the appropriate Java for Linux at: @@ -78,8 +80,9 @@ In configurations (b) and (c) Swift will launch its own workers, and then communicate using its own TCP protocol. +} -INSTALL +\section{INSTALLATION}{ mkdir ~/RPackages ~/RLibrary # if not already created cd ~/RPackages @@ -87,8 +90,11 @@ R CMS INSTALL -l ~/RLibrary Swift_0.1.tar.gz export R_LIBS=~/RLibrary -QUICK START +export GLOBUS_HOSTNAME=10.0.0.200 # Eg for Merlot: internal address of the login node +} +\section{QUICK START}{ + In a shell (outside of R) start the local Swift server: $HOME/RLibrary/Swift/exec/swift-start local @@ -100,7 +106,7 @@ These will produce output similar to: -\\\\\\VERBATIM +\preformatted{ > require(Swift) Loading required package: Swift @@ -127,13 +133,14 @@ [[1]] [1] 4505 - ==> test 1.1 passed > -////////////// VERBATIM +} -CONFIGURE SERVERS +} +\section{CONFIGURE SERVERS}{ + edit configure-site-NAME in exec/ can put local cores into an ssh pool @@ -143,8 +150,9 @@ access remote systems via ssh Export SWIFTR_TMP in your environment +} -START SERVERS +\section{START SERVERS}{ # do this outside of R - BEFORE trying to run R Swift functions @@ -163,15 +171,17 @@ These Swift servers can be started and left running, across R runs options(swift.server="local") # or "pbsman" or "ssh" +} -HELLO WORLD TEST +\section{HELLO WORLD TEST}{ # Start swift local server as above require(Swift) basicSwiftTest() +} -RUN FULL TEST +\section{RUN FULL TEST}{ As a regular user: @@ -180,9 +190,9 @@ # Then -n=10 # 1o times through full test loop +n=10 # 10 times through full test loop -swiftTestLoop(n) +testLoop(n) Testing from the source tree: @@ -190,10 +200,10 @@ source("Swift/tests/TestSwift.R") or R CMD TEST etc? FIXME +} +\section{STOPPING SWIFT SERVERS}{ -STOPPING SWIFT SERVERS - The following ps command is useful for displaying the many background swift processes. I keep this aliased as "mp" (my processes): @@ -217,8 +227,10 @@ Occaasionally a killall R and/or killall java is required -USAGE +} +\section{USAGE}{ + Swift returns Error object when remote side fails. swiftapply( ) @@ -242,22 +254,17 @@ To be developed: swiftSapply, ... -TESTS AND EXAMPLES +} -basicSwiftTest() +\section{OPENMX EXAMPLES}{ -runAllSwiftTests() - -testloop(n) - - -OPENMX EXAMPLES - This section is specific to users of the OpenMX R package for structural equation modeling. -USING ADDITIONAL PARALLEL ENVIRONMENTS +} +\section{USING ADDITIONAL PARALLEL ENVIRONMENTS}{ + 3) ssh confiured for password-free login (to run on remote worker nodes) Ability to ssh to server machines (without password: agents, master @@ -272,14 +279,16 @@ (document ssh tricks here for pw-less access) +} +\section{DIRECTORY STRUCTURE USED FOR SWIFT RUNTIME}{ +tbd +} -DIRECTORY STRUCTURE USED FOR SWIFT RUNTIME +\section{PROCEESS STRUCTURE OF SWIFT RUNTIME}{ -PROCEESS STRUCTURE USE FOR SWIFT RUNTIME +\preformatted{ -vvvv VERBATIM - vanquish$ mp UID PID PPID PGID SID C STIME TTY TIME CMD wilde 3621 3553 3553 3553 0 19:17 ? 00:00:00 sshd: wilde at pts/1 @@ -299,12 +308,11 @@ wilde 4455 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar wilde 4270 1 3726 3622 0 19:38 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar wilde 4160 1 3726 3622 0 19:36 pts/1 00:00:00 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --ar -vanquish$ +vanquish$ } -^^^^^ VERBATIM +} +\section{DEBUGGING AND TROUBLESHOOTING}{ -DEBUGGING AND TROUBLESHOOTING - * manual mode * logs to look at @@ -315,23 +323,22 @@ You should see periodic status update lines such as the following: -vvvvvvvvv VERBATIM +\preformatted{ +tbd} - - - -^^^^^^^^^ VERBATIM - * reporting bugs: what to send (FIXME: need swiftsnapshot script) * setting Swift worker logging with $HOME/.globus/coasters/loglevel file. This file should contain a single text integer: 0=most detailed, 4=least detaild, 5=off. This is an interim log control mechanism and may be deprecated in the future. +} -CHECKOUT AND BUILD SWIFT R PACKAGE FROM SVN -WITH COMPILED SWIFT BINARY RELEASE (TRUNK) FROM SVN +\section{CHECKOUT AND BUILD SWIFT R PACKAGE FROM SVN}{ +(with compiled swift binary release (trunk) from svn) + +\preformatted{ cd ~ svn checkout https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SwiftR cd SwiftR/Swift @@ -340,10 +347,12 @@ wget http://www.ci.uchicago.edu/~wilde/swift.rNNNN.cog.rNNNN.tar.gz cd ~/SwiftR -./install.sh # generates a .gz package in ~/public_html/*.gz +./install.sh # generates a .gz package in ~/public_html/*.gz} -CAVEATS +} +\section{CAVEATS}{ + Swift requires Sun Java 1.4 or above; preferably 1.6. It will not run under gcj (GNU Java) although it is getting closer to being able to and may work - to some extent in in some settings. You need to ensure @@ -352,8 +361,27 @@ In addition, the environment variable CLASSPATH should not be set. Variables set in the initialze script must typically be set in global -environment ( var <<- value ); +environment ( var <<- value ); These conventions may need to be +revisted. The initialexpr script is passed in the same saved/loaded R +object as R functions and arguments. Thus if the expr needs to be run +before these objects can be loaded, then an alternate mathod of +intialization needs to be used. (Eg the original method of passing th +einitvar as a separate file). +All Swift servers do a \verb{require(methods)} call when they +start. Its not clear if this is unversally desired. It was not done by +default in Rserver but seems to be in interactive R. + +Running on systems like Merlot with very restrctive firewalls requires +that the user first export the env var GLOBUS_HOSTNAME, set to the IP +address of the network interface by which the worker nodes can connect +to the login host on which the R client and start-swift commands are +running. + +pbs and pbsf servers try to clean up all worker nodes used with a +killlall -u $USER. This can be made more precise to avoid killing jobs +on shared worker nodes. + The following caveats are high prioiry on the FIXME list: You MUST start the Swift server before running a swiftapply() call @@ -385,13 +413,13 @@ \author{ -Swift R package developed by Michael Wilde - Swift was developed by: Mihael Hategan, Ben Clifford, Justin Wozniak, Yong Zhao, Ian Foster, and Michael Wilde with contributions from Sarah Kenny, Ioan Raicu, Luiz Gadelha, Allan Espinosa, Zhao Zhang, David Kelly, Jon Monette, Glen Hocky, Tom Uram, Wenjun Wu, and other users. +Swift R package developed by Michael Wilde and the OpenMx project + Maintainer: Michael Wilde } From noreply at svn.ci.uchicago.edu Thu Oct 14 08:51:28 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 Oct 2010 08:51:28 -0500 (CDT) Subject: [Swift-commit] r3682 - SwiftApps/SwiftR Message-ID: <20101014135128.BF8679CC84@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-14 08:51:28 -0500 (Thu, 14 Oct 2010) New Revision: 3682 Modified: SwiftApps/SwiftR/TODO Log: edited todo list Modified: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO 2010-10-14 13:47:37 UTC (rev 3681) +++ SwiftApps/SwiftR/TODO 2010-10-14 13:51:28 UTC (rev 3682) @@ -1,5 +1,18 @@ *** TO DO LIST: +Thu: +why script.pl appears in coasters dir for manual coasters? + +more flexibility in ssh mehanism + +need globus_hostname? + +job cleanup improvement + +tcsh ok? + + + BUG: Passivate failing on swift pbs config on pads. ^^^^^^^^^^^^^^^^^^ From noreply at svn.ci.uchicago.edu Sun Oct 17 09:15:27 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 17 Oct 2010 09:15:27 -0500 (CDT) Subject: [Swift-commit] r3683 - in SwiftApps/SwiftR/Swift: exec man Message-ID: <20101017141527.18DB59CC9E@svn.ci.uchicago.edu> Author: wilde Date: 2010-10-17 09:15:22 -0500 (Sun, 17 Oct 2010) New Revision: 3683 Modified: SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/Swift/man/Swift-package.Rd Log: correct shell syntax error in ssh server code. Small doc updates. Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-14 13:51:28 UTC (rev 3682) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-17 14:15:22 UTC (rev 3683) @@ -259,7 +259,7 @@ elif [ $server = ssh ]; then if [ $cores -eq 0 ]; then - cores = $defaultRemoteCores + cores=$defaultRemoteCores fi source $SWIFTRBIN/configure-server-ssh $cores $time Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-10-14 13:51:28 UTC (rev 3682) +++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-10-17 14:15:22 UTC (rev 3683) @@ -5,62 +5,139 @@ \alias{swiftLapply} \docType{package} \title{ -R interface to Swift parallel scripting language +R interface to for parallel apply() calls using Swift } \description{ -Description: Routines to invoke R functions and Swift scripts on -remote resources through Swift. R functions can be remotely executed -in parallel in a manner similar to Snow using a list of argument -lists. Eventually more general Swift functions can be embedded and -invked remotely as well. +Apply R functions to lists of arguments in parallel, on distributed +resources. Resources selection and access, scheduling, and throttling +are provided using the Swift parallel scripting language, +transparently to the R user. +Currently supports local multicore CPUs; clusters runnng PBS; and +ad-hoc clusters composed of one or more remote multicore systems +accessible via SSH. + +Remote resources are managed by Swift worker agents started by SSH or +the cluster scheduler, and accessed via a Swift protocol. Each pool of +resources is called a "Swift server". Multiple Swift servers can be +accessed from one R client, + } + \details{ \tabular{ll}{ Package: \tab Swift\cr Type: \tab Package\cr -Version: \tab 1.0\cr -Date: \tab 2010-02-25\cr +Version: \tab 0.1.0\cr +Date: \tab 2010-10-15\cr License: \tab Globus Toolkit Public License v3 (based on Apache License 2.0): http://www.globus.org/toolkit/legal/4.0/license-v3.html \cr LazyLoad: \tab yes\cr } +FIXME: Use the traditional R format for the initial part of the man page. + +The main function in this package is: + +resultList = swiftapply(function,listOfArgumentLists) + +swiftapply() invokes function on each of the arguments lists in +listOfArgumentLists, and returns a list of the result of the invocation in the corresponding +member of resultList. Each invocation is executed in parallel in an +external, independent copy of R. + +This is the most general function in the Swift package, as each +arglist is specified in full and can contain unique values for all +arguments for each invocation of function. + +Options that determine the behavior of swiftapply can be set via +options(swift.*=value) or via optional keyword arguments. + To use this package, create a list of argument lists, and then invoke: -swiftapply(function,arglists). -As a preliminary interface, you can set R options() to control Swift's -operation: +\verb{resultList = swiftapply(function, listOfArgumentLists)} +For example: + +\preformatted{ +arglists=list() +for(i in 1:10) { + arglists[[i]] = list(i) +} +resultlist = swiftapply(sqrt,arglists) +} + +The same conventions as used by the Snowfall package are used for swiftLapply, analogous to sfLapply: + +\preformatted{ +r = swiftLapply(seq(1,10),sqrt) +} + +Currenty swiftLapply is the only one implemented (i.e. swiftSapply etc are not yet provided but will be soon). + +Arbitrary R objects can be passed. For example: + +\preformatted{ + data(cars) + data(trees) + sumstuff <- function(treedata, cardata) { + sum(treedata$Height, cardata$dist) + } + args = list(trees, cars) + arglist = rep(list(args), 10) + res = swiftapply(sumstuff, arglist) +} + +As a preliminary interface, you can set R options() to control the +operation of the functions in the Swift package: + options(swift.callsperbatch=n) # n = number of R calls to perform in each Swift job. -options(swift.site=sitename) # sitename = "local" to run on the -current host and "pbs" to submit to a local PBS cluster. +options(swift.server="servername") # servername = "local" to run on +the current host, "pbs" to submit to a local PBS cluster, "pbsf" to +run on clusters such as Merlot which have firewalls that restrict +outbound cnnectivity from the worker nodes to the Swift server running +on the login node. +options(swift.keepwork=TRUE) # Retain the temporary files that the +Swift functions use to pass R data from client t remote R +servers. This is useful for debugging. + } \section{PREREQUISITES}{ +To run Swift, you need a Java runtime environment (JRE) installed on +the client machine (where the client R workspace will be executed). +Worker nodes (remote resources) only need Perl (to run the Swift +worker agent). + +Remote resources can be accessed via SSH or the PBS batch scheduler. + +Details: + 1) Sun Java 1.4 or higher (preferably 1.6) installed and in your PATH -Download the appropriate Java for Linux at: - http://www.java.com/en/download/manual.jsp (JREs) +Download the appropriate Java Runtime (JRE) for Linux at:\verb{ + http://www.java.com/en/download/manual.jsp} -Typically either 32 bit with this link: - Linux (self-extracting file) filesize: 19.9 +Typically either 32 bit with this link:\verb{ + Linux (self-extracting file) filesize: 19.9} -Or 64 bit with this link - Linux x64 * filesize: 19.3 MB +Or 64 bit with this link\verb{ + Linux x64 * filesize: 19.3 MB } Its better to install the JDK (~80MB) from: http://www.oracle.com/technetwork/java/javase/downloads/jdk6-jsp-136632.html (JDKs) -(This will enable you to compile Swift fixes from the Swift development trunk) +(This will enable you to compile Swift revisions from the Swift development trunk) -Mac OS X: Download ... +Mac OS X: Download ... FIXME: Mac OS X is not yet tested for use with +this package. Few issues are expected, but some shell command +differences may affect the scripts in this package. Verify that you have Sun Java installed and in your PATH correctly by doing: @@ -72,14 +149,26 @@ 2) R v2.11 or higher in your PATH (on client and server machines) +(Testing has been done on R 2.11. The package is likely to operate on +older R versions as well, but has not yet been validated on them). + 3) Access to Parallel Resources - You can run Swift R package using: (a) Multiple cores on your local login host or workstation/laptop + (b) One or more remote machines, possibly each a multicore, accessed via ssh + (c) Clusters running PBS, SGE, or Condor schedulers In configurations (b) and (c) Swift will launch its own workers, and then communicate using its own TCP protocol. + +Swift workers must be able to connect back to the Swift server on TCP +ports in the range of 30000 and higher. (FIXME: determine specifics). +If this is not available on a cluster (e.g., Merlot), then the pbsf +server will tunnel the Swift port over the standard ssh port, assuming +that is reachable. + } \section{INSTALLATION}{ @@ -93,116 +182,106 @@ export GLOBUS_HOSTNAME=10.0.0.200 # Eg for Merlot: internal address of the login node } -\section{QUICK START}{ +\section{QUICK_START}{ +\preformatted{ In a shell (outside of R) start the local Swift server: -$HOME/RLibrary/Swift/exec/swift-start local +$HOME/RLibrary/Swift/exec/swift-start local # +export R_LIBS=$HOME/RLibrary + R > require(Swift) ->basicSwiftTest() ->runAllSwiftTests() +> basicSwiftTest() # should take about 1 second +> runAllSwiftTests() # should take < 60 seconds -These will produce output similar to: -\preformatted{ +} -> require(Swift) -Loading required package: Swift -> basicSwiftTest() +} -*** Starting test 1.1 *** +\section{ENVIRONMENT_VARIABLES}{ -Test of local do.call(sumstuff) -local result= -[1] 4505 +If used, these variables must be exported in the UNIX environment in which +swift-start is executed: -Test of swiftapply(sumstuff,arglist) +SWIFTR_TMP sets the root directory below which Swift will maintain its +directory structure. Defaults to /tmp. For, e.g., the PADS cluster, +its best to set this to /scratch/local, as /tmp is very limited in +space. Seldom needed. -Swift properties: - swiftserver = local - callsperbatch = 1 - runmode = service - tmpdir = /tmp - workerhosts = localhost - initialize = initVar1 <<- 19; initVar2 <<- sqrt(400)+3 - -Swift request is in /tmp/wilde/SwiftR/requests.P04114/R0000000 -Swift result: -[[1]] -[1] 4505 - -==> test 1.1 passed -> +GLOBUS_HOSTNAME should be set to the IP address of the login host if +it contains multiple network interfaces (see /sbin/ifconfig) and if +the worker nodes can only reach the login host on a subset of these +interfaces. } -} +\section{START_SERVERS}{ -\section{CONFIGURE SERVERS}{ +To run swiftapply() and any of the swiftXapply() functions, you first +start one or more "Swift servers" on your local host (where you will +run the R client workspace. -edit configure-site-NAME in exec/ +Currently you must do this manually and in your login shell, outside +of R - BEFORE trying to run R Swift functions. If you run swiftapply() +without a Swift server running, your R session will hang and you will +need to kill it. This issue will be resolved shortly. -can put local cores into an ssh pool +The start-swift command (and all related shell scripts) are located in the installed package "exec" directry, so its handy to set a shell variable to point there: -swift $HOME/.ssh/auth-defaults file for additional shs servers +SWIFT=/Swift/ -access remote systems via ssh +Examples of starting the Swift server follow. -Export SWIFTR_TMP in your environment -} +To run N parallel R servers on the local host, one for each core: -\section{START SERVERS}{ +\verb{$SWIFT/exec/start-swift} -# do this outside of R - BEFORE trying to run R Swift functions +To run 4 R servers: -SWIFT=/Swift/ +\verb{$SWIFT/exec/start-swift -c 4} -$SWIFT/exec/start-swift local +To run 4 R servers on each of two hosts that can be reach by ssh: -or +\verb{$SIFT/exec/start-swift -s ssh -c 4 -h "hostname1 hostname2"} -$SWIFT/exec/start-swift pbsman +To run 8 R servers for 30 minutes on each of 3 nodes of the Merlot cluster, run this on the login host "merlot", using its "serial" queue: -or +\verb{$SWIFT/exec/start-swift -s pbsf -c 8 -n 3 -t 00:30:00 -q serial} -$SWIFT/exec/start-swift ssh host1 ... hostN - These Swift servers can be started and left running, across R runs options(swift.server="local") # or "pbsman" or "ssh" } -\section{HELLO WORLD TEST}{ +\section{TESTS}{ +Running a hello world test: + +\preformatted{ # Start swift local server as above require(Swift) basicSwiftTest() } -\section{RUN FULL TEST}{ +Running a full test -As a regular user: - +\preformatted{ require(Swift) -fullSwiftTest() +runAllSwiftTests() +} -# Then +Running full tests n times: -n=10 # 10 times through full test loop - +\preformatted{ testLoop(n) +} - -Testing from the source tree: - -source("Swift/tests/TestSwift.R") - -or R CMD TEST etc? FIXME } -\section{STOPPING SWIFT SERVERS}{ +\section{STOPPING_SWIFT_SERVERS}{ The following ps command is useful for displaying the many background swift processes. I keep this aliased as "mp" (my processes): @@ -256,14 +335,14 @@ } -\section{OPENMX EXAMPLES}{ +\section{OPENMX_EXAMPLES}{ This section is specific to users of the OpenMX R package for structural equation modeling. } -\section{USING ADDITIONAL PARALLEL ENVIRONMENTS}{ +\section{USING_OTHER_PARALLEL_ENVIRONMENTS}{ 3) ssh confiured for password-free login (to run on remote worker nodes) @@ -315,8 +394,21 @@ * manual mode +You can get Swift to stop after it has produced the call batch file +(an R save()'d object file. You can manually load this in another R +workspace, inspect the contents, manually onvoke the remote call, and +end the data back. This is useful if issues arise in the transparency +of function, argument, and retrun value marshalling, and if there are +concerns about the transparency of the remote execution. + * logs to look at +See the section on the directory structure of Swift services. + +"info" files returned from each execution can contain messages if the remote R jobs fail to launch. + +Each R server has its own log, on the host on which the server executes. + * is my swift server responding? tail -f $TMP/$USER/SwiftR/swift.local/swift.stdouterr @@ -382,7 +474,7 @@ killlall -u $USER. This can be made more precise to avoid killing jobs on shared worker nodes. -The following caveats are high prioiry on the FIXME list: +The following caveats are high priority on the FIXME list: You MUST start the Swift server before running a swiftapply() call from R. Otherwise R hangs and must be killed and restarted. @@ -390,24 +482,28 @@ When the FIFOs (named pipes) which are used to communicate from R to Swift get hung, you need to use kill or Quit to break out of R. -There is no automatic restart yet if swift dies in its server -loop. +There is no automatic restart yet if swift dies in its server loop. In +particular, parsing errors, eg on the Swift initialexpr text, can +cause the R and hence the Swift server to exit. The Only lapply is implemented (also SwiftApply) - need to see if we can cut down arg passing overhead for many of the apply() cases. Log records build up fast; these will be reduced as we get more -confidence with the code and shake out bugs +confidence with the code and shake out bugs, +There is no asynchronous swiftapply call yet. Each call must complete +before control is returned to the R command loop. + +Only one instance of each server type (i.e., local, pbs, pbsf, ssh) +can be started at a time. + Lower priority issues are: There is no easy way yet to alter Swift configuration file variables -such as number of cores to use, etc. Do this for now by editing an -existing configuration under Swift/exec/conigure-swift-NNN where NNN -is the Swift server name. -Each swiftapply() is pretty noisy - it echos its options, etc. This -will quiet down and "verbose" will be made an option. +Swift echoes its options on every call. It will be made silent and +"verbose" will be made an option. } From noreply at svn.ci.uchicago.edu Sun Oct 17 14:44:22 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 17 Oct 2010 14:44:22 -0500 (CDT) Subject: [Swift-commit] r3684 - trunk/libexec Message-ID: <20101017194422.5027C9CC86@svn.ci.uchicago.edu> Author: hategan Date: 2010-10-17 14:44:22 -0500 (Sun, 17 Oct 2010) New Revision: 3684 Modified: trunk/libexec/_swiftwrap Log: fixed argument passing when using parameter files Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-10-17 14:15:22 UTC (rev 3683) +++ trunk/libexec/_swiftwrap 2010-10-17 19:44:22 UTC (rev 3684) @@ -128,6 +128,16 @@ return 1 } +genScripts() { + echo "#!/bin/bash" > run.sh + echo -n "\"$EXEC\" " >> run.sh + for CMDARG in "${CMDARGS[@]}"; do + echo -n "\"$CMDARG\" " >> run.sh + done + echo "1>\"$STDOUT\" 2>\"$STDERR\"" >> run.sh + chmod +x run.sh +} + cdm_local_output() { L=$1 @@ -273,7 +283,7 @@ CMDARGS=("$line") FIRST=0 else - CMDARGS=("${CMDARGS[*]}" "$line") + CMDARGS=("${CMDARGS[@]}" "$line") fi done < <(grep -E "^-a " $PARAMFILE | cut -d " " -f 2-) else @@ -403,16 +413,12 @@ if [ "$KICKSTART" == "" ]; then if [ "$STDIN" == "" ]; then if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then - echo "#!/bin/bash" > run.sh - echo "\"$EXEC\" \"${CMDARGS[@]}\" 1>\"$STDOUT\" 2>\"$STDERR\"" >> run.sh - chmod +x run.sh + genScripts fi "$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" else if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then - echo "#!/bin/bash" > run.sh - echo "\"$EXEC\" \"${CMDARGS[@]}\" 1>\"$STDOUT\" 2>\"$STDERR\" <\"$STDIN\"" >> run.sh - chmod +x run.sh + genScripts fi "$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN" fi From noreply at svn.ci.uchicago.edu Sun Oct 17 21:46:15 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 17 Oct 2010 21:46:15 -0500 (CDT) Subject: [Swift-commit] r3685 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20101018024615.E4AD99CC9E@svn.ci.uchicago.edu> Author: hategan Date: 2010-10-17 21:46:15 -0500 (Sun, 17 Oct 2010) New Revision: 3685 Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java Log: attempt to fix deadlock Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2010-10-17 19:44:22 UTC (rev 3684) +++ trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2010-10-18 02:46:15 UTC (rev 3685) @@ -33,22 +33,24 @@ log(leaf, value); - synchronized (var.getRoot()) { // TODO want to do a type check here, for runtime type checking // and pull out the appropriate internal value from value if it // is a DSHandle. There is no need (I think? maybe numerical casting?) // for type conversion here; but would be useful to have // type checking. - synchronized (value.getRoot()) { - if (!value.isClosed()) { - throw new FutureNotYetAvailable(addFutureListener(stack, value)); - } - deepCopy(leaf, value, stack); + synchronized (value.getRoot()) { + if (!value.isClosed()) { + throw new FutureNotYetAvailable(addFutureListener(stack, value)); } + } + synchronized (var.getRoot()) { + deepCopy(leaf, value, stack); + if (var.getParent() != null && var.getParent().getType().isArray()) { markAsAvailable(stack, leaf.getParent(), leaf.getPathFromRoot().getLast()); } } + return null; } catch (FutureNotYetAvailable fnya) { From noreply at svn.ci.uchicago.edu Tue Oct 19 13:53:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 19 Oct 2010 13:53:24 -0500 (CDT) Subject: [Swift-commit] r3686 - trunk/tests/language/should-not-work Message-ID: <20101019185324.551679CC86@svn.ci.uchicago.edu> Author: skenny Date: 2010-10-19 13:53:24 -0500 (Tue, 19 Oct 2010) New Revision: 3686 Added: trunk/tests/language/should-not-work/118-keyword-misuse-stdout.swift Log: test for use of keyword stdout Added: trunk/tests/language/should-not-work/118-keyword-misuse-stdout.swift =================================================================== --- trunk/tests/language/should-not-work/118-keyword-misuse-stdout.swift (rev 0) +++ trunk/tests/language/should-not-work/118-keyword-misuse-stdout.swift 2010-10-19 18:53:24 UTC (rev 3686) @@ -0,0 +1,14 @@ + //THIS-SCRIPT-SHOULD-FAIL +type file {} + + +app (messagefile stdout, messagefile b) greeting(string m) { + echo m stdout=@filename(a) stderr=@filename(b); +} + +file e; +file f; + +(e,f) = greeting("hello world"); + + From noreply at svn.ci.uchicago.edu Sun Oct 24 22:12:21 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 Oct 2010 22:12:21 -0500 (CDT) Subject: [Swift-commit] r3687 - trunk/examples Message-ID: <20101025031221.B8D3E9CC86@svn.ci.uchicago.edu> Author: davidk Date: 2010-10-24 22:12:18 -0500 (Sun, 24 Oct 2010) New Revision: 3687 Modified: trunk/examples/iterate.swift trunk/examples/manyparam.swift trunk/examples/parameter.swift trunk/examples/q5.swift trunk/examples/restart.swift trunk/examples/second_procedure.swift trunk/examples/types.swift Log: Bug fixes to tutorial and updates for consistency Modified: trunk/examples/iterate.swift =================================================================== --- trunk/examples/iterate.swift 2010-10-19 18:53:24 UTC (rev 3686) +++ trunk/examples/iterate.swift 2010-10-25 03:12:18 UTC (rev 3687) @@ -1,15 +1,11 @@ type counterfile; -(counterfile t) echo(string m) { - app { +app (counterfile t) echo(string m) { echo m stdout=@filename(t); - } } -(counterfile t) countstep(counterfile i) { - app { +app (counterfile t) countstep(counterfile i) { wcl @filename(i) @filename(t); - } } counterfile a[] ; Modified: trunk/examples/manyparam.swift =================================================================== --- trunk/examples/manyparam.swift 2010-10-19 18:53:24 UTC (rev 3686) +++ trunk/examples/manyparam.swift 2010-10-25 03:12:18 UTC (rev 3687) @@ -1,9 +1,7 @@ type messagefile; -(messagefile t) greeting (string s) { - app { - echo s stdout=@filename(t); - } +app (messagefile t) greeting (string s) { + echo s stdout=@filename(t); } messagefile english <"english.txt">; Modified: trunk/examples/parameter.swift =================================================================== --- trunk/examples/parameter.swift 2010-10-19 18:53:24 UTC (rev 3686) +++ trunk/examples/parameter.swift 2010-10-25 03:12:18 UTC (rev 3687) @@ -1,9 +1,7 @@ type messagefile; -(messagefile t) greeting (string s) { - app { - echo s stdout=@filename(t); - } +app (messagefile t) greeting (string s) { + echo s stdout=@filename(t); } messagefile outfile <"hello2.txt">; Modified: trunk/examples/q5.swift =================================================================== --- trunk/examples/q5.swift 2010-10-19 18:53:24 UTC (rev 3686) +++ trunk/examples/q5.swift 2010-10-25 03:12:18 UTC (rev 3687) @@ -1,9 +1,7 @@ type messagefile; -(messagefile t) greeting (string s[]) { - app { - echo s[0] s[1] s[2] stdout=@filename(t); - } +app (messagefile t) greeting (string s[]) { + echo s[0] s[1] s[2] stdout=@filename(t); } messagefile outfile <"q5out.txt">; Modified: trunk/examples/restart.swift =================================================================== --- trunk/examples/restart.swift 2010-10-19 18:53:24 UTC (rev 3686) +++ trunk/examples/restart.swift 2010-10-25 03:12:18 UTC (rev 3687) @@ -1,27 +1,19 @@ type file; -(file f) touch() { - app { +app (file f) touch() { touch @f; - } } -(file f) processL(file inp) { - app { +app (file f) processL(file inp) { echo "processL" stdout=@f; - } } -(file f) processR(file inp) { - app { +app (file f) processR(file inp) { broken "process" stdout=@f; - } } -(file f) join(file left, file right) { - app { +app (file f) join(file left, file right) { echo "join" @left @right stdout=@f; - } } file f = touch(); Modified: trunk/examples/second_procedure.swift =================================================================== --- trunk/examples/second_procedure.swift 2010-10-19 18:53:24 UTC (rev 3686) +++ trunk/examples/second_procedure.swift 2010-10-25 03:12:18 UTC (rev 3687) @@ -1,15 +1,11 @@ -type messagefile {} +type messagefile; -(messagefile t) greeting (string s) { - app { - echo s stdout=@filename(t); - } +app (messagefile t) greeting (string s) { + echo s stdout=@filename(t); } -(messagefile o) capitalise(messagefile i) { - app { - tr "[a-z]" "[A-Z]" stdin=@filename(i) stdout=@filename(o); - } +app (messagefile o) capitalise(messagefile i) { + tr "[a-z]" "[A-Z]" stdin=@filename(i) stdout=@filename(o); } messagefile hellofile <"hello.txt">; Modified: trunk/examples/types.swift =================================================================== --- trunk/examples/types.swift 2010-10-19 18:53:24 UTC (rev 3686) +++ trunk/examples/types.swift 2010-10-25 03:12:18 UTC (rev 3687) @@ -1,14 +1,12 @@ -type messagefile {} +type messagefile; type details { string name; int pies; } -(messagefile t) greeting (details d) { - app { - echo "Hello. Your name is" d.name "and you have eaten" d.pies "pies." stdout=@filename(t); - } +app (messagefile t) greeting (details d) { + echo "Hello. Your name is" d.name "and you have eaten" d.pies "pies." stdout=@filename(t); } details person; From noreply at svn.ci.uchicago.edu Tue Oct 26 13:34:20 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 13:34:20 -0500 (CDT) Subject: [Swift-commit] r3688 - trunk/tests Message-ID: <20101026183420.A05B89CC86@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 13:34:20 -0500 (Tue, 26 Oct 2010) New Revision: 3688 Modified: trunk/tests/nightly.sh Log: Minor fixes Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-10-25 03:12:18 UTC (rev 3687) +++ trunk/tests/nightly.sh 2010-10-26 18:34:20 UTC (rev 3688) @@ -20,7 +20,7 @@ # and a *.timeout specifier # The scripts may setup and inspect files in RUNDIR including exec.out, # which must be accessed in stdout.txt, because the currently running -# tested process writes to exec.out, stdout.txt is a copy +# tested process writes to exec.out, stdout.txt is a copy. # The GROUP scripts can read the GROUP variable # The timeout number in the *.timeout file overrides the default # timeout @@ -151,7 +151,7 @@ BRANCH=trunk #BRANCH="branches/tests-01" -SCRIPTDIR=$( dirname $0 ) +SCRIPTDIR=$( cd $( dirname $0 ) ; /bin/pwd ) SWIFTCOUNT=0 From noreply at svn.ci.uchicago.edu Tue Oct 26 14:00:33 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 14:00:33 -0500 (CDT) Subject: [Swift-commit] r3689 - trunk/libexec Message-ID: <20101026190033.CDE569CC86@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 14:00:33 -0500 (Tue, 26 Oct 2010) New Revision: 3689 Modified: trunk/libexec/_swiftwrap Log: Enable basic MPICH/Coasters functionality Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-10-26 18:34:20 UTC (rev 3688) +++ trunk/libexec/_swiftwrap 2010-10-26 19:00:33 UTC (rev 3689) @@ -2,6 +2,9 @@ # this script must be invoked inside of bash, not plain sh # note that this script modifies $IFS +# Toggle debugging output from debug() +DEBUG=1 + infosection() { echo >& "$INFO" echo "_____________________________________________________________________________" >& "$INFO" @@ -12,6 +15,8 @@ } info() { + infosection "command line" + echo $COMMANDLINE 2>&1 >& "$INFO" infosection "uname -a" uname -a 2>&1 >& "$INFO" infosection "id" @@ -28,8 +33,6 @@ infosection "/proc/meminfo" cat /proc/meminfo 2>&1 >& "$INFO" fi - infosection "command line" - echo $COMMANDLINE 2>&1 >& "$INFO" if [ -f "$STDOUT" ] ; then infosection "stdout" cat $STDOUT >& "$INFO" @@ -48,6 +51,10 @@ echo "$@" >& "$INFO" } +debug() { + [ $DEBUG == 1 ] && echo "$@" >& "$INFO" +} + fail() { EC=$1 shift @@ -135,7 +142,7 @@ echo -n "\"$CMDARG\" " >> run.sh done echo "1>\"$STDOUT\" 2>\"$STDERR\"" >> run.sh - chmod +x run.sh + chmod +x run.sh } cdm_local_output() @@ -222,8 +229,13 @@ mkdir -p $INFODIR closeinfo -rm -f "$INFODIR/${ID}-info" -openinfo "$INFODIR/${ID}-info" +if [ -z $MPI_RANK ]; then + INFOFILE="$INFODIR/${ID}-info" +else + INFOFILE="$INFODIR/${ID}-${MPI_RANK}-info" +fi +rm -f $INFOFILE +openinfo "$INFOFILE" logstate "LOG_START" infosection "Wrapper" @@ -328,6 +340,7 @@ fi fi +debug "PID=$$" log "PWD=$PWD" log "DIR=$DIR" log "EXEC=$EXEC" @@ -349,6 +362,8 @@ checkError 254 "Failed to create job directory $DIR" log "Created job directory: $DIR" +if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then + logstate "CREATE_INPUTDIR" for D in $DIRS ; do mkdir -p "$DIR/$D" 2>&1 >>"$INFO" @@ -400,10 +415,17 @@ done fi +fi # MPI_RANK==0 + +debug "Moving to jobdir: $DIR" +cd $DIR +if [ $? != 0 ]; then + log "PWD: $PWD" + log $( find . ) + fail 254 "Could not cd to: $DIR" +fi logstate "EXECUTE" -cd $DIR - if [ ! -f "$EXEC" ]; then fail 254 "The executable $EXEC does not exist" fi @@ -447,12 +469,17 @@ fi fi +log "Moving back to workflow directory $WFDIR" cd $WFDIR +if [ $? != 0 ]; then + fail 254 "Could not cd to workflow directory: $WFDIR" +fi -log "Moving back to workflow directory $WFDIR" logstate "EXECUTE_DONE" log "Job ran successfully" +if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then + MISSING= for O in $OUTF ; do if [ ! -f "$DIR/$O" ]; then @@ -464,6 +491,7 @@ fi done if [ "$MISSING" != "" ]; then + log $( find . ) fail 254 "The following output files were not created by the application: $MISSING" fi @@ -487,12 +515,17 @@ touch $WFDIR/status/${JOBDIR}/${ID}-success fi +else + # Allow rank 0 to write output + sleep 1 +fi # MPI_RANK==0 + logstate "END" closeinfo if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then - mkdir -p "$WFDIR/info/$JOBDIR" + mkdir -p "$WFDIR/info/$JOBDIR" mv "$INFODIR/${ID}-info" "$WFDIR/info/$JOBDIR/${ID}-info" fi From noreply at svn.ci.uchicago.edu Tue Oct 26 14:09:07 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 14:09:07 -0500 (CDT) Subject: [Swift-commit] r3690 - usertools Message-ID: <20101026190907.89C849CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 14:09:06 -0500 (Tue, 26 Oct 2010) New Revision: 3690 Added: usertools/mpich/ Log: Location for MPICH patches From noreply at svn.ci.uchicago.edu Tue Oct 26 14:11:15 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 14:11:15 -0500 (CDT) Subject: [Swift-commit] r3691 - usertools/mpich Message-ID: <20101026191115.3342B9CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 14:11:15 -0500 (Tue, 26 Oct 2010) New Revision: 3691 Added: usertools/mpich/README.txt Log: Start on README Added: usertools/mpich/README.txt =================================================================== --- usertools/mpich/README.txt (rev 0) +++ usertools/mpich/README.txt 2010-10-26 19:11:15 UTC (rev 3691) @@ -0,0 +1,3 @@ +Contains the following patches: + +mpich2-1.3.diff From noreply at svn.ci.uchicago.edu Tue Oct 26 14:37:12 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 14:37:12 -0500 (CDT) Subject: [Swift-commit] r3692 - trunk/tests Message-ID: <20101026193712.56A629CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 14:37:11 -0500 (Tue, 26 Oct 2010) New Revision: 3692 Added: trunk/tests/mpi/ Log: Location for tests with MPICH/Coasters From noreply at svn.ci.uchicago.edu Tue Oct 26 14:47:15 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 14:47:15 -0500 (CDT) Subject: [Swift-commit] r3693 - trunk/tests/mpi Message-ID: <20101026194715.4CA3C9CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 14:47:15 -0500 (Tue, 26 Oct 2010) New Revision: 3693 Added: trunk/tests/mpi/100-input.txt trunk/tests/mpi/100-mci.check.sh trunk/tests/mpi/100-mci.clean.sh trunk/tests/mpi/100-mci.setup.sh trunk/tests/mpi/100-mci.swift trunk/tests/mpi/100-mpi-cp.c trunk/tests/mpi/sites.template.xml trunk/tests/mpi/tc.template.data Log: Import first MPI test (sketch) Added: trunk/tests/mpi/100-input.txt =================================================================== --- trunk/tests/mpi/100-input.txt (rev 0) +++ trunk/tests/mpi/100-input.txt 2010-10-26 19:47:15 UTC (rev 3693) @@ -0,0 +1 @@ +hello Added: trunk/tests/mpi/100-mci.check.sh =================================================================== --- trunk/tests/mpi/100-mci.check.sh (rev 0) +++ trunk/tests/mpi/100-mci.check.sh 2010-10-26 19:47:15 UTC (rev 3693) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +[ -f 100-output.txt ] || exit 1 + +CONTENTS1=$( cat 100-input.txt ) +CONTENTS2=$( cat 100-output.txt ) + +[[ $CONTENTS1 == $CONTENTS2 ]] || exit 1 + +exit 0 Property changes on: trunk/tests/mpi/100-mci.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/mpi/100-mci.clean.sh =================================================================== --- trunk/tests/mpi/100-mci.clean.sh (rev 0) +++ trunk/tests/mpi/100-mci.clean.sh 2010-10-26 19:47:15 UTC (rev 3693) @@ -0,0 +1,8 @@ +#!/bin/bash + +set -x + +rm -v 100-input.txt || exit 1 +rm -v 100-output.txt || exit 1 + +exit 0 Property changes on: trunk/tests/mpi/100-mci.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/mpi/100-mci.setup.sh =================================================================== --- trunk/tests/mpi/100-mci.setup.sh (rev 0) +++ trunk/tests/mpi/100-mci.setup.sh 2010-10-26 19:47:15 UTC (rev 3693) @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +which mpicc || exit 1 + +mpicc -std=gnu99 $GROUP/100-mpi-cp.c -o mpi-cp || exit 1 + +cp -v $GROUP/100-input.txt . || exit 1 + +exit 0 Property changes on: trunk/tests/mpi/100-mci.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/mpi/100-mci.swift =================================================================== --- trunk/tests/mpi/100-mci.swift (rev 0) +++ trunk/tests/mpi/100-mci.swift 2010-10-26 19:47:15 UTC (rev 3693) @@ -0,0 +1,12 @@ + +type file; + +app (file o) copy(file i) +{ + mpi_cp @i @o; +} + +file input<"100-input.txt">; +file output<"100-output.txt">; + +output = copy(input); Added: trunk/tests/mpi/100-mpi-cp.c =================================================================== --- trunk/tests/mpi/100-mpi-cp.c (rev 0) +++ trunk/tests/mpi/100-mpi-cp.c 2010-10-26 19:47:15 UTC (rev 3693) @@ -0,0 +1,135 @@ +/* + * Simple Hydra test + */ + +#include +#include +#include +#include +#include + +#include + +#define max(a, b) (((a) > (b)) ? (a) : (b)) +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +void read_input(const char *restrict filename, char** data, int* size); +void write_output(const char *restrict filename, char* data, int size); +void startup(void); + +const int chunk = 64*1024; + +int main(int argc, char* argv[]) +{ + int mpi_size, mpi_rank; + + MPI_Init(&argc, &argv); + + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + startup(); + + // system("/bin/hostname"); + // printf("size: %i\n", mpi_size); + // printf("rank: %i\n", mpi_rank); + + char* data; + int size; + + assert(argc == 3); + + if (mpi_rank == 0) + { + read_input(argv[1], &data, &size); + printf("size: %i\n", size); + MPI_Send(&size, 1, MPI_INT, + 1, 1, MPI_COMM_WORLD); + MPI_Send(data, size, MPI_CHAR, + 1, 1, MPI_COMM_WORLD); + } + else + { + MPI_Status status; + + MPI_Recv(&size, 1, MPI_INT, + 0, 1, MPI_COMM_WORLD, &status); + printf("size: %i\n", size); + + data = malloc(size); + assert(data); + MPI_Recv(data, size, MPI_CHAR, + 0, 1, MPI_COMM_WORLD, &status); + + write_output(argv[2], data, size); + } + + MPI_Finalize(); + return EXIT_SUCCESS; +} + +void startup() +{ + // system("/bin/hostname"); + // printf("size: %i\n", mpi_size); + // printf("rank: %i\n", mpi_rank); + char* pwd = getenv("PWD"); + printf("PWD: %s\n", pwd); + fflush(NULL); +} + +void read_input(const char* restrict filename, char** data, int* size) +{ + char* result; + FILE *file; + + printf("read: %s\n", filename); + + struct stat filestat; + + int error = stat(filename, &filestat); + assert(error == 0); + int s = filestat.st_size; + int actual, c; + + result = malloc(s); + + file = fopen(filename, "r"); + assert(file); + + int count = 0; + while (count < s) + { + c = min(chunk, s-count); + actual = fread(result+count, 1, c, file); + count += actual; + } + + fclose(file); + + *size = s; + *data = result; +} + +void write_output(const char* restrict filename, + char* data, int size) +{ + FILE *file; + + int actual, c; + + printf("write: %s\n", filename); + + file = fopen(filename, "w"); + assert(file); + + int count = 0; + while (count < size) + { + c = min(chunk, size-count); + actual = fwrite(data+count, 1, c, file); + count += actual; + } + + fclose(file); +} Added: trunk/tests/mpi/sites.template.xml =================================================================== --- trunk/tests/mpi/sites.template.xml (rev 0) +++ trunk/tests/mpi/sites.template.xml 2010-10-26 19:47:15 UTC (rev 3693) @@ -0,0 +1,24 @@ + + + + + + _WORK_ + file + + + + + + _HOST_ + 2.55 + 10000 + 4 + 8 + 1000 + 1 + 4 + _WORK_ + + + Added: trunk/tests/mpi/tc.template.data =================================================================== --- trunk/tests/mpi/tc.template.data (rev 0) +++ trunk/tests/mpi/tc.template.data 2010-10-26 19:47:15 UTC (rev 3693) @@ -0,0 +1,24 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null +localhost cp /bin/cp INSTALLED INTEL32::LINUX null + +# hydra-tests-2 + +coasterslocal mpi_cp _PWD_/mpi-cp INSTALLED INTEL32::LINUX globus::hostCount=2 + +# coasterslocal mpi_cp /bin/false INSTALLED INTEL32::LINUX globus::hostCount=2 From noreply at svn.ci.uchicago.edu Tue Oct 26 14:48:45 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 14:48:45 -0500 (CDT) Subject: [Swift-commit] r3694 - trunk/tests/mpi Message-ID: <20101026194845.20AB49CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 14:48:44 -0500 (Tue, 26 Oct 2010) New Revision: 3694 Added: trunk/tests/mpi/README.txt Log: New README Added: trunk/tests/mpi/README.txt =================================================================== --- trunk/tests/mpi/README.txt (rev 0) +++ trunk/tests/mpi/README.txt 2010-10-26 19:48:44 UTC (rev 3694) @@ -0,0 +1,6 @@ + +Test MPICH/Coasters integration + +Be sure the correct MPICH is in your PATH before running nightly.sh + +See: http://www.ci.uchicago.edu/wiki/bin/view/SWFT/CoastersMpi From noreply at svn.ci.uchicago.edu Tue Oct 26 14:49:21 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 14:49:21 -0500 (CDT) Subject: [Swift-commit] r3695 - usertools/mpich Message-ID: <20101026194921.599D89CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 14:49:20 -0500 (Tue, 26 Oct 2010) New Revision: 3695 Modified: usertools/mpich/README.txt Log: Refer to wiki page Modified: usertools/mpich/README.txt =================================================================== --- usertools/mpich/README.txt 2010-10-26 19:48:44 UTC (rev 3694) +++ usertools/mpich/README.txt 2010-10-26 19:49:20 UTC (rev 3695) @@ -1,3 +1,5 @@ Contains the following patches: mpich2-1.3.diff + +See: http://www.ci.uchicago.edu/wiki/bin/view/SWFT/CoastersMpi From noreply at svn.ci.uchicago.edu Tue Oct 26 15:03:19 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 15:03:19 -0500 (CDT) Subject: [Swift-commit] r3696 - usertools/mpich Message-ID: <20101026200319.3D6F79CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-26 15:03:18 -0500 (Tue, 26 Oct 2010) New Revision: 3696 Added: usertools/mpich/mpich2-1.3.diff Log: MPICH patch for 1.3 Added: usertools/mpich/mpich2-1.3.diff =================================================================== --- usertools/mpich/mpich2-1.3.diff (rev 0) +++ usertools/mpich/mpich2-1.3.diff 2010-10-26 20:03:18 UTC (rev 3696) @@ -0,0 +1,99 @@ +Index: src/pm/hydra/tools/bootstrap/external/external_init.c +=================================================================== +--- src/pm/hydra/tools/bootstrap/external/external_init.c (revision 7377) ++++ src/pm/hydra/tools/bootstrap/external/external_init.c (working copy) +@@ -89,3 +89,8 @@ + { + return external_init(); + } ++ ++HYD_status HYDT_bsci_manual_init(void) ++{ ++ return external_init(); ++} +Index: src/pm/hydra/tools/bootstrap/external/external_launch.c +=================================================================== +--- src/pm/hydra/tools/bootstrap/external/external_launch.c (revision 7377) ++++ src/pm/hydra/tools/bootstrap/external/external_launch.c (working copy) +@@ -268,6 +268,12 @@ + HYDU_print_strlist(targs + offset); + } + ++ if (!strcmp(HYDT_bsci_info.bootstrap, "manual")) { ++ printf("HYDRA_MANUAL_LINE: "); ++ HYDU_print_strlist(targs + offset); ++ continue; ++ } ++ + /* The stdin pointer will be some value for process_id 0; for + * everyone else, it's a dummy value. We don't just pass it + * NULL, as older versions of ssh seem to freak out when no +@@ -304,6 +310,13 @@ + HYDU_ERR_POP(status, "demux returned error registering fd\n"); + } + ++ fprintf(stderr, "%c\n", 'A'); ++ ++ if (!strcmp(HYDT_bsci_info.bootstrap, "manual")) { ++ printf("HYDRA_MANUAL_DONE:\n"); ++ fflush(stdout); ++ } ++ + fn_exit: + HYDU_free_strlist(targs); + if (path) +Index: src/pm/hydra/configure.in +=================================================================== +--- src/pm/hydra/configure.in (revision 7377) ++++ src/pm/hydra/configure.in (working copy) +@@ -146,7 +146,7 @@ + ######################################################################### + AC_ARG_WITH(hydra-bss, [ --with-hydra-bss=name - Boot-strap Server (ssh,rsh,fork,slurm,ll,lsf,sge,persist)], + [ hydra_bss=$withval ], +- [ hydra_bss="ssh,rsh,fork,slurm,ll,lsf,sge,persist" ]) ++ [ hydra_bss="ssh,rsh,fork,slurm,ll,lsf,sge,persist,manual" ]) + AC_MSG_CHECKING(boot-strap server) + AC_MSG_RESULT($hydra_bss) + hydra_bss_names="`echo $hydra_bss | sed -e 's/:/ /g' -e 's/,/ /g'`" +@@ -187,6 +187,10 @@ + hydra_bss_persist=true + available_bss=`echo $available_bss persist` + ;; ++ manual) ++ hydra_bss_manual=true ++ available_bss=`echo $available_bss manual` ++ ;; + *) + ;; + esac +@@ -377,7 +381,7 @@ + have_plpa=yes + AC_DEFINE(HAVE_PLPA,1,[Define if plpa is available]) + fi +- ;; ++ ;; + hwloc) + if test "$with_hwloc_prefix" = "embedded" ; then + PAC_CONFIG_SUBDIR([tools/bind/hwloc/hwloc],[have_hwloc=yes],[have_hwloc=no]) +@@ -451,7 +455,7 @@ + AC_DEFINE(HAVE_BLCR,1,[Define if blcr is available]) + available_ckpointlib=`echo $available_ckpointlib blcr` + fi +- ;; ++ ;; + *) + ;; + esac +Index: src/pm/hydra/pm/pmiserv/pmiserv_utils.c +=================================================================== +--- src/pm/hydra/pm/pmiserv/pmiserv_utils.c (revision 7377) ++++ src/pm/hydra/pm/pmiserv/pmiserv_utils.c (working copy) +@@ -90,7 +90,7 @@ + } + + proxy_args[arg++] = HYDU_strdup("--enable-stdin"); +- proxy_args[arg++] = HYDU_int_to_str(enable_stdin); ++ proxy_args[arg++] = HYDU_int_to_str(0); // (enable_stdin); + + proxy_args[arg++] = HYDU_strdup("--proxy-id"); + proxy_args[arg++] = NULL; From noreply at svn.ci.uchicago.edu Tue Oct 26 16:59:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 16:59:32 -0500 (CDT) Subject: [Swift-commit] r3698 - trunk/docs/guides Message-ID: <20101026215932.14D699CC8D@svn.ci.uchicago.edu> Author: skenny Date: 2010-10-26 16:59:31 -0500 (Tue, 26 Oct 2010) New Revision: 3698 Modified: trunk/docs/guides/quickstartguide.php Log: code for reallyquickstartguide now saved, commented out, in the quickstartguide Modified: trunk/docs/guides/quickstartguide.php =================================================================== --- trunk/docs/guides/quickstartguide.php 2010-10-26 21:30:39 UTC (rev 3697) +++ trunk/docs/guides/quickstartguide.php 2010-10-26 21:59:31 UTC (rev 3698) @@ -1,5 +1,5 @@ -Swift Quick Start Guide +Swift Quick Start Guide
@@ -18,17 +18,14 @@
-

Swift Quick Start Guide

Source Control $LastChangedRevision: 2810 $

Abstract

? +

Swift Quick start Guide

Source Control $LastChangedRevision: 2810 $

Abstract

? - The impatient may find the Swift Really Quick Start - Guide to be more convenient. + This guide describes the steps needed to download, install, + configure, and run the basic examples for Swift. If you are + using a pre-installed version of Swift, you can skip + directly to the configuration + section. - This guide describes the steps needed to download, install, - configure, and run the basic examples for Swift. If you are - using a pre-installed version of Swift, you can skip - directly to the configuration - section. -

Table of Contents

1. Downloading a Swift Distribution
1.1. Stable Releases
1.2. Nightly Builds
1.3. Source Repository
2. Installing a Swift Binary Package
3. Configuring Swift
3.1. Grid Security
3.2. Swift Properties
4. Running Swift Examples

1.?Downloading a Swift Distribution

There are three main ways of getting the Swift implementation: stable releases, nightly builds, and the source code repository. @@ -190,3 +187,97 @@ var pageTracker = _gat._getTracker("UA-106257-5"); pageTracker._trackPageview(); } catch(err) {} + + +

+ + + + + + + +
+ +

Swift Really Quick Start Guide

Source control $LastChangedRevision: 2646 $

Abstract

? + + This guide is a compressed version of the Swift Quick Start + Guide. + +

Table of Contents

1. Swift Really Quick Start Guide

1.?Swift Really Quick Start Guide

  • + + Download + Swift. + +

  • + + Unpack it and add the swift-xyz/bin directory to your + PATH. + +

  • + + Make sure you have your user certificate, a valid GSI proxy + certificate, and the proper CA root certificates in either + ~/.globus/certificates or + /etc/grid-security/certificates. + +

  • + + Edit swift-xyz/etc/swift.properties. You + should add your numeric IP address there + (ip.address=x.y.z.w). + +

  • + + Use the example site catalog and transformation catalog (they + are configured for local submission): + +

    +cd swift-xyz/etc
+cp sites.xml.example sites.xml
+cp tc.data.example tc.data
+

    + +

  • + + Use swift file.dtm to compile and execute + file.dtm. + +

  • + + Use swift -resume file-<runid>.?.rlog + file.dtm to resume a failed run. + +

+
+ + + + + +
+ + + + +--> From noreply at svn.ci.uchicago.edu Tue Oct 26 17:04:30 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 26 Oct 2010 17:04:30 -0500 (CDT) Subject: [Swift-commit] r3699 - trunk/docs/guides Message-ID: <20101026220430.7D89D9CC8D@svn.ci.uchicago.edu> Author: skenny Date: 2010-10-26 17:04:30 -0500 (Tue, 26 Oct 2010) New Revision: 3699 Modified: trunk/docs/guides/quickstartguide.php Log: removed reallyquickstartguide code and link Modified: trunk/docs/guides/quickstartguide.php =================================================================== --- trunk/docs/guides/quickstartguide.php 2010-10-26 21:59:31 UTC (rev 3698) +++ trunk/docs/guides/quickstartguide.php 2010-10-26 22:04:30 UTC (rev 3699) @@ -187,97 +187,3 @@ var pageTracker = _gat._getTracker("UA-106257-5"); pageTracker._trackPageview(); } catch(err) {} - - -
- - - - - - - -
- -

Swift Really Quick Start Guide

Source control $LastChangedRevision: 2646 $

Abstract

? - - This guide is a compressed version of the Swift Quick Start - Guide. - -

Table of Contents

1. Swift Really Quick Start Guide

1.?Swift Really Quick Start Guide

  • - - Download - Swift. - -

  • - - Unpack it and add the swift-xyz/bin directory to your - PATH. - -

  • - - Make sure you have your user certificate, a valid GSI proxy - certificate, and the proper CA root certificates in either - ~/.globus/certificates or - /etc/grid-security/certificates. - -

  • - - Edit swift-xyz/etc/swift.properties. You - should add your numeric IP address there - (ip.address=x.y.z.w). - -

  • - - Use the example site catalog and transformation catalog (they - are configured for local submission): - -

    -cd swift-xyz/etc
-cp sites.xml.example sites.xml
-cp tc.data.example tc.data
-

    - -

  • - - Use swift file.dtm to compile and execute - file.dtm. - -

  • - - Use swift -resume file-<runid>.?.rlog - file.dtm to resume a failed run. - -

-
- - - - - -
- - - - ---> From noreply at svn.ci.uchicago.edu Wed Oct 27 13:24:01 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 27 Oct 2010 13:24:01 -0500 (CDT) Subject: [Swift-commit] r3700 - trunk/tests Message-ID: <20101027182401.D94959CC86@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-27 13:24:01 -0500 (Wed, 27 Oct 2010) New Revision: 3700 Modified: trunk/tests/nightly.sh Log: Check for errors with ifconfig Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-10-26 22:04:30 UTC (rev 3699) +++ trunk/tests/nightly.sh 2010-10-27 18:24:01 UTC (rev 3700) @@ -760,8 +760,15 @@ out package "swift-$DATE.tar.gz" } -GLOBUS_HOSTNAME=$( ifconfig | grep inet | head -1 | cut -d ':' -f 2 | \ +if which ifconfig > /dev/null; then + IFCONFIG=ifconfig +else + IFCONFIG=/sbin/ifconfig +fi +$IFCONFIG > /dev/null || crash "Cannot run ifconfig!" +GLOBUS_HOSTNAME=$( $IFCONFIG | grep inet | head -1 | cut -d ':' -f 2 | \ awk '{print $1}' ) +[ $? != 0 ] && crash "Could not obtain GLOBUS_HOSTNAME!" group_sites_xml() { TEMPLATE=$GROUP/sites.template.xml if [ -f $TEMPLATE ]; then From noreply at svn.ci.uchicago.edu Wed Oct 27 14:46:49 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 27 Oct 2010 14:46:49 -0500 (CDT) Subject: [Swift-commit] r3701 - trunk/tests Message-ID: <20101027194649.B58999CC95@svn.ci.uchicago.edu> Author: wozniak Date: 2010-10-27 14:46:49 -0500 (Wed, 27 Oct 2010) New Revision: 3701 Modified: trunk/tests/nightly.sh Log: Clarifications Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-10-27 18:24:01 UTC (rev 3700) +++ trunk/tests/nightly.sh 2010-10-27 19:46:49 UTC (rev 3701) @@ -8,8 +8,8 @@ # Run nightly.sh -h for quick help # When something goes wrong, find and check tests.log or use -v -# Code is checked out into TOPDIR -# Swift is installed in its source tree +# Code is checked out into TOPDIR (PWD by default) (PATH is not used) +# Swift is compiled and installed in its source tree # The run is executed in RUNDIR (TOPDIR/RUNDIRBASE) # The build test is started in TOPDIR # Everything for a Swift test is written in its RUNDIR From noreply at svn.ci.uchicago.edu Thu Oct 28 18:13:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 28 Oct 2010 18:13:24 -0500 (CDT) Subject: [Swift-commit] r3702 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20101028231324.C04BFFAC2@svn.ci.uchicago.edu> Author: hategan Date: 2010-10-28 18:13:24 -0500 (Thu, 28 Oct 2010) New Revision: 3702 Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: disabled console debugger Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-10-27 19:46:49 UTC (rev 3701) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-10-28 23:13:24 UTC (rev 3702) @@ -381,7 +381,7 @@ if (hash == null) { hash = new WrapperMap(); stack.firstFrame().setVar(VDL_FUTURE_WRAPPER_MAP, hash); - InHook.install(new Monitor(hash)); + //InHook.install(new Monitor(hash)); } return hash; } @@ -485,7 +485,7 @@ } protected static void closeShallow(VariableStack stack, DSHandle handle) throws ExecutionException { - synchronized(handle.getRoot()) { + synchronized (handle.getRoot()) { handle.closeShallow(); getFutureWrapperMap(stack).close(handle); } From noreply at svn.ci.uchicago.edu Sat Oct 30 16:33:13 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 30 Oct 2010 16:33:13 -0500 (CDT) Subject: [Swift-commit] r3703 - trunk/bin Message-ID: <20101030213313.E0DF79CC8D@svn.ci.uchicago.edu> Author: davidk Date: 2010-10-30 16:33:13 -0500 (Sat, 30 Oct 2010) New Revision: 3703 Modified: trunk/bin/swift Log: Updated swift shell script to use bash sockets for usage tracking rather than netcat Modified: trunk/bin/swift =================================================================== --- trunk/bin/swift 2010-10-28 23:13:24 UTC (rev 3702) +++ trunk/bin/swift 2010-10-30 21:33:13 UTC (rev 3703) @@ -116,11 +116,11 @@ SWIFT_USAGE_STATS=0 fi -NC=`which nc 2>&1` -if [ ! -x "$NC" ]; then - NC=`which netcat 2>&1` - if [ ! -x "$NC" ]; then - echo "warning: unable to find nc" +BASH="/bin/bash" +if [ ! -x "$BASH" ]; then + BASH=`which bash 2>&1` + if [ ! -x "$BASH" ]; then + echo "warning: unable to find bash" SWIFT_USAGE_STATS=0 fi fi @@ -139,7 +139,7 @@ USERID=`echo $USERID $HOSTNAME $MAC | $MD5SUM |cut -d ' ' -f1` DATE=`date` KEY=`echo $USERID $DATE $$ | $MD5SUM | cut -d ' ' -f1` - echo $KEY $USERID $SCRIPTID $SCRIPTLENGTH -1 0| $NC -w 10 -n -u $LISTENER $PORT & + $BASH -c "echo $KEY $USERID $SCRIPTID $SCRIPTLENGTH -1 0 > /dev/udp/$LISTENER/$PORT" & fi ### EXECUTE ############ @@ -154,7 +154,7 @@ return_code=$? if [ "$SWIFT_USAGE_STATS" = 1 ]; then - echo $KEY $USERID $SCRIPTID $SCRIPTLENGTH $return_code 1| $NC -w 10 -n -u $LISTENER $PORT & + $BASH -c "echo $KEY $USERID $SCRIPTID $SCRIPTLENGTH $return_code 1 > /dev/udp/$LISTENER/$PORT" & fi exit $return_code