[Swift-commit] r3655 - in SwiftApps/SwiftR: . Swift/R Swift/exec Swift/tests
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Fri Oct 1 19:18:26 CDT 2010
Author: wilde
Date: 2010-10-01 19:18:26 -0500 (Fri, 01 Oct 2010)
New Revision: 3655
Added:
SwiftApps/SwiftR/Swift/exec/configure-site-local
SwiftApps/SwiftR/Swift/exec/configure-site-pbs
SwiftApps/SwiftR/Swift/exec/start-swift
Modified:
SwiftApps/SwiftR/Swift/R/Swift.R
SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
SwiftApps/SwiftR/Swift/exec/rserver.swift
SwiftApps/SwiftR/Swift/tests/TestSwift.R
SwiftApps/SwiftR/TODO
Log:
Start of changes to split out sites into separate servers. Added mutex to EvalRBatchPersistent for local provider throttling problems. Simplified reserver.swift back down to a single site. configure-pbs is preliminary version, not yet working.
Modified: SwiftApps/SwiftR/Swift/R/Swift.R
===================================================================
--- SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-01 20:48:46 UTC (rev 3654)
+++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-10-02 00:18:26 UTC (rev 3655)
@@ -1,11 +1,11 @@
-swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL, runmode=NULL, initialize=NULL, workerhosts=NULL, keepwork=NULL )
+swiftapply <- function( func, arglists, swiftserver=NULL, callsperbatch=NULL, runmode=NULL, initialize=NULL, workerhosts=NULL, keepwork=NULL )
{
# Set Swift default options if not passed as keyword paramets or pre-set by user
- if(is.null(site))
- site <- getOption("swift.site")
- if(is.null(site))
- site<-"local"
+ if(is.null(swiftserver))
+ site <- getOption("swift.server")
+ if(is.null(swiftserver))
+ swiftserver<-"local"
if(is.null(callsperbatch))
callsperbatch <- getOption("swift.callsperbatch")
@@ -36,7 +36,7 @@
keepwork <- FALSE;
cat("\nSwift properties:\n")
- cat(" site =",site,"\n");
+ cat(" swiftserver =", swiftserver,"\n");
cat(" callsperbatch =", callsperbatch,"\n")
cat(" runmode =", runmode,"\n")
cat(" initialize =", initialize,"\n")
@@ -45,15 +45,32 @@
user <- Sys.info()[["user"]]
+ # Initialize globals if first call in this workspace
+
+ requestdirbase = mget("swift.requestdirbase",envir=globalenv(),ifnotfound=list(NULL))[[1]] # FIXME: state more elegantly?
+ if(is.null(requestdirbase)) {
+ requestdirbase = sprintf("/tmp/%s/SwiftR/requests.P%.5d",user,Sys.getpid())
+ dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE)
+ assign("swift.requestdirbase",requestdirbase,globalenv())
+ requestid = 0;
+ assign("swift.requestid",0,globalenv())
+ }
+ else {
+ requestid = get("swift.requestid",envir=globalenv()) + 1;
+ assign("swift.requestid",requestid,globalenv())
+ }
+
# Execute the calls in batches
# rundir <- system("mktemp -d SwiftR.run.XXXX",intern=TRUE) # FIXME: put this in /tmp/$USER/SwiftR/requests
- basedir = paste("/tmp/",user,"/SwiftR/requests",sep="")
- dir.create(basedir,recursive=TRUE,showWarnings=FALSE)
- reqdir <- system(paste("mktemp -d ", basedir, "/SwiftR.run.XXXX",sep=""),intern=TRUE)
+# basedir = paste("/tmp/",user,"/SwiftR/requests",sep="")
+# reqdir <- system(paste("mktemp -d ", basedir, "/SwiftR.run.XXXX",sep=""),intern=TRUE)
+ reqdir = sprintf("%s/R%.7d",requestdirbase,requestid)
+ dir.create(reqdir,showWarnings=FALSE)
cat("Swift request is in",reqdir,"\n")
- cat(initialize,file=paste(reqdir,"/initialize.R",sep="")) # FIXME: honor this???
+ cat(initialize,file=paste(reqdir,"/initialize.R",sep="")) # FIXME: honor this on calls after the first in an R server?
+ # FIXME: eliminate this oevrhead on each call to swiftapply
narglists <- length(arglists) # number of arglists to process
batch <- 1 # Next arglist batch number to fill
@@ -81,7 +98,7 @@
if( runmode == "manual" ) { # Prompt for return (empty line) to continue; assumes user ran a manual R to process the call.
cat("Manual Swift Run:\n run dir: ", getwd(), "/", reqdir,"\n\n")
cat(" swift script: ", RunSwiftScript, "\n")
- cat(" site: ", site,"\n")
+ cat(" swiftserver: ", swiftserver,"\n")
cat(" swiftapplyScript: ", swiftapplyScript,"\n")
cat(" Use RunAllR.sh to process and press return when complete:")
system(paste("cp ", system.file(package="Swift","exec/RunAllR.sh"), reqdir))
@@ -89,13 +106,13 @@
}
else if (runmode == "script") {
RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh")
- system(paste(RunSwiftScript,reqdir,site,swiftapplyScript,"\"",workerhosts,"\""))
+ system(paste(RunSwiftScript,reqdir,swiftserver,swiftapplyScript,"\"",workerhosts,"\""))
}
else { # runmode == "service" # FIXME: check and post error if not "service"
# Send request to service
- swiftServerDir = paste("/tmp/",user,"/SwiftR/swiftserver",sep="")
+ swiftServerDir = paste("/tmp/",user,"/SwiftR/swift.",swiftserver,sep="")
cat(initialize,file=paste(swiftServerDir,"/initialize.R",sep="")) # FIXME
Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-01 20:48:46 UTC (rev 3654)
+++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-10-02 00:18:26 UTC (rev 3655)
@@ -95,6 +95,15 @@
# Ready to talk to the server: send request and read response
+while true; do
+ mkdir $SLOTDIR/mutex
+ if [ $? != 0 ]; then
+ sleep 1;
+ else
+ break;
+ fi
+done
+
echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo
touch $SLOTDIR/lastwrite
@@ -102,4 +111,6 @@
head -3 < $SLOTDIR/fromR.fifo # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit")
+rmdir $SLOTDIR/mutex
+
# Fixme: how to get exceptions and stdout/stderr text from R server ???
\ No newline at end of file
Added: SwiftApps/SwiftR/Swift/exec/configure-site-local
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-site-local (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/configure-site-local 2010-10-02 00:18:26 UTC (rev 3655)
@@ -0,0 +1,61 @@
+#! /bin/bash
+
+throttleOneCore="-0.001"
+localcores=5 # FIXME: parameterize: localthreads=N
+
+#### DBDBDBDB vvvvvv
+#localcores=1
+
+cat >tc <<END
+fork bashlocal /bin/bash null null null
+END
+
+cat >sites.xml <<END
+<config>
+ <pool handle="fork">
+ <execution provider="local" url="none" />
+ <profile key="jobThrottle" namespace="karajan">0.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" url="none" />
+ <workdirectory>$(pwd)/swiftwork</workdirectory>
+ </pool>
+END
+
+for i in `seq -w 0 $((localcores-1))`; do
+
+# FIXME: how to get the right PATH for R in here?
+# use /bin/sh and count on users PATH?
+# then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename)
+
+cat >>tc <<END
+local${i} bash /bin/bash null null ENV::PATH="$PATH";ENV::SWIFT_JOB_SLOT="${i}";ENV::SWIFT_WORKER_PID="$$"
+END
+
+cat >>sites.xml <<END
+ <pool handle="local${i}">
+ <execution provider="local" url="none" />
+ <profile key="jobThrottle" namespace="karajan">$throttleOneCore</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" url="none" />
+ <workdirectory>$(pwd)/swiftwork</workdirectory>
+ </pool>
+END
+
+done
+
+# <profile namespace="karajan" key="jobsPerCpu">1</profile>
+
+echo '</config>' >>sites.xml
+
+cat >cf <<END
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+lazy.errors=false
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
+throttle.host.submit=1
+END
+
+
Property changes on: SwiftApps/SwiftR/Swift/exec/configure-site-local
___________________________________________________________________
Name: svn:executable
+ *
Added: SwiftApps/SwiftR/Swift/exec/configure-site-pbs
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-site-pbs (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/configure-site-pbs 2010-10-02 00:18:26 UTC (rev 3655)
@@ -0,0 +1,60 @@
+#! /bin/bash
+
+throttlePBS=.31
+
+cat >tc <<END
+fork bashlocal /bin/bash null null null
+pbscoast bash /bin/bash null null ENV::PATH="$PATH";ENV::SWIFT_JOB_SLOT="${i}";ENV::SWIFT_WORKER_PID="$$"
+END
+
+cat >sites.xml <<END
+<config>
+ <pool handle="fork">
+ <execution provider="local" url="none" />
+ <profile key="jobThrottle" namespace="karajan">0.15</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" url="none" />
+ <workdirectory>$(pwd)/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="pbsdirect">
+ <execution provider="pbs" url="none" />
+ <profile namespace="globus" key="queue">fast</profile>
+ <profile namespace="globus" key="maxwalltime">00:59:00</profile>
+ <profile key="jobThrottle" namespace="karajan">$throttlePBS</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" url="none" />
+ <workdirectory>$HOME/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="pbscoast">
+ <execution provider="coaster" url="none" jobmanager="local:pbs"/>
+
+ <profile namespace="globus" key="queue">fast</profile>
+ <profile namespace="globus" key="maxTime">3500</profile>
+ <profile namespace="globus" key="maxWallTime">00:00:01</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="workersPerNode">1</profile>
+ <profile namespace="karajan" key="jobThrottle">2.55</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+
+ <filesystem provider="local" url="none"/>
+ <workdirectory>/home/wilde/swiftwork</workdirectory>
+ </pool>
+</config>
+END
+
+cat >cf <<END
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+lazy.errors=false
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
+throttle.host.submit=1
+END
+
+
Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/rserver.swift 2010-10-01 20:48:46 UTC (rev 3654)
+++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2010-10-02 00:18:26 UTC (rev 3655)
@@ -5,31 +5,11 @@
# needs to select between EvalRPersistent and simple RunR
# enable persistent for local mode (not sure how)
-app (external e, RData result, file stout, file sterr) runRdefault (file shellscript, file initRScript, file RServerScript, RData rcall)
+app (external e, RData result, file stout, file sterr) runR (file shellscript, file initRScript, file RServerScript, RData rcall)
{
- bashdefault @shellscript @initRScript @RServerScript @rcall @result stdout=@stout stderr=@sterr;
+ bash @shellscript @initRScript @RServerScript @rcall @result stdout=@stout stderr=@sterr;
}
-app (external e, RData result, file stout, file sterr) runRlocal (file shellscript, file initRScript, file RServerScript, RData rcall)
-{
- bashlocal @shellscript @initRScript @RServerScript @rcall @result stdout=@stout stderr=@sterr;
-}
-
-app (external e, RData result, file stout, file sterr) runRpbs (file shellscript, file initRScript, file RServerScript, RData rcall)
-{
- bashpbs @shellscript @initRScript @RServerScript @rcall @result stdout=@stout stderr=@sterr;
-}
-
-app (external e, RData result, file stout, file sterr) runRservice (file shellscript, file initRScript, file RServerScript, RData rcall)
-{
- bashservice @shellscript @initRScript @RServerScript @rcall @result stdout=@stout stderr=@sterr;
-}
-
-app (external e, RData result, file stout, file sterr) RunRv1 (file shellscript, RData rcall)
-{
- bash @shellscript @rcall @result stdout=@stout stderr=@sterr;
-}
-
app ack (external e[])
{
# bash "-c" "echo SIGNAL DONE >/dev/tty; echo done > /tmp/SwiftR/swiftserver/resultpipe";
@@ -47,22 +27,8 @@
file initScript <"initialize.R">;
file rsScript <"SwiftRServer.sh">;
- string location=@arg("location","local"); # FIXME: pass the execution site in as an argument (selects tc entry)
-
foreach c, i in rcalls {
- # FIXME: swift needs a better way to express the dynamic function selection below
- if(location=="default") {
- (e[i], results[i],stout[i], sterr[i]) = runRdefault(runRscript,initScript,rsScript,c);
- }
- if(location=="local") {
- (e[i], results[i],stout[i], sterr[i]) = runRlocal(runRscript,initScript,rsScript,c);
- }
- if(location=="service") {
- (e[i], results[i],stout[i], sterr[i]) = runRservice(runRscript,initScript,rsScript,c);
- }
- if(location=="pbs") {
- (e[i], results[i],stout[i], sterr[i]) = runRpbs(runRscript,initScript,rsScript,c);
- }
+ (e[i], results[i],stout[i], sterr[i]) = runR(runRscript,initScript,rsScript,c);
}
}
@@ -74,7 +40,7 @@
boolean done;
string dir;
-# FIXME: read swiftserver dir via @args
+ # FIXME: read swiftserver dir via @args
dir = readData(requestPipeName); # Reads direct from this local pipe. Assumes Swift started in right dir.
trace("rserver: got dir", dir);
@@ -84,10 +50,11 @@
if (dir=="done") { done=true; } else { done=false;}
-# file f<"/tmp/SwiftR/swiftserver/resultpipe">;
-# f = writeData("completed\n"); # Doesnt work: runs as soon as the block is entered.
-# want: tracef("%k completed\n", "my/responsepipe.fifo", wait);
-# %k waits for wait to be fully closed; then sends formatted string to specified file(doing open, write, close).
+ # file f<"/tmp/SwiftR/swiftserver/resultpipe">;
+ # f = writeData("completed\n"); # Doesnt work: runs as soon as the block is entered.
+ # want: tracef("%k completed\n", "my/responsepipe.fifo", wait);
+ # %k waits for wait to be fully closed; then sends formatted string to specified file(doing open, write, close).
+
ack(wait);
} until (done);
Added: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-10-02 00:18:26 UTC (rev 3655)
@@ -0,0 +1,45 @@
+#! /bin/bash
+
+site=$1
+
+# FIXME: check args and use better arg parsing
+
+tmp=/tmp # FIXME: allow this to change eg for sites with main tmp dir elsewhere
+throttleOneCore="-0.001"
+localcores=5 # FIXME: parameterize: localthreads=N
+
+SWIFTRBIN=$(cd $(dirname $0); pwd)
+rundir=$tmp/$USER/SwiftR/swift.local # rundir prefix # FIXME: handle multiple concurent independent swift servers per user
+mkdir -p $(dirname $rundir)
+trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success
+rm -rf $rundir
+ln -s $trundir $rundir
+cd $rundir
+
+script=$SWIFTRBIN/rserver.swift
+cp $script $SWIFTRBIN/passive-coaster-swift $SWIFTRBIN/swift.properties $rundir
+script=$(basename $script)
+cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.sh} .
+
+# FIXME: rework this script to transfer all shells and rscripts
+# needed, and to copy in the R prelude for the R server processes (to
+# include for example the OpenMx library) NOTE: Both were done in older version of this script.
+
+# rm -f requestpipe resultpipe
+mkfifo requestpipe resultpipe
+
+#FIXME JUNK app=/bin/bash
+# FIXME: remove these fossils:
+#serviceport=1985
+#site=local
+#location=$1
+#stagingmethod=$2
+
+source $SWIFTRBIN/configure-site-$1
+
+$SWIFTRBIN/../swift/bin/swift -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& swift.stdouterr </dev/null
+
+# wait-for-worker-port
+
+# $SWIFTRBIN/start-workers-$1 $workerport
+
Property changes on: SwiftApps/SwiftR/Swift/exec/start-swift
___________________________________________________________________
Name: svn:executable
+ *
Modified: SwiftApps/SwiftR/Swift/tests/TestSwift.R
===================================================================
--- SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-10-01 20:48:46 UTC (rev 3654)
+++ SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-10-02 00:18:26 UTC (rev 3655)
@@ -12,6 +12,33 @@
options(swift.initialize=initcmds) # Set here; used in test group 4
+# To paste for quick testing: ###########
+
+if (FALSE) {
+
+require(Swift)
+
+initcmds <- "
+initVar1 <- 19
+initVar2 <- sqrt(400)+3
+"
+
+failures=0
+
+options(swift.initialize=initcmds) # Set here; used in test group 4
+
+sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) }
+data(cars)
+data(trees)
+
+args=list(trees,cars)
+arglist = rep(list(args),1)
+
+swiftres = swiftapply(sumstuff,arglist)
+
+
+} ####################################
+
# NOTE: swift.initialize statements must be set before any swiftapply() calls when running in "service" mode.
runAllTests <- function()
Modified: SwiftApps/SwiftR/TODO
===================================================================
--- SwiftApps/SwiftR/TODO 2010-10-01 20:48:46 UTC (rev 3654)
+++ SwiftApps/SwiftR/TODO 2010-10-02 00:18:26 UTC (rev 3655)
@@ -1,6 +1,15 @@
*** TO DO LIST:
+MAIN
+- coaster provider data mover
+- tracef to replace job
+- interrupt a pipe read in R
+- develop the common config-site-$SITE scripts
+- finalize() method to clean up request dir?
+- test with swift logging turned way down
+
+
See if we can get Swift to just shut down the channel and start it again?
- no extended idle timer
- start new swift worker
More information about the Swift-commit
mailing list