[Swift-commit] r4043 - in SwiftApps/SwiftR/Swift: . R exec man
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Fri Jan 28 15:12:17 CST 2011
Author: tga
Date: 2011-01-28 15:12:17 -0600 (Fri, 28 Jan 2011)
New Revision: 4043
Modified:
SwiftApps/SwiftR/Swift/DESCRIPTION
SwiftApps/SwiftR/Swift/R/Swift.R
SwiftApps/SwiftR/Swift/R/Workers.R
SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
SwiftApps/SwiftR/Swift/exec/start-swift
SwiftApps/SwiftR/Swift/man/Swift-package.Rd
Log:
* Moved tests to separate file
* Now cleans up swift server directory by default when done.
Modified: SwiftApps/SwiftR/Swift/DESCRIPTION
===================================================================
--- SwiftApps/SwiftR/Swift/DESCRIPTION 2011-01-27 23:29:10 UTC (rev 4042)
+++ SwiftApps/SwiftR/Swift/DESCRIPTION 2011-01-28 21:12:17 UTC (rev 4043)
@@ -1,8 +1,8 @@
Package: Swift
Type: Package
Title: R interface to Swift parallel scripting languaage
-Version: 0.1.1
-Date: 2010-02-25
+Version: 0.1.2
+Date: 2010-02-28
Author: Michael Wilde
Maintainer: Michael Wilde <wilde at mcs.anl.gov>
Description: Routines to invoke R functions on remote resources through Swift.
Modified: SwiftApps/SwiftR/Swift/R/Swift.R
===================================================================
--- SwiftApps/SwiftR/Swift/R/Swift.R 2011-01-27 23:29:10 UTC (rev 4042)
+++ SwiftApps/SwiftR/Swift/R/Swift.R 2011-01-28 21:12:17 UTC (rev 4043)
@@ -230,6 +230,7 @@
}
names(rlist) = names(arglists)
if( ! keepwork ) {
+ cat("Removing ", reqdir, "\n")
unlink(reqdir,recursive=TRUE)
}
return(rlist)
@@ -246,462 +247,3 @@
names(arglists) = names(tlist)
swiftapply(func, arglists)
}
-#####
-#* checking R code for possible problems ... NOTE
-#swiftapply: no visible binding for '<<-' assignment to 'swiftprops'
-#swiftapply: no visible binding for global variable 'swiftprops'
-#swiftapply: no visible binding for global variable 'result'
-
-
-#------------------ Tests moved here from test dir:
-
-
-initSwiftTestOptions <- function()
-{
- options(swift.site="service")
- options(swift.keepwork=TRUE)
- initcmds <- "initVar1 <<- 19; initVar2 <<- sqrt(400)+3"
- options(swift.initialexpr=initcmds) # Set here; used in test group 4
-}
-
-
-
-
-swiftTest_1.1 <- function()
-{
-
- initSwiftTestOptions()
-
- cat("\n*** Starting test 1.1 ***\n\n")
-
- sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) }
- data(cars)
- data(trees)
-
- args=list(trees,cars)
- arglist = rep(list(args),1)
-
- cat("Test of local do.call(sumstuff)\n")
- localres = do.call(sumstuff,args)
- cat("local result=\n")
- print(localres)
-
- cat("\nTest of swiftapply(sumstuff,arglist)\n")
- swiftres = swiftapply(sumstuff,arglist)
- cat("Swift result:\n")
- print(swiftres)
-
- if(identical(localres,swiftres[[1]])) {
- cat("\n==> test 1.1 passed\n")
- } else {
- cat("\n==> test 1.1 FAILED !!!!!\n")
- }
-}
-
-basicSwiftTest <- function() { swiftTest_1.1() }
-
-# .... more tests from below to move here
-swiftTest_4.1 <- function()
-{
- sumivars <- function() { initVar1+initVar2 }
-
- args=list()
- arglist = rep(list(args),1)
-
- localres = 42
-
- cat("\nTest of swiftapply(sumivars,arglist)\n")
- swiftres = swiftapply(sumivars,arglist)
- cat("Swift result:\n")
- print(swiftres)
-
- if(identical(localres,swiftres[[1]])) {
- cat("\n==> test 4.1 passed\n")
- } else {
- cat("\n==> test 4.1 FAILED !!!!!\n")
- }
-}
-
-swiftTest_4.2 <- function()
-{
-
- options(swift.initialexpr="initVar3 <<- 123; initVar4 <<- 100");
-
- mulivars <- function() { initVar3*initVar4 }
-
- args=list()
- arglist = rep(list(args),1)
-
- localres = 12300;
-
- cat("\nTest of swiftapply(mulivars,arglist)\n")
- swiftres = swiftapply(mulivars,arglist)
- cat("Swift result:\n")
- print(swiftres)
-
- if(identical(localres,swiftres[[1]])) {
- cat("\n==> test 4.2 passed\n")
- } else {
- cat("\n==> test 4.2 FAILED !!!!!\n")
- }
-}
-
-# Performance tests
-
-swiftTest_6.1 <- function(delay,ncalls)
-{
- options(swift.initialexpr="initVar3 <<- 123; initVar4 <<- 100");
-
- timed <- function(delay) { Sys.sleep(delay); delay }
-
- args=list(delay)
- arglist = rep(list(args),ncalls)
-
- cat("\nTest of swiftapply(delay,arglist)\n")
-
- startTime = proc.time()[["elapsed"]]
- swiftres = swiftapply(timed,arglist)
- endTime = proc.time()[["elapsed"]]
- runTime <- endTime - startTime
-
- cat("\n\n ===> Total elapsed unit test time = ",runTime," seconds.\n\n")
-
- cat("Swift result:\n")
- print(swiftres[[1]])
-
- if(identical(delay,swiftres[[1]])) {
- cat("\n==> test 6.1 passed\n")
- } else {
- cat("\n==> test 6.1 FAILED !!!!!\n")
- }
-
-}
-
-runAllSwiftTests <- function()
-{
- # Launch workerif nothing already running
- # testPid will be NULL if nothihg launched
- testPid <- tryCatch(
- (function() { swiftapply(log, list(list(1)),quiet=TRUE) ;
- return(NULL); })(),
- error=function(x) {swiftInit();})
-
- ### FIXME: Save prior options here: restore them when tests are done.
- ### Recovery if interrrupted?
-
- failures=0
-
- startTime = proc.time()[["elapsed"]]
-
- cat("\n*** Starting test group 1 - functions on simple data structures ***\n\n")
-
- swiftTest_1.1()
-
- ##### Test 1.2
-
- # test 10 remote calls
-
- sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) }
- data(cars)
- data(trees)
-
- args=list(trees,cars)
- arglist <- rep(list(args),10)
-
- localres = do.call(sumstuff,args)
-
- cat("\n*** Test 1.2.1: 10 calls to substuff()\n")
- swiftres <- swiftapply(sumstuff,arglist)
- cat("Swift result:\n")
- format(swiftres)
-
- diffs <- 0
- for(i in 1:length(swiftres) ) {
- if( !identical(swiftres[[i]],localres) ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format( swiftres[[i]] )))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 1.2.1 passed\n")
- } else {
- cat("\n!!!==> test 1.2.1 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
- cat("\n*** Test 1.2.2: 10 calls to substuff() - callsperbatch=10\n")
- swiftres = swiftapply(sumstuff,arglist,callsperbatch=10)
- cat("Swift result:\n")
- format(swiftres)
-
- diffs <- 0
- for(i in 1:length(swiftres) ) {
- if( !identical(swiftres[[i]],localres) ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format( swiftres[[i]] )))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 1.2.2 passed\n")
- } else {
- cat("\n!!!==> test 1.2.2 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
- cat("\n*** Test 1.2.3: 10 calls to substuff() - callsperbatch=2\n")
- swiftres = swiftapply(sumstuff,arglist,callsperbatch=2)
- cat("Swift result:\n")
- format(swiftres)
-
- diffs <- 0
- for(i in 1:length(swiftres) ) {
- if( !identical(swiftres[[i]],localres) ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i, format( swiftres[[i]] )))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 1.2.3 passed\n")
- } else {
- cat("\n!!!==> test 1.2.3 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
- cat("\n*** Test 1.2.4: 10 calls to substuff() - callsperbatch=3\n")
- swiftres = swiftapply(sumstuff,arglist,callsperbatch=3)
- swiftres <- swiftapply(sumstuff,arglist)
- cat("Swift result:\n")
- format(swiftres)
-
- diffs <- 0
- for(i in 1:length(swiftres) ) {
- if( !identical(swiftres[[i]],localres) ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i, format( swiftres[[i]] )))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 1.2.4 passed\n")
- } else {
- cat("\n!!!==> test 1.2.4 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
- # swiftres = swiftapply(sumstuff,arglist,callsperbatch=2,site="pbs")
- # test variations on local vs ssh vs pbs; coasters vs non; etc.
-
-
-
- ##### Test Group 2
-
- cat("\n*** Starting test group 2 - test matrix passing***\n")
-
- matfunc <- function( m1, m2 )
- {
- (1/m1) %*% m2
- }
-
- n <- 5
- m1 <- array(sin(1:n**2), dim=c(n,n))
- m2 <- t(m1)
-
- localres = matfunc(m1,m2)
-
- cat("\n*** Test 2.1: 100 calls to matfunc(dim=5x5) - callsperbatch=9\n")
-
- args=list(m1,m2)
- arglist <- rep(list(args),100)
-
- swiftres = swiftapply(matfunc,arglist,callsperbatch=9)
-
- diffs <- 0
- #for(i in 1:length(swiftres) ) {
- for(i in c(seq(1,100,10),100)) {
- if( !all.equal(swiftres[[i]],localres) ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]])))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 2.1 passed\n")
- } else {
- cat("\n!!!==> test 2.2 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
- n <- 237
- n <- 50
- m1 <- array(sin(1:n**2), dim=c(n,n))
- m2 <- t(m1)
-
- localres = matfunc(m1,m2)
-
- cat("\n*** Test 2.2: 123 calls to matfunc(dim=bigger) - callsperbatch=7\n") # FIXME make n easy to adjust and print actual value
-
- args=list(m1,m2)
- arglist <- rep(list(args),123)
-
- swiftres = swiftapply(matfunc,arglist,callsperbatch=7)
-
- diffs <- 0
- #for(i in 1:length(swiftres) ) {
- for(i in c(seq(1,length(swiftres),10),length(swiftres))) {
-
- if( !all.equal(swiftres[[i]],localres) ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]])))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 2.2 passed\n")
- } else {
- cat("\n!!!==> test 2.2 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
-
- ##### Test Group 3
-
- cat("\n*** Starting test group 3 - test list element and name passing***\n")
-
- # Test if list element names are being sent and returned correctly
-
- n <- 5
- m1 <- array(sin(1:n**2), dim=c(n,n))
- m2 <- t(m1)
-
- inlist = list()
- inlist[[1]]=123
- inlist[[2]]=456
- inlist$name1=789
- inlist$name2=987
- inlist$mat1 = m1
- inlist[[99]] = m2
-
- listfunc <- function(ilist)
- {
- olist = ilist
- olist$sum = ilist[[1]] + ilist[[2]] + ilist$name1 + ilist$name2
- olist$names = names(ilist)
- olist$mprod = ilist$mat1 %*% ilist[[99]]
- return(olist)
- }
- localres = listfunc(inlist)
-
- cat("\n*** Starting test 3.1 - 4 calls in one batch of 5 ***\n")
-
- args=list(inlist)
- arglist <- rep(list(args),4)
-
- swiftres = swiftapply(listfunc,arglist,callsperbatch=5)
-
- diffs <- 0
- for(i in 1:length(swiftres) ) {
- if( !all.equal(swiftres[[i]],localres) ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d=%s\n",i,format(swiftres[[i]])))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 3.1 passed\n")
- } else {
- cat("\n!!!==> test 3.1 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
- cat("\n*** Starting test 3.2 - 99 calls in batches of 11 ***\n")
-
- args=list(inlist)
- arglist <- rep(list(args),99)
-
- swiftres = swiftapply(listfunc,arglist,callsperbatch=11)
-
- diffs <- 0
- for(i in 1:length(swiftres) ) {
- if( !all.equal(swiftres[[i]],localres) ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]])))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 3.2 passed\n")
- } else {
- cat("\n!!!==> test 3.2 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
- ##### Test Group 4 # test initialexpr string
-
- cat("\n*** Starting test group 4 - test remote R service initialization string ***\n")
-
- swiftTest_4.1()
- swiftTest_4.2()
-
-
-
-
- ##### Test Group 5 # test error handling
-
- cat("\n*** Starting test group 5 - test remote R service error ***\n")
-
- arglist = list(list(1.0),list(2.0),list("3.0"),list(4.0),list(5.0))
-
- cat("\nTest of swiftapply(sumivars,arglist)\n")
- swiftres = swiftapply(log,arglist)
- cat("Swift result:\n")
- print(swiftres)
-
- goodres = c("numeric","numeric","try-error","numeric","numeric")
-
- diffs <- 0
- for(i in 1:length(swiftres) ) {
- if( class(swiftres[[i]]) != goodres[i] ) {
- diffs <- diffs + 1
- if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]])))
- }
- }
-
- if(diffs == 0) {
- cat("\n==> test 5.1 passed\n")
- } else {
- cat("\n!!!==> test 5.1 failed.\n")
- cat(sprintf(" %d result elements failed to match.\n",diffs));
- failures=failures+1
- }
-
- endTime <- proc.time()[["elapsed"]]
- runTime <- endTime - startTime
-
- cat("\n\n ===> Total elapsed test time = ",runTime," seconds.\n\n")
- if (!is.null(testPid))
- swiftShutdown(testPid)
-} # end function runAllTests
-
-#options(swift.site="local")
-#runAllTests()
-
-testloop <- function(npass)
-{
- for(i in 1:npass) {
- cat("\n\n\n ***** Starting test pass ", i, " ***** \n\n\n");
- runAllSwiftTests()
- cat("\n\n\n ***** Completed test pass ", i, " ***** \n\n\n");
- system("sleep 3")
- }
-}
Modified: SwiftApps/SwiftR/Swift/R/Workers.R
===================================================================
--- SwiftApps/SwiftR/Swift/R/Workers.R 2011-01-27 23:29:10 UTC (rev 4042)
+++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-01-28 21:12:17 UTC (rev 4043)
@@ -6,7 +6,7 @@
parEnv=NULL, workmode=NULL,
throttle=NULL, queue=NULL,
rcmd=NULL, time=NULL,
- workerLogging=NULL )
+ workerLogging=NULL,keepworkdir=NULL)
{
# server: which server backend to use to acquire workers
# for example, local runs tasks on the local machine
@@ -121,6 +121,10 @@
cmdString <- paste(cmdString, "-w", shQuote(workerLogging))
}
+ if(is.null(keepworkdir))
+ keepworkdir <- getOption("swift.keepworkdir")
+ if(!is.null(keepworkdir) && keepworkdir)
+ cmdString <- paste(cmdString, "-k")
# launch asynchronously
Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-01-27 23:29:10 UTC (rev 4042)
+++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-01-28 21:12:17 UTC (rev 4043)
@@ -128,4 +128,4 @@
echo DB: Freed $SLOTDIR/mutex
-# Fixme: how to get exceptions and stdout/stderr text from R server ???
\ No newline at end of file
+# Fixme: how to get exceptions and stdout/stderr text from R server ???
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2011-01-27 23:29:10 UTC (rev 4042)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-01-28 21:12:17 UTC (rev 4043)
@@ -5,13 +5,6 @@
export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap
-# Standard clenuup actions
-function stdcleanup {
- # don't accept any more requests: unlink fifo from filesystem
- if [ -p requestpipe ]; then
- rm requestpipe
- fi
-}
# Define internal functions
@@ -280,7 +273,8 @@
echo Started workers from batch job $(cat $jobidfile)
else
echo Batch queue submission failed, exiting.
- stdcleanup
+ stdcleanup_start
+ stdcleanup_end
exit 1
fi
}
@@ -346,6 +340,7 @@
-s server local local, pbs, sge, ssh, pbsf (for firewalled worker nodes)
-t time 00:30:00 hh:mm:ss, for PBS and SGE only
-w wkloglvl NONE NONE, ERROR, WARN, INFO, DEBUG, TRACE
+ -k keepdir No argument, if flag is set, will keep working directory
Examples:
@@ -380,6 +375,7 @@
project=NONE
parEnv=NONE
workerLogging=ERROR
+keepdir=FALSE
rcmd=ssh # rcmd: ssh (typical) or qrsh (eg for siraf with node login restrictions)
workmode=slot # slot: start one worker on each slot; node: start one worker for all slots on a node
@@ -399,6 +395,7 @@
-s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge; shift ;;
-t) time=$2; verify-not-null time $time; shift ;;
-w) workerLogging=$2; verify-is-one-of workerLoggingLevel $workerLogging NONE ERROR WARN INFO DEBUG TRACE; shift ;;
+ -k) keepdir=TRUE ;;
*) usage; exit 1 ;;
esac
shift
@@ -410,10 +407,28 @@
rundir=$tmp/$USER/SwiftR/swift.$server # rundir prefix # FIXME: handle multiple concurent independent swift servers per user
mkdir -p $(dirname $rundir)
trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success
-rm -rf $rundir
+rm -f $rundir
ln -s $trundir $rundir
-cd $rundir
+cd $trundir
+
+
+# Standard clenuup actions
+function stdcleanup_start {
+ # don't accept any more requests: unlink fifo from filesystem
+ if [ -p requestpipe ]; then
+ rm requestpipe
+ fi
+}
+function stdcleanup_end {
+ # Check to see if we should leave
+ # logs, etc around
+ if [ "$keepdir" = "FALSE" ]; then
+ cd /
+ rm -rf "$trundir"
+ fi
+}
+
echo Running in $trundir "(linked to $rundir)"
script=$SWIFTRBIN/rserver.swift
@@ -443,7 +458,7 @@
source $SWIFTRBIN/configure-server-local $cores
function onexit {
- stdcleanup
+ stdcleanup_start
# Find and terminate R workers: they should register their PiD
# in a standard location based on the pid of this start-swift
# script
@@ -452,6 +467,7 @@
kill `cat $rwork/R.pid` &> /dev/null
rm -rf $rwork
done
+ stdcleanup_end
exit 0
}
@@ -473,7 +489,7 @@
TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap
function onexit {
- stdcleanup
+ stdcleanup_start
coasterservicepid="" # null: saved in case we go back to using coaster servers
trap - $TRAPEVENTS
sshpids=$(cat $sshpidfile)
@@ -488,6 +504,7 @@
if [ "_$sshpids$starterpid$coasterservicepid" != _ ]; then
echo kill $sshpids $starterpid $coasterservicepid >& /dev/null
fi
+ stdcleanup_end
# exit cleanly
exit 0
}
@@ -511,7 +528,7 @@
TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap
function onexit {
- stdcleanup
+ stdcleanup_start
coasterservicepid="" # null: saved in case we go back to using coaster servers
trap - $TRAPEVENTS
@@ -524,6 +541,7 @@
qdel "$jobid"
fi
# eit cleanly
+ stdcleanup_end
exit 0
}
Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd
===================================================================
--- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2011-01-27 23:29:10 UTC (rev 4042)
+++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2011-01-28 21:12:17 UTC (rev 4043)
@@ -27,8 +27,8 @@
\tabular{ll}{
Package: \tab Swift\cr
Type: \tab Package\cr
-Version: \tab 0.1.1\cr
-Date: \tab 2011-01-26\cr
+Version: \tab 0.1.2\cr
+Date: \tab 2011-01-28\cr
License: \tab Globus Toolkit Public License v3 (based on Apache License 2.0):
http://www.globus.org/toolkit/legal/4.0/license-v3.html \cr
LazyLoad: \tab yes\cr
More information about the Swift-commit
mailing list