[Swift-commit] r5118 - in SwiftApps/SwiftR: . Swift/R Swift/man Swift/tests Swift/tests/perf Swift/tests/platform-specific
tga at ci.uchicago.edu
tga at ci.uchicago.edu
Wed Sep 14 21:54:05 CDT 2011
Author: tga
Date: 2011-09-14 21:54:05 -0500 (Wed, 14 Sep 2011)
New Revision: 5118
Added:
SwiftApps/SwiftR/Swift/tests/README
SwiftApps/SwiftR/Swift/tests/perf/
SwiftApps/SwiftR/Swift/tests/perf/param_size.R
SwiftApps/SwiftR/Swift/tests/perf/serialisation_overhead.R
SwiftApps/SwiftR/Swift/tests/platform-specific/
SwiftApps/SwiftR/Swift/tests/platform-specific/batch_test.R
SwiftApps/SwiftR/Swift/tests/platform-specific/cobalt_test.sub
SwiftApps/SwiftR/Swift/tests/platform-specific/pbs_test.sub
Removed:
SwiftApps/SwiftR/Swift/man/swiftShutdown.Rd
SwiftApps/SwiftR/Swift/tests/batch_test.R
SwiftApps/SwiftR/Swift/tests/cobalt_test.sub
SwiftApps/SwiftR/Swift/tests/param_size.R
SwiftApps/SwiftR/Swift/tests/pbs_test.sub
SwiftApps/SwiftR/Swift/tests/serialisation_overhead.R
Modified:
SwiftApps/SwiftR/Makefile
SwiftApps/SwiftR/Swift/R/Workers.R
SwiftApps/SwiftR/Swift/man/swiftExport.Rd
SwiftApps/SwiftR/Swift/man/swiftInit.Rd
SwiftApps/SwiftR/Swift/man/swiftLibrary.Rd
SwiftApps/SwiftR/Swift/man/swiftapply.Rd
Log:
Fixed most warnings from R package check utility (aside from missing docs for test functions)
Modified: SwiftApps/SwiftR/Makefile
===================================================================
--- SwiftApps/SwiftR/Makefile 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Makefile 2011-09-15 02:54:05 UTC (rev 5118)
@@ -2,7 +2,7 @@
include Swift/src/make.include
# All files that go into package
-PKG_FILES = $(shell find Swift/R -name '*.R'; find Swift/man -name '*.Rd'; find Swift/exec/ -type f -not -path '*/.svn*'; find Swift/inst/ -type f -not -path '*/.svn*')
+PKG_FILES = $(shell find Swift -not -path '*/.svn*')
PKG_FILES += Swift/DESCRIPTION
Modified: SwiftApps/SwiftR/Swift/R/Workers.R
===================================================================
--- SwiftApps/SwiftR/Swift/R/Workers.R 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-09-15 02:54:05 UTC (rev 5118)
@@ -114,7 +114,7 @@
if( is.null(hosts) ) {
if (server == "ssh") {
- error(paste("Need to provide hosts list for ssh server."))
+ stop(paste("Need to provide hosts list for ssh server."))
}
}
else {
@@ -286,8 +286,8 @@
cat(paste("Terminating worker", worker$pid, "of type",
worker$server, "\n"))
}
- cmdString <- file.path(.find.package("Swift"), "exec/killtree &> /dev/null ")
- killCmd <- paste(cmdString, worker$pid)
+ cmdString <- file.path(.find.package("Swift"), "exec/killtree")
+ killCmd <- paste(cmdString, worker$pid, " &> /dev/null")
system(killCmd, wait=FALSE)
}
Modified: SwiftApps/SwiftR/Swift/man/swiftExport.Rd
===================================================================
--- SwiftApps/SwiftR/Swift/man/swiftExport.Rd 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/man/swiftExport.Rd 2011-09-15 02:54:05 UTC (rev 5118)
@@ -19,8 +19,8 @@
}
\usage{
-swiftExport(..., list = NULL, file = NULL)
-swiftExportAll()
+swiftExport(..., list = NULL, file = NULL, quiet=getOption("swift.quiet"))
+swiftExportAll(file = NULL)
swiftRemoveAll()
}
%- maybe also 'usage' for other objects documented here.
@@ -37,6 +37,9 @@
If not provided, a file will automatically be chosen.
A directory can be specified via the swift.exportdir option.
}
+ \item{quiet}{
+ Whether informational messages should be suppressed
+ }
}
\details{
In many cases if a data or a function is referenced within a function
@@ -81,9 +84,6 @@
Maintainer: Michael Wilde <wilde at mcs.anl.gov>
}
-\note{
-%% ~~further notes~~
-}
%% ~Make other sections like Warning with \section{Warning }{....} ~
Modified: SwiftApps/SwiftR/Swift/man/swiftInit.Rd
===================================================================
--- SwiftApps/SwiftR/Swift/man/swiftInit.Rd 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/man/swiftInit.Rd 2011-09-15 02:54:05 UTC (rev 5118)
@@ -1,27 +1,26 @@
\name{swiftInit}
+\alias{swiftShutdown}
\alias{swiftInit}
%- Also NEED an '\alias' for EACH other topic documented here.
-\title{Starting Swift Workers
+\title{Starting and Stopping Swift Workers
}
\description{
-This function prepares for parallel Swift jobs to be run by initialising a pool of workers. Depending on the arguments, these workers may be on the local host, or on a remote host.
+\code{swiftInit} prepares for parallel Swift jobs to be run by initialising a pool of workers. Depending on the arguments, these workers may be on the local host, or on a remote host. \code{swiftShutdown} shuts down workers that were started by \code{swiftInit}.
}
\usage{
-swiftInit(cores = NULL, server = NULL, hosts = NULL, nodes = NULL, project = NULL, parEnv = NULL, workmode = NULL, throttle = NULL, queue = NULL, rcmd = NULL, time = NULL, workerLogging = NULL)
+swiftInit(cores = getOption("swift.cores"), server = getOption("swift.server"), hosts = getOption("swift.hosts"), nodes = getOption("swift.nodes"), project = getOption("swift.project"), parEnv = getOption("swift.parenv"), kernel = getOption("swift.kernel"), workmode = getOption("swift.workmode"), throttle = getOption("swift.throttle"), retries = getOption("swift.retries"), queue = getOption("swift.queue"), rcmd = getOption("swift.rcmd"), time = getOption("swift.time"), workerLogging = getOption("swift.workerLogging"), swiftLogging = getOption("swift.swiftLogging"), keepworkdir = getOption("swift.keepworkdir"), tc.file = getOption("swift.tc.file"), cf.file = getOption("swift.cf.file"), sites.file = getOption("swift.sites.file"))
+swiftShutdown(handle = NULL, all=FALSE)
}
-%- maybe also 'usage' for other objects documented here.
\arguments{
- All arguments to swiftInit() are optional.
- If any of the arguments are not provided to swiftInit, this function
- tries to obtain a value from Swift's options mechanism. The option
+ All arguments to \code{swiftInit} are optional, however some for some
+ values of the \code{server} argument other arguments may need to be
+ specified directly or via a Swift option.
+
+ If any of the arguments are not provided to swiftInit, the function
+ tries to obtain a value using \code{getOption}. The option
names correspond to the argument names prefixed with "swift.", for example
"swift.cores" or "swift.server".
- This function does not necessarily require any arguments to be set directly
- or indirectly through options, however depending on your server setting
- and local configuration you may need to provide some argument.
-
-
\item{cores}{
The number of cores per host. The default values vary from 2 to 8 depending on the server type.
}
@@ -62,8 +61,17 @@
}
\item{throttle}{
The throttle setting to be used by Swift: controls the rate of sending jobs
- to workers.
+ to workers. 0 means that only two function invocations are run at a time.
+ 1 means that 102 function invocations can be run concurrently. The general
+ formula for maximum number of parallel function invocations is
+ 2 + 100 * throttle.
+
}
+ \item{retries}{
+ The number of times Swift should retry a task if it encounters an execution error.
+ R code which throws errors will not be re-executed, only errors external to R
+ such as computer failures will trigger a retry.
+}
\item{queue}{
The scheduler queue to put jobs in. This is only relevant for PBS
and SGE.
@@ -84,18 +92,49 @@
\item{workerLogging}{
For testing purposes: the swift worker logging level.
}
+ \item{swiftLogging}{
+ For testing purposes: if set to true, log additional trace information to
+ Swift log files.
+}
+ \item{keepworkdir}{
+ For debugging purposes: whether to keep the contents of the Swift
+ working directory.
+}
+ \item{tc.file}{
+ If server is set to custom, the path to the Swift transformation catalog
+ file.
+}
+ \item{sites.file}{
+ If server is set to custom, the path to the Swift site file
+ file.
+}
+ \item{cf.file}{
+ If server is set to custom, the path to the Swift configuration
+ file.
+}
+ \item{handle}{
+ The handle value returned by "\code{swiftInit}" when a worker was
+ started up. If provided, only that worker will be shut down.
+ If handle is not provided and \code{all} is FALSE, then the most
+ recently started worker that is still running will be shut down.
}
+ \item{all}{
+ logical; if set to TRUE, shut down all running R workers, rather
+ than just workers from a single \code{swiftInit} call.
+}
+}
\details{
- As a result of calling swiftInit(), a swift server process is started
+ As a result of calling \code{swiftInit}, a swift server process is started
in the background. This process will be cleaned up automatically when
- you exit your R session. Alternatively, swiftShutdown can shut down
+ you exit your R session. Alternatively, \code{swiftShutdown} can shut down
some or all server processes without closing R.
+
}
\value{
- swiftInit returns a handle to the swift server process started.
- This handle can be used later as an argument to swiftShutdown to
+ \code{swiftInit} returns a handle to the swift server process started.
+ This handle can be used later as an argument to \code{swiftShutdown} to
shut down the server process. The actual data in the handle may
change in later versions of Swift.
}
@@ -112,9 +151,6 @@
Maintainer: Michael Wilde <wilde at mcs.anl.gov>
}
-\note{
-%% ~~further notes~~
-}
%% ~Make other sections like Warning with \section{Warning }{....} ~
@@ -134,7 +170,7 @@
swiftShutdown(serverHandle)
# Now run remotely via ssh on 2 remote machines called foo and bar
-options(swift.server="ssh")
+\dontrun{options(swift.server="ssh")
swiftInit(hosts="foo bar")
swiftLapply(list(3, 4), log)
# ssh servers will keep running
@@ -148,7 +184,8 @@
swiftLapply(c(3, 4), log)
# Close all the workers we started up
-swiftShutdown()
+swiftShutdown(all=TRUE)
+}
}
% Add one or more standard keywords, see file 'KEYWORDS' in the
Modified: SwiftApps/SwiftR/Swift/man/swiftLibrary.Rd
===================================================================
--- SwiftApps/SwiftR/Swift/man/swiftLibrary.Rd 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/man/swiftLibrary.Rd 2011-09-15 02:54:05 UTC (rev 5118)
@@ -56,9 +56,6 @@
Maintainer: Michael Wilde <wilde at mcs.anl.gov>
}
-\note{
-%% ~~further notes~~
-}
%% ~Make other sections like Warning with \section{Warning }{....} ~
@@ -67,8 +64,13 @@
\code{\link{Swift-package}}
}
\examples{
+library(Swift)
+swiftInit()
+swiftLibrary(stats)
+swiftLapply(list(0.1, 0.2, 0.3),
+ function (x) { pbeta(x, 2, 3) })
+swiftShutdown()
-
}
% Add one or more standard keywords, see file 'KEYWORDS' in the
% R documentation directory.
Deleted: SwiftApps/SwiftR/Swift/man/swiftShutdown.Rd
===================================================================
--- SwiftApps/SwiftR/Swift/man/swiftShutdown.Rd 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/man/swiftShutdown.Rd 2011-09-15 02:54:05 UTC (rev 5118)
@@ -1,60 +0,0 @@
-\name{swiftShutdown}
-\alias{swiftShutdown}
-\title{
-Shut Down Swift Workers
-}
-\description{
-%% ~~ A concise (1-5 lines) description of what the function does. ~~
-}
-\usage{
-swiftShutdown(handle = NULL, all=FALSE)
-}
-\arguments{
- \item{handle}{
- The handle value returned by "\code{swiftInit}" when a worker was
- started up. If provided, only that worker will be shut down.
- If handle is not provided and \code{all} is FALSE, then the most
- recently started worker will be shut down.
-
-}
- \item{all}{
- Whether to terminate all workers started within this R session.
-}
-}
-\details{
-%% ~~ If necessary, more details than the description above ~~
-}
-\value{
- No return value
-}
-\references{
-\url{http://www.ci.uchicago.edu/swift}
-}
-\author{
-Swift was developed by: Mihael Hategan, Ben Clifford, Justin Wozniak,
-Yong Zhao, Ian Foster, and Michael Wilde with contributions from Sarah
-Kenny, Ioan Raicu, Luiz Gadelha, Allan Espinosa, Zhao Zhang, David
-Kelly, Jon Monette, Glen Hocky, Tom Uram, Wenjun Wu, and other users.
-
-Swift R package developed by Michael Wilde, Tim Armstrong and the OpenMx project
-
-Maintainer: Michael Wilde <wilde at mcs.anl.gov>
-}
-\note{
-%% ~~further notes~~
-}
-
-%% ~Make other sections like Warning with \section{Warning }{....} ~
-
-\seealso{
-\code{\link{Swift-package}}
-\code{\link{swiftInit}}
-
-}
-\examples{
-
-}
-% Add one or more standard keywords, see file 'KEYWORDS' in the
-% R documentation directory.
-\keyword{ ~kwd1 }
-\keyword{ ~kwd2 }% __ONLY ONE__ keyword per line
Modified: SwiftApps/SwiftR/Swift/man/swiftapply.Rd
===================================================================
--- SwiftApps/SwiftR/Swift/man/swiftapply.Rd 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/man/swiftapply.Rd 2011-09-15 02:54:05 UTC (rev 5118)
@@ -21,11 +21,11 @@
swiftapply is a more general function that takes additional arguments.
}
\usage{
-swiftapply(func, arglists, server = NULL, callsperbatch = NULL, runmode = NULL, initialexpr = NULL, workerhosts = NULL, keepwork = NULL, tmpdir = NULL, timeout = NULL, quiet = FALSE)
+swiftapply(func, arglists, server = NULL, callsperbatch = NULL, runmode = NULL, initialexpr = NULL, workerhosts = NULL, keepwork = NULL, tmpdir = NULL, timeout = NULL, quiet = NULL, printTiming = NULL)
swiftLapply(tlist, func, ...)
-swiftSapply(tlist, func, ...)
+swiftSapply(tlist, func, ..., simplify = TRUE, USE.NAMES = TRUE)
}
\arguments{
The first two arguments are required for all variations of the
@@ -45,7 +45,7 @@
\item{arglists}{
A list of argument lists to be given to the function.
}
- \item{tlists}{
+ \item{tlist}{
For swiftLapply and swiftSapply, a list which contains the first
argument for each function invocation.
}
@@ -57,7 +57,7 @@
}
\item{server}{
The swift server type to use to run. The possible values are
- "local", "ssh", "pbs", "sge" and "pbsf", the same as swiftInit.
+ "local", "ssh", "pbs", "sge" and "pbsf", the same as \verb{swiftInit}.
The most recently started server of the specified type will be used
to execute the apply call. If not specified, then
the swift.server option will be used. If that is not specified, the
@@ -99,10 +99,22 @@
\item{quiet}{
Whether to suppress informational output about the job.
}
+ \item{printTiming}{
+ If set to TRUE, prints timing information on components of the
+ apply operation.
}
+ \item{simplify}{
+ Whether to simplify the return value of \code{swiftSapply} following the
+ same rules as \code{sapply}.
+}
+ \item{USE.NAMES}{
+ Same behaviour as the equivalent argument to \code{sapply}.
+
+}
+}
\details{
-Before running swiftapply for the first time in a session you will likely
-want to run swiftInit to start a Swift server.
+Before running \verb{swiftapply} for the first time in a session you will likely
+want to run \verb{swiftInit} to start a Swift server.
}
\value{
Returns a list with return values corresponding to the argument
@@ -125,9 +137,6 @@
Maintainer: Michael Wilde <wilde at mcs.anl.gov>
}
-\note{
-%% ~~further notes~~
-}
%% ~Make other sections like Warning with \section{Warning }{....} ~
@@ -137,12 +146,30 @@
\code{\link{Swift-package}}
}
\examples{
-##---- Should be DIRECTLY executable !! ----
-##-- ==> Define data, use random,
-##-- or do help(data=index) for the standard data sets.
+\dontshow{library(Swift)}
+\dontshow{swiftInit()}
+swiftapply(log, list(list(5),list(6)))
-swiftapply(log, list(list(5),list(6)))
+# swiftLapply returns result as list
+swiftLapply(list(1,2,3), function (x) { 2 * x })
+swiftLapply(c(1,2,3), function (x) { 2 * x })
+
+# swiftSapply will coerce result to vector
+swiftSapply(c(1,2,3), function (x) { 2 * x })
+
+pow <- function (x, y) {
+ x ^ y
}
+
+# swiftapply allows multiple argument functions by taking a list of lists
+swiftapply(pow, list(list(2,2), list(2,3), list(7,3)))
+
+# swiftLapply and swiftSapply allow only one value to be provided for the
+# second and subsequent arguments for the multiple function invocations
+swiftLapply(list(1,2,3,4,5,6,7,8,9), pow, 2)
+swiftSapply(list(1,2,3,4,5,6,7,8,9), pow, 3)
+\dontshow{swiftShutdown()}
+}
% Add one or more standard keywords, see file 'KEYWORDS' in the
% R documentation directory.
\keyword{ ~kwd1 }
Added: SwiftApps/SwiftR/Swift/tests/README
===================================================================
--- SwiftApps/SwiftR/Swift/tests/README (rev 0)
+++ SwiftApps/SwiftR/Swift/tests/README 2011-09-15 02:54:05 UTC (rev 5118)
@@ -0,0 +1,7 @@
+perf/
+ Tests used to gather performance measurements
+platform-specific/
+ Tests which need to be run in a specific environment (e.g. on a cluster
+ or on the head node of a cluster with a particular scheduler)
+OpenMx
+ Benchmarks and tests using the OpenMx package
Deleted: SwiftApps/SwiftR/Swift/tests/batch_test.R
===================================================================
--- SwiftApps/SwiftR/Swift/tests/batch_test.R 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/tests/batch_test.R 2011-09-15 02:54:05 UTC (rev 5118)
@@ -1,38 +0,0 @@
-require(Swift)
-
-server <- commandArgs(TRUE)[[1]]
-
-
-hostinfo1 <- getNodeList()
-print("Auto:")
-print(hostinfo1)
-
-if (server == "cobalt") {
- hostinfo2 <- getNodeList("cobalt")
- print("cobalt:")
- print(hostinfo2)
-}
-if (server %in% c("pbs", "pbsf")) {
- hostinfo2 <- getNodeList("pbs")
- print("pbs:")
- print(hostinfo2)
-
- hostinfo3 <- getNodeList("pbsf")
- print("pbsf:")
- print(hostinfo3)
-}
-if (server == "sge") {
- hostinfo2 <- getNodeList("sge")
- print("sge:")
- print(hostinfo2)
-
- hostinfo3 <- getNodeList("pbsf")
- print("pbsf:")
- print(hostinfo3)
-}
-
-print ("Hosts:")
-print(hostinfo1[[1]])
-
-runAllSwiftTests(server="ssh", hosts=hostinfo1[[1]], cores=8)
-
Deleted: SwiftApps/SwiftR/Swift/tests/cobalt_test.sub
===================================================================
--- SwiftApps/SwiftR/Swift/tests/cobalt_test.sub 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/tests/cobalt_test.sub 2011-09-15 02:54:05 UTC (rev 5118)
@@ -1,12 +0,0 @@
-#!/bin/sh
-export WORKER_LOGGING_LEVEL=ERROR
-echo '***' COBALT_NODEFILE file: $COBALT_NODEFILE
-cat $COBALT_NODEFILE
-echo '***' unique nodes are:
-sort < $COBALT_NODEFILE|uniq
-
-echo GLOBUS_HOSTNAME $GLOBUS_HOSTNAME
-export GLOBUS_HOSTNAME=""
-Rscript `dirname $0`/batch_test.R cobalt
-
-
Deleted: SwiftApps/SwiftR/Swift/tests/param_size.R
===================================================================
--- SwiftApps/SwiftR/Swift/tests/param_size.R 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/tests/param_size.R 2011-09-15 02:54:05 UTC (rev 5118)
@@ -1,57 +0,0 @@
-require(Swift)
-source("Swift/tests/perf_tests.R")
-
-cores <- 4
-intsize <- 4
-
-param.bytes <- lapply(1:24, function (x) {list(cores, 4 * (2**x))})
-
-id <- function (x) {x}
-
-test <- function (arr) {
- res <- swiftLapply(arr, id)
- length(res)
- return (NULL)
-}
-makeArray <- function (procs, arr.bytes) {
- #arrsize is in bytes
- args <- list()
- for (i in 1:procs) {
- args[[i]] <- as.vector(sample(1:1000, arr.bytes/intsize, replace=T), "integer")
- }
- return(args)
-}
-
-ts <- makeTestSuite(groups=list(
- makePerfTestGroup(mode="swift", name="ArgSize",
- f=test, prep=makeArray,
- allargs=param.bytes,
- server="local", cores=cores),
- makePerfTestGroup(mode="swift", name="ArgSize",
- f=test, prep=makeArray,
- allargs=param.bytes,
- server="ssh", cores=1, hosts="nettle wapato dandelion cattail")
- ))
-res <- runTestSuite(ts)
-
-anl <- mergeGroupResults(analyseSuitePerf(res,
- argnames=c("procs", "arr.bytes"), perfparams=c('server')))
-
-
-
-loc <- subset(anl, server=="local")
-ssh <- subset(anl, server=="ssh")
-plot(x=ssh$arr.bytes/1024, y=ssh$time, col='red', ylab="Time (s)", xlab="Argument size (bytes)",
-# log="xy", ylim=c(0.1, 500),
- xaxt="n")
-title(main="Effect of argument size on execution time", line=2)
-title(main="Execution time of identity function with one function invocation per core", line=1, font.main=4, cex.main=0.8)
-axis(1, at=ssh$arr.bytes/1024,
- labels=lapply(ssh$arr.bytes,
- function (n) { if (n >= 1024*1024) {sprintf("%dM", n/(1024*1024))}
- else if (n >= 1024 ) {sprintf("%dK", n/1024)}
- else {sprintf("%d", n)}})
- )
-points(x=loc$arr.bytes/1024, y=loc$time)
-legend("topleft", legend=c("Local (4 cores)", "Ssh (4 hosts, 1 core per host)"),
- fill=c("black", "red"))
Deleted: SwiftApps/SwiftR/Swift/tests/pbs_test.sub
===================================================================
--- SwiftApps/SwiftR/Swift/tests/pbs_test.sub 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/tests/pbs_test.sub 2011-09-15 02:54:05 UTC (rev 5118)
@@ -1,33 +0,0 @@
-#PBS -S /bin/sh
-#PBS -N SwiftR-batch
-#PBS -m n
-#PBS -l nodes=1
-#PBS -l walltime=00:30:00
-
-HOST=$(echo http://192.5.86.6:48998 | sed -e 's,^http://,,' -e 's/:.*//')
-PORT=$(echo http://192.5.86.6:48998 | sed -e 's,^.*:,,')
-echo '***' PBS_NODEFILE file: $PBS_NODEFILE
-cat $PBS_NODEFILE
-echo '***' unique nodes are:
-sort < $PBS_NODEFILE|uniq
-
-echo GLOBUS_HOSTNAME=$GLOBUS_HOSTNAME
-#export PATH=/autonfs/home/tga/R-2.12.1/bin:$PATH
-
-if which Rscript; then
- echo Rscript binary found!
-else
- echo Rscript binary not found in:
- echo PATH=$PATH
- echo Maybe you need to set PATH in this sub file
- exit 1
-fi
-
-
-#Rscript `dirname $0`/batch_test.R pbs
-if [ -f $PBS_O_WORKDIR/Swift/tests/batch_test.R ]; then
- Rscript $PBS_O_WORKDIR/Swift/tests/batch_test.R pbs
-else
- echo Could not find batch_test.R. Maybe try modifying this .sub file with the absolute path of the script.
- exit 1
-fi
Added: SwiftApps/SwiftR/Swift/tests/perf/param_size.R
===================================================================
--- SwiftApps/SwiftR/Swift/tests/perf/param_size.R (rev 0)
+++ SwiftApps/SwiftR/Swift/tests/perf/param_size.R 2011-09-15 02:54:05 UTC (rev 5118)
@@ -0,0 +1,57 @@
+require(Swift)
+source("Swift/tests/perf_tests.R")
+
+cores <- 4
+intsize <- 4
+
+param.bytes <- lapply(1:24, function (x) {list(cores, 4 * (2**x))})
+
+id <- function (x) {x}
+
+test <- function (arr) {
+ res <- swiftLapply(arr, id)
+ length(res)
+ return (NULL)
+}
+makeArray <- function (procs, arr.bytes) {
+ #arrsize is in bytes
+ args <- list()
+ for (i in 1:procs) {
+ args[[i]] <- as.vector(sample(1:1000, arr.bytes/intsize, replace=T), "integer")
+ }
+ return(args)
+}
+
+ts <- makeTestSuite(groups=list(
+ makePerfTestGroup(mode="swift", name="ArgSize",
+ f=test, prep=makeArray,
+ allargs=param.bytes,
+ server="local", cores=cores),
+ makePerfTestGroup(mode="swift", name="ArgSize",
+ f=test, prep=makeArray,
+ allargs=param.bytes,
+ server="ssh", cores=1, hosts="nettle wapato dandelion cattail")
+ ))
+res <- runTestSuite(ts)
+
+anl <- mergeGroupResults(analyseSuitePerf(res,
+ argnames=c("procs", "arr.bytes"), perfparams=c('server')))
+
+
+
+loc <- subset(anl, server=="local")
+ssh <- subset(anl, server=="ssh")
+plot(x=ssh$arr.bytes/1024, y=ssh$time, col='red', ylab="Time (s)", xlab="Argument size (bytes)",
+# log="xy", ylim=c(0.1, 500),
+ xaxt="n")
+title(main="Effect of argument size on execution time", line=2)
+title(main="Execution time of identity function with one function invocation per core", line=1, font.main=4, cex.main=0.8)
+axis(1, at=ssh$arr.bytes/1024,
+ labels=lapply(ssh$arr.bytes,
+ function (n) { if (n >= 1024*1024) {sprintf("%dM", n/(1024*1024))}
+ else if (n >= 1024 ) {sprintf("%dK", n/1024)}
+ else {sprintf("%d", n)}})
+ )
+points(x=loc$arr.bytes/1024, y=loc$time)
+legend("topleft", legend=c("Local (4 cores)", "Ssh (4 hosts, 1 core per host)"),
+ fill=c("black", "red"))
Added: SwiftApps/SwiftR/Swift/tests/perf/serialisation_overhead.R
===================================================================
--- SwiftApps/SwiftR/Swift/tests/perf/serialisation_overhead.R (rev 0)
+++ SwiftApps/SwiftR/Swift/tests/perf/serialisation_overhead.R 2011-09-15 02:54:05 UTC (rev 5118)
@@ -0,0 +1,70 @@
+
+require(Swift)
+source("Swift/tests/perf_tests.R")
+source("Swift/tests/OpenMx/BootstrapParallelBigger.R")
+
+args = 128
+res = list(n=c(), create=c(), prepro=c(), write=c(), writesep=c(), writesepsplit=c())
+i = 1
+for (n in c(1,2,4,8,16,32,48, 64, 96, 128, 192, 256, 384, 512)) {
+ cat("\n", n, "\n\n")
+ res$n[[i]] = n
+
+ startTime = proc.time()[["elapsed"]]
+ model <- buildModels(75, 100, n)
+ endTime = proc.time()[["elapsed"]]
+ res$create[[i]] = endTime - startTime
+ cat(paste("Took", endTime - startTime, "s to build models\n"))
+
+
+
+ startTime = proc.time()[["elapsed"]]
+ dataList <- generateDataList(model)
+ dshare <- shareData(model)
+ independents <- getAllIndependents(dshare)
+ endTime = proc.time()[["elapsed"]]
+ res$prepro[[i]] = endTime - startTime
+ cat(paste("Took", endTime - startTime, "s to process model\n"))
+
+
+ x = list(mxRun, independents)
+ wStartTime = proc.time()[["elapsed"]]
+ save(x, file="blah.Rdata")
+ wEndTime = proc.time()[["elapsed"]]
+ cat(paste("Took", wEndTime - wStartTime, "s to write to disk models\n"))
+ res$write[[i]] = wEndTime - wStartTime
+
+ wStartTime = proc.time()[["elapsed"]]
+ for (j in 1:n) {
+ x = list(mxRun, independents[[j]])
+ save(x, file=paste("blah", j, ".Rdata", sep=""))
+ }
+ wEndTime = proc.time()[["elapsed"]]
+ cat(paste("Took", wEndTime - wStartTime, "s to write to disk models individually\n"))
+ res$writesep[[i]] = wEndTime - wStartTime
+
+
+ wStartTime = proc.time()[["elapsed"]]
+ y = list(mxRun)
+ save(y, file=paste("mxRun.Rdata", sep=""))
+ for (j in 1:n) {
+ x = independents[[j]]
+ save(x, file=paste("blah", j, ".Rdata", sep=""))
+ }
+ wEndTime = proc.time()[["elapsed"]]
+ cat(paste("Took", wEndTime - wStartTime, "s to write to disk models individually split\n"))
+ res$writesepsplit[[i]] = wEndTime - wStartTime
+
+ i <- i + 1
+}
+plot(res$n, res$writesep, col="green", type="l", xlab="No. replicas", ylab="Time (s)")
+points(res$n, res$write, col="red", type="l")
+points(res$n, res$writesepsplit, col="blue", type="l")
+points(res$n, res$create, col="black", type="l")
+points(res$n, res$prepro, col="brown", type="l")
+
+legend("topleft",
+ legend=c("write separately", "write single file",
+ "write function\nand args separately", "time to create omx models",
+ "time to identify independent models"),
+ fill=c("green", "red", 'blue', 'black', 'brown'))
Added: SwiftApps/SwiftR/Swift/tests/platform-specific/batch_test.R
===================================================================
--- SwiftApps/SwiftR/Swift/tests/platform-specific/batch_test.R (rev 0)
+++ SwiftApps/SwiftR/Swift/tests/platform-specific/batch_test.R 2011-09-15 02:54:05 UTC (rev 5118)
@@ -0,0 +1,38 @@
+require(Swift)
+
+server <- commandArgs(TRUE)[[1]]
+
+
+hostinfo1 <- getNodeList()
+print("Auto:")
+print(hostinfo1)
+
+if (server == "cobalt") {
+ hostinfo2 <- getNodeList("cobalt")
+ print("cobalt:")
+ print(hostinfo2)
+}
+if (server %in% c("pbs", "pbsf")) {
+ hostinfo2 <- getNodeList("pbs")
+ print("pbs:")
+ print(hostinfo2)
+
+ hostinfo3 <- getNodeList("pbsf")
+ print("pbsf:")
+ print(hostinfo3)
+}
+if (server == "sge") {
+ hostinfo2 <- getNodeList("sge")
+ print("sge:")
+ print(hostinfo2)
+
+ hostinfo3 <- getNodeList("pbsf")
+ print("pbsf:")
+ print(hostinfo3)
+}
+
+print ("Hosts:")
+print(hostinfo1[[1]])
+
+runAllSwiftTests(server="ssh", hosts=hostinfo1[[1]], cores=8)
+
Added: SwiftApps/SwiftR/Swift/tests/platform-specific/cobalt_test.sub
===================================================================
--- SwiftApps/SwiftR/Swift/tests/platform-specific/cobalt_test.sub (rev 0)
+++ SwiftApps/SwiftR/Swift/tests/platform-specific/cobalt_test.sub 2011-09-15 02:54:05 UTC (rev 5118)
@@ -0,0 +1,12 @@
+#!/bin/sh
+export WORKER_LOGGING_LEVEL=ERROR
+echo '***' COBALT_NODEFILE file: $COBALT_NODEFILE
+cat $COBALT_NODEFILE
+echo '***' unique nodes are:
+sort < $COBALT_NODEFILE|uniq
+
+echo GLOBUS_HOSTNAME $GLOBUS_HOSTNAME
+export GLOBUS_HOSTNAME=""
+Rscript `dirname $0`/batch_test.R cobalt
+
+
Property changes on: SwiftApps/SwiftR/Swift/tests/platform-specific/cobalt_test.sub
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/SwiftR/Swift/tests/platform-specific/pbs_test.sub
===================================================================
--- SwiftApps/SwiftR/Swift/tests/platform-specific/pbs_test.sub (rev 0)
+++ SwiftApps/SwiftR/Swift/tests/platform-specific/pbs_test.sub 2011-09-15 02:54:05 UTC (rev 5118)
@@ -0,0 +1,33 @@
+#PBS -S /bin/sh
+#PBS -N SwiftR-batch
+#PBS -m n
+#PBS -l nodes=1
+#PBS -l walltime=00:30:00
+
+HOST=$(echo http://192.5.86.6:48998 | sed -e 's,^http://,,' -e 's/:.*//')
+PORT=$(echo http://192.5.86.6:48998 | sed -e 's,^.*:,,')
+echo '***' PBS_NODEFILE file: $PBS_NODEFILE
+cat $PBS_NODEFILE
+echo '***' unique nodes are:
+sort < $PBS_NODEFILE|uniq
+
+echo GLOBUS_HOSTNAME=$GLOBUS_HOSTNAME
+#export PATH=/autonfs/home/tga/R-2.12.1/bin:$PATH
+
+if which Rscript; then
+ echo Rscript binary found!
+else
+ echo Rscript binary not found in:
+ echo PATH=$PATH
+ echo Maybe you need to set PATH in this sub file
+ exit 1
+fi
+
+
+#Rscript `dirname $0`/batch_test.R pbs
+if [ -f $PBS_O_WORKDIR/Swift/tests/batch_test.R ]; then
+ Rscript $PBS_O_WORKDIR/Swift/tests/batch_test.R pbs
+else
+ echo Could not find batch_test.R. Maybe try modifying this .sub file with the absolute path of the script.
+ exit 1
+fi
Deleted: SwiftApps/SwiftR/Swift/tests/serialisation_overhead.R
===================================================================
--- SwiftApps/SwiftR/Swift/tests/serialisation_overhead.R 2011-09-14 23:06:12 UTC (rev 5117)
+++ SwiftApps/SwiftR/Swift/tests/serialisation_overhead.R 2011-09-15 02:54:05 UTC (rev 5118)
@@ -1,70 +0,0 @@
-
-require(Swift)
-source("Swift/tests/perf_tests.R")
-source("Swift/tests/OpenMx/BootstrapParallelBigger.R")
-
-args = 128
-res = list(n=c(), create=c(), prepro=c(), write=c(), writesep=c(), writesepsplit=c())
-i = 1
-for (n in c(1,2,4,8,16,32,48, 64, 96, 128, 192, 256, 384, 512)) {
- cat("\n", n, "\n\n")
- res$n[[i]] = n
-
- startTime = proc.time()[["elapsed"]]
- model <- buildModels(75, 100, n)
- endTime = proc.time()[["elapsed"]]
- res$create[[i]] = endTime - startTime
- cat(paste("Took", endTime - startTime, "s to build models\n"))
-
-
-
- startTime = proc.time()[["elapsed"]]
- dataList <- generateDataList(model)
- dshare <- shareData(model)
- independents <- getAllIndependents(dshare)
- endTime = proc.time()[["elapsed"]]
- res$prepro[[i]] = endTime - startTime
- cat(paste("Took", endTime - startTime, "s to process model\n"))
-
-
- x = list(mxRun, independents)
- wStartTime = proc.time()[["elapsed"]]
- save(x, file="blah.Rdata")
- wEndTime = proc.time()[["elapsed"]]
- cat(paste("Took", wEndTime - wStartTime, "s to write to disk models\n"))
- res$write[[i]] = wEndTime - wStartTime
-
- wStartTime = proc.time()[["elapsed"]]
- for (j in 1:n) {
- x = list(mxRun, independents[[j]])
- save(x, file=paste("blah", j, ".Rdata", sep=""))
- }
- wEndTime = proc.time()[["elapsed"]]
- cat(paste("Took", wEndTime - wStartTime, "s to write to disk models individually\n"))
- res$writesep[[i]] = wEndTime - wStartTime
-
-
- wStartTime = proc.time()[["elapsed"]]
- y = list(mxRun)
- save(y, file=paste("mxRun.Rdata", sep=""))
- for (j in 1:n) {
- x = independents[[j]]
- save(x, file=paste("blah", j, ".Rdata", sep=""))
- }
- wEndTime = proc.time()[["elapsed"]]
- cat(paste("Took", wEndTime - wStartTime, "s to write to disk models individually split\n"))
- res$writesepsplit[[i]] = wEndTime - wStartTime
-
- i <- i + 1
-}
-plot(res$n, res$writesep, col="green", type="l", xlab="No. replicas", ylab="Time (s)")
-points(res$n, res$write, col="red", type="l")
-points(res$n, res$writesepsplit, col="blue", type="l")
-points(res$n, res$create, col="black", type="l")
-points(res$n, res$prepro, col="brown", type="l")
-
-legend("topleft",
- legend=c("write separately", "write single file",
- "write function\nand args separately", "time to create omx models",
- "time to identify independent models"),
- fill=c("green", "red", 'blue', 'black', 'brown'))
More information about the Swift-commit
mailing list