[Swift-commit] r3297 - in SwiftApps/SwiftR: . Swift Swift/R Swift/exec Swift/man Swift/tests
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Fri Apr 30 11:36:30 CDT 2010
Author: wilde
Date: 2010-04-30 11:36:30 -0500 (Fri, 30 Apr 2010)
New Revision: 3297
Added:
SwiftApps/SwiftR/Swift/
SwiftApps/SwiftR/Swift/DESCRIPTION
SwiftApps/SwiftR/Swift/R/
SwiftApps/SwiftR/Swift/R/Swift.R
SwiftApps/SwiftR/Swift/exec/
SwiftApps/SwiftR/Swift/exec/RunR.sh
SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh
SwiftApps/SwiftR/Swift/exec/swiftapply.swift
SwiftApps/SwiftR/Swift/man/
SwiftApps/SwiftR/Swift/man/Swift-package.Rd
SwiftApps/SwiftR/Swift/tests/
SwiftApps/SwiftR/Swift/tests/TestSwift.R
Removed:
SwiftApps/SwiftR/RunR.sh
SwiftApps/SwiftR/RunSwiftScript.sh
SwiftApps/SwiftR/Swift.R
SwiftApps/SwiftR/TestSwift.R
SwiftApps/SwiftR/swiftapply.swift
Modified:
SwiftApps/SwiftR/TODO
Log:
Created initial version of Swift package dir and relocated files to it.
Deleted: SwiftApps/SwiftR/RunR.sh
===================================================================
--- SwiftApps/SwiftR/RunR.sh 2010-04-30 15:19:39 UTC (rev 3296)
+++ SwiftApps/SwiftR/RunR.sh 2010-04-30 16:36:30 UTC (rev 3297)
@@ -1,13 +0,0 @@
-#! /usr/bin/env Rscript
-
-argv = commandArgs(TRUE)
-
-load(argv[1]);
-
-result=list()
-for(c in 1:length(rcall$arglistbatch)) {
- # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status)
- result[[c]] = do.call( rcall$func, rcall$arglistbatch[[c]] )
-}
-
-save(result,file=argv[2])
Deleted: SwiftApps/SwiftR/RunSwiftScript.sh
===================================================================
--- SwiftApps/SwiftR/RunSwiftScript.sh 2010-04-30 15:19:39 UTC (rev 3296)
+++ SwiftApps/SwiftR/RunSwiftScript.sh 2010-04-30 16:36:30 UTC (rev 3297)
@@ -1,32 +0,0 @@
-rundir=$1
-site=$2
-
-cd $rundir
-
-cat >tc <<EOF
-$site RunR /home/wilde/SwiftR/RunR.sh null null null
-EOF
-
-cat >sites.xml <<EOF
-<config>
- <pool handle="local">
- <execution provider="local" url="none" />
- <profile namespace="karajan" key="initialScore">10000</profile>
- <profile namespace="karajan" key="jobThrottle">.11</profile>
- <filesystem provider="local"/>
- <workdirectory>$(pwd)</workdirectory>
- </pool>
- <pool handle="pbs">
- <profile namespace="globus" key="maxwalltime">00:00:10</profile>
- <profile namespace="globus" key="maxtime">1800</profile>
- <execution provider="coaster" url="none" jobManager="local:pbs"/>
- <profile namespace="globus" key="workersPerNode">1</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <profile namespace="karajan" key="jobThrottle">5.99</profile>
- <filesystem provider="local"/>
- <workdirectory>$(pwd)</workdirectory>
- </pool>
-</config>
-EOF
-
-swift -tc.file tc -sites.file sites.xml ../swiftapply.swift
Added: SwiftApps/SwiftR/Swift/DESCRIPTION
===================================================================
--- SwiftApps/SwiftR/Swift/DESCRIPTION (rev 0)
+++ SwiftApps/SwiftR/Swift/DESCRIPTION 2010-04-30 16:36:30 UTC (rev 3297)
@@ -0,0 +1,10 @@
+Package: Swift
+Type: Package
+Title: R interface to Swift parallel scripting languaage
+Version: 0.1
+Date: 2010-02-25
+Author: Michael Wilde
+Maintainer: Michael Wilde <wilde at mcs.anl.gov>
+Description: Routines to invoke R functions on remote resources through Swift.
+License: Apache License
+LazyLoad: yes
Copied: SwiftApps/SwiftR/Swift/R/Swift.R (from rev 3254, SwiftApps/SwiftR/Swift.R)
===================================================================
--- SwiftApps/SwiftR/Swift/R/Swift.R (rev 0)
+++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-04-30 16:36:30 UTC (rev 3297)
@@ -0,0 +1,98 @@
+swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL )
+{
+ # Move swiftprops into a Swift namespace
+
+# if(!exists("swiftprops")) {
+# swiftprops <<-list()
+# swiftprops$site <<- "local"
+# swiftprops$callsperbatch <<- 1
+# }
+
+ if (is.null(getOption("swift.site")))
+ options(swift.site="local");
+ if (is.null(getOption("swift.callsperbatch")))
+ options(swift.callsperbatch=1)
+
+ # Set Swift properties
+
+# if(is.null(site))
+# if( is.null(swiftprops) || is.null(swiftprops$site))
+# site <- "local"
+# else
+# site <- swiftprops$site
+# if(is.null(callsperbatch))
+# if( is.null(swiftprops) || is.null(swiftprops$callsperbatch))
+# callsperbatch <- 1
+# else
+# Callsperbatch <- swiftprops$callsperbatch
+
+ if(is.null(site))
+ site <- getOption("swift.site")
+ if(is.null(callsperbatch))
+ callsperbatch <- getOption("swift.callsperbatch")
+
+ cat("\nSwift R properties:\n")
+ cat(" site =",site,"\n");
+ cat(" callsperbatch =",callsperbatch,"\n")
+ cat("\nCurrent dir: ",getwd(),"\n");
+
+ # Execute the calls in batches
+
+ rundir <- system("mktemp -d SwiftR.run.XXX",intern=TRUE)
+ cat("Swift running in",rundir,"\n")
+ narglists <- length(arglists) # number of arglists to process
+ batch <- 1 # Next arglist batch number to fill
+ arglist <- 1 # Next arglist number to insert
+ while(arglist <= narglists) {
+ arglistsleft <- narglists - arglist + 1
+ if(arglistsleft >= callsperbatch) {
+ batchsize <- callsperbatch
+ }
+ else {
+ batchsize <- arglistsleft
+ }
+ arglistbatch <- list()
+ for(i in 1 : batchsize) {
+ arglistbatch[[i]] <- arglists[[arglist]]
+ arglist <- arglist +1
+ }
+ rcall <- list(func=func,arglistbatch=arglistbatch)
+ save(rcall,file=paste(rundir,"/cbatch.",as.character(batch),".Rdata",sep=""))
+ batch <- batch + 1;
+ }
+ nbatches <- batch - 1
+ RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh")
+ RunRScript <- system.file(package="Swift","exec/RunR.sh")
+ swiftapplyScript <- system.file(package="Swift","exec/swiftapply.swift")
+ system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript,sep=" "))
+
+ # Fetch the batch results
+
+ rno <- 1
+ rlist <- list()
+ for(batch in 1:nbatches) {
+ result <- NULL
+ load(paste(rundir,"/rbatch.",as.character(batch),".Rdata",sep=""))
+ nresults <- length(result)
+ for(r in 1:nresults) {
+ rlist[[rno]] <- result[[r]]
+ rno <- rno + 1
+ }
+ }
+ return(rlist)
+}
+
+swiftLapply <- function( tlist, func, ... )
+{
+ arglists <- list()
+ narglists <- length(tlist)
+ for(i in 1 : narglists) {
+ arglists[[i]] <- list(tlist[[i]], ...);
+ }
+ swiftapply(func, arglists)
+}
+#####
+#* checking R code for possible problems ... NOTE
+#swiftapply: no visible binding for '<<-' assignment to 'swiftprops'
+#swiftapply: no visible binding for global variable 'swiftprops'
+#swiftapply: no visible binding for global variable 'result'
Copied: SwiftApps/SwiftR/Swift/exec/RunR.sh (from rev 3251, SwiftApps/SwiftR/RunR.sh)
===================================================================
--- SwiftApps/SwiftR/Swift/exec/RunR.sh (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/RunR.sh 2010-04-30 16:36:30 UTC (rev 3297)
@@ -0,0 +1,13 @@
+#! /usr/bin/env Rscript
+
+argv = commandArgs(TRUE)
+
+load(argv[1]);
+
+result=list()
+for(c in 1:length(rcall$arglistbatch)) {
+ # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status)
+ result[[c]] = do.call( rcall$func, rcall$arglistbatch[[c]] )
+}
+
+save(result,file=argv[2])
Copied: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh (from rev 3254, SwiftApps/SwiftR/RunSwiftScript.sh)
===================================================================
--- SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-04-30 16:36:30 UTC (rev 3297)
@@ -0,0 +1,35 @@
+rundir=$1
+site=$2
+script=$3
+runR=$4
+
+cd $rundir
+
+cat >tc <<EOF
+$site BadRunR $runR null null null
+$site RunR $runR null null ENV::R_TESTS="";ENV::PATH="$PATH"
+EOF
+
+cat >sites.xml <<EOF
+<config>
+ <pool handle="local">
+ <execution provider="local" url="none" />
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="karajan" key="jobThrottle">.11</profile>
+ <filesystem provider="local"/>
+ <workdirectory>$(pwd)</workdirectory>
+ </pool>
+ <pool handle="pbs">
+ <profile namespace="globus" key="maxwalltime">00:00:10</profile>
+ <profile namespace="globus" key="maxtime">1800</profile>
+ <execution provider="coaster" url="none" jobManager="local:pbs"/>
+ <profile namespace="globus" key="workersPerNode">1</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="karajan" key="jobThrottle">5.99</profile>
+ <filesystem provider="local"/>
+ <workdirectory>$(pwd)</workdirectory>
+ </pool>
+</config>
+EOF
+
+swift -tc.file tc -sites.file sites.xml $script
Copied: SwiftApps/SwiftR/Swift/exec/swiftapply.swift (from rev 3251, SwiftApps/SwiftR/swiftapply.swift)
===================================================================
--- SwiftApps/SwiftR/Swift/exec/swiftapply.swift (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-04-30 16:36:30 UTC (rev 3297)
@@ -0,0 +1,13 @@
+type RFile;
+
+app (RFile result) RunR (RFile rcall)
+{
+ RunR @rcall @result;
+}
+
+RFile rcalls[] <simple_mapper; prefix="cbatch.", suffix=".Rdata", padding=0>;
+RFile results[] <simple_mapper; prefix="rbatch.", suffix=".Rdata", padding=0>;
+
+foreach c, i in rcalls {
+ results[i] = RunR(c);
+}
Added: SwiftApps/SwiftR/Swift/man/Swift-package.Rd
===================================================================
--- SwiftApps/SwiftR/Swift/man/Swift-package.Rd (rev 0)
+++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-04-30 16:36:30 UTC (rev 3297)
@@ -0,0 +1,62 @@
+\name{Swift-package}
+\alias{Swift-package}
+\alias{Swift}
+\alias{swiftapply}
+\alias{swiftLapply}
+\docType{package}
+\title{
+R interface to Swift parallel scripting language
+}
+\description{
+Description: Routines to invoke R functions and Swift scripts on remote resources through Swift.
+R functions can be remotely executed in parallel in a manner similar to Snow using a list of argument lists.
+Eventually more general Swift functions can be embedded and invked remotely as well.
+}
+\details{
+\tabular{ll}{
+Package: \tab Swift\cr
+Type: \tab Package\cr
+Version: \tab 1.0\cr
+Date: \tab 2010-02-25\cr
+License: \tab Globus Toolkit Public License v3 (based on Apache License 2.0): http://www.globus.org/toolkit/legal/4.0/license-v3.html \cr
+LazyLoad: \tab yes\cr
+}
+To use this package, create a list of argument lists, and then invoke: swiftapply(function,arglists).
+
+As a preliminary interface, you can set R options() to control Swift's operation:
+
+options(swift.callsperbatch=n) # n = number of R calls to perform in each Swift job.
+
+options(swift.site=sitename) # sitename = "local" to run on the current host and "pbs" to submit to a local PBS cluster.
+
+}
+\author{
+Michael Wilde
+
+Maintainer: Michael Wilde <wilde at mcs.anl.gov>
+}
+\references{
+http://www.ci/uchicago.edu/swift
+}
+\keyword{ parallel and distributed execution }
+%%\seealso{
+%%~~ Optional links to other man pages, e.g. ~~
+%%~~ \code{\link[<pkg>:<pkg>-package]{<pkg>}} ~~
+%%}
+\examples{
+
+require("boot")
+sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) }
+
+args=list(ducks,dogs)
+arglist = rep(list(args),9)
+res = swiftapply(sumcrits,arglist)
+
+res = swiftapply(sumcrits,arglist,callsperbatch=10)
+
+res = swiftapply(sumcrits,arglist,callsperbatch=2)
+
+res = swiftapply(sumcrits,arglist,callsperbatch=3)
+
+# res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs")
+}
Copied: SwiftApps/SwiftR/Swift/tests/TestSwift.R (from rev 3253, SwiftApps/SwiftR/TestSwift.R)
===================================================================
--- SwiftApps/SwiftR/Swift/tests/TestSwift.R (rev 0)
+++ SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-04-30 16:36:30 UTC (rev 3297)
@@ -0,0 +1,68 @@
+require(boot)
+#source("Swift.R")
+require(Swift)
+
+sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) }
+
+args=list(ducks,dogs)
+arglist = rep(list(args),9)
+
+###
+require("boot")
+sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) }
+
+args=list(ducks,dogs)
+arglist = rep(list(args),9)
+res = swiftapply(sumcrits,arglist)
+
+res = swiftapply(sumcrits,arglist,callsperbatch=10)
+
+res = swiftapply(sumcrits,arglist,callsperbatch=2)
+
+res = swiftapply(sumcrits,arglist,callsperbatch=3)
+
+# res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs")
+###
+
+
+if(TRUE) { # Basic tests
+
+ res = do.call(sumcrits,args)
+ cat("Test of do.call(sumcrits)\n")
+ print(res)
+
+ cat("\nTest of swiftapply(sumcrits,arglist)\n")
+ res = swiftapply(sumcrits,arglist)
+ print(res)
+}
+
+if(FALSE) { # Test various batch sizes
+
+ cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n")
+ res = swiftapply(sumcrits,arglist,callsperbatch=10)
+ print(res)
+
+ cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=2)\n")
+ res = swiftapply(sumcrits,arglist,callsperbatch=2)
+ print(res)
+
+ cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=3)\n")
+ res = swiftapply(sumcrits,arglist,callsperbatch=3)
+ print(res)
+
+ cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=20)\n")
+ res = swiftapply(sumcrits,arglist,callsperbatch=20)
+ print(res)
+}
+
+if(FALSE) { # Larger-scale tests
+
+ cat("\nTest of swiftapply(sumcrits,arglist[1000],callsperbatch=1)\n")
+
+ arglist = rep(list(args),100)
+# res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs")
+ res = swiftapply(sumcrits,arglist,callsperbatch=2)
+
+ print(res[[1]])
+ print(res[[1000]])
+}
Deleted: SwiftApps/SwiftR/Swift.R
===================================================================
--- SwiftApps/SwiftR/Swift.R 2010-04-30 15:19:39 UTC (rev 3296)
+++ SwiftApps/SwiftR/Swift.R 2010-04-30 16:36:30 UTC (rev 3297)
@@ -1,67 +0,0 @@
-swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL )
-{
- # Move swiftprops into a Swift namespace
-
- if(!exists("swiftprops")) {
- swiftprops <<-list()
- swiftprops$site <<- "local"
- swiftprops$callsperbatch <<- 1
- }
-
- # Set Swift properties
-
- if(is.null(site))
- if( is.null(swiftprops) || is.null(swiftprops$site))
- site <- "local"
- else
- site <- swiftprops$site
- if(is.null(callsperbatch))
- if( is.null(swiftprops) || is.null(swiftprops$callsperbatch))
- callsperbatch <- 1
- else
- callsperbatch <- swiftprops$callsperbatch
- cat("\nSwift R properties:\n")
- cat(" site =",site,"\n");
- cat(" callsperbatch =",callsperbatch,"\n")
-
- # Execute the calls in batches
-
- rundir <- system("mktemp -d SwiftR.run.XXX",intern=TRUE)
- cat("Swift running in",rundir,"\n")
- narglists <- length(arglists) # number of arglists to process
- batch <- 1 # Next arglist batch number to fill
- arglist <- 1 # Next arglist number to insert
- while(arglist <= narglists) {
- arglistsleft <- narglists - arglist + 1
- if(arglistsleft >= callsperbatch) {
- batchsize <- callsperbatch
- }
- else {
- batchsize <- arglistsleft
- }
- arglistbatch <- list()
- for(i in 1 : batchsize) {
- arglistbatch[[i]] <- arglists[[arglist]]
- arglist <- arglist +1
- }
- rcall <- list(func=func,arglistbatch=arglistbatch)
- save(rcall,file=paste(rundir,"/cbatch.",as.character(batch),".Rdata",sep=""))
- batch <- batch + 1;
- }
- nbatches <- batch - 1
- system(paste("./RunSwiftScript.sh",rundir,site,sep=" "))
-
- # Fetch the batch results
-
- rno <- 1
- rlist <- list()
- for(batch in 1:nbatches) {
- load(paste(rundir,"/rbatch.",as.character(batch),".Rdata",sep=""))
- nresults <- length(result)
- for(r in 1:nresults) {
- rlist[[rno]] <- result[[r]]
- rno <- rno + 1
- }
- }
- return(rlist)
-}
Modified: SwiftApps/SwiftR/TODO
===================================================================
--- SwiftApps/SwiftR/TODO 2010-04-30 15:19:39 UTC (rev 3296)
+++ SwiftApps/SwiftR/TODO 2010-04-30 16:36:30 UTC (rev 3297)
@@ -1,4 +1,42 @@
+*** NOTES on where everything lives:
+
+Am testing on PADS
+~/SwiftR is my "project" main working dir
+
+R is under ~/R and ~/R/pads (compiled for PADS; the ~/R/bin/R executable gets a library error on pads)
+OpenMx source tree checked out under: ~/SwiftR/OpenMx
+
+R packages are installed under: ~/RPackages
+
+--
+
+(Note: dont yet know if we do or do not need separate compiles between
+communicado, bridles, pads. teraport, and other systems on the CI
+net. Hopefully nit; if we do, weill need to create a tree to R
+releases each with a separate subtree for user-installed packages)
+Seems that we do, at least for PADS, we get this error:
+
+login1$ ~/R/bin/R
+/home/wilde/R/lib64/R/bin/exec/R:
+ error while loading shared libraries: libreadline.so.4:
+ cannot open shared object file: No such file or directory
+login1$
+
+--
+
+
+Swift package *source* (tbd) is under:
+
+~/SwiftR/Swift (the "Swift" package)
+
+Swift package is installed under:
+
+
+
+
+*** TO DO LIST:
+
x n args
x batch
x into svn
Deleted: SwiftApps/SwiftR/TestSwift.R
===================================================================
--- SwiftApps/SwiftR/TestSwift.R 2010-04-30 15:19:39 UTC (rev 3296)
+++ SwiftApps/SwiftR/TestSwift.R 2010-04-30 16:36:30 UTC (rev 3297)
@@ -1,48 +0,0 @@
-require(boot)
-source("Swift.R")
-
-sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) }
-
-args=list(ducks,dogs)
-arglist = rep(list(args),9)
-
-if(TRUE) { # Basic tests
-
- res = do.call(sumcrits,args)
- cat("Test of do.call(sumcrits)\n")
- print(res)
-
- cat("\nTest of swiftapply(sumcrits,arglist)\n")
- res = swiftapply(sumcrits,arglist)
- print(res)
-}
-
-if(FALSE) { # Test various batch sizes
-
- cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n")
- res = swiftapply(sumcrits,arglist,callsperbatch=10)
- print(res)
-
- cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=2)\n")
- res = swiftapply(sumcrits,arglist,callsperbatch=2)
- print(res)
-
- cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=3)\n")
- res = swiftapply(sumcrits,arglist,callsperbatch=3)
- print(res)
-
- cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=20)\n")
- res = swiftapply(sumcrits,arglist,callsperbatch=20)
- print(res)
-}
-
-if(FALSE) { # Larger-scale tests
-
- cat("\nTest of swiftapply(sumcrits,arglist[1000],callsperbatch=1)\n")
-
- arglist = rep(list(args),1000)
- res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs")
-
- print(res[[1]])
- print(res[[1000]])
-}
Deleted: SwiftApps/SwiftR/swiftapply.swift
===================================================================
--- SwiftApps/SwiftR/swiftapply.swift 2010-04-30 15:19:39 UTC (rev 3296)
+++ SwiftApps/SwiftR/swiftapply.swift 2010-04-30 16:36:30 UTC (rev 3297)
@@ -1,13 +0,0 @@
-type RFile;
-
-app (RFile result) RunR (RFile rcall)
-{
- RunR @rcall @result;
-}
-
-RFile rcalls[] <simple_mapper; prefix="cbatch.", suffix=".Rdata", padding=0>;
-RFile results[] <simple_mapper; prefix="rbatch.", suffix=".Rdata", padding=0>;
-
-foreach c, i in rcalls {
- results[i] = RunR(c);
-}
More information about the Swift-commit
mailing list