From noreply at svn.ci.uchicago.edu Tue Feb 1 02:31:13 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 1 Feb 2011 02:31:13 -0600 (CST) Subject: [Swift-commit] r4046 - branches/release-0.92/tests Message-ID: <20110201083113.B20F69CC6B@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-01 02:31:12 -0600 (Tue, 01 Feb 2011) New Revision: 4046 Modified: branches/release-0.92/tests/run-nightly.sh Log: generic run-nightly Modified: branches/release-0.92/tests/run-nightly.sh =================================================================== --- branches/release-0.92/tests/run-nightly.sh 2011-01-30 09:03:27 UTC (rev 4045) +++ branches/release-0.92/tests/run-nightly.sh 2011-02-01 08:31:12 UTC (rev 4046) @@ -1,21 +1,23 @@ #!/bin/bash -# example file for setting up environment and running nightly.sh +# sets up environment and runs nightly.sh # given a groupslistfile, will run nightly.sh on that site GROUPSLISTFILE=$1 # E.g., groups/group-all-local.sh # user-specific variables -export WORK=/ci/projects/cnari/swift_work/skenny -export QUEUE=short -export PROJECT=CI-IBN000039 +export WORK=$2 +export QUEUE=$3 +export PROJECT=$4 # branch testing -export COG_VERSION=branches/4.1.8 -export SWIFT_VERSION=branches/release-0.92 +export COG_VERSION=$5 +export SWIFT_VERSION=$6 -/home/skenny/swift_runs/tests/nightly.sh -g $GROUPSLISTFILE +#$2/nightly.sh -g $GROUPSLISTFILE +$2/nightly.sh -g $GROUPSLISTFILE + exit 0 From noreply at svn.ci.uchicago.edu Tue Feb 1 02:55:10 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 1 Feb 2011 02:55:10 -0600 (CST) Subject: [Swift-commit] r4047 - branches/release-0.92/tests Message-ID: <20110201085510.929B79CC94@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-01 02:55:10 -0600 (Tue, 01 Feb 2011) New Revision: 4047 Added: branches/release-0.92/tests/sitetester Log: new version of meta.sh Added: branches/release-0.92/tests/sitetester =================================================================== --- branches/release-0.92/tests/sitetester (rev 0) +++ branches/release-0.92/tests/sitetester 2011-02-01 08:55:10 UTC (rev 4047) @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +import commands +import os + +# the workdir for a given site needs to exist on the site you're testing +# and should contain run-nightly & nightly.sh +# this script will ssh onto the site, pass run-nightly the necesary variables +# for running nightly.sh + +class Site: + + def __init__(self,login,logtype,workdir,queue,project): + self.login = login + self.logintype = logtype + self.workdir = workdir + self.queue = queue + self.project = project + + def testSite(self,cogv,swiftv,test_type): + cmdline = self.logintype+" "+self.login+" "+self.workdir+"/run-nightly.sh "+self.workdir+"/"+test_type+" "+self.workdir+" "+self.queue+" "+self.project+" "+cogv+" "+swiftv + print "running......"+cmdline + res = commands.getoutput(cmdline) + print res + cmdline2 = "scp -r "+s+":"+rundir+" ." + res2 = commands.getoutput(cmdline) + + + + +#---------------------main-------------------------------------------------------- + +sites = [] + +s_logins = ['skenny at login2.pads.ci.uchicago.edu','login-abe.ncsa.teragrid.org','login1-qb.loni-lsu.teragrid.org'] +logintype = ['ssh','gsissh','gsissh'] +workdirs = ['/home/skenny/swift_runs/tests','/u/ac/skenny/swift_runs/tests','/home/skenny/swift_runs/tests'] +queues = ['short','debug','workq'] +projects = ['CI-IBN000039','TG-DBS080004N','TG-DBS080004N'] + +test_type = "groups/local-pbs-coasters.sh" +cogv = 'branches/4.1.8' +swiftv = 'branches/release-0.92' + +rundir = commands.getoutput('date +\"%Y-%m-%d\"') +rundir = "run-"+rundir + +for idx,s in enumerate(s_logins): + sites.append(Site(s,logintype[idx],workdirs[idx],queues[idx],projects[idx])) + +# loop over all or run single + +sites[1].testSite(cogv,swiftv,test_type) +sites[2].testSite(cogv,swiftv,test_type) + + + + + Property changes on: branches/release-0.92/tests/sitetester ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Wed Feb 2 10:13:20 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 2 Feb 2011 10:13:20 -0600 (CST) Subject: [Swift-commit] r4048 - in SwiftApps/SwiftR/mxtests: . snowfall swift Message-ID: <20110202161320.300F99CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-02 10:13:19 -0600 (Wed, 02 Feb 2011) New Revision: 4048 Added: SwiftApps/SwiftR/mxtests/snowfall/ SwiftApps/SwiftR/mxtests/snowfall/BootstrapParallelBigger.R SwiftApps/SwiftR/mxtests/snowfall/RAM-3Factor-96Indicators-covdata-a.R SwiftApps/SwiftR/mxtests/swift/ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R SwiftApps/SwiftR/mxtests/swift/RAM-3Factor-96Indicators-covdata-a.R Removed: SwiftApps/SwiftR/mxtests/BootstrapParallelBigger.R Log: Checking in Snowfall versions of Mx benchmarks, and creating copies to convert to swift. Deleted: SwiftApps/SwiftR/mxtests/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/BootstrapParallelBigger.R 2011-02-01 08:55:10 UTC (rev 4047) +++ SwiftApps/SwiftR/mxtests/BootstrapParallelBigger.R 2011-02-02 16:13:19 UTC (rev 4048) @@ -1,118 +0,0 @@ -# -# Copyright 2007-2010 The OpenMx Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require(OpenMx) -require(snowfall) - -hostfile <- Sys.getenv(c("PBS_NODEFILE")) - -hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] -ncpus <- length(hostnames) - -sfSetMaxCPUs(ncpus) -sfInit(parallel=TRUE, cpus=ncpus, socketHosts=hostnames) -sfLibrary(OpenMx) - -set.seed(10) - -# parameters for the simulation: lambda = factor loadings, -# specifics = specific variances -nVar <- 75 -nObs <- 10000 -nReps <- 1024 -goodStartValues <- TRUE -if (!is.logical(goodStartValues)) { - stop("'goodStartValues' should be logical. Try again.") -} - -lambda <- matrix(1:nVar*2/nVar, nVar, 1) -specifics <- diag(nVar) -chl <- chol(lambda %*% t(lambda) + specifics) - -# indices for parameters and hessian estimate in results -pStrt <- 3 -pEnd <- pStrt + 2*nVar - 1 -hStrt <- pEnd + 1 -hEnd <- hStrt + 2*nVar - 1 - -# dimension names for OpenMx -dn <- list() -dn[[1]] <- paste("Var", 1:nVar, sep="") -dn[[2]] <- dn[[1]] - -# function to get a covariance matrix -randomCov <- function(nObs, nVar, chl, dn) { - x <- matrix(rnorm(nObs*nVar), nObs, nVar) - x <- x %*% chl - thisCov <- cov(x) - dimnames(thisCov) <- dn - return(thisCov) -} - -createNewModel <- function(index, prefix, model) { - modelname <- paste(prefix, index, sep='') - model at data@observed <- randomCov(nObs, nVar, chl, dn) - model at name <- modelname - return(model) -} - -getStats <- function(model) { - retval <- c(model at output$status[[1]], - max(abs(model at output$gradient)), - model at output$estimate, - sqrt(diag(solve(model at output$hessian)))) - return(retval) -} - - -# initialize obsCov for MxModel -obsCov <- randomCov(nObs, nVar, chl, dn) - -# results matrix: get results for each simulation -results <- matrix(0, nReps, hEnd) -dnr <- c("inform", "maxAbsG", paste("lambda", 1:nVar, sep=""), - paste("specifics", 1:nVar, sep=""), - paste("hessLambda", 1:nVar, sep=""), - paste("hessSpecifics", 1:nVar, sep="")) -dimnames(results)[[2]] <- dnr - -# instantiate MxModel -template <- mxModel(name="stErrSim", - mxMatrix(name="lambda", type="Full", nrow=nVar, ncol=1, - free=TRUE, values=(1:nVar*2/nVar)*goodStartValues), - mxMatrix(name="specifics", type="Diag", nrow=nVar, - free=TRUE, values=rep(1, nVar)), - mxAlgebra(lambda %*% t(lambda) + specifics, - name="preCov", dimnames=dn), - mxData(observed=obsCov, type="cov", numObs=nObs), - mxMLObjective(covariance='preCov'), - independent = TRUE) - -topModel <- mxModel(name = 'container') - -sfExportAll() - -submodels <- lapply(1:nReps, createNewModel, 'stErrSim', template) - -names(submodels) <- omxExtractNames(submodels) -topModel at submodels <- submodels - -modelResults <- mxRun(topModel, silent=TRUE, suppressWarnings=TRUE) - -print(ncpus) -print(modelResults at output$wallTime) - -sfStop() - Added: SwiftApps/SwiftR/mxtests/snowfall/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/snowfall/BootstrapParallelBigger.R (rev 0) +++ SwiftApps/SwiftR/mxtests/snowfall/BootstrapParallelBigger.R 2011-02-02 16:13:19 UTC (rev 4048) @@ -0,0 +1,118 @@ +# +# Copyright 2007-2010 The OpenMx Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require(OpenMx) +require(snowfall) + +hostfile <- Sys.getenv(c("PBS_NODEFILE")) + +hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] +ncpus <- length(hostnames) + +sfSetMaxCPUs(ncpus) +sfInit(parallel=TRUE, cpus=ncpus, socketHosts=hostnames) +sfLibrary(OpenMx) + +set.seed(10) + +# parameters for the simulation: lambda = factor loadings, +# specifics = specific variances +nVar <- 75 +nObs <- 10000 +nReps <- 1024 +goodStartValues <- TRUE +if (!is.logical(goodStartValues)) { + stop("'goodStartValues' should be logical. Try again.") +} + +lambda <- matrix(1:nVar*2/nVar, nVar, 1) +specifics <- diag(nVar) +chl <- chol(lambda %*% t(lambda) + specifics) + +# indices for parameters and hessian estimate in results +pStrt <- 3 +pEnd <- pStrt + 2*nVar - 1 +hStrt <- pEnd + 1 +hEnd <- hStrt + 2*nVar - 1 + +# dimension names for OpenMx +dn <- list() +dn[[1]] <- paste("Var", 1:nVar, sep="") +dn[[2]] <- dn[[1]] + +# function to get a covariance matrix +randomCov <- function(nObs, nVar, chl, dn) { + x <- matrix(rnorm(nObs*nVar), nObs, nVar) + x <- x %*% chl + thisCov <- cov(x) + dimnames(thisCov) <- dn + return(thisCov) +} + +createNewModel <- function(index, prefix, model) { + modelname <- paste(prefix, index, sep='') + model at data@observed <- randomCov(nObs, nVar, chl, dn) + model at name <- modelname + return(model) +} + +getStats <- function(model) { + retval <- c(model at output$status[[1]], + max(abs(model at output$gradient)), + model at output$estimate, + sqrt(diag(solve(model at output$hessian)))) + return(retval) +} + + +# initialize obsCov for MxModel +obsCov <- randomCov(nObs, nVar, chl, dn) + +# results matrix: get results for each simulation +results <- matrix(0, nReps, hEnd) +dnr <- c("inform", "maxAbsG", paste("lambda", 1:nVar, sep=""), + paste("specifics", 1:nVar, sep=""), + paste("hessLambda", 1:nVar, sep=""), + paste("hessSpecifics", 1:nVar, sep="")) +dimnames(results)[[2]] <- dnr + +# instantiate MxModel +template <- mxModel(name="stErrSim", + mxMatrix(name="lambda", type="Full", nrow=nVar, ncol=1, + free=TRUE, values=(1:nVar*2/nVar)*goodStartValues), + mxMatrix(name="specifics", type="Diag", nrow=nVar, + free=TRUE, values=rep(1, nVar)), + mxAlgebra(lambda %*% t(lambda) + specifics, + name="preCov", dimnames=dn), + mxData(observed=obsCov, type="cov", numObs=nObs), + mxMLObjective(covariance='preCov'), + independent = TRUE) + +topModel <- mxModel(name = 'container') + +sfExportAll() + +submodels <- lapply(1:nReps, createNewModel, 'stErrSim', template) + +names(submodels) <- omxExtractNames(submodels) +topModel at submodels <- submodels + +modelResults <- mxRun(topModel, silent=TRUE, suppressWarnings=TRUE) + +print(ncpus) +print(modelResults at output$wallTime) + +sfStop() + Added: SwiftApps/SwiftR/mxtests/snowfall/RAM-3Factor-96Indicators-covdata-a.R =================================================================== --- SwiftApps/SwiftR/mxtests/snowfall/RAM-3Factor-96Indicators-covdata-a.R (rev 0) +++ SwiftApps/SwiftR/mxtests/snowfall/RAM-3Factor-96Indicators-covdata-a.R 2011-02-02 16:13:19 UTC (rev 4048) @@ -0,0 +1,133 @@ +# --------------------------------------------------------------------- +# Program: RAM-3Factor-12Indicators.R +# Author: Steven M. Boker +# Date: Fri Jul 30 13:45:12 EDT 2010 +# +# This program is a factor model using standard RAM. +# +# --------------------------------------------------------------------- +# Revision History +# -- Fri Jul 30 13:45:12 EDT 2010 +# Created RAM-3Factor-12Indicators.R. +# +# --------------------------------------------------------------------- + +# ---------------------------------- +# Read libraries and set options. + +library(OpenMx) +library(snowfall) + +hostfile <- Sys.getenv(c("PBS_NODEFILE")) + +hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] +ncpus <- length(hostnames) + +sfSetMaxCPUs(ncpus) +sfInit(parallel=TRUE, cpus=ncpus, socketHosts = hostnames) +sfLibrary(OpenMx) + +options(width=100) +set.seed(10) + +# --------------------------------------------------------------------- +# Data for factor model. + +numberSubjects <- 1000 +numberFactors <- 3 +numberIndPerFactor <- 32 +numberIndicators <- numberIndPerFactor*numberFactors # must be a multiple of numberFactors + +XMatrix <- matrix(rnorm(numberSubjects*numberFactors, mean=0, sd=1), numberSubjects, numberFactors) + +tLoadings <- c(1, seq(.5, .9, length.out=(numberIndPerFactor-1)), rep(0, numberIndPerFactor*2), + rep(0, numberIndPerFactor*1), 1, seq(.5, .9, length.out=(numberIndPerFactor-1)), rep(0, numberIndPerFactor*1), + rep(0, numberIndPerFactor*2), 1, seq(.5, .9, length.out=(numberIndPerFactor-1))) +BMatrix <- matrix(tLoadings, numberFactors, numberIndicators, byrow=TRUE) +UMatrix <- matrix(rnorm(numberSubjects*numberIndicators, mean=0, sd=1), numberSubjects, numberIndicators) +YMatrix <- XMatrix %*% BMatrix + UMatrix + +cor(XMatrix) + +dimnames(YMatrix) <- list(NULL, paste("X", 1:numberIndicators, sep="")) + +YFrame <- data.frame(YMatrix) + +round(cor(YFrame), 3) +round(cov(YFrame), 3) + +indicators <- paste("X", 1:numberIndicators, sep="") +totalVars <- numberIndicators + numberFactors + +# ---------------------------------- +# Build an orthogonal simple structure factor model + +latents <- paste("F", 1:numberFactors, sep="") + +uniqueLabels <- paste("U_", indicators, sep="") +meanLabels <- paste("M_", latents, sep="") +factorVarLabels <- paste("Var_", latents, sep="") + +latents1 <- latents[1] +indicators1 <- indicators[1:numberIndPerFactor] +loadingLabels1 <- paste("b_F1", indicators[1:numberIndPerFactor], sep="") +latents2 <- latents[2] +indicators2 <- indicators[numberIndPerFactor+(1:numberIndPerFactor)] +loadingLabels2 <- paste("b_F2", indicators[numberIndPerFactor+(1:numberIndPerFactor)], sep="") +latents3 <- latents[3] +indicators3 <- indicators[(2*numberIndPerFactor)+(1:numberIndPerFactor)] +loadingLabels3 <- paste("b_F3", indicators[(2*numberIndPerFactor)+(1:numberIndPerFactor)], sep="") + +threeFactorOrthogonal <- mxModel("threeFactorOrthogonal", + type="RAM", + manifestVars=c(indicators), + latentVars=c(latents,"dummy1"), + mxPath(from=latents1, to=indicators1, + arrows=1, all=TRUE, + free=TRUE, values=.2, + labels=loadingLabels1), + mxPath(from=latents2, to=indicators2, + arrows=1, all=TRUE, + free=TRUE, values=.2, + labels=loadingLabels2), + mxPath(from=latents3, to=indicators3, + arrows=1, all=TRUE, + free=TRUE, values=.2, + labels=loadingLabels3), + mxPath(from=latents1, to=indicators1[1], + arrows=1, + free=FALSE, values=1), + mxPath(from=latents2, to=indicators2[1], + arrows=1, + free=FALSE, values=1), + mxPath(from=latents3, to=indicators3[1], + arrows=1, + free=FALSE, values=1), + mxPath(from=indicators, + arrows=2, + free=TRUE, values=.2, + labels=uniqueLabels), + mxPath(from=latents, + arrows=2, + free=TRUE, values=.8, + labels=factorVarLabels), + mxPath(from="one", to=indicators, + arrows=1, free=FALSE, values=0), + mxPath(from="one", to=c(latents), + arrows=1, free=TRUE, values=.1, + labels=meanLabels), + mxCI(c('A', 'S')), + mxData(observed=cov(YFrame), means=mean(YFrame), + numObs=nrow(YFrame), type="cov") + ) + +threeFactorOrthogonalOut <- mxRun(threeFactorOrthogonal) +threeFactorCI <- omxParallelCI(threeFactorOrthogonalOut) + +totalTime <- threeFactorCI at output$wallTime +seqTime <- threeFactorOrthogonalOut at output$wallTime + +print(ncpus) +print(totalTime - seqTime) + +sfStop() Copied: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R (from rev 4047, SwiftApps/SwiftR/mxtests/BootstrapParallelBigger.R) =================================================================== --- SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R (rev 0) +++ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-02 16:13:19 UTC (rev 4048) @@ -0,0 +1,117 @@ +# +# Copyright 2007-2010 The OpenMx Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require(OpenMx) +require(Swift) + +#hostfile <- Sys.getenv(c("PBS_NODEFILE")) + +#hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] +#ncpus <- length(hostnames) + +options(swift.initialexpr="library(OpenMx)") + + +set.seed(10) + +# parameters for the simulation: lambda = factor loadings, +# specifics = specific variances +nVar <- 75 +nObs <- 10000 +nReps <- 1024 +goodStartValues <- TRUE +if (!is.logical(goodStartValues)) { + stop("'goodStartValues' should be logical. Try again.") +} + +lambda <- matrix(1:nVar*2/nVar, nVar, 1) +specifics <- diag(nVar) +chl <- chol(lambda %*% t(lambda) + specifics) + +# indices for parameters and hessian estimate in results +pStrt <- 3 +pEnd <- pStrt + 2*nVar - 1 +hStrt <- pEnd + 1 +hEnd <- hStrt + 2*nVar - 1 + +# dimension names for OpenMx +dn <- list() +dn[[1]] <- paste("Var", 1:nVar, sep="") +dn[[2]] <- dn[[1]] + +# function to get a covariance matrix +randomCov <- function(nObs, nVar, chl, dn) { + x <- matrix(rnorm(nObs*nVar), nObs, nVar) + x <- x %*% chl + thisCov <- cov(x) + dimnames(thisCov) <- dn + return(thisCov) +} + +createNewModel <- function(index, prefix, model) { + modelname <- paste(prefix, index, sep='') + model at data@observed <- randomCov(nObs, nVar, chl, dn) + model at name <- modelname + return(model) +} + +getStats <- function(model) { + retval <- c(model at output$status[[1]], + max(abs(model at output$gradient)), + model at output$estimate, + sqrt(diag(solve(model at output$hessian)))) + return(retval) +} + + +# initialize obsCov for MxModel +obsCov <- randomCov(nObs, nVar, chl, dn) + +# results matrix: get results for each simulation +results <- matrix(0, nReps, hEnd) +dnr <- c("inform", "maxAbsG", paste("lambda", 1:nVar, sep=""), + paste("specifics", 1:nVar, sep=""), + paste("hessLambda", 1:nVar, sep=""), + paste("hessSpecifics", 1:nVar, sep="")) +dimnames(results)[[2]] <- dnr + +# instantiate MxModel +template <- mxModel(name="stErrSim", + mxMatrix(name="lambda", type="Full", nrow=nVar, ncol=1, + free=TRUE, values=(1:nVar*2/nVar)*goodStartValues), + mxMatrix(name="specifics", type="Diag", nrow=nVar, + free=TRUE, values=rep(1, nVar)), + mxAlgebra(lambda %*% t(lambda) + specifics, + name="preCov", dimnames=dn), + mxData(observed=obsCov, type="cov", numObs=nObs), + mxMLObjective(covariance='preCov'), + independent = TRUE) + +topModel <- mxModel(name = 'container') + +sfExportAll() + +submodels <- lapply(1:nReps, createNewModel, 'stErrSim', template) + +names(submodels) <- omxExtractNames(submodels) +topModel at submodels <- submodels + +modelResults <- mxRun(topModel, silent=TRUE, suppressWarnings=TRUE) + +print(ncpus) +print(modelResults at output$wallTime) + +sfStop() + Property changes on: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R ___________________________________________________________________ Name: svn:mergeinfo + Added: SwiftApps/SwiftR/mxtests/swift/RAM-3Factor-96Indicators-covdata-a.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/RAM-3Factor-96Indicators-covdata-a.R (rev 0) +++ SwiftApps/SwiftR/mxtests/swift/RAM-3Factor-96Indicators-covdata-a.R 2011-02-02 16:13:19 UTC (rev 4048) @@ -0,0 +1,133 @@ +# --------------------------------------------------------------------- +# Program: RAM-3Factor-12Indicators.R +# Author: Steven M. Boker +# Date: Fri Jul 30 13:45:12 EDT 2010 +# +# This program is a factor model using standard RAM. +# +# --------------------------------------------------------------------- +# Revision History +# -- Fri Jul 30 13:45:12 EDT 2010 +# Created RAM-3Factor-12Indicators.R. +# +# --------------------------------------------------------------------- + +# ---------------------------------- +# Read libraries and set options. + +library(OpenMx) +library(snowfall) + +hostfile <- Sys.getenv(c("PBS_NODEFILE")) + +hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] +ncpus <- length(hostnames) + +sfSetMaxCPUs(ncpus) +sfInit(parallel=TRUE, cpus=ncpus, socketHosts = hostnames) +sfLibrary(OpenMx) + +options(width=100) +set.seed(10) + +# --------------------------------------------------------------------- +# Data for factor model. + +numberSubjects <- 1000 +numberFactors <- 3 +numberIndPerFactor <- 32 +numberIndicators <- numberIndPerFactor*numberFactors # must be a multiple of numberFactors + +XMatrix <- matrix(rnorm(numberSubjects*numberFactors, mean=0, sd=1), numberSubjects, numberFactors) + +tLoadings <- c(1, seq(.5, .9, length.out=(numberIndPerFactor-1)), rep(0, numberIndPerFactor*2), + rep(0, numberIndPerFactor*1), 1, seq(.5, .9, length.out=(numberIndPerFactor-1)), rep(0, numberIndPerFactor*1), + rep(0, numberIndPerFactor*2), 1, seq(.5, .9, length.out=(numberIndPerFactor-1))) +BMatrix <- matrix(tLoadings, numberFactors, numberIndicators, byrow=TRUE) +UMatrix <- matrix(rnorm(numberSubjects*numberIndicators, mean=0, sd=1), numberSubjects, numberIndicators) +YMatrix <- XMatrix %*% BMatrix + UMatrix + +cor(XMatrix) + +dimnames(YMatrix) <- list(NULL, paste("X", 1:numberIndicators, sep="")) + +YFrame <- data.frame(YMatrix) + +round(cor(YFrame), 3) +round(cov(YFrame), 3) + +indicators <- paste("X", 1:numberIndicators, sep="") +totalVars <- numberIndicators + numberFactors + +# ---------------------------------- +# Build an orthogonal simple structure factor model + +latents <- paste("F", 1:numberFactors, sep="") + +uniqueLabels <- paste("U_", indicators, sep="") +meanLabels <- paste("M_", latents, sep="") +factorVarLabels <- paste("Var_", latents, sep="") + +latents1 <- latents[1] +indicators1 <- indicators[1:numberIndPerFactor] +loadingLabels1 <- paste("b_F1", indicators[1:numberIndPerFactor], sep="") +latents2 <- latents[2] +indicators2 <- indicators[numberIndPerFactor+(1:numberIndPerFactor)] +loadingLabels2 <- paste("b_F2", indicators[numberIndPerFactor+(1:numberIndPerFactor)], sep="") +latents3 <- latents[3] +indicators3 <- indicators[(2*numberIndPerFactor)+(1:numberIndPerFactor)] +loadingLabels3 <- paste("b_F3", indicators[(2*numberIndPerFactor)+(1:numberIndPerFactor)], sep="") + +threeFactorOrthogonal <- mxModel("threeFactorOrthogonal", + type="RAM", + manifestVars=c(indicators), + latentVars=c(latents,"dummy1"), + mxPath(from=latents1, to=indicators1, + arrows=1, all=TRUE, + free=TRUE, values=.2, + labels=loadingLabels1), + mxPath(from=latents2, to=indicators2, + arrows=1, all=TRUE, + free=TRUE, values=.2, + labels=loadingLabels2), + mxPath(from=latents3, to=indicators3, + arrows=1, all=TRUE, + free=TRUE, values=.2, + labels=loadingLabels3), + mxPath(from=latents1, to=indicators1[1], + arrows=1, + free=FALSE, values=1), + mxPath(from=latents2, to=indicators2[1], + arrows=1, + free=FALSE, values=1), + mxPath(from=latents3, to=indicators3[1], + arrows=1, + free=FALSE, values=1), + mxPath(from=indicators, + arrows=2, + free=TRUE, values=.2, + labels=uniqueLabels), + mxPath(from=latents, + arrows=2, + free=TRUE, values=.8, + labels=factorVarLabels), + mxPath(from="one", to=indicators, + arrows=1, free=FALSE, values=0), + mxPath(from="one", to=c(latents), + arrows=1, free=TRUE, values=.1, + labels=meanLabels), + mxCI(c('A', 'S')), + mxData(observed=cov(YFrame), means=mean(YFrame), + numObs=nrow(YFrame), type="cov") + ) + +threeFactorOrthogonalOut <- mxRun(threeFactorOrthogonal) +threeFactorCI <- omxParallelCI(threeFactorOrthogonalOut) + +totalTime <- threeFactorCI at output$wallTime +seqTime <- threeFactorOrthogonalOut at output$wallTime + +print(ncpus) +print(totalTime - seqTime) + +sfStop() Property changes on: SwiftApps/SwiftR/mxtests/swift/RAM-3Factor-96Indicators-covdata-a.R ___________________________________________________________________ Name: svn:mergeinfo + From noreply at svn.ci.uchicago.edu Wed Feb 2 10:46:44 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 2 Feb 2011 10:46:44 -0600 (CST) Subject: [Swift-commit] r4049 - SwiftApps/SwiftR/Swift/R Message-ID: <20110202164644.910549CC6B@svn.ci.uchicago.edu> Author: tga Date: 2011-02-02 10:46:44 -0600 (Wed, 02 Feb 2011) New Revision: 4049 Added: SwiftApps/SwiftR/Swift/R/Tests.R Log: Forgot to check in new tests file on previous commit. Added: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R (rev 0) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-02 16:46:44 UTC (rev 4049) @@ -0,0 +1,451 @@ + + +initSwiftTestOptions <- function() +{ + options(swift.site="service") + options(swift.keepwork=TRUE) + initcmds <- "initVar1 <<- 19; initVar2 <<- sqrt(400)+3" + options(swift.initialexpr=initcmds) # Set here; used in test group 4 +} + + + + +swiftTest_1.1 <- function() +{ + + initSwiftTestOptions() + + cat("\n*** Starting test 1.1 ***\n\n") + + sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } + data(cars) + data(trees) + + args=list(trees,cars) + arglist = rep(list(args),1) + + cat("Test of local do.call(sumstuff)\n") + localres = do.call(sumstuff,args) + cat("local result=\n") + print(localres) + + cat("\nTest of swiftapply(sumstuff,arglist)\n") + swiftres = swiftapply(sumstuff,arglist) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres[[1]])) { + cat("\n==> test 1.1 passed\n") + } else { + cat("\n==> test 1.1 FAILED !!!!!\n") + } +} + +basicSwiftTest <- function() { swiftTest_1.1() } + +# .... more tests from below to move here +swiftTest_4.1 <- function() +{ + sumivars <- function() { initVar1+initVar2 } + + args=list() + arglist = rep(list(args),1) + + localres = 42 + + cat("\nTest of swiftapply(sumivars,arglist)\n") + swiftres = swiftapply(sumivars,arglist) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres[[1]])) { + cat("\n==> test 4.1 passed\n") + } else { + cat("\n==> test 4.1 FAILED !!!!!\n") + } +} + +swiftTest_4.2 <- function() +{ + + options(swift.initialexpr="initVar3 <<- 123; initVar4 <<- 100"); + + mulivars <- function() { initVar3*initVar4 } + + args=list() + arglist = rep(list(args),1) + + localres = 12300; + + cat("\nTest of swiftapply(mulivars,arglist)\n") + swiftres = swiftapply(mulivars,arglist) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres[[1]])) { + cat("\n==> test 4.2 passed\n") + } else { + cat("\n==> test 4.2 FAILED !!!!!\n") + } +} + +# Performance tests + +swiftTest_6.1 <- function(delay,ncalls) +{ + options(swift.initialexpr="initVar3 <<- 123; initVar4 <<- 100"); + + timed <- function(delay) { Sys.sleep(delay); delay } + + args=list(delay) + arglist = rep(list(args),ncalls) + + cat("\nTest of swiftapply(delay,arglist)\n") + + startTime = proc.time()[["elapsed"]] + swiftres = swiftapply(timed,arglist) + endTime = proc.time()[["elapsed"]] + runTime <- endTime - startTime + + cat("\n\n ===> Total elapsed unit test time = ",runTime," seconds.\n\n") + + cat("Swift result:\n") + print(swiftres[[1]]) + + if(identical(delay,swiftres[[1]])) { + cat("\n==> test 6.1 passed\n") + } else { + cat("\n==> test 6.1 FAILED !!!!!\n") + } + +} + +runAllSwiftTests <- function() +{ + # Launch workerif nothing already running + # testPid will be NULL if nothihg launched + testPid <- tryCatch( + (function() { swiftapply(log, list(list(1)),quiet=TRUE) ; + return(NULL); })(), + error=function(x) {swiftInit();}) + + ### FIXME: Save prior options here: restore them when tests are done. + ### Recovery if interrrupted? + + failures=0 + + startTime = proc.time()[["elapsed"]] + + cat("\n*** Starting test group 1 - functions on simple data structures ***\n\n") + + swiftTest_1.1() + + ##### Test 1.2 + + # test 10 remote calls + + sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } + data(cars) + data(trees) + + args=list(trees,cars) + arglist <- rep(list(args),10) + + localres = do.call(sumstuff,args) + + cat("\n*** Test 1.2.1: 10 calls to substuff()\n") + swiftres <- swiftapply(sumstuff,arglist) + cat("Swift result:\n") + format(swiftres) + + diffs <- 0 + for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format( swiftres[[i]] ))) + } + } + + if(diffs == 0) { + cat("\n==> test 1.2.1 passed\n") + } else { + cat("\n!!!==> test 1.2.1 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + cat("\n*** Test 1.2.2: 10 calls to substuff() - callsperbatch=10\n") + swiftres = swiftapply(sumstuff,arglist,callsperbatch=10) + cat("Swift result:\n") + format(swiftres) + + diffs <- 0 + for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format( swiftres[[i]] ))) + } + } + + if(diffs == 0) { + cat("\n==> test 1.2.2 passed\n") + } else { + cat("\n!!!==> test 1.2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + cat("\n*** Test 1.2.3: 10 calls to substuff() - callsperbatch=2\n") + swiftres = swiftapply(sumstuff,arglist,callsperbatch=2) + cat("Swift result:\n") + format(swiftres) + + diffs <- 0 + for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i, format( swiftres[[i]] ))) + } + } + + if(diffs == 0) { + cat("\n==> test 1.2.3 passed\n") + } else { + cat("\n!!!==> test 1.2.3 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + cat("\n*** Test 1.2.4: 10 calls to substuff() - callsperbatch=3\n") + swiftres = swiftapply(sumstuff,arglist,callsperbatch=3) + swiftres <- swiftapply(sumstuff,arglist) + cat("Swift result:\n") + format(swiftres) + + diffs <- 0 + for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i, format( swiftres[[i]] ))) + } + } + + if(diffs == 0) { + cat("\n==> test 1.2.4 passed\n") + } else { + cat("\n!!!==> test 1.2.4 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + # swiftres = swiftapply(sumstuff,arglist,callsperbatch=2,site="pbs") + # test variations on local vs ssh vs pbs; coasters vs non; etc. + + + + ##### Test Group 2 + + cat("\n*** Starting test group 2 - test matrix passing***\n") + + matfunc <- function( m1, m2 ) + { + (1/m1) %*% m2 + } + + n <- 5 + m1 <- array(sin(1:n**2), dim=c(n,n)) + m2 <- t(m1) + + localres = matfunc(m1,m2) + + cat("\n*** Test 2.1: 100 calls to matfunc(dim=5x5) - callsperbatch=9\n") + + args=list(m1,m2) + arglist <- rep(list(args),100) + + swiftres = swiftapply(matfunc,arglist,callsperbatch=9) + + diffs <- 0 + #for(i in 1:length(swiftres) ) { + for(i in c(seq(1,100,10),100)) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) + } + } + + if(diffs == 0) { + cat("\n==> test 2.1 passed\n") + } else { + cat("\n!!!==> test 2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + n <- 237 + n <- 50 + m1 <- array(sin(1:n**2), dim=c(n,n)) + m2 <- t(m1) + + localres = matfunc(m1,m2) + + cat("\n*** Test 2.2: 123 calls to matfunc(dim=bigger) - callsperbatch=7\n") # FIXME make n easy to adjust and print actual value + + args=list(m1,m2) + arglist <- rep(list(args),123) + + swiftres = swiftapply(matfunc,arglist,callsperbatch=7) + + diffs <- 0 + #for(i in 1:length(swiftres) ) { + for(i in c(seq(1,length(swiftres),10),length(swiftres))) { + + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) + } + } + + if(diffs == 0) { + cat("\n==> test 2.2 passed\n") + } else { + cat("\n!!!==> test 2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + + ##### Test Group 3 + + cat("\n*** Starting test group 3 - test list element and name passing***\n") + + # Test if list element names are being sent and returned correctly + + n <- 5 + m1 <- array(sin(1:n**2), dim=c(n,n)) + m2 <- t(m1) + + inlist = list() + inlist[[1]]=123 + inlist[[2]]=456 + inlist$name1=789 + inlist$name2=987 + inlist$mat1 = m1 + inlist[[99]] = m2 + + listfunc <- function(ilist) + { + olist = ilist + olist$sum = ilist[[1]] + ilist[[2]] + ilist$name1 + ilist$name2 + olist$names = names(ilist) + olist$mprod = ilist$mat1 %*% ilist[[99]] + return(olist) + } + localres = listfunc(inlist) + + cat("\n*** Starting test 3.1 - 4 calls in one batch of 5 ***\n") + + args=list(inlist) + arglist <- rep(list(args),4) + + swiftres = swiftapply(listfunc,arglist,callsperbatch=5) + + diffs <- 0 + for(i in 1:length(swiftres) ) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d=%s\n",i,format(swiftres[[i]]))) + } + } + + if(diffs == 0) { + cat("\n==> test 3.1 passed\n") + } else { + cat("\n!!!==> test 3.1 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + cat("\n*** Starting test 3.2 - 99 calls in batches of 11 ***\n") + + args=list(inlist) + arglist <- rep(list(args),99) + + swiftres = swiftapply(listfunc,arglist,callsperbatch=11) + + diffs <- 0 + for(i in 1:length(swiftres) ) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) + } + } + + if(diffs == 0) { + cat("\n==> test 3.2 passed\n") + } else { + cat("\n!!!==> test 3.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + ##### Test Group 4 # test initialexpr string + + cat("\n*** Starting test group 4 - test remote R service initialization string ***\n") + + swiftTest_4.1() + swiftTest_4.2() + + + + + ##### Test Group 5 # test error handling + + cat("\n*** Starting test group 5 - test remote R service error ***\n") + + arglist = list(list(1.0),list(2.0),list("3.0"),list(4.0),list(5.0)) + + cat("\nTest of swiftapply(sumivars,arglist)\n") + swiftres = swiftapply(log,arglist) + cat("Swift result:\n") + print(swiftres) + + goodres = c("numeric","numeric","try-error","numeric","numeric") + + diffs <- 0 + for(i in 1:length(swiftres) ) { + if( class(swiftres[[i]]) != goodres[i] ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) + } + } + + if(diffs == 0) { + cat("\n==> test 5.1 passed\n") + } else { + cat("\n!!!==> test 5.1 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 + } + + endTime <- proc.time()[["elapsed"]] + runTime <- endTime - startTime + + cat("\n\n ===> Total elapsed test time = ",runTime," seconds.\n\n") + if (!is.null(testPid)) + swiftShutdown(testPid) +} # end function runAllTests + +#options(swift.site="local") +#runAllTests() + +testloop <- function(npass) +{ + for(i in 1:npass) { + cat("\n\n\n ***** Starting test pass ", i, " ***** \n\n\n"); + runAllSwiftTests() + cat("\n\n\n ***** Completed test pass ", i, " ***** \n\n\n"); + system("sleep 3") + } +} From noreply at svn.ci.uchicago.edu Wed Feb 2 10:50:13 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 2 Feb 2011 10:50:13 -0600 (CST) Subject: [Swift-commit] r4050 - SwiftApps/SwiftR/Swift/R Message-ID: <20110202165013.D57CD9CC6B@svn.ci.uchicago.edu> Author: tga Date: 2011-02-02 10:50:13 -0600 (Wed, 02 Feb 2011) New Revision: 4050 Modified: SwiftApps/SwiftR/Swift/R/Workers.R Log: Allowed hosts to be provided as a list instead of just as a single string. Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-02 16:46:44 UTC (rev 4049) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-02 16:50:13 UTC (rev 4050) @@ -62,9 +62,14 @@ if(is.null(hosts)) hosts <- getOption("swift.hosts") - if (is.null(hosts)) - hosts <- 1 if(! is.null(hosts) ) { + if (length(hosts) > 1) { + # Concatenate list of hosts + hosts <- paste(hosts, collapse=" ") + } + else if (length(hosts) == 0) { + stop(paste("Zero length host argument:", hosts)) + } cmdString <- paste(cmdString, "-h", shQuote(hosts) ) } @@ -85,6 +90,8 @@ else { if(is.null(nodes)) nodes <- getOption("swift.nodes") + if(is.null(nodes)) + nodes <- 1 # Default value if(! is.null(nodes) ) { cmdString <- paste(cmdString, "-n", shQuote(nodes)) } From noreply at svn.ci.uchicago.edu Wed Feb 2 11:36:20 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 2 Feb 2011 11:36:20 -0600 (CST) Subject: [Swift-commit] r4051 - in SwiftApps/SwiftR/Swift: R exec Message-ID: <20110202173620.E19669CC6B@svn.ci.uchicago.edu> Author: tga Date: 2011-02-02 11:36:20 -0600 (Wed, 02 Feb 2011) New Revision: 4051 Modified: SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/start-swift Log: Security fix: made sure that all of the temporary files created by various components of SwiftR are only readable/writeable by current user. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2011-02-02 16:50:13 UTC (rev 4050) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2011-02-02 17:36:20 UTC (rev 4051) @@ -1,3 +1,8 @@ +# Modes for directories and files created by SwiftR +# We only give permissions to the current user. +kDIR_MODE <- "700" +kFILE_MODE <- "600" + swiftapply <- function( func, arglists, server=NULL, callsperbatch=NULL, @@ -110,7 +115,8 @@ } else { requestdirbase = sprintf("%s/%s/SwiftR/requests.P%.5d",tmpdir,user,Sys.getpid()) - dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE) + dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE, + mode=kDIR_MODE) options(swift.requestdirbase=requestdirbase) requestid = 0; } @@ -119,7 +125,7 @@ # Execute the calls in batches reqdir = sprintf("%s/R%.7d",requestdirbase,requestid) - dir.create(reqdir,recursive=TRUE,showWarnings=FALSE) + dir.create(reqdir,recursive=TRUE,showWarnings=FALSE,mode=kDIR_MODE) if (! quiet) { cat("Swift request is in",reqdir,"\n") } Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-02 16:50:13 UTC (rev 4050) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-02 17:36:20 UTC (rev 4051) @@ -3,6 +3,11 @@ # Arguments: inputBatchSaveFile outputBatchSaveFile # bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; +# Set restrictive umask for duration of script +# This value prevents any other users from reading +# or writing +umask "077" + #if [ $# != 4 ]; then if [ $# != 3 ]; then echo $0: expecting 4 arguments, got $#: $* 1>&2 Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-02 16:50:13 UTC (rev 4050) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-02 17:36:20 UTC (rev 4051) @@ -4,6 +4,10 @@ export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap +# Set the umask to prevent any access by other users: +# there is no reason why any other user should need to look at +# the temporary files, etc that we create +umask 077 # Define internal functions From noreply at svn.ci.uchicago.edu Wed Feb 2 18:16:51 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 2 Feb 2011 18:16:51 -0600 (CST) Subject: [Swift-commit] r4052 - in SwiftApps/SwiftR: Swift/R mxtests/swift Message-ID: <20110203001651.8EF309CC6B@svn.ci.uchicago.edu> Author: tga Date: 2011-02-02 18:16:51 -0600 (Wed, 02 Feb 2011) New Revision: 4052 Modified: SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R Log: Initial work and notes on new swift functions to match snowfall functions: * swiftLibrary * swiftExportAll Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2011-02-02 17:36:20 UTC (rev 4051) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2011-02-03 00:16:51 UTC (rev 4052) @@ -58,11 +58,27 @@ # service: send requests to Swift service loop via fifo # script: run swift for each request, via RunSwiftScript.sh (currently broken) # manual: for testing, let user run remote R server manually + + # Build a list of "Library" statements for all of the libraries + # already specified through swiftLibrary commands. + lib <- getOption("swift.libraries") + if (!is.null(lib)) { + # library statements + stmts <- lapply(lib, + function (l) { + return (sprintf("library(%s);", l)); + }) + libstr = paste(stmts, collapse=" ") + } + else { + libstr = "" + } + if(is.null(initialexpr)) initialexpr <- getOption("swift.initialexpr") - if(is.null(initialexpr)) - initialexpr <- ""; + # Have library imports before other expressions + initialexpr <- paste(libstr, initialexpr, collapse=" "); if(is.null(workerhosts)) workerhosts <- getOption("swift.workerhosts") @@ -253,3 +269,67 @@ names(arglists) = names(tlist) swiftapply(func, arglists) } + +swiftLibrary <- function (newlib, reset=FALSE) +{ + # Add a library to be included on all workers + # The package can be provided directly or alternatively + # the package name can be provided. + if (reset) { + libs = list() + } + else { + libs <- getOption("swift.libraries") + if (is.null(libs)) + libs <- list() + } + for (l in newlib) { + if (!is.character(l)) { + # Maybe library was provided unquoted + #FIXME: doesn't work because l is evaluated above + l <- deparse(substitute(l)) + } + libs[[length(libs) + 1]] <- l + } + options(swift.libraries=libs) +} +#TODO: not implemented +.swiftExport <- function (...) { + # List of object names (as R symbols or as strings) + # These will be passed directly to save() to be serialized + + #TODO: + # We want the protocol to support workers coming and going from the + # cluster. We want to rely on less implicit state. Ie. if we run + # swiftExport, then a worker is added to cluster, then an apply call + # is run, we want the worker to include the exported objects. + # One approach to doing this is to enforce the export at apply + # time. + # + # + # When swiftExport() is called: + # * Save all of the objects to a file + # - Q:Can we assume its on a shared filesystem? + # * MAYBE: put the file somewhere accessible by all workers + # * append the file's path to a stored list of files + # * MAYBE LATER: asynchronously start swift tasks to distribute + # the data, to get a headstart on the work + # When swiftApply() is called: + # - Inform all worker R processes through some mechanism (TBD) + # of the list of all exported data files. (extra file?) + # - The worker goes through the list in order to ensure that name + # conflicts are resolved in the way you would expect. + # If it has loaded a file previously, it does nothing. + # If it hasn't loaded the file, it accesses the file (mechanism TBD) + # and loads it into the global namespace. + # data + # + # swiftRemoveAll(): + # delete the list of exported files from above and have a single + # entry which instructs workers to delete all their data. This is + # mainly important because it eliminates possibility of using huge + # amounts of memory. + # swiftRemove() + # Implementation of this function is somewhat problematic, as it only + # partially undoes previous work +} Modified: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-02 17:36:20 UTC (rev 4051) +++ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-03 00:16:51 UTC (rev 4052) @@ -16,13 +16,13 @@ require(OpenMx) require(Swift) -#hostfile <- Sys.getenv(c("PBS_NODEFILE")) +hostfile <- Sys.getenv(c("PBS_NODEFILE")) -#hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] -#ncpus <- length(hostnames) +hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] -options(swift.initialexpr="library(OpenMx)") +swiftSess <- swiftInit(server="ssh", cores=8, hosts=hostnames) +swiftLibrary("OpenMx") set.seed(10) @@ -101,11 +101,11 @@ topModel <- mxModel(name = 'container') -sfExportAll() +#sfExportAll() -submodels <- lapply(1:nReps, createNewModel, 'stErrSim', template) +submodels <- swiftLapply(1:nReps, createNewModel, 'stErrSim', template) -names(submodels) <- omxExtractNames(submodels) +names(submodels) <- imxExtractNames(submodels) topModel at submodels <- submodels modelResults <- mxRun(topModel, silent=TRUE, suppressWarnings=TRUE) @@ -113,5 +113,4 @@ print(ncpus) print(modelResults at output$wallTime) -sfStop() - +swiftShutdown(swiftSess) From noreply at svn.ci.uchicago.edu Thu Feb 3 11:56:20 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 3 Feb 2011 11:56:20 -0600 (CST) Subject: [Swift-commit] r4053 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110203175620.2F95D9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-03 11:56:19 -0600 (Thu, 03 Feb 2011) New Revision: 4053 Added: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R Removed: SwiftApps/SwiftR/Swift/exec/SwiftRServer.sh Modified: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh SwiftApps/SwiftR/Swift/exec/rserver.swift SwiftApps/SwiftR/Swift/exec/start-merlot SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/Swift/exec/start-swift-Rserver SwiftApps/SwiftR/Swift/exec/swiftapply.swift Log: * Fixed extension of SwiftRServer.sh to .R to reflect the fact it is an R script * added in error handling for invalid initialExpr values: previously the R workers would fail silently without reporting the problem back. Modified: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2011-02-03 00:16:51 UTC (rev 4052) +++ SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2011-02-03 17:56:19 UTC (rev 4053) @@ -73,7 +73,7 @@ # FIXME: fossil: was here: cat >RunR.sh <; file runRscript <"EvalRBatchPersistent.sh">; - file rsScript <"SwiftRServer.sh">; + file rsScript <"SwiftRServer.R">; foreach c, i in rcalls { (e[i], results[i],stout[i], sterr[i]) = runR(runRscript,rsScript,c); Modified: SwiftApps/SwiftR/Swift/exec/start-merlot =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-merlot 2011-02-03 00:16:51 UTC (rev 4052) +++ SwiftApps/SwiftR/Swift/exec/start-merlot 2011-02-03 17:56:19 UTC (rev 4053) @@ -215,7 +215,7 @@ #cp $script $SWIFTRBIN/passive-coaster-swift $SWIFTRBIN/swift.properties $rundir cp $script . script=$(basename $script) -cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.sh} . +cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.R} . # DONE: FIXME: rework this script to transfer all shells and rscripts # needed, and to copy in the R prelude for the R server processes (to Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-03 00:16:51 UTC (rev 4052) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-03 17:56:19 UTC (rev 4053) @@ -439,7 +439,7 @@ #cp $script $SWIFTRBIN/passive-coaster-swift $SWIFTRBIN/swift.properties $rundir cp $script . script=$(basename $script) -cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.sh} . +cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.R} . # DONE: FIXME: rework this script to transfer all shells and rscripts # needed, and to copy in the R prelude for the R server processes (to Modified: SwiftApps/SwiftR/Swift/exec/start-swift-Rserver =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-Rserver 2011-02-03 00:16:51 UTC (rev 4052) +++ SwiftApps/SwiftR/Swift/exec/start-swift-Rserver 2011-02-03 17:56:19 UTC (rev 4053) @@ -218,7 +218,7 @@ fi -cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.sh} . +cp $SWIFTRBIN/{EvalRBatchPersistent.sh,SwiftRServer.R} . script=$(basename $script) @@ -259,4 +259,4 @@ exit -## swift -tc.file tc -sites.file sites.xml rserver.swift \ No newline at end of file +## swift -tc.file tc -sites.file sites.xml rserver.swift Modified: SwiftApps/SwiftR/Swift/exec/swiftapply.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2011-02-03 00:16:51 UTC (rev 4052) +++ SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2011-02-03 17:56:19 UTC (rev 4053) @@ -20,7 +20,7 @@ #file runRscript <"RunR.sh">; file runRscript <"EvalRBatchPersistent.sh">; file initScript <"initialize.R">; -file rsScript <"SwiftRServer.sh">; +file rsScript <"SwiftRServer.R">; foreach c, i in rcalls { (results[i],stout[i], sterr[i]) = runR(runRscript,initScript,rsScript,c); From noreply at svn.ci.uchicago.edu Thu Feb 3 12:37:27 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 3 Feb 2011 12:37:27 -0600 (CST) Subject: [Swift-commit] r4054 - SwiftApps/SwiftR/Swift/R Message-ID: <20110203183727.13FDD9CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-03 12:37:26 -0600 (Thu, 03 Feb 2011) New Revision: 4054 Modified: SwiftApps/SwiftR/Swift/R/Swift.R Log: Working versions of swiftLibrary and swiftDetach functions which work analagously to built-in library and detach fucntions (but with fewer options). These functions are implemented in such a way that the sequence of library and detach calls is stored in R and added to the initialExpr for all the swift workers, so new workers can be brought online and have the correct libraries loaded without intervention from the user. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2011-02-03 17:56:19 UTC (rev 4053) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2011-02-03 18:37:26 UTC (rev 4054) @@ -61,14 +61,26 @@ # Build a list of "Library" statements for all of the libraries # already specified through swiftLibrary commands. - lib <- getOption("swift.libraries") + lib <- getOption(".swift.packages") if (!is.null(lib)) { # library statements stmts <- lapply(lib, - function (l) { - return (sprintf("library(%s);", l)); + function (libcmd) { + verb <- libcmd[1]; + if (verb == "library") { + return (sprintf("library(%s);", libcmd[2])); + } + else if (verb == "detach") { + return (sprintf("try(detach(package:%s),silent=T);", + libcmd[2])); + } + else { + error(paste("invalid verb ", verb, " in ", + ".swift.packages option")) + } }) libstr = paste(stmts, collapse=" ") + cat("libstr: ", libstr) } else { libstr = "" @@ -270,31 +282,93 @@ swiftapply(func, arglists) } -swiftLibrary <- function (newlib, reset=FALSE) -{ +swiftRemoveAll <- function () { + # Cleans up all data in global namespace on workers + options(.swift.exports=list(c("removeAll"))) +} + + +swiftLibrary <- function (packname) { # Add a library to be included on all workers # The package can be provided directly or alternatively # the package name can be provided. - if (reset) { - libs = list() + # If reset is true, then previously added libraries won't be + # reimported on new workers. + + # Check to see if it is a string, if it is not a string + # then we will deparse it to get the expression text + if (!is.character(substitute(packname))) { + # Maybe library was provided unquoted + packname <- deparse(substitute(packname)) } + if (! packname %in% installed.packages()) { + # Warn users in case they made a typo + warning(paste("Package", packname, + "was not a installed package in this instance of R,", + "but may be installed on SwiftR workers")) + } + plist <- getOption(".swift.packages") + if (is.null(plist)) { + plist <- list() + } else { - libs <- getOption("swift.libraries") - if (is.null(libs)) - libs <- list() + # scan libs for previous detaches and libraries of + # this package. Delete all of them + plist = Filter(function(packcmd) + { return (packcmd[2] != packname);}, + plist) } - for (l in newlib) { - if (!is.character(l)) { - # Maybe library was provided unquoted - #FIXME: doesn't work because l is evaluated above - l <- deparse(substitute(l)) - } - libs[[length(libs) + 1]] <- l + # Add the library command to the end of the list of commands to be + # executed in slave R instances + plist[[length(plist) + 1]] <- c("library", packname) + + options(.swift.packages=plist) +} + + +swiftDetach <- function (name) { + # name is an string or identifier such as "package:OpenMx", + # following the same pattern as the built in R detach() function + # Detaches a library from workers and ensures that it will no longer be + # imported. Note that the effect on dependent imports is a little + # messy and behaves differently on existing and new workers. + # If package A requires package B, and we import A on a worker, + # package B will also be imported on the worker. By detaching package A + # we don't also detach B. This contrasts to a fresh worker, where + # package B will not be imported. + if (!is.character(substitute(name))) { + # Maybe package was provided unquoted + name <- deparse(substitute(name)) } - options(swift.libraries=libs) + + #TODO: remove from options(".swift.packages") + # Scan through list and remove any attach or detach of this name + # add a detach cmd to the end + if (substr(name, 1, 8) != "package:") + stop("Can only detach packages. Package must be specified as package:PackageName") + # Get bit after "package:" + packname = substr(name, 9, nchar(name)) + plist = getOption(".swift.packages") + if (is.null(plist)) { + plist = list() + } + else { + # Previous export/detach commands are now redundant + plist = Filter(function(packcmd) + { return (packcmd[2] != packname);}, + plist) + } + plist[[length(plist) + 1]] = c("detach", packname) + + options(.swift.packages=plist) } + +swiftExportAll <- function () { + return (swiftExport(list=ls(globalenv()))) +} + #TODO: not implemented -.swiftExport <- function (...) { +.swiftExport <- function (..., list=NULL) { # List of object names (as R symbols or as strings) # These will be passed directly to save() to be serialized @@ -332,4 +406,16 @@ # swiftRemove() # Implementation of this function is somewhat problematic, as it only # partially undoes previous work + + + #Pseudocode: + # TODO: choose file location + # expFile <- ??? + save(..., list=list, file=expFile) + exportList <- getOption(".swift.exports") + if (is.null(exportList)) + exportList = list() #TODO: start with removeAll command? + exportList[[length(exportList) + 1]] = c("export", expFile) + options(.swift.exports=exportList) + } From noreply at svn.ci.uchicago.edu Thu Feb 3 14:00:40 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 3 Feb 2011 14:00:40 -0600 (CST) Subject: [Swift-commit] r4055 - in SwiftApps/SwiftR: . Swift Swift/R Message-ID: <20110203200040.9BC729CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-03 14:00:40 -0600 (Thu, 03 Feb 2011) New Revision: 4055 Added: SwiftApps/SwiftR/Swift/NAMESPACE SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/R/Export.R SwiftApps/SwiftR/Swift/R/Library.R Removed: SwiftApps/SwiftR/Swift/R/Swift.R Modified: SwiftApps/SwiftR/Makefile Log: Refactored R code a bit: * Added NAMESPACE declaration so that only functions intended to be exported will be exported * Split Swift.R into Export.R, Library.R and Apply.R as it was getting rather logn * Refactored swiftapply to use some subprocedures. Modified: SwiftApps/SwiftR/Makefile =================================================================== --- SwiftApps/SwiftR/Makefile 2011-02-03 18:37:26 UTC (rev 4054) +++ SwiftApps/SwiftR/Makefile 2011-02-03 20:00:40 UTC (rev 4055) @@ -3,6 +3,7 @@ PKG_FILES = $(shell find Swift/R -name '*.R'; find Swift/man -name '*.Rd'; find Swift/exec/ -type f -not -path '*/.svn*'; find Swift/inst/ -type f -not -path '*/.svn*') PKG_FILES += Swift/DESCRIPTION +PKG_FILES += Swift/NAMESPACE # Extract the version number from the R package description file Added: SwiftApps/SwiftR/Swift/NAMESPACE =================================================================== --- SwiftApps/SwiftR/Swift/NAMESPACE (rev 0) +++ SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-03 20:00:40 UTC (rev 4055) @@ -0,0 +1,10 @@ +export(swiftapply) +export(swiftLapply) +export(swiftInit) +export(swiftShutdown) +export(swiftLibrary) +export(swiftDetach) + +export(runAllSwiftTests) +export(basicSwiftTest) +exportPattern("^swiftTest") Copied: SwiftApps/SwiftR/Swift/R/Apply.R (from rev 4054, SwiftApps/SwiftR/Swift/R/Swift.R) =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R (rev 0) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-03 20:00:40 UTC (rev 4055) @@ -0,0 +1,301 @@ +# Modes for directories and files created by SwiftR +# We only give permissions to the current user. +kDIR_MODE <- "700" +kFILE_MODE <- "600" + +swiftapply <- function( func, arglists, + server=NULL, + callsperbatch=NULL, + runmode=NULL, + initialexpr=NULL, + workerhosts=NULL, + keepwork=NULL, + tmpdir=NULL, + timeout=NULL, + quiet=FALSE) +{ + # Set Swift default options if not passed as keywords or pre-set by user + if(is.null(server)) + server <- getOption("swift.server") + if(is.null(server)) + server<-"local" + + if(is.null(callsperbatch)) + callsperbatch <- getOption("swift.callsperbatch") + + if(is.null(callsperbatch)) { + # auto batching: + # start with 1 batch per core + wc <- workerCount(server) + if (is.null(wc)) { + if (! quiet ) + cat(paste("Information about current server of type", + server, "not found")) + callsperbatch <- 1 + } + else { + batches_per_worker <- getOption("swift.batchesperworker") + if (is.null(batches_per_worker)) + batches_per_worker <- 1 + target_batches <- wc * batches_per_worker + # We want to have at least batches_per_worker batches of calls + # for each worker. If it doesn't divide evenly, some will get + # batches_per_worker + 1 batches. + # callsperbatch < 1 doesn't make sense, hence the use of max + callsperbatch = max(1, floor(length(arglists)/target_batches)) +# if (! quiet ) +# cat(paste("Selected callsperbatch of", callsperbatch, +# "automatically, based on worker count of",wc, +# "and arg count of ", length(arglists))) + } + } + #cat("Got ", length(arglists), " arguments with batching factor ", callsperbatch) + + if(is.null(runmode)) + runmode <- getOption("swift.runmode") + if(is.null(runmode)) + runmode <- "service" + # service: send requests to Swift service loop via fifo + # script: run swift for each request, via RunSwiftScript.sh (currently broken) + # manual: for testing, let user run remote R server manually + + # Build a list of "Library" statements for all of the libraries + # already specified through swiftLibrary commands. + lib <- getOption(".swift.packages") + if (!is.null(lib)) { + # library statements + stmts <- lapply(lib, + function (libcmd) { + verb <- libcmd[1]; + if (verb == "library") { + return (sprintf("library(%s);", libcmd[2])); + } + else if (verb == "detach") { + return (sprintf("try(detach(package:%s),silent=T);", + libcmd[2])); + } + else { + error(paste("invalid verb ", verb, " in ", + ".swift.packages option")) + } + }) + libstr = paste(stmts, collapse=" ") + cat("libstr: ", libstr) + } + else { + libstr = "" + } + + + if(is.null(initialexpr)) + initialexpr <- getOption("swift.initialexpr") + # Have library imports before other expressions + initialexpr <- paste(libstr, initialexpr, collapse=" "); + + if(is.null(workerhosts)) + workerhosts <- getOption("swift.workerhosts") + if(is.null(workerhosts)) + workerhosts <- "localhost"; + + if(is.null(keepwork)) + keepwork <- getOption("swift.keepwork") + if(is.null(keepwork)) + keepwork <- FALSE; + + if(is.null(tmpdir)) + tmpdir <- getOption("swift.tmpdir") + if(is.null(tmpdir)) + tmpdir <- Sys.getenv("SWIFTR_TMP"); + + if(tmpdir=="") { + if(is.null(tmpdir)) + tmpdir <- getOption("swift.tmpdir") + tmpdir <- "/tmp"; + } + + if(is.null(quiet)) { + quiet <- getOption("swift.quiet") + if(is.null(quiet)) + quiet <- FALSE + } + + if (! quiet) { + cat("\nswiftapply to ", length(arglists), " arg lists.\n") + cat("\nSwift properties:\n") + cat(" server =", server,"\n") + cat(" callsperbatch =", callsperbatch,"\n") + cat(" runmode =", runmode,"\n") + cat(" tmpdir =", tmpdir,"\n") + cat(" workerhosts =", workerhosts,"\n") + cat(" initialexpr =", initialexpr,"\n\n") + } + + # Execute the calls in batches + reqdir <- setupRequestDir(tmpdir=tmpdir) + if (! quiet) { + cat("Swift request is in",reqdir,"\n") + } + + nbatches <- writeRequestBatches(func, arglists, initialexpr, + reqdir, callsperbatch) + + swiftapplyScript <- system.file(package="Swift","exec/swiftapply.swift") + if( runmode == "manual" ) { # Prompt for return (empty line) to continue; assumes user ran a manual R to process the call. + cat("Manual Swift Run:\n run dir: ", getwd(), "/", reqdir,"\n\n") + cat(" swift script: ", RunSwiftScript, "\n") + cat(" server: ", server,"\n") + cat(" swiftapplyScript: ", swiftapplyScript,"\n") + cat(" Use RunAllR.sh to process and press return when complete:") + system(paste("cp ", system.file(package="Swift","exec/RunAllR.sh"), reqdir)) + readLines(n=1) + } + else if (runmode == "script") { + RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh") + system(paste(RunSwiftScript,reqdir,server,swiftapplyScript,"\"",workerhosts,"\"")) + } + else { # runmode == "service" # FIXME: check and post error if not "service" + + # Send request to service + user <- Sys.info()[["user"]] + swiftServerDir = file.path(tmpdir,user,"SwiftR", + paste("swift.",server,sep="")) + + requestPipeName=file.path(swiftServerDir,"requestpipe") + resultPipeName=file.path(swiftServerDir,"resultpipe") + + # fifo will block irrecoverably if there is no reader on the + # other end of the requestPipe. This is bad. The swift worker + # script is responsible for deleting the request pipe when it + # shuts down, so we know if the requestPipe still exists there + # should still be a worker (or the worker crashed in a funny way). + if (file.exists(requestPipeName)) { + # there is a race condition here if the fifo disappears in + # between checking for existence and opening the fifo, but + # the timeout will catch that unlikely case + writeTimeout <- 4000 + success <- writeFifo(requestPipeName,paste(reqdir,"\n",sep=""), + timeout=writeTimeout) + if (! success) { + stop(paste("timeout of", writeTimeout, + "ms exceeded when attempting to", + "rendezvous with swift server: maybe it is not running or", + "it has crashed")) + } + + # Wait for reply from service + res <- readFifo(resultPipeName, timeout=timeout) + if ((! length(res) == 0) && is.na(res)) { + stop(paste("Timeout of ", timeout, "ms exceeded when waiting", + "for response from swift server")) + } + + # Check that the message was correct + successMsg <- "done" + if (res[[1]] != successMsg) + stop(paste("Got unexpected message '", + paste(res, collapse="\n"),"' on fifo ", + "aborting job", sep="")) + } + else { + stop(paste("Have you run swiftInit?\n", + "It appears that no SwiftR servers of type", server, + "are running, as no request pipe exists in", + swiftServerDir)) + } + } + + # Fetch the batch results + + rno <- 1 + rlist <- list() + for(batch in 1:nbatches) { + result <- NULL + load(file.path(reqdir, + paste("/rbatch.",as.character(batch),".Rdata",sep=""))) + nresults <- length(result) + for(r in 1:nresults) { + rlist[[rno]] <- result[[r]] +if(class(result[[r]]) == "try-error") { +cat("ERROR in eval: ", result[[r]], "\n"); +} + #DB cat("swiftapply: result rno=",rno,":\n") # FIXME: for logging + #DB cat(rlist[[rno]]@output$gradient,"\n") + rno <- rno + 1 + } + } + names(rlist) = names(arglists) + if( ! keepwork ) { + cat("Removing ", reqdir, "\n") + unlink(reqdir,recursive=TRUE) + } + return(rlist) +} + +swiftLapply <- function( tlist, func, ... ) +{ + if (length(tlist) == 0) return(tlist) + arglists <- list() + narglists <- length(tlist) + for(i in 1 : narglists) { + arglists[[i]] <- list(tlist[[i]], ...); + } + names(arglists) = names(tlist) + swiftapply(func, arglists) +} + + +setupRequestDir <- function (tmpdir) { + # Initialize globals if first call in this invocation of R + # Use the options mechanism so that setting is tied + # to lifetime of this R process. If we stored this in a global + # variable, it is possible that, say, directory requests.55555/R0000005 + # is created, the user exits the session without saving, and therefore + # the swift.requestid counter is out of step with the file system + requestdirbase = getOption("swift.requestdirbase") + if(!is.null(requestdirbase)) { + requestid = getOption("swift.requestid") + 1; + } + else { + requestdirbase = sprintf("%s/%s/SwiftR/requests.P%.5d",tmpdir, + Sys.info()[["user"]],Sys.getpid()) + dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE, + mode=kDIR_MODE) + options(swift.requestdirbase=requestdirbase) + requestid = 0; + } + options(swift.requestid=requestid) + reqdir = sprintf("%s/R%.7d",requestdirbase,requestid) + dir.create(reqdir,recursive=TRUE,showWarnings=FALSE,mode=kDIR_MODE) + return (reqdir) +} + + +writeRequestBatches <- function (func, arglists, initialexpr, + reqdir, callsperbatch) { + # Write the function call info out to cbatch.?.RData files in reqdir + # in batches of size specified by callsperbatch + # returns the number of batches written + narglists <- length(arglists) # number of arglists to process + batch <- 1 # Next arglist batch number to fill + arglist <- 1 # Next arglist number to insert + while(arglist <= narglists) { + arglistsleft <- narglists - arglist + 1 + if(arglistsleft >= callsperbatch) { + batchsize <- callsperbatch + } + else { + batchsize <- arglistsleft + } + arglistbatch <- list() + for(i in 1 : batchsize) { + arglistbatch[[i]] <- arglists[[arglist]] + arglist <- arglist +1 + } + rcall <- list(initializer=initialexpr,func=func,arglistbatch=arglistbatch) + save(rcall, + file=file.path(reqdir, + paste("cbatch.",as.character(batch),".Rdata",sep=""))) + batch <- batch + 1; + } + return (batch - 1) +} Property changes on: SwiftApps/SwiftR/Swift/R/Apply.R ___________________________________________________________________ Name: svn:mergeinfo + Added: SwiftApps/SwiftR/Swift/R/Export.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Export.R (rev 0) +++ SwiftApps/SwiftR/Swift/R/Export.R 2011-02-03 20:00:40 UTC (rev 4055) @@ -0,0 +1,64 @@ + +swiftRemoveAll <- function () { + # Cleans up all data in global namespace on workers + options(.swift.exports=list(c("removeAll"))) +} + + + +swiftExportAll <- function () { + return (swiftExport(list=ls(globalenv()))) +} + +#TODO: not implemented +.swiftExport <- function (..., list=NULL) { + # List of object names (as R symbols or as strings) + # These will be passed directly to save() to be serialized + + #TODO: + # We want the protocol to support workers coming and going from the + # cluster. We want to rely on less implicit state. Ie. if we run + # swiftExport, then a worker is added to cluster, then an apply call + # is run, we want the worker to include the exported objects. + # One approach to doing this is to enforce the export at apply + # time. + # + # + # When swiftExport() is called: + # * Save all of the objects to a file + # - Q:Can we assume its on a shared filesystem? + # * MAYBE: put the file somewhere accessible by all workers + # * append the file's path to a stored list of files + # * MAYBE LATER: asynchronously start swift tasks to distribute + # the data, to get a headstart on the work + # When swiftApply() is called: + # - Inform all worker R processes through some mechanism (TBD) + # of the list of all exported data files. (extra file?) + # - The worker goes through the list in order to ensure that name + # conflicts are resolved in the way you would expect. + # If it has loaded a file previously, it does nothing. + # If it hasn't loaded the file, it accesses the file (mechanism TBD) + # and loads it into the global namespace. + # data + # + # swiftRemoveAll(): + # delete the list of exported files from above and have a single + # entry which instructs workers to delete all their data. This is + # mainly important because it eliminates possibility of using huge + # amounts of memory. + # swiftRemove() + # Implementation of this function is somewhat problematic, as it only + # partially undoes previous work + + + #Pseudocode: + # TODO: choose file location + # expFile <- ??? + save(..., list=list, file=expFile) + exportList <- getOption(".swift.exports") + if (is.null(exportList)) + exportList = list() #TODO: start with removeAll command? + exportList[[length(exportList) + 1]] = c("export", expFile) + options(.swift.exports=exportList) + +} Added: SwiftApps/SwiftR/Swift/R/Library.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Library.R (rev 0) +++ SwiftApps/SwiftR/Swift/R/Library.R 2011-02-03 20:00:40 UTC (rev 4055) @@ -0,0 +1,75 @@ + +swiftLibrary <- function (packname) { + # Add a library to be included on all workers + # The package can be provided directly or alternatively + # the package name can be provided. + # If reset is true, then previously added libraries won't be + # reimported on new workers. + + # Check to see if it is a string, if it is not a string + # then we will deparse it to get the expression text + if (!is.character(substitute(packname))) { + # Maybe library was provided unquoted + packname <- deparse(substitute(packname)) + } + if (! packname %in% installed.packages()) { + # Warn users in case they made a typo + warning(paste("Package", packname, + "was not a installed package in this instance of R,", + "but may be installed on SwiftR workers")) + } + plist <- getOption(".swift.packages") + if (is.null(plist)) { + plist <- list() + } + else { + # scan libs for previous detaches and libraries of + # this package. Delete all of them + plist = Filter(function(packcmd) + { return (packcmd[2] != packname);}, + plist) + } + # Add the library command to the end of the list of commands to be + # executed in slave R instances + plist[[length(plist) + 1]] <- c("library", packname) + + options(.swift.packages=plist) +} + + +swiftDetach <- function (name) { + # name is an string or identifier such as "package:OpenMx", + # following the same pattern as the built in R detach() function + # Detaches a library from workers and ensures that it will no longer be + # imported. Note that the effect on dependent imports is a little + # messy and behaves differently on existing and new workers. + # If package A requires package B, and we import A on a worker, + # package B will also be imported on the worker. By detaching package A + # we don't also detach B. This contrasts to a fresh worker, where + # package B will not be imported. + if (!is.character(substitute(name))) { + # Maybe package was provided unquoted + name <- deparse(substitute(name)) + } + + #TODO: remove from options(".swift.packages") + # Scan through list and remove any attach or detach of this name + # add a detach cmd to the end + if (substr(name, 1, 8) != "package:") + stop("Can only detach packages. Package must be specified as package:PackageName") + # Get bit after "package:" + packname = substr(name, 9, nchar(name)) + plist = getOption(".swift.packages") + if (is.null(plist)) { + plist = list() + } + else { + # Previous export/detach commands are now redundant + plist = Filter(function(packcmd) + { return (packcmd[2] != packname);}, + plist) + } + plist[[length(plist) + 1]] = c("detach", packname) + + options(.swift.packages=plist) +} Deleted: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2011-02-03 18:37:26 UTC (rev 4054) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2011-02-03 20:00:40 UTC (rev 4055) @@ -1,421 +0,0 @@ -# Modes for directories and files created by SwiftR -# We only give permissions to the current user. -kDIR_MODE <- "700" -kFILE_MODE <- "600" - -swiftapply <- function( func, arglists, - server=NULL, - callsperbatch=NULL, - runmode=NULL, - initialexpr=NULL, - workerhosts=NULL, - keepwork=NULL, - tmpdir=NULL, - timeout=NULL, - quiet=FALSE) -{ - # Set Swift default options if not passed as keywords or pre-set by user - if(is.null(server)) - server <- getOption("swift.server") - if(is.null(server)) - server<-"local" - - if(is.null(callsperbatch)) - callsperbatch <- getOption("swift.callsperbatch") - - if(is.null(callsperbatch)) { - # auto batching: - # start with 1 batch per core - wc <- workerCount(server) - if (is.null(wc)) { - if (! quiet ) - cat(paste("Information about current server of type", - server, "not found")) - callsperbatch <- 1 - } - else { - batches_per_worker <- getOption("swift.batchesperworker") - if (is.null(batches_per_worker)) - batches_per_worker <- 1 - target_batches <- wc * batches_per_worker - # We want to have at least batches_per_worker batches of calls - # for each worker. If it doesn't divide evenly, some will get - # batches_per_worker + 1 batches. - # callsperbatch < 1 doesn't make sense, hence the use of max - callsperbatch = max(1, floor(length(arglists)/target_batches)) -# if (! quiet ) -# cat(paste("Selected callsperbatch of", callsperbatch, -# "automatically, based on worker count of",wc, -# "and arg count of ", length(arglists))) - } - } - #cat("Got ", length(arglists), " arguments with batching factor ", callsperbatch) - - if(is.null(runmode)) - runmode <- getOption("swift.runmode") - if(is.null(runmode)) - runmode <- "service" - # service: send requests to Swift service loop via fifo - # script: run swift for each request, via RunSwiftScript.sh (currently broken) - # manual: for testing, let user run remote R server manually - - # Build a list of "Library" statements for all of the libraries - # already specified through swiftLibrary commands. - lib <- getOption(".swift.packages") - if (!is.null(lib)) { - # library statements - stmts <- lapply(lib, - function (libcmd) { - verb <- libcmd[1]; - if (verb == "library") { - return (sprintf("library(%s);", libcmd[2])); - } - else if (verb == "detach") { - return (sprintf("try(detach(package:%s),silent=T);", - libcmd[2])); - } - else { - error(paste("invalid verb ", verb, " in ", - ".swift.packages option")) - } - }) - libstr = paste(stmts, collapse=" ") - cat("libstr: ", libstr) - } - else { - libstr = "" - } - - - if(is.null(initialexpr)) - initialexpr <- getOption("swift.initialexpr") - # Have library imports before other expressions - initialexpr <- paste(libstr, initialexpr, collapse=" "); - - if(is.null(workerhosts)) - workerhosts <- getOption("swift.workerhosts") - if(is.null(workerhosts)) - workerhosts <- "localhost"; - - if(is.null(keepwork)) - keepwork <- getOption("swift.keepwork") - if(is.null(keepwork)) - keepwork <- FALSE; - - if(is.null(tmpdir)) - tmpdir <- getOption("swift.tmpdir") - if(is.null(tmpdir)) - tmpdir <- Sys.getenv("SWIFTR_TMP"); - - if(tmpdir=="") { - if(is.null(tmpdir)) - tmpdir <- getOption("swift.tmpdir") - tmpdir <- "/tmp"; - } - - if(is.null(quiet)) { - quiet <- getOption("swift.quiet") - if(is.null(quiet)) - quiet <- FALSE - } - - if (! quiet) { - cat("\nswiftapply to ", length(arglists), " arg lists.\n") - cat("\nSwift properties:\n") - cat(" server =", server,"\n") - cat(" callsperbatch =", callsperbatch,"\n") - cat(" runmode =", runmode,"\n") - cat(" tmpdir =", tmpdir,"\n") - cat(" workerhosts =", workerhosts,"\n") - cat(" initialexpr =", initialexpr,"\n\n") - } - user <- Sys.info()[["user"]] - - # Initialize globals if first call in this invocation of R - # Use the options mechanism so that setting is tied - # to lifetime of this R process. If we stored this in a global - # variable, it is possible that, say, directory requests.55555/R0000005 - # is created, the user exits the session without saving, and therefore - # the swift.requestid counter is out of step with the file system - requestdirbase = getOption("swift.requestdirbase") - if(!is.null(requestdirbase)) { - requestid = getOption("swift.requestid") + 1; - } - else { - requestdirbase = sprintf("%s/%s/SwiftR/requests.P%.5d",tmpdir,user,Sys.getpid()) - dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE, - mode=kDIR_MODE) - options(swift.requestdirbase=requestdirbase) - requestid = 0; - } - options(swift.requestid=requestid) - - # Execute the calls in batches - - reqdir = sprintf("%s/R%.7d",requestdirbase,requestid) - dir.create(reqdir,recursive=TRUE,showWarnings=FALSE,mode=kDIR_MODE) - if (! quiet) { - cat("Swift request is in",reqdir,"\n") - } - - narglists <- length(arglists) # number of arglists to process - batch <- 1 # Next arglist batch number to fill - arglist <- 1 # Next arglist number to insert - while(arglist <= narglists) { - arglistsleft <- narglists - arglist + 1 - if(arglistsleft >= callsperbatch) { - batchsize <- callsperbatch - } - else { - batchsize <- arglistsleft - } - arglistbatch <- list() - for(i in 1 : batchsize) { - arglistbatch[[i]] <- arglists[[arglist]] - arglist <- arglist +1 - } - rcall <- list(initializer=initialexpr,func=func,arglistbatch=arglistbatch) - save(rcall,file=paste(reqdir,"/cbatch.",as.character(batch),".Rdata",sep="")) - batch <- batch + 1; - } - nbatches <- batch - 1 - swiftapplyScript <- system.file(package="Swift","exec/swiftapply.swift") - - if( runmode == "manual" ) { # Prompt for return (empty line) to continue; assumes user ran a manual R to process the call. - cat("Manual Swift Run:\n run dir: ", getwd(), "/", reqdir,"\n\n") - cat(" swift script: ", RunSwiftScript, "\n") - cat(" server: ", server,"\n") - cat(" swiftapplyScript: ", swiftapplyScript,"\n") - cat(" Use RunAllR.sh to process and press return when complete:") - system(paste("cp ", system.file(package="Swift","exec/RunAllR.sh"), reqdir)) - readLines(n=1) - } - else if (runmode == "script") { - RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh") - system(paste(RunSwiftScript,reqdir,server,swiftapplyScript,"\"",workerhosts,"\"")) - } - else { # runmode == "service" # FIXME: check and post error if not "service" - - # Send request to service - - swiftServerDir = paste(tmpdir,"/",user,"/SwiftR/swift.",server,sep="") - - requestPipeName=paste(swiftServerDir,"/requestpipe",sep="") - resultPipeName=paste(swiftServerDir,"/resultpipe",sep="") - - # fifo will block irrecoverably if there is no reader on the - # other end of the requestPipe. This is bad. The swift worker - # script is responsible for deleting the request pipe when it - # shuts down, so we know if the requestPipe still exists there - # should still be a worker (or the worker crashed in a funny way). - if (file.exists(requestPipeName)) { - # there is a race condition here if the fifo disappears in - # between checking for existence and opening the fifo, but - # the timeout will catch that unlikely case - writeTimeout <- 4000 - success <- writeFifo(requestPipeName,paste(reqdir,"\n",sep=""), - timeout=writeTimeout) - if (! success) { - stop(paste("timeout of", writeTimeout, - "ms exceeded when attempting to", - "rendezvous with swift server: maybe it is not running or", - "it has crashed")) - } - - # Wait for reply from service - res <- readFifo(resultPipeName, timeout=timeout) - if ((! length(res) == 0) && is.na(res)) { - stop(paste("Timeout of ", timeout, "ms exceeded when waiting", - "for response from swift server")) - } - - # Check that the message was correct - successMsg <- "done" - if (res[[1]] != successMsg) - stop(paste("Got unexpected message '", - paste(res, collapse="\n"),"' on fifo ", - "aborting job", sep="")) - } - else { - stop(paste("Have you run swiftInit?\n", - "It appears that no SwiftR servers of type", server, - "are running, as no request pipe exists in", - swiftServerDir)) - } - } - - # Fetch the batch results - - rno <- 1 - rlist <- list() - for(batch in 1:nbatches) { - result <- NULL - load(paste(reqdir,"/rbatch.",as.character(batch),".Rdata",sep="")) - nresults <- length(result) - for(r in 1:nresults) { - rlist[[rno]] <- result[[r]] -if(class(result[[r]]) == "try-error") { -cat("ERROR in eval: ", result[[r]], "\n"); -} - #DB cat("swiftapply: result rno=",rno,":\n") # FIXME: for logging - #DB cat(rlist[[rno]]@output$gradient,"\n") - rno <- rno + 1 - } - } - names(rlist) = names(arglists) - if( ! keepwork ) { - cat("Removing ", reqdir, "\n") - unlink(reqdir,recursive=TRUE) - } - return(rlist) -} - -swiftLapply <- function( tlist, func, ... ) -{ - if (length(tlist) == 0) return(tlist) - arglists <- list() - narglists <- length(tlist) - for(i in 1 : narglists) { - arglists[[i]] <- list(tlist[[i]], ...); - } - names(arglists) = names(tlist) - swiftapply(func, arglists) -} - -swiftRemoveAll <- function () { - # Cleans up all data in global namespace on workers - options(.swift.exports=list(c("removeAll"))) -} - - -swiftLibrary <- function (packname) { - # Add a library to be included on all workers - # The package can be provided directly or alternatively - # the package name can be provided. - # If reset is true, then previously added libraries won't be - # reimported on new workers. - - # Check to see if it is a string, if it is not a string - # then we will deparse it to get the expression text - if (!is.character(substitute(packname))) { - # Maybe library was provided unquoted - packname <- deparse(substitute(packname)) - } - if (! packname %in% installed.packages()) { - # Warn users in case they made a typo - warning(paste("Package", packname, - "was not a installed package in this instance of R,", - "but may be installed on SwiftR workers")) - } - plist <- getOption(".swift.packages") - if (is.null(plist)) { - plist <- list() - } - else { - # scan libs for previous detaches and libraries of - # this package. Delete all of them - plist = Filter(function(packcmd) - { return (packcmd[2] != packname);}, - plist) - } - # Add the library command to the end of the list of commands to be - # executed in slave R instances - plist[[length(plist) + 1]] <- c("library", packname) - - options(.swift.packages=plist) -} - - -swiftDetach <- function (name) { - # name is an string or identifier such as "package:OpenMx", - # following the same pattern as the built in R detach() function - # Detaches a library from workers and ensures that it will no longer be - # imported. Note that the effect on dependent imports is a little - # messy and behaves differently on existing and new workers. - # If package A requires package B, and we import A on a worker, - # package B will also be imported on the worker. By detaching package A - # we don't also detach B. This contrasts to a fresh worker, where - # package B will not be imported. - if (!is.character(substitute(name))) { - # Maybe package was provided unquoted - name <- deparse(substitute(name)) - } - - #TODO: remove from options(".swift.packages") - # Scan through list and remove any attach or detach of this name - # add a detach cmd to the end - if (substr(name, 1, 8) != "package:") - stop("Can only detach packages. Package must be specified as package:PackageName") - # Get bit after "package:" - packname = substr(name, 9, nchar(name)) - plist = getOption(".swift.packages") - if (is.null(plist)) { - plist = list() - } - else { - # Previous export/detach commands are now redundant - plist = Filter(function(packcmd) - { return (packcmd[2] != packname);}, - plist) - } - plist[[length(plist) + 1]] = c("detach", packname) - - options(.swift.packages=plist) -} - -swiftExportAll <- function () { - return (swiftExport(list=ls(globalenv()))) -} - -#TODO: not implemented -.swiftExport <- function (..., list=NULL) { - # List of object names (as R symbols or as strings) - # These will be passed directly to save() to be serialized - - #TODO: - # We want the protocol to support workers coming and going from the - # cluster. We want to rely on less implicit state. Ie. if we run - # swiftExport, then a worker is added to cluster, then an apply call - # is run, we want the worker to include the exported objects. - # One approach to doing this is to enforce the export at apply - # time. - # - # - # When swiftExport() is called: - # * Save all of the objects to a file - # - Q:Can we assume its on a shared filesystem? - # * MAYBE: put the file somewhere accessible by all workers - # * append the file's path to a stored list of files - # * MAYBE LATER: asynchronously start swift tasks to distribute - # the data, to get a headstart on the work - # When swiftApply() is called: - # - Inform all worker R processes through some mechanism (TBD) - # of the list of all exported data files. (extra file?) - # - The worker goes through the list in order to ensure that name - # conflicts are resolved in the way you would expect. - # If it has loaded a file previously, it does nothing. - # If it hasn't loaded the file, it accesses the file (mechanism TBD) - # and loads it into the global namespace. - # data - # - # swiftRemoveAll(): - # delete the list of exported files from above and have a single - # entry which instructs workers to delete all their data. This is - # mainly important because it eliminates possibility of using huge - # amounts of memory. - # swiftRemove() - # Implementation of this function is somewhat problematic, as it only - # partially undoes previous work - - - #Pseudocode: - # TODO: choose file location - # expFile <- ??? - save(..., list=list, file=expFile) - exportList <- getOption(".swift.exports") - if (is.null(exportList)) - exportList = list() #TODO: start with removeAll command? - exportList[[length(exportList) + 1]] = c("export", expFile) - options(.swift.exports=exportList) - -} From noreply at svn.ci.uchicago.edu Thu Feb 3 14:19:49 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 3 Feb 2011 14:19:49 -0600 (CST) Subject: [Swift-commit] r4056 - in SwiftApps/SwiftR/Swift: . R Message-ID: <20110203201949.D44739CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-03 14:19:49 -0600 (Thu, 03 Feb 2011) New Revision: 4056 Modified: SwiftApps/SwiftR/Swift/NAMESPACE SwiftApps/SwiftR/Swift/R/Library.R Log: Added swiftDetachAll() function to reset the libraries loaded on worker machines. Modified: SwiftApps/SwiftR/Swift/NAMESPACE =================================================================== --- SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-03 20:00:40 UTC (rev 4055) +++ SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-03 20:19:49 UTC (rev 4056) @@ -4,6 +4,7 @@ export(swiftShutdown) export(swiftLibrary) export(swiftDetach) +export(swiftDetachAll) export(runAllSwiftTests) export(basicSwiftTest) Modified: SwiftApps/SwiftR/Swift/R/Library.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Library.R 2011-02-03 20:00:40 UTC (rev 4055) +++ SwiftApps/SwiftR/Swift/R/Library.R 2011-02-03 20:19:49 UTC (rev 4056) @@ -31,11 +31,32 @@ } # Add the library command to the end of the list of commands to be # executed in slave R instances - plist[[length(plist) + 1]] <- c("library", packname) + plist[[length(plist) + 1]] <- c("library", packname) options(.swift.packages=plist) } +swiftDetachAll <- function () { + # Detaches all packages previously loaded through swiftLibrary + plist = getOption(".swift.packages") + if (is.null(plist)) { + # Nothing added before + return + } + # replace all c("library", libname) entries in list with: + # c("detach", libname) + plist = lapply(plist, + function (ent) { + if (ent[1] == "library") { + c("detach", ent[2]) + } + else { + # already detached + ent + } + }) + options(.swift.packages=plist) +} swiftDetach <- function (name) { # name is an string or identifier such as "package:OpenMx", From noreply at svn.ci.uchicago.edu Thu Feb 3 15:40:13 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 3 Feb 2011 15:40:13 -0600 (CST) Subject: [Swift-commit] r4057 - SwiftApps/SwiftR/Swift/R Message-ID: <20110203214013.178A69CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-03 15:40:12 -0600 (Thu, 03 Feb 2011) New Revision: 4057 Modified: SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/R/Export.R SwiftApps/SwiftR/Swift/R/Library.R Log: Further refactoring of Apply module to make swiftapply less monolithic and more readable. Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-03 20:19:49 UTC (rev 4056) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-03 21:40:12 UTC (rev 4057) @@ -1,7 +1,11 @@ -# Modes for directories and files created by SwiftR + + +# The Modes for directories and files created by SwiftR # We only give permissions to the current user. kDIR_MODE <- "700" kFILE_MODE <- "600" +swiftapplyScript <- system.file(package="Swift","exec/swiftapply.swift") +RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh") swiftapply <- function( func, arglists, server=NULL, @@ -15,110 +19,35 @@ quiet=FALSE) { # Set Swift default options if not passed as keywords or pre-set by user - if(is.null(server)) - server <- getOption("swift.server") - if(is.null(server)) - server<-"local" + if(is.null(server)) server <- getOption("swift.server") + if(is.null(server)) server<-"local" - if(is.null(callsperbatch)) - callsperbatch <- getOption("swift.callsperbatch") + if(is.null(callsperbatch)) callsperbatch <- getOption("swift.callsperbatch") + if(is.null(callsperbatch)) + callsperbatch <- chooseBatchSize(length(arglists), server) - if(is.null(callsperbatch)) { - # auto batching: - # start with 1 batch per core - wc <- workerCount(server) - if (is.null(wc)) { - if (! quiet ) - cat(paste("Information about current server of type", - server, "not found")) - callsperbatch <- 1 - } - else { - batches_per_worker <- getOption("swift.batchesperworker") - if (is.null(batches_per_worker)) - batches_per_worker <- 1 - target_batches <- wc * batches_per_worker - # We want to have at least batches_per_worker batches of calls - # for each worker. If it doesn't divide evenly, some will get - # batches_per_worker + 1 batches. - # callsperbatch < 1 doesn't make sense, hence the use of max - callsperbatch = max(1, floor(length(arglists)/target_batches)) -# if (! quiet ) -# cat(paste("Selected callsperbatch of", callsperbatch, -# "automatically, based on worker count of",wc, -# "and arg count of ", length(arglists))) - } - } - #cat("Got ", length(arglists), " arguments with batching factor ", callsperbatch) + # service: send requests to Swift service loop via fifo + # script: run swift for each request, via RunSwiftScript.sh (broken) + # manual: for testing, let user run remote R server manually + if(is.null(runmode)) runmode <- getOption("swift.runmode") + if(is.null(runmode)) runmode <- "service" - if(is.null(runmode)) - runmode <- getOption("swift.runmode") - if(is.null(runmode)) - runmode <- "service" - # service: send requests to Swift service loop via fifo - # script: run swift for each request, via RunSwiftScript.sh (currently broken) - # manual: for testing, let user run remote R server manually - - # Build a list of "Library" statements for all of the libraries - # already specified through swiftLibrary commands. - lib <- getOption(".swift.packages") - if (!is.null(lib)) { - # library statements - stmts <- lapply(lib, - function (libcmd) { - verb <- libcmd[1]; - if (verb == "library") { - return (sprintf("library(%s);", libcmd[2])); - } - else if (verb == "detach") { - return (sprintf("try(detach(package:%s),silent=T);", - libcmd[2])); - } - else { - error(paste("invalid verb ", verb, " in ", - ".swift.packages option")) - } - }) - libstr = paste(stmts, collapse=" ") - cat("libstr: ", libstr) - } - else { - libstr = "" - } - - - if(is.null(initialexpr)) - initialexpr <- getOption("swift.initialexpr") + if(is.null(initialexpr)) initialexpr <- getOption("swift.initialexpr") # Have library imports before other expressions - initialexpr <- paste(libstr, initialexpr, collapse=" "); + initialexpr <- paste(buildLibStr(), initialexpr, collapse=" "); - if(is.null(workerhosts)) - workerhosts <- getOption("swift.workerhosts") - if(is.null(workerhosts)) - workerhosts <- "localhost"; + if(is.null(workerhosts)) workerhosts <- getOption("swift.workerhosts") + if(is.null(workerhosts)) workerhosts <- "localhost"; - if(is.null(keepwork)) - keepwork <- getOption("swift.keepwork") - if(is.null(keepwork)) - keepwork <- FALSE; + if(is.null(keepwork)) keepwork <- getOption("swift.keepwork") + if(is.null(keepwork)) keepwork <- FALSE; - if(is.null(tmpdir)) - tmpdir <- getOption("swift.tmpdir") - if(is.null(tmpdir)) - tmpdir <- Sys.getenv("SWIFTR_TMP"); + if(is.null(tmpdir)) tmpdir <- chooseTmpDir() - if(tmpdir=="") { - if(is.null(tmpdir)) - tmpdir <- getOption("swift.tmpdir") - tmpdir <- "/tmp"; - } + if(is.null(quiet)) quiet <- getOption("swift.quiet") + if(is.null(quiet)) quiet <- FALSE - if(is.null(quiet)) { - quiet <- getOption("swift.quiet") - if(is.null(quiet)) - quiet <- FALSE - } - + # Inform user of some call info if (! quiet) { cat("\nswiftapply to ", length(arglists), " arg lists.\n") cat("\nSwift properties:\n") @@ -132,15 +61,14 @@ # Execute the calls in batches reqdir <- setupRequestDir(tmpdir=tmpdir) - if (! quiet) { - cat("Swift request is in",reqdir,"\n") - } - nbatches <- writeRequestBatches(func, arglists, initialexpr, reqdir, callsperbatch) - swiftapplyScript <- system.file(package="Swift","exec/swiftapply.swift") - if( runmode == "manual" ) { # Prompt for return (empty line) to continue; assumes user ran a manual R to process the call. + if (! quiet) cat("Swift request written to: ",reqdir,"\n") + + if( runmode == "manual" ) { + # Prompt for return (empty line) to continue; assumes user ran a + # manual R to process the call. cat("Manual Swift Run:\n run dir: ", getwd(), "/", reqdir,"\n\n") cat(" swift script: ", RunSwiftScript, "\n") cat(" server: ", server,"\n") @@ -150,11 +78,10 @@ readLines(n=1) } else if (runmode == "script") { - RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh") - system(paste(RunSwiftScript,reqdir,server,swiftapplyScript,"\"",workerhosts,"\"")) + system(paste(RunSwiftScript,reqdir,server,swiftapplyScript, + shQuote(workerhosts))) } - else { # runmode == "service" # FIXME: check and post error if not "service" - + else if (runmode == "service") { # Send request to service user <- Sys.info()[["user"]] swiftServerDir = file.path(tmpdir,user,"SwiftR", @@ -162,74 +89,24 @@ requestPipeName=file.path(swiftServerDir,"requestpipe") resultPipeName=file.path(swiftServerDir,"resultpipe") - - # fifo will block irrecoverably if there is no reader on the - # other end of the requestPipe. This is bad. The swift worker - # script is responsible for deleting the request pipe when it - # shuts down, so we know if the requestPipe still exists there - # should still be a worker (or the worker crashed in a funny way). - if (file.exists(requestPipeName)) { - # there is a race condition here if the fifo disappears in - # between checking for existence and opening the fifo, but - # the timeout will catch that unlikely case - writeTimeout <- 4000 - success <- writeFifo(requestPipeName,paste(reqdir,"\n",sep=""), - timeout=writeTimeout) - if (! success) { - stop(paste("timeout of", writeTimeout, - "ms exceeded when attempting to", - "rendezvous with swift server: maybe it is not running or", - "it has crashed")) - } - - # Wait for reply from service - res <- readFifo(resultPipeName, timeout=timeout) - if ((! length(res) == 0) && is.na(res)) { - stop(paste("Timeout of ", timeout, "ms exceeded when waiting", - "for response from swift server")) - } - - # Check that the message was correct - successMsg <- "done" - if (res[[1]] != successMsg) - stop(paste("Got unexpected message '", - paste(res, collapse="\n"),"' on fifo ", - "aborting job", sep="")) + # Try sending: this function will cause error if it fails + sendServiceRequest(requestPipeName, reqdir, server) + + res <- getServiceResponse(resultPipeName, timeout) + # Check that the message was correct + if (res[[1]] != "done") { + stop(paste("Got unexpected message '", + paste(res, collapse="\n"),"' on fifo ", + "aborting job", sep="")) } - else { - stop(paste("Have you run swiftInit?\n", - "It appears that no SwiftR servers of type", server, - "are running, as no request pipe exists in", - swiftServerDir)) - } } + else { + stop(paste("Invalid runmode", runmode)) + } # Fetch the batch results - - rno <- 1 - rlist <- list() - for(batch in 1:nbatches) { - result <- NULL - load(file.path(reqdir, - paste("/rbatch.",as.character(batch),".Rdata",sep=""))) - nresults <- length(result) - for(r in 1:nresults) { - rlist[[rno]] <- result[[r]] -if(class(result[[r]]) == "try-error") { -cat("ERROR in eval: ", result[[r]], "\n"); + return(fetchBatchResults(reqdir, nbatches, arglists, keepwork)) } - #DB cat("swiftapply: result rno=",rno,":\n") # FIXME: for logging - #DB cat(rlist[[rno]]@output$gradient,"\n") - rno <- rno + 1 - } - } - names(rlist) = names(arglists) - if( ! keepwork ) { - cat("Removing ", reqdir, "\n") - unlink(reqdir,recursive=TRUE) - } - return(rlist) -} swiftLapply <- function( tlist, func, ... ) { @@ -299,3 +176,111 @@ } return (batch - 1) } + +fetchBatchResults <- function (reqdir, nbatches, arglists, keepwork) { + rno <- 1 + rlist <- list() + for(batch in 1:nbatches) { + # The result in the file will be named "result" + result <- NULL + load(file.path(reqdir, + paste("rbatch.",as.character(batch),".Rdata",sep=""))) + nresults <- length(result) + for(r in 1:nresults) { + rlist[[rno]] <- result[[r]] + if(inherits(result[[r]], "try-error")) { + cat("ERROR in eval: ", result[[r]], "\n"); + } + #DB cat("swiftapply: result rno=",rno,":\n") # FIXME: for logging + #DB cat(rlist[[rno]]@output$gradient,"\n") + rno <- rno + 1 + } + } + names(rlist) = names(arglists) + if( ! keepwork ) { + cat("Removing ", reqdir, "\n") + unlink(reqdir,recursive=TRUE) + } + return(rlist) +} + +chooseBatchSize <- function (numargs, server) { + # Automatic selection of worker count + # start with 1 batch per core + wc <- workerCount(server) + if (is.null(wc)) { + if (! quiet ) + cat(paste("Information about current server of type", + server, "not found")) + callsperbatch <- 1 + } + else { + batches_per_worker <- getOption("swift.batchesperworker") + if (is.null(batches_per_worker)) + batches_per_worker <- 1 + target_batches <- wc * batches_per_worker + # We want to have at least batches_per_worker batches of calls + # for each worker. If it doesn't divide evenly, some will get + # batches_per_worker + 1 batches. + # callsperbatch < 1 doesn't make sense, hence the use of max + callsperbatch = max(1, floor(numargs/target_batches)) + } +} + +chooseTmpDir <- function () { + # Choose temporary directory based on global settings + # with several fallbacks + tmpdir <- getOption("swift.tmpdir") + if(is.null(tmpdir)) + tmpdir <- Sys.getenv("SWIFTR_TMP"); + + if(tmpdir=="") { + tmpdir <- "/tmp"; + } +} + +sendServiceRequest <- function (requestPipeName, reqdir, server=NULL) { + # fifo will block irrecoverably if there is no reader on the + # other end of the requestPipe. This is bad. The swift worker + # script is responsible for deleting the request pipe when it + # shuts down, so we know if the requestPipe still exists there + # should still be a worker (or the worker crashed in a funny way). + if (file.exists(requestPipeName)) { + # there is a race condition here if the fifo disappears in + # between checking for existence and opening the fifo, but + # the timeout will catch that unlikely case + writeTimeout <- 4000 + success <- writeFifo(requestPipeName,paste(reqdir,"\n",sep=""), + timeout=writeTimeout) + if (! success) { + stop(paste("timeout of", writeTimeout, + "ms exceeded when attempting to", + "rendezvous with SwiftR server of type", server, ".\n", + "Maybe it is not running or it has crashed")) + } + } + else { + stop(paste("Have you run swiftInit?\n", + "It appears that no SwiftR servers of type", server, + "are running, as no request pipe exists at", + requestPipeName)) + } + +} + +getServiceResponse <- function (resultPipeName, timeout) { + # Blocking wait for response from service on a pipe + # returns the string response. Raises error if timeout occurs. + + # Wait for reply from service + res <- readFifo(resultPipeName, timeout=timeout) + if (is.na(res)) { + stop(paste("Timeout of ", timeout, "ms exceeded when waiting", + "for response from swift server")) + } + if (length(res) == 0) { + stop(paste("Zero length response on named pipe ", resultPipeName)) + } + return (res) +} + Modified: SwiftApps/SwiftR/Swift/R/Export.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Export.R 2011-02-03 20:19:49 UTC (rev 4056) +++ SwiftApps/SwiftR/Swift/R/Export.R 2011-02-03 21:40:12 UTC (rev 4057) @@ -1,4 +1,44 @@ - +# EXPORT MECHANISM: +#================== +# We want the protocol to support workers coming and going from the +# cluster. We also want to be robust to worker failures. This +# means that we can't rely on state being retained at workers. +# I.e. If a new worker is added, we need it to receive all of the +# relevant exported data. However, for the purposes of efficiency, +# we don't want workers to have to repeatedly reload data they have +# already loaded. +# +# In order to achieve this, we will have the workers perform imports +# at apply time. The worker will receive information as a part of the +# +# +# +# +# When swiftExport() is called: +# * Save all of the objects to a file +# - Q:Can we assume its on a shared filesystem? +# * MAYBE: put the file somewhere accessible by all workers +# * append the file's path to a stored list of files +# * MAYBE LATER: asynchronously start swift tasks to distribute +# the data, to get a headstart on the work +# When swiftApply() is called: +# - Inform all worker R processes through some mechanism (TBD) +# of the list of all exported data files. (extra file?) +# - The worker goes through the list in order to ensure that name +# conflicts are resolved in the way you would expect. +# If it has loaded a file previously, it does nothing. +# If it hasn't loaded the file, it accesses the file (mechanism TBD) +# and loads it into the global namespace. +# data +# +# swiftRemoveAll(): +# delete the list of exported files from above and have a single +# entry which instructs workers to delete all their data. This is +# mainly important because it eliminates possibility of using huge +# amounts of memory. +# swiftRemove() +# Implementation of this function is somewhat problematic, as it only +# partially undoes previous work swiftRemoveAll <- function () { # Cleans up all data in global namespace on workers options(.swift.exports=list(c("removeAll"))) @@ -7,6 +47,7 @@ swiftExportAll <- function () { + # Exports all functions and data in global environment return (swiftExport(list=ls(globalenv()))) } @@ -15,40 +56,6 @@ # List of object names (as R symbols or as strings) # These will be passed directly to save() to be serialized - #TODO: - # We want the protocol to support workers coming and going from the - # cluster. We want to rely on less implicit state. Ie. if we run - # swiftExport, then a worker is added to cluster, then an apply call - # is run, we want the worker to include the exported objects. - # One approach to doing this is to enforce the export at apply - # time. - # - # - # When swiftExport() is called: - # * Save all of the objects to a file - # - Q:Can we assume its on a shared filesystem? - # * MAYBE: put the file somewhere accessible by all workers - # * append the file's path to a stored list of files - # * MAYBE LATER: asynchronously start swift tasks to distribute - # the data, to get a headstart on the work - # When swiftApply() is called: - # - Inform all worker R processes through some mechanism (TBD) - # of the list of all exported data files. (extra file?) - # - The worker goes through the list in order to ensure that name - # conflicts are resolved in the way you would expect. - # If it has loaded a file previously, it does nothing. - # If it hasn't loaded the file, it accesses the file (mechanism TBD) - # and loads it into the global namespace. - # data - # - # swiftRemoveAll(): - # delete the list of exported files from above and have a single - # entry which instructs workers to delete all their data. This is - # mainly important because it eliminates possibility of using huge - # amounts of memory. - # swiftRemove() - # Implementation of this function is somewhat problematic, as it only - # partially undoes previous work #Pseudocode: Modified: SwiftApps/SwiftR/Swift/R/Library.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Library.R 2011-02-03 20:19:49 UTC (rev 4056) +++ SwiftApps/SwiftR/Swift/R/Library.R 2011-02-03 21:40:12 UTC (rev 4057) @@ -94,3 +94,32 @@ options(.swift.packages=plist) } + +buildLibStr <- function () { + # Build a list of "Library" statements for all of the libraries + # already specified through swiftLibrary commands. + lib <- getOption(".swift.packages") + if (!is.null(lib)) { + # library statements + stmts <- lapply(lib, + function (libcmd) { + verb <- libcmd[1]; + if (verb == "library") { + return (sprintf("library(%s);", libcmd[2])); + } + else if (verb == "detach") { + return (sprintf("try(detach(package:%s),silent=T);", + libcmd[2])); + } + else { + error(paste("invalid verb ", verb, " in ", + ".swift.packages option")) + } + }) + libstr = paste(stmts, collapse=" ") + } + else { + libstr = "" + } + return (libstr) +} From noreply at svn.ci.uchicago.edu Thu Feb 3 16:22:37 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 3 Feb 2011 16:22:37 -0600 (CST) Subject: [Swift-commit] r4058 - SwiftApps/SwiftR/Swift/R Message-ID: <20110203222237.1B03B9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-03 16:22:36 -0600 (Thu, 03 Feb 2011) New Revision: 4058 Modified: SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/R/Export.R SwiftApps/SwiftR/Swift/R/Workers.R Log: Added auto-detection of local cpus. Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-03 21:40:12 UTC (rev 4057) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-03 22:22:36 UTC (rev 4058) @@ -117,7 +117,7 @@ arglists[[i]] <- list(tlist[[i]], ...); } names(arglists) = names(tlist) - swiftapply(func, arglists) + swiftapply(func, arglists, ...) } Modified: SwiftApps/SwiftR/Swift/R/Export.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Export.R 2011-02-03 21:40:12 UTC (rev 4057) +++ SwiftApps/SwiftR/Swift/R/Export.R 2011-02-03 22:22:36 UTC (rev 4058) @@ -39,23 +39,20 @@ # swiftRemove() # Implementation of this function is somewhat problematic, as it only # partially undoes previous work -swiftRemoveAll <- function () { - # Cleans up all data in global namespace on workers - options(.swift.exports=list(c("removeAll"))) -} - swiftExportAll <- function () { # Exports all functions and data in global environment return (swiftExport(list=ls(globalenv()))) } #TODO: not implemented -.swiftExport <- function (..., list=NULL) { +swiftExport <- function (..., list=NULL) { # List of object names (as R symbols or as strings) # These will be passed directly to save() to be serialized + cat("Got here") + cat("Got here") #Pseudocode: @@ -69,3 +66,8 @@ options(.swift.exports=exportList) } + +swiftRemoveAll <- function () { + # Cleans up all data in global namespace on workers + options(.swift.exports=list(c("removeAll"))) +} Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-03 21:40:12 UTC (rev 4057) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-03 22:22:36 UTC (rev 4058) @@ -1,6 +1,5 @@ - swiftInit <- function( cores=NULL, server=NULL, hosts=NULL, nodes=NULL, project=NULL, parEnv=NULL, workmode=NULL, @@ -49,12 +48,17 @@ if(is.null(cores)) cores <- getOption("swift.cores") if (is.null(cores)) { - if (server == "local") - cores <- 2 - else if (server == "ssh") + if (server == "local") { + cores <- getOption("swift.system.cores") + if (is.null(cores)) + cores <- 2 + } + else if (server == "ssh") { cores <- 4 - else + } + else { cores <- 8 + } } if(! is.null(cores) ) { cmdString <- paste(cmdString, "-c", shQuote(cores)) @@ -213,6 +217,10 @@ return (worker$cores * worker$nodes) } +.onLoad <- function (libname, packagename) { + detectSystemInfo() +} + .First.lib <- function(libname, packagename) { # When the library is loaded, set up the # list of workers @@ -257,3 +265,24 @@ } } +detectSystemInfo <- function () { + # Do initial detection and setting of options to + # reflect system info. DOesn't guarantee options will be set + os <- tolower(R.version$os) + corecount <- 0 + if (substr(os, 1, 5) == "linux") { + corecount <- try(as.numeric(system( + "grep -c '^processor' /proc/cpuinfo", intern=T))) + } + #if (substr(os, 1, 5) == "darwin") { + #} + #TODO: detect cores on other systems: + # * MAC OS X + # * BSD? + # + + if (!inherits(corecount, "try-error") && corecount >= 1){ + options(swift.system.cores=corecount) + } +} + From noreply at svn.ci.uchicago.edu Thu Feb 3 18:28:50 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 3 Feb 2011 18:28:50 -0600 (CST) Subject: [Swift-commit] r4059 - in SwiftApps/SwiftR/Swift: . R exec Message-ID: <20110204002850.25CB19CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-03 18:28:49 -0600 (Thu, 03 Feb 2011) New Revision: 4059 Modified: SwiftApps/SwiftR/Swift/NAMESPACE SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/R/Export.R SwiftApps/SwiftR/Swift/exec/SwiftRServer.R SwiftApps/SwiftR/Swift/exec/start-swift Log: Checking in ongoing work on export mechanism. It is not quite working yet, but there are only a couple of problems left ot resolve. Modified: SwiftApps/SwiftR/Swift/NAMESPACE =================================================================== --- SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-03 22:22:36 UTC (rev 4058) +++ SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-04 00:28:49 UTC (rev 4059) @@ -6,6 +6,10 @@ export(swiftDetach) export(swiftDetachAll) +export(swiftExport) +export(swiftExportAll) +export(swiftRemoveAll) + export(runAllSwiftTests) export(basicSwiftTest) exportPattern("^swiftTest") Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-03 22:22:36 UTC (rev 4058) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-04 00:28:49 UTC (rev 4059) @@ -148,7 +148,9 @@ writeRequestBatches <- function (func, arglists, initialexpr, - reqdir, callsperbatch) { + reqdir, callsperbatch, exportlist=NULL) { + + if (is.null(exportlist)) exportlist <- getOption(".swift.exports") # Write the function call info out to cbatch.?.RData files in reqdir # in batches of size specified by callsperbatch # returns the number of batches written @@ -168,7 +170,9 @@ arglistbatch[[i]] <- arglists[[arglist]] arglist <- arglist +1 } - rcall <- list(initializer=initialexpr,func=func,arglistbatch=arglistbatch) + rcall <- list(initializer=initialexpr, + imports=exportlist, + func=func,arglistbatch=arglistbatch) save(rcall, file=file.path(reqdir, paste("cbatch.",as.character(batch),".Rdata",sep=""))) @@ -274,13 +278,13 @@ # Wait for reply from service res <- readFifo(resultPipeName, timeout=timeout) + if (length(res) == 0) { + stop(paste("Zero length response on named pipe ", resultPipeName)) + } if (is.na(res)) { stop(paste("Timeout of ", timeout, "ms exceeded when waiting", "for response from swift server")) } - if (length(res) == 0) { - stop(paste("Zero length response on named pipe ", resultPipeName)) - } return (res) } Modified: SwiftApps/SwiftR/Swift/R/Export.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Export.R 2011-02-03 22:22:36 UTC (rev 4058) +++ SwiftApps/SwiftR/Swift/R/Export.R 2011-02-04 00:28:49 UTC (rev 4059) @@ -47,22 +47,17 @@ } #TODO: not implemented -swiftExport <- function (..., list=NULL) { +swiftExport <- function (..., list=NULL, expFile=NULL) { # List of object names (as R symbols or as strings) # These will be passed directly to save() to be serialized - - cat("Got here") - cat("Got here") - - - #Pseudocode: + # TODO: For now,while testing, must manually specify location # TODO: choose file location # expFile <- ??? save(..., list=list, file=expFile) exportList <- getOption(".swift.exports") if (is.null(exportList)) exportList = list() #TODO: start with removeAll command? - exportList[[length(exportList) + 1]] = c("export", expFile) + exportList[[length(exportList) + 1]] = c("import", expFile) options(.swift.exports=exportList) } Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-03 22:22:36 UTC (rev 4058) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-04 00:28:49 UTC (rev 4059) @@ -46,6 +46,8 @@ } doInit <- function(initializer) { + #print(sprintf("received initializer=%s latestInitializer=%s\n", + # initializer, latestInitializer)); if( initializer != latestInitializer) { initialExpr <- parse(text=initializer) eval(initialExpr) @@ -54,32 +56,95 @@ } } +failBatch <- function(rcall, try.error, resultBatchFileName) { + # This function is called when an error occurs and + # no valid argument values can be produced + # It fills a result array with the try.error object + # and writes it to resultBatchFileName + result <- list() + for (c in 1:length(rcall$arglistbatch)) { + result[[c]] <- try.error + } + save(result,file=resultBatchFileName) +} + runBatch <- function( callBatchFileName, resultBatchFileName ) { - result <- list() success <- try(load(callBatchFileName)); - if (!inherits(success, "try-error")) { - success <- try(doInit(rcall$initializer)) + if (inherits(success, "try-error")) { + failBatch(rcall, success, resultBatchFileName) + return } + success <- try(doInit(rcall$initializer)) if (inherits(success, "try-error")) { - # Load or initialiser failed, return an error for all elements - for (c in 1:length(rcall$arglistbatch)) { - result[[c]] <- success - } - save(result,file=resultBatchFileName) + failBatch(rcall, success, resultBatchFileName) + return } - else { - # print(sprintf("received initializer=%s latestInitializer=%s\n",initializer, latestInitializer)); + success <- try(loadImports(rcall$imports)) + if (inherits(success, "try-error")) { + failBatch(rcall, success, resultBatchFileName) + } - for(c in 1:length(rcall$arglistbatch)) { - # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) - result[[c]] = try(do.call( rcall$func, - as.list(rcall$arglistbatch[[c]] ))) - } - save(result,file=resultBatchFileName) + result <- list() + for(c in 1:length(rcall$arglistbatch)) { + # FIXME: run this under try/catch and save error status + # in results object (need to make it a list: rval + error status) + result[[c]] = try(do.call( rcall$func, + as.list(rcall$arglistbatch[[c]] ))) } + save(result,file=resultBatchFileName) } +loadImports <- function (importlist) { + # importlist is a list of directives to be carried out in sequence. + # There are two distinct directives which appear in the first position: + # - removeAll removes all global R objects + # - import comes with an argument in the second position which is + # the location of an R file to import + for (imp in importlist) { + directive <- imp[1] + if (directive == "import") { + # Load the contents of the specified file + # into the global environment + file <- imp[2] + if (!exists(".swift.imported")) { + .swift.imported <<- new.env(hash=T, parent=emptyenv()) + doImport <- TRUE + } + else { + # check to see if already imported + doImport <- !exists(file, envir=.swift.imported) + } + if (doImport) { + #TODO: load can fail with warning + load(file, envir=globalenv()) + # if an error occurs here, assume calling function + # will catch it + assign(file, TRUE, envir=.swift.imported) + cat("Loaded file ", file, "\n") + } + else { + cat("Ignored file ", file, "\n") + } + } + else if (directive == "removeAll") { + # delete all visible objects in global environment + # TODO: need to record that we did this removeAll + cat("Deleting ", ls(envir=globalenv())) + #TODO: this fails because it deletes the functions in + # this file from the environment + rm(list=ls(envir=globalenv())) + # reset record of imported items + if (exists(".swift.imported", envir=globalenv())) + rm(.swift.imported, envir=globalenv()) + cat("Deleted all") + } + else { + stop(paste("Invalid import directive", directive)) + } + } +} + cat("calling server: dir=",fifoDir,"\n") setwd(fifoDir) SwiftRFifoServer(fifoDir) Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-03 22:22:36 UTC (rev 4058) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-04 00:28:49 UTC (rev 4059) @@ -466,11 +466,14 @@ # Find and terminate R workers: they should register their PiD # in a standard location based on the pid of this start-swift # script - for rwork in `(shopt -s nullglob; echo ../Rworkers/worker.$$/*/)` - do - kill `cat $rwork/R.pid` &> /dev/null - rm -rf $rwork - done + if [ "$keepdir" = "FALSE" ] + then + for rwork in `(shopt -s nullglob; echo ../Rworkers/worker.$$/*/)` + do + kill `cat $rwork/R.pid` &> /dev/null + rm -rf $rwork + done + fi stdcleanup_end exit 0 } From noreply at svn.ci.uchicago.edu Fri Feb 4 12:08:57 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 4 Feb 2011 12:08:57 -0600 (CST) Subject: [Swift-commit] r4060 - SwiftApps/SwiftR Message-ID: <20110204180857.63CE79CC6B@svn.ci.uchicago.edu> Author: tga Date: 2011-02-04 12:08:57 -0600 (Fri, 04 Feb 2011) New Revision: 4060 Added: SwiftApps/SwiftR/IMMEDIATE-TODO Log: Checking in prioritised TODO list Added: SwiftApps/SwiftR/IMMEDIATE-TODO =================================================================== --- SwiftApps/SwiftR/IMMEDIATE-TODO (rev 0) +++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-04 18:08:57 UTC (rev 4060) @@ -0,0 +1,75 @@ + +HIGH: +-- Test pbsf (on PADS?) + +HIGH: +in swift.properties we should make sure we dont return info logs we should make sure that the amount of logging on both the worker side and the client side is as low as it can go +* Turn off bashrc processing, etc shell script + +HIGH: +- tests +-- simple calibration tests (n args, arg sizes, durations, etc) +-- openmx tests + --- OmxParallelCI + --- OmxParallelBootstrap + +HIGH: +- perf approach + -- socket? + -- swift fast branch? + -- swift times? + -- micro studies on provider staging etc. + +HIGH: + - maybe needed for Bigger parallel bootstrp + Look at all OmxNNN parallel calls - see if any are used that we dont yet handle. + +MID: +- saner approach to channels: channel per request to avoid the issue + of what happens if a "done" is never read + +MID: +- Wiki: screen + +MID: +- startup notes from Tim Bates + -- rlib rpackage sugg + +MID: +- SGE for Tim bates + -- ranger -- needs accounts + -- ibi + -- siraf + -- godzilla (if updated) + +MID: +- user testing in general (ssh, pbs, sge) +* SGE testing: ranger, siraf (low priority) + +LOW: (unless needed by immediate OpenMx app or test) +- complete sf compat functions (sapply, lapply -> for openMx, based on usage) + +LOW: +- run modes + -- workspace stays up, server(s) are transient-> current + -- workspace is run as a job under batch scheduler -> new feature + +LOW: +-- perf issue wrt arg list copies (very low prio) + +LOW: +- Batch option: number of jobs + +LOW: + idea: +* swiftInit() should wait until batch job is submitted successfully, or + at least one ssh client starts correctly + +LOW: +* swiftStat to check state of worker processes + +LOW: +* feasibility of multithreaded server: + - each request provides a reply channel + - swift server forks off a worker thread for + each request From noreply at svn.ci.uchicago.edu Fri Feb 4 12:42:48 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 4 Feb 2011 12:42:48 -0600 (CST) Subject: [Swift-commit] r4061 - SwiftApps/SwiftR Message-ID: <20110204184248.4DE389CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-04 12:42:48 -0600 (Fri, 04 Feb 2011) New Revision: 4061 Modified: SwiftApps/SwiftR/IMMEDIATE-TODO Log: Checking in notes from OpenMx meeting about potential benchmarks. Modified: SwiftApps/SwiftR/IMMEDIATE-TODO =================================================================== --- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-04 18:08:57 UTC (rev 4060) +++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-04 18:42:48 UTC (rev 4061) @@ -1,4 +1,35 @@ +1) swift bcast if exportAll files/data and/or xmit this via prov staging w/ caching of duplicates +2) Fast branch: working? faster? + +3) speed: where is bottleneck? how to measure and tune? +- ParallelCI and ParallelBoostrap benchmarks +- smaller focused micro-tests to create plots of speed vs param size and efficiency vs runtime of the evaluated function. + +4) SGE: +- test in IBI +- test on Ranger +- get this to Tim Bates to test on Eddie () + +5) Timing: dramatic bencmark results for new proposal. + + +VERY HIGH: +-- Benchmarks for grant app - grant app is due 21 Feb +-- New results by around 11 Feb + - Synthetic benchmark + - Parallel bootstrap + - Parallel CI + - smaller focused micro-tests to create plots of speed vs param size and efficiency vs runtime of the evaluated function. + - Sarah Kenny FMRI + - parallel OpenMx Tests +-- Candidate systems: + - Eureka/cobalt + - Beagle? (Prelim: crays crow) + - Sicortex (prob. not) + - BG/P +-- Benchmarks on average user machines + HIGH: -- Test pbsf (on PADS?) From noreply at svn.ci.uchicago.edu Mon Feb 7 11:21:08 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 7 Feb 2011 11:21:08 -0600 (CST) Subject: [Swift-commit] r4062 - in SwiftApps/SwiftR/Swift: R exec Message-ID: <20110207172108.E69429CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-07 11:21:08 -0600 (Mon, 07 Feb 2011) New Revision: 4062 Modified: SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/R/Export.R SwiftApps/SwiftR/Swift/R/Workers.R SwiftApps/SwiftR/Swift/exec/SwiftRServer.R Log: * swiftExport, swiftExportAll and swiftRemoveAll functions implemented. - However, they rely on the user manually specifying an export file on a filesystem readable by the workers. THis will be fixed next. - These functions are implemented using a session abstraction which allows workers newly joining a swift cluster to "catch up" on previous exports when an apply call is made, while not requiring workers to repeatedly import the same files * Functions on host now run in a separate environment from the global environment, which can be tidied up when removeAll is invoked. It is no longer necessary to use the <<- operator in swift initial expressions: any assignment made with the <- operator in an initializer will be visible to any function being run. * Any references to variables or function made in the body of an implied function that match the name of an exported variable/function resolve correctly * Bugfix: count number of cores correct for ssh workers * Bugfix: pass quiet argument to where it is required Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-04 18:42:48 UTC (rev 4061) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 17:21:08 UTC (rev 4062) @@ -24,7 +24,7 @@ if(is.null(callsperbatch)) callsperbatch <- getOption("swift.callsperbatch") if(is.null(callsperbatch)) - callsperbatch <- chooseBatchSize(length(arglists), server) + callsperbatch <- chooseBatchSize(length(arglists), server, quiet=quiet) # service: send requests to Swift service loop via fifo # script: run swift for each request, via RunSwiftScript.sh (broken) @@ -150,7 +150,11 @@ writeRequestBatches <- function (func, arglists, initialexpr, reqdir, callsperbatch, exportlist=NULL) { - if (is.null(exportlist)) exportlist <- getOption(".swift.exports") + if (is.null(exportlist)) exportlist <- getOption(".swift.session") + if (is.null(exportlist)) { + exportlist <- newSession() + options(.swift.session = exportlist) + } # Write the function call info out to cbatch.?.RData files in reqdir # in batches of size specified by callsperbatch # returns the number of batches written @@ -208,7 +212,7 @@ return(rlist) } -chooseBatchSize <- function (numargs, server) { +chooseBatchSize <- function (numargs, server, quiet=FALSE) { # Automatic selection of worker count # start with 1 batch per core wc <- workerCount(server) Modified: SwiftApps/SwiftR/Swift/R/Export.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Export.R 2011-02-04 18:42:48 UTC (rev 4061) +++ SwiftApps/SwiftR/Swift/R/Export.R 2011-02-07 17:21:08 UTC (rev 4062) @@ -47,22 +47,37 @@ } #TODO: not implemented -swiftExport <- function (..., list=NULL, expFile=NULL) { +swiftExport <- function (..., list=NULL, file=NULL) { # List of object names (as R symbols or as strings) # These will be passed directly to save() to be serialized # TODO: For now,while testing, must manually specify location # TODO: choose file location # expFile <- ??? - save(..., list=list, file=expFile) - exportList <- getOption(".swift.exports") - if (is.null(exportList)) - exportList = list() #TODO: start with removeAll command? - exportList[[length(exportList) + 1]] = c("import", expFile) - options(.swift.exports=exportList) + if (is.null(list)) + save(..., file=file) + else + save(..., list=list, file=file) + session <- getOption(".swift.session") + if (is.null(session)) + session <- newSession() + session$exports[[length(session$exports) + 1]] <- file + options(.swift.session=session) } +newSession <- function () { + exportList= list() + exportList$session <- as.integer(runif(1, min=0, max=.Machine$integer.max)) + exportList$exports <- list() + return (exportList) +} + swiftRemoveAll <- function () { # Cleans up all data in global namespace on workers - options(.swift.exports=list(c("removeAll"))) + # This is achieved by starting a new "session" + # Use session numbers to track whether a worker has the + # correct data + # TODO: think about how to clean up data files. + # it is hard to know when they can be safely removed + options(.swift.session=newSession()) } Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-04 18:42:48 UTC (rev 4061) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-07 17:21:08 UTC (rev 4062) @@ -66,7 +66,12 @@ if(is.null(hosts)) hosts <- getOption("swift.hosts") - if(! is.null(hosts) ) { + if( is.null(hosts) ) { + if (server == "ssh") { + error(paste("Need to provide hosts list for ssh server.")) + } + } + else { if (length(hosts) > 1) { # Concatenate list of hosts hosts <- paste(hosts, collapse=" ") @@ -95,7 +100,10 @@ if(is.null(nodes)) nodes <- getOption("swift.nodes") if(is.null(nodes)) - nodes <- 1 # Default value + if (server == "ssh") + nodes <- length(strsplit(hosts, " ")) + else + nodes <- 1 # Default value if(! is.null(nodes) ) { cmdString <- paste(cmdString, "-n", shQuote(nodes)) } Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-04 18:42:48 UTC (rev 4061) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-07 17:21:08 UTC (rev 4062) @@ -7,7 +7,7 @@ cat(Sys.getpid(),file=paste(fifoDir,"/R.pid",sep="")) -latestInitializer <- ""; +.current.initializer <<- "" SwiftRFifoServer <- function( fifoBasename ) { @@ -45,14 +45,12 @@ } } -doInit <- function(initializer) { +doInit <- function(initializer, env) { #print(sprintf("received initializer=%s latestInitializer=%s\n", # initializer, latestInitializer)); - if( initializer != latestInitializer) { - initialExpr <- parse(text=initializer) - eval(initialExpr) - latestInitializer <<- initializer - # print(sprintf("after eval: latestInitializer=%s\n", latestInitializer)); + if( initializer != .current.initializer) { + eval(parse(text=initializer), envir=env) + .current.initializer <<- initializer } } @@ -73,16 +71,35 @@ success <- try(load(callBatchFileName)); if (inherits(success, "try-error")) { failBatch(rcall, success, resultBatchFileName) - return + return() } - success <- try(doInit(rcall$initializer)) + success <- try(loadImports(rcall$imports)) if (inherits(success, "try-error")) { failBatch(rcall, success, resultBatchFileName) - return + return() } - success <- try(loadImports(rcall$imports)) + + if (exists(".current.envir", envir=globalenv())) { + env = get(".current.envir", envir=globalenv()) + } + else { + # Create a new environment to work in + if (is.null(importlist) || is.null(importlist$session)) { + # If the client provides no info about the environment, + # we should create a new environment to work in + env = new.env(parent=parent.env(globalenv())) + } + else { + newSession(importlist$session) + env = get(".current.envir", envir=globalenv()) + } + } + environment(rcall$func) <- env + + success <- try(doInit(rcall$initializer, env)) if (inherits(success, "try-error")) { failBatch(rcall, success, resultBatchFileName) + return() } result <- list() @@ -95,52 +112,61 @@ save(result,file=resultBatchFileName) } +newSession <- function (session) { + cat("Setting up new session\n", file=stderr()) + #Setup new session + .current.session <<- session + # Use an environment as a hash table to track imports + .current.imported <<- new.env(hash=T, parent=emptyenv()) + # Create an environment to store imported items in + # and to evaluate apply calls in. Note that the environment + # does not point to the global environment as its parent, which means + # means that assignments and variable lookusp will not touch the global + # environment by default, and will be contained in this session. + # However, the <<- assignment operator and the get, assign, etc function + # can still access the global enviroment if used in the applied function. + + .current.envir <<- new.env(parent=parent.env(globalenv())) + + .current.initializer <<- "" +} + loadImports <- function (importlist) { - # importlist is a list of directives to be carried out in sequence. - # There are two distinct directives which appear in the first position: - # - removeAll removes all global R objects - # - import comes with an argument in the second position which is - # the location of an R file to import - for (imp in importlist) { - directive <- imp[1] - if (directive == "import") { - # Load the contents of the specified file - # into the global environment - file <- imp[2] - if (!exists(".swift.imported")) { - .swift.imported <<- new.env(hash=T, parent=emptyenv()) - doImport <- TRUE - } - else { - # check to see if already imported - doImport <- !exists(file, envir=.swift.imported) - } - if (doImport) { - #TODO: load can fail with warning - load(file, envir=globalenv()) - # if an error occurs here, assume calling function - # will catch it - assign(file, TRUE, envir=.swift.imported) - cat("Loaded file ", file, "\n") - } - else { - cat("Ignored file ", file, "\n") - } + # First check whether the session id has changed + cat(paste("Import list:", importlist$exports, "\n"), file=stderr()) + cat(paste("New Session:", importlist$session, "\n"), file=stderr()) + cat(paste("Old Session:", try(get(".current.session", envir=globalenv())), + "\n"), file=stderr()) + doSetup <- FALSE + if (!is.null(importlist)) { + if (exists(".current.session", envir=globalenv())) { + if (importlist$session != .current.session) + doSetup <- TRUE } - else if (directive == "removeAll") { - # delete all visible objects in global environment - # TODO: need to record that we did this removeAll - cat("Deleting ", ls(envir=globalenv())) - #TODO: this fails because it deletes the functions in - # this file from the environment - rm(list=ls(envir=globalenv())) - # reset record of imported items - if (exists(".swift.imported", envir=globalenv())) - rm(.swift.imported, envir=globalenv()) - cat("Deleted all") + else { + doSetup <- TRUE } + } + if (doSetup) { + newSession(importlist$session) + } + + for (file in importlist$exports) { + # Load the contents of the specified file + # into the global environment + + cat("File: ", file, "\n") + # check to see if already imported + if (!exists(file, envir=.current.imported)) { + #TODO: load can fail with warning + load(file, envir=.current.envir) + # if an error occurs here, assume calling function + # will catch it + assign(file, TRUE, envir=.current.imported) + cat("Loaded file ", file, "\n") + } else { - stop(paste("Invalid import directive", directive)) + cat("Ignored file ", file, "\n") } } } From noreply at svn.ci.uchicago.edu Mon Feb 7 11:25:38 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 7 Feb 2011 11:25:38 -0600 (CST) Subject: [Swift-commit] r4063 - SwiftApps/SwiftR/Swift/R Message-ID: <20110207172538.7904D9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-07 11:25:33 -0600 (Mon, 07 Feb 2011) New Revision: 4063 Modified: SwiftApps/SwiftR/Swift/R/Tests.R Log: Checking in changes to tests reflecting that it is no longer necessary (and deprecated) to use <<- assignment operator in initialexpr Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-07 17:21:08 UTC (rev 4062) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-07 17:25:33 UTC (rev 4063) @@ -4,7 +4,7 @@ { options(swift.site="service") options(swift.keepwork=TRUE) - initcmds <- "initVar1 <<- 19; initVar2 <<- sqrt(400)+3" + initcmds <- "initVar1 <- 19; initVar2 <- sqrt(400)+3" options(swift.initialexpr=initcmds) # Set here; used in test group 4 } @@ -69,7 +69,7 @@ swiftTest_4.2 <- function() { - options(swift.initialexpr="initVar3 <<- 123; initVar4 <<- 100"); + options(swift.initialexpr="initVar3 <- 123; initVar4 <- 100"); mulivars <- function() { initVar3*initVar4 } @@ -94,7 +94,7 @@ swiftTest_6.1 <- function(delay,ncalls) { - options(swift.initialexpr="initVar3 <<- 123; initVar4 <<- 100"); + options(swift.initialexpr="initVar3 <- 123; initVar4 <- 100"); timed <- function(delay) { Sys.sleep(delay); delay } From noreply at svn.ci.uchicago.edu Mon Feb 7 15:04:05 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 7 Feb 2011 15:04:05 -0600 (CST) Subject: [Swift-commit] r4064 - in SwiftApps/SwiftR/Swift: R exec Message-ID: <20110207210405.47E059CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-07 15:04:05 -0600 (Mon, 07 Feb 2011) New Revision: 4064 Added: SwiftApps/SwiftR/Swift/R/Init.R Modified: SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/R/Export.R SwiftApps/SwiftR/Swift/R/Tests.R SwiftApps/SwiftR/Swift/R/Workers.R SwiftApps/SwiftR/Swift/exec/SwiftRServer.R Log: Added automatic selection of directory for exports, cleanup of exported file, tests of export functionality. Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 17:25:33 UTC (rev 4063) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 21:04:05 UTC (rev 4064) @@ -127,21 +127,21 @@ # to lifetime of this R process. If we stored this in a global # variable, it is possible that, say, directory requests.55555/R0000005 # is created, the user exits the session without saving, and therefore - # the swift.requestid counter is out of step with the file system + # the .swift.requestid counter is out of step with the file system requestdirbase = getOption("swift.requestdirbase") if(!is.null(requestdirbase)) { - requestid = getOption("swift.requestid") + 1; + requestid = getOption(".swift.requestid") + 1; } else { - requestdirbase = sprintf("%s/%s/SwiftR/requests.P%.5d",tmpdir, - Sys.info()[["user"]],Sys.getpid()) + requestdirbase = file.path(tmpdir, Sys.info()[["user"]],"SwiftR", + sprintf("requests.P%.5d",Sys.getpid())) dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE, mode=kDIR_MODE) options(swift.requestdirbase=requestdirbase) requestid = 0; } - options(swift.requestid=requestid) - reqdir = sprintf("%s/R%.7d",requestdirbase,requestid) + options(.swift.requestid=requestid) + reqdir = file.path(requestdirbase, sprintf("R%.7d",requestid)) dir.create(reqdir,recursive=TRUE,showWarnings=FALSE,mode=kDIR_MODE) return (reqdir) } Modified: SwiftApps/SwiftR/Swift/R/Export.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Export.R 2011-02-07 17:25:33 UTC (rev 4063) +++ SwiftApps/SwiftR/Swift/R/Export.R 2011-02-07 21:04:05 UTC (rev 4064) @@ -41,18 +41,42 @@ # partially undoes previous work -swiftExportAll <- function () { +swiftExportAll <- function (file=NULL) { # Exports all functions and data in global environment - return (swiftExport(list=ls(globalenv()))) + swiftExport(list=ls(globalenv()), file=file) } -#TODO: not implemented swiftExport <- function (..., list=NULL, file=NULL) { # List of object names (as R symbols or as strings) # These will be passed directly to save() to be serialized - # TODO: For now,while testing, must manually specify location - # TODO: choose file location - # expFile <- ??? + if (is.null(file)) { + dir <- getExportDir() + if (is.null(dir)) { + error(paste("Could not determine an export directory, try", + "setting the", + "swift.exportdir option to a directory accessible by this R", + "session and the Swift worker processes")) + } + # Separate directory for each new R session to keep files together + # and avoid conflicts. + cat("Export directory: ", dir, "\n") + dir.create(dir, recursive=T, showWarnings=FALSE, mode=kDIR_MODE) + + # add the file, keeping a counter to use in the file name + expid = getOption(".swift.exportid") + if (is.null(expid)) expid = 0 + options(.swift.exportid = expid + 1) + file <- file.path(dir, sprintf("E%.7d",expid)) + cat("Export file: ", file, "\n") + + # Keep track of files for later cleanup + expfiles <- getOption(".swift.exportfiles") + if (is.null(expfiles)) expfiles = list() + expfiles[[length(expfiles) + 1]] <- file + options(.swift.exportfiles=expfiles) + } + + #TODO: file mode? if (is.null(list)) save(..., file=file) else @@ -81,3 +105,34 @@ # it is hard to know when they can be safely removed options(.swift.session=newSession()) } + +getExportDir <- function () { + d <- getOption("swift.exportdir") + if (! is.null(d)) d <- file.path(d, ".swiftr", "exports") + + # Home directory is a candidate + d <- Sys.getenv("HOME")[[1]] + if (d == "") { + return (NULL) + } + else { + d <- file.path(d, ".swiftr") + return (file.path(d, sprintf("exports.P%.5d",Sys.getpid()))) + } +} + +# Function called when Swift is unloaded or session ends +# that is responsible for cleanup +removeExports <- function () { + cat("removing exports\n") + expfiles <- getOption(".swift.exportfiles") + for (file in expfiles) { + #cat("removing", file, "\n") + file.remove(file) + } + options(.swift.exportfiles=NULL) + dir <- getExportDir() + if (!is.null(dir)) { + unlink(dir, recursive=T) + } +} Added: SwiftApps/SwiftR/Swift/R/Init.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Init.R (rev 0) +++ SwiftApps/SwiftR/Swift/R/Init.R 2011-02-07 21:04:05 UTC (rev 4064) @@ -0,0 +1,80 @@ +# This file contains functions involved in starting up and shutting down +# SwiftR + +.onLoad <- function (libname, packagename) { + detectSystemInfo() +} + +.First.lib <- function(libname, packagename) { + # When the library is loaded, set up the + # list of workers + .swift.workers <<- list() +} + +swiftCleanup <- function () { + # Shut down workers + swiftShutdown() + # Clean up exported files + removeExports() + +} + +.Last.lib <- function(p) +{ + # If the library is unloaded we need to do cleanup + swiftCleanup() + removeHook() +} + +# Hook to perform cleanup of workers upon shutting down an R +# session +addHook <- function() { + # Replace the user's last function with ours + # If .UserLast already exists don't worry about it + # as we've already added our hook + if (!exists(".UserLast")) { + if (!exists(".Last")) { + # Create a dummy function + .UserLast <<- function () {} + + } + else { + .UserLast <<- .Last + } + + .Last <<- function () { + swiftCleanup() + .UserLast() + removeHook() + } + } +} + +removeHook <- function() { + if (exists(".UserLast", where=".GlobalEnv")) { + .Last <<- .UserLast + rm(".UserLast", pos=".GlobalEnv") + } +} + +detectSystemInfo <- function () { + # Do initial detection and setting of options to + # reflect system info. DOesn't guarantee options will be set + os <- tolower(R.version$os) + corecount <- 0 + if (substr(os, 1, 5) == "linux") { + corecount <- try(as.numeric(system( + "grep -c '^processor' /proc/cpuinfo", intern=T))) + } + #if (substr(os, 1, 5) == "darwin") { + #} + #TODO: detect cores on other systems: + # * MAC OS X + # * BSD? + # + + if (!inherits(corecount, "try-error") && corecount >= 1){ + options(swift.system.cores=corecount) + } +} + Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-07 17:25:33 UTC (rev 4063) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-07 21:04:05 UTC (rev 4064) @@ -42,6 +42,98 @@ } } +#1.3.* tests export functionality +swiftTest_1.3.1 <- function() +{ + + initSwiftTestOptions() + + cat("\n*** Starting test 1.3.1 ***\n\n") + y <- 1 + inc <- function(x) { x + y } + + arglist = list(1,2,3,4,5,6,7,8,9) + + cat("Test of local lapply\n") + localres = lapply(arglist, inc) + cat("local result=\n") + print(localres) + + # Test swiftExport + swiftExport(y) + cat("\nTest of swiftLapply\n") + swiftres = swiftLapply(arglist, inc) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres)) { + cat("\n==> test 1.3.1 passed\n") + } else { + cat("\n==> test 1.3.1 FAILED !!!!!\n") + } +} + +swiftTest_1.3.2 <- function() +{ + # Test swiftExportAll() + + cat("\n*** Starting test 1.3.2 ***\n\n") + y <<- 1 + inc <- function(x) { x + y } + + arglist = list(1,2,3,4,5,6,7,8,9) + + cat("Test of local lapply\n") + localres = lapply(arglist, inc) + cat("local result=\n") + print(localres) + + # Test swiftExport + swiftExportAll() + cat("\nTest of swiftLapply\n") + swiftres = swiftLapply(arglist, inc) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres)) { + cat("\n==> test 1.3.2 passed\n") + } else { + cat("\n==> test 1.3.2 FAILED !!!!!\n") + } +} + + +swiftTest_1.3.3 <- function() +{ + + initSwiftTestOptions() + + cat("\n*** Starting test 1.3.3 ***\n\n") + y <- 1 + inc <- function(x) { x + y } + + arglist = list(1,2,3,4) + + # Test swiftExportAll + swiftExportAll() + cat("\nTest of swiftLapply\n") + swiftres = swiftLapply(arglist, inc) + cat("Swift result before removal:\n") + print(swiftres) + + swiftRemoveAll() + swiftres = swiftLapply(arglist, inc) + cat("Swift result after removal:\n") + for (e in swiftres) { + if (!inherits(e, "try-error")) { + cat("\n==> test 1.3.3 FAILED !!!!!\n") + return() + } + } + cat("\n==> test 1.3.3 passed\n") +} + + basicSwiftTest <- function() { swiftTest_1.1() } # .... more tests from below to move here @@ -242,8 +334,10 @@ # swiftres = swiftapply(sumstuff,arglist,callsperbatch=2,site="pbs") # test variations on local vs ssh vs pbs; coasters vs non; etc. + swiftTest_1.3.1() + swiftTest_1.3.2() + swiftTest_1.3.3() - ##### Test Group 2 cat("\n*** Starting test group 2 - test matrix passing***\n") Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-07 17:25:33 UTC (rev 4063) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-07 21:04:05 UTC (rev 4064) @@ -225,72 +225,3 @@ return (worker$cores * worker$nodes) } -.onLoad <- function (libname, packagename) { - detectSystemInfo() -} - -.First.lib <- function(libname, packagename) { - # When the library is loaded, set up the - # list of workers - .swift.workers <<- list() -} - -.Last.lib <- function(p) -{ - # If the library is unloaded we need to do cleanup - swiftShutdown() - removeHook() -} - -# Hook to perform cleanup of workers upon shutting down an R -# session -addHook <- function() { - # Replace the user's last function with ours - # If .UserLast already exists don't worry about it - # as we've already added our hook - if (!exists(".UserLast")) { - if (!exists(".Last")) { - # Create a dummy function - .UserLast <<- function () {} - - } - else { - .UserLast <<- .Last - } - - .Last <<- function () { - swiftShutdown() - .UserLast() - removeHook() - } - } -} - -removeHook <- function() { - if (exists(".UserLast", where=".GlobalEnv")) { - .Last <<- .UserLast - rm(".UserLast", pos=".GlobalEnv") - } -} - -detectSystemInfo <- function () { - # Do initial detection and setting of options to - # reflect system info. DOesn't guarantee options will be set - os <- tolower(R.version$os) - corecount <- 0 - if (substr(os, 1, 5) == "linux") { - corecount <- try(as.numeric(system( - "grep -c '^processor' /proc/cpuinfo", intern=T))) - } - #if (substr(os, 1, 5) == "darwin") { - #} - #TODO: detect cores on other systems: - # * MAC OS X - # * BSD? - # - - if (!inherits(corecount, "try-error") && corecount >= 1){ - options(swift.system.cores=corecount) - } -} - Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-07 17:25:33 UTC (rev 4063) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-07 21:04:05 UTC (rev 4064) @@ -70,7 +70,8 @@ { success <- try(load(callBatchFileName)); if (inherits(success, "try-error")) { - failBatch(rcall, success, resultBatchFileName) + failBatch(rcall, paste(callBatchFileName, "could not be opened"), + resultBatchFileName) return() } success <- try(loadImports(rcall$imports)) @@ -133,10 +134,10 @@ loadImports <- function (importlist) { # First check whether the session id has changed - cat(paste("Import list:", importlist$exports, "\n"), file=stderr()) - cat(paste("New Session:", importlist$session, "\n"), file=stderr()) - cat(paste("Old Session:", try(get(".current.session", envir=globalenv())), - "\n"), file=stderr()) + #cat(paste("Import list:", importlist$exports, "\n"), file=stderr()) + #cat(paste("New Session:", importlist$session, "\n"), file=stderr()) + #cat(paste("Old Session:", try(get(".current.session", envir=globalenv())), + # "\n"), file=stderr()) doSetup <- FALSE if (!is.null(importlist)) { if (exists(".current.session", envir=globalenv())) { @@ -155,7 +156,7 @@ # Load the contents of the specified file # into the global environment - cat("File: ", file, "\n") + #cat("File: ", file, "\n") # check to see if already imported if (!exists(file, envir=.current.imported)) { #TODO: load can fail with warning @@ -166,7 +167,7 @@ cat("Loaded file ", file, "\n") } else { - cat("Ignored file ", file, "\n") + #cat("Ignored file ", file, "\n") } } } From noreply at svn.ci.uchicago.edu Mon Feb 7 16:07:51 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 7 Feb 2011 16:07:51 -0600 (CST) Subject: [Swift-commit] r4065 - in SwiftApps/SwiftR/Swift: R exec man Message-ID: <20110207220751.BB0D99CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-07 16:07:51 -0600 (Mon, 07 Feb 2011) New Revision: 4065 Modified: SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/R/Init.R SwiftApps/SwiftR/Swift/R/Workers.R SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/Swift/exec/start-swift-daemon SwiftApps/SwiftR/Swift/man/swiftapply.Rd Log: * Made the way the swift server was selected more consistent in swiftapply * If a server type is specified through an argument or the toptions mechanism, we use the most recently launched server of that type. If no server type is specified, then we use the most recently launched server. * swiftShutdown now shuts down the most recently started server by default. Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 21:04:05 UTC (rev 4064) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 22:07:51 UTC (rev 4065) @@ -20,7 +20,6 @@ { # Set Swift default options if not passed as keywords or pre-set by user if(is.null(server)) server <- getOption("swift.server") - if(is.null(server)) server<-"local" if(is.null(callsperbatch)) callsperbatch <- getOption("swift.callsperbatch") if(is.null(callsperbatch)) @@ -83,9 +82,7 @@ } else if (runmode == "service") { # Send request to service - user <- Sys.info()[["user"]] - swiftServerDir = file.path(tmpdir,user,"SwiftR", - paste("swift.",server,sep="")) + swiftServerDir = getWorkerDir(server) requestPipeName=file.path(swiftServerDir,"requestpipe") resultPipeName=file.path(swiftServerDir,"resultpipe") Modified: SwiftApps/SwiftR/Swift/R/Init.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Init.R 2011-02-07 21:04:05 UTC (rev 4064) +++ SwiftApps/SwiftR/Swift/R/Init.R 2011-02-07 22:07:51 UTC (rev 4065) @@ -13,7 +13,7 @@ swiftCleanup <- function () { # Shut down workers - swiftShutdown() + swiftShutdown(all=TRUE) # Clean up exported files removeExports() Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-07 21:04:05 UTC (rev 4064) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-07 22:07:51 UTC (rev 4065) @@ -149,12 +149,15 @@ # launch asynchronously # for now, we will rely on the shell script's output to inform # the user if there was a problem with the workers - pid <- system(cmdString, intern=TRUE) + out <- system(cmdString, intern=TRUE) + pid <- out[1] + workdir <- out[2] cat("Started worker manager with pid ", pid, "\n") output <- list() output$pid <- pid output$server <- server + output$workdir <- workdir output$cores <- cores output$nodes <- nodes @@ -165,19 +168,13 @@ # this process exits addHook() - # by default, set the swift.server option so that - # subsequent swift*apply calls use the newly created - # server - if (is.null(getOption("swift.server"))) - options(swift.server=server) # Sleep to give start-swift time to set up fifos,etc Sys.sleep(2) return (output) } -swiftShutdown <- function(handle=NULL) -{ +swiftShutdown <- function(handle=NULL, all=FALSE) { if (!exists(".swift.workers") || is.null(.swift.workers) || length(.swift.workers) == 0) { @@ -185,10 +182,19 @@ return } if (is.null(handle)) { - workers = .swift.workers - .swift.workers <<- list() + if (all) { + workers <- .swift.workers + .swift.workers <<- list() + cat("Shutting down all Swift worker processes\n") + } + else { + # Remove the last started worker + workers <- .swift.workers[length(.swift.workers)] + .swift.workers <<- .swift.workers[1:length(.swift.workers)-1] + } } else { + # Split between matching workers and non-matching workers workers <- Filter( function (worker) { return (worker$pid == as.character(handle$pid)) ; }, .swift.workers) @@ -196,9 +202,9 @@ function (worker) { return (worker$pid != as.character(handle$pid)) ; }, .swift.workers) } - cat("Shutting down Swift worker processes\n") # shut down all worker processes using kill for (worker in workers) { + cat("Terminating worker", worker$pid, " of type ",worker$server, "\n") cmdString <- file.path(.find.package("Swift"), "exec/killtree &> /dev/null ") killCmd <- paste(cmdString, worker$pid) system(killCmd, wait=FALSE) @@ -212,16 +218,49 @@ # TODO: this is a bit flakey as there is no guarantee # the currently active worker of this type was launched within # R + worker <- getWorker(server) + if (is.null(worker)) { + return (NULL) + } + else { + return (worker$cores * worker$nodes) + } +} + +# Find the most recently launched instance of a server type. If +# no argument provided, the most recent server of any type +getWorker <- function (server=NULL) { if (!exists(".swift.workers") || is.null(.swift.workers)) { return (NULL) } - worker <- Find( - function (worker) { return (worker$server == server) ; }, - .swift.workers, right=TRUE) - if (is.null(worker)) - return (NULL) - else - return (worker$cores * worker$nodes) + if (is.null(server)) { + return (.swift.workers[[length(.swift.workers)]]) + } + else { + worker <- Find( + function (worker) { return (worker$server == server) ; }, + .swift.workers, right=TRUE) + return (worker) + } } +getWorkerDir <- function (server=NULL, tmpdir=NULL) { + worker <- getWorker(server) + if (!is.null(worker)) { + return (worker$workdir) + } + else { + if (!is.null(server)) { + if (is.null(tmpdir)) tmpdir <- chooseTmpDir() + user <- Sys.info()[["user"]] + return (file.path(tmpdir,user,"SwiftR", + paste("swift.",server,sep=""))) + } + else { + stop(paste("No SwiftR servers launched within R and no server type", + "specified, can't identify a likely location for a swiftR", + "service")) + } + } +} Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-07 21:04:05 UTC (rev 4064) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-07 22:07:51 UTC (rev 4065) @@ -39,7 +39,7 @@ { get-contact LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env? - LOGDIR=/tmp/$USER/SwiftR/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way? + LOGDIR=$tmp/$USER/SwiftR/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way? # mkdir -p $LOGDIR # is done with the ssh command, below @@ -378,6 +378,7 @@ queue=NONE project=NONE parEnv=NONE +workdir=NONE workerLogging=ERROR keepdir=FALSE @@ -400,6 +401,7 @@ -t) time=$2; verify-not-null time $time; shift ;; -w) workerLogging=$2; verify-is-one-of workerLoggingLevel $workerLogging NONE ERROR WARN INFO DEBUG TRACE; shift ;; -k) keepdir=TRUE ;; + -d) workdir=$2; verify-not-null workdir $workdir; shift ;; *) usage; exit 1 ;; esac shift @@ -410,7 +412,17 @@ rundir=$tmp/$USER/SwiftR/swift.$server # rundir prefix # FIXME: handle multiple concurent independent swift servers per user mkdir -p $(dirname $rundir) -trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success + +# Setup a working directory +if [ "$workdir" = NONE ] +then + trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success +else + echo Working in $workdir + trundir=$workdir + mkdir -p $workdir +fi + rm -f $rundir ln -s $trundir $rundir cd $trundir Modified: SwiftApps/SwiftR/Swift/exec/start-swift-daemon =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-daemon 2011-02-07 21:04:05 UTC (rev 4064) +++ SwiftApps/SwiftR/Swift/exec/start-swift-daemon 2011-02-07 22:07:51 UTC (rev 4065) @@ -8,13 +8,16 @@ # the pid of start-swift ssscript=`dirname $0`/start-swift -# Start as detached daemon, with output going to stdout +# Choose a working directory +tmp=${SWIFTR_TMP:-/tmp} +workdir=$(mktemp -d $tmp/$USER/SwiftR/swift.XXXX) # FIXME: check success - +# Start as detached daemon, with output going to stderr # Start up a subprocess with a new process group # childpid will be of form '[jobno] pid' -$ssscript "$@" 1>&2 & +$ssscript "$@" -d $workdir 1>&2 & childpid=$! echo ${childpid} +echo ${workdir} Modified: SwiftApps/SwiftR/Swift/man/swiftapply.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/swiftapply.Rd 2011-02-07 21:04:05 UTC (rev 4064) +++ SwiftApps/SwiftR/Swift/man/swiftapply.Rd 2011-02-07 22:07:51 UTC (rev 4065) @@ -44,7 +44,9 @@ The swift server type to use to run. The possible values are "local", "ssh", "pbs", "sge" and "pbsf", the same as swiftInit. The most recently started server of the specified type will be used - to execute the apply call. The default value is "local". + to execute the apply call. If not specified, then + the swift.server option will be used. If that is not specified, the + most recently launched server of any type will be used. } \item{callsperbatch}{ The number of function calls to group together into a single batch. From noreply at svn.ci.uchicago.edu Mon Feb 7 16:50:54 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 7 Feb 2011 16:50:54 -0600 (CST) Subject: [Swift-commit] r4066 - in SwiftApps/SwiftR: Swift/R mxtests/swift Message-ID: <20110207225054.1B45C9CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-07 16:50:53 -0600 (Mon, 07 Feb 2011) New Revision: 4066 Modified: SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/R/Workers.R SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R Log: Minor fix for text, and converting openmx test Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 22:07:51 UTC (rev 4065) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 22:50:53 UTC (rev 4066) @@ -266,8 +266,8 @@ } else { stop(paste("Have you run swiftInit?\n", - "It appears that no SwiftR servers of type", server, - "are running, as no request pipe exists at", + "It appears that the SwiftR server is not running", + ", as no request pipe exists at", requestPipeName)) } Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-07 22:07:51 UTC (rev 4065) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-07 22:50:53 UTC (rev 4066) @@ -204,7 +204,8 @@ } # shut down all worker processes using kill for (worker in workers) { - cat("Terminating worker", worker$pid, " of type ",worker$server, "\n") + cat(paste("Terminating worker", worker$pid, "of type", + worker$server, "\n")) cmdString <- file.path(.find.package("Swift"), "exec/killtree &> /dev/null ") killCmd <- paste(cmdString, worker$pid) system(killCmd, wait=FALSE) Modified: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-07 22:07:51 UTC (rev 4065) +++ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-07 22:50:53 UTC (rev 4066) @@ -101,7 +101,7 @@ topModel <- mxModel(name = 'container') -#sfExportAll() +swiftExportAll() submodels <- swiftLapply(1:nReps, createNewModel, 'stErrSim', template) From noreply at svn.ci.uchicago.edu Mon Feb 7 17:31:18 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 7 Feb 2011 17:31:18 -0600 (CST) Subject: [Swift-commit] r4067 - SwiftApps/SwiftR/Swift/R Message-ID: <20110207233118.DF3209CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-07 17:31:18 -0600 (Mon, 07 Feb 2011) New Revision: 4067 Modified: SwiftApps/SwiftR/Swift/R/Apply.R Log: Introduced a bug in swiftLapply previously by passing argument unnecessarily to swiftapply. Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 22:50:53 UTC (rev 4066) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-07 23:31:18 UTC (rev 4067) @@ -114,7 +114,7 @@ arglists[[i]] <- list(tlist[[i]], ...); } names(arglists) = names(tlist) - swiftapply(func, arglists, ...) + swiftapply(func, arglists) } From noreply at svn.ci.uchicago.edu Tue Feb 8 08:58:03 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 8 Feb 2011 08:58:03 -0600 (CST) Subject: [Swift-commit] r4068 - SwiftApps/SwiftR/Swift/R Message-ID: <20110208145803.9E8B89CCC7@svn.ci.uchicago.edu> Author: tga Date: 2011-02-08 08:58:03 -0600 (Tue, 08 Feb 2011) New Revision: 4068 Modified: SwiftApps/SwiftR/Swift/R/Workers.R Log: Bugfix: didn't handle case where no workers started from R. Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-07 23:31:18 UTC (rev 4067) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-08 14:58:03 UTC (rev 4068) @@ -150,8 +150,11 @@ # for now, we will rely on the shell script's output to inform # the user if there was a problem with the workers out <- system(cmdString, intern=TRUE) - pid <- out[1] - workdir <- out[2] + if (length(out) != 2) + stop(paste("Unexpected output from start-swift: '", out, "'", + "Launching may have failed")) + pid <- out[[1]] + workdir <- out[[2]] cat("Started worker manager with pid ", pid, "\n") output <- list() @@ -232,7 +235,8 @@ # no argument provided, the most recent server of any type getWorker <- function (server=NULL) { if (!exists(".swift.workers") - || is.null(.swift.workers)) { + || is.null(.swift.workers + || length(.swift.workers) == 0)) { return (NULL) } if (is.null(server)) { From noreply at svn.ci.uchicago.edu Tue Feb 8 09:13:36 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 8 Feb 2011 09:13:36 -0600 (CST) Subject: [Swift-commit] r4069 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110208151336.86C189CCC7@svn.ci.uchicago.edu> Author: tga Date: 2011-02-08 09:13:35 -0600 (Tue, 08 Feb 2011) New Revision: 4069 Modified: SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/Swift/exec/start-swift-daemon Log: Check that directory creation went ok. Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-08 14:58:03 UTC (rev 4068) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-08 15:13:35 UTC (rev 4069) @@ -1,6 +1,6 @@ #! /bin/bash -#set -x +set -x export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap @@ -417,6 +417,11 @@ if [ "$workdir" = NONE ] then trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success + if [ "$?" != "0" ] + then + echo "Could not create temporary directory under $tmp/$USER/SwiftR" + exit 1 + fi else echo Working in $workdir trundir=$workdir Modified: SwiftApps/SwiftR/Swift/exec/start-swift-daemon =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-daemon 2011-02-08 14:58:03 UTC (rev 4068) +++ SwiftApps/SwiftR/Swift/exec/start-swift-daemon 2011-02-08 15:13:35 UTC (rev 4069) @@ -10,8 +10,15 @@ # Choose a working directory tmp=${SWIFTR_TMP:-/tmp} -workdir=$(mktemp -d $tmp/$USER/SwiftR/swift.XXXX) # FIXME: check success +mkdir -p $tmp/$USER/SwiftR +workdir=$(mktemp -d $tmp/$USER/SwiftR/swift.XXXX) +if [ "$?" != "0" ] +then + echo "Could not create temporary directory under $tmp/$USER/SwiftR" + exit 1 +fi + # Start as detached daemon, with output going to stderr # Start up a subprocess with a new process group # childpid will be of form '[jobno] pid' From noreply at svn.ci.uchicago.edu Tue Feb 8 09:28:07 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 8 Feb 2011 09:28:07 -0600 (CST) Subject: [Swift-commit] r4070 - SwiftApps/SwiftR/Swift/R Message-ID: <20110208152807.7384B9CCC7@svn.ci.uchicago.edu> Author: tga Date: 2011-02-08 09:28:07 -0600 (Tue, 08 Feb 2011) New Revision: 4070 Modified: SwiftApps/SwiftR/Swift/R/Workers.R Log: Bugfix in getWorker() - introduced a logical error. Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-08 15:13:35 UTC (rev 4069) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-08 15:28:07 UTC (rev 4070) @@ -235,8 +235,8 @@ # no argument provided, the most recent server of any type getWorker <- function (server=NULL) { if (!exists(".swift.workers") - || is.null(.swift.workers - || length(.swift.workers) == 0)) { + || is.null(.swift.workers) + || (length(.swift.workers) == 0)) { return (NULL) } if (is.null(server)) { From noreply at svn.ci.uchicago.edu Tue Feb 8 17:07:25 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 8 Feb 2011 17:07:25 -0600 (CST) Subject: [Swift-commit] r4071 - SwiftApps/SwiftR/Swift/R Message-ID: <20110208230725.38CE29CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-08 17:07:24 -0600 (Tue, 08 Feb 2011) New Revision: 4071 Modified: SwiftApps/SwiftR/Swift/R/Export.R Log: Fix to export that resulted in save not resolving some variables correctly. Modified: SwiftApps/SwiftR/Swift/R/Export.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Export.R 2011-02-08 15:28:07 UTC (rev 4070) +++ SwiftApps/SwiftR/Swift/R/Export.R 2011-02-08 23:07:24 UTC (rev 4071) @@ -78,9 +78,9 @@ #TODO: file mode? if (is.null(list)) - save(..., file=file) + save(..., file=file, envir=parent.frame()) else - save(..., list=list, file=file) + save(..., list=list, file=file, envir=parent.frame()) session <- getOption(".swift.session") if (is.null(session)) session <- newSession() From noreply at svn.ci.uchicago.edu Wed Feb 9 13:17:27 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 9 Feb 2011 13:17:27 -0600 (CST) Subject: [Swift-commit] r4072 - SwiftApps/SwiftR/Swift/R Message-ID: <20110209191727.B91559CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-09 13:17:27 -0600 (Wed, 09 Feb 2011) New Revision: 4072 Modified: SwiftApps/SwiftR/Swift/R/Apply.R Log: Removed write timeout by default: this behaved poorly when the swift workers took some time to start up. Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-08 23:07:24 UTC (rev 4071) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-09 19:17:27 UTC (rev 4072) @@ -87,7 +87,7 @@ requestPipeName=file.path(swiftServerDir,"requestpipe") resultPipeName=file.path(swiftServerDir,"resultpipe") # Try sending: this function will cause error if it fails - sendServiceRequest(requestPipeName, reqdir, server) + sendServiceRequest(requestPipeName, reqdir, server, timeout=timeout) res <- getServiceResponse(resultPipeName, timeout) # Check that the message was correct @@ -244,7 +244,7 @@ } } -sendServiceRequest <- function (requestPipeName, reqdir, server=NULL) { +sendServiceRequest <- function (requestPipeName, reqdir, server=NULL, timeout=NULL) { # fifo will block irrecoverably if there is no reader on the # other end of the requestPipe. This is bad. The swift worker # script is responsible for deleting the request pipe when it @@ -254,11 +254,10 @@ # there is a race condition here if the fifo disappears in # between checking for existence and opening the fifo, but # the timeout will catch that unlikely case - writeTimeout <- 4000 success <- writeFifo(requestPipeName,paste(reqdir,"\n",sep=""), - timeout=writeTimeout) + timeout=timeout) if (! success) { - stop(paste("timeout of", writeTimeout, + stop(paste("timeout of", timeout, "ms exceeded when attempting to", "rendezvous with SwiftR server of type", server, ".\n", "Maybe it is not running or it has crashed")) From noreply at svn.ci.uchicago.edu Wed Feb 9 13:38:14 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 9 Feb 2011 13:38:14 -0600 (CST) Subject: [Swift-commit] r4073 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110209193814.58F199CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-09 13:38:14 -0600 (Wed, 09 Feb 2011) New Revision: 4073 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/SwiftRServer.R Log: Added error handling. Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-09 19:17:27 UTC (rev 4072) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-09 19:38:14 UTC (rev 4073) @@ -91,7 +91,9 @@ mkfifo $SLOTDIR/toR.fifo mkfifo $SLOTDIR/fromR.fifo chmod +x $RServerScript - PATH=.:$PATH $RServerScript $SLOTDIR >& $SLOTDIR/R.log & # launch R server + echo "$0: INFO: Launching $RServerScript $SLOTDIR" + PATH=.:$PATH + strace $RServerScript $SLOTDIR >& $SLOTDIR/R.log & # launch R server # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" else @@ -125,8 +127,15 @@ echo dummy stderr response 1>&2 # FIXME - testing if this is the provider staging problem (not xfering zero len stderr) -head -3 < $SLOTDIR/fromR.fifo # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit") +res=$(head -3 < $SLOTDIR/fromR.fifo) # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit") +echo "$res" +# Test if R server reported an error +if echo "$res" | grep -q '^ERROR:' +then + exit 1 +fi + echo DB: Got response rmdir $SLOTDIR/mutex Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-09 19:17:27 UTC (rev 4072) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-09 19:38:14 UTC (rev 4073) @@ -2,6 +2,8 @@ require(methods) argv = commandArgs(TRUE) +print("SwiftRServer arguments:") +print(argv) fifoDir = argv[1]; # FIXME: test for valid arguments @@ -11,8 +13,8 @@ SwiftRFifoServer <- function( fifoBasename ) { - inFifoName = paste(fifoDir,"/toR.fifo",sep="") - outFifoName = paste(fifoDir,"/fromR.fifo",sep="") + inFifoName = file.path(fifoDir,"toR.fifo") + outFifoName = file.path(fifoDir,"fromR.fifo") dir = getwd() cat("SwiftRServer starting in dir:",dir,"\n"); repeat { @@ -45,6 +47,14 @@ } } +doFail <- function( error ) +{ + outFifoName = file.path(fifoDir,"fromR.fifo") + outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) + cat(file=outfifo, paste("ERROR: R server failed with error", gsub("\n", " ", error))) + close(outfifo) +} + doInit <- function(initializer, env) { #print(sprintf("received initializer=%s latestInitializer=%s\n", # initializer, latestInitializer)); @@ -174,5 +184,5 @@ cat("calling server: dir=",fifoDir,"\n") setwd(fifoDir) -SwiftRFifoServer(fifoDir) -cat("returned from server: dir=",fifoDir,"\n") +tryCatch(SwiftRFifoServer(fifoDir), error= + cat("returned from server: dir=",fifoDir,"\n")) From noreply at svn.ci.uchicago.edu Wed Feb 9 13:49:31 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 9 Feb 2011 13:49:31 -0600 (CST) Subject: [Swift-commit] r4074 - in SwiftApps/SwiftR/Swift: R exec Message-ID: <20110209194931.BE9229CCC7@svn.ci.uchicago.edu> Author: tga Date: 2011-02-09 13:49:31 -0600 (Wed, 09 Feb 2011) New Revision: 4074 Modified: SwiftApps/SwiftR/Swift/R/Fifo.R SwiftApps/SwiftR/Swift/exec/fifowrite Log: Fixed bug in no-timeout implementation of fifo and added error checking to fifowrite script. Modified: SwiftApps/SwiftR/Swift/R/Fifo.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Fifo.R 2011-02-09 19:38:14 UTC (rev 4073) +++ SwiftApps/SwiftR/Swift/R/Fifo.R 2011-02-09 19:49:31 UTC (rev 4074) @@ -27,7 +27,7 @@ if (is.null(timeout)) { - cmdString <- bin + cmdString <- paste(bin, fifopath) } else { cmdString <- paste(bin, fifopath, "-t", timeout) Modified: SwiftApps/SwiftR/Swift/exec/fifowrite =================================================================== --- SwiftApps/SwiftR/Swift/exec/fifowrite 2011-02-09 19:38:14 UTC (rev 4073) +++ SwiftApps/SwiftR/Swift/exec/fifowrite 2011-02-09 19:49:31 UTC (rev 4074) @@ -6,6 +6,13 @@ fifo=$1 shift + +if [ ! -p $fifo ] +then + echo $fifo is not a named pipe + exit 1 +fi + timeout=0 msg="" From noreply at svn.ci.uchicago.edu Wed Feb 9 15:13:56 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 9 Feb 2011 15:13:56 -0600 (CST) Subject: [Swift-commit] r4075 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110209211356.6AE029CCC7@svn.ci.uchicago.edu> Author: tga Date: 2011-02-09 15:13:56 -0600 (Wed, 09 Feb 2011) New Revision: 4075 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh Log: Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-09 19:49:31 UTC (rev 4074) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-09 21:13:56 UTC (rev 4075) @@ -93,7 +93,9 @@ chmod +x $RServerScript echo "$0: INFO: Launching $RServerScript $SLOTDIR" PATH=.:$PATH - strace $RServerScript $SLOTDIR >& $SLOTDIR/R.log & # launch R server + #echo PATH: $PATH > $SLOTDIR/R.log + #which Rscript >> $SLOTDIR/R.log + $RServerScript $SLOTDIR >> $SLOTDIR/R.log 2>&1 & # launch R server # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" else From noreply at svn.ci.uchicago.edu Wed Feb 9 16:09:06 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 9 Feb 2011 16:09:06 -0600 (CST) Subject: [Swift-commit] r4076 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110209220906.9F8FD9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-09 16:09:06 -0600 (Wed, 09 Feb 2011) New Revision: 4076 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/SwiftRServer.R Log: modified so that failure running R batch would result in an error message correctly being sent down the pipe Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-09 21:13:56 UTC (rev 4075) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-09 22:09:06 UTC (rev 4076) @@ -50,12 +50,14 @@ fi } + trap "onexit 1" 1 trap "onexit 2" 2 trap "onexit 3" 3 trap "onexit 15" 15 trap "onexit EXIT" EXIT + function idletimer { cd $1 while true; do @@ -92,9 +94,10 @@ mkfifo $SLOTDIR/fromR.fifo chmod +x $RServerScript echo "$0: INFO: Launching $RServerScript $SLOTDIR" - PATH=.:$PATH + PATH=.:$PATH #echo PATH: $PATH > $SLOTDIR/R.log #which Rscript >> $SLOTDIR/R.log + echo $PATH $RServerScript $SLOTDIR >> $SLOTDIR/R.log 2>&1 & # launch R server # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" @@ -122,26 +125,31 @@ done echo DB: Obtained $SLOTDIR/mutex -echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo -touch $SLOTDIR/lastwrite +if echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo +then + touch $SLOTDIR/lastwrite -echo DB: Sent request + echo DB: Sent request -echo dummy stderr response 1>&2 # FIXME - testing if this is the provider staging problem (not xfering zero len stderr) + echo dummy stderr response 1>&2 # FIXME - testing if this is the provider staging problem (not xfering zero len stderr) -res=$(head -3 < $SLOTDIR/fromR.fifo) # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit") -echo "$res" -# Test if R server reported an error -if echo "$res" | grep -q '^ERROR:' -then - exit 1 -fi + res=$(cat < $SLOTDIR/fromR.fifo) + echo DB: Got response: $res + # Test if R server reported an error + if echo "$res" | grep -q '^ERROR:' + then + exit 1 + fi -echo DB: Got response + rmdir $SLOTDIR/mutex -rmdir $SLOTDIR/mutex + echo DB: Freed $SLOTDIR/mutex +else + echo "ERROR: Could not write to fifo ok" + rmdir $SLOTDIR/mutex -echo DB: Freed $SLOTDIR/mutex + exit 1 +fi # Fixme: how to get exceptions and stdout/stderr text from R server ??? Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-09 21:13:56 UTC (rev 4075) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-09 22:09:06 UTC (rev 4076) @@ -21,24 +21,25 @@ cat("SwiftRServer at top of loop is in dir:",getwd(),"\n"); setwd(dir) # FIXME: not yet sure what is changing the CWD infifo <- fifo(inFifoName,open="rb",blocking=TRUE) - outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) cmd <- scan(infifo,what=list("","",""),nlines=1,flush=FALSE,quiet=FALSE,fill=TRUE) op = cmd[[1]] callBatchFileName = cmd[[2]] resultBatchFileName = cmd[[3]] cat("cmd: op=",op," call batch=",callBatchFileName," result batch=",resultBatchFileName,"\n"); - cat(file=outfifo, "Read from sock: cmd=",cmd[[1]], cmd[[2]],"\n"); + #cat(file=outfifo, "Read from sock: cmd=",cmd[[1]], cmd[[2]],"\n"); if(is.null(op)) { cat("op is NULL\n") } else { if( identical(op,"quit")) { + outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) cat(file=outfifo,"Server closing down.\n") break } if( identical(op,"run")) { - cat(file=outfifo, "About to run batch file: ", callBatchFileName,"\n"); + #cat(file=outfifo, "About to run batch file: ", callBatchFileName,"\n"); runBatch(callBatchFileName,resultBatchFileName) + outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) cat(file=outfifo, "Batch completed: result batch file: ", resultBatchFileName,"\n"); } } @@ -184,5 +185,5 @@ cat("calling server: dir=",fifoDir,"\n") setwd(fifoDir) -tryCatch(SwiftRFifoServer(fifoDir), error= - cat("returned from server: dir=",fifoDir,"\n")) +tryCatch(SwiftRFifoServer(fifoDir), error=doFail) +cat("returned from server: dir=",fifoDir,"\n")) From noreply at svn.ci.uchicago.edu Thu Feb 10 08:32:43 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 10 Feb 2011 08:32:43 -0600 (CST) Subject: [Swift-commit] r4077 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110210143243.CDAE09CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-10 08:32:42 -0600 (Thu, 10 Feb 2011) New Revision: 4077 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Removed echoing of start-swift script Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-09 22:09:06 UTC (rev 4076) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-10 14:32:42 UTC (rev 4077) @@ -1,6 +1,6 @@ #! /bin/bash -set -x +#set -x export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap From noreply at svn.ci.uchicago.edu Thu Feb 10 08:36:11 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 10 Feb 2011 08:36:11 -0600 (CST) Subject: [Swift-commit] r4078 - in SwiftApps/SwiftR: . Swift Swift/R Swift/tests Message-ID: <20110210143611.0EE399CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-10 08:36:10 -0600 (Thu, 10 Feb 2011) New Revision: 4078 Added: SwiftApps/SwiftR/Swift/R/TestFramework.R SwiftApps/SwiftR/Swift/tests/runtests.R Removed: SwiftApps/SwiftR/Swift/tests/TestSwift.R Modified: SwiftApps/SwiftR/Makefile SwiftApps/SwiftR/Swift/NAMESPACE SwiftApps/SwiftR/Swift/R/Tests.R Log: Revamped test suite: now we represent testsuite as a data structure, and provide functions to run tests, collect results and timing. and present a summary of test correctness, performance. Added a target "test" to the makefile. This serves two purposes: * Encourage regular running of test suite and addition of tests to preserver developer sanity * Basis for benchmarks: timing results for each test function are collected in a list * Tobe implemented later: parameters to runtests file to allow testing of different servers, etc. Modified: SwiftApps/SwiftR/Makefile =================================================================== --- SwiftApps/SwiftR/Makefile 2011-02-10 14:32:42 UTC (rev 4077) +++ SwiftApps/SwiftR/Makefile 2011-02-10 14:36:10 UTC (rev 4078) @@ -37,6 +37,9 @@ check: $(TBALL) R CMD check $(TBALL) +test: install + ./Swift/tests/runtests.R + clean: rm -rf Swift/inst/swift/* rm $(TBALL) Modified: SwiftApps/SwiftR/Swift/NAMESPACE =================================================================== --- SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-10 14:32:42 UTC (rev 4077) +++ SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-10 14:36:10 UTC (rev 4078) @@ -13,3 +13,10 @@ export(runAllSwiftTests) export(basicSwiftTest) exportPattern("^swiftTest") +export(makeTestGroup) +export(runTestGroup) +export(runTestSuite) +exportPattern("^testGroup") +export(swiftTestSuite) +export(printTestGroup) +export(mkTest) Added: SwiftApps/SwiftR/Swift/R/TestFramework.R =================================================================== --- SwiftApps/SwiftR/Swift/R/TestFramework.R (rev 0) +++ SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-10 14:36:10 UTC (rev 4078) @@ -0,0 +1,66 @@ +# Generic functions for running and displaying results +# from a suite of tests + +mkTest <- function (f) { + test = list() + test$name = substitute(f) + test$fun = f + test$passed = NULL + return (test) +} + +makeTestGroup <- function (name, tlist, init=NULL) { + group = list() + group$init = init + group$tests = tlist + return (group) +} + +runTestGroup <- function (group) { + cat("\n*** Starting test group ", group$name, "***\n") + if (!is.null(group$init)) + group$init() + for (i in 1:length(group$tests)) { + test <- group$tests[[i]] + cat("\n== Starting test ", test$name, " ==\n") + startTime = proc.time()[["elapsed"]] + group$tests[[i]]$passed <- try(test$fun()) + endTime <- proc.time()[["elapsed"]] + runTime <- endTime - startTime + group$tests[[i]]$time <- runTime + + } + return (group) +} + +printTestGroup <- function (group) { + cat("\n*** Test group ", group$name, "***\n") + resFmt <- function(res) { + if (is.null(res)) { + "???" + } else { + if (inherits(res, "try-error")) + {paste("ERROR: '", res, "'") } + else + {if (res) "PASSED" else "FAILED"} + } + } + timeFmt <- function (time) { + if (is.null(time)) + "???" + else + sprintf("%.2f", time) + } + for (test in group$tests) { + cat(paste(test$name, ": ", resFmt(test$passed), + " (", timeFmt(test$time), ")\n", sep="")) + } +} + +runTestSuite <- function (suite) { + suite$setup() + resSuite <- lapply(suite$tests, runTestGroup) + lapply(resSuite, printTestGroup) + suite$teardown() + return (NULL) +} Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-10 14:32:42 UTC (rev 4077) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-10 14:36:10 UTC (rev 4078) @@ -1,21 +1,18 @@ - -initSwiftTestOptions <- function() -{ +initSwiftTestOptions <- function() { options(swift.site="service") options(swift.keepwork=TRUE) initcmds <- "initVar1 <- 19; initVar2 <- sqrt(400)+3" - options(swift.initialexpr=initcmds) # Set here; used in test group 4 + options(swift.initialexpr=initcmds) # Set here; used in test group} + # FIXME: Save prior options here: restore them when tests are done. } +basicSwiftTest <- function() { swiftTest_1.1() } -swiftTest_1.1 <- function() -{ +swiftTest_1.1 <- function() { - initSwiftTestOptions() - cat("\n*** Starting test 1.1 ***\n\n") sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } @@ -37,207 +34,18 @@ if(identical(localres,swiftres[[1]])) { cat("\n==> test 1.1 passed\n") + return (TRUE) } else { cat("\n==> test 1.1 FAILED !!!!!\n") + return (FALSE) } } -#1.3.* tests export functionality -swiftTest_1.3.1 <- function() -{ - initSwiftTestOptions() - - cat("\n*** Starting test 1.3.1 ***\n\n") - y <- 1 - inc <- function(x) { x + y } - - arglist = list(1,2,3,4,5,6,7,8,9) - - cat("Test of local lapply\n") - localres = lapply(arglist, inc) - cat("local result=\n") - print(localres) - - # Test swiftExport - swiftExport(y) - cat("\nTest of swiftLapply\n") - swiftres = swiftLapply(arglist, inc) - cat("Swift result:\n") - print(swiftres) - - if(identical(localres,swiftres)) { - cat("\n==> test 1.3.1 passed\n") - } else { - cat("\n==> test 1.3.1 FAILED !!!!!\n") - } -} - -swiftTest_1.3.2 <- function() -{ - # Test swiftExportAll() - - cat("\n*** Starting test 1.3.2 ***\n\n") - y <<- 1 - inc <- function(x) { x + y } - - arglist = list(1,2,3,4,5,6,7,8,9) - - cat("Test of local lapply\n") - localres = lapply(arglist, inc) - cat("local result=\n") - print(localres) - - # Test swiftExport - swiftExportAll() - cat("\nTest of swiftLapply\n") - swiftres = swiftLapply(arglist, inc) - cat("Swift result:\n") - print(swiftres) - - if(identical(localres,swiftres)) { - cat("\n==> test 1.3.2 passed\n") - } else { - cat("\n==> test 1.3.2 FAILED !!!!!\n") - } -} - - -swiftTest_1.3.3 <- function() -{ - - initSwiftTestOptions() - - cat("\n*** Starting test 1.3.3 ***\n\n") - y <- 1 - inc <- function(x) { x + y } - - arglist = list(1,2,3,4) - - # Test swiftExportAll - swiftExportAll() - cat("\nTest of swiftLapply\n") - swiftres = swiftLapply(arglist, inc) - cat("Swift result before removal:\n") - print(swiftres) - - swiftRemoveAll() - swiftres = swiftLapply(arglist, inc) - cat("Swift result after removal:\n") - for (e in swiftres) { - if (!inherits(e, "try-error")) { - cat("\n==> test 1.3.3 FAILED !!!!!\n") - return() - } - } - cat("\n==> test 1.3.3 passed\n") -} - - -basicSwiftTest <- function() { swiftTest_1.1() } - -# .... more tests from below to move here -swiftTest_4.1 <- function() -{ - sumivars <- function() { initVar1+initVar2 } - - args=list() - arglist = rep(list(args),1) - - localres = 42 - - cat("\nTest of swiftapply(sumivars,arglist)\n") - swiftres = swiftapply(sumivars,arglist) - cat("Swift result:\n") - print(swiftres) - - if(identical(localres,swiftres[[1]])) { - cat("\n==> test 4.1 passed\n") - } else { - cat("\n==> test 4.1 FAILED !!!!!\n") - } -} - -swiftTest_4.2 <- function() -{ - - options(swift.initialexpr="initVar3 <- 123; initVar4 <- 100"); - - mulivars <- function() { initVar3*initVar4 } - - args=list() - arglist = rep(list(args),1) - - localres = 12300; - - cat("\nTest of swiftapply(mulivars,arglist)\n") - swiftres = swiftapply(mulivars,arglist) - cat("Swift result:\n") - print(swiftres) - - if(identical(localres,swiftres[[1]])) { - cat("\n==> test 4.2 passed\n") - } else { - cat("\n==> test 4.2 FAILED !!!!!\n") - } -} - -# Performance tests - -swiftTest_6.1 <- function(delay,ncalls) -{ - options(swift.initialexpr="initVar3 <- 123; initVar4 <- 100"); - - timed <- function(delay) { Sys.sleep(delay); delay } - - args=list(delay) - arglist = rep(list(args),ncalls) - - cat("\nTest of swiftapply(delay,arglist)\n") - - startTime = proc.time()[["elapsed"]] - swiftres = swiftapply(timed,arglist) - endTime = proc.time()[["elapsed"]] - runTime <- endTime - startTime - - cat("\n\n ===> Total elapsed unit test time = ",runTime," seconds.\n\n") - - cat("Swift result:\n") - print(swiftres[[1]]) - - if(identical(delay,swiftres[[1]])) { - cat("\n==> test 6.1 passed\n") - } else { - cat("\n==> test 6.1 FAILED !!!!!\n") - } - -} - -runAllSwiftTests <- function() -{ - # Launch workerif nothing already running - # testPid will be NULL if nothihg launched - testPid <- tryCatch( - (function() { swiftapply(log, list(list(1)),quiet=TRUE) ; - return(NULL); })(), - error=function(x) {swiftInit();}) - - ### FIXME: Save prior options here: restore them when tests are done. - ### Recovery if interrrupted? - - failures=0 - - startTime = proc.time()[["elapsed"]] - - cat("\n*** Starting test group 1 - functions on simple data structures ***\n\n") - - swiftTest_1.1() - - ##### Test 1.2 - +swiftTest_1.2.1 <- function () { # test 10 remote calls - - sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } + sumstuff <- function(treedata,cardata) + { sum( treedata$Height, cardata$dist ) } data(cars) data(trees) @@ -246,7 +54,6 @@ localres = do.call(sumstuff,args) - cat("\n*** Test 1.2.1: 10 calls to substuff()\n") swiftres <- swiftapply(sumstuff,arglist) cat("Swift result:\n") format(swiftres) @@ -261,13 +68,27 @@ if(diffs == 0) { cat("\n==> test 1.2.1 passed\n") + return (TRUE) } else { cat("\n!!!==> test 1.2.1 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } +} - cat("\n*** Test 1.2.2: 10 calls to substuff() - callsperbatch=10\n") +swiftTest_1.2.2 <- function () { + sumstuff <- function(treedata,cardata) + { sum( treedata$Height, cardata$dist ) } + data(cars) + data(trees) + + args=list(trees,cars) + arglist <- rep(list(args),10) + + localres = do.call(sumstuff,args) + + cat("*** 10 calls to sumstuff() - callsperbatch=10\n") swiftres = swiftapply(sumstuff,arglist,callsperbatch=10) cat("Swift result:\n") format(swiftres) @@ -282,13 +103,27 @@ if(diffs == 0) { cat("\n==> test 1.2.2 passed\n") + return (TRUE) } else { cat("\n!!!==> test 1.2.2 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } +} - cat("\n*** Test 1.2.3: 10 calls to substuff() - callsperbatch=2\n") +swiftTest_1.2.3 <- function () { + sumstuff <- function(treedata,cardata) + { sum( treedata$Height, cardata$dist ) } + data(cars) + data(trees) + + args=list(trees,cars) + arglist <- rep(list(args),10) + + localres = do.call(sumstuff,args) + + cat("*** 10 calls to substuff() - callsperbatch=2\n") swiftres = swiftapply(sumstuff,arglist,callsperbatch=2) cat("Swift result:\n") format(swiftres) @@ -303,13 +138,29 @@ if(diffs == 0) { cat("\n==> test 1.2.3 passed\n") + return (TRUE) } else { cat("\n!!!==> test 1.2.3 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } - cat("\n*** Test 1.2.4: 10 calls to substuff() - callsperbatch=3\n") +} + + +swiftTest_1.2.4 <- function () { + sumstuff <- function(treedata,cardata) + { sum( treedata$Height, cardata$dist ) } + data(cars) + data(trees) + + args=list(trees,cars) + arglist <- rep(list(args),10) + + localres = do.call(sumstuff,args) + + cat("*** 10 calls to substuff() - callsperbatch=3\n") swiftres = swiftapply(sumstuff,arglist,callsperbatch=3) swiftres <- swiftapply(sumstuff,arglist) cat("Swift result:\n") @@ -325,23 +176,180 @@ if(diffs == 0) { cat("\n==> test 1.2.4 passed\n") + return (TRUE) } else { cat("\n!!!==> test 1.2.4 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } +} - # swiftres = swiftapply(sumstuff,arglist,callsperbatch=2,site="pbs") - # test variations on local vs ssh vs pbs; coasters vs non; etc. +#1.3.* tests export functionality +swiftTest_1.3.1 <- function() { - swiftTest_1.3.1() - swiftTest_1.3.2() - swiftTest_1.3.3() + cat("\n*** Starting test 1.3.1 ***\n\n") + y <- 1 + inc <- function(x) { x + y } - ##### Test Group 2 + arglist = list(1,2,3,4,5,6,7,8,9) - cat("\n*** Starting test group 2 - test matrix passing***\n") + cat("Test of local lapply\n") + localres = lapply(arglist, inc) + cat("local result=\n") + print(localres) + # Test swiftExport + swiftExport(y) + rm(y) # Remove y here so that swift can't resolve locally + cat("\nTest of swiftLapply\n") + swiftres = swiftLapply(arglist, inc) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres)) { + cat("\n==> test 1.3.1 passed\n") + return(TRUE) + } else { + cat("\n==> test 1.3.1 FAILED !!!!!\n") + return(FALSE) + } +} + +swiftTest_1.3.2 <- function() { + # Test swiftExportAll() + + cat("\n*** Starting test 1.3.2 ***\n\n") + # put in global environment so we can test global env + swift.tmp.y <<- 1 + inc <- function(x) { x + swift.tmp.y } + + arglist = list(1,2,3,4,5,6,7,8,9) + + cat("Test of local lapply\n") + localres = lapply(arglist, inc) + cat("local result=\n") + print(localres) + + # Test swiftExport + swiftExportAll() + + # Remove y here so that swift can't resolve locally + rm(swift.tmp.y, envir=globalenv()) + cat("\nTest of swiftLapply\n") + swiftres = swiftLapply(arglist, inc) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres)) { + cat("\n==> test 1.3.2 passed\n") + return(TRUE) + } else { + cat("\n==> test 1.3.2 FAILED !!!!!\n") + return (FALSE) + } +} + + +swiftTest_1.3.3 <- function() { + + cat("\n*** Starting test 1.3.3 ***\n\n") + y <- 1 + inc <- function(x) { x + y } + + arglist = list(1,2,3,4) + + # Test swiftExportAll + swiftExportAll() + rm(y) # Remove y here so that swift can't resolve locally + cat("\nTest of swiftLapply\n") + swiftres1 <- swiftLapply(arglist, inc) + cat("Swift result before removal:\n") + print(swiftres1) + + swiftRemoveAll() + inc2 <- function(x) { x + y } + + swiftres2 <- swiftLapply(arglist, inc) + cat("Swift result after removal:\n") + print(swiftres2) + for (e in swiftres2) { + if (!inherits(e, "try-error")) { + cat("\n==> test 1.3.3 FAILED !!!!!\n") + return (FALSE) + return() + } + } + cat("\n==> test 1.3.3 passed\n") + return(TRUE) +} + +swiftTest_1.3.4 <- function () { + # This test demonstrates a case where a variable + # does not need to be exported. + cat("\n*** Starting test 1.3.4 ***\n\n") + swiftRemoveAll() # Cleanup + y <- 1 + inc <- function(x) { x + y } + # DONT export + arglist = list(1,2,3,4,5,6,7,8,9) + + cat("Test of local lapply\n") + localres = lapply(arglist, inc) + cat("local result=\n") + print(localres) + + cat("\nTest of swiftLapply\n") + swiftres = swiftLapply(arglist, inc) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres)) { + cat("\n==> test 1.3.4 passed\n") + return(TRUE) + } else { + cat("\n==> test 1.3.4 FAILED !!!!!\n") + return (FALSE) + } +} + + +swiftTest_1.3.5 <- function () { + # This test demonstrates a case where a variable + # does not need to be exported + + cat("\n*** Starting test 1.3.5 ***\n\n") + swiftRemoveAll() # Cleanup + f_inc <- function (z) { z + 1 } + f <- function () { + # Use another function + inc <- function(x) { f_inc(x) } + # DONT export + arglist = list(1,2,3,4,5,6,7,8,9) + + cat("Test of local lapply\n") + localres = lapply(arglist, inc) + cat("local result=\n") + print(localres) + + cat("\nTest of swiftLapply\n") + swiftres = swiftLapply(arglist, inc) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres)) { + cat("\n==> test 1.3.5 passed\n") + return (TRUE) + } else { + cat("\n==> test 1.3.5 FAILED !!!!!\n") + return (FALSE) + } + } + f() +} + + +swiftTest_2.1 <- function() { matfunc <- function( m1, m2 ) { (1/m1) %*% m2 @@ -361,9 +369,8 @@ swiftres = swiftapply(matfunc,arglist,callsperbatch=9) diffs <- 0 - #for(i in 1:length(swiftres) ) { for(i in c(seq(1,100,10),100)) { - if( !all.equal(swiftres[[i]],localres) ) { + if( !all.equal(swiftres[[i]],localres) ) { diffs <- diffs + 1 if( diffs < 10 ) cat(sprintf("res[%d]=%s\n",i,format(swiftres[[i]]))) } @@ -371,12 +378,20 @@ if(diffs == 0) { cat("\n==> test 2.1 passed\n") + return (TRUE) } else { - cat("\n!!!==> test 2.2 failed.\n") + cat("\n!!!==> test 2.1 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } +} +swiftTest_2.2 <- function() { + matfunc <- function( m1, m2 ) + { + (1/m1) %*% m2 + } n <- 237 n <- 50 m1 <- array(sin(1:n**2), dim=c(n,n)) @@ -384,7 +399,7 @@ localres = matfunc(m1,m2) - cat("\n*** Test 2.2: 123 calls to matfunc(dim=bigger) - callsperbatch=7\n") # FIXME make n easy to adjust and print actual value + cat("\n*** 123 calls to matfunc(dim=bigger) - callsperbatch=7\n") # FIXME make n easy to adjust and print actual value args=list(m1,m2) arglist <- rep(list(args),123) @@ -392,7 +407,6 @@ swiftres = swiftapply(matfunc,arglist,callsperbatch=7) diffs <- 0 - #for(i in 1:length(swiftres) ) { for(i in c(seq(1,length(swiftres),10),length(swiftres))) { if( !all.equal(swiftres[[i]],localres) ) { @@ -403,19 +417,16 @@ if(diffs == 0) { cat("\n==> test 2.2 passed\n") + return (TRUE) } else { cat("\n!!!==> test 2.2 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } +} - - ##### Test Group 3 - - cat("\n*** Starting test group 3 - test list element and name passing***\n") - - # Test if list element names are being sent and returned correctly - +data_3.1 <- function () { n <- 5 m1 <- array(sin(1:n**2), dim=c(n,n)) m2 <- t(m1) @@ -427,23 +438,28 @@ inlist$name2=987 inlist$mat1 = m1 inlist[[99]] = m2 + return (inlist) +} - listfunc <- function(ilist) - { - olist = ilist - olist$sum = ilist[[1]] + ilist[[2]] + ilist$name1 + ilist$name2 - olist$names = names(ilist) - olist$mprod = ilist$mat1 %*% ilist[[99]] - return(olist) - } - localres = listfunc(inlist) +listfunc_3.1 <- function(ilist) { + olist = ilist + olist$sum = ilist[[1]] + ilist[[2]] + ilist$name1 + ilist$name2 + olist$names = names(ilist) + olist$mprod = ilist$mat1 %*% ilist[[99]] + return(olist) +} +swiftTest_3.1 <- function() { + #Test if list element names are being sent and returned correctly + inlist <- data_3.1() + localres = listfunc_3.1(inlist) + cat("\n*** Starting test 3.1 - 4 calls in one batch of 5 ***\n") args=list(inlist) arglist <- rep(list(args),4) - swiftres = swiftapply(listfunc,arglist,callsperbatch=5) + swiftres = swiftapply(listfunc_3.1,arglist,callsperbatch=5) diffs <- 0 for(i in 1:length(swiftres) ) { @@ -455,18 +471,25 @@ if(diffs == 0) { cat("\n==> test 3.1 passed\n") + return (TRUE) } else { cat("\n!!!==> test 3.1 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } +} +swiftTest_3.2 <- function() { + inlist <- data_3.1() + localres = listfunc_3.1(inlist) cat("\n*** Starting test 3.2 - 99 calls in batches of 11 ***\n") + inlist <- data_3.1() args=list(inlist) arglist <- rep(list(args),99) - swiftres = swiftapply(listfunc,arglist,callsperbatch=11) + swiftres = swiftapply(listfunc_3.1,arglist,callsperbatch=11) diffs <- 0 for(i in 1:length(swiftres) ) { @@ -478,26 +501,67 @@ if(diffs == 0) { cat("\n==> test 3.2 passed\n") + return (TRUE) } else { cat("\n!!!==> test 3.2 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } - ##### Test Group 4 # test initialexpr string +} - cat("\n*** Starting test group 4 - test remote R service initialization string ***\n") +swiftTest_4.1 <- function() { + sumivars <- function() { initVar1+initVar2 } - swiftTest_4.1() - swiftTest_4.2() + args=list() + arglist = rep(list(args),1) + localres = 42 + cat("\nTest of swiftapply(sumivars,arglist)\n") + swiftres = swiftapply(sumivars,arglist) + cat("Swift result:\n") + print(swiftres) + if(identical(localres,swiftres[[1]])) { + cat("\n==> test 4.1 passed\n") + return (TRUE) + } else { + cat("\n==> test 4.1 FAILED !!!!!\n") + return (FALSE) + } +} - ##### Test Group 5 # test error handling +swiftTest_4.2 <- function() { - cat("\n*** Starting test group 5 - test remote R service error ***\n") + options(swift.initialexpr="initVar3 <- 123; initVar4 <- 100"); + mulivars <- function() { initVar3*initVar4 } + + args=list() + arglist = rep(list(args),1) + + localres = 12300; + + cat("\nTest of swiftapply(mulivars,arglist)\n") + swiftres = swiftapply(mulivars,arglist) + cat("Swift result:\n") + print(swiftres) + + if(identical(localres,swiftres[[1]])) { + cat("\n==> test 4.2 passed\n") + return (TRUE) + } else { + cat("\n==> test 4.2 FAILED !!!!!\n") + return (FALSE) + } +} + +swiftTest_5.1 <- function () { + # Test error handling + cat("\n*** Starting test group 5 - test ***\n") + arglist = list(list(1.0),list(2.0),list("3.0"),list(4.0),list(5.0)) cat("\nTest of swiftapply(sumivars,arglist)\n") @@ -517,22 +581,62 @@ if(diffs == 0) { cat("\n==> test 5.1 passed\n") + return (TRUE) } else { cat("\n!!!==> test 5.1 failed.\n") cat(sprintf(" %d result elements failed to match.\n",diffs)); failures=failures+1 + return (FALSE) } +} +swiftTest_6.1.1 <- function () { + swiftTest_6.1(2, 20) +} + +swiftTest_6.1 <- function(delay,ncalls) { + options(swift.initialexpr="initVar3 <- 123; initVar4 <- 100"); + + timed <- function(delay) { Sys.sleep(delay); delay } + + args=list(delay) + arglist = rep(list(args),ncalls) + + cat("\nTest of swiftapply(delay,arglist)\n") + + startTime = proc.time()[["elapsed"]] + swiftres = swiftapply(timed,arglist) + endTime = proc.time()[["elapsed"]] + runTime <- endTime - startTime + + cat("\n\n ===> Total elapsed unit test time = ",runTime," seconds.\n\n") + + cat("Swift result:\n") + print(swiftres[[1]]) + + if(identical(delay,swiftres[[1]])) { + cat("\n==> test 6.1 passed\n") + return (TRUE) + } else { + cat("\n==> test 6.1 FAILED !!!!!\n") + return (FALSE) + } + +} + +runAllSwiftTests <- function() { + + startTime = proc.time()[["elapsed"]] + + + runTestSuite(swiftTestSuite) + endTime <- proc.time()[["elapsed"]] runTime <- endTime - startTime cat("\n\n ===> Total elapsed test time = ",runTime," seconds.\n\n") - if (!is.null(testPid)) - swiftShutdown(testPid) -} # end function runAllTests +} -#options(swift.site="local") -#runAllTests() testloop <- function(npass) { @@ -543,3 +647,52 @@ system("sleep 3") } } + + +testGroup1.1 <- list( init=NULL, + name="1.1 Sanity Check", + tests = list(mkTest(swiftTest_1.1))) + +testGroup1.2 <- list( init=NULL, + name="1.2 Basic Test - Adding data sets", + tests = list(mkTest(swiftTest_1.2.1), + mkTest(swiftTest_1.2.2), mkTest(swiftTest_1.2.3), + mkTest(swiftTest_1.2.4))) + +testGroup1.3 <- list( init=NULL, + name="1.3 - Export functionality", + tests = list(mkTest(swiftTest_1.3.1), + mkTest(swiftTest_1.3.2), mkTest(swiftTest_1.3.3), + mkTest(swiftTest_1.3.4), mkTest(swiftTest_1.3.5)) ) + +testGroup2 <- list( init=NULL, + name="2 - test matrix passing", + tests = list(mkTest(swiftTest_2.1), + mkTest(swiftTest_2.2)) ) + +testGroup3 <- list( init=NULL, + name="3 - test list element and name passing", + tests = list(mkTest(swiftTest_3.1), + mkTest(swiftTest_3.2))) + +testGroup4 <- list( init=NULL, + name="4 - test remote R service initialization string", + tests = list(mkTest(swiftTest_4.1), + mkTest(swiftTest_4.2))) + +testGroup5 <- list( init=NULL, + name="5 - remote R service error ", + tests = list(mkTest(swiftTest_5.1))) + +testGroup6 <- list( init=NULL, + name="6 - performance tests", + tests = list(mkTest(swiftTest_6.1.1))) + + +swiftTestSuite <- list( + setup=function () { initSwiftTestOptions(); + swiftInit() }, # swiftInit controlled via options + tests=list(testGroup1.1, testGroup2, testGroup1.2, testGroup1.3, testGroup3, testGroup4, + testGroup5, testGroup6), + teardown=function () { swiftShutdown() }) + Deleted: SwiftApps/SwiftR/Swift/tests/TestSwift.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/TestSwift.R 2011-02-10 14:32:42 UTC (rev 4077) +++ SwiftApps/SwiftR/Swift/tests/TestSwift.R 2011-02-10 14:36:10 UTC (rev 4078) @@ -1,99 +0,0 @@ - -##### Setup - -# To paste for quick testing: ########### - -if (FALSE) { - -require(Swift) - -initcmds <- " -initVar1 <- 19 -initVar2 <- sqrt(400)+3 -" - -failures=0 - -options(swift.initialize=initcmds) # Set here; used in test group 4 - -sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } -data(cars) -data(trees) - -args=list(trees,cars) -arglist = rep(list(args),1) - -swiftres = swiftapply(sumstuff,arglist) - - -} #################################### - -require(Swift) - -runAllSwiftTests() - - - -if(FALSE) { # Test various batch sizes - - cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") - res = swiftapply(sumcrits,arglist,callsperbatch=10) - print(res) - - cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=2)\n") - res = swiftapply(sumcrits,arglist,callsperbatch=2) - print(res) - - cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=3)\n") - res = swiftapply(sumcrits,arglist,callsperbatch=3) - print(res) - - cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=20)\n") - res = swiftapply(sumcrits,arglist,callsperbatch=20) - print(res) -} - -if(FALSE) { # Larger-scale tests - - cat("\nTest of swiftapply(sumcrits,arglist[1000],callsperbatch=1)\n") - - arglist = rep(list(args),100) -# res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs") - res = swiftapply(sumcrits,arglist,callsperbatch=2) - - print(res[[1]]) - print(res[[1000]]) -} - -if(FALSE) { # OLD test 5 - -#### Test Group 5: test passive mode (make this start of multiple config tests?) - -cat("\n*** Starting test 5.1 ***\n\n") - -options(swift.site="passive") - -sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } -data(cars) -data(trees) - -args=list(trees,cars) -arglist = rep(list(args),1) - -cat("Test of local do.call(sumstuff)\n") -localres = do.call(sumstuff,args) -cat("local result=\n") -print(localres) - -cat("\nTest of swiftapply(sumstuff,arglist)\n") -swiftres = swiftapply(sumstuff,arglist) -cat("Swift result:\n") -print(swiftres) - -if(identical(localres,swiftres[[1]])) { - cat("\n==> test 1.1 passed\n") -} else { - cat("\n==> test 1.1 FAILED !!!!!\n") -} - -} # end if(FALSE) Added: SwiftApps/SwiftR/Swift/tests/runtests.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/runtests.R (rev 0) +++ SwiftApps/SwiftR/Swift/tests/runtests.R 2011-02-10 14:36:10 UTC (rev 4078) @@ -0,0 +1,5 @@ +#!/bin/env Rscript + +library(Swift) +#TODO: take command line options to setup options +runAllSwiftTests() Property changes on: SwiftApps/SwiftR/Swift/tests/runtests.R ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Thu Feb 10 08:42:13 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 10 Feb 2011 08:42:13 -0600 (CST) Subject: [Swift-commit] r4079 - SwiftApps/SwiftR/Swift/man Message-ID: <20110210144213.DD08B9CC95@svn.ci.uchicago.edu> Author: tga Date: 2011-02-10 08:42:13 -0600 (Thu, 10 Feb 2011) New Revision: 4079 Modified: SwiftApps/SwiftR/Swift/man/swiftapply.Rd Log: Corrected documentation to correctly explain tiemout behaviour for swiftInit Modified: SwiftApps/SwiftR/Swift/man/swiftapply.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/swiftapply.Rd 2011-02-10 14:36:10 UTC (rev 4078) +++ SwiftApps/SwiftR/Swift/man/swiftapply.Rd 2011-02-10 14:42:13 UTC (rev 4079) @@ -76,7 +76,9 @@ } \item{timeout}{ If provided, swiftapply will return if a response is not received by - the swift server in the specified time window in milliseconds. + the swift server in the specified time window in milliseconds. The timeout + applies to two time intervals: receiving an acknowledgement from the swift + server that the request was received, and receiving the final result. } \item{quiet}{ From noreply at svn.ci.uchicago.edu Thu Feb 10 08:46:29 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 10 Feb 2011 08:46:29 -0600 (CST) Subject: [Swift-commit] r4080 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110210144629.EB9A59CC95@svn.ci.uchicago.edu> Author: tga Date: 2011-02-10 08:46:29 -0600 (Thu, 10 Feb 2011) New Revision: 4080 Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R Log: Changed how environments are managed within the R server: the previous scheme where we created an independent environment to run functions in was problematic as the approach wiped out closures attached to functions which prevented applied functions correctly resolving non-global variables or library functions in its closure. Now exported data goes into the global environment of the R server, and all of the functions and data required for the R server to run are stored within variables beginning with "." in the R global environment. This allows imported data to be cleaned out of the global environment when needed. Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-10 14:42:13 UTC (rev 4079) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-10 14:46:29 UTC (rev 4080) @@ -1,189 +1,188 @@ #! /usr/bin/env Rscript -require(methods) -argv = commandArgs(TRUE) -print("SwiftRServer arguments:") -print(argv) +library(methods) -fifoDir = argv[1]; # FIXME: test for valid arguments -cat(Sys.getpid(),file=paste(fifoDir,"/R.pid",sep="")) -.current.initializer <<- "" +# Enclose everything within .main environment so that we can use global environmen tfor process data +# Note; Everything in the global environment that belong to this script should be prefixed +# with a .: other thrings are assumed to belong to script. +.main <- function (argv) { -SwiftRFifoServer <- function( fifoBasename ) -{ - inFifoName = file.path(fifoDir,"toR.fifo") - outFifoName = file.path(fifoDir,"fromR.fifo") - dir = getwd() - cat("SwiftRServer starting in dir:",dir,"\n"); - repeat { - cat("SwiftRServer at top of loop is in dir:",getwd(),"\n"); - setwd(dir) # FIXME: not yet sure what is changing the CWD - infifo <- fifo(inFifoName,open="rb",blocking=TRUE) - cmd <- scan(infifo,what=list("","",""),nlines=1,flush=FALSE,quiet=FALSE,fill=TRUE) - op = cmd[[1]] - callBatchFileName = cmd[[2]] - resultBatchFileName = cmd[[3]] - cat("cmd: op=",op," call batch=",callBatchFileName," result batch=",resultBatchFileName,"\n"); - #cat(file=outfifo, "Read from sock: cmd=",cmd[[1]], cmd[[2]],"\n"); - if(is.null(op)) { - cat("op is NULL\n") - } - else { - if( identical(op,"quit")) { - outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) - cat(file=outfifo,"Server closing down.\n") - break - } - if( identical(op,"run")) { - #cat(file=outfifo, "About to run batch file: ", callBatchFileName,"\n"); - runBatch(callBatchFileName,resultBatchFileName) - outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) - cat(file=outfifo, "Batch completed: result batch file: ", resultBatchFileName,"\n"); - } - } - close(infifo) - close(outfifo) - } -} + print("SwiftRServer arguments:") + print(argv) -doFail <- function( error ) -{ - outFifoName = file.path(fifoDir,"fromR.fifo") - outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) - cat(file=outfifo, paste("ERROR: R server failed with error", gsub("\n", " ", error))) - close(outfifo) -} + fifoDir = argv[1]; # FIXME: test for valid arguments -doInit <- function(initializer, env) { - #print(sprintf("received initializer=%s latestInitializer=%s\n", - # initializer, latestInitializer)); - if( initializer != .current.initializer) { - eval(parse(text=initializer), envir=env) - .current.initializer <<- initializer - } -} + cat(Sys.getpid(),file=paste(fifoDir,"/R.pid",sep="")) -failBatch <- function(rcall, try.error, resultBatchFileName) { - # This function is called when an error occurs and - # no valid argument values can be produced - # It fills a result array with the try.error object - # and writes it to resultBatchFileName - result <- list() - for (c in 1:length(rcall$arglistbatch)) { - result[[c]] <- try.error + .current.initializer <<- "" + + SwiftRFifoServer <- function( fifoBasename ) + { + inFifoName = file.path(fifoDir,"toR.fifo") + outFifoName = file.path(fifoDir,"fromR.fifo") + dir = getwd() + cat("SwiftRServer starting in dir:",dir,"\n"); + repeat { + cat("SwiftRServer at top of loop is in dir:",getwd(),"\n"); + setwd(dir) # FIXME: not yet sure what is changing the CWD + infifo <- fifo(inFifoName,open="rb",blocking=TRUE) + cmd <- scan(infifo,what=list("","",""),nlines=1,flush=FALSE,quiet=FALSE,fill=TRUE) + op = cmd[[1]] + callBatchFileName = cmd[[2]] + resultBatchFileName = cmd[[3]] + cat("DB: cmd: op=",op," call batch=",callBatchFileName," result batch=",resultBatchFileName,"\n"); + if(is.null(op)) { + cat("op is NULL\n") + outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) + cat(file=outfifo,"op is NULL.\n") + } + else { + if( identical(op,"quit")) { + outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) + cat(file=outfifo,"Server closing down.\n") + break + } + if( identical(op,"run")) { + cat("DB: About to run batch file: ", callBatchFileName,"\n"); + runBatch(callBatchFileName,resultBatchFileName) + outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) + cat(file=outfifo, "Batch completed: result batch file: ", resultBatchFileName,"\n"); + } + } + close(infifo) + close(outfifo) + } } - save(result,file=resultBatchFileName) -} -runBatch <- function( callBatchFileName, resultBatchFileName ) -{ - success <- try(load(callBatchFileName)); - if (inherits(success, "try-error")) { - failBatch(rcall, paste(callBatchFileName, "could not be opened"), - resultBatchFileName) - return() - } - success <- try(loadImports(rcall$imports)) - if (inherits(success, "try-error")) { - failBatch(rcall, success, resultBatchFileName) - return() - } + doFail <- function( error ) + { + outFifoName = file.path(fifoDir,"fromR.fifo") + outfifo <- fifo(outFifoName,open="wb",blocking=TRUE) + cat(paste("ERROR: R server failed with error", gsub("\n", " ", error)), "\n") + cat(file=outfifo, paste("ERROR: R server failed with error", gsub("\n", " ", error)), "\n") + close(outfifo) + } - if (exists(".current.envir", envir=globalenv())) { - env = get(".current.envir", envir=globalenv()) - } - else { - # Create a new environment to work in - if (is.null(importlist) || is.null(importlist$session)) { - # If the client provides no info about the environment, - # we should create a new environment to work in - env = new.env(parent=parent.env(globalenv())) + doInit <- function(initializer) { + #print(sprintf("received initializer=%s latestInitializer=%s\n", + # initializer, latestInitializer)); + if( initializer != .current.initializer) { + cat("DB: Running initializer:", initializer, "\n", file=stderr()) + eval(parse(text=initializer), envir=globalenv()) + .current.initializer <<- initializer + } } - else { - newSession(importlist$session) - env = get(".current.envir", envir=globalenv()) + + failBatch <- function(rcall, try.error, resultBatchFileName) { + # This function is called when an error occurs and + # no valid argument values can be produced + # It fills a result array with the try.error object + # and writes it to resultBatchFileName + result <- list() + cat("DB: Failing with error:", try.error, "\n", file=stderr()) + for (c in 1:length(rcall$arglistbatch)) { + result[[c]] <- try.error + } + save(result,file=resultBatchFileName) } - } - environment(rcall$func) <- env - success <- try(doInit(rcall$initializer, env)) - if (inherits(success, "try-error")) { - failBatch(rcall, success, resultBatchFileName) - return() - } + runBatch <- function( callBatchFileName, resultBatchFileName ) + { + # Load contents into local environment + success <- try(load(callBatchFileName, envir=environment())); + if (inherits(success, "try-error")) { + failBatch(rcall, paste(callBatchFileName, "could not be opened"), + resultBatchFileName) + return() + } + success <- try(loadImports(rcall$imports)) + if (inherits(success, "try-error")) { + failBatch(rcall, success, resultBatchFileName) + return() + } - result <- list() - for(c in 1:length(rcall$arglistbatch)) { - # FIXME: run this under try/catch and save error status - # in results object (need to make it a list: rval + error status) - result[[c]] = try(do.call( rcall$func, - as.list(rcall$arglistbatch[[c]] ))) - } - save(result,file=resultBatchFileName) -} -newSession <- function (session) { - cat("Setting up new session\n", file=stderr()) - #Setup new session - .current.session <<- session - # Use an environment as a hash table to track imports - .current.imported <<- new.env(hash=T, parent=emptyenv()) - # Create an environment to store imported items in - # and to evaluate apply calls in. Note that the environment - # does not point to the global environment as its parent, which means - # means that assignments and variable lookusp will not touch the global - # environment by default, and will be contained in this session. - # However, the <<- assignment operator and the get, assign, etc function - # can still access the global enviroment if used in the applied function. + success <- try(doInit(rcall$initializer)) + if (inherits(success, "try-error")) { + failBatch(rcall, success, resultBatchFileName) + return() + } - .current.envir <<- new.env(parent=parent.env(globalenv())) + cat("DB: Doing apply\n", file=stderr()) + result <- list() + for(c in 1:length(rcall$arglistbatch)) { + # FIXME: run this under try/catch and save error status + # in results object (need to make it a list: rval + error status) + result[[c]] = try(do.call( rcall$func, + as.list(rcall$arglistbatch[[c]] ))) + } + cat("DB: Saving Results\n", file=stderr()) + save(result,file=resultBatchFileName) + cat("DB: Results saved\n", file=stderr()) + } - .current.initializer <<- "" -} + newSession <- function (session) { + cat("DB: Setting up new session\n", file=stderr()) + cat("DB: Deleting:", ls(envir=globalenv()), "\n", file=stderr()) + #Setup new session + .current.session <<- session + # Use an environment as a hash table to track imports + .current.imported <<- new.env(hash=T, parent=emptyenv()) + + # Clear out global environment of anything not starting + # with a . - ie. things put there by previous imports + rm(list=ls(envir=globalenv()), envir=globalenv()) -loadImports <- function (importlist) { - # First check whether the session id has changed - #cat(paste("Import list:", importlist$exports, "\n"), file=stderr()) - #cat(paste("New Session:", importlist$session, "\n"), file=stderr()) - #cat(paste("Old Session:", try(get(".current.session", envir=globalenv())), - # "\n"), file=stderr()) - doSetup <- FALSE - if (!is.null(importlist)) { - if (exists(".current.session", envir=globalenv())) { - if (importlist$session != .current.session) + .current.initializer <<- "" + cat("DB: reset env\n") + } + + loadImports <- function (importlist) { + # First check whether the session id has changed + #cat(paste("Import list:", importlist$exports, "\n"), file=stderr()) + #cat(paste("New Session:", importlist$session, "\n"), file=stderr()) + #cat(paste("Old Session:", try(get(".current.session", envir=globalenv())), + # "\n"), file=stderr()) + doSetup <- FALSE + if (!is.null(importlist)) { + if (exists(".current.session", envir=globalenv())) { + if (importlist$session != .current.session) + doSetup <- TRUE + } + else { doSetup <- TRUE + } } - else { - doSetup <- TRUE + if (doSetup) { + newSession(importlist$session) } - } - if (doSetup) { - newSession(importlist$session) - } - for (file in importlist$exports) { - # Load the contents of the specified file - # into the global environment + for (file in importlist$exports) { + # Load the contents of the specified file + # into the global environment - #cat("File: ", file, "\n") - # check to see if already imported - if (!exists(file, envir=.current.imported)) { - #TODO: load can fail with warning - load(file, envir=.current.envir) - # if an error occurs here, assume calling function - # will catch it - assign(file, TRUE, envir=.current.imported) - cat("Loaded file ", file, "\n") + #cat("File: ", file, "\n") + # check to see if already imported + if (!exists(file, envir=.current.imported)) { + #TODO: load can fail with warning + load(file, envir=globalenv()) + # if an error occurs here, assume calling function + # will catch it + assign(file, TRUE, envir=.current.imported) + cat("Loaded file ", file, "\n") + } + else { + #cat("Ignored file ", file, "\n") + } } - else { - #cat("Ignored file ", file, "\n") - } } + + cat("calling server: dir=",fifoDir,"\n") + setwd(fifoDir) + tryCatch(SwiftRFifoServer(fifoDir), error=doFail) + cat("returned from server: dir=",fifoDir,"\n") } -cat("calling server: dir=",fifoDir,"\n") -setwd(fifoDir) -tryCatch(SwiftRFifoServer(fifoDir), error=doFail) -cat("returned from server: dir=",fifoDir,"\n")) +.main(commandArgs(TRUE)) From noreply at svn.ci.uchicago.edu Thu Feb 10 11:40:15 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 10 Feb 2011 11:40:15 -0600 (CST) Subject: [Swift-commit] r4081 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110210174015.C67F09CC95@svn.ci.uchicago.edu> Author: tga Date: 2011-02-10 11:40:02 -0600 (Thu, 10 Feb 2011) New Revision: 4081 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Fix for pbsf: point to correct configuration script Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-10 14:46:29 UTC (rev 4080) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-10 17:40:02 UTC (rev 4081) @@ -544,9 +544,13 @@ cores=$defaultClusterCores fi echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle + if [ $server = pbsf ] + then + source $SWIFTRBIN/configure-server-pbsf $cores + else + source $SWIFTRBIN/configure-server-${server} $cores + fi - source $SWIFTRBIN/configure-server-${server} $cores - jobidfile=${out/stdouterr/jobid} TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap From noreply at svn.ci.uchicago.edu Thu Feb 10 13:15:45 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 10 Feb 2011 13:15:45 -0600 (CST) Subject: [Swift-commit] r4082 - in SwiftApps/SwiftR: . Swift Swift/R Swift/tests Message-ID: <20110210191545.5989D9CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-10 13:15:41 -0600 (Thu, 10 Feb 2011) New Revision: 4082 Added: SwiftApps/SwiftR/Swift/R/Bench.R Modified: SwiftApps/SwiftR/IMMEDIATE-TODO SwiftApps/SwiftR/Makefile SwiftApps/SwiftR/Swift/NAMESPACE SwiftApps/SwiftR/Swift/R/TestFramework.R SwiftApps/SwiftR/Swift/R/Tests.R SwiftApps/SwiftR/Swift/tests/runtests.R Log: Added initial benchmark which simply does a configurable sleep. Added infrastructure to allow easy construction of parameterized performance tests. Modified: SwiftApps/SwiftR/IMMEDIATE-TODO =================================================================== --- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-10 17:40:02 UTC (rev 4081) +++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-10 19:15:41 UTC (rev 4082) @@ -77,6 +77,10 @@ - user testing in general (ssh, pbs, sge) * SGE testing: ranger, siraf (low priority) +MID: +* Cleanup ssh worker processes: add a watchdog that detects when worker.pl has + gone away + LOW: (unless needed by immediate OpenMx app or test) - complete sf compat functions (sapply, lapply -> for openMx, based on usage) Modified: SwiftApps/SwiftR/Makefile =================================================================== --- SwiftApps/SwiftR/Makefile 2011-02-10 17:40:02 UTC (rev 4081) +++ SwiftApps/SwiftR/Makefile 2011-02-10 19:15:41 UTC (rev 4082) @@ -38,7 +38,7 @@ R CMD check $(TBALL) test: install - ./Swift/tests/runtests.R + Rscript Swift/tests/runtests.R clean: rm -rf Swift/inst/swift/* Modified: SwiftApps/SwiftR/Swift/NAMESPACE =================================================================== --- SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-10 17:40:02 UTC (rev 4081) +++ SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-10 19:15:41 UTC (rev 4082) @@ -11,12 +11,17 @@ export(swiftRemoveAll) export(runAllSwiftTests) -export(basicSwiftTest) -exportPattern("^swiftTest") export(makeTestGroup) +export(makeParamTestGroup) export(runTestGroup) export(runTestSuite) +export(printTestGroup) +export(mkTest) + +export(basicSwiftTest) +exportPattern("^swiftTest") exportPattern("^testGroup") export(swiftTestSuite) -export(printTestGroup) -export(mkTest) + +export(mkPerfTest) +export(doPerfTest) Added: SwiftApps/SwiftR/Swift/R/Bench.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Bench.R (rev 0) +++ SwiftApps/SwiftR/Swift/R/Bench.R 2011-02-10 19:15:41 UTC (rev 4082) @@ -0,0 +1,75 @@ + + +swiftTest_6.1.1 <- function () { + swiftTest_6.1(2, 20) +} + +sleeper <- function(delay,ncalls) { + options(swift.initialexpr="initVar3 <- 123; initVar4 <- 100"); + + timed <- function(delay) { Sys.sleep(delay); delay } + + args=list(delay) + arglist = rep(list(args),ncalls) + + cat("\nTest of swiftapply(delay,arglist)\n") + + startTime = proc.time()[["elapsed"]] + swiftres = swiftapply(timed,arglist) + endTime = proc.time()[["elapsed"]] + runTime <- endTime - startTime + + cat("\n\n ===> Ran for ",runTime," seconds.\n\n") + + cat("Swift result:\n") + print(swiftres[[1]]) + + if(identical(delay,swiftres[[1]])) { + cat("\n==> sleeper passed\n") + return (TRUE) + } else { + cat("\n==> sleeper FAILED !!!!!\n") + return (FALSE) + } + +} + +id <- function () { return (1) } + +mkPerfTest <- function() { + # Have an initial test that will block until + # resources are ready + warmUpGroup <- makeTestGroup( + name="Wait for server to start", + tests=list(mkTest(id))) + + perfTestGroup1 <- makeParamTestGroup( + name="1 - basic performance test", + f=sleeper, + allargs=list( + list(1, 10), + list(1, 20), + list(1, 50), + list(1, 100), + list(2, 10), + list(2, 20), + list(2, 50), + list(2, 100), + list(3, 10), + list(3, 20), + list(3, 50), + list(3, 100), + list(4, 10), + list(4, 20), + list(4, 50), + list(4, 100))) + swiftTestSuite <- list( + setup=function () { initSwiftTestOptions(); + swiftInit() }, # swiftInit controlled via options + tests=list(warmUpGroup, perfTestGroup1), + teardown=function () { swiftShutdown() }) + return (swiftTestSuite) +} +doPerfTest <- function () { + runTestSuite(mkPerfTest()) +} Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R =================================================================== --- SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-10 17:40:02 UTC (rev 4081) +++ SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-10 19:15:41 UTC (rev 4082) @@ -3,23 +3,53 @@ mkTest <- function (f) { test = list() - test$name = substitute(f) + test$name = deparse(substitute(f)) test$fun = f test$passed = NULL + test$time <- NULL return (test) } -makeTestGroup <- function (name, tlist, init=NULL) { +makeTestGroup <- function (name, tests, setup=NULL, teardown=NULL) { group = list() - group$init = init - group$tests = tlist + group$name <- name + group$setup = setup + group$tests = tests + group$teardown = teardown return (group) } +makeParamTestGroup <- function (name, f, allargs, + setup=NULL, teardown=NULL) { + + + buildClosure <- function (f, args) { + force(f) + force(args) + return (function () { do.call(f, args) }) + } + fname = deparse(substitute(f)) + + tests = list() + for (args in allargs) { + test = list() + args <- as.list(args) + test$name = paste(fname, substring(deparse(args), 5) , sep="") + test$fun <- buildClosure(f, args) + test$args <- args # Store for later analysis + test$passed <- NULL + test$time <- NULL + tests[[length(tests) + 1]] <- test + } + + return (makeTestGroup(name, tests, setup, teardown)) +} + + runTestGroup <- function (group) { cat("\n*** Starting test group ", group$name, "***\n") - if (!is.null(group$init)) - group$init() + if (!is.null(group$setup)) + group$setup() for (i in 1:length(group$tests)) { test <- group$tests[[i]] cat("\n== Starting test ", test$name, " ==\n") @@ -30,6 +60,8 @@ group$tests[[i]]$time <- runTime } + if (!is.null(group$teardown)) + group$teardown() return (group) } @@ -49,7 +81,7 @@ if (is.null(time)) "???" else - sprintf("%.2f", time) + sprintf("%.2fs", time) } for (test in group$tests) { cat(paste(test$name, ": ", resFmt(test$passed), @@ -60,7 +92,8 @@ runTestSuite <- function (suite) { suite$setup() resSuite <- lapply(suite$tests, runTestGroup) + suite$teardown() + lapply(resSuite, printTestGroup) - suite$teardown() - return (NULL) + return (resSuite) } Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-10 17:40:02 UTC (rev 4081) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-10 19:15:41 UTC (rev 4082) @@ -590,40 +590,7 @@ } } -swiftTest_6.1.1 <- function () { - swiftTest_6.1(2, 20) -} -swiftTest_6.1 <- function(delay,ncalls) { - options(swift.initialexpr="initVar3 <- 123; initVar4 <- 100"); - - timed <- function(delay) { Sys.sleep(delay); delay } - - args=list(delay) - arglist = rep(list(args),ncalls) - - cat("\nTest of swiftapply(delay,arglist)\n") - - startTime = proc.time()[["elapsed"]] - swiftres = swiftapply(timed,arglist) - endTime = proc.time()[["elapsed"]] - runTime <- endTime - startTime - - cat("\n\n ===> Total elapsed unit test time = ",runTime," seconds.\n\n") - - cat("Swift result:\n") - print(swiftres[[1]]) - - if(identical(delay,swiftres[[1]])) { - cat("\n==> test 6.1 passed\n") - return (TRUE) - } else { - cat("\n==> test 6.1 FAILED !!!!!\n") - return (FALSE) - } - -} - runAllSwiftTests <- function() { startTime = proc.time()[["elapsed"]] @@ -649,50 +616,47 @@ } -testGroup1.1 <- list( init=NULL, +testGroup1.1 <- makeTestGroup( name="1.1 Sanity Check", tests = list(mkTest(swiftTest_1.1))) -testGroup1.2 <- list( init=NULL, +testGroup1.2 <- makeTestGroup( name="1.2 Basic Test - Adding data sets", tests = list(mkTest(swiftTest_1.2.1), mkTest(swiftTest_1.2.2), mkTest(swiftTest_1.2.3), mkTest(swiftTest_1.2.4))) -testGroup1.3 <- list( init=NULL, +testGroup1.3 <- makeTestGroup( name="1.3 - Export functionality", tests = list(mkTest(swiftTest_1.3.1), mkTest(swiftTest_1.3.2), mkTest(swiftTest_1.3.3), mkTest(swiftTest_1.3.4), mkTest(swiftTest_1.3.5)) ) -testGroup2 <- list( init=NULL, +testGroup2 <- makeTestGroup( name="2 - test matrix passing", tests = list(mkTest(swiftTest_2.1), mkTest(swiftTest_2.2)) ) -testGroup3 <- list( init=NULL, +testGroup3 <- makeTestGroup( name="3 - test list element and name passing", tests = list(mkTest(swiftTest_3.1), mkTest(swiftTest_3.2))) -testGroup4 <- list( init=NULL, +testGroup4 <- makeTestGroup( name="4 - test remote R service initialization string", tests = list(mkTest(swiftTest_4.1), mkTest(swiftTest_4.2))) -testGroup5 <- list( init=NULL, +testGroup5 <- makeTestGroup( name="5 - remote R service error ", tests = list(mkTest(swiftTest_5.1))) -testGroup6 <- list( init=NULL, - name="6 - performance tests", - tests = list(mkTest(swiftTest_6.1.1))) swiftTestSuite <- list( setup=function () { initSwiftTestOptions(); swiftInit() }, # swiftInit controlled via options tests=list(testGroup1.1, testGroup2, testGroup1.2, testGroup1.3, testGroup3, testGroup4, - testGroup5, testGroup6), + testGroup5), teardown=function () { swiftShutdown() }) Modified: SwiftApps/SwiftR/Swift/tests/runtests.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/runtests.R 2011-02-10 17:40:02 UTC (rev 4081) +++ SwiftApps/SwiftR/Swift/tests/runtests.R 2011-02-10 19:15:41 UTC (rev 4082) @@ -1,4 +1,4 @@ -#!/bin/env Rscript +#!/usr/bin/env Rscript library(Swift) #TODO: take command line options to setup options From noreply at svn.ci.uchicago.edu Thu Feb 10 15:10:53 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 10 Feb 2011 15:10:53 -0600 (CST) Subject: [Swift-commit] r4083 - in SwiftApps/SwiftR/Swift: . R Message-ID: <20110210211053.345979CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-10 15:10:50 -0600 (Thu, 10 Feb 2011) New Revision: 4083 Modified: SwiftApps/SwiftR/Swift/NAMESPACE SwiftApps/SwiftR/Swift/R/TestFramework.R Log: Added function to collate results from performance tests into vectors. Modified: SwiftApps/SwiftR/Swift/NAMESPACE =================================================================== --- SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-10 19:15:41 UTC (rev 4082) +++ SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-10 21:10:50 UTC (rev 4083) @@ -25,3 +25,4 @@ export(mkPerfTest) export(doPerfTest) +export(analysePerf) Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R =================================================================== --- SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-10 19:15:41 UTC (rev 4082) +++ SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-10 21:10:50 UTC (rev 4083) @@ -45,7 +45,30 @@ return (makeTestGroup(name, tests, setup, teardown)) } +analysePerf <- function (testGroupResults, nargs) { + # Build a list of vectors of performance results for easier analysis + # [[1]] -> first arg + # [[2]] -> second arg + # ... ... ....... + # [[n]] -> nth arg + # $name + # $passed + # $time + collated <- list() + for (i in 1:nargs) { + collated[[i]] <- unlist(lapply(testGroupResults$tests, + function(test) {test$args[[i]]})) + } + collated$name <- unlist(lapply(testGroupResults$tests, + function(test) {test$name })) + collated$passed <-unlist(lapply(testGroupResults$tests, + function(test) {test$passed })) + collated$time <- unlist(lapply(testGroupResults$tests, + function(test) {test$time })) + return (collated) +} + runTestGroup <- function (group) { cat("\n*** Starting test group ", group$name, "***\n") if (!is.null(group$setup)) From noreply at svn.ci.uchicago.edu Thu Feb 10 17:25:54 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 10 Feb 2011 17:25:54 -0600 (CST) Subject: [Swift-commit] r4084 - in SwiftApps/SwiftR: Swift Swift/R mxtests/swift Message-ID: <20110210232554.3D17E9CC95@svn.ci.uchicago.edu> Author: tga Date: 2011-02-10 17:25:49 -0600 (Thu, 10 Feb 2011) New Revision: 4084 Added: SwiftApps/SwiftR/mxtests/swift/MakeTests.R Modified: SwiftApps/SwiftR/Swift/NAMESPACE SwiftApps/SwiftR/Swift/R/TestFramework.R SwiftApps/SwiftR/Swift/R/Tests.R SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R Log: Added infrastructure for comparative tests of swift and snowfall. Working towards unifying the performance testing so that the same test can be run on swift, snowfall, and serially. now running basic performance tests for the bootstrap example. Modified: SwiftApps/SwiftR/Swift/NAMESPACE =================================================================== --- SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-10 21:10:50 UTC (rev 4083) +++ SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-10 23:25:49 UTC (rev 4084) @@ -15,6 +15,7 @@ export(makeParamTestGroup) export(runTestGroup) export(runTestSuite) +export(makeTestSuite) export(printTestGroup) export(mkTest) @@ -26,3 +27,4 @@ export(mkPerfTest) export(doPerfTest) export(analysePerf) + Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R =================================================================== --- SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-10 21:10:50 UTC (rev 4083) +++ SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-10 23:25:49 UTC (rev 4084) @@ -112,10 +112,16 @@ } } +makeTestSuite <- function(tests, setup=NULL, teardown=NULL) { + return (list(tests=tests, setup=setup, teardown=teardown)) +} + runTestSuite <- function (suite) { - suite$setup() + if (!is.null(suite$setup)) + suite$setup() resSuite <- lapply(suite$tests, runTestGroup) - suite$teardown() + if (!is.null(suite$teardown)) + suite$teardown() lapply(resSuite, printTestGroup) return (resSuite) Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-10 21:10:50 UTC (rev 4083) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-10 23:25:49 UTC (rev 4084) @@ -653,7 +653,7 @@ -swiftTestSuite <- list( +swiftTestSuite <- makeTestSuite( setup=function () { initSwiftTestOptions(); swiftInit() }, # swiftInit controlled via options tests=list(testGroup1.1, testGroup2, testGroup1.2, testGroup1.3, testGroup3, testGroup4, Modified: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-10 21:10:50 UTC (rev 4083) +++ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-10 23:25:49 UTC (rev 4084) @@ -1,4 +1,4 @@ -# +#!/bin/env Rscript # Copyright 2007-2010 The OpenMx Project # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,102 +15,153 @@ require(OpenMx) require(Swift) +source("mxtests/swift/MakeTests.R") -hostfile <- Sys.getenv(c("PBS_NODEFILE")) +#hostfile <- Sys.getenv(c("PBS_NODEFILE"))[[1]] -hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] +#hostnames <- read.table(hostfile, stringsAsFactors=FALSE)[[1]] +#cat("Running on hosts:", hostnames, "\n") -swiftSess <- swiftInit(server="ssh", cores=8, hosts=hostnames) -swiftLibrary("OpenMx") +main <- function () { + swiftLibrary(OpenMx) -set.seed(10) + swiftSess <- swiftSetup(server="local", cores=4) -# parameters for the simulation: lambda = factor loadings, -# specifics = specific variances -nVar <- 75 -nObs <- 10000 -nReps <- 1024 -goodStartValues <- TRUE -if (!is.logical(goodStartValues)) { - stop("'goodStartValues' should be logical. Try again.") + nVar <- 75 + nObs <- 10000 + nReps <- 1024 + + perfExperiment(nVar, nObs, nReps) + swiftShutdown(swiftSess) } -lambda <- matrix(1:nVar*2/nVar, nVar, 1) -specifics <- diag(nVar) -chl <- chol(lambda %*% t(lambda) + specifics) -# indices for parameters and hessian estimate in results -pStrt <- 3 -pEnd <- pStrt + 2*nVar - 1 -hStrt <- pEnd + 1 -hEnd <- hStrt + 2*nVar - 1 +bootstrapTest <- function (nVar, nObs, nReps) { + set.seed(10) + topModel <- buildModels(nVar, nObs, nReps) -# dimension names for OpenMx -dn <- list() -dn[[1]] <- paste("Var", 1:nVar, sep="") -dn[[2]] <- dn[[1]] + modelResults <- mxRun(topModel, silent=TRUE, suppressWarnings=TRUE) -# function to get a covariance matrix -randomCov <- function(nObs, nVar, chl, dn) { - x <- matrix(rnorm(nObs*nVar), nObs, nVar) - x <- x %*% chl - thisCov <- cov(x) - dimnames(thisCov) <- dn - return(thisCov) + print(modelResults at output$wallTime) + return (modelResults) } -createNewModel <- function(index, prefix, model) { - modelname <- paste(prefix, index, sep='') - model at data@observed <- randomCov(nObs, nVar, chl, dn) - model at name <- modelname - return(model) -} -getStats <- function(model) { - retval <- c(model at output$status[[1]], - max(abs(model at output$gradient)), - model at output$estimate, - sqrt(diag(solve(model at output$hessian)))) - return(retval) + + + #nVar, nObs, nReps +#testArgs <- list( +# list(75, 10000, 4), +# list(75, 10000, 8), +# list(75, 10000, 16), +# list(75, 10000, 32)) + +testArgs <- list( + list(75, 1000, 4), + list(75, 1000, 8), + list(75, 1000, 16), + list(75, 1000, 32)) + +makeBootstrapTestGroup <- function (...) { + makePerfTestGroup( + name="BootstrapParallelBigger", + f=bootstrapTest, + allargs=testArgs, + ... ) } -# initialize obsCov for MxModel -obsCov <- randomCov(nObs, nVar, chl, dn) -# results matrix: get results for each simulation -results <- matrix(0, nReps, hEnd) -dnr <- c("inform", "maxAbsG", paste("lambda", 1:nVar, sep=""), - paste("specifics", 1:nVar, sep=""), - paste("hessLambda", 1:nVar, sep=""), - paste("hessSpecifics", 1:nVar, sep="")) -dimnames(results)[[2]] <- dnr -# instantiate MxModel -template <- mxModel(name="stErrSim", - mxMatrix(name="lambda", type="Full", nrow=nVar, ncol=1, - free=TRUE, values=(1:nVar*2/nVar)*goodStartValues), - mxMatrix(name="specifics", type="Diag", nrow=nVar, - free=TRUE, values=rep(1, nVar)), - mxAlgebra(lambda %*% t(lambda) + specifics, - name="preCov", dimnames=dn), - mxData(observed=obsCov, type="cov", numObs=nObs), - mxMLObjective(covariance='preCov'), - independent = TRUE) +buildModels <- function (nVar, nObs, nReps) { + # parameters for the simulation: lambda = factor loadings, + # specifics = specific variances + goodStartValues <- TRUE + if (!is.logical(goodStartValues)) { + stop("'goodStartValues' should be logical. Try again.") + } -topModel <- mxModel(name = 'container') + lambda <- matrix(1:nVar*2/nVar, nVar, 1) + specifics <- diag(nVar) + chl <- chol(lambda %*% t(lambda) + specifics) -swiftExportAll() + # indices for parameters and hessian estimate in results + pStrt <- 3 + pEnd <- pStrt + 2*nVar - 1 + hStrt <- pEnd + 1 + hEnd <- hStrt + 2*nVar - 1 -submodels <- swiftLapply(1:nReps, createNewModel, 'stErrSim', template) + # dimension names for OpenMx + dn <- list() + dn[[1]] <- paste("Var", 1:nVar, sep="") + dn[[2]] <- dn[[1]] -names(submodels) <- imxExtractNames(submodels) -topModel at submodels <- submodels + # function to get a covariance matrix + randomCov <- function(nObs, nVar, chl, dn) { + x <- matrix(rnorm(nObs*nVar), nObs, nVar) + x <- x %*% chl + thisCov <- cov(x) + dimnames(thisCov) <- dn + return(thisCov) + } -modelResults <- mxRun(topModel, silent=TRUE, suppressWarnings=TRUE) + createNewModel <- function(index, prefix, model) { + modelname <- paste(prefix, index, sep='') + model at data@observed <- randomCov(nObs, nVar, chl, dn) + model at name <- modelname + return(model) + } -print(ncpus) -print(modelResults at output$wallTime) + getStats <- function(model) { + retval <- c(model at output$status[[1]], + max(abs(model at output$gradient)), + model at output$estimate, + sqrt(diag(solve(model at output$hessian)))) + return(retval) + } -swiftShutdown(swiftSess) + + # initialize obsCov for MxModel + obsCov <- randomCov(nObs, nVar, chl, dn) + + # results matrix: get results for each simulation +# results <- matrix(0, nReps, hEnd) +# dnr <- c("inform", "maxAbsG", paste("lambda", 1:nVar, sep=""), +# paste("specifics", 1:nVar, sep=""), +# paste("hessLambda", 1:nVar, sep=""), +# paste("hessSpecifics", 1:nVar, sep="")) +# dimnames(results)[[2]] <- dnr + + # instantiate MxModel + template <- mxModel(name="stErrSim", + mxMatrix(name="lambda", type="Full", nrow=nVar, ncol=1, + free=TRUE, values=(1:nVar*2/nVar)*goodStartValues), + mxMatrix(name="specifics", type="Diag", nrow=nVar, + free=TRUE, values=rep(1, nVar)), + mxAlgebra(lambda %*% t(lambda) + specifics, + name="preCov", dimnames=dn), + mxData(observed=obsCov, type="cov", numObs=nObs), + mxMLObjective(covariance='preCov'), + independent = TRUE) + + topModel <- mxModel(name = 'container') + + # Don't need to export as local environment will automatically + # be sent + #swiftExportAll() + submodels <- omxLapply(1:nReps, createNewModel, 'stErrSim', template) + cat(length(submodels), "submodels created\n") + names(submodels) <- imxExtractNames(submodels) + topModel at submodels <- submodels + + return (topModel) +} + +swiftTestSuite <- makeTestSuite( + tests=list( + makeBootstrapTestGroup(mode="swift", server="local", cores=2), + makeBootstrapTestGroup(mode="swift", server="local", cores=4), + makeBootstrapTestGroup(mode="swift", server="ssh", cores=8, hosts="nettle antares deneb"), + makeBootstrapTestGroup(mode="serial") + )) Added: SwiftApps/SwiftR/mxtests/swift/MakeTests.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/MakeTests.R (rev 0) +++ SwiftApps/SwiftR/mxtests/swift/MakeTests.R 2011-02-10 23:25:49 UTC (rev 4084) @@ -0,0 +1,64 @@ + + + +swiftSetup <- function (...) { + library(Swift) + swiftSess <- swiftInit(...) + # Run some small jobs to esnure workers are started and warmed up + swiftLapply(rep(1, swiftSess$cores * swiftSess$nodes * 2), + function (x) { x }) + return (swiftSess) +} + +swiftTearDown <- function (...) { + swiftShutdown() + detach(package:Swift) +} + +sfSetup <- function(..., cpus=ncpus) { + library(snowfall) + sfInit(..., cpus=ncpus) + # warmup + sfLapply(rep(1, ncpus * 2), + function (x) { x }) + +} + +sfTearDown <- function (...) { + sfStop() + detach(package:snowfall) +} + + + +makePerfTestGroup <- function (mode, name, f, allargs, ...) { + params <- list(...) + paramstr <- paste(names(params), + rep("=", length(params)), + lapply(params, deparse), + sep="", collapse=", ") + if (mode == "swift") { + tg <- makeParamTestGroup(name=paste("swift_",name, " ", paramstr,sep=""), + f=f, + allargs=allargs, + setup=function() {swiftSetup(...)}, + tearDown <- swiftTearDown) + } + else if (mode == "snowfall") { + tg <- makeParamTestGroup(name=paste("sf_",name, paramstr, sep=""), + f=f, + allargs=allargs, + setup=function() {sfSetup(...)}, + tearDown <- sfTearDown) + + } + else { + print("Making serial test") + tg <- makeParamTestGroup(name=paste("serial_",name, paramstr, sep=""), + f=f, + allargs=allargs, + setup=function() + {try(detach(package:Swift)); try(detach(package:Snowfall))} ) + } + return (tg) +} From noreply at svn.ci.uchicago.edu Fri Feb 11 10:36:09 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 11 Feb 2011 10:36:09 -0600 (CST) Subject: [Swift-commit] r4085 - SwiftApps/SwiftR/Swift/R Message-ID: <20110211163609.08B6F9CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-11 10:36:08 -0600 (Fri, 11 Feb 2011) New Revision: 4085 Modified: SwiftApps/SwiftR/Swift/R/Workers.R Log: Fixed bug with counting number of nodes for ssh. Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-10 23:25:49 UTC (rev 4084) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-11 16:36:08 UTC (rev 4085) @@ -100,8 +100,10 @@ if(is.null(nodes)) nodes <- getOption("swift.nodes") if(is.null(nodes)) - if (server == "ssh") - nodes <- length(strsplit(hosts, " ")) + if (server == "ssh") { + nodes <- length(strsplit(hosts, " ", fixed=T)[[1]]) + cat("Node code for hosts ", hosts, " is ", nodes) + } else nodes <- 1 # Default value if(! is.null(nodes) ) { From noreply at svn.ci.uchicago.edu Fri Feb 11 15:07:52 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 11 Feb 2011 15:07:52 -0600 (CST) Subject: [Swift-commit] r4086 - in SwiftApps/SwiftR: Swift/R mxtests/swift Message-ID: <20110211210752.957239CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-11 15:07:52 -0600 (Fri, 11 Feb 2011) New Revision: 4086 Modified: SwiftApps/SwiftR/Swift/R/Bench.R SwiftApps/SwiftR/Swift/R/TestFramework.R SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R SwiftApps/SwiftR/mxtests/swift/MakeTests.R Log: Continued improvements to performance testing Modified: SwiftApps/SwiftR/Swift/R/Bench.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Bench.R 2011-02-11 16:36:08 UTC (rev 4085) +++ SwiftApps/SwiftR/Swift/R/Bench.R 2011-02-11 21:07:52 UTC (rev 4086) @@ -34,35 +34,57 @@ } -id <- function () { return (1) } +id <- function () { + swiftLapply(rep(list(1), 100), + function (x) {x}) +} mkPerfTest <- function() { # Have an initial test that will block until # resources are ready warmUpGroup <- makeTestGroup( name="Wait for server to start", - tests=list(mkTest(id))) + tests=rep(list(mkTest(id)), 5)) perfTestGroup1 <- makeParamTestGroup( name="1 - basic performance test", f=sleeper, allargs=list( - list(1, 10), - list(1, 20), - list(1, 50), - list(1, 100), - list(2, 10), - list(2, 20), - list(2, 50), - list(2, 100), - list(3, 10), - list(3, 20), - list(3, 50), - list(3, 100), - list(4, 10), - list(4, 20), - list(4, 50), - list(4, 100))) + list(1, 8), + list(1, 16), + list(1, 32), + list(1, 48), + list(1, 64), + list(1, 96), + list(1, 128), + list(2, 8), + list(2, 16), + list(2, 32), + list(2, 48), + list(2, 64), + list(2, 96), + list(2, 128), + list(3, 8), + list(3, 16), + list(3, 32), + list(3, 48), + list(3, 64), + list(3, 96), + list(3, 128), + list(4, 8), + list(4, 16), + list(4, 32), + list(4, 48), + list(4, 64), + list(4, 96), + list(4, 128), + list(5, 8), + list(5, 16), + list(5, 32), + list(5, 48), + list(5, 64), + list(5, 96), + list(5, 128))) swiftTestSuite <- list( setup=function () { initSwiftTestOptions(); swiftInit() }, # swiftInit controlled via options @@ -70,6 +92,7 @@ teardown=function () { swiftShutdown() }) return (swiftTestSuite) } + doPerfTest <- function () { runTestSuite(mkPerfTest()) } Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R =================================================================== --- SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-11 16:36:08 UTC (rev 4085) +++ SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-11 21:07:52 UTC (rev 4086) @@ -45,7 +45,7 @@ return (makeTestGroup(name, tests, setup, teardown)) } -analysePerf <- function (testGroupResults, nargs) { +analysePerf <- function (testGroupResults, nargs=0) { # Build a list of vectors of performance results for easier analysis # [[1]] -> first arg # [[2]] -> second arg @@ -55,17 +55,32 @@ # $passed # $time collated <- list() - for (i in 1:nargs) { - collated[[i]] <- unlist(lapply(testGroupResults$tests, - function(test) {test$args[[i]]})) + if (nargs >= 1) { + for (i in 1:nargs) { + collated[[i]] <- unlist(lapply(testGroupResults$tests, + function(test) {test$args[[i]]})) + } } + n <- length(testGroupResults$tests) + collated$name <- unlist(lapply(testGroupResults$tests, function(test) {test$name })) collated$passed <-unlist(lapply(testGroupResults$tests, function(test) {test$passed })) collated$time <- unlist(lapply(testGroupResults$tests, function(test) {test$time })) - return (collated) + + # Store the pnames as vectors + if (!is.null(testGroupResults$perfparams)) { + pnames <- names(testGroupResults$perfparams) + if (!is.na(pnames) && length(pnames) > 0) { + for (i in 1:length(pnames)) { + collated[[pnames[[i]]]] <- rep(testGroupResults$perfparams[[i]], + n) + } + } + } + return (data.frame(collated)) } @@ -91,13 +106,16 @@ printTestGroup <- function (group) { cat("\n*** Test group ", group$name, "***\n") resFmt <- function(res) { - if (is.null(res)) { - "???" - } else { - if (inherits(res, "try-error")) - {paste("ERROR: '", res, "'") } - else - {if (res) "PASSED" else "FAILED"} + if (is.null(res)) { + "???" + } else if (inherits(res, "try-error")) { + paste("ERROR: '", res, "'") + } else if (identical(res, TRUE)) { + "PASSED" + } else if (identical(res, FALSE)) { + "FAILED" + } else { + "" } } timeFmt <- function (time) { @@ -119,10 +137,12 @@ runTestSuite <- function (suite) { if (!is.null(suite$setup)) suite$setup() - resSuite <- lapply(suite$tests, runTestGroup) + resSuite <- suite + resSuite$tests <- lapply(suite$tests, runTestGroup) + if (!is.null(suite$teardown)) suite$teardown() - lapply(resSuite, printTestGroup) + lapply(resSuite$tests, printTestGroup) return (resSuite) } Modified: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-11 16:36:08 UTC (rev 4085) +++ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-11 21:07:52 UTC (rev 4086) @@ -23,20 +23,8 @@ #cat("Running on hosts:", hostnames, "\n") -main <- function () { - swiftLibrary(OpenMx) - swiftSess <- swiftSetup(server="local", cores=4) - nVar <- 75 - nObs <- 10000 - nReps <- 1024 - - perfExperiment(nVar, nObs, nReps) - swiftShutdown(swiftSess) -} - - bootstrapTest <- function (nVar, nObs, nReps) { set.seed(10) topModel <- buildModels(nVar, nObs, nReps) @@ -58,16 +46,26 @@ # list(75, 10000, 32)) testArgs <- list( - list(75, 1000, 4), - list(75, 1000, 8), - list(75, 1000, 16), - list(75, 1000, 32)) + list(75, 100, 4), + list(75, 100, 8), + list(75, 100, 16), + list(75, 100, 32), + list(75, 100, 64), + list(75, 100, 128), + list(75, 100, 256), + list(75, 100, 256 + 128), + list(75, 100, 512), + list(75, 100, 786), + list(75, 100, 1024), + list(75, 100, 1024 + 512), + list(75, 100, 2048 ) + ) -makeBootstrapTestGroup <- function (...) { +makeBootstrapTestGroup <- function (first=1, last=length(testArgs), ...) { makePerfTestGroup( name="BootstrapParallelBigger", f=bootstrapTest, - allargs=testArgs, + allargs=testArgs[first:last], ... ) } @@ -160,8 +158,32 @@ swiftTestSuite <- makeTestSuite( tests=list( - makeBootstrapTestGroup(mode="swift", server="local", cores=2), - makeBootstrapTestGroup(mode="swift", server="local", cores=4), - makeBootstrapTestGroup(mode="swift", server="ssh", cores=8, hosts="nettle antares deneb"), - makeBootstrapTestGroup(mode="serial") + # 16 cores + makeBootstrapTestGroup(mode="swift", server="ssh", cores=2, + hosts=c("nettle", "wapato", "dandelion", "cattail", + "chicory", "echinacea", "amaranth", "black-cohosh"), + first=3, last=5), + # 8 cores + makeBootstrapTestGroup(mode="swift", server="ssh", cores=2, + hosts="nettle wapato dandelion cattail", + first=2, last=5), + makeBootstrapTestGroup(mode="swift", server="local", cores=4, + last=4), + makeBootstrapTestGroup(mode="swift", server="local", cores=2, + last=4), + makeBootstrapTestGroup(mode="swift", server="local", cores=1, + last=2), + makeBootstrapTestGroup(mode="swift", server="local", cores=1, + last=2), + makeBootstrapTestGroup(mode="serial", last=2) )) + +padsTestSuite <- makeTestSuite( + tests=list( + makeBootstrapTestGroup(mode="swift", server="pbs", cores=1, nodes=1), + makeBootstrapTestGroup(mode="swift", server="pbs", cores=8, nodes=1), + makeBootstrapTestGroup(mode="swift", server="pbs", cores=8, nodes=2), + makeBootstrapTestGroup(mode="swift", server="pbs", cores=8, nodes=3), + makeBootstrapTestGroup(mode="swift", server="pbs", cores=8, nodes=4), + makeBootstrapTestGroup(mode="swift", server="pbs", cores=8, nodes=5) + )) Modified: SwiftApps/SwiftR/mxtests/swift/MakeTests.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/MakeTests.R 2011-02-11 16:36:08 UTC (rev 4085) +++ SwiftApps/SwiftR/mxtests/swift/MakeTests.R 2011-02-11 21:07:52 UTC (rev 4086) @@ -4,9 +4,12 @@ swiftSetup <- function (...) { library(Swift) swiftSess <- swiftInit(...) + # Wait to start + Sys.sleep(10) # Run some small jobs to esnure workers are started and warmed up - swiftLapply(rep(1, swiftSess$cores * swiftSess$nodes * 2), - function (x) { x }) + swiftapply(function (x) { x }, + rep(1, swiftSess$cores * swiftSess$nodes * 2), + callsperbatch=1) return (swiftSess) } @@ -41,6 +44,7 @@ tg <- makeParamTestGroup(name=paste("swift_",name, " ", paramstr,sep=""), f=f, allargs=allargs, + perfparams <- params, setup=function() {swiftSetup(...)}, tearDown <- swiftTearDown) } @@ -48,6 +52,7 @@ tg <- makeParamTestGroup(name=paste("sf_",name, paramstr, sep=""), f=f, allargs=allargs, + perfparams <- params, setup=function() {sfSetup(...)}, tearDown <- sfTearDown) @@ -57,6 +62,7 @@ tg <- makeParamTestGroup(name=paste("serial_",name, paramstr, sep=""), f=f, allargs=allargs, + perfparams <- list(), setup=function() {try(detach(package:Swift)); try(detach(package:Snowfall))} ) } From noreply at svn.ci.uchicago.edu Fri Feb 11 15:08:10 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 11 Feb 2011 15:08:10 -0600 (CST) Subject: [Swift-commit] r4087 - SwiftApps/SwiftR/Swift/R Message-ID: <20110211210810.AD5619CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-11 15:08:10 -0600 (Fri, 11 Feb 2011) New Revision: 4087 Modified: SwiftApps/SwiftR/Swift/R/Workers.R Log: removed unnecessary print statement Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-11 21:07:52 UTC (rev 4086) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-11 21:08:10 UTC (rev 4087) @@ -102,7 +102,7 @@ if(is.null(nodes)) if (server == "ssh") { nodes <- length(strsplit(hosts, " ", fixed=T)[[1]]) - cat("Node code for hosts ", hosts, " is ", nodes) +# cat("Node count for hosts '", hosts, "' is ", nodes) } else nodes <- 1 # Default value From noreply at svn.ci.uchicago.edu Mon Feb 14 09:52:06 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 14 Feb 2011 09:52:06 -0600 (CST) Subject: [Swift-commit] r4088 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110214155206.0EA0B9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-14 09:52:05 -0600 (Mon, 14 Feb 2011) New Revision: 4088 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: bugfix: start-swift script now executes cleanup code if swift itself exits. Previously cleanup code was only executed if start-swift exited by receiving a signal. Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-11 21:08:10 UTC (rev 4087) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-14 15:52:05 UTC (rev 4088) @@ -468,6 +468,8 @@ out=swift.stdouterr touch $out +# Function to run on termination of swift +exitcmd="" if [ $server = local ]; then @@ -497,10 +499,10 @@ TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap trap onexit $TRAPEVENTS + exitcmd=onexit - elif [ $server = ssh ]; then - + if [ $cores -eq 0 ]; then cores=$defaultSshCores fi @@ -534,6 +536,7 @@ } trap onexit $TRAPEVENTS + exitcmd=onexit wait-and-start-ssh-workers & starterpid=$! @@ -574,6 +577,7 @@ } trap onexit $TRAPEVENTS + exitcmd=onexit wait-and-start-batch-workers & starterpid=$! @@ -581,3 +585,5 @@ fi $SWIFTRBIN/../swift/bin/swift -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& $out Author: hategan Date: 2011-02-14 12:42:48 -0600 (Mon, 14 Feb 2011) New Revision: 4089 Modified: trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java Log: made the array file mapper static; while not necessarily true that the files argument is closed, the code assumes so. Under that assumption, the mapper is then static Modified: trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2011-02-14 15:52:05 UTC (rev 4088) +++ trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2011-02-14 18:42:48 UTC (rev 4089) @@ -60,6 +60,6 @@ } public boolean isStatic() { - return false; + return true; } } From noreply at svn.ci.uchicago.edu Mon Feb 14 15:58:55 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 14 Feb 2011 15:58:55 -0600 (CST) Subject: [Swift-commit] r4090 - in SwiftApps/SwiftR/Swift: R exec man Message-ID: <20110214215855.7B7809CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-14 15:58:55 -0600 (Mon, 14 Feb 2011) New Revision: 4090 Added: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt Modified: SwiftApps/SwiftR/Swift/R/Workers.R SwiftApps/SwiftR/Swift/exec/configure-server-pbs SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/Swift/man/swiftInit.Rd Log: Adding in a backend for cobalt. --THIS line, and those below, will be ignored-- M Swift/R/Workers.R A Swift/exec/configure-server-cobalt M Swift/exec/start-swift M Swift/exec/configure-server-pbs M Swift/man/swiftInit.Rd Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-14 18:42:48 UTC (rev 4089) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-14 21:58:55 UTC (rev 4090) @@ -2,7 +2,7 @@ swiftInit <- function( cores=NULL, server=NULL, hosts=NULL, nodes=NULL, project=NULL, - parEnv=NULL, workmode=NULL, + parEnv=NULL, kernel=NULL, workmode=NULL, throttle=NULL, queue=NULL, rcmd=NULL, time=NULL, workerLogging=NULL,keepworkdir=NULL) @@ -87,6 +87,13 @@ if(! is.null(parEnv) ) { cmdString <- paste(cmdString, "-e", shQuote(parEnv)) } + + if(is.null(kernel)) + kernel <- getOption("swift.kernel") + + if(! is.null(kernel) ) { + cmdString <- paste(cmdString, "-kernel", shQuote(kernel)) + } if(is.null(workmode)) workmode <- getOption("swift.workmode") Added: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt (rev 0) +++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-02-14 21:58:55 UTC (rev 4090) @@ -0,0 +1,50 @@ +#! /bin/bash + +# configuration for cobalt with manually-started Swift workers (passive coasters) + +cores=$1 + +throttle=5.0 # allow approximately 500 concurrent jobs + +cat >tc <sites.xml < + + + + 0.15 + 10000 + + $(pwd)/swiftwork + + + + + passive + $cores + $throttle + 10000 + + $HOME/swiftwork + + + +END + +cat >cf <cf < $sshpidfile } + # FIXME: does PBS need same workers-per-node logic as SGE? - make-pbs-submit-file() { if [ $queue != NONE ]; then @@ -254,15 +254,14 @@ IDLETIMEOUT=$((60*60*240)) # 10 days: FIXME: make this a command line arg # FIXME: set up for capturing batch job id: rm -rf remotepid.* # FIXME: should not be needed if we start in a new dir each time - make-${server}-submit-file + if [ "$server" != "cobalt" ]; then + make-${server}-submit-file + fi #FIXME: doesn't work for SGE on IBI cluster as there is additional text # returned by qsub - if [ "${server}" != "sge" ] + if [ "${server}" = "sge" ] then - qsub batch.sub >$jobidfile - succ=$? - else # Sun grid engine inconviently returns a bunch of text surrounding # the job id. There is no documented way to obtain the job number # directly from qsub. We will parse out the first number in this text @@ -270,6 +269,38 @@ # this was tested on). qsub batch.sub | sed 's/[^0-9 ]//g' | awk '{ print $1 }' > $jobidfile succ=$? + elif [ "${server}" = "cobalt" ] + then + # cobalt arguments are specified through command ine rather than + # through a submit file + #cobalt qsub statement with relevant parameters + # queue name + if [ "$queue" != NONE ]; then + queueDirective="-q $queue" + else + queueDirective="" + fi + if [ "$kernel" != NONE ]; then + kernDirective="--kernel $kernel" + else + kernDirective="" + fi + if [ "$project" != NONE ]; then + projDirective="-A $project" + else + projDirective="" + fi + # for now, rely on swift worker to fork off worker processes + # so we have multiple workers per node. In future could + # add support for virtual node mode, etc + qsub -t "$time" -n $nodes $queueDirective $kernDirective \ + $projDirective -O SwiftR-workers \ + --env WORKER_LOGGING_LEVEL=$workerLogging \ + /usr/bin/perl $SWIFTBIN/worker.pl $CONTACT \ + $HOME/.globus/coasters $IDLETIMEOUT + else + qsub batch.sub >$jobidfile + succ=$? fi if [ $? -eq 0 ] @@ -336,13 +367,13 @@ -c cores 2,4,5 >= 1 (default is: local 2; ssh 4; cluster 8) -e parEnv site specific, SGE only -h hosts 1 list of hosts, quoted as one argument, space separated - -m workmode node node: start one worker for all slots on a node; slot (one worker on each slot) + -m workmode node node: start one worker for all slots on a node; slot (one worker on each slot) (Currently ignored) -n nodes 1 -p throttle 10 >= 1 - -q queue site speific (PBS, SGE) + -q queue site speific (PBS, SGE, Cobalt) -r rcmd ssh site specific, SGE only, typically ssh. qrsh for siraf cluster - -s server local local, pbs, sge, ssh, pbsf (for firewalled worker nodes) - -t time 00:30:00 hh:mm:ss, for PBS and SGE only + -s server local local, pbs, sge, ssh, pbsf,cobalt (for firewalled worker nodes) + -t time 00:30:00 hh:mm:ss, for PBS, Cobalt and SGE only -w wkloglvl NONE NONE, ERROR, WARN, INFO, DEBUG, TRACE -k keepdir No argument, if flag is set, will keep working directory @@ -378,6 +409,7 @@ queue=NONE project=NONE parEnv=NONE +kernel=NONE workdir=NONE workerLogging=ERROR keepdir=FALSE @@ -391,13 +423,14 @@ -A) project=$2; verify-not-null project $project; shift ;; -c) cores=$2; verify-is-numeric cores $cores; shift ;; -e) parEnv=$2; verify-not-null parEnv $parEnv; shift ;; + -kernel) kernel=$2; verify-not-null kernel $kernel; shift ;; -h) hosts=$2; verify-not-null hosts $hosts; shift ;; -m) workmode=$2; verify-is-one-of workmode $workmode slot node; shift ;; -n) nodes=$2; verify-is-numeric nodes $nodes; shift ;; -p) throttle=$2; verify-is-numeric throttle $throttle; shift ;; -q) queue=$2; verify-not-null queue $queue; shift ;; -r) rcmd=$2; verify-is-one-of rcmd $rcmd ssh qrsh; shift ;; - -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge; shift ;; + -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge cobalt; shift ;; -t) time=$2; verify-not-null time $time; shift ;; -w) workerLogging=$2; verify-is-one-of workerLoggingLevel $workerLogging NONE ERROR WARN INFO DEBUG TRACE; shift ;; -k) keepdir=TRUE ;; @@ -541,7 +574,8 @@ wait-and-start-ssh-workers & starterpid=$! -elif [ \( $server = pbs \) -o \( $server = pbsf \) -o \( $server = sge \) ]; then +elif [ \( $server = pbs \) -o \( $server = pbsf \) -o \( $server = sge \) \ + -o \( $server = cobalt \) ]; then if [ $cores -eq 0 ]; then cores=$defaultClusterCores Modified: SwiftApps/SwiftR/Swift/man/swiftInit.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/swiftInit.Rd 2011-02-14 18:42:48 UTC (rev 4089) +++ SwiftApps/SwiftR/Swift/man/swiftInit.Rd 2011-02-14 21:58:55 UTC (rev 4090) @@ -26,7 +26,7 @@ The number of cores per host. The default values vary from 2 to 8 depending on the server type. } \item{server}{ - One of: "local", "ssh", "pbs", "sge", "pbsf". + One of: "local", "ssh", "pbs", "sge", "pbsf", "cobalt". How Swift will run the jobs: for example, if "local" is chosen, they will be run on the local machine, or if "pbs" is chosen, they will be run through the pbs scheduler. @@ -40,10 +40,11 @@ } \item{nodes}{ The number of nodes to request from the batch scheduler. This only - is used if server is "pbs", "sge" or "pbsf". + is used if server is "pbs", "sge", "pbsf" and "cobalt". } \item{project}{ - The project name passed to the PBS or SGE batch scheduler. Site-specific. + The project name passed to the PBS, Cobalt or SGE batch scheduler. + Site-specific. } \item{parEnv}{ SGE only. This is the parallel environment setting passed to the @@ -51,6 +52,9 @@ multi-node jobs with SwiftR on Sun Grid Engine sites. The "mpi" environment is often a suitable choice. } + \item{kernel} { + Cobalt only - the kernel to run on the compute nodes. + } \item{workmode}{ Can be "node" or "slot". If "node", one worker is started for all slots on a node. From noreply at svn.ci.uchicago.edu Mon Feb 14 17:10:42 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 14 Feb 2011 17:10:42 -0600 (CST) Subject: [Swift-commit] r4091 - usertools/plotter/src/plotter Message-ID: <20110214231042.DAB25FC41@svn.ci.uchicago.edu> Author: wozniak Date: 2011-02-14 17:10:42 -0600 (Mon, 14 Feb 2011) New Revision: 4091 Modified: usertools/plotter/src/plotter/Lines.java Log: Add support for bw, misc cleanups Modified: usertools/plotter/src/plotter/Lines.java =================================================================== --- usertools/plotter/src/plotter/Lines.java 2011-02-14 21:58:55 UTC (rev 4090) +++ usertools/plotter/src/plotter/Lines.java 2011-02-14 23:10:42 UTC (rev 4091) @@ -1,16 +1,14 @@ package plotter; +import gnu.getopt.Getopt; + import java.awt.Color; import java.awt.geom.Rectangle2D; - import java.io.*; import java.util.*; -import gnu.getopt.Getopt; - import org.apache.commons.io.IOUtils; import org.apache.xmlgraphics.java2d.ps.EPSDocumentGraphics2D; - import org.jfree.chart.ChartFactory; import org.jfree.chart.JFreeChart; import org.jfree.chart.axis.NumberAxis; @@ -30,8 +28,8 @@ public static boolean bw = false; - static int w = 400; - static int h = 400; + static int width = 400; + static int height = 400; // null indicates the value was not set by the user static Double xmin = null; @@ -64,11 +62,11 @@ g2d.setGraphicContext (new org.apache.xmlgraphics.java2d.GraphicContext()); - rectangle = new Rectangle2D.Double(0, 0, w, h); + rectangle = new Rectangle2D.Double(0, 0, width, height); g2d.setGraphicContext (new org.apache.xmlgraphics.java2d.GraphicContext()); - g2d.setupDocument(out, w, h); + g2d.setupDocument(out, width, height); } catch (IOException e) { @@ -170,8 +168,8 @@ { switch (c) { - case 'v': - verbose = true; + case 'v': + verbose = true; } } @@ -268,16 +266,17 @@ width (output image width) height (output image height) xmin, xmax, ymin, ymax (auto-selected if not given) + bw (Black and white, true/false, default false) */ static void scanProperties() { String tmp; tmp = properties.getProperty("width"); if (tmp != null) - w = Integer.parseInt(tmp.trim()); + width = Integer.parseInt(tmp.trim()); tmp = properties.getProperty("height"); if (tmp != null) - h = Integer.parseInt(tmp.trim()); + height = Integer.parseInt(tmp.trim()); tmp = properties.getProperty("xmin"); if (tmp != null) xmin = Double.parseDouble(tmp); @@ -290,6 +289,9 @@ tmp = properties.getProperty("ymax"); if (tmp != null) ymax = Double.parseDouble(tmp); + tmp = properties.getProperty("bw"); + if (tmp != null) + bw = Boolean.parseBoolean(tmp); } /** From noreply at svn.ci.uchicago.edu Mon Feb 14 17:11:25 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 14 Feb 2011 17:11:25 -0600 (CST) Subject: [Swift-commit] r4092 - usertools/plotter Message-ID: <20110214231125.945DCFC41@svn.ci.uchicago.edu> Author: wozniak Date: 2011-02-14 17:11:25 -0600 (Mon, 14 Feb 2011) New Revision: 4092 Modified: usertools/plotter/ Log: Set ignores Property changes on: usertools/plotter ___________________________________________________________________ Name: svn:ignore + bin From noreply at svn.ci.uchicago.edu Mon Feb 14 20:06:14 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 14 Feb 2011 20:06:14 -0600 (CST) Subject: [Swift-commit] r4093 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110215020614.890849CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-14 20:06:14 -0600 (Mon, 14 Feb 2011) New Revision: 4093 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Removed debug flag in script, added by acciedent ` Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-14 23:11:25 UTC (rev 4092) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-15 02:06:14 UTC (rev 4093) @@ -1,6 +1,6 @@ #! /bin/bash -set -x +#set -x export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap From noreply at svn.ci.uchicago.edu Mon Feb 14 20:07:14 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 14 Feb 2011 20:07:14 -0600 (CST) Subject: [Swift-commit] r4094 - SwiftApps/SwiftR/Swift/man Message-ID: <20110215020714.43DB99CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-14 20:07:14 -0600 (Mon, 14 Feb 2011) New Revision: 4094 Modified: SwiftApps/SwiftR/Swift/man/swiftInit.Rd Log: Fixed markup error Modified: SwiftApps/SwiftR/Swift/man/swiftInit.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/swiftInit.Rd 2011-02-15 02:06:14 UTC (rev 4093) +++ SwiftApps/SwiftR/Swift/man/swiftInit.Rd 2011-02-15 02:07:14 UTC (rev 4094) @@ -53,7 +53,7 @@ environment is often a suitable choice. } \item{kernel} { - Cobalt only - the kernel to run on the compute nodes. + Cobalt only. the kernel to run on the compute nodes. } \item{workmode}{ Can be "node" or "slot". From noreply at svn.ci.uchicago.edu Mon Feb 14 20:08:21 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 14 Feb 2011 20:08:21 -0600 (CST) Subject: [Swift-commit] r4095 - SwiftApps/SwiftR/Swift/man Message-ID: <20110215020821.55E599CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-14 20:08:21 -0600 (Mon, 14 Feb 2011) New Revision: 4095 Modified: SwiftApps/SwiftR/Swift/man/swiftInit.Rd Log: Ooops, didn't fix markup error before. Modified: SwiftApps/SwiftR/Swift/man/swiftInit.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/swiftInit.Rd 2011-02-15 02:07:14 UTC (rev 4094) +++ SwiftApps/SwiftR/Swift/man/swiftInit.Rd 2011-02-15 02:08:21 UTC (rev 4095) @@ -52,7 +52,7 @@ multi-node jobs with SwiftR on Sun Grid Engine sites. The "mpi" environment is often a suitable choice. } - \item{kernel} { + \item{kernel}{ Cobalt only. the kernel to run on the compute nodes. } \item{workmode}{ From noreply at svn.ci.uchicago.edu Tue Feb 15 11:23:41 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 11:23:41 -0600 (CST) Subject: [Swift-commit] r4096 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110215172341.B3A0E9CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-15 11:23:41 -0600 (Tue, 15 Feb 2011) New Revision: 4096 Modified: SwiftApps/SwiftR/Swift/exec/fifowrite Log: Fix to prevent fifowrite processes breeding out of control: signal is now caught so if an apply call is Ctrl-Ced then the fifo write shell will clean up properly. Modified: SwiftApps/SwiftR/Swift/exec/fifowrite =================================================================== --- SwiftApps/SwiftR/Swift/exec/fifowrite 2011-02-15 02:08:21 UTC (rev 4095) +++ SwiftApps/SwiftR/Swift/exec/fifowrite 2011-02-15 17:23:41 UTC (rev 4096) @@ -33,6 +33,16 @@ esac done +function cleanup() { + if [ "$tout_pid" != "" ]; then + # stop timeout process from lingering + kill $tout_pid &> /dev/null + fi + if [ "$catpid" != "" ]; then + kill $catpid &> /dev/null + fi +} + # fork off a process to read from fifo and write to # temp file. # echo message to fifo, fork off a process. The remainder @@ -41,6 +51,7 @@ echo -n "$msg" > "$fifo" & catpid=$! tout_pid="" + if [ "$timeout" -gt 0 ]; then stimeout=`echo "scale=3; $timeout/1000.0" | bc` @@ -54,17 +65,18 @@ ) &> /dev/null & tout_pid=$! fi + # ignore kill signal when child is killed: # at that time wait will wake up and we can exit normally trap "" 15 if wait $catpid then + catpid="" # cat exited normally - if [ ! "$tout_pid" = "" ]; then - # stop timeout process from lingering - kill $tout_pid &> /dev/null - fi + cleanup exit 0 else + catpid="" + cleanup exit 2 fi From noreply at svn.ci.uchicago.edu Tue Feb 15 11:29:19 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 11:29:19 -0600 (CST) Subject: [Swift-commit] r4097 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110215172919.6A7BD9CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-15 11:29:19 -0600 (Tue, 15 Feb 2011) New Revision: 4097 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Added in working directory and log file locations for SwiftR Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-15 17:23:41 UTC (rev 4096) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-15 17:29:19 UTC (rev 4097) @@ -294,16 +294,20 @@ # so we have multiple workers per node. In future could # add support for virtual node mode, etc qsub -t "$time" -n $nodes $queueDirective $kernDirective \ - $projDirective -O SwiftR-workers \ + $projDirective -e `pwd`/SwiftR-workers.err \ + -o `pwd`/SwiftR-workers.out --debuglog `pwd`/cobalt.log \ + --cwd / \ --env WORKER_LOGGING_LEVEL=$workerLogging \ /usr/bin/perl $SWIFTBIN/worker.pl $CONTACT \ - $HOME/.globus/coasters $IDLETIMEOUT + $HOME/.globus/coasters $IDLETIMEOUT >$jobidfile + #TODO: change current working directory + succ=$? else qsub batch.sub >$jobidfile succ=$? fi - if [ $? -eq 0 ] + if [ $succ -eq 0 ] then echo Started workers from batch job $(cat $jobidfile) else From noreply at svn.ci.uchicago.edu Tue Feb 15 13:40:24 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 13:40:24 -0600 (CST) Subject: [Swift-commit] r4098 - branches/release-0.92/etc/sites Message-ID: <20110215194024.A766F9CC80@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-15 13:40:24 -0600 (Tue, 15 Feb 2011) New Revision: 4098 Removed: branches/release-0.92/etc/sites/teraport-remote-ssh/ Log: cleanup of etc/sites dir From noreply at svn.ci.uchicago.edu Tue Feb 15 13:48:18 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 13:48:18 -0600 (CST) Subject: [Swift-commit] r4099 - branches/release-0.92/etc/sites Message-ID: <20110215194818.C17D39CC80@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-15 13:48:18 -0600 (Tue, 15 Feb 2011) New Revision: 4099 Removed: branches/release-0.92/etc/sites/teraport-local-pbs/ Log: cleanup From noreply at svn.ci.uchicago.edu Tue Feb 15 13:48:30 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 13:48:30 -0600 (CST) Subject: [Swift-commit] r4100 - branches/release-0.92/etc/sites Message-ID: <20110215194830.BD2A69CC80@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-15 13:48:30 -0600 (Tue, 15 Feb 2011) New Revision: 4100 Removed: branches/release-0.92/etc/sites/teraport-local-pbs-coasters/ Log: cleanup From noreply at svn.ci.uchicago.edu Tue Feb 15 13:49:01 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 13:49:01 -0600 (CST) Subject: [Swift-commit] r4101 - branches/release-0.92/etc/sites Message-ID: <20110215194901.2AC519CC80@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-15 13:49:01 -0600 (Tue, 15 Feb 2011) New Revision: 4101 Removed: branches/release-0.92/etc/sites/teraport-remote-pbs-coasters-ssh/ Log: cleanup From noreply at svn.ci.uchicago.edu Tue Feb 15 13:59:52 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 13:59:52 -0600 (CST) Subject: [Swift-commit] r4102 - branches/release-0.92/etc/sites Message-ID: <20110215195952.A5F289CC80@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-15 13:59:52 -0600 (Tue, 15 Feb 2011) New Revision: 4102 Added: branches/release-0.92/etc/sites/local-pbs-coasters.xml Log: template for use by sites file generator Added: branches/release-0.92/etc/sites/local-pbs-coasters.xml =================================================================== --- branches/release-0.92/etc/sites/local-pbs-coasters.xml (rev 0) +++ branches/release-0.92/etc/sites/local-pbs-coasters.xml 2011-02-15 19:59:52 UTC (rev 4102) @@ -0,0 +1,17 @@ + + + + + 1 + 7200 + 1 + 1 + 1 + 1 + _QUEUE_ + 5.99 + 10000 + _PROJECT_ + _WORK_ + + From noreply at svn.ci.uchicago.edu Tue Feb 15 15:26:14 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 15:26:14 -0600 (CST) Subject: [Swift-commit] r4103 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110215212614.28F7D9CC80@svn.ci.uchicago.edu> Author: tga Date: 2011-02-15 15:26:14 -0600 (Tue, 15 Feb 2011) New Revision: 4103 Modified: SwiftApps/SwiftR/Swift/exec/fiforead SwiftApps/SwiftR/Swift/exec/fifowrite Log: Added interrupt handler to cleanup fiforead/write processes if cancelled from R. Modified: SwiftApps/SwiftR/Swift/exec/fiforead =================================================================== --- SwiftApps/SwiftR/Swift/exec/fiforead 2011-02-15 19:59:52 UTC (rev 4102) +++ SwiftApps/SwiftR/Swift/exec/fiforead 2011-02-15 21:26:14 UTC (rev 4103) @@ -26,11 +26,28 @@ done outfile=`mktemp` +function cleanup { + if [ "$tout_pid" != "" ]; then + # stop timeout process from lingering + kill $tout_pid &> /dev/null + fi + if [ "$catpid" != "" ]; then + # stop timeout process from lingering + kill $catpid &> /dev/null + fi +} + + # fork off a process to read from fifo and write to # temp file +tout_pid="" +catpid="" + cat $fifo &> "$outfile" & + catpid=$! -tout_pid="" +trap "cleanup" EXIT 1 2 3 15 + if [ "$timeout" -gt 0 ]; then stimeout=`echo "scale=3; $timeout/1000.0" | bc` @@ -48,21 +65,19 @@ # are closed tout_pid=$! fi -# ignore kill signal when child is killed: -# at that time wait will wake up and we can exit normally -trap "" 15 + if wait $catpid then + catpid="" # cat exited normally - if [ ! "$tout_pid" = "" ]; then - # stop timeout process from lingering - kill $tout_pid &> /dev/null - fi + cleanup # Print output cat "$outfile" rm "$outfile" exit 0 else + catpid="" + cleanup echo "_FIFOTIMEOUT_" rm "$outfile" #TODO: kill timeout thread Modified: SwiftApps/SwiftR/Swift/exec/fifowrite =================================================================== --- SwiftApps/SwiftR/Swift/exec/fifowrite 2011-02-15 19:59:52 UTC (rev 4102) +++ SwiftApps/SwiftR/Swift/exec/fifowrite 2011-02-15 21:26:14 UTC (rev 4103) @@ -51,6 +51,7 @@ echo -n "$msg" > "$fifo" & catpid=$! tout_pid="" +trap "cleanup" EXIT 1 2 3 15 if [ "$timeout" -gt 0 ]; then @@ -66,9 +67,6 @@ tout_pid=$! fi -# ignore kill signal when child is killed: -# at that time wait will wake up and we can exit normally -trap "" 15 if wait $catpid then catpid="" From noreply at svn.ci.uchicago.edu Tue Feb 15 15:47:59 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 15:47:59 -0600 (CST) Subject: [Swift-commit] r4104 - SwiftApps/SwiftR/Swift/R Message-ID: <20110215214759.03E119CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-15 15:47:58 -0600 (Tue, 15 Feb 2011) New Revision: 4104 Modified: SwiftApps/SwiftR/Swift/R/Workers.R Log: Bugfix: Missing processing of project argument to swfitInit Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-15 21:26:14 UTC (rev 4103) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-15 21:47:58 UTC (rev 4104) @@ -130,6 +130,12 @@ cmdString <- paste(cmdString, "-q", shQuote(queue)) } + if(is.null(project)) + project <- getOption("swift.project") + if(! is.null(project) ) { + cmdString <- paste(cmdString, "-A", shQuote(project)) + } + if(is.null(rcmd)) rcmd <- getOption("swift.rcmd") if(! is.null(rcmd) ) { From noreply at svn.ci.uchicago.edu Tue Feb 15 15:48:43 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 15:48:43 -0600 (CST) Subject: [Swift-commit] r4105 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110215214843.ADF599CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-15 15:48:43 -0600 (Tue, 15 Feb 2011) New Revision: 4105 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Tweaks to cobalt qsub commadn - not quite working yet Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-15 21:47:58 UTC (rev 4104) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-15 21:48:43 UTC (rev 4105) @@ -293,12 +293,14 @@ # for now, rely on swift worker to fork off worker processes # so we have multiple workers per node. In future could # add support for virtual node mode, etc + #CONTACT="http://10.40.9.151:$(echo $CONTACT | sed -e 's,^.*:,,')" + logdir=`pwd` qsub -t "$time" -n $nodes $queueDirective $kernDirective \ - $projDirective -e `pwd`/SwiftR-workers.err \ - -o `pwd`/SwiftR-workers.out --debuglog `pwd`/cobalt.log \ + $projDirective -e $logdir/SwiftR-workers.err \ + -o $logdir/SwiftR-workers.out --debuglog $logdir/cobalt.log \ --cwd / \ --env WORKER_LOGGING_LEVEL=$workerLogging \ - /usr/bin/perl $SWIFTBIN/worker.pl $CONTACT \ + /usr/bin/perl $SWIFTBIN/worker.pl $CONTACT SwiftR-workers \ $HOME/.globus/coasters $IDLETIMEOUT >$jobidfile #TODO: change current working directory succ=$? @@ -607,7 +609,7 @@ kill $starterpid &> /dev/null fi if [ "_$jobid" != _ ]; then - qdel "$jobid" + qdel "$jobid" &> /dev/null fi # eit cleanly stdcleanup_end From noreply at svn.ci.uchicago.edu Tue Feb 15 17:31:55 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 17:31:55 -0600 (CST) Subject: [Swift-commit] r4106 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110215233155.8773E9CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-15 17:31:55 -0600 (Tue, 15 Feb 2011) New Revision: 4106 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Got SwiftR working on Eureka!! Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-15 21:48:43 UTC (rev 4105) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-15 23:31:55 UTC (rev 4106) @@ -162,6 +162,27 @@ END } +make-cobalt-submit-file() +{ +# Most cobalt options are passed through the command line, we just need to +# include relevant arguments for worker.pl in here. + +cat > $HOME/batch.sub <$jobidfile - #TODO: change current working directory + $projDirective -e $wkdir/SwiftR-workers.err \ + -o $wkdir/SwiftR-workers.out --debuglog $wkdir/cobalt.log \ + --cwd / $HOME/batch.sub > $jobidfile succ=$? else - qsub batch.sub >$jobidfile + qsub batch.sub > $jobidfile succ=$? fi From noreply at svn.ci.uchicago.edu Tue Feb 15 17:34:26 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 17:34:26 -0600 (CST) Subject: [Swift-commit] r4107 - SwiftApps/SwiftR/Swift/R Message-ID: <20110215233426.BAE849CC94@svn.ci.uchicago.edu> Author: tga Date: 2011-02-15 17:34:26 -0600 (Tue, 15 Feb 2011) New Revision: 4107 Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R Log: Fix to TestFramework: it had inserted test results prematurely into a data frame. This didn't work in all circumstances because in some cases we had non-string and non-numeric data in the result lists. Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R =================================================================== --- SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-15 23:31:55 UTC (rev 4106) +++ SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-15 23:34:26 UTC (rev 4107) @@ -80,7 +80,7 @@ } } } - return (data.frame(collated)) + return (collated) } From noreply at svn.ci.uchicago.edu Tue Feb 15 22:53:06 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 15 Feb 2011 22:53:06 -0600 (CST) Subject: [Swift-commit] r4108 - usertools/plotter Message-ID: <20110216045306.9C34A9CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2011-02-15 22:53:06 -0600 (Tue, 15 Feb 2011) New Revision: 4108 Modified: usertools/plotter/classpath.zsh Log: Fix classpath on Cygwin Modified: usertools/plotter/classpath.zsh =================================================================== --- usertools/plotter/classpath.zsh 2011-02-15 23:34:26 UTC (rev 4107) +++ usertools/plotter/classpath.zsh 2011-02-16 04:53:06 UTC (rev 4108) @@ -5,9 +5,11 @@ if [[ $( uname ) == CYGWIN* ]] then typeset -T CP cp ";" - cp+=( ${classpath} ) - cp+=${PLOTTERS}/src - cp+=( ${PLOTTERS}/lib/*.jar ) + cp+=c:/cygwin${PLOTTERS}/src + for jar in ${PLOTTERS}/lib/*.jar + do + cp+=c:/cygwin${jar} + done else typeset -T CP cp CP=${CLASSPATH}:${PLOTTERS}/src From noreply at svn.ci.uchicago.edu Wed Feb 16 12:50:01 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 16 Feb 2011 12:50:01 -0600 (CST) Subject: [Swift-commit] r4109 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110216185001.81B3F9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-16 12:50:01 -0600 (Wed, 16 Feb 2011) New Revision: 4109 Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift Log: Added in trace informationf or debugging rserver.swift script. Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-16 04:53:06 UTC (rev 4108) +++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-16 18:50:01 UTC (rev 4109) @@ -50,9 +50,9 @@ string dir; # FIXME: read swiftserver dir via @args - + trace("top of loop: rserver waiting for input on", requestPipeName); dir = readData(requestPipeName); # Reads direct from this local pipe. Assumes Swift started in right dir. - # trace("rserver: got dir", dir); + trace("rserver: got dir", dir); external wait[]; wait = apply(dir); From noreply at svn.ci.uchicago.edu Wed Feb 16 14:23:28 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 16 Feb 2011 14:23:28 -0600 (CST) Subject: [Swift-commit] r4110 - in branches/release-0.92: . bin Message-ID: <20110216202328.9446B9CC80@svn.ci.uchicago.edu> Author: davidk Date: 2011-02-16 14:23:28 -0600 (Wed, 16 Feb 2011) New Revision: 4110 Added: branches/release-0.92/bin/gensites Modified: branches/release-0.92/build.xml Log: Initial commit of gensites script Added: branches/release-0.92/bin/gensites =================================================================== --- branches/release-0.92/bin/gensites (rev 0) +++ branches/release-0.92/bin/gensites 2011-02-16 20:23:28 UTC (rev 4110) @@ -0,0 +1,193 @@ +#!/bin/sh + +# crash: Report a problem and exit +crash() +{ + MSG=$1 + echo ${MSG} + exit 1 +} + +# get_value: Return the right hand side of an equals +# statment in swift.properties format: gensites.foo=bar +# will return bar +get_value() +{ + echo $1 |cut -d "=" -f2 +} + +# Verify an argument is not null +verify_not_null() +{ + argname=$1; shift + if [ _$1 != _ ]; then + return 0; + else + echo $0: value for $argname can not be null + exit 1 + fi +} + +# Print usage information and exit +usage() +{ + cat << END + + usage: gensites template [-p properties.file] [-L template_directory] [-h] + + template Name of template to use + -p properties.file Specify a swift.properties to use + -L template_directory Specify a non-standard template directory + -T List all templates available + -h Help / usage information + + Examples: + + Create a site configuration file for sites.xml using default properties.file in current directory + $ gensites pads > sites.xml + + Use a specific properties file for a site + $ gensites -p sites.properties pads > sites.xml + + Specify a non-standard directory where templates are located + $ gensites -L template.dir pads > sites.xml + + +END +exit 0; +} + +# Parse command line arguments +while [ $# -gt 0 ] +do + case "$1" in + -p) PROPERTIES_FILE=$2; verify_not_null properties_file $PROPERTIES_FILE; shift ;; + -L) TEMPLATE_DIRECTORY=$2; verify_not_null template_directory $TEMPLATE_DIRECTORY; shift ;; + -T) LIST_TEMPLATES=1 ;; + -h|-help) usage ;; + *) TEMPLATE=$1 ;; + esac + shift +done + +# Determine SWIFT_HOME +if [ -z "$SWIFT_HOME" ]; then + SWIFT_BIN_DIR=`which swift` + + # From SVN + SWIFT_HOME=`dirname $SWIFT_BIN_DIR`"/../../../etc/sites" + + # From a distribution + if [ ! -d "$SWIFT_HOME" ]; then + SWIFT_HOME=`dirname $SWIFT_BIN_DIR`"../../etc/sites" + fi + + # Can't find swift path, give up + if [ ! -d "$SWIFT_HOME" ]; then + crash "Unable to determine SWIFT_HOME. Please set manually." + fi +fi + +# List templates +if [ ! -z "$LIST_TEMPLATES" ]; then + ls -1 $SWIFT_HOME/* | xargs -n1 basename + ls -1 $HOME/.swift/sites/* | xargs -n1 basename + exit 0 +fi + +# Require template name to be specified by user +if [ -z "$TEMPLATE" ]; then + crash "Not specified: TEMPLATE" +fi + +# Use custom template directory if asked +if [ -d "$TEMPLATE_DIRECTORY" ]; then + TEMPLATE_PATH="$TEMPLATE_DIRECTORY/$TEMPLATE" +fi + +# Locate template +if [ ! -f "$TEMPLATE_PATH" ]; then + if [ -f "$TEMPLATE" ]; then + $TEMPLATE_PATH = $TEMPLATE + elif [ -f "$SWIFT_HOME/$TEMPLATE" ]; then + TEMPLATE_PATH=$SWIFT_HOME/$TEMPLATE + elif [ -f "$HOME/.swift/sites/$TEMPLATE" ]; then + TEMPLATE_PATH=$HOME/.swift/sites/$TEMPLATE + fi + if [ ! -f "$TEMPLATE_PATH" ]; then + crash "Cannot find template for $TEMPLATE" + fi +fi + + +# Ensure a properties file exists +if [ ! -f "$PROPERTIES_FILE" ]; then + PROPERTIES_FILE="swift.properties" + if [ ! -f "$PROPERTIES_FILE" ]; then + crash "Unable to find a valid properties file! Please specify a valid file with the -p option" + fi +fi + +# Parse values into variables for later sed processing +WORK=`pwd`"/work" +while read line +do + case "$line" in + $TEMPLATE".work="*) + WORK=`get_value $line` + ;; + $TEMPLATE".globus_hostname="*) + GLOBUS_HOSTNAME=`get_value $line` + ;; + $TEMPLATE".nodes="*) + NODES=`get_value $line` + ;; + $TEMPLATE".project="*) + PROJECT=`get_value $line` + ;; + $TEMPLATE".queue="*) + QUEUE=`get_value $line` + ;; + $TEMPLATE".n_gran="*) + N_GRAN=`get_value $line` + ;; + $TEMPLATE".n_max="*) + MAXTIME=`get_value $line` + ;; + esac +done < $PROPERTIES_FILE + +# Test for mandatory variables +[[ ${WORK} == "" ]] && crash "Not specified: WORK" + +# Verify that the variables by the template are defined +for TOKEN in PROJECT QUEUE N_GRAN N_MAX SLOTS + do + if grep _${TOKEN}_ ${TEMPLATE_PATH} > /dev/null + then + if ! declare -p ${TOKEN} > /dev/null + then + printenv + crash "Not specified: ${TOKEN}" + fi + fi +done + +# Replace values +SEDFILE=`mktemp` +{ + echo "s/_NODES_/${NODES}/" + echo "s/_HOST_/${GLOBUS_HOSTNAME}/" + echo "s at _WORK_@${WORK}@" + echo "s/_PROJECT_/${PROJECT}/" + echo "s/_QUEUE_/${QUEUE}/" + echo "s/_N_GRAN_/${N_GRAN}/" + echo "s/_N_MAX_/${N_MAX}/" + echo "s/_SLOTS_/${SLOTS}/" + echo "s/_MAXTIME_/${MAXTIME}/" + echo "s at _SERVICE_COASTERS_@${SERVICE_COASTERS:-NO_URL_GIVEN}@" + echo "s at _SERVICE_PORT_@${SERVICE_PORT:-NO_PORT_GIVEN}@" +} > $SEDFILE + +sed -f $SEDFILE < ${TEMPLATE_PATH} +rm $SEDFILE Property changes on: branches/release-0.92/bin/gensites ___________________________________________________________________ Name: svn:executable + * Modified: branches/release-0.92/build.xml =================================================================== --- branches/release-0.92/build.xml 2011-02-16 18:50:01 UTC (rev 4109) +++ branches/release-0.92/build.xml 2011-02-16 20:23:28 UTC (rev 4110) @@ -86,6 +86,7 @@ + From noreply at svn.ci.uchicago.edu Wed Feb 16 16:38:58 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 16 Feb 2011 16:38:58 -0600 (CST) Subject: [Swift-commit] r4111 - in SwiftApps/SwiftR: Swift Swift/R Swift/man mxtests/swift Message-ID: <20110216223859.011879CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-16 16:38:58 -0600 (Wed, 16 Feb 2011) New Revision: 4111 Added: SwiftApps/SwiftR/Swift/man/swiftExport.Rd SwiftApps/SwiftR/Swift/man/swiftLibrary.Rd Modified: SwiftApps/SwiftR/Swift/DESCRIPTION SwiftApps/SwiftR/Swift/R/Tests.R SwiftApps/SwiftR/Swift/man/Swift-package.Rd SwiftApps/SwiftR/Swift/man/swiftShutdown.Rd SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R Log: Working towards new release: added in documentation of new features. Modified: SwiftApps/SwiftR/Swift/DESCRIPTION =================================================================== --- SwiftApps/SwiftR/Swift/DESCRIPTION 2011-02-16 20:23:28 UTC (rev 4110) +++ SwiftApps/SwiftR/Swift/DESCRIPTION 2011-02-16 22:38:58 UTC (rev 4111) @@ -1,8 +1,8 @@ Package: Swift Type: Package Title: R interface to Swift parallel scripting languaage -Version: 0.1.2 -Date: 2010-02-28 +Version: 0.1.3 +Date: 2010-02-16 Author: Michael Wilde Maintainer: Michael Wilde Description: Routines to invoke R functions on remote resources through Swift. Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-16 20:23:28 UTC (rev 4110) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-16 22:38:58 UTC (rev 4111) @@ -9,7 +9,11 @@ -basicSwiftTest <- function() { swiftTest_1.1() } +basicSwiftTest <- function() { + swiftInit() + swiftTest_1.1() + swiftShutdown() +} swiftTest_1.1 <- function() { Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2011-02-16 20:23:28 UTC (rev 4110) +++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2011-02-16 22:38:58 UTC (rev 4111) @@ -222,7 +222,6 @@ \preformatted{ > require(Swift) > options(swift.server="local") -> swiftInit(cores=4) # start up Swift on your local machine > basicSwiftTest() # should take about 1 second > runAllSwiftTests() # should take < 60 seconds @@ -258,18 +257,15 @@ To run 4 parallel R servers on the local host: -\verb{options(swift.server="local")} -\verb{swiftInit(cores=4)} +\verb{swiftInit(server="local", cores=4)} To run 4 R servers on each of two hosts that can be reach by ssh: -\verb{options(swift.server="ssh")} -\verb{swiftInit(cores="4", hosts="hostname1 hostname2")} +\verb{swiftInit(server="ssh", cores="4", hosts="hostname1 hostname2")} To run 8 R servers for 30 minutes on each of 3 nodes of the Merlot cluster, run this on the login host "merlot", using its "serial" queue: -\verb{options(swift.server="pbsf")} -\verb{swiftInit(cores=8, nodes=3, time="00:30:00", +\verb{swiftInit(server="pbsf", cores=8, nodes=3, time="00:30:00", queue="serial")} These Swift servers can be started and left running for multiple @@ -287,7 +283,6 @@ # Start swift local server as above require(Swift) -initSwift() basicSwiftTest() } @@ -295,7 +290,7 @@ \preformatted{ require(Swift) -initSwift() +# runAllSwiftTests automatically starts up and shuts down a swift server runAllSwiftTests() } @@ -415,14 +410,19 @@ In addition, the environment variable CLASSPATH should not be set. -Variables set in the initialze script must typically be set in global -environment ( var <<- value ); These conventions may need to be -revisted. The initialexpr script is passed in the same saved/loaded R +Variables set in the initialize script will appear in the global +environment of the remote workers. +The initialexpr script is passed in the same saved/loaded R object as R functions and arguments. Thus if the expr needs to be run before these objects can be loaded, then an alternate mathod of intialization needs to be used. (Eg the original method of passing th einitvar as a separate file). +A set of libraries to be loaded remotely can be maintained with the +\code{swiftLibrary} and \code{swiftDetach} calls. In some cases +it is desirable to send data back and forth using the \code{swiftExport} +call. + All Swift servers do a \verb{require(methods)} call when they start. Its not clear if this is unversally desired. It was not done by default in Rserver but seems to be in interactive R. @@ -439,10 +439,6 @@ The following caveats are high priority on the FIXME list: -There is no automatic restart yet if swift dies in its server loop. In -particular, parsing errors, eg on the Swift initialexpr text, can -cause the R and hence the Swift server to exit - Only lapply is implemented (also SwiftApply) - need to see if we can cut down arg passing overhead for many of the apply() cases. @@ -452,9 +448,6 @@ There is no asynchronous swiftapply call yet. Each call must complete before control is returned to the R command loop. -Only one instance of each server type (i.e., local, pbs, pbsf, ssh) -can be started at a time. - Lower priority issues are: There is no easy way yet to alter Swift configuration file variables @@ -484,6 +477,8 @@ \code{\link{swiftInit}} \code{\link{swiftShutdown}} \code{\link{swiftapply}} +\code{\link{swiftLibrary}} +\code{\link{swiftExport}} } \examples{ library(Swift) Added: SwiftApps/SwiftR/Swift/man/swiftExport.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/swiftExport.Rd (rev 0) +++ SwiftApps/SwiftR/Swift/man/swiftExport.Rd 2011-02-16 22:38:58 UTC (rev 4111) @@ -0,0 +1,124 @@ +\name{swiftExport} +\alias{swiftExport} +\alias{swiftExportAll} +\alias{swiftRemoveAll} +%- Also NEED an '\alias' for EACH other topic documented here. +\title{ +Exporting global data to remove R workspaces. +} +\description{ + \code{swiftExport} provides a mechanism to export data and functions to remote + R workspaces. It is mainly useful in the case where a function being + used in a swift*apply call references functions or data in the global + workspace. \code{swiftExportAll} exports all non-hidden variables in + the global environment of the R session (i.e. those shown by an + invocation of \code{ls()} at the interactive prompt). + \code{swiftRemoveAll} removes all previously exported data from + remote R workspaces. + +} + +\usage{ +swiftExport(..., list = NULL, file = NULL) +swiftExportAll() +swiftRemoveAll() +} +%- maybe also 'usage' for other objects documented here. +\arguments{ + \item{\dots}{ + Names of data items to export, either as strings or R identifiers. +} + \item{list}{ + A list of strings which name data items to export. +} + \item{file}{ + Optional: a file which is on a file system shared between the local + R session and remote R workers in which to store the exported data. + If not provided, a file will automatically be chosen. + A directory can be specified via the swift.exportdir option. +} +} +\details{ +In many cases if a data or a function is referenced within a function +given to swift*apply it will automatically be transferred when the +apply call is carried out. This happens if it is a local +variable in some function, or if the reference refers to something +defined in a package that is loaded on the remote machines. + +However, functions or data in the global environment (e.g. those defined +directly in an interactive R session, or assignments using the \code{<<-} +operator are not automatically transferred to remote R workspaces +when a swift*apply call is carried out. + +If you are working with a large data set, it can be more efficient to store +the data in the global environment and use swiftExport to transfer the +data. R workers load exported data on demand, +so if it is referenced in multiple apply calls, the exported data only needs +t be transferred once. + +swiftExport currently assumes that the swift worker processors share a file +system with the current machine. The directory through which exported data +is transferred can be specified with the swift.exportdir option. + +The set of exported data persists across multiple swift sessions: if you shut +down a swift session and start a new one with \code{swiftInit}, the set of +exported data will be the same until you run \code{swiftRemoveAll}. + +} +\value{ + No values are returned. +} +\references{ +\url{http://www.ci.uchicago.edu/swift} +} +\author{ +Swift was developed by: Mihael Hategan, Ben Clifford, Justin Wozniak, +Yong Zhao, Ian Foster, and Michael Wilde with contributions from Sarah +Kenny, Ioan Raicu, Luiz Gadelha, Allan Espinosa, Zhao Zhang, David +Kelly, Jon Monette, Glen Hocky, Tom Uram, Wenjun Wu, and other users. + +Swift R package developed by Michael Wilde, Tim Armstrong and the OpenMx project + +Maintainer: Michael Wilde +} +\note{ +%% ~~further notes~~ +} + +%% ~Make other sections like Warning with \section{Warning }{....} ~ + +\seealso{ +\code{\link{Swift-package}} +} +\examples{ + +swiftInit(server="local", cores=2) + +# global data +mydata <- c(1,2,3,4) + +getter <- function (i) { mydata[[i]] } + +# Won't be able to find mydata +swiftLapply(1:4, getter) + +swiftExport(mydata) +# or: swiftExport(list=c("mydata")) +# This should work better! +swiftLapply(1:4, getter) + +swiftRemoveAll() +# won't work again +swiftLapply(1:4, getter) + +swiftExportAll() +# works again +swiftLapply(1:4, getter) + +swiftShutdown() + +} +% Add one or more standard keywords, see file 'KEYWORDS' in the +% R documentation directory. +\keyword{ ~kwd1 } +\keyword{ ~kwd2 }% __ONLY ONE__ keyword per line Added: SwiftApps/SwiftR/Swift/man/swiftLibrary.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/swiftLibrary.Rd (rev 0) +++ SwiftApps/SwiftR/Swift/man/swiftLibrary.Rd 2011-02-16 22:38:58 UTC (rev 4111) @@ -0,0 +1,76 @@ +\name{swiftLibrary} +\alias{swiftLibrary} +\alias{swiftDetach} +\alias{swiftDetachAll} +\title{ + Loading and unloading remote Swift libraries. +} +\description{ +These functions allow packages to be loaded and unloaded on remote R instances +managed by SwiftR. +} +\usage{ +swiftLibrary(packname) +swiftDetach(packname) +swiftDetachAll() +} +\arguments{ + \item{packname}{ + The name of the package to attach or detach. + The arguments accepted are the same as the standard R \code{library} + and \code{detach} functions. They can either be provided as R identifiers + (e.g. \code{library(Swift)}) or as strings (e.g. \code{library("Swift")}). + swiftDetach accepts package names with a "package:" prefix in the same + was as \code{detach} (e.g. \code{detach(package:Swift)}). +} +} +\details{ +If a library is added through the \code{swiftLibrary} call, +it will be loaded for any future swift*apply calls. The library can be unloaded using \code{swiftDetach} or \code{swiftDetachAll}. +The implementation guarantees that any subsequent swift*apply calls are run in +R workspaces with the appropriate set of packages loaded. + +Note that the package will not be loaded into the current R workspace. + +If you pair up \code{swiftLibrary} and +\code{swiftDetach} calls with the local versions, +you will have the same libraries loaded on remote R workspaces and in the local +workspace. + +\code{swiftDetachAll} unloads all libraries previously loaded using this +mechanism. +} +\value{ + No value is returned. +} +\references{ +\url{http://www.ci.uchicago.edu/swift} +} +\author{ +Swift was developed by: Mihael Hategan, Ben Clifford, Justin Wozniak, +Yong Zhao, Ian Foster, and Michael Wilde with contributions from Sarah +Kenny, Ioan Raicu, Luiz Gadelha, Allan Espinosa, Zhao Zhang, David +Kelly, Jon Monette, Glen Hocky, Tom Uram, Wenjun Wu, and other users. + +Swift R package developed by Michael Wilde, Tim Armstrong and the OpenMx project + +Maintainer: Michael Wilde +} +\note{ +%% ~~further notes~~ +} + +%% ~Make other sections like Warning with \section{Warning }{....} ~ + +\seealso{ +%% ~~objects to See Also as \code{\link{help}}, ~~~ +\code{\link{Swift-package}} +} +\examples{ + + +} +% Add one or more standard keywords, see file 'KEYWORDS' in the +% R documentation directory. +\keyword{ ~kwd1 } +\keyword{ ~kwd2 }% __ONLY ONE__ keyword per line Modified: SwiftApps/SwiftR/Swift/man/swiftShutdown.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/swiftShutdown.Rd 2011-02-16 20:23:28 UTC (rev 4110) +++ SwiftApps/SwiftR/Swift/man/swiftShutdown.Rd 2011-02-16 22:38:58 UTC (rev 4111) @@ -7,17 +7,20 @@ %% ~~ A concise (1-5 lines) description of what the function does. ~~ } \usage{ -swiftShutdown(handle = NULL) +swiftShutdown(handle = NULL, all=FALSE) } \arguments{ \item{handle}{ The handle value returned by "\code{swiftInit}" when a worker was started up. If provided, only that worker will be shut down. - If handle is not provided, all workers started during this R - session will be shut down. + If handle is not provided and \code{all} is FALSE, then the most + recently started worker will be shut down. } + \item{all}{ + Whether to terminate all workers started within this R session. } +} \details{ %% ~~ If necessary, more details than the description above ~~ } Modified: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-16 20:23:28 UTC (rev 4110) +++ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-16 22:38:58 UTC (rev 4111) @@ -53,7 +53,6 @@ list(75, 100, 64), list(75, 100, 128), list(75, 100, 256), - list(75, 100, 256 + 128), list(75, 100, 512), list(75, 100, 786), list(75, 100, 1024), From noreply at svn.ci.uchicago.edu Thu Feb 17 10:51:50 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 17 Feb 2011 10:51:50 -0600 (CST) Subject: [Swift-commit] r4112 - SwiftApps/SwiftR Message-ID: <20110217165150.D57DAFC41@svn.ci.uchicago.edu> Author: tga Date: 2011-02-17 10:51:50 -0600 (Thu, 17 Feb 2011) New Revision: 4112 Modified: SwiftApps/SwiftR/Makefile Log: Make automatically includes svn revision info in release tarball. Modified: SwiftApps/SwiftR/Makefile =================================================================== --- SwiftApps/SwiftR/Makefile 2011-02-16 22:38:58 UTC (rev 4111) +++ SwiftApps/SwiftR/Makefile 2011-02-17 16:51:50 UTC (rev 4112) @@ -45,6 +45,7 @@ rm $(TBALL) $(TBALL): Swift/inst/swift/bin/swift $(PKG_FILES) Makefile + if [ -d .svn ]; then svn info > Swift/svninfo; fi R CMD build Swift From noreply at svn.ci.uchicago.edu Thu Feb 17 12:05:36 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 17 Feb 2011 12:05:36 -0600 (CST) Subject: [Swift-commit] r4113 - in SwiftApps/SwiftR: Swift/R mxtests/swift Message-ID: <20110217180536.2A253FC41@svn.ci.uchicago.edu> Author: tga Date: 2011-02-17 12:05:36 -0600 (Thu, 17 Feb 2011) New Revision: 4113 Modified: SwiftApps/SwiftR/Swift/R/Bench.R SwiftApps/SwiftR/Swift/R/TestFramework.R SwiftApps/SwiftR/Swift/R/Tests.R SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R Log: Renamed test suite attribute "tests" to "groups" for clarity and to eliminate ambiguity with the property of the group Modified: SwiftApps/SwiftR/Swift/R/Bench.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Bench.R 2011-02-17 16:51:50 UTC (rev 4112) +++ SwiftApps/SwiftR/Swift/R/Bench.R 2011-02-17 18:05:36 UTC (rev 4113) @@ -88,7 +88,7 @@ swiftTestSuite <- list( setup=function () { initSwiftTestOptions(); swiftInit() }, # swiftInit controlled via options - tests=list(warmUpGroup, perfTestGroup1), + groups=list(warmUpGroup, perfTestGroup1), teardown=function () { swiftShutdown() }) return (swiftTestSuite) } Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R =================================================================== --- SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-17 16:51:50 UTC (rev 4112) +++ SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-17 18:05:36 UTC (rev 4113) @@ -130,19 +130,19 @@ } } -makeTestSuite <- function(tests, setup=NULL, teardown=NULL) { - return (list(tests=tests, setup=setup, teardown=teardown)) +makeTestSuite <- function(groups, setup=NULL, teardown=NULL) { + return (list(groups=groups, setup=setup, teardown=teardown)) } runTestSuite <- function (suite) { if (!is.null(suite$setup)) suite$setup() resSuite <- suite - resSuite$tests <- lapply(suite$tests, runTestGroup) + resSuite$groups <- lapply(suite$groups, runTestGroup) if (!is.null(suite$teardown)) suite$teardown() - lapply(resSuite$tests, printTestGroup) + lapply(resSuite$groups, printTestGroup) return (resSuite) } Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-17 16:51:50 UTC (rev 4112) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-17 18:05:36 UTC (rev 4113) @@ -660,7 +660,7 @@ swiftTestSuite <- makeTestSuite( setup=function () { initSwiftTestOptions(); swiftInit() }, # swiftInit controlled via options - tests=list(testGroup1.1, testGroup2, testGroup1.2, testGroup1.3, testGroup3, testGroup4, + groups=list(testGroup1.1, testGroup2, testGroup1.2, testGroup1.3, testGroup3, testGroup4, testGroup5), teardown=function () { swiftShutdown() }) Modified: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-17 16:51:50 UTC (rev 4112) +++ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-17 18:05:36 UTC (rev 4113) @@ -156,7 +156,7 @@ } swiftTestSuite <- makeTestSuite( - tests=list( + groups=list( # 16 cores makeBootstrapTestGroup(mode="swift", server="ssh", cores=2, hosts=c("nettle", "wapato", "dandelion", "cattail", @@ -178,7 +178,7 @@ )) padsTestSuite <- makeTestSuite( - tests=list( + groups=list( makeBootstrapTestGroup(mode="swift", server="pbs", cores=1, nodes=1), makeBootstrapTestGroup(mode="swift", server="pbs", cores=8, nodes=1), makeBootstrapTestGroup(mode="swift", server="pbs", cores=8, nodes=2), From noreply at svn.ci.uchicago.edu Thu Feb 17 14:33:16 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 17 Feb 2011 14:33:16 -0600 (CST) Subject: [Swift-commit] r4114 - in SwiftApps/SwiftR: Swift/R mxtests/swift Message-ID: <20110217203316.86F58FC41@svn.ci.uchicago.edu> Author: tga Date: 2011-02-17 14:33:16 -0600 (Thu, 17 Feb 2011) New Revision: 4114 Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R SwiftApps/SwiftR/mxtests/swift/MakeTests.R Log: Properly handle performance params. Modified: SwiftApps/SwiftR/Swift/R/TestFramework.R =================================================================== --- SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-17 18:05:36 UTC (rev 4113) +++ SwiftApps/SwiftR/Swift/R/TestFramework.R 2011-02-17 20:33:16 UTC (rev 4114) @@ -20,7 +20,7 @@ } makeParamTestGroup <- function (name, f, allargs, - setup=NULL, teardown=NULL) { + setup=NULL, teardown=NULL, perfparams=NULL) { buildClosure <- function (f, args) { @@ -39,13 +39,14 @@ test$args <- args # Store for later analysis test$passed <- NULL test$time <- NULL + test$perfparams <- perfparams tests[[length(tests) + 1]] <- test } return (makeTestGroup(name, tests, setup, teardown)) } -analysePerf <- function (testGroupResults, nargs=0) { +analysePerf <- function (testGroupResults, nargs=0, perfparams=NULL) { # Build a list of vectors of performance results for easier analysis # [[1]] -> first arg # [[2]] -> second arg @@ -71,13 +72,10 @@ function(test) {test$time })) # Store the pnames as vectors - if (!is.null(testGroupResults$perfparams)) { - pnames <- names(testGroupResults$perfparams) - if (!is.na(pnames) && length(pnames) > 0) { - for (i in 1:length(pnames)) { - collated[[pnames[[i]]]] <- rep(testGroupResults$perfparams[[i]], - n) - } + if (!is.null(perfparams)) { + for (pname in perfparams) { + collated[[pname]] <- unlist(lapply(testGroupResults$tests, + function(test) {test$perfparams[[pname]]} )) } } return (collated) Modified: SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-17 18:05:36 UTC (rev 4113) +++ SwiftApps/SwiftR/mxtests/swift/BootstrapParallelBigger.R 2011-02-17 20:33:16 UTC (rev 4114) @@ -149,7 +149,8 @@ #swiftExportAll() submodels <- omxLapply(1:nReps, createNewModel, 'stErrSim', template) cat(length(submodels), "submodels created\n") - names(submodels) <- imxExtractNames(submodels) +# names(submodels) <- imxExtractNames(submodels) + names(submodels) <- omxExtractNames(submodels) topModel at submodels <- submodels return (topModel) Modified: SwiftApps/SwiftR/mxtests/swift/MakeTests.R =================================================================== --- SwiftApps/SwiftR/mxtests/swift/MakeTests.R 2011-02-17 18:05:36 UTC (rev 4113) +++ SwiftApps/SwiftR/mxtests/swift/MakeTests.R 2011-02-17 20:33:16 UTC (rev 4114) @@ -44,17 +44,17 @@ tg <- makeParamTestGroup(name=paste("swift_",name, " ", paramstr,sep=""), f=f, allargs=allargs, - perfparams <- params, + perfparams = params, setup=function() {swiftSetup(...)}, - tearDown <- swiftTearDown) + teardown = swiftTearDown) } else if (mode == "snowfall") { tg <- makeParamTestGroup(name=paste("sf_",name, paramstr, sep=""), f=f, allargs=allargs, - perfparams <- params, + perfparams = params, setup=function() {sfSetup(...)}, - tearDown <- sfTearDown) + teardown = sfTearDown) } else { @@ -62,7 +62,7 @@ tg <- makeParamTestGroup(name=paste("serial_",name, paramstr, sep=""), f=f, allargs=allargs, - perfparams <- list(), + perfparams = list(), setup=function() {try(detach(package:Swift)); try(detach(package:Snowfall))} ) } From noreply at svn.ci.uchicago.edu Thu Feb 17 17:50:37 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 17 Feb 2011 17:50:37 -0600 (CST) Subject: [Swift-commit] r4115 - SwiftApps/SwiftR Message-ID: <20110217235037.A0D32FC41@svn.ci.uchicago.edu> Author: tga Date: 2011-02-17 17:50:37 -0600 (Thu, 17 Feb 2011) New Revision: 4115 Modified: SwiftApps/SwiftR/Makefile Log: Fix makefile to work correctly on machines with no svn installation. Modified: SwiftApps/SwiftR/Makefile =================================================================== --- SwiftApps/SwiftR/Makefile 2011-02-17 20:33:16 UTC (rev 4114) +++ SwiftApps/SwiftR/Makefile 2011-02-17 23:50:37 UTC (rev 4115) @@ -45,7 +45,7 @@ rm $(TBALL) $(TBALL): Swift/inst/swift/bin/swift $(PKG_FILES) Makefile - if [ -d .svn ]; then svn info > Swift/svninfo; fi + if [ -d .svn -a -x "`which svn`" ]; then svn info > Swift/svninfo; fi R CMD build Swift From noreply at svn.ci.uchicago.edu Fri Feb 18 15:59:31 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 18 Feb 2011 15:59:31 -0600 (CST) Subject: [Swift-commit] r4116 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110218215931.2E0FB9CC80@svn.ci.uchicago.edu> Author: tga Date: 2011-02-18 15:59:30 -0600 (Fri, 18 Feb 2011) New Revision: 4116 Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt SwiftApps/SwiftR/Swift/exec/start-swift Log: Set cobalt to use provider staging. Fixed bug where cobalt submit file was overwritten before it could be used. Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-02-17 23:50:37 UTC (rev 4115) +++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-02-18 21:59:30 UTC (rev 4116) @@ -30,7 +30,8 @@ $throttle 10000 - $HOME/swiftwork + $(pwd)/swiftwork + $(pwd)/swiftscratch @@ -42,8 +43,8 @@ execution.retries=0 lazy.errors=false status.mode=provider -use.provider.staging=false -provider.staging.pin.swiftfiles=false +use.provider.staging=true +provider.staging.pin.swiftfiles=true #throttle.host.submit=1 END Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-17 23:50:37 UTC (rev 4115) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-18 21:59:30 UTC (rev 4116) @@ -1,6 +1,6 @@ #! /bin/bash -#set -x +set -x export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap @@ -81,12 +81,13 @@ # FIXME: does PBS need same workers-per-node logic as SGE? make-pbs-submit-file() { + SUBMIT_FILE=$1 if [ $queue != NONE ]; then queueDirective="#PBS -q $queue" else queueDirective="" fi -cat >batch.sub < $SUBMIT_FILE <batch.sub < $SUBMIT_FILE < $HOME/batch.sub < $SUBMIT_FILE <batch.sub < $SUBMIT_FILE < $jobidfile + qsub $SUBMIT_FILE | sed 's/[^0-9 ]//g' | awk '{ print $1 }' > $jobidfile succ=$? elif [ "${server}" = "cobalt" ] then @@ -317,10 +328,10 @@ qsub -t "$time" -n $nodes $queueDirective $kernDirective \ $projDirective -e $wkdir/SwiftR-workers.err \ -o $wkdir/SwiftR-workers.out --debuglog $wkdir/cobalt.log \ - --cwd / $HOME/batch.sub > $jobidfile + --cwd / $SUBMIT_FILE > $jobidfile succ=$? else - qsub batch.sub > $jobidfile + qsub $SUBMIT_FILE > $jobidfile succ=$? fi @@ -501,6 +512,10 @@ if [ "$keepdir" = "FALSE" ]; then cd / rm -rf "$trundir" + if [ "$server" = "cobalt" ]; then + # Submit file in different directory + rm $SUBMIT_FILE + fi fi } From noreply at svn.ci.uchicago.edu Fri Feb 18 15:59:56 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 18 Feb 2011 15:59:56 -0600 (CST) Subject: [Swift-commit] r4117 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110218215956.EBE559CC80@svn.ci.uchicago.edu> Author: tga Date: 2011-02-18 15:59:56 -0600 (Fri, 18 Feb 2011) New Revision: 4117 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Need to stop checking in start-swift with echoing. Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-18 21:59:30 UTC (rev 4116) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-18 21:59:56 UTC (rev 4117) @@ -1,6 +1,6 @@ #! /bin/bash -set -x +#set -x export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap From noreply at svn.ci.uchicago.edu Fri Feb 18 16:50:16 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 18 Feb 2011 16:50:16 -0600 (CST) Subject: [Swift-commit] r4118 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110218225016.CFEBC9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-18 16:50:16 -0600 (Fri, 18 Feb 2011) New Revision: 4118 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Added in cleanup code on some exit branches previously missing it. Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-18 21:59:56 UTC (rev 4117) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-18 22:50:16 UTC (rev 4118) @@ -28,6 +28,8 @@ if [ "_$uriline" = _ ]; then echo "$0: No passive state message from Swift - exiting." + stdcleanup_start + stdcleanup_end exit 1 fi @@ -119,6 +121,8 @@ if [ _$GLOBUS_HOSTNAME = _ ]; then echo GLOBUS_HOSTNAME must be set to worker-reachable address of submit host for pbsf server mode. usage + stdcleanup_start + stdcleanup_end exit 1 fi if [ $queue != NONE ]; then From noreply at svn.ci.uchicago.edu Fri Feb 18 17:10:53 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 18 Feb 2011 17:10:53 -0600 (CST) Subject: [Swift-commit] r4119 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110218231053.C36F49CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-18 17:10:53 -0600 (Fri, 18 Feb 2011) New Revision: 4119 Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbs SwiftApps/SwiftR/Swift/exec/configure-server-sge SwiftApps/SwiftR/Swift/exec/configure-server-ssh Log: Switched to provider staging and locals cratch dirs. Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbs =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2011-02-18 22:50:16 UTC (rev 4118) +++ SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2011-02-18 23:10:53 UTC (rev 4119) @@ -32,7 +32,8 @@ $throttlePBS 10000 - $HOME/swiftwork + $(pwd)/swiftwork + $(pwd)/swiftscratch @@ -44,8 +45,8 @@ execution.retries=0 lazy.errors=false status.mode=provider -use.provider.staging=false -provider.staging.pin.swiftfiles=false +use.provider.staging=true +provider.staging.pin.swiftfiles=true #throttle.host.submit=1 END Modified: SwiftApps/SwiftR/Swift/exec/configure-server-sge =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-sge 2011-02-18 22:50:16 UTC (rev 4118) +++ SwiftApps/SwiftR/Swift/exec/configure-server-sge 2011-02-18 23:10:53 UTC (rev 4119) @@ -33,8 +33,8 @@ $throttle 10000 - $HOME/swiftwork - /tmp/$USER/swiftwork + $(pwd)/swiftwork + $pwd/swiftscratch proxy Modified: SwiftApps/SwiftR/Swift/exec/configure-server-ssh =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-ssh 2011-02-18 22:50:16 UTC (rev 4118) +++ SwiftApps/SwiftR/Swift/exec/configure-server-ssh 2011-02-18 23:10:53 UTC (rev 4119) @@ -25,7 +25,8 @@ 2.55 10000 - $HOME/swiftwork + $(pwd)/swiftwork + $(pwd)/swiftscratch @@ -37,8 +38,8 @@ execution.retries=0 lazy.errors=false status.mode=provider -use.provider.staging=false -provider.staging.pin.swiftfiles=false +use.provider.staging=true +provider.staging.pin.swiftfiles=true #throttle.host.submit=1 END From noreply at svn.ci.uchicago.edu Fri Feb 18 23:48:29 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 18 Feb 2011 23:48:29 -0600 (CST) Subject: [Swift-commit] r4121 - in branches/release-0.92: bin etc/sites Message-ID: <20110219054829.9670A9CCBF@svn.ci.uchicago.edu> Author: davidk Date: 2011-02-18 23:48:29 -0600 (Fri, 18 Feb 2011) New Revision: 4121 Added: branches/release-0.92/etc/sites/intrepid branches/release-0.92/etc/sites/local branches/release-0.92/etc/sites/local-pbs-coasters branches/release-0.92/etc/sites/pads branches/release-0.92/etc/sites/queenbee branches/release-0.92/etc/sites/sge-local branches/release-0.92/etc/sites/ssh branches/release-0.92/etc/sites/ssh-pbs-coasters branches/release-0.92/etc/sites/surveyor Removed: branches/release-0.92/etc/sites/local-pbs-coasters.xml Modified: branches/release-0.92/bin/gensites Log: Gensites updates: now uses pbs style directives in swift.properties, ability to use a default value across all templates, various other fixes Added templates from test suite Modified: branches/release-0.92/bin/gensites =================================================================== --- branches/release-0.92/bin/gensites 2011-02-19 04:31:50 UTC (rev 4120) +++ branches/release-0.92/bin/gensites 2011-02-19 05:48:29 UTC (rev 4121) @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # crash: Report a problem and exit crash() @@ -13,7 +13,7 @@ # will return bar get_value() { - echo $1 |cut -d "=" -f2 + echo $2 |cut -d "=" -f2 } # Verify an argument is not null @@ -74,12 +74,8 @@ if [ -z "$SWIFT_HOME" ]; then SWIFT_BIN_DIR=`which swift` - # From SVN - SWIFT_HOME=`dirname $SWIFT_BIN_DIR`"/../../../etc/sites" - - # From a distribution if [ ! -d "$SWIFT_HOME" ]; then - SWIFT_HOME=`dirname $SWIFT_BIN_DIR`"../../etc/sites" + SWIFT_HOME=`dirname $SWIFT_BIN_DIR`"/../etc/sites" fi # Can't find swift path, give up @@ -90,8 +86,16 @@ # List templates if [ ! -z "$LIST_TEMPLATES" ]; then - ls -1 $SWIFT_HOME/* | xargs -n1 basename - ls -1 $HOME/.swift/sites/* | xargs -n1 basename + # Templates in etc/sites + for file in `ls -1 $SWIFT_HOME/*` + do + basename $file + done + # Templates in .swift/sites + for file in `ls -1 $HOME/.swift/sites/*` + do + basename $file + done exit 0 fi @@ -133,25 +137,25 @@ while read line do case "$line" in - $TEMPLATE".work="*) + \#$TEMPLATE" work="*|'#gensites work='*) WORK=`get_value $line` ;; - $TEMPLATE".globus_hostname="*) + \#$TEMPLATE" globus_hostname="*|'#gensites globus_hostname=') GLOBUS_HOSTNAME=`get_value $line` ;; - $TEMPLATE".nodes="*) + \#$TEMPLATE" nodes="*|'#gensites nodes='*) NODES=`get_value $line` ;; - $TEMPLATE".project="*) + \#$TEMPLATE" project="*|'#gensites project='*) PROJECT=`get_value $line` ;; - $TEMPLATE".queue="*) + \#$TEMPLATE" queue="*|'#gensites queue='*) QUEUE=`get_value $line` ;; - $TEMPLATE".n_gran="*) + \#$TEMPLATE" n_gran="*|'#gensites n_gran='*) N_GRAN=`get_value $line` ;; - $TEMPLATE".n_max="*) + \#$TEMPLATE" n_max="*|'#gensites n_max='*) MAXTIME=`get_value $line` ;; esac @@ -167,7 +171,6 @@ then if ! declare -p ${TOKEN} > /dev/null then - printenv crash "Not specified: ${TOKEN}" fi fi Added: branches/release-0.92/etc/sites/intrepid =================================================================== --- branches/release-0.92/etc/sites/intrepid (rev 0) +++ branches/release-0.92/etc/sites/intrepid 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,32 @@ + + + + + + _WORK_ + + 0.04 + file + + + + + + + _HOST_ + _PROJECT_ + _QUEUE_ + zeptoos + true + 21 + 10000 + 1 + DEBUG + 1 + 900 + 64 + 64 + _WORK_ + + + Added: branches/release-0.92/etc/sites/local =================================================================== --- branches/release-0.92/etc/sites/local (rev 0) +++ branches/release-0.92/etc/sites/local 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,9 @@ + + + + + 10000 + .31 + _WORK_ + + Copied: branches/release-0.92/etc/sites/local-pbs-coasters (from rev 4120, branches/release-0.92/etc/sites/local-pbs-coasters.xml) =================================================================== --- branches/release-0.92/etc/sites/local-pbs-coasters (rev 0) +++ branches/release-0.92/etc/sites/local-pbs-coasters 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,17 @@ + + + + + 1 + 7200 + 1 + 1 + 1 + 1 + _QUEUE_ + 5.99 + 10000 + _PROJECT_ + _WORK_ + + Deleted: branches/release-0.92/etc/sites/local-pbs-coasters.xml =================================================================== --- branches/release-0.92/etc/sites/local-pbs-coasters.xml 2011-02-19 04:31:50 UTC (rev 4120) +++ branches/release-0.92/etc/sites/local-pbs-coasters.xml 2011-02-19 05:48:29 UTC (rev 4121) @@ -1,17 +0,0 @@ - - - - - 1 - 7200 - 1 - 1 - 1 - 1 - _QUEUE_ - 5.99 - 10000 - _PROJECT_ - _WORK_ - - Added: branches/release-0.92/etc/sites/pads =================================================================== --- branches/release-0.92/etc/sites/pads (rev 0) +++ branches/release-0.92/etc/sites/pads 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,8 @@ + + + + + 0 + _WORK_ + + Added: branches/release-0.92/etc/sites/queenbee =================================================================== --- branches/release-0.92/etc/sites/queenbee (rev 0) +++ branches/release-0.92/etc/sites/queenbee 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,8 @@ + + + + + 0 + _WORK_ + + Added: branches/release-0.92/etc/sites/sge-local =================================================================== --- branches/release-0.92/etc/sites/sge-local (rev 0) +++ branches/release-0.92/etc/sites/sge-local 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,10 @@ + + + + threaded + .49 + 10000 + + _WORK_ + + Added: branches/release-0.92/etc/sites/ssh =================================================================== --- branches/release-0.92/etc/sites/ssh (rev 0) +++ branches/release-0.92/etc/sites/ssh 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,8 @@ + + + + + 0 + _WORK_ + + Added: branches/release-0.92/etc/sites/ssh-pbs-coasters =================================================================== --- branches/release-0.92/etc/sites/ssh-pbs-coasters (rev 0) +++ branches/release-0.92/etc/sites/ssh-pbs-coasters 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,15 @@ + + + + + 3000 + 8 + 1 + 1 + 10 + short + 0.5 + 10000 + _WORK_ + + Added: branches/release-0.92/etc/sites/surveyor =================================================================== --- branches/release-0.92/etc/sites/surveyor (rev 0) +++ branches/release-0.92/etc/sites/surveyor 2011-02-19 05:48:29 UTC (rev 4121) @@ -0,0 +1,22 @@ + + + + + + _HOST_ + _PROJECT_ + _QUEUE_ + zeptoos + true + 21 + 10000 + 1 + DEBUG + 1 + 900 + 64 + 64 + _WORK_ + + + From noreply at svn.ci.uchicago.edu Fri Feb 18 22:31:51 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 18 Feb 2011 22:31:51 -0600 (CST) Subject: [Swift-commit] r4120 - branches/release-0.92/etc/sites Message-ID: <20110219043151.1FFB59CC80@svn.ci.uchicago.edu> Author: davidk Date: 2011-02-18 22:31:50 -0600 (Fri, 18 Feb 2011) New Revision: 4120 Removed: branches/release-0.92/etc/sites/cnari-abe/ branches/release-0.92/etc/sites/cnari-local/ branches/release-0.92/etc/sites/cnari-queenbee/ branches/release-0.92/etc/sites/cnari-ranger/ branches/release-0.92/etc/sites/local-pbs-coasters/ branches/release-0.92/etc/sites/local-pbs/ branches/release-0.92/etc/sites/local/ branches/release-0.92/etc/sites/pads-local-pbs-coasters/ branches/release-0.92/etc/sites/pads-local-pbs/ branches/release-0.92/etc/sites/pads-remote-pbs-coasters-ssh/ branches/release-0.92/etc/sites/pads-remote-ssh/ branches/release-0.92/etc/sites/ssh/ Log: Removing swiftconfig templates to be replaced with gensites templates with similar names From noreply at svn.ci.uchicago.edu Sat Feb 19 15:29:00 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 19 Feb 2011 15:29:00 -0600 (CST) Subject: [Swift-commit] r4122 - branches/release-0.92/bin Message-ID: <20110219212900.A3DD69CC80@svn.ci.uchicago.edu> Author: davidk Date: 2011-02-19 15:29:00 -0600 (Sat, 19 Feb 2011) New Revision: 4122 Modified: branches/release-0.92/bin/gensites Log: Gensites swift.properties syntax changes and variable evaluation Modified: branches/release-0.92/bin/gensites =================================================================== --- branches/release-0.92/bin/gensites 2011-02-19 05:48:29 UTC (rev 4121) +++ branches/release-0.92/bin/gensites 2011-02-19 21:29:00 UTC (rev 4122) @@ -9,11 +9,12 @@ } # get_value: Return the right hand side of an equals -# statment in swift.properties format: gensites.foo=bar +# statment in swift.properties format: #site sitename option=value # will return bar get_value() { - echo $2 |cut -d "=" -f2 + for last; do true; done + eval echo $last |cut -d "=" -f2 } # Verify an argument is not null @@ -137,25 +138,25 @@ while read line do case "$line" in - \#$TEMPLATE" work="*|'#gensites work='*) + "#site $TEMPLATE work="*|'#site work='*) WORK=`get_value $line` ;; - \#$TEMPLATE" globus_hostname="*|'#gensites globus_hostname=') + "#site $TEMPLATE globus_hostname="*|'#site globus_hostname=') GLOBUS_HOSTNAME=`get_value $line` ;; - \#$TEMPLATE" nodes="*|'#gensites nodes='*) + "#site $TEMPLATE nodes="*|'#site nodes='*) NODES=`get_value $line` ;; - \#$TEMPLATE" project="*|'#gensites project='*) + "#site $TEMPLATE project="*|'#site project='*) PROJECT=`get_value $line` ;; - \#$TEMPLATE" queue="*|'#gensites queue='*) + "#site $TEMPLATE queue="*|'#site queue='*) QUEUE=`get_value $line` ;; - \#$TEMPLATE" n_gran="*|'#gensites n_gran='*) + "#site $TEMPLATE n_gran="*|'#site n_gran='*) N_GRAN=`get_value $line` ;; - \#$TEMPLATE" n_max="*|'#gensites n_max='*) + "#site $TEMPLATE n_max="*|'#site n_max='*) MAXTIME=`get_value $line` ;; esac From noreply at svn.ci.uchicago.edu Sat Feb 19 15:45:04 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 19 Feb 2011 15:45:04 -0600 (CST) Subject: [Swift-commit] r4123 - branches/release-0.92/bin Message-ID: <20110219214504.D11519CC92@svn.ci.uchicago.edu> Author: davidk Date: 2011-02-19 15:45:03 -0600 (Sat, 19 Feb 2011) New Revision: 4123 Modified: branches/release-0.92/bin/gensites Log: Small update to gensites usage and better handling of errors while listing templates Modified: branches/release-0.92/bin/gensites =================================================================== --- branches/release-0.92/bin/gensites 2011-02-19 21:29:00 UTC (rev 4122) +++ branches/release-0.92/bin/gensites 2011-02-19 21:45:03 UTC (rev 4123) @@ -34,7 +34,7 @@ { cat << END - usage: gensites template [-p properties.file] [-L template_directory] [-h] + usage: gensites template [-p properties.file] [-L template_directory] [-h] [-T] template Name of template to use -p properties.file Specify a swift.properties to use @@ -65,7 +65,7 @@ -p) PROPERTIES_FILE=$2; verify_not_null properties_file $PROPERTIES_FILE; shift ;; -L) TEMPLATE_DIRECTORY=$2; verify_not_null template_directory $TEMPLATE_DIRECTORY; shift ;; -T) LIST_TEMPLATES=1 ;; - -h|-help) usage ;; + -h|-help|-?) usage ;; *) TEMPLATE=$1 ;; esac shift @@ -88,12 +88,12 @@ # List templates if [ ! -z "$LIST_TEMPLATES" ]; then # Templates in etc/sites - for file in `ls -1 $SWIFT_HOME/*` + for file in `ls -1 $SWIFT_HOME/* 2>/dev/null` do basename $file done # Templates in .swift/sites - for file in `ls -1 $HOME/.swift/sites/*` + for file in `ls -1 $HOME/.swift/sites/* 2>/dev/null` do basename $file done @@ -124,7 +124,6 @@ fi fi - # Ensure a properties file exists if [ ! -f "$PROPERTIES_FILE" ]; then PROPERTIES_FILE="swift.properties" From noreply at svn.ci.uchicago.edu Mon Feb 21 14:55:42 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 14:55:42 -0600 (CST) Subject: [Swift-commit] r4124 - in SwiftApps/SwiftR/Swift: . R Message-ID: <20110221205542.4BCF89CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 14:55:41 -0600 (Mon, 21 Feb 2011) New Revision: 4124 Modified: SwiftApps/SwiftR/Swift/NAMESPACE SwiftApps/SwiftR/Swift/R/Tests.R Log: Added in option to pass swiftInit parameters directly to test runner functions to save unnecessary option setting. Modified: SwiftApps/SwiftR/Swift/NAMESPACE =================================================================== --- SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-19 21:45:03 UTC (rev 4123) +++ SwiftApps/SwiftR/Swift/NAMESPACE 2011-02-21 20:55:41 UTC (rev 4124) @@ -22,7 +22,6 @@ export(basicSwiftTest) exportPattern("^swiftTest") exportPattern("^testGroup") -export(swiftTestSuite) export(mkPerfTest) export(doPerfTest) Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-19 21:45:03 UTC (rev 4123) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-21 20:55:41 UTC (rev 4124) @@ -9,8 +9,8 @@ -basicSwiftTest <- function() { - swiftInit() +basicSwiftTest <- function(...) { + swiftInit(...) swiftTest_1.1() swiftShutdown() } @@ -595,12 +595,12 @@ } -runAllSwiftTests <- function() { +runAllSwiftTests <- function(...) { startTime = proc.time()[["elapsed"]] - runTestSuite(swiftTestSuite) + runTestSuite(makeFullTestSuite(...)) endTime <- proc.time()[["elapsed"]] runTime <- endTime - startTime @@ -656,11 +656,12 @@ tests = list(mkTest(swiftTest_5.1))) - -swiftTestSuite <- makeTestSuite( +makeFullTestSuite <- function (...) { + initArgs <- list(...) + makeTestSuite( setup=function () { initSwiftTestOptions(); - swiftInit() }, # swiftInit controlled via options + do.call(swiftInit, initArgs) }, # swiftInit controlled via options groups=list(testGroup1.1, testGroup2, testGroup1.2, testGroup1.3, testGroup3, testGroup4, testGroup5), teardown=function () { swiftShutdown() }) - +} From noreply at svn.ci.uchicago.edu Mon Feb 21 15:06:15 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 15:06:15 -0600 (CST) Subject: [Swift-commit] r4125 - in SwiftApps/SwiftR/Swift: R exec Message-ID: <20110221210615.8B8229CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 15:06:15 -0600 (Mon, 21 Feb 2011) New Revision: 4125 Modified: SwiftApps/SwiftR/Swift/R/Workers.R SwiftApps/SwiftR/Swift/exec/start-swift Log: Added --trace option to trace start-swift Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-21 20:55:41 UTC (rev 4124) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-21 21:06:15 UTC (rev 4125) @@ -160,6 +160,8 @@ if(!is.null(keepworkdir) && keepworkdir) cmdString <- paste(cmdString, "-k") + if (!is.null(getOption("swift.trace")) && getOption("swift.trace")) + cmdString <- paste(cmdString, "--trace") # launch asynchronously # for now, we will rely on the shell script's output to inform Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-21 20:55:41 UTC (rev 4124) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-21 21:06:15 UTC (rev 4125) @@ -1,7 +1,5 @@ #! /bin/bash -#set -x - export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap # Set the umask to prevent any access by other users: @@ -412,6 +410,7 @@ -t time 00:30:00 hh:mm:ss, for PBS, Cobalt and SGE only -w wkloglvl NONE NONE, ERROR, WARN, INFO, DEBUG, TRACE -k keepdir No argument, if flag is set, will keep working directory + --trace If provided, echo bash commands Examples: @@ -470,6 +469,7 @@ -t) time=$2; verify-not-null time $time; shift ;; -w) workerLogging=$2; verify-is-one-of workerLoggingLevel $workerLogging NONE ERROR WARN INFO DEBUG TRACE; shift ;; -k) keepdir=TRUE ;; + --trace) set -x ;; -d) workdir=$2; verify-not-null workdir $workdir; shift ;; *) usage; exit 1 ;; esac From noreply at svn.ci.uchicago.edu Mon Feb 21 15:22:50 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 15:22:50 -0600 (CST) Subject: [Swift-commit] r4126 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110221212250.A239A9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 15:22:50 -0600 (Mon, 21 Feb 2011) New Revision: 4126 Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: A previous change broke pbsf - checking in a fix Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-21 21:06:15 UTC (rev 4125) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-21 21:22:50 UTC (rev 4126) @@ -623,7 +623,7 @@ echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle if [ $server = pbsf ] then - source $SWIFTRBIN/configure-server-pbsf $cores + source $SWIFTRBIN/configure-server-pbs $cores else source $SWIFTRBIN/configure-server-${server} $cores fi From noreply at svn.ci.uchicago.edu Mon Feb 21 16:15:00 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 16:15:00 -0600 (CST) Subject: [Swift-commit] r4127 - in SwiftApps/SwiftR/Swift: R exec Message-ID: <20110221221500.C8F829CC82@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 16:15:00 -0600 (Mon, 21 Feb 2011) New Revision: 4127 Modified: SwiftApps/SwiftR/Swift/R/Workers.R SwiftApps/SwiftR/Swift/exec/configure-server-cobalt SwiftApps/SwiftR/Swift/exec/configure-server-pbs SwiftApps/SwiftR/Swift/exec/configure-server-sge SwiftApps/SwiftR/Swift/exec/configure-server-ssh SwiftApps/SwiftR/Swift/exec/start-swift Log: Revamped throttle setting to be based on core and node count rather than hardwired. Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-21 21:22:50 UTC (rev 4126) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-21 22:15:00 UTC (rev 4127) @@ -118,8 +118,14 @@ } } - if(is.null(throttle)) + if(is.null(throttle)) { throttle <- getOption("swift.throttle") + if (is.null(throttle)) { + # number of simultaneous jobs / 10: + # Round up to nearest integer + throttle <- as.integer((cores * nodes / 10.0) + 1) + } + } if(! is.null(throttle) ) { cmdString <- paste(cmdString, "-p", shQuote(throttle)) } Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-02-21 21:22:50 UTC (rev 4126) +++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-02-21 22:15:00 UTC (rev 4127) @@ -3,9 +3,8 @@ # configuration for cobalt with manually-started Swift workers (passive coasters) cores=$1 +throttle=$2 -throttle=5.0 # allow approximately 500 concurrent jobs - cat >tc <tc <tc <sites.xml < Modified: SwiftApps/SwiftR/Swift/exec/configure-server-ssh =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-ssh 2011-02-21 21:22:50 UTC (rev 4126) +++ SwiftApps/SwiftR/Swift/exec/configure-server-ssh 2011-02-21 22:15:00 UTC (rev 4127) @@ -1,8 +1,10 @@ #! /bin/bash cores=$1 -time=$2 +throttle=$2 +time=$3 + cat >tc <= 1 + -p throttle 10 >= 1 integer -q queue site speific (PBS, SGE, Cobalt) -r rcmd ssh site specific, SGE only, typically ssh. qrsh for siraf cluster -s server local local, pbs, sge, ssh, pbsf,cobalt (for firewalled worker nodes) @@ -433,8 +433,6 @@ time="00:30:00" nodes=1 queue=short -#throttleOneCore="-0.001" FIXME: Remove -#throttleOneCore="0.00" FIXME: Remove cores=0 defaultLocalCores=2 defaultSshCores=4 @@ -581,7 +579,7 @@ fi echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle - source $SWIFTRBIN/configure-server-ssh $cores $time + source $SWIFTRBIN/configure-server-ssh $cores $throttle $time sshpidfile=${out/stdouterr/workerpids} @@ -623,9 +621,9 @@ echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle if [ $server = pbsf ] then - source $SWIFTRBIN/configure-server-pbs $cores + source $SWIFTRBIN/configure-server-pbs $cores $throttle else - source $SWIFTRBIN/configure-server-${server} $cores + source $SWIFTRBIN/configure-server-${server} $cores $throttle fi jobidfile=${out/stdouterr/jobid} From noreply at svn.ci.uchicago.edu Mon Feb 21 16:54:29 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 16:54:29 -0600 (CST) Subject: [Swift-commit] r4128 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110221225429.D9C569CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 16:54:29 -0600 (Mon, 21 Feb 2011) New Revision: 4128 Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift Log: Disabled .bashrc and .bash_profile processing for the frequently run Eval...Persistent.sh script. Currently correct PATH settings, etc are inherited from the context of worker.pl so it is redundant. This might have to be reconsidered in presence of different execution mechanisms. Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-21 22:15:00 UTC (rev 4127) +++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-21 22:54:29 UTC (rev 4128) @@ -10,7 +10,7 @@ app (external e, RData result, file stout, file sterr) runR (file shellscript, file RServerScript, RData rcall) { - bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; + bash "--noprofile" @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; } app ack (external e[]) From noreply at svn.ci.uchicago.edu Mon Feb 21 17:07:34 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 17:07:34 -0600 (CST) Subject: [Swift-commit] r4129 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110221230734.35A689CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 17:07:34 -0600 (Mon, 21 Feb 2011) New Revision: 4129 Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt SwiftApps/SwiftR/Swift/exec/configure-server-local SwiftApps/SwiftR/Swift/exec/configure-server-pbs SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto SwiftApps/SwiftR/Swift/exec/configure-server-sge SwiftApps/SwiftR/Swift/exec/configure-server-ssh Log: Disabled wrapper log transfers and made sure site directories were cleaned up. Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-02-21 22:54:29 UTC (rev 4128) +++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-02-21 23:07:34 UTC (rev 4129) @@ -38,7 +38,7 @@ cat >cf <' >>sites.xml cat >cf <cf <cf <cf <cf < Author: tga Date: 2011-02-21 17:38:51 -0600 (Mon, 21 Feb 2011) New Revision: 4130 Modified: SwiftApps/SwiftR/Swift/R/Tests.R Log: Minor refactoring of tests. Modified: SwiftApps/SwiftR/Swift/R/Tests.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-21 23:07:34 UTC (rev 4129) +++ SwiftApps/SwiftR/Swift/R/Tests.R 2011-02-21 23:38:51 UTC (rev 4130) @@ -1,11 +1,4 @@ -initSwiftTestOptions <- function() { - options(swift.site="service") - options(swift.keepwork=TRUE) - initcmds <- "initVar1 <- 19; initVar2 <- sqrt(400)+3" - options(swift.initialexpr=initcmds) # Set here; used in test group} - # FIXME: Save prior options here: restore them when tests are done. -} @@ -658,9 +651,11 @@ makeFullTestSuite <- function (...) { initArgs <- list(...) - makeTestSuite( - setup=function () { initSwiftTestOptions(); - do.call(swiftInit, initArgs) }, # swiftInit controlled via options + initArgs[['keepwork']] <- TRUE + makeTestSuite( + setup=function () { options(swift.runmode='service') + options(swift.initialexpr="initVar1 <- 19; initVar2 <- sqrt(400)+3") + do.call(swiftInit, initArgs) }, # swiftInit controlled via options groups=list(testGroup1.1, testGroup2, testGroup1.2, testGroup1.3, testGroup3, testGroup4, testGroup5), teardown=function () { swiftShutdown() }) From noreply at svn.ci.uchicago.edu Mon Feb 21 18:06:43 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 18:06:43 -0600 (CST) Subject: [Swift-commit] r4131 - in SwiftApps/SwiftR/Swift: R exec Message-ID: <20110222000643.A2EDE9CC82@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 18:06:43 -0600 (Mon, 21 Feb 2011) New Revision: 4131 Modified: SwiftApps/SwiftR/Swift/R/Workers.R SwiftApps/SwiftR/Swift/exec/start-swift Log: Made swift logging far less verbose by default. Added an option to reenable it for debugging purposes. Modified: SwiftApps/SwiftR/Swift/R/Workers.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-21 23:38:51 UTC (rev 4130) +++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-02-22 00:06:43 UTC (rev 4131) @@ -5,7 +5,8 @@ parEnv=NULL, kernel=NULL, workmode=NULL, throttle=NULL, queue=NULL, rcmd=NULL, time=NULL, - workerLogging=NULL,keepworkdir=NULL) + workerLogging=NULL,swiftLogging=NULL, + keepworkdir=NULL) { # server: which server backend to use to acquire workers # for example, local runs tasks on the local machine @@ -161,6 +162,12 @@ cmdString <- paste(cmdString, "-w", shQuote(workerLogging)) } + if(is.null(swiftLogging)) + swiftLogging <- getOption("swift.swiftLogging") + if( (! is.null(swiftLogging)) && swiftLogging ) { + cmdString <- paste(cmdString, "-L") + } + if(is.null(keepworkdir)) keepworkdir <- getOption("swift.keepworkdir") if(!is.null(keepworkdir) && keepworkdir) Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-21 23:38:51 UTC (rev 4130) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-22 00:06:43 UTC (rev 4131) @@ -445,6 +445,7 @@ kernel=NONE workdir=NONE workerLogging=ERROR +swiftLoggingFlag="-minimal.logging" keepdir=FALSE rcmd=ssh # rcmd: ssh (typical) or qrsh (eg for siraf with node login restrictions) @@ -466,6 +467,7 @@ -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge cobalt; shift ;; -t) time=$2; verify-not-null time $time; shift ;; -w) workerLogging=$2; verify-is-one-of workerLoggingLevel $workerLogging NONE ERROR WARN INFO DEBUG TRACE; shift ;; + -L) swiftLoggingFlag="" ;; # swift default is lots of logging -k) keepdir=TRUE ;; --trace) set -x ;; -d) workdir=$2; verify-not-null workdir $workdir; shift ;; @@ -656,6 +658,6 @@ fi -$SWIFTRBIN/../swift/bin/swift -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& $out & $out Author: tga Date: 2011-02-21 18:12:22 -0600 (Mon, 21 Feb 2011) New Revision: 4132 Modified: SwiftApps/SwiftR/IMMEDIATE-TODO Log: Updates to TODO list. Modified: SwiftApps/SwiftR/IMMEDIATE-TODO =================================================================== --- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-22 00:06:43 UTC (rev 4131) +++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-22 00:12:22 UTC (rev 4132) @@ -1,7 +1,4 @@ -1) swift bcast if exportAll files/data and/or xmit this via prov staging w/ caching of duplicates -2) Fast branch: working? faster? - 3) speed: where is bottleneck? how to measure and tune? - ParallelCI and ParallelBoostrap benchmarks - smaller focused micro-tests to create plots of speed vs param size and efficiency vs runtime of the evaluated function. @@ -11,33 +8,24 @@ - test on Ranger - get this to Tim Bates to test on Eddie () -5) Timing: dramatic bencmark results for new proposal. VERY HIGH: --- Benchmarks for grant app - grant app is due 21 Feb --- New results by around 11 Feb - - Synthetic benchmark +-- Benchmarks + Timing: dramatic bencmark results for new proposal. - Parallel bootstrap - Parallel CI - - smaller focused micro-tests to create plots of speed vs param size and efficiency vs runtime of the evaluated function. + - smaller focused micro-tests looking at param size - Sarah Kenny FMRI - parallel OpenMx Tests --- Candidate systems: - - Eureka/cobalt - - Beagle? (Prelim: crays crow) - - Sicortex (prob. not) - - BG/P --- Benchmarks on average user machines + - Candidate systems: + - Eureka/cobalt - need to finish up + - Beagle - might be working + - Sicortex (NO) + - BG/P (NO) -HIGH: --- Test pbsf (on PADS?) HIGH: -in swift.properties we should make sure we dont return info logs we should make sure that the amount of logging on both the worker side and the client side is as low as it can go -* Turn off bashrc processing, etc shell script - -HIGH: - tests -- simple calibration tests (n args, arg sizes, durations, etc) -- openmx tests @@ -81,6 +69,17 @@ * Cleanup ssh worker processes: add a watchdog that detects when worker.pl has gone away +MID: +* monitor batch jobs to make sure they launch ok and keep the user in the loop +* restart batch jobs automatically if time runs out + +MID: +* Fast branch: working? faster? + +MID: +Investigate use of swift broadcase: + swift bcast if exportAll files/data and/or xmit this via prov staging w/ caching of duplicates + LOW: (unless needed by immediate OpenMx app or test) - complete sf compat functions (sapply, lapply -> for openMx, based on usage) @@ -108,3 +107,16 @@ - each request provides a reply channel - swift server forks off a worker thread for each request + +========================================================================= +Completed: +========================================================================= +HIGH: +-- Test pbsf (on PADS?) + +HIGH: +-- Check job throttling setting +* Turn off bashrc processing, etc shell script + +HIGH: +in swift.properties we should make sure we dont return info logs we should make sure that the amount of logging on both the worker side and the client side is as low as it can go From noreply at svn.ci.uchicago.edu Mon Feb 21 20:07:07 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 20:07:07 -0600 (CST) Subject: [Swift-commit] r4133 - SwiftApps/SwiftR Message-ID: <20110222020707.0D49E9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 20:07:06 -0600 (Mon, 21 Feb 2011) New Revision: 4133 Modified: SwiftApps/SwiftR/IMMEDIATE-TODO Log: aAdded a couple of changes to the TODO list. Modified: SwiftApps/SwiftR/IMMEDIATE-TODO =================================================================== --- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-22 00:12:22 UTC (rev 4132) +++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-22 02:07:06 UTC (rev 4133) @@ -40,9 +40,13 @@ -- micro studies on provider staging etc. HIGH: - - maybe needed for Bigger parallel bootstrp - Look at all OmxNNN parallel calls - see if any are used that we dont yet handle. +Coaster timeout problem: + He (Mihael) will also look at a better fix to the coaster timeout problem, but for now, you should integrate the timeout change from my trunk/cog/modules/provider-coaster/src/* into your test trunk/ + Otherwise, you'll find that your coaster workers quit after a few minutes of inactivity and then start-swift needs to be killed, workers cleanup up, and start-swift restarted. + + + MID: - saner approach to channels: channel per request to avoid the issue of what happens if a "done" is never read @@ -80,6 +84,9 @@ Investigate use of swift broadcase: swift bcast if exportAll files/data and/or xmit this via prov staging w/ caching of duplicates +MID: +Implement omxXXX parallel calls, update openmx code + LOW: (unless needed by immediate OpenMx app or test) - complete sf compat functions (sapply, lapply -> for openMx, based on usage) @@ -120,3 +127,7 @@ HIGH: in swift.properties we should make sure we dont return info logs we should make sure that the amount of logging on both the worker side and the client side is as low as it can go + +HIGH: + Look at all OmxNNN parallel calls - see if any are used that we dont yet handle. + - It turns out that they are not currently needed From noreply at svn.ci.uchicago.edu Mon Feb 21 20:12:12 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 21 Feb 2011 20:12:12 -0600 (CST) Subject: [Swift-commit] r4134 - SwiftApps/SwiftR Message-ID: <20110222021212.6311C9CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-21 20:12:12 -0600 (Mon, 21 Feb 2011) New Revision: 4134 Modified: SwiftApps/SwiftR/IMMEDIATE-TODO Log: One more note Modified: SwiftApps/SwiftR/IMMEDIATE-TODO =================================================================== --- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-22 02:07:06 UTC (rev 4133) +++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-22 02:12:12 UTC (rev 4134) @@ -45,8 +45,9 @@ He (Mihael) will also look at a better fix to the coaster timeout problem, but for now, you should integrate the timeout change from my trunk/cog/modules/provider-coaster/src/* into your test trunk/ Otherwise, you'll find that your coaster workers quit after a few minutes of inactivity and then start-swift needs to be killed, workers cleanup up, and start-swift restarted. +HIGH: +See email about pbs breaking on UVa cluster - MID: - saner approach to channels: channel per request to avoid the issue of what happens if a "done" is never read From noreply at svn.ci.uchicago.edu Tue Feb 22 10:37:27 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 22 Feb 2011 10:37:27 -0600 (CST) Subject: [Swift-commit] r4135 - text/parco10submission Message-ID: <20110222163727.5CA0D9CC80@svn.ci.uchicago.edu> Author: wilde Date: 2011-02-22 10:37:27 -0600 (Tue, 22 Feb 2011) New Revision: 4135 Modified: text/parco10submission/paper.bib text/parco10submission/paper.tex Log: Made corrections requested by reviewer: x = fixed as suggested, o = was already fixed (not sure what version was reviewed) NOTED: needs more work > p. 12: "It is important to understood" Not in the svn version - what version was this reviewer reading? x > p. 12: becomes"in scope" [space missing] x > p. 12: "The invocation of mappers... are" ['are' should be 'is'] x > p. 13: My dictionary says 'de facto' not de-facto x > p. 13: "underly" should be "underlie" [underly is not a word] o > p. 14: "automatically selection" Not in svn version o > p. 14: semntics Not in svn version x > p. 17: applicatoin x > p. 19: "discoveries...continues to drive" wrong [should be 'continue'] already fixed but changed to match suggestion. o > p. 19: My dictionary says self-contained, not "self contained" already fixed. x > p. 20: line -9: maybe "find N files"? [doesn't make sense to me as is] x > p. 22: line 11: geos[] . should be geos[]. (extraneous space) x > p. 22: "for statement" should be "for the statement" x > p. 22: line -8 "of every input tiles" should be "of every input tile" already fixed, but improved. x > p. 25: 7*27*10=1890 not 1690 (also see p. 26 where 1670 is mentioned) o > p. 25: "These structure" [should be structures] x > p. 25: "selectable energy function to used by" [should be 'to be used > by'] x > Fig. 3: acronym SEM is undefined (also 2nd paragraph of section 5.2) NOTED: WE WANT TO CHANGE THIS FIG > Fig. 4: lower half: color of "Processors" differs left vs. right; why? NOTED: TO REPLACE o > p. 32: "rational" should be "rationale" NOTED: Needs further improvement. o > p. 32: "goals to providing" should be "goals of providing" x > p. 33: line -12 "environment" should be "environments" x NOTED > p. 34: "on single parallel system" - another typo x NOTED: > p. 35: "thousands of small accesses to the filesystem" might not > only be a bottleneck, it may significantly degrade > responsiveness for every process on the system x > p. 38: "J. H G" -> J. H. G." ? Modified: text/parco10submission/paper.bib =================================================================== --- text/parco10submission/paper.bib 2011-02-22 02:12:12 UTC (rev 4134) +++ text/parco10submission/paper.bib 2011-02-22 16:37:27 UTC (rev 4135) @@ -195,15 +195,30 @@ pages = {39--59} } - at article{Futures, + at article{OLDFutures, title = {{The Incremental Garbage Collection of Processes}}, - author = {H G Baker, Jr. and C Hewitt }, + author = {H. G. Baker, Jr. and C Hewitt }, journal = {{Proceedings of Symposium on AI and Programming Languages, ACM SIGPLAN Notices}}, volume = {12(8)}, year = 1977, pages = {55--59} } + at inproceedings{Futures, + author = {Baker,Jr., Henry C. and Hewitt, Carl}, + title = {The incremental garbage collection of processes}, + booktitle = {Proceedings of the 1977 symposium on Artificial intelligence and programming languages}, + year = {1977}, + pages = {55--59}, + numpages = {5}, + url = {http://doi.acm.org/10.1145/800228.806932}, + doi = {http://doi.acm.org/10.1145/800228.806932}, + acmid = {806932}, + publisher = {ACM}, + address = {New York, NY, USA}, + keywords = {Eager evaluation, Garbage collection, Lazy evaluation, Multiprocessing systems, Processor scheduling}, +} + @misc{LONIPIPELINE, title="LONI Pipeline http://pipeline.loni.ucla.edu/" } Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2011-02-22 02:12:12 UTC (rev 4134) +++ text/parco10submission/paper.tex 2011-02-22 16:37:27 UTC (rev 4135) @@ -43,7 +43,7 @@ \author[ci,mcs]{Michael Wilde\corref{cor1}} \ead{wilde at mcs.anl.gov} \author[ci]{Mihael Hategan} \author[mcs]{Justin M. Wozniak} -\author[alcf]{Ben Clifford} +\author[astro]{Ben Clifford} \author[ci]{Daniel S. Katz} \author[ci,mcs,cs]{Ian Foster} @@ -51,8 +51,8 @@ \address[ci]{Computation Institute, University of Chicago and Argonne National Laboratory} \address[mcs]{Mathematics and Computer Science Division, Argonne National Laboratory} -\address[alcf]{Argonne Leadership Computing Facility, Argonne National Laboratory } \address[cs]{Department of Computer Science, University of Chicago } +\address[astro]{Department of Astronomy and Astrophysics, University of Chicago} \begin{abstract} @@ -823,7 +823,7 @@ Swift mappers can operate on files stored on the local machine in the directory where the {\tt swift} command is executing, or they can map any files accessible to the local machine, using absolute pathnames. Custom mappers (and some of the built-in mappers) can also map variables to files specified by URIs for access from remote servers via protocols such as GridFTP or HTTP, as described in section \ref{Execution}. Mappers can interact with structure fields and array elements in a simple and useful manner. New mappers can be added to Swift either as Java classes or as simple, external executable scripts or programs coded in any language. -Mappers can operate both as input mappers (which map files to be processed as application inputs) and as output mappers (which specify the names of files to be produced by applications). It is important to understand that mapping a variable is a different operation from setting the value of a variable. Variables of mapped-file type are mapped (conceptually) when the variable becomes``in scope,'' but they are set when a statement assigns them a value. The invocation of mappers (including external executable mappers) are completely synchronized with the Swift parallel execution model. +Mappers can operate both as input mappers (which map files to be processed as application inputs) and as output mappers (which specify the names of files to be produced by applications). It is important to understand that mapping a variable is a different operation from setting the value of a variable. Variables of mapped-file type are mapped (conceptually) when the variable becomes ``in scope,'' but they are set when a statement assigns them a value. Mapper invocations (and invocations of external mapper executables) are completely synchronized with the Swift parallel execution model. This ability to abstract the processing of files by programs as if they were in-memory objects and to process them with an implicitly parallel programming model is Swift's most valuable and noteworthy contribution. @@ -903,7 +903,7 @@ portability), that they will run in any particular order with respect to other application invocations in a script (except those implied by data dependency), or that their working directories will or will not be -cleaned up after execution. In addition, applications should strive to avoid side effects that could limit both their location independence and the determinism (either actual or de-facto) of the overall results of Swift scripts that call them. +cleaned up after execution. In addition, applications should strive to avoid side effects that could limit both their location independence and the determinism (either actual or de facto) of the overall results of Swift scripts that call them. Consider the following \verb|app| declaration for the \verb|rotate| function: @@ -942,7 +942,7 @@ \section{The Swift runtime environment} \label{Execution} -The Swift runtime environment comprises a set of services providing the parallel, distributed, and reliable execution that underly the simple Swift language model. A notable contribution of Swift is the extent to which the language model has been kept simple by factoring the complexity of these issues out of the language and implementing them in the runtime environment. Notable features of this environment include the following: +The Swift runtime environment comprises a set of services providing the parallel, distributed, and reliable execution that underlie the simple Swift language model. A notable contribution of Swift is the extent to which the language model has been kept simple by factoring the complexity of these issues out of the language and implementing them in the runtime environment. Notable features of this environment include the following: \begin{itemize} @@ -1078,7 +1078,7 @@ provides a natural way to deal both with many transient errors, such as temporary network loss, and with many changes in site state. -Some errors are more permanent; for example, an applicatoin +Some errors are more permanent; for example, an application program may have a bug that causes it to always fail when given a particular set of inputs. In this case, Swift's retry mechanism will not help; each job will be tried a number of times, and each @@ -1172,8 +1172,8 @@ time and with respect to themselves at other times. The load that a particular site will bear varies over time, and sites can fail in unusual ways. Swift's site scoring mechanism deals well with this situation in -the majority of cases. However, discovery of new and unusual failure -modes continues to drive the implementation of increasingly robust fault tolerance +the majority of cases. However, discoveries of new and unusual failure +modes continue to drive the implementation of increasingly robust fault tolerance mechanisms. When running jobs on dynamically discovered sites, it is likely that @@ -1185,7 +1185,7 @@ files are described as mapped input files in the same way as input data files and are passed as a parameter to the application executable. Swift's existing input file management then -stages-in the application files once per site per run. +stages in the application files once per site per run. \section{Applications} \label{Applications} @@ -1228,9 +1228,9 @@ 5 million 1-byte pixels (5.7 MB), covering 2400 x 2400 250-meter squares, based on a specific map projection. -The Swift script analyzes the dataset to find the files with the N -largest total area of any requested sets of land-cover typesr It then produces a new dataset with viewable -color images of those closest-matching data tiles. +The Swift script analyzes the dataset to select the top N files ranked by +total area of specified sets of land-cover types. It then produces a new dataset with viewable +color images of those selected data tiles. (A color rendering step is required, since the input datasets are not viewable images; their pixel values are land-use codes.) A typical invocation of this script would be ``\emph{Find the top 12 urban tiles}'' or ``\emph{Find the 16 tiles with the most forest and grassland.}'' Since this script is used for tutorial purposes, the application programs it calls are simple shell scripts that use fast, generic image-processing applications to process the MODIS data. Thus, the example executes quickly while serving as a realistic tutorial script for much more compute-intensive satellite data-processing applications. @@ -1258,15 +1258,15 @@ At lines 55--57 the script performs its first computation using a {\tt foreach} loop to invoke {\tt getLandUse} in parallel on each file mapped to the elements of {\tt geos[]}. Since 317 files were mapped (in lines 47--48), the loop will submit 317 instances of the application in parallel to the execution provider. These will execute with a degree of parallelism subject to available resources. %\katznote{DONE: is this strictly true? Do you want to say that it will enable 317 instances to be runnable in parallel, but the number that are actually run in parallel depends on the hardware available to Swift, or something like that?} -At lines 52--53 the result of each computation is placed in a file mapped to the array {\tt land} and named by the regular expression translation based on the file names mapped to {\tt geos[]} . +At lines 52--53 the result of each computation is placed in a file mapped to the array {\tt land} and named by the regular expression translation based on the file names mapped to {\tt geos[]}. Thus the landuse histogram for file {\tt h00v08.tif} would be written into file {\tt h00v08.landuse.freq} and would be considered by Swift to be of type {\tt landuse}. Once all the land usage histograms have been computed, the script can then execute {\tt analyzeLandUse} at line 63 to find the requested number of highest tiles (files) with a specific land cover combination. The Swift runtime system uses futures to ensure that this analysis function is not invoked until all of its input files have computed and transported to the computation site chosen to run the analysis program. All of these steps take place automatically, using the relatively simple and location-independent Swift expressions shown. The output files to be used for the result are specified in the declarations at lines 61--62. % \katznote{DONE: should these lines have a space inserted before the ``<'' to match the previous lines? Same question for 67-68... } -To visualize the results, the application function {\tt markMap} invoked at line 68 will generate an image of a world map using the MODIS projection system and indicate the selected tiles matching the analysis criteria. Since this statement depends on the output of the analysis ({\tt topSelected}), it will wait for statement at line 63 to complete before commencing. +To visualize the results, the application function {\tt markMap} invoked at line 68 will generate an image of a world map using the MODIS projection system and indicate the selected tiles matching the analysis criteria. Since this statement depends on the output of the analysis ({\tt topSelected}), it will wait for the statement at line 63 to complete before commencing. -For additional visualization, the script assembles a full map of all the input tiles, placed in their proper grid location on the MODIS world map projection, and again marking the selected tiles. Since this operation needs true-color images of every input tiles these are computed---again in parallel---with 317 jobs generated by the foreach statement at lines 76--78. The power of Swift's implicit parallelization is shown vividly here: since the {\tt colorMODIS} call at line 77 depends only on the input array {\tt geos}, these 317 application invocations are submitted in parallel with the initial 317 parallel executions of the {\tt getLandUse} application at line 56. The script concludes at line 83 by assembling a montage of all the colored tiles and writing this image file to a web-accessible directory for viewing. +For additional visualization, the script assembles a full map of all the input tiles, placed in their proper grid location on the MODIS world map projection, and with the selected tiles marked. Since this operation needs true-color images of every input tiles these are computed---again in parallel---with 317 jobs generated by the foreach statement at lines 76--78. The power of Swift's implicit parallelization is shown vividly here: since the {\tt colorMODIS} call at line 77 depends only on the input array {\tt geos}, these 317 application invocations are submitted in parallel with the initial 317 parallel executions of the {\tt getLandUse} application at line 56. The script concludes at line 83 by assembling a montage of all the colored tiles and writing this image file to a web-accessible directory for viewing. \pagebreak ~ @@ -1304,7 +1304,7 @@ describing which particles are in the cavity. Each script run covers a simulation space of 7 radii by 27 centers by -10 models, requiring 1,690 jobs per run. Three model systems are +10 models, requiring 1,890 jobs per run. Three model systems are investigated for a total of 90 runs. Swift mappers enable metadata describing these aspects to be encoded in the data files of the simulation campaigns to assist in managing the large volume of file data. @@ -1320,11 +1320,11 @@ The single application called by this script is the {\tt glassRun} program wrapped in the app function at lines 21--29. Note that rather than defining main program logic in ``open" (top-level) code, the script places all the program logic in the function {\tt GlassRun}, invoked by the single statement at line 80. This approach enables the simulation script to be defined in a library that can be imported into other Swift scripts to perform entire campaigns or campaign subsets. The {\tt GlassRun} function starts by extracting a large set of science parameters from the Swift command line at lines 33--48 using the {\tt @arg()} function. It uses the built-in function {\tt readData} at lines 42--43 to read prepared lists of molecular radii and centroids from parameter files to define the primary physical dimensions of the simulation space. -A selectable energy function to used by the simulation application is specified as a parameter at line 48. +A selectable energy function to be used by the simulation application is specified as a parameter at line 48. At lines 57 and 61, the script leverages Swift flexible dynamic arrays to create a 3D array for input and a 4D array of structures for outputs. These data structures, whose leaf elements consist entirely of mapped files, are set by using the external mappers specified for the input array at lines 57--59 and for the output array of structures at lines 61--63. Note that many of the science parameters are passed to the mappers, which in turn are used by the input mapper to locate files within the large, multilevel directory structure of the campaign and by the output mapper to create new directory and file naming conventions for the campaign outputs. The mappers apply the common, useful practice of using scientific metadata to determine directory and file names. -The entire body of the {\tt GlassRun} is a four-level nesting of \verb|foreach| statements at lines 65--77. These loops perform a parallel parameter sweep over all combinations of radius, centroid, model, and job number within the simulation space. A single run of the script immediately expands to an independent parallel invocation of the simulation application for each point in the space: 1,670 jobs for the minimum case of a 7 x 27 x 10 x 1 space. Note that the {\tt if} statement at line 69 causes the simulation execution to be skipped if it has already been performed, as determined by a ``\verb|NULL|'' file name returned by the mapper for the output of a given job in the simulation space. In the current campaign the fourth dimension ({\tt nsub}) of the simulation space is fixed at one. This value could be increased to define subconfigurations that would perform better Monte Carlo averaging, with a multiplicative increase in the number of jobs. This is currently set to one because there are ample starting configurations, but if this was not the case (as in earlier campaigns) the script could run repeated simulations with different random seeds. +The entire body of the {\tt GlassRun} is a four-level nesting of \verb|foreach| statements at lines 65--77. These loops perform a parallel parameter sweep over all combinations of radius, centroid, model, and job number within the simulation space. A single run of the script immediately expands to an independent parallel invocation of the simulation application for each point in the space: 1,890 jobs for the minimum case of a 7 x 27 x 10 x 1 space. Note that the {\tt if} statement at line 69 causes the simulation execution to be skipped if it has already been performed, as determined by a ``\verb|NULL|'' file name returned by the mapper for the output of a given job in the simulation space. In the current campaign the fourth dimension ({\tt nsub}) of the simulation space is fixed at one. This value could be increased to define subconfigurations that would perform better Monte Carlo averaging, with a multiplicative increase in the number of jobs. This is currently set to one because there are ample starting configurations, but if this was not the case (as in earlier campaigns) the script could run repeated simulations with different random seeds. The advantages of managing a simulation campaign in this manner are borne out well by Hocky's experience: the expression of the campaign is a well-structured, high-level script, devoid of details about file naming, synchronization of parallel tasks, location and state of remote computing resources, or explicit explicit data transfer. Hocky was able to leverage local cluster resources on many occasions, but at any time he could count on his script's acquiring on the order of 1,000 compute cores from 6 to 18 sites of the Open Science Grid. When executing on the OSG, he leveraged Swift's capability to replicate jobs that were waiting in queues at more congested sites, and automatically sent them to sites where resources were available and jobs were being processed at better rates. All these actions would have represented a huge distraction from his primary scientific simulation campaign if he had been required to use or to script lower-level abstractions where parallelism and r emote distribution were the manual responsibility of the programmer. @@ -1425,7 +1425,7 @@ Previously published measurements of Swift performance on several scientific applications provide evidence that its parallel distributed programming model can be implemented with sufficient scalability and efficiency to make it a practical tool for large-scale parallel application scripting. -The performance of Swift submitting jobs over the wide-area network from the University of Chicago to the teraGrid Ranger cluster at TACC is shown in Figure~\ref{SEMplots} (from \cite{CNARI_2009}). The figure plots an SEM workload of 131,072 jobs for four brain regions and two experimental conditions. This workflow completed in approximately 3 hours. The logs from the {\tt swift\_plot\_log} utility show the high degree of concurrent overlap between job execution and input and output file staging to remote computing resources. +The performance of Swift submitting jobs over the wide-area network from the University of Chicago to the teraGrid Ranger cluster at TACC is shown in Figure~\ref{SEMplots} (from \cite{CNARI_2009}). The figure plots a structural equation modeling (SEM) workload of 131,072 jobs for four brain regions and two experimental conditions. This workflow completed in approximately 3 hours. The logs from the {\tt swift\_plot\_log} utility show the high degree of concurrent overlap between job execution and input and output file staging to remote computing resources. The workflows were developed on and submitted (to Ranger) from a single-core Linux workstation at the University of Chicago running an Intel Xeon 3.20 GHz CPU. Data staging was performed by using the Globus GridFTP protocol, and job execution was performed over the Globus GRAM~2 protocol. During the third hour of the workflow, Swift achieved very high utilization of the 2,048 allocated processor cores and a steady rate of input and output transfers. The first two hours of the run were more bursty, because of fluctuating grid conditions and data server loads. @@ -1506,7 +1506,7 @@ %% tasks. We use the term workflow as defined by (Taylor %% et. al. 2006). So we often call a Swift script a workflow. -% Ousterhout in (Ousterhout 1998) eloquently laid out the rational and +% Ousterhout in (Ousterhout 1998) eloquently laid out the rationale and % motivation for scripting languages. As the creator of Tcl [ref], he % described here the difference between programming and scripting, and % the place of each in the scheme of applying computers to solving @@ -1517,6 +1517,10 @@ the scheme of applying computers to solving problems have previously been presented~\cite{Scripting_1998}. +%%% ^^^^^^ IMPROVE THIS; dont separate scripting from programming....^^^^ + +%%% Review the text below VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV + Coordination languages and systems such as Linda~\cite{LINDA}, Strand~\cite{STRAND_1989}, and PCN~\cite{PCN_1993} allow composition of distributed or parallel components, but they usually require the components @@ -1551,8 +1555,8 @@ programming tool for the specification and execution of large parallel computations on large quantities of data and facilitating the utilization of large distributed resources. However,the two -differ in many aspects. The -MapReduce programming model supports key-value pairs as +differ in many aspects. +The MapReduce programming model supports key-value pairs as input or output datasets and two types of computation functions, map and reduce; Swift provides a type system and allows the definition of complex data structures and arbitrary computational @@ -1625,8 +1629,10 @@ Walker et al.~\cite{DataFlowShell} have recently developed extensions to BASH that allow a user to define a dataflow graph, including the concepts -of fork, join, cycles, and key-value aggregation, but execute on single parallel system or cluster. +of fork, join, cycles, and key-value aggregation, but which execute on single parallel systems or clusters. +%%% No data types or mapping ^^^; Check latest state of this re the "executes on" point. + A few groups have been working on parallel and distributed versions of make~\cite{GXPmake, makeflow}. These tools use the concept of ``virtual data,'' where the user defines the processing by which data is created and then calls for the final data product. The make-like tools determine what processing is needed to get from the existing files to the final product, which includes running processing tasks. If this is run on a distributed system, data movement also must be handled by the tools. In comparison, Swift is a language, which may be slightly @@ -1675,6 +1681,11 @@ are developing a set of collective data management primitives ~\cite{CDM_2009}. +% ^^^ reviewer points out: +% > p. 35: "thousands of small accesses to the filesystem" might not +% > only be a bottleneck, it may significantly degrade +% > responsiveness for every process on the system + \subsection{Provenance} \label{Provenance} From noreply at svn.ci.uchicago.edu Tue Feb 22 10:57:24 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 22 Feb 2011 10:57:24 -0600 (CST) Subject: [Swift-commit] r4136 - text/parco10submission Message-ID: <20110222165724.D7D319CC82@svn.ci.uchicago.edu> Author: dsk Date: 2011-02-22 10:57:24 -0600 (Tue, 22 Feb 2011) New Revision: 4136 Modified: text/parco10submission/paper.tex Log: fixing missing reference to Section 5 at end of Section 1 Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2011-02-22 16:37:27 UTC (rev 4135) +++ text/parco10submission/paper.tex 2011-02-22 16:57:24 UTC (rev 4136) @@ -233,7 +233,9 @@ implementation, including the distributed architecture that enables applications to run on distributed resources. Section~\ref{Applications} describes real-world applications using -Swift on scientific projects. Section~\ref{Related} relates Swift to +Swift on scientific projects. Section~\ref{Performance} +provides performance results. +Section~\ref{Related} relates Swift to other systems. Section~\ref{Future} highlights ongoing and future work in the Swift project. Section~\ref{Conclusion} offers @@ -1339,9 +1341,9 @@ %\end{Verbatim} %\end{verbatim} -\section{Performance characteristics} -\label{Performance} +\section{Performance characteristics\label{Performance}} + We present here a few additional measurements of Swift performance and highlight a few previously published results. \subsection{Synthetic benchmark results} From noreply at svn.ci.uchicago.edu Tue Feb 22 13:35:00 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 22 Feb 2011 13:35:00 -0600 (CST) Subject: [Swift-commit] r4137 - SwiftApps/SwiftR Message-ID: <20110222193500.5F8239CC82@svn.ci.uchicago.edu> Author: tga Date: 2011-02-22 13:35:00 -0600 (Tue, 22 Feb 2011) New Revision: 4137 Modified: SwiftApps/SwiftR/IMMEDIATE-TODO Log: TODO update. Modified: SwiftApps/SwiftR/IMMEDIATE-TODO =================================================================== --- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-22 16:57:24 UTC (rev 4136) +++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-22 19:35:00 UTC (rev 4137) @@ -3,17 +3,13 @@ - ParallelCI and ParallelBoostrap benchmarks - smaller focused micro-tests to create plots of speed vs param size and efficiency vs runtime of the evaluated function. -4) SGE: -- test in IBI -- test on Ranger -- get this to Tim Bates to test on Eddie () VERY HIGH: -- Benchmarks Timing: dramatic bencmark results for new proposal. - - Parallel bootstrap + - Parallel bootstrap DONE - Parallel CI - smaller focused micro-tests looking at param size - Sarah Kenny FMRI @@ -38,9 +34,13 @@ -- swift fast branch? -- swift times? -- micro studies on provider staging etc. - + HIGH: +- test on Ranger + +HIGH: + Coaster timeout problem: He (Mihael) will also look at a better fix to the coaster timeout problem, but for now, you should integrate the timeout change from my trunk/cog/modules/provider-coaster/src/* into your test trunk/ Otherwise, you'll find that your coaster workers quit after a few minutes of inactivity and then start-swift needs to be killed, workers cleanup up, and start-swift restarted. From noreply at svn.ci.uchicago.edu Wed Feb 23 16:48:57 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 23 Feb 2011 16:48:57 -0600 (CST) Subject: [Swift-commit] r4138 - in branches/release-0.92/tests/providers: . local-sge-coasters Message-ID: <20110223224857.2A5699CC92@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-23 16:48:54 -0600 (Wed, 23 Feb 2011) New Revision: 4138 Added: branches/release-0.92/tests/providers/local-sge-coasters/ branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.check.sh branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.setup.sh branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.swift branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.timeout branches/release-0.92/tests/providers/local-sge-coasters/catsn.0001.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0002.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0003.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0004.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0005.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0006.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0007.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0008.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0009.out.expected branches/release-0.92/tests/providers/local-sge-coasters/catsn.0010.out.expected branches/release-0.92/tests/providers/local-sge-coasters/data.txt branches/release-0.92/tests/providers/local-sge-coasters/sites.template.xml branches/release-0.92/tests/providers/local-sge-coasters/tc.template.data branches/release-0.92/tests/providers/local-sge-coasters/title.txt Log: for ranger testing Added: branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.check.sh =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.check.sh (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.check.sh 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +for count in `seq --format "%04.f" 1 1 10` +do + [ -f catsn.$count.out ] || exit 1 + CONTENTS1=$( cat catsn.$count.out.expected ) + CONTENTS2=$( cat catsn.$count.out ) + [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1 +done +exit 0 Property changes on: branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.check.sh ___________________________________________________________________ Name: svn:executable + * Added: branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.setup.sh =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.setup.sh (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.setup.sh 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1,4 @@ +#!/bin/bash + +cp -v $GROUP/data.txt . || exit 1 +cp -v $GROUP/*expected . || exit 1 Property changes on: branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.swift =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.swift (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.swift 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1,15 @@ +type file; + +app (file o) cat (file i) +{ + cat @i stdout=@o; +} + +string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +string char[] = @strsplit(t, ""); + +file out[]; +foreach j in [1:@toint(@arg("n","10"))] { + file data<"data.txt">; + out[j] = cat(data); +} Added: branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.timeout =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.timeout (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/001-catsn-local-sge-coasters.timeout 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1,2 @@ +7200 + Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0001.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0001.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0001.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0002.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0002.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0002.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0003.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0003.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0003.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0004.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0004.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0004.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0005.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0005.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0005.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0006.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0006.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0006.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0007.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0007.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0007.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0008.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0008.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0008.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0009.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0009.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0009.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/catsn.0010.out.expected =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/catsn.0010.out.expected (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/catsn.0010.out.expected 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/data.txt =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/data.txt (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/data.txt 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Hello world Added: branches/release-0.92/tests/providers/local-sge-coasters/sites.template.xml =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/sites.template.xml (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/sites.template.xml 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1,17 @@ + + + + + 1 + 7200 + 1 + 1 + 1 + 1 + _QUEUE_ + 5.99 + 10000 + _PROJECT_ + _WORK_ + + Added: branches/release-0.92/tests/providers/local-sge-coasters/tc.template.data =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/tc.template.data (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/tc.template.data 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1,8 @@ +local-sge-coasters echo /bin/echo INSTALLED INTEL32::LINUX null +local-sge-coasters cat /bin/cat INSTALLED INTEL32::LINUX null +local-sge-coasters ls /bin/ls INSTALLED INTEL32::LINUX null +local-sge-coasters grep /bin/grep INSTALLED INTEL32::LINUX null +local-sge-coasters sort /bin/sort INSTALLED INTEL32::LINUX null +local-sge-coasters paste /bin/paste INSTALLED INTEL32::LINUX null +local-sge-coasters wc /usr/bin/wc INSTALLED INTEL32::LINUX null + Added: branches/release-0.92/tests/providers/local-sge-coasters/title.txt =================================================================== --- branches/release-0.92/tests/providers/local-sge-coasters/title.txt (rev 0) +++ branches/release-0.92/tests/providers/local-sge-coasters/title.txt 2011-02-23 22:48:54 UTC (rev 4138) @@ -0,0 +1 @@ +Local PBS with Coasters Configuration Test From noreply at svn.ci.uchicago.edu Wed Feb 23 16:50:01 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 23 Feb 2011 16:50:01 -0600 (CST) Subject: [Swift-commit] r4139 - branches/release-0.92/tests/groups Message-ID: <20110223225001.91BA19CC92@svn.ci.uchicago.edu> Author: skenny Date: 2011-02-23 16:50:01 -0600 (Wed, 23 Feb 2011) New Revision: 4139 Added: branches/release-0.92/tests/groups/local-sge-coasters.sh Log: ranger testing Added: branches/release-0.92/tests/groups/local-sge-coasters.sh =================================================================== --- branches/release-0.92/tests/groups/local-sge-coasters.sh (rev 0) +++ branches/release-0.92/tests/groups/local-sge-coasters.sh 2011-02-23 22:50:01 UTC (rev 4139) @@ -0,0 +1,6 @@ + +# GROUPLIST definition to run sge tests + +GROUPLIST=( + $TESTDIR/providers/local-sge-coasters \ +) From noreply at svn.ci.uchicago.edu Thu Feb 24 16:49:20 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 24 Feb 2011 16:49:20 -0600 (CST) Subject: [Swift-commit] r4140 - in SwiftApps/SwiftR: . Swift Message-ID: <20110224224920.565F49CC82@svn.ci.uchicago.edu> Author: tga Date: 2011-02-24 16:49:20 -0600 (Thu, 24 Feb 2011) New Revision: 4140 Modified: SwiftApps/SwiftR/IMMEDIATE-TODO SwiftApps/SwiftR/Swift/DESCRIPTION Log: Checking in 0.1.4 release Modified: SwiftApps/SwiftR/IMMEDIATE-TODO =================================================================== --- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-23 22:50:01 UTC (rev 4139) +++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-02-24 22:49:20 UTC (rev 4140) @@ -94,6 +94,12 @@ LOW: - run modes -- workspace stays up, server(s) are transient-> current + ?? Protocol: + - SwiftR detects that there is a job still runing, doesn't + shut down server on exit + - SwiftR removes result pipe + - Swift script writes a done message to result directory + and then exits loop -- workspace is run as a job under batch scheduler -> new feature LOW: Modified: SwiftApps/SwiftR/Swift/DESCRIPTION =================================================================== --- SwiftApps/SwiftR/Swift/DESCRIPTION 2011-02-23 22:50:01 UTC (rev 4139) +++ SwiftApps/SwiftR/Swift/DESCRIPTION 2011-02-24 22:49:20 UTC (rev 4140) @@ -1,8 +1,8 @@ Package: Swift Type: Package Title: R interface to Swift parallel scripting languaage -Version: 0.1.3 -Date: 2010-02-16 +Version: 0.1.4 +Date: 2010-02-24 Author: Michael Wilde Maintainer: Michael Wilde Description: Routines to invoke R functions on remote resources through Swift. From noreply at svn.ci.uchicago.edu Thu Feb 24 17:00:09 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 24 Feb 2011 17:00:09 -0600 (CST) Subject: [Swift-commit] r4141 - SwiftApps/SwiftR/Swift Message-ID: <20110224230009.588A99CC82@svn.ci.uchicago.edu> Author: tga Date: 2011-02-24 17:00:09 -0600 (Thu, 24 Feb 2011) New Revision: 4141 Removed: SwiftApps/SwiftR/Swift/tests/ Log: Removed unused tests directory. From noreply at svn.ci.uchicago.edu Thu Feb 24 17:25:51 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 24 Feb 2011 17:25:51 -0600 (CST) Subject: [Swift-commit] r4142 - in SwiftApps/SwiftR/Swift: . exec tests Message-ID: <20110224232551.0FDE29CC82@svn.ci.uchicago.edu> Author: tga Date: 2011-02-24 17:25:50 -0600 (Thu, 24 Feb 2011) New Revision: 4142 Added: SwiftApps/SwiftR/Swift/tests/ SwiftApps/SwiftR/Swift/tests/runtests.R Removed: SwiftApps/SwiftR/Swift/tests/runtests.R Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/rserver.swift Log: Oops I shouldn't have deleted that. Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-24 23:00:09 UTC (rev 4141) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-24 23:25:50 UTC (rev 4142) @@ -83,6 +83,31 @@ RPIDS= +TIMEOUT=5 + +function start_timeout { + ppid=$$ + trap "timeout_handler" SIGHUP + ( + sleep $TIMEOUT + kill -1 $ppid &> /dev/null #SIGHUP + ) & +} + +function stop_timeout { + trap "" SIGHUP +} + +function timeout_handler { + echo 'Timed out waiting to contact R process' + echo 'R log follows:' + echo '=====================' + cat $SLOTDIR/R.log + exit 1 +} + + + # Try to make slotdir. # If the mkdir succeeds, this is the first request to the slot, # so we create a new R server and send the current request to it; @@ -99,18 +124,18 @@ #which Rscript >> $SLOTDIR/R.log echo $PATH $RServerScript $SLOTDIR >> $SLOTDIR/R.log 2>&1 & # launch R server + start_timeout # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" else # wait to make sure fifo exists # fromR fifo is created last, so wait for that one - if [ ! -p $SLOTDIR/fromR.fifo ]; then - sleep 5 # FIXME: try a few times, or wait longer? - if [ ! -p $SLOTDIR/fromR.fifo ]; then - echo "$0: ERROR: FIFO $SLOTDIR/fromR.fifo did not appear within 5 seconds.\n" - exit 1 + start_timeout + while 1; do + if [ -p $SLOTDIR/fromR.fifo ]; then + break fi - fi + done fi # Ready to talk to the server: send request and read response @@ -130,7 +155,7 @@ touch $SLOTDIR/lastwrite echo DB: Sent request - + stop_timeout # started up ok echo dummy stderr response 1>&2 # FIXME - testing if this is the provider staging problem (not xfering zero len stderr) res=$(cat < $SLOTDIR/fromR.fifo) Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-24 23:00:09 UTC (rev 4141) +++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-24 23:25:50 UTC (rev 4142) @@ -10,7 +10,7 @@ app (external e, RData result, file stout, file sterr) runR (file shellscript, file RServerScript, RData rcall) { - bash "--noprofile" @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; + bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; } app ack (external e[]) Copied: SwiftApps/SwiftR/Swift/tests (from rev 4140, SwiftApps/SwiftR/Swift/tests) Deleted: SwiftApps/SwiftR/Swift/tests/runtests.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/runtests.R 2011-02-24 22:49:20 UTC (rev 4140) +++ SwiftApps/SwiftR/Swift/tests/runtests.R 2011-02-24 23:25:50 UTC (rev 4142) @@ -1,5 +0,0 @@ -#!/usr/bin/env Rscript - -library(Swift) -#TODO: take command line options to setup options -runAllSwiftTests() Copied: SwiftApps/SwiftR/Swift/tests/runtests.R (from rev 4140, SwiftApps/SwiftR/Swift/tests/runtests.R) =================================================================== --- SwiftApps/SwiftR/Swift/tests/runtests.R (rev 0) +++ SwiftApps/SwiftR/Swift/tests/runtests.R 2011-02-24 23:25:50 UTC (rev 4142) @@ -0,0 +1,5 @@ +#!/usr/bin/env Rscript + +library(Swift) +#TODO: take command line options to setup options +runAllSwiftTests() From noreply at svn.ci.uchicago.edu Thu Feb 24 17:27:03 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 24 Feb 2011 17:27:03 -0600 (CST) Subject: [Swift-commit] r4143 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110224232703.C4FDA9CC82@svn.ci.uchicago.edu> Author: tga Date: 2011-02-24 17:27:03 -0600 (Thu, 24 Feb 2011) New Revision: 4143 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/rserver.swift Log: oops... didn't mean to check in these changes yet. Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-24 23:25:50 UTC (rev 4142) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-24 23:27:03 UTC (rev 4143) @@ -83,31 +83,6 @@ RPIDS= -TIMEOUT=5 - -function start_timeout { - ppid=$$ - trap "timeout_handler" SIGHUP - ( - sleep $TIMEOUT - kill -1 $ppid &> /dev/null #SIGHUP - ) & -} - -function stop_timeout { - trap "" SIGHUP -} - -function timeout_handler { - echo 'Timed out waiting to contact R process' - echo 'R log follows:' - echo '=====================' - cat $SLOTDIR/R.log - exit 1 -} - - - # Try to make slotdir. # If the mkdir succeeds, this is the first request to the slot, # so we create a new R server and send the current request to it; @@ -124,18 +99,18 @@ #which Rscript >> $SLOTDIR/R.log echo $PATH $RServerScript $SLOTDIR >> $SLOTDIR/R.log 2>&1 & # launch R server - start_timeout # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" else # wait to make sure fifo exists # fromR fifo is created last, so wait for that one - start_timeout - while 1; do - if [ -p $SLOTDIR/fromR.fifo ]; then - break + if [ ! -p $SLOTDIR/fromR.fifo ]; then + sleep 5 # FIXME: try a few times, or wait longer? + if [ ! -p $SLOTDIR/fromR.fifo ]; then + echo "$0: ERROR: FIFO $SLOTDIR/fromR.fifo did not appear within 5 seconds.\n" + exit 1 fi - done + fi fi # Ready to talk to the server: send request and read response @@ -155,7 +130,7 @@ touch $SLOTDIR/lastwrite echo DB: Sent request - stop_timeout # started up ok + echo dummy stderr response 1>&2 # FIXME - testing if this is the provider staging problem (not xfering zero len stderr) res=$(cat < $SLOTDIR/fromR.fifo) Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-24 23:25:50 UTC (rev 4142) +++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-24 23:27:03 UTC (rev 4143) @@ -10,7 +10,7 @@ app (external e, RData result, file stout, file sterr) runR (file shellscript, file RServerScript, RData rcall) { - bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; + bash "--noprofile" @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; } app ack (external e[]) From noreply at svn.ci.uchicago.edu Fri Feb 25 09:45:04 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 25 Feb 2011 09:45:04 -0600 (CST) Subject: [Swift-commit] r4144 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110225154504.042899CC82@svn.ci.uchicago.edu> Author: tga Date: 2011-02-25 09:45:03 -0600 (Fri, 25 Feb 2011) New Revision: 4144 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/rserver.swift Log: Partially implemented timeout when Rscript fails to launch on worker. Now need to pass the error from start-swift to R Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-24 23:27:03 UTC (rev 4143) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-25 15:45:03 UTC (rev 4144) @@ -3,6 +3,8 @@ # Arguments: inputBatchSaveFile outputBatchSaveFile # bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; +set -x + # Set restrictive umask for duration of script # This value prevents any other users from reading # or writing @@ -83,6 +85,39 @@ RPIDS= +TIMEOUT=5 + +function start_timeout { + ppid=$$ + echo start_timeout + trap "timeout_handler" SIGHUP + ( + echo timeout_handler for $ppid sleeping + sleep ${TIMEOUT}s + echo timeout_handler work up + kill -1 $ppid &> /dev/null #SIGHUP + if [ "$1" != "" ]; then + kill -1 $1 &> /dev/null #SIGHUP + fi + ) & +} + +function stop_timeout { + #DEBUG + echo stop_timeout + trap "" SIGHUP +} + +function timeout_handler { + echo 'Timed out waiting to contact R process' + echo 'R log follows:' + echo '=====================' + cat $SLOTDIR/R.log + exit 1 +} + + + # Try to make slotdir. # If the mkdir succeeds, this is the first request to the slot, # so we create a new R server and send the current request to it; @@ -104,17 +139,16 @@ else # wait to make sure fifo exists # fromR fifo is created last, so wait for that one - if [ ! -p $SLOTDIR/fromR.fifo ]; then - sleep 5 # FIXME: try a few times, or wait longer? - if [ ! -p $SLOTDIR/fromR.fifo ]; then - echo "$0: ERROR: FIFO $SLOTDIR/fromR.fifo did not appear within 5 seconds.\n" - exit 1 + while 1; do + if [ -p $SLOTDIR/fromR.fifo ]; then + break fi - fi + done fi # Ready to talk to the server: send request and read response +start_timeout while true; do mkdir $SLOTDIR/mutex if [ $? != 0 ]; then @@ -123,14 +157,20 @@ break; fi done +stop_timeout echo DB: Obtained $SLOTDIR/mutex -if echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo +echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo & +echopid=$! +echo echopid $echopid +start_timeout $echopid + +if wait $echopid then touch $SLOTDIR/lastwrite echo DB: Sent request - + stop_timeout # started up ok echo dummy stderr response 1>&2 # FIXME - testing if this is the provider staging problem (not xfering zero len stderr) res=$(cat < $SLOTDIR/fromR.fifo) @@ -146,6 +186,7 @@ echo DB: Freed $SLOTDIR/mutex else + stop_timeout echo "ERROR: Could not write to fifo ok" rmdir $SLOTDIR/mutex Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-24 23:27:03 UTC (rev 4143) +++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-02-25 15:45:03 UTC (rev 4144) @@ -10,7 +10,7 @@ app (external e, RData result, file stout, file sterr) runR (file shellscript, file RServerScript, RData rcall) { - bash "--noprofile" @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; + bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; } app ack (external e[]) From noreply at svn.ci.uchicago.edu Fri Feb 25 16:50:13 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 25 Feb 2011 16:50:13 -0600 (CST) Subject: [Swift-commit] r4145 - in SwiftApps/SwiftR/Swift: R exec Message-ID: <20110225225013.6AEA09CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-25 16:50:13 -0600 (Fri, 25 Feb 2011) New Revision: 4145 Modified: SwiftApps/SwiftR/Swift/R/Apply.R SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/start-swift Log: Finished adding in detection of when R worker process fails to start up. Modified: SwiftApps/SwiftR/Swift/R/Apply.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-25 15:45:03 UTC (rev 4144) +++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-02-25 22:50:13 UTC (rev 4145) @@ -91,7 +91,12 @@ res <- getServiceResponse(resultPipeName, timeout) # Check that the message was correct - if (res[[1]] != "done") { + if (substring(res[[1]], 1, 6) == "error:") { + stop(paste("Received error message from Swift '", res, + "'. The swift server has shut down, rerun swiftInit to" + ," restart", sep="")) + } + else if (res[[1]] != "done") { stop(paste("Got unexpected message '", paste(res, collapse="\n"),"' on fifo ", "aborting job", sep="")) Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-25 15:45:03 UTC (rev 4144) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-25 22:50:13 UTC (rev 4145) @@ -3,7 +3,6 @@ # Arguments: inputBatchSaveFile outputBatchSaveFile # bash @shellscript @RServerScript @rcall @result stdout=@stout stderr=@sterr; -set -x # Set restrictive umask for duration of script # This value prevents any other users from reading @@ -89,12 +88,9 @@ function start_timeout { ppid=$$ - echo start_timeout trap "timeout_handler" SIGHUP ( - echo timeout_handler for $ppid sleeping sleep ${TIMEOUT}s - echo timeout_handler work up kill -1 $ppid &> /dev/null #SIGHUP if [ "$1" != "" ]; then kill -1 $1 &> /dev/null #SIGHUP @@ -104,7 +100,6 @@ function stop_timeout { #DEBUG - echo stop_timeout trap "" SIGHUP } @@ -129,10 +124,10 @@ mkfifo $SLOTDIR/fromR.fifo chmod +x $RServerScript echo "$0: INFO: Launching $RServerScript $SLOTDIR" - PATH=.:$PATH #echo PATH: $PATH > $SLOTDIR/R.log #which Rscript >> $SLOTDIR/R.log - echo $PATH + + echo LD_LIBRARYPATH= $LD_LIBRARY_PATH $RServerScript $SLOTDIR >> $SLOTDIR/R.log 2>&1 & # launch R server # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" @@ -162,7 +157,6 @@ echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo & echopid=$! -echo echopid $echopid start_timeout $echopid if wait $echopid Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-25 15:45:03 UTC (rev 4144) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-02-25 22:50:13 UTC (rev 4145) @@ -558,14 +558,15 @@ # Find and terminate R workers: they should register their PiD # in a standard location based on the pid of this start-swift # script - if [ "$keepdir" = "FALSE" ] - then - for rwork in `(shopt -s nullglob; echo ../Rworkers/worker.$$/*/)` - do + for rwork in `(shopt -s nullglob; echo ../Rworkers/worker.$$/*/)` + do + if [ -f "$rwork/R.pid" ]; then kill `cat $rwork/R.pid` &> /dev/null + fi + if [ "$keepdir" = "FALSE" ]; then rm -rf $rwork - done - fi + fi + done stdcleanup_end exit 0 } @@ -659,5 +660,12 @@ fi $SWIFTRBIN/../swift/bin/swift $swiftLoggingFlag -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& $out resultpipe & + echopid=$! + sleep 5 + kill $echopid &> /dev/null +fi -# Do any cleanup if swift exits in this manner From noreply at svn.ci.uchicago.edu Fri Feb 25 19:23:53 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 25 Feb 2011 19:23:53 -0600 (CST) Subject: [Swift-commit] r4146 - branches/release-0.92/bin Message-ID: <20110226012353.93B319CC92@svn.ci.uchicago.edu> Author: davidk Date: 2011-02-25 19:23:53 -0600 (Fri, 25 Feb 2011) New Revision: 4146 Modified: branches/release-0.92/bin/gensites Log: gensites: better error handling, fixed search path, list specific templates Modified: branches/release-0.92/bin/gensites =================================================================== --- branches/release-0.92/bin/gensites 2011-02-25 22:50:13 UTC (rev 4145) +++ branches/release-0.92/bin/gensites 2011-02-26 01:23:53 UTC (rev 4146) @@ -4,7 +4,7 @@ crash() { MSG=$1 - echo ${MSG} + echo ${MSG} >&2 exit 1 } @@ -34,13 +34,14 @@ { cat << END - usage: gensites template [-p properties.file] [-L template_directory] [-h] [-T] + usage: gensites template [-p properties.file] [-L template_directory] [-h] [-T] [-l] template Name of template to use -p properties.file Specify a swift.properties to use -L template_directory Specify a non-standard template directory -T List all templates available -h Help / usage information + -l List the contents of a specific template Examples: @@ -64,6 +65,7 @@ case "$1" in -p) PROPERTIES_FILE=$2; verify_not_null properties_file $PROPERTIES_FILE; shift ;; -L) TEMPLATE_DIRECTORY=$2; verify_not_null template_directory $TEMPLATE_DIRECTORY; shift ;; + -l) LIST_SPECIFIC_TEMPLATE=1 ;; -T) LIST_TEMPLATES=1 ;; -h|-help|-?) usage ;; *) TEMPLATE=$1 ;; @@ -85,8 +87,8 @@ fi fi -# List templates -if [ ! -z "$LIST_TEMPLATES" ]; then +# List all templates +if [ ! -z $LIST_TEMPLATES ]; then # Templates in etc/sites for file in `ls -1 $SWIFT_HOME/* 2>/dev/null` do @@ -120,15 +122,24 @@ TEMPLATE_PATH=$HOME/.swift/sites/$TEMPLATE fi if [ ! -f "$TEMPLATE_PATH" ]; then - crash "Cannot find template for $TEMPLATE" + crash "Cannot find template for $TEMPLATE" fi fi +# List a specific template +if [ ! -z $LIST_SPECIFIC_TEMPLATE ]; then + cat $TEMPLATE_PATH + exit 0 +fi + # Ensure a properties file exists if [ ! -f "$PROPERTIES_FILE" ]; then PROPERTIES_FILE="swift.properties" if [ ! -f "$PROPERTIES_FILE" ]; then - crash "Unable to find a valid properties file! Please specify a valid file with the -p option" + PROPERTIES_FILE="$HOME/.swift/swift.properties" + if [ ! -f "$PROPERTIES_FILE" ]; then + crash "Unable to find a valid properties file! Please specify a valid file with the -p option" + fi fi fi @@ -166,14 +177,13 @@ # Verify that the variables by the template are defined for TOKEN in PROJECT QUEUE N_GRAN N_MAX SLOTS - do - if grep _${TOKEN}_ ${TEMPLATE_PATH} > /dev/null - then - if ! declare -p ${TOKEN} > /dev/null - then - crash "Not specified: ${TOKEN}" +do + # Need but can't find, due + if grep _${TOKEN}_ $TEMPLATE_PATH > /dev/null; then + if [ -z "${!TOKEN}" ]; then + crash "Not specified: ${TOKEN}" + fi fi - fi done # Replace values @@ -192,5 +202,6 @@ echo "s at _SERVICE_PORT_@${SERVICE_PORT:-NO_PORT_GIVEN}@" } > $SEDFILE -sed -f $SEDFILE < ${TEMPLATE_PATH} + +sed -f $SEDFILE < $TEMPLATE_PATH rm $SEDFILE From noreply at svn.ci.uchicago.edu Sat Feb 26 15:43:39 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 26 Feb 2011 15:43:39 -0600 (CST) Subject: [Swift-commit] r4147 - in branches/release-0.92: etc examples Message-ID: <20110226214339.889A09CC80@svn.ci.uchicago.edu> Author: davidk Date: 2011-02-26 15:43:39 -0600 (Sat, 26 Feb 2011) New Revision: 4147 Removed: branches/release-0.92/examples/array_iteration.swift branches/release-0.92/examples/arraymapper.swift branches/release-0.92/examples/diamond.dtm branches/release-0.92/examples/file_counter.dtm branches/release-0.92/examples/helloworld.dtm branches/release-0.92/examples/helloworld_named.dtm branches/release-0.92/examples/q8.swift branches/release-0.92/examples/range.dtm Modified: branches/release-0.92/etc/tc.data Log: Verified that all the tutorial scripts will run in 0.92. Added 'wc' to default tc.data which is used by one of the tutorial examples. Removed some of the older scripts which are not mentioned in the tutorial and also no longer work. Modified: branches/release-0.92/etc/tc.data =================================================================== --- branches/release-0.92/etc/tc.data 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/etc/tc.data 2011-02-26 21:43:39 UTC (rev 4147) @@ -16,3 +16,4 @@ localhost sort /bin/sort INSTALLED INTEL32::LINUX null localhost paste /bin/paste INSTALLED INTEL32::LINUX null localhost cp /bin/cp INSTALLED INTEL32::LINUX null +localhost wc /usr/bin/wc INSTALLED INTEL32::LINUX null Deleted: branches/release-0.92/examples/array_iteration.swift =================================================================== --- branches/release-0.92/examples/array_iteration.swift 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/examples/array_iteration.swift 2011-02-26 21:43:39 UTC (rev 4147) @@ -1,17 +0,0 @@ -type file {} - -(file f) echo (string s) { - app { - echo s stdout=@filename(f); - } -} - -(file fa[]) echo_batch (string sa[]) { - foreach string s, i in sa { - fa[i] = echo(s); - } -} - -string sa[] = ["hello","hi there","how are you"]; -file fa[]; -fa = echo_batch(sa); Deleted: branches/release-0.92/examples/arraymapper.swift =================================================================== --- branches/release-0.92/examples/arraymapper.swift 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/examples/arraymapper.swift 2011-02-26 21:43:39 UTC (rev 4147) @@ -1,8 +0,0 @@ -type file {}; - -file files[]; - -foreach f in files { - print(f); -} - Deleted: branches/release-0.92/examples/diamond.dtm =================================================================== --- branches/release-0.92/examples/diamond.dtm 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/examples/diamond.dtm 2011-02-26 21:43:39 UTC (rev 4147) @@ -1,33 +0,0 @@ -type file {}; - -(file f) generate (float p1) { - app { - generate "-aTOP -T4" "-p" p1 "-o" @f; - } -} - -(file f2) process (file f1, string name, float p2) { - app { - process "-a" name "-T4" "-p" p2 "-i" @f1 "-o" @f2; - } -} - -(file f3) combine (file f1, file f2) { - app { - combine "-aBOTTOM -T4" "-i" @f1 @f2 "-o" @f3; - } -} - -(file fd) diamond (float p1, float p2) { - file fa; - file fb; - file fc; - - fa = generate(p1); - fb = process(fa, "LEFT", p2); - fc = process(fa, "RIGHT", p2); - fd = combine(fb, fc); -} - -file final<"FINAL">; -final = diamond(1, 100); Deleted: branches/release-0.92/examples/file_counter.dtm =================================================================== --- branches/release-0.92/examples/file_counter.dtm 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/examples/file_counter.dtm 2011-02-26 21:43:39 UTC (rev 4147) @@ -1,20 +0,0 @@ -// a two-step workflow that essentially does -// ls "/bin" | wc - -type file {} - -(file f) ls (string s) { - app { - ls s stdout=@filename(f); - } -} - -(file c) wc (file f) { - app { - wc stdin=@filename(f) stdout=@filename(c); - } -} - -file list, count; -list = ls("/bin"); -count = wc(list); Deleted: branches/release-0.92/examples/helloworld.dtm =================================================================== --- branches/release-0.92/examples/helloworld.dtm 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/examples/helloworld.dtm 2011-02-26 21:43:39 UTC (rev 4147) @@ -1,8 +0,0 @@ -type file {} //define a type for file -(file t) echo (string s) { //procedure declaration - app { - echo s stdout=@filename(t); //redirect stdout to a file - } -} - -file hw = echo("hello world"); //procedure call Deleted: branches/release-0.92/examples/helloworld_named.dtm =================================================================== --- branches/release-0.92/examples/helloworld_named.dtm 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/examples/helloworld_named.dtm 2011-02-26 21:43:39 UTC (rev 4147) @@ -1,9 +0,0 @@ -type file {} //define a type for file -(file t) echo (string s) { //procedure declaration - app { - echo s stdout=@filename(t); //redirect stdout to a file - } -} - -file hw<"helloworld.txt">; //name the output file -hw = echo("hello world"); //procedure call Deleted: branches/release-0.92/examples/q8.swift =================================================================== --- branches/release-0.92/examples/q8.swift 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/examples/q8.swift 2011-02-26 21:43:39 UTC (rev 4147) @@ -1,14 +0,0 @@ - -type file {} - -(file t) echo (string s) { - app { - echo s stdout=@filename(t); - } -} - -file inputFiles[] ; - -file o <"foo.out">; -o = echo(@filenames(inputFiles)); - Deleted: branches/release-0.92/examples/range.dtm =================================================================== --- branches/release-0.92/examples/range.dtm 2011-02-26 01:23:53 UTC (rev 4146) +++ branches/release-0.92/examples/range.dtm 2011-02-26 21:43:39 UTC (rev 4147) @@ -1,5 +0,0 @@ -int nums[] = [0:20:2]; // generate a list 0, 2, 4, 6, 8 ... - -foreach num in nums { - print(num); -} From noreply at svn.ci.uchicago.edu Mon Feb 28 13:32:13 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 28 Feb 2011 13:32:13 -0600 (CST) Subject: [Swift-commit] r4148 - text/parco10submission Message-ID: <20110228193213.9F5709CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2011-02-28 13:32:13 -0600 (Mon, 28 Feb 2011) New Revision: 4148 Added: text/parco10submission/ResponseToReviews2.txt Log: Adding Added: text/parco10submission/ResponseToReviews2.txt =================================================================== --- text/parco10submission/ResponseToReviews2.txt (rev 0) +++ text/parco10submission/ResponseToReviews2.txt 2011-02-28 19:32:13 UTC (rev 4148) @@ -0,0 +1,14 @@ + +Reviewer's comment: + +The paper is good as it stands except there are still a large number +of typos. Without trying hard I found a couple dozen small problems +which are easy to correct. The authors must to do a thorough +proofreading of the document before it can be ready for publication. + +Here are the things I found (page numbers refer to numbers on the +printed pages, not to page numbers inside acrobat reader): + +Response: + +All typos were addressed and corrected. From noreply at svn.ci.uchicago.edu Mon Feb 28 13:32:27 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 28 Feb 2011 13:32:27 -0600 (CST) Subject: [Swift-commit] r4149 - text/parco10submission Message-ID: <20110228193227.8CF719CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2011-02-28 13:32:27 -0600 (Mon, 28 Feb 2011) New Revision: 4149 Added: text/parco10submission/ResponseToReviews1.txt Log: Rename Copied: text/parco10submission/ResponseToReviews1.txt (from rev 4137, text/parco10submission/ResponseToReviews.txt) =================================================================== --- text/parco10submission/ResponseToReviews1.txt (rev 0) +++ text/parco10submission/ResponseToReviews1.txt 2011-02-28 19:32:27 UTC (rev 4149) @@ -0,0 +1,374 @@ + +---------- Forwarded message ---------- +Date: Thu, 14 Oct 2010 13:29:09 +From: Parallel Computing +To: wozniak at mcs.anl.gov +Cc: worleyph at ornl.gov, Rupak.Biswas at nasa.gov, Rajesh.Nishtala at gmail.com, + loliker at lbl.gov +Subject: Your Submission PARCO-D-10-00054 + +Ms. Ref. No.: PARCO-D-10-00054 +Title: Swift: A language for distributed parallel scripting +Parallel Computing + +Dear Justin, + +Reviewers have now commented on your paper. You will see that they are advising that you revise your manuscript. If you are prepared to undertake the work required, I would be pleased to reconsider my decision. + +For your guidance, reviewers' comments are appended below. + +If you decide to revise the work, please submit a list of changes or a rebuttal against each point which is being raised when you submit the revised manuscript. + +Your revision should be submitted before Nov 11, 2010. + +To submit a revision, please go to http://ees.elsevier.com/parco/ and login as an Author. On your Main Menu page is a folder entitled "Submissions Needing Revision". You will find your submission record there. + +When submitting your revised manuscript, please ensure that you upload the source files (e.g. (La)TeX or Word; (La)TeX files as 1 comprehensive file and each figure and table separately). Uploading a PDF file at this stage will create delays should your manuscript be finally accepted for publication. If your revised submission does not include the source files, we will contact you to request them. + +Yours sincerely, + +Patrick Haven Worley, Ph.D. +Associate Editor +Parallel Computing + +********************************************************** + +Reviewers' comments: + +Dear Authors, + Thank you for your submission to the Parallel Computing special issue on Emerging Programming Paradigms for Large-Scale Scientific Computing. As there were several concerns about the manuscript, we ask that you update your submission to address the reviewer feedback, and include a point by point response to the reviewer comments. + +With kind regards, +ParCo Guest Editors: +Rupak Biswas, Rajesh Nishtala, Lenny Oliker + + +**************************************************************************************** + +Reviewer #1: The paper describes Swift, a scripting language for distributed parallel applications. I have mixed feelings about this paper. I can see the usefulness of a scripting language like Swift. The problems Swift was designed to address are certainly interesting and important. + +On the other hand, I don't see much scientific merit in the paper. The paper reads more like a Swift user manual than a scientific paper. For the language design, the only thing that might be novel is the notion of mapped type, but I consider it to be quite minor. I also don't see any new ideas in the data-flow dependency based execution model. + +>>> Response: + +We believe that our discussion of related work shows that there is no other language that does what Swift does. We also believe that the decisions we have made in creating Swift, as a simple minimal language with a function model for evaluating a large set of individual applications on both parallel and distributed systems of extreme scale using the concept of single-assignment futures to highly effectively exploit implicit parallelism provides the scientific merit of the paper. + +<<< + +For the distributed execution, one important missing piece is performance evaluation. Data locality is very important for data-intensive applications. As I understand it, data have to be moved in and out the clusters. So, understanding the cost of scheduling and data transfer is very important to validate the Swift design. Perhaps, it was +published somewhere else, but it would be nice to discuss it in this paper. + +>>> Response: + +We have added a new section, "5. Performance Characteristics" in +response to this point. Additional tests are being developed and run, +so these results may be further refined before publication. Some +results from prior publications have been cited and included here, +which show the overlap of data transfer and processing to address the +issue above. + +<<< + +Here are some more detailed comments: + +1. Swift uses restart log to reuse the results of successfully +completed components. The paper mentioned "appropriate manual +intervention". This seems to be something you can almost completely +automate. Based on my experiences with large-scale and long running +applications, this can be very useful. + +>>> Automation of restart + +The "manual intervention" referred to the correction of whatever +caused the script to fail. For example, a missing data file. Since the +Swift restart mechanism *is* fully automated, this phrase was removed. + +<<< + +2. Swift reduces job submission cost using clustering. It is not +clear to me if a subgraph can be batched together by your clustering +technique. This obviously requires a little bit of analysis of the +data-flow graph to do it properly. But it could be quite useful to +achieve better data locality. + +>>> Clustering + +This section has been clarified. Swift will group tasks together based +on their expected time duration and their readiness to run. So a +cluster batch could include tasks from multiple sub-graphs. But its +based on + +<<< + +3. In terms of programming models, modern systems such as Microsoft's +DryadLINQ and Google's FlumeJava successfully integrate data-flow +constructs into state of the art programming languages (C# and Java). +This integration approach is quite nice and powerful. It would be nice +if the authors can compare Swift with these two systems. + +>>> Response: + +We have added comparisons to Dryad and FlumeJava in the related work section. + +<<< + +Reviewer #2: The paper presents a powerful high-level scripting language, SwiftScript, for performing a massive number of tasks/jobs coupled with collections of file-based data. It describes details of the language syntax and semantics, which is based on data flow with support of arrays and procedures, and discusses the implementation and several use cases with the main focus on a grid. Although a similar work was published before, this paper gives elaborated summary of the technical details, which is useful for general audience. + +The current work is an attempt to overcome issues in dealing with a massive amount of simulations and data either on a grid or on a massive parallel system, such as large data files, various dependencies, fault tolerance. The framework simplifies the process in the scripting design by an "app" interface to the actual command line for program execution and a "mapper" to map data structures with actual files without the need of going into details of how the program actually gets executed and without explicitly specifying the types of the files involved. The paper also describes the implementation that is built on top of the CoG Karajan workflow engine with built-in fault tolerance and demonstrates the usefulness of the system with a number of examples. + +The primary focus of the work is for the grid applications, although the paper has indicated the applicability to other systems, such as more tightly coupled massive computing systems. + +For those who might not be familiar with the Karajan language, it would be useful to add a reference to the related work. + +>>> Response: + +We have added such a reference + +<<< + +It would be helpful to include some discussion on the "auto-parallelization" capability (achieved via data flow analysis?). + +>>> auto-parallelization + +This is now discussed in much more detail, in section 2. + +<<< + +Out of the four application examples presented, two of the cases (4.3 and 4.4) do not contain enough details to support the discussion; deleting the two examples should not affect the clarity of the paper. + +>>> Examples: clarify or delete + +We have completely revised the application example section (4). It +now shows only two app examples, but does so in much more precise +detail, to provide a better understanding of what using Swift entails. + +<<< + +It would be helpful to elaborate more in example 4.2 on how each task/job gets scheduled onto Ranger nodes or how Swift interacts with the local batch job scheduler, which would in turn help audience understand better how SwiftScript could be used for a certain class of applications on a more tightly coupled massive computing environment (such as +parameter studies). + +>>> 4.2 - task scheduling on Ranger + +As Sec 4 is revised considerably, this information has been included +in Secs. 2 & 3. + +<<< + +There are a few minor typos in the manuscript. +For instance, on Page 7 section 2.2, in the code snippet: + + output[ix] = rotate(frames, 180); + +Should "frames" be "f" in this case? + +>>> Response: + +The typo has been corrected. + +<<< + +Reviewer #3: This is an interesting paper aimed at the practical problem of +managing a large ensemble of computations with possible dependencies +among the tasks. The general outline and structure of the paper is +fine. There are a number of small errors which should have been +caught by proofreading the manuscript. + +>>> Response: + +typos and grammar have been corrected by a fresh complete proofreading. + +<<< + +The most substantive comment I have concerns examples 4.3 and 4.4. I +believe that, as they stand, these are too sparsely annotated to add +significant value to the paper. So I suggest either documenting these +examples more carefully, so that a reader new to Swift can understand +them, or else omit these examples entirely. + +>>> examples 4.3 and 4.4: annotate or delete + +Addressed and revised completely, as mentioned above. + +<<< + +Further comments: + +1. Sometimes "Swift" is used and sometimes "SwiftScript" is used. Is +there any reason to have two different terms? From the website it +appears that "SwiftScript" referred to an earlier version of the +language, so perhaps "SwiftScript" should be replaced by "Swift" +everywhere. + +>>> Response: + +This has been done. SwiftScript no longer appears. + +<<< + +2. It's a bit awkward that "single assignment" is used in section 2.1 +but not defined until section 2.3. + +>>> Response: + +This has been fixed. + +<<< + +3. The example on p. 5 appears erroneous, it seems the rotate +procedure should have an angle input in addition to the image input +(this is corrected on p.6). + +>>> Response: + +This has been fixed. + +<<< + +4. In section 2.2, should rotate be invoked as "rotate(f, 180)" instead +of "rotate(frames, 180)"? + +>>> Response: + +This has been fixed. + +<<< + +5. There are some acronyms in the paper that should, I believe, be +defined. In some cases the acronym can be omitted, for example I +believe RDBMS is only used once so there is really no need for the +acronym. Other acronyms appear in Figure 1: CoG, OSG, AWS; these +probably should be defined. Other acronyms: GRAM, fMRI (and FMRI, +cf. Fig. 2, which should probably be fMRI). + +>>> Response: + +RDBMS has been removed. +FMRI has been removed. +GRAM doesn't seem to need explanation, it's cited where first used. +The acronyms in Figure 1 are defined in the caption. + +<<< + +6. All these appear: "stage in", "stage-in", "stagein". Please be +consistent (similarly for stage out). + +>>> Response: + +This has been fixed. + +<<< + +7. The mysterious numbers '12, 1000, 1000, "81 3 3"' in example 4.1 +might merit an explanation. + +>>> + +It seems clear that these are parameters passed into the app program; +I don't think explanation is needed + +<<< + +8. The Figure numbering in the text of section 4.1 needs correction. + +>>> + +fixed. + +<<< + +9. In the text of section 4.1, I believe the variables or, iv, +direction, overwrite, and ov should be in a different font for clarity. +In the text of section 4.1 "n x n" should be in a different font and +"n2" is clearly wrong. + +>>> + +fixed. + +<<< + +10. Section 4.2, what is an "emblem experiment"? + +>>> + +This has been removed in the revision of Sec 4. + +<<< + +11. The margins in sections 4.2 and 4.4 need fixing as some lines +run completely off the page. + +>>> + +This has been addressed in the revision of Sec 4. + +<<< + +12. "Karajan" is mentioned several times, there really should be short +definition of it and a reference to it in the bibliography. + +>>> Response: + +This has been fixed. + +<<< + +13. Many of the references look incomplete; journal references really +should have page numbers, some references are missing a year. +Reference 8 severely mangles "Bresnahan". + +>>> Response: + +This has been fixed. + +<<< + +14. It is somewhat disingenuous to refer to thousands and millions of +cores (section 6.2). All systems I know of are managed as nodes, +where each node might have 8 or 12 or 16 cores. This is no essential +simplification, I know, but why not be honest and refer to nodes +instead of cores? + +>>> Response: + +In Swift, individual tasks generally run on cores, so cores is the correct +term. + +<<< + + +15. "Swift is implemented by compiling to a Karajan program". Why +would one want to _compile_ a scripting language? It seems more natural +(to this naive reader) to have an interpreter or a translator. + +>>> Response: + +This has been fixed. We now more accurately say: +Swift is implemented by generating and executing a Karajan program. + +<<< + +16. The coaster idea looks quite interesting, could this be expanded, +or could an example with coasters be constructed? + +>>> <<< + +17. Table 1, 1st row, 3rd column: should it be f->data.txt instead of +f->file.txt? + +>>> Response: + +This has been fixed. + +<<< + +18. There are many (too many to list) typos, missing words, mistakes +such as "en queued" instead of "enqueued", subject/verb mismatches of +number and/or tense. A careful proofreading is sorely needed. + + +>>> Response: + +This has been fixed. + +<<< From noreply at svn.ci.uchicago.edu Mon Feb 28 13:32:41 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 28 Feb 2011 13:32:41 -0600 (CST) Subject: [Swift-commit] r4150 - text/parco10submission Message-ID: <20110228193241.148359CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2011-02-28 13:32:40 -0600 (Mon, 28 Feb 2011) New Revision: 4150 Removed: text/parco10submission/ResponseToReviews.txt Log: Drop this for ResponseToReviews1 Deleted: text/parco10submission/ResponseToReviews.txt =================================================================== --- text/parco10submission/ResponseToReviews.txt 2011-02-28 19:32:27 UTC (rev 4149) +++ text/parco10submission/ResponseToReviews.txt 2011-02-28 19:32:40 UTC (rev 4150) @@ -1,374 +0,0 @@ - ----------- Forwarded message ---------- -Date: Thu, 14 Oct 2010 13:29:09 -From: Parallel Computing -To: wozniak at mcs.anl.gov -Cc: worleyph at ornl.gov, Rupak.Biswas at nasa.gov, Rajesh.Nishtala at gmail.com, - loliker at lbl.gov -Subject: Your Submission PARCO-D-10-00054 - -Ms. Ref. No.: PARCO-D-10-00054 -Title: Swift: A language for distributed parallel scripting -Parallel Computing - -Dear Justin, - -Reviewers have now commented on your paper. You will see that they are advising that you revise your manuscript. If you are prepared to undertake the work required, I would be pleased to reconsider my decision. - -For your guidance, reviewers' comments are appended below. - -If you decide to revise the work, please submit a list of changes or a rebuttal against each point which is being raised when you submit the revised manuscript. - -Your revision should be submitted before Nov 11, 2010. - -To submit a revision, please go to http://ees.elsevier.com/parco/ and login as an Author. On your Main Menu page is a folder entitled "Submissions Needing Revision". You will find your submission record there. - -When submitting your revised manuscript, please ensure that you upload the source files (e.g. (La)TeX or Word; (La)TeX files as 1 comprehensive file and each figure and table separately). Uploading a PDF file at this stage will create delays should your manuscript be finally accepted for publication. If your revised submission does not include the source files, we will contact you to request them. - -Yours sincerely, - -Patrick Haven Worley, Ph.D. -Associate Editor -Parallel Computing - -********************************************************** - -Reviewers' comments: - -Dear Authors, - Thank you for your submission to the Parallel Computing special issue on Emerging Programming Paradigms for Large-Scale Scientific Computing. As there were several concerns about the manuscript, we ask that you update your submission to address the reviewer feedback, and include a point by point response to the reviewer comments. - -With kind regards, -ParCo Guest Editors: -Rupak Biswas, Rajesh Nishtala, Lenny Oliker - - -**************************************************************************************** - -Reviewer #1: The paper describes Swift, a scripting language for distributed parallel applications. I have mixed feelings about this paper. I can see the usefulness of a scripting language like Swift. The problems Swift was designed to address are certainly interesting and important. - -On the other hand, I don't see much scientific merit in the paper. The paper reads more like a Swift user manual than a scientific paper. For the language design, the only thing that might be novel is the notion of mapped type, but I consider it to be quite minor. I also don't see any new ideas in the data-flow dependency based execution model. - ->>> Response: - -We believe that our discussion of related work shows that there is no other language that does what Swift does. We also believe that the decisions we have made in creating Swift, as a simple minimal language with a function model for evaluating a large set of individual applications on both parallel and distributed systems of extreme scale using the concept of single-assignment futures to highly effectively exploit implicit parallelism provides the scientific merit of the paper. - -<<< - -For the distributed execution, one important missing piece is performance evaluation. Data locality is very important for data-intensive applications. As I understand it, data have to be moved in and out the clusters. So, understanding the cost of scheduling and data transfer is very important to validate the Swift design. Perhaps, it was -published somewhere else, but it would be nice to discuss it in this paper. - ->>> Response: - -We have added a new section, "5. Performance Characteristics" in -response to this point. Additional tests are being developed and run, -so these results may be further refined before publication. Some -results from prior publications have been cited and included here, -which show the overlap of data transfer and processing to address the -issue above. - -<<< - -Here are some more detailed comments: - -1. Swift uses restart log to reuse the results of successfully -completed components. The paper mentioned "appropriate manual -intervention". This seems to be something you can almost completely -automate. Based on my experiences with large-scale and long running -applications, this can be very useful. - ->>> Automation of restart - -The "manual intervention" referred to the correction of whatever -caused the script to fail. For example, a missing data file. Since the -Swift restart mechanism *is* fully automated, this phrase was removed. - -<<< - -2. Swift reduces job submission cost using clustering. It is not -clear to me if a subgraph can be batched together by your clustering -technique. This obviously requires a little bit of analysis of the -data-flow graph to do it properly. But it could be quite useful to -achieve better data locality. - ->>> Clustering - -This section has been clarified. Swift will group tasks together based -on their expected time duration and their readiness to run. So a -cluster batch could include tasks from multiple sub-graphs. But its -based on - -<<< - -3. In terms of programming models, modern systems such as Microsoft's -DryadLINQ and Google's FlumeJava successfully integrate data-flow -constructs into state of the art programming languages (C# and Java). -This integration approach is quite nice and powerful. It would be nice -if the authors can compare Swift with these two systems. - ->>> Response: - -We have added comparisons to Dryad and FlumeJava in the related work section. - -<<< - -Reviewer #2: The paper presents a powerful high-level scripting language, SwiftScript, for performing a massive number of tasks/jobs coupled with collections of file-based data. It describes details of the language syntax and semantics, which is based on data flow with support of arrays and procedures, and discusses the implementation and several use cases with the main focus on a grid. Although a similar work was published before, this paper gives elaborated summary of the technical details, which is useful for general audience. - -The current work is an attempt to overcome issues in dealing with a massive amount of simulations and data either on a grid or on a massive parallel system, such as large data files, various dependencies, fault tolerance. The framework simplifies the process in the scripting design by an "app" interface to the actual command line for program execution and a "mapper" to map data structures with actual files without the need of going into details of how the program actually gets executed and without explicitly specifying the types of the files involved. The paper also describes the implementation that is built on top of the CoG Karajan workflow engine with built-in fault tolerance and demonstrates the usefulness of the system with a number of examples. - -The primary focus of the work is for the grid applications, although the paper has indicated the applicability to other systems, such as more tightly coupled massive computing systems. - -For those who might not be familiar with the Karajan language, it would be useful to add a reference to the related work. - ->>> Response: - -We have added such a reference - -<<< - -It would be helpful to include some discussion on the "auto-parallelization" capability (achieved via data flow analysis?). - ->>> auto-parallelization - -This is now discussed in much more detail, in section 2. - -<<< - -Out of the four application examples presented, two of the cases (4.3 and 4.4) do not contain enough details to support the discussion; deleting the two examples should not affect the clarity of the paper. - ->>> Examples: clarify or delete - -We have completely revised the application example section (4). It -now shows only two app examples, but does so in much more precise -detail, to provide a better understanding of what using Swift entails. - -<<< - -It would be helpful to elaborate more in example 4.2 on how each task/job gets scheduled onto Ranger nodes or how Swift interacts with the local batch job scheduler, which would in turn help audience understand better how SwiftScript could be used for a certain class of applications on a more tightly coupled massive computing environment (such as -parameter studies). - ->>> 4.2 - task scheduling on Ranger - -As Sec 4 is revised considerably, this information has been included -in Secs. 2 & 3. - -<<< - -There are a few minor typos in the manuscript. -For instance, on Page 7 section 2.2, in the code snippet: - - output[ix] = rotate(frames, 180); - -Should "frames" be "f" in this case? - ->>> Response: - -The typo has been corrected. - -<<< - -Reviewer #3: This is an interesting paper aimed at the practical problem of -managing a large ensemble of computations with possible dependencies -among the tasks. The general outline and structure of the paper is -fine. There are a number of small errors which should have been -caught by proofreading the manuscript. - ->>> Response: - -typos and grammar have been corrected by a fresh complete proofreading. - -<<< - -The most substantive comment I have concerns examples 4.3 and 4.4. I -believe that, as they stand, these are too sparsely annotated to add -significant value to the paper. So I suggest either documenting these -examples more carefully, so that a reader new to Swift can understand -them, or else omit these examples entirely. - ->>> examples 4.3 and 4.4: annotate or delete - -Addressed and revised completely, as mentioned above. - -<<< - -Further comments: - -1. Sometimes "Swift" is used and sometimes "SwiftScript" is used. Is -there any reason to have two different terms? From the website it -appears that "SwiftScript" referred to an earlier version of the -language, so perhaps "SwiftScript" should be replaced by "Swift" -everywhere. - ->>> Response: - -This has been done. SwiftScript no longer appears. - -<<< - -2. It's a bit awkward that "single assignment" is used in section 2.1 -but not defined until section 2.3. - ->>> Response: - -This has been fixed. - -<<< - -3. The example on p. 5 appears erroneous, it seems the rotate -procedure should have an angle input in addition to the image input -(this is corrected on p.6). - ->>> Response: - -This has been fixed. - -<<< - -4. In section 2.2, should rotate be invoked as "rotate(f, 180)" instead -of "rotate(frames, 180)"? - ->>> Response: - -This has been fixed. - -<<< - -5. There are some acronyms in the paper that should, I believe, be -defined. In some cases the acronym can be omitted, for example I -believe RDBMS is only used once so there is really no need for the -acronym. Other acronyms appear in Figure 1: CoG, OSG, AWS; these -probably should be defined. Other acronyms: GRAM, fMRI (and FMRI, -cf. Fig. 2, which should probably be fMRI). - ->>> Response: - -RDBMS has been removed. -FMRI has been removed. -GRAM doesn't seem to need explanation, it's cited where first used. -The acronyms in Figure 1 are defined in the caption. - -<<< - -6. All these appear: "stage in", "stage-in", "stagein". Please be -consistent (similarly for stage out). - ->>> Response: - -This has been fixed. - -<<< - -7. The mysterious numbers '12, 1000, 1000, "81 3 3"' in example 4.1 -might merit an explanation. - ->>> - -It seems clear that these are parameters passed into the app program; -I don't think explanation is needed - -<<< - -8. The Figure numbering in the text of section 4.1 needs correction. - ->>> - -fixed. - -<<< - -9. In the text of section 4.1, I believe the variables or, iv, -direction, overwrite, and ov should be in a different font for clarity. -In the text of section 4.1 "n x n" should be in a different font and -"n2" is clearly wrong. - ->>> - -fixed. - -<<< - -10. Section 4.2, what is an "emblem experiment"? - ->>> - -This has been removed in the revision of Sec 4. - -<<< - -11. The margins in sections 4.2 and 4.4 need fixing as some lines -run completely off the page. - ->>> - -This has been addressed in the revision of Sec 4. - -<<< - -12. "Karajan" is mentioned several times, there really should be short -definition of it and a reference to it in the bibliography. - ->>> Response: - -This has been fixed. - -<<< - -13. Many of the references look incomplete; journal references really -should have page numbers, some references are missing a year. -Reference 8 severely mangles "Bresnahan". - ->>> Response: - -This has been fixed. - -<<< - -14. It is somewhat disingenuous to refer to thousands and millions of -cores (section 6.2). All systems I know of are managed as nodes, -where each node might have 8 or 12 or 16 cores. This is no essential -simplification, I know, but why not be honest and refer to nodes -instead of cores? - ->>> Response: - -In Swift, individual tasks generally run on cores, so cores is the correct -term. - -<<< - - -15. "Swift is implemented by compiling to a Karajan program". Why -would one want to _compile_ a scripting language? It seems more natural -(to this naive reader) to have an interpreter or a translator. - ->>> Response: - -This has been fixed. We now more accurately say: -Swift is implemented by generating and executing a Karajan program. - -<<< - -16. The coaster idea looks quite interesting, could this be expanded, -or could an example with coasters be constructed? - ->>> <<< - -17. Table 1, 1st row, 3rd column: should it be f->data.txt instead of -f->file.txt? - ->>> Response: - -This has been fixed. - -<<< - -18. There are many (too many to list) typos, missing words, mistakes -such as "en queued" instead of "enqueued", subject/verb mismatches of -number and/or tense. A careful proofreading is sorely needed. - - ->>> Response: - -This has been fixed. - -<<< From noreply at svn.ci.uchicago.edu Mon Feb 28 13:38:44 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 28 Feb 2011 13:38:44 -0600 (CST) Subject: [Swift-commit] r4153 - text/parco10submission/notes Message-ID: <20110228193844.ED9499CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2011-02-28 13:38:44 -0600 (Mon, 28 Feb 2011) New Revision: 4153 Added: text/parco10submission/notes/paper-bib.txt Log: Adding Added: text/parco10submission/notes/paper-bib.txt =================================================================== --- text/parco10submission/notes/paper-bib.txt (rev 0) +++ text/parco10submission/notes/paper-bib.txt 2011-02-28 19:38:44 UTC (rev 4153) @@ -0,0 +1,13 @@ + +To create paper-bib.tex, + +1) just run lx.zsh +2) paper.tex paper.bbl > paper-bib.tex +3) do some minor editing in paper.tex to put things in order + +Then use + +pdflatex paper-bib + +to create paper-bib.pdf + From noreply at svn.ci.uchicago.edu Mon Feb 28 15:48:24 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 28 Feb 2011 15:48:24 -0600 (CST) Subject: [Swift-commit] r4154 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110228214824.90E849CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-28 15:48:24 -0600 (Mon, 28 Feb 2011) New Revision: 4154 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh Log: Fix to batch. Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-28 19:38:44 UTC (rev 4153) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-28 21:48:24 UTC (rev 4154) @@ -128,6 +128,7 @@ #which Rscript >> $SLOTDIR/R.log echo LD_LIBRARYPATH= $LD_LIBRARY_PATH + PATH=.:$PATH $RServerScript $SLOTDIR >> $SLOTDIR/R.log 2>&1 & # launch R server # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" From noreply at svn.ci.uchicago.edu Mon Feb 28 16:35:41 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 28 Feb 2011 16:35:41 -0600 (CST) Subject: [Swift-commit] r4155 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110228223541.904179CC92@svn.ci.uchicago.edu> Author: tga Date: 2011-02-28 16:35:41 -0600 (Mon, 28 Feb 2011) New Revision: 4155 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/SwiftRServer.R SwiftApps/SwiftR/Swift/exec/configure-server-local Log: Got SwiftR working on Beagle login node finally by passing LD_LIBRARY_PATH. Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-28 21:48:24 UTC (rev 4154) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-02-28 22:35:41 UTC (rev 4155) @@ -129,6 +129,7 @@ echo LD_LIBRARYPATH= $LD_LIBRARY_PATH PATH=.:$PATH + echo PATH= $PATH $RServerScript $SLOTDIR >> $SLOTDIR/R.log 2>&1 & # launch R server # idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it echo "$0: INFO: Launched $RServerScript $SLOTDIR Rscript" Modified: SwiftApps/SwiftR/Swift/exec/SwiftRServer.R =================================================================== --- SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-28 21:48:24 UTC (rev 4154) +++ SwiftApps/SwiftR/Swift/exec/SwiftRServer.R 2011-02-28 22:35:41 UTC (rev 4155) @@ -93,8 +93,7 @@ # Load contents into local environment success <- try(load(callBatchFileName, envir=environment())); if (inherits(success, "try-error")) { - failBatch(rcall, paste(callBatchFileName, "could not be opened"), - resultBatchFileName) + stop(paste(callBatchFileName, "could not be opened")) return() } success <- try(loadImports(rcall$imports)) Modified: SwiftApps/SwiftR/Swift/exec/configure-server-local =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-local 2011-02-28 21:48:24 UTC (rev 4154) +++ SwiftApps/SwiftR/Swift/exec/configure-server-local 2011-02-28 22:35:41 UTC (rev 4155) @@ -44,6 +44,7 @@ $throttleOneCore 10000 + $LD_LIBRARY_PATH $(pwd)/swiftwork From noreply at svn.ci.uchicago.edu Mon Feb 28 17:34:04 2011 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 28 Feb 2011 17:34:04 -0600 (CST) Subject: [Swift-commit] r4156 - SwiftApps/SwiftR/Swift/exec Message-ID: <20110228233404.3FB669CC80@svn.ci.uchicago.edu> Author: tga Date: 2011-02-28 17:34:04 -0600 (Mon, 28 Feb 2011) New Revision: 4156 Added: SwiftApps/SwiftR/Swift/exec/configure-server-crayxt Modified: SwiftApps/SwiftR/Swift/exec/start-swift Log: Got SwiftR to point where it can launch coaster workers on beagle compute nodes, but haven't successfully got those to register with the coaster server. Copied: SwiftApps/SwiftR/Swift/exec/configure-server-crayxt (from rev 4154, SwiftApps/SwiftR/Swift/exec/configure-server-pbs) =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-crayxt (rev 0) +++ SwiftApps/SwiftR/Swift/exec/configure-server-crayxt 2011-02-28 23:34:04 UTC (rev 4156) @@ -0,0 +1,52 @@ +#! /bin/bash + +# configuration for PBS with manually-started Swift workers (passive coasters) + +cores=$1 + +throttlePBS=$2 + +cat >tc <sites.xml < + + + + 0.15 + 10000 + + $(pwd)/swiftwork + + + + + passive + $cores + $throttlePBS + 10000 + + $(pwd)/swiftwork + $(pwd)/swiftscratch + + + +END + +cat >cf < $SUBMIT_FILE <= 1 integer -q queue site speific (PBS, SGE, Cobalt) -r rcmd ssh site specific, SGE only, typically ssh. qrsh for siraf cluster - -s server local local, pbs, sge, ssh, pbsf,cobalt (for firewalled worker nodes) + -s server local local, pbs, sge, ssh, pbsf,cobalt,crayxt (for firewalled worker nodes) -t time 00:30:00 hh:mm:ss, for PBS, Cobalt and SGE only -w wkloglvl NONE NONE, ERROR, WARN, INFO, DEBUG, TRACE -k keepdir No argument, if flag is set, will keep working directory @@ -464,7 +495,7 @@ -p) throttle=$2; verify-is-numeric throttle $throttle; shift ;; -q) queue=$2; verify-not-null queue $queue; shift ;; -r) rcmd=$2; verify-is-one-of rcmd $rcmd ssh qrsh; shift ;; - -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge cobalt; shift ;; + -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge cobalt crayxt; shift ;; -t) time=$2; verify-not-null time $time; shift ;; -w) workerLogging=$2; verify-is-one-of workerLoggingLevel $workerLogging NONE ERROR WARN INFO DEBUG TRACE; shift ;; -L) swiftLoggingFlag="" ;; # swift default is lots of logging @@ -616,7 +647,7 @@ starterpid=$! elif [ \( $server = pbs \) -o \( $server = pbsf \) -o \( $server = sge \) \ - -o \( $server = cobalt \) ]; then + -o \( $server = cobalt \) -o \( $server = crayxt \) ]; then if [ $cores -eq 0 ]; then cores=$defaultClusterCores