From noreply at svn.ci.uchicago.edu Wed Feb 3 15:12:49 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 3 Feb 2010 15:12:49 -0600 (CST) Subject: [Swift-commit] r3221 - in trunk: etc libexec src/org/globus/swift src/org/globus/swift/data src/org/globus/swift/data/policy src/org/globus/swift/data/util Message-ID: <20100203211249.EC4619D17C@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-03 15:12:49 -0600 (Wed, 03 Feb 2010) New Revision: 3221 Added: trunk/libexec/cdm.pl trunk/src/org/globus/swift/data/ trunk/src/org/globus/swift/data/Director.java trunk/src/org/globus/swift/data/Query.java trunk/src/org/globus/swift/data/policy/ trunk/src/org/globus/swift/data/policy/Default.java trunk/src/org/globus/swift/data/policy/Direct.java trunk/src/org/globus/swift/data/policy/Policy.java trunk/src/org/globus/swift/data/util/ trunk/src/org/globus/swift/data/util/LineReader.java Modified: trunk/etc/log4j.properties trunk/libexec/_swiftwrap trunk/libexec/vdl-int.k trunk/libexec/vdl-lib.xml trunk/libexec/vdl.k Log: General framework for CDM functionality, including DIRECT policy. Modified: trunk/etc/log4j.properties =================================================================== --- trunk/etc/log4j.properties 2010-01-25 22:00:15 UTC (rev 3220) +++ trunk/etc/log4j.properties 2010-02-03 21:12:49 UTC (rev 3221) @@ -26,4 +26,6 @@ log4j.logger.org.globus.cog.abstraction.impl.common.task.TaskImpl=DEBUG log4j.logger.org.griphyn.vdl.karajan.lib.GetFieldValue=DEBUG log4j.logger.org.griphyn.vdl.engine.Karajan=INFO -log4j.logger.org.globus.cog.abstraction.coaster.rlog=INFO \ No newline at end of file +log4j.logger.org.globus.cog.abstraction.coaster.rlog=INFO + +log4j.logger.org.globus.swift.data.Director=DEBUG Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-01-25 22:00:15 UTC (rev 3220) +++ trunk/libexec/_swiftwrap 2010-02-03 21:12:49 UTC (rev 3221) @@ -113,6 +113,37 @@ exec 3>&- } +cdm() { + log "CDM: $@" + + local JOBDIR=$1 # Given jobdir + local MODE=$2 # INPUT or OUTPUT + local FILE=$3 # User file + local POLICY=$4 # DIRECT, BROADCAST, ... + shift 4 + local ARGS=$@ + + case $POLICY in + DIRECT) + DIRECT_DIR=${ARGS[0]} + log "CDM[DIRECT]: Linking $JOBDIR/$FILE to $DIRECT_DIR/$FILE" + if [ $MODE == "INPUT" ]; then + [ -f "$DIRECT_DIR/$FILE" ] + checkError 254 "CDM[DIRECT]: $DIRECT_DIR/$FILE does not exist!" + ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE + checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!" + elif [ $MODE == "OUTPUT" ]; then + touch $DIRECT_DIR/$FILE + checkError 254 "CDM[DIRECT]: Touching $DIRECT_DIR/$FILE failed!" + ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE + checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!" + else + fail 254 "Unknown MODE: $MODE" + fi + ;; + esac +} + COMMANDLINE=$@ # get the parent directory of the directory containing _swiftwrap, to use @@ -162,7 +193,6 @@ logstate "LOG_START" infosection "Wrapper" - getarg "-e" "$@" EXEC=$VALUE shift $SHIFTCOUNT @@ -195,6 +225,10 @@ KICKSTART=$VALUE shift $SHIFTCOUNT +getarg "-cdmfile" "$@" +CDM_FILE=$VALUE +shift $SHIFTCOUNT + getarg "-status" "$@" STATUSMODE=$VALUE shift $SHIFTCOUNT @@ -259,6 +293,7 @@ log "INF=$INF" log "OUTF=$OUTF" log "KICKSTART=$KICKSTART" +log "CDM_FILE=$CDM_FILE" log "ARGS=$@" log "ARGC=$#" @@ -276,24 +311,45 @@ log "Created output directory: $DIR/$D" done +log "PWD: $( pwd )" +log "FIND: $( find . )" + logstate "LINK_INPUTS" for L in $INF ; do + CDM_POLICY=$( perl shared/cdm.pl $L < shared/$CDM_FILE ) + log "CDM_POLICY: $CDM_POLICY" + if [ $CDM_POLICY != "DEFAULT" ]; then + eval cdm $DIR "INPUT" $L $CDM_POLICY + break + fi if [ $COPYNOTLINK = 1 ]; then cp "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO checkError 254 "Failed to copy input file $L" log "Copied input: $WFDIR/shared/$L to $DIR/$L" else + [ -f $WFDIR/shared/$L ] + checkError 254 "Could not locate input file: $L" ln -s "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO checkError 254 "Failed to link input file $L" log "Linked input: $WFDIR/shared/$L to $DIR/$L" fi done +logstate "LINK_CDM_OUTPUTS" +for L in $OUTF ; do + CDM_POLICY=$( perl shared/cdm.pl $L < shared/$CDM_FILE ) + log "CDM_POLICY: $CDM_POLICY" + if [ $CDM_POLICY != "DEFAULT" ]; then + eval cdm $DIR "OUTPUT" $L $CDM_POLICY + eval CDM_OUTPUT_$L="SKIP" + break + fi +done + logstate "EXECUTE" cd $DIR -#ls >>$WRAPPERLOG if [ ! -f "$EXEC" ]; then fail 254 "The executable $EXEC does not exist" fi @@ -349,6 +405,7 @@ MISSING= for O in $OUTF ; do + log "checking: $O" if [ ! -f "$DIR/$O" ]; then if [ "$MISSING" == "" ]; then MISSING=$O @@ -361,19 +418,25 @@ fail 254 "The following output files were not created by the application: $MISSING" fi -logstate "MOVING_OUTPUTS" +logstate "MOVING_OUTPUTS $OUTF" for O in $OUTF ; do - mv "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >& "$INFO" - checkError 254 "Failed to move output file $O to shared directory" + log "output: $O ${CDM_OUTPUTS}" + if [ $( eval echo "$"$( echo CDM_OUTPUT_$O ) ) != "SKIP" ]; then + mv "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >& "$INFO" + checkError 254 "Failed to move output file $O to shared directory" + else + log "CDM: SKIP OUTPUT: $O" + fi done logstate "RM_JOBDIR" -rm -rf "$DIR" 2>&1 >& "$INFO" +# rm -rf "$DIR" 2>&1 >& "$INFO" checkError 254 "Failed to remove job directory $DIR" if [ "$STATUSMODE" = "files" ]; then logstate "TOUCH_SUCCESS" touch $WFDIR/status/${JOBDIR}/${ID}-success + log "TOUCH CODE: $WFDIR/status/${JOBDIR}/${ID}-success $?" fi logstate "END" Added: trunk/libexec/cdm.pl =================================================================== --- trunk/libexec/cdm.pl (rev 0) +++ trunk/libexec/cdm.pl 2010-02-03 21:12:49 UTC (rev 3221) @@ -0,0 +1,46 @@ + +# CDM Lookup + +sub print_hash +{ + my $hash = $_[0]; + foreach (keys %$hash) + { + print "$_ : $$hash{$_} \n"; + } +} + +sub cdm_lookup +{ + my $hash = $_[0]; + my $file = $_[1]; + + $result = "DEFAULT"; + foreach (keys %$hash) + { + $pattern = $_; + if ($file =~ /$pattern/) + { + print("$$hash{$pattern}\n"); + last; + } + } +} + +# Command-line arguments: +$file = $ARGV[0]; + +# Read fs.data off of stdin: +%map = (); +while () +{ + chomp; + next if $_ eq ""; + + @tokens = split(/ /, $_); + $key = shift(@tokens); + $rest = join(' ', @tokens); + $map{$key} = $rest; +} + +cdm_lookup(\%map, $file); Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-01-25 22:00:15 UTC (rev 3220) +++ trunk/libexec/vdl-int.k 2010-02-03 21:12:49 UTC (rev 3221) @@ -215,6 +215,7 @@ ) ) ) + log(LOG:INFO, "END cleanups={cleanup}") ) element(cleanupFiles, [files, host] @@ -245,6 +246,21 @@ element(doStagein, [jobid, files, dir, host] log(LOG:INFO, "START jobid={jobid} - Staging in files") + + cdmfile := cdm:file() + log(LOG:DEBUG, "cdmfile: {cdmfile}") + log(LOG:DEBUG, "swift.home: {swift.home}") + libexec := "{swift.home}/libexec" + + if (cdmfile != "" then( + doStageinFile(provider="file", srchost="localhost", srcfile=basename(cdmfile), + srcdir=dirname(cdmfile), desthost=host, destdir=dir, + size=file:size(cdmfile), policy="DEFAULT") + doStageinFile(provider="file", srchost="localhost", srcfile="cdm.pl", + srcdir=libexec, desthost=host, destdir=dir, + size=file:size("{libexec}/cdm.pl}"), policy="DEFAULT") + )) + uParallelFor(file, files provider := provider(file) srchost := hostname(file) @@ -253,25 +269,35 @@ filename := basename(file) size := file:size("{srcdir}/{filename}", host=srchost, provider=provider) - vdl:cacheAddAndLockFile(filename, destdir, host, size - cleanupFiles(cacheFilesToRemove, host) - log(LOG:DEBUG, "FILE_STAGE_IN_START file={file} srchost={srchost} srcdir={srcdir} srcname={filename} ", - "desthost={host} destdir={destdir} provider={provider}") - restartOnError(".*", 2 - task:transfer(srcprovider=provider, srchost=srchost, srcfile=filename, - srcdir=srcdir, desthost=host, destdir=destdir) - ) - log(LOG:DEBUG, "FILE_STAGE_IN_END file={file} srchost={srchost} srcdir={srcdir} srcname={filename} ", - "desthost={host} destdir={destdir} provider={provider}") - ) - ) + policy := cdm:query(query=file) + log(LOG:DEBUG, "policy: {file} : {policy}") + + doStageinFile(provider=provider, srchost=srchost, srcfile=filename, + srcdir=srcdir, desthost=host, destdir=destdir, size=size, policy=policy) + ) log(LOG:INFO, "END jobid={jobid} - Staging in finished") ) - + + element(doStageinFile, [provider, srchost, srcfile, srcdir, desthost, destdir, size, policy] + vdl:cacheAddAndLockFile(srcfile, destdir, desthost, size + cleanupFiles(cacheFilesToRemove, desthost) + + log(LOG:DEBUG, "FILE_STAGE_IN_START file={srcfile} srchost={srchost} srcdir={srcdir} srcname={srcfile} ", + "desthost={desthost} destdir={destdir} provider={provider}") + if (policy == "DEFAULT" then( + restartOnError(".*", 2 + task:transfer(srcprovider=provider, srchost=srchost, srcfile=srcfile, + srcdir=srcdir, desthost=desthost, destdir=destdir))) + else(log(LOG:DEBUG, "FILE_STAGE_IN_SKIP file={srcfile} policy={policy}"))) + log(LOG:DEBUG, "FILE_STAGE_IN_END file={srcfile} srchost={srchost} srcdir={srcdir} srcname={srcfile} ", + "desthost={desthost} destdir={destdir} provider={provider}") + ) + ) + element(doStageout, [jobid, stageouts, dir, host] log(LOG:INFO, "START jobid={jobid} - Staging out files") done := list( - uparallelFor(pv, stageouts + uParallelFor(pv, stageouts [path, var] := each(pv) file := vdl:absfilename(vdl:getfield(var, path = path)) provider := vdl:provider(file) @@ -286,9 +312,12 @@ "destdir={ldir} desthost={dhost} provider={provider}") //make sure we do have the directory on the client side dir:make(ldir, host=dhost, provider=provider) - restartOnError(".*", 2 - task:transfer(srchost=host, srcfile=bname, - srcdir=rdir, destdir=ldir, desthost=dhost, destprovider=provider) + policy := cdm:query(query=file) + if (policy == "DEFAULT" then( + restartOnError(".*", 2 + task:transfer(srchost=host, srcfile=bname, + srcdir=rdir, destdir=ldir, desthost=dhost, destprovider=provider))) + else(log(LOG:DEBUG, "FILE_STAGE_OUT_SKIP srcname={bname}")) ) log(LOG:DEBUG, "FILE_STAGE_OUT_END srcname={bname} srcdir={rdir} srchost={host} ", "destdir={ldir} desthost={dhost} provider={provider}") @@ -437,6 +466,7 @@ "-if ",flatten(infiles(stagein)),nl(), "-of ",flatten(outfiles(stageout)),nl(), "-k ",kickstart,nl(), + "-cdmfile ",cdm:file(),nl(), "-status ",statusMode,nl()) for(argiterator, arguments sys:file:write(wrapfile,append=true,"-a ",argiterator,nl()) @@ -497,7 +527,8 @@ "-if", flatten(infiles(stagein)), "-of", flatten(outfiles(stageout)), "-k", kickstart, - "-status", statusMode + "-cdmfile", cdm:file(), + "-status", statusMode, "-a", maybe(each(arguments)) ) directory=wfdir Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-01-25 22:00:15 UTC (rev 3220) +++ trunk/libexec/vdl-lib.xml 2010-02-03 21:12:49 UTC (rev 3221) @@ -97,4 +97,10 @@ + + + + + + Modified: trunk/libexec/vdl.k =================================================================== --- trunk/libexec/vdl.k 2010-01-25 22:00:15 UTC (rev 3220) +++ trunk/libexec/vdl.k 2010-02-03 21:12:49 UTC (rev 3221) @@ -155,7 +155,9 @@ if( sys:not(anyerrors) then( //hmm, you can append to channels! + log(LOG:DEBUG, "Starting cleanups") append(warnings, from(warnings, cleanups(cleanup))) + log(LOG:DEBUG, "Ending cleanups") ) else( log(LOG:INFO, "Errors detected. Cleanup not done.") Added: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java (rev 0) +++ trunk/src/org/globus/swift/data/Director.java 2010-02-03 21:12:49 UTC (rev 3221) @@ -0,0 +1,67 @@ + +package org.globus.swift.data; + +import java.io.File; +import java.io.IOException; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.log4j.Logger; +import org.globus.swift.data.policy.Policy; +import org.globus.swift.data.util.LineReader; +import org.griphyn.vdl.karajan.Loader; + +/** + * Manages CDM policies for files based on pattern matching. + * @author wozniak + * */ + +public class Director { + private static final Logger logger = Logger.getLogger(Director.class); + + /** + Save the location of the given CDM policy file + */ + static File policyFile; + + /** + Maps from Patterns to Policies + */ + static Map map; + + static { + map = new LinkedHashMap(); + } + + public static void load(File policyFile) throws IOException { + logger.info("loading: " + policyFile); + Director.policyFile = policyFile; + LineReader lines = new LineReader(); + List list = lines.read(policyFile); + for (Iterator it = list.iterator(); it.hasNext(); ) { + String s = (String) it.next(); + addLine(s); + } + } + + static void addLine(String s) { + String[] tokens = LineReader.tokenize(s); + Pattern pattern = Pattern.compile(tokens[0]); + Policy policy = Policy.valueOf(tokens[1]); + map.put(pattern, policy); + } + + public static Policy lookup(String file) { + for (Iterator it = map.keySet().iterator(); it.hasNext(); ) { + Pattern pattern = (Pattern) it.next(); + Matcher matcher = pattern.matcher(file); + if (matcher.matches()) + return (Policy) map.get(pattern); + } + return Policy.DEFAULT; + } +} Added: trunk/src/org/globus/swift/data/Query.java =================================================================== --- trunk/src/org/globus/swift/data/Query.java (rev 0) +++ trunk/src/org/globus/swift/data/Query.java 2010-02-03 21:12:49 UTC (rev 3221) @@ -0,0 +1,36 @@ +package org.globus.swift.data; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; + +import org.globus.swift.data.policy.Policy; + +public class Query extends FunctionsCollection { + + public static final Arg PA_QUERY = new Arg.Positional("query"); + + static { + setArguments("cdm_query", new Arg[]{ PA_QUERY }); + setArguments("cdm_file", new Arg[]{}); + } + + public String cdm_query(VariableStack stack) throws ExecutionException { + String file = (String) PA_QUERY.getValue(stack); + Policy policy = Director.lookup(file); + return policy.toString(); + } + + public String cdm_file(VariableStack stack) throws ExecutionException { + String file = ""; + if (Director.policyFile != null) + file = Director.policyFile.toString(); + return file; + } +} Added: trunk/src/org/globus/swift/data/policy/Default.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Default.java (rev 0) +++ trunk/src/org/globus/swift/data/policy/Default.java 2010-02-03 21:12:49 UTC (rev 3221) @@ -0,0 +1,9 @@ + +package org.globus.swift.data.policy; + +public class Default extends Policy { + + public String toString() { + return "DEFAULT"; + } +} Added: trunk/src/org/globus/swift/data/policy/Direct.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Direct.java (rev 0) +++ trunk/src/org/globus/swift/data/policy/Direct.java 2010-02-03 21:12:49 UTC (rev 3221) @@ -0,0 +1,8 @@ +package org.globus.swift.data.policy; + +public class Direct extends Policy { + + public String toString() { + return "DIRECT"; + } +} Added: trunk/src/org/globus/swift/data/policy/Policy.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Policy.java (rev 0) +++ trunk/src/org/globus/swift/data/policy/Policy.java 2010-02-03 21:12:49 UTC (rev 3221) @@ -0,0 +1,21 @@ + +package org.globus.swift.data.policy; + +import java.util.List; + +public class Policy { + + public static Policy DEFAULT = valueOf("default"); + + public Policy() + {} + + public static Policy valueOf(String token) { + if (token.compareToIgnoreCase("default") == 0) + return new Default(); + else if (token.compareToIgnoreCase("direct") == 0) + return new Direct(); + + return null; + } +} Added: trunk/src/org/globus/swift/data/util/LineReader.java =================================================================== --- trunk/src/org/globus/swift/data/util/LineReader.java (rev 0) +++ trunk/src/org/globus/swift/data/util/LineReader.java 2010-02-03 21:12:49 UTC (rev 3221) @@ -0,0 +1,78 @@ + +package org.globus.swift.data.util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; + +public class LineReader { + + public LineReader() + {} + + public static List read(File file) throws FileNotFoundException { + BufferedReader reader = + new BufferedReader(new FileReader(file)); + return read(reader); + } + + public static List read(String s) { + BufferedReader reader = + new BufferedReader(new StringReader(s)); + return read(reader); + } + + public static List read(BufferedReader reader) { + List result = new ArrayList(); + try + { + String prevline = ""; + String line = ""; + while ((line = reader.readLine()) != null) { + int hash = line.indexOf("#"); + if (hash >= 0) + line = line.substring(0,hash); + line = (prevline + " " + line).trim(); + if (line.endsWith("\\")) { + line = line.substring(0, line.length()-2); + prevline = line; + continue; + } + else { + prevline = ""; + line = line.trim(); + if (line.length() > 0) + result.add(line); + } + } + reader.close(); + } + catch (IOException e) + { + System.out.println("LineReader: I/O problem."); + return null; + } + return result; + } + + public static String[] tokenize(String line) { + if (line == null) + return null; + List words = new ArrayList(); + String[] ws = line.split("\\s"); + for (int i = 0; i < ws.length; i++) + if (ws[i].length() > 0) + words.add(ws[i]); + String[] result = new String[words.size()]; + for (int i = 0; i < words.size(); i++) + result[i] = (String) words.get(i); + return result; + } +} From noreply at svn.ci.uchicago.edu Wed Feb 3 16:23:18 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 3 Feb 2010 16:23:18 -0600 (CST) Subject: [Swift-commit] r3222 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20100203222318.5B4549CC89@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-03 16:23:18 -0600 (Wed, 03 Feb 2010) New Revision: 3222 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: Should have been included in r3221 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-02-03 21:12:49 UTC (rev 3221) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-02-03 22:23:18 UTC (rev 3222) @@ -38,6 +38,7 @@ import org.globus.cog.karajan.workflow.service.channels.AbstractKarajanChannel; import org.globus.cog.util.ArgumentParser; import org.globus.cog.util.ArgumentParserException; +import org.globus.swift.data.Director; import org.griphyn.vdl.engine.Karajan; import org.griphyn.vdl.karajan.functions.ConfigProperty; import org.griphyn.vdl.karajan.lib.Execute; @@ -64,6 +65,7 @@ public static final String ARG_VERBOSE = "verbose"; public static final String ARG_DEBUG = "debug"; public static final String ARG_LOGFILE = "logfile"; + public static final String ARG_CDMFILE = "cdm.file"; public static final String ARG_RUNID = "runid"; public static final String ARG_TUI = "tui"; public static final String ARG_RECOMPILE = "recompile"; @@ -90,6 +92,9 @@ if (ap.isPresent(ARG_MONITOR)) { new Monitor().start(); } + if (ap.isPresent(ARG_CDMFILE)) { + loadCDM(ap); + } if (!ap.hasValue(ArgumentParser.DEFAULT)) { error("No SwiftScript program specified"); } @@ -209,6 +214,20 @@ System.err.print("For usage information: swift -help\n\n"); } + static void loadCDM(ArgumentParser ap) { + String cdmString = null; + try { + cdmString = ap.getStringValue(ARG_CDMFILE); + File cdmFile = new File(cdmString); + Director.load(cdmFile); + } + catch (IOException e) { + logger.debug("Detailed exception:", e); + error("Could not start execution.\n\t" + + "Could not read given CDM policy file: " + cdmString); + } + } + public static String compile(String project) throws FileNotFoundException, ParsingException, IncorrectInvocationException, CompilationException, IOException { @@ -396,6 +415,11 @@ + "uses the name of the SwiftScript program being run and additional information to make the name unique.", "file", ArgumentParser.OPTIONAL); ap + .addOption( + ARG_CDMFILE, + "Specifies a CDM policy file.", + "file", ArgumentParser.OPTIONAL); + ap .addOption( ARG_RUNID, "Specifies the run identifier. This must be unique for every invocation of a workflow and is used in several places to keep files from different executions cleanly separated. By default, a datestamp and random number are used to generate a run identifier.", From noreply at svn.ci.uchicago.edu Wed Feb 3 16:23:42 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 3 Feb 2010 16:23:42 -0600 (CST) Subject: [Swift-commit] r3223 - trunk/libexec Message-ID: <20100203222342.927DD9CC89@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-03 16:23:42 -0600 (Wed, 03 Feb 2010) New Revision: 3223 Modified: trunk/libexec/_swiftwrap Log: Fix to handle multiple input files. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-02-03 22:23:18 UTC (rev 3222) +++ trunk/libexec/_swiftwrap 2010-02-03 22:23:42 UTC (rev 3223) @@ -316,11 +316,12 @@ logstate "LINK_INPUTS" for L in $INF ; do + log "L: $L" CDM_POLICY=$( perl shared/cdm.pl $L < shared/$CDM_FILE ) log "CDM_POLICY: $CDM_POLICY" if [ $CDM_POLICY != "DEFAULT" ]; then eval cdm $DIR "INPUT" $L $CDM_POLICY - break + continue fi if [ $COPYNOTLINK = 1 ]; then cp "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO From noreply at svn.ci.uchicago.edu Thu Feb 4 12:33:35 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 4 Feb 2010 12:33:35 -0600 (CST) Subject: [Swift-commit] r3224 - trunk/libexec Message-ID: <20100204183335.EB6859CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-04 12:33:35 -0600 (Thu, 04 Feb 2010) New Revision: 3224 Modified: trunk/libexec/cdm.pl Log: Bug fix for CDM DEFAULT case. Modified: trunk/libexec/cdm.pl =================================================================== --- trunk/libexec/cdm.pl 2010-02-03 22:23:42 UTC (rev 3223) +++ trunk/libexec/cdm.pl 2010-02-04 18:33:35 UTC (rev 3224) @@ -21,10 +21,11 @@ $pattern = $_; if ($file =~ /$pattern/) { - print("$$hash{$pattern}\n"); + $result = "$$hash{$pattern}\n"; last; } } + print "$result\n"; } # Command-line arguments: From noreply at svn.ci.uchicago.edu Thu Feb 4 12:35:53 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 4 Feb 2010 12:35:53 -0600 (CST) Subject: [Swift-commit] r3225 - trunk/libexec Message-ID: <20100204183553.C70709CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-04 12:35:53 -0600 (Thu, 04 Feb 2010) New Revision: 3225 Modified: trunk/libexec/_swiftwrap Log: Output fixes for CDM files. Also contains whitespace style corrections. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-02-04 18:33:35 UTC (rev 3224) +++ trunk/libexec/_swiftwrap 2010-02-04 18:35:53 UTC (rev 3225) @@ -1,7 +1,7 @@ # this script must be invoked inside of bash, not plain sh infosection() { - echo >& "$INFO" + echo >& "$INFO" echo "_____________________________________________________________________________" >& "$INFO" echo >& "$INFO" echo " $1" >& "$INFO" @@ -98,7 +98,7 @@ done VALUE="${VALUE:1}" elif [ "$PARAMFILE" != "" ] && grep -E "^$NAME " $PARAMFILE ; then - VALUE=$(grep -E "^$NAME " $PARAMFILE | cut -d ' ' -f 2-) + VALUE=$(grep -E "^$NAME " $PARAMFILE | cut -d ' ' -f 2-) else fail 254 "Missing $NAME argument" fi @@ -113,35 +113,48 @@ exec 3>&- } +contains() { + ARRAY=$1 + X=$2 + + for a in ${!ARRAY} + do + if [[ ${a} == ${X} ]]; then + return 0 + fi + done + return 1 +} + cdm() { - log "CDM: $@" + log "CDM: $@" - local JOBDIR=$1 # Given jobdir - local MODE=$2 # INPUT or OUTPUT - local FILE=$3 # User file - local POLICY=$4 # DIRECT, BROADCAST, ... - shift 4 - local ARGS=$@ + local JOBDIR=$1 # Given jobdir + local MODE=$2 # INPUT or OUTPUT + local FILE=$3 # User file + local POLICY=$4 # DIRECT, BROADCAST, ... + shift 4 + local ARGS=$@ - case $POLICY in - DIRECT) - DIRECT_DIR=${ARGS[0]} - log "CDM[DIRECT]: Linking $JOBDIR/$FILE to $DIRECT_DIR/$FILE" - if [ $MODE == "INPUT" ]; then - [ -f "$DIRECT_DIR/$FILE" ] - checkError 254 "CDM[DIRECT]: $DIRECT_DIR/$FILE does not exist!" - ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE - checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!" - elif [ $MODE == "OUTPUT" ]; then - touch $DIRECT_DIR/$FILE - checkError 254 "CDM[DIRECT]: Touching $DIRECT_DIR/$FILE failed!" - ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE - checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!" - else - fail 254 "Unknown MODE: $MODE" - fi - ;; - esac + case $POLICY in + DIRECT) + DIRECT_DIR=${ARGS[0]} + log "CDM[DIRECT]: Linking $JOBDIR/$FILE to $DIRECT_DIR/$FILE" + if [ $MODE == "INPUT" ]; then + [ -f "$DIRECT_DIR/$FILE" ] + checkError 254 "CDM[DIRECT]: $DIRECT_DIR/$FILE does not exist!" + ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE + checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!" + elif [ $MODE == "OUTPUT" ]; then + touch $DIRECT_DIR/$FILE + checkError 254 "CDM[DIRECT]: Touching $DIRECT_DIR/$FILE failed!" + ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE + checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!" + else + fail 254 "Unknown MODE: $MODE" + fi + ;; + esac } COMMANDLINE=$@ @@ -257,13 +270,13 @@ fi if [ "X$SCRATCH" != "X" ]; then - log "Job directory mode is: local copy" - DIR=$SCRATCH/$JOBDIR/$ID - COPYNOTLINK=1 + log "Job directory mode is: local copy" + DIR=$SCRATCH/$JOBDIR/$ID + COPYNOTLINK=1 else - log "Job directory mode is: link on shared filesystem" - DIR=jobs/$JOBDIR/$ID - COPYNOTLINK=0 + log "Job directory mode is: link on shared filesystem" + DIR=jobs/$JOBDIR/$ID + COPYNOTLINK=0 fi PATH=$PATH:/bin:/usr/bin @@ -337,13 +350,13 @@ done logstate "LINK_CDM_OUTPUTS" +SKIPPED_OUTPUT=() for L in $OUTF ; do CDM_POLICY=$( perl shared/cdm.pl $L < shared/$CDM_FILE ) log "CDM_POLICY: $CDM_POLICY" if [ $CDM_POLICY != "DEFAULT" ]; then eval cdm $DIR "OUTPUT" $L $CDM_POLICY - eval CDM_OUTPUT_$L="SKIP" - break + SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L ) fi done @@ -406,7 +419,6 @@ MISSING= for O in $OUTF ; do - log "checking: $O" if [ ! -f "$DIR/$O" ]; then if [ "$MISSING" == "" ]; then MISSING=$O @@ -421,12 +433,12 @@ logstate "MOVING_OUTPUTS $OUTF" for O in $OUTF ; do - log "output: $O ${CDM_OUTPUTS}" - if [ $( eval echo "$"$( echo CDM_OUTPUT_$O ) ) != "SKIP" ]; then - mv "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >& "$INFO" - checkError 254 "Failed to move output file $O to shared directory" + log "output: $O" + if ! contains SKIPPED_OUTPUT $O ; then + mv -v "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >& "$INFO" + checkError 254 "Failed to move output file $O to shared directory" else - log "CDM: SKIP OUTPUT: $O" + log "CDM: SKIPPED_OUTPUT: $O" fi done @@ -445,10 +457,14 @@ closeinfo if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then - mkdir -p "$WFDIR/info/$JOBDIR" + mkdir -p "$WFDIR/info/$JOBDIR" mv "$INFODIR/${ID}-info" "$WFDIR/info/$JOBDIR/${ID}-info" fi # ensure we exit with a 0 after a successful execution exit 0 +# Local Variables: +# mode: sh +# sh-basic-offset: 8 +# End: From noreply at svn.ci.uchicago.edu Thu Feb 4 17:31:53 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 4 Feb 2010 17:31:53 -0600 (CST) Subject: [Swift-commit] r3226 - trunk/src/org/globus/swift/data Message-ID: <20100204233153.60B9C9CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-04 17:31:53 -0600 (Thu, 04 Feb 2010) New Revision: 3226 Modified: trunk/src/org/globus/swift/data/Director.java Log: Provide a simple command-line interface for policy queries. Modified: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java 2010-02-04 18:35:53 UTC (rev 3225) +++ trunk/src/org/globus/swift/data/Director.java 2010-02-04 23:31:53 UTC (rev 3226) @@ -64,4 +64,31 @@ } return Policy.DEFAULT; } + + /** + * Check the policy effect of name with respect to policy_file + * @param args {name, policy_file} + */ + public static void main(String[] args) { + if (args.length != 2) { + System.out.println("Incorrect args"); + System.exit(1); + } + + try { + + String name = args[0]; + File policyFile = new File(args[1]); + if (! policyFile.exists()) { + System.out.println("Policy file does not exist: " + + args[1]); + } + load(policyFile); + Policy policy = lookup(name); + System.out.println(name + ": " + policy); + } catch (Exception e) { + e.printStackTrace(); + System.exit(2); + } + } } From noreply at svn.ci.uchicago.edu Thu Feb 4 17:34:56 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 4 Feb 2010 17:34:56 -0600 (CST) Subject: [Swift-commit] r3227 - trunk/libexec Message-ID: <20100204233456.4803D9CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-04 17:34:56 -0600 (Thu, 04 Feb 2010) New Revision: 3227 Modified: trunk/libexec/_swiftwrap Log: Better use of CDM. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-02-04 23:31:53 UTC (rev 3226) +++ trunk/libexec/_swiftwrap 2010-02-04 23:34:56 UTC (rev 3227) @@ -126,9 +126,20 @@ return 1 } -cdm() { - log "CDM: $@" +cdm_lookup() { + FILE=$1 + CDM_FILE=$2 + RESULT="DEFAULT" + if [ -f shared/cdm.pl ]; then + RESULT=$( perl shared/cdm.pl $FILE < $CDM_FILE ) + fi + echo $RESULT +} + +cdm_action() { + log "CDM_ACTION: $@" + local JOBDIR=$1 # Given jobdir local MODE=$2 # INPUT or OUTPUT local FILE=$3 # User file @@ -200,6 +211,7 @@ checkEmpty "$JOBDIR" "Missing job directory prefix" mkdir -p $INFODIR closeinfo + rm -f "$INFODIR/${ID}-info" openinfo "$INFODIR/${ID}-info" @@ -330,10 +342,10 @@ logstate "LINK_INPUTS" for L in $INF ; do log "L: $L" - CDM_POLICY=$( perl shared/cdm.pl $L < shared/$CDM_FILE ) - log "CDM_POLICY: $CDM_POLICY" + CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE ) + log "CDM_POLICY: $CDM_POLICY" if [ $CDM_POLICY != "DEFAULT" ]; then - eval cdm $DIR "INPUT" $L $CDM_POLICY + eval cdm_action $DIR "INPUT" $L $CDM_POLICY continue fi if [ $COPYNOTLINK = 1 ]; then @@ -352,7 +364,7 @@ logstate "LINK_CDM_OUTPUTS" SKIPPED_OUTPUT=() for L in $OUTF ; do - CDM_POLICY=$( perl shared/cdm.pl $L < shared/$CDM_FILE ) + CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE ) log "CDM_POLICY: $CDM_POLICY" if [ $CDM_POLICY != "DEFAULT" ]; then eval cdm $DIR "OUTPUT" $L $CDM_POLICY From noreply at svn.ci.uchicago.edu Thu Feb 4 17:47:12 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 4 Feb 2010 17:47:12 -0600 (CST) Subject: [Swift-commit] r3228 - trunk/libexec Message-ID: <20100204234712.8E9E99CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-04 17:47:12 -0600 (Thu, 04 Feb 2010) New Revision: 3228 Modified: trunk/libexec/_swiftwrap Log: Bug fix to CDM call. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-02-04 23:34:56 UTC (rev 3227) +++ trunk/libexec/_swiftwrap 2010-02-04 23:47:12 UTC (rev 3228) @@ -365,9 +365,9 @@ SKIPPED_OUTPUT=() for L in $OUTF ; do CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE ) - log "CDM_POLICY: $CDM_POLICY" + log "CDM_POLICY: $L $CDM_POLICY" if [ $CDM_POLICY != "DEFAULT" ]; then - eval cdm $DIR "OUTPUT" $L $CDM_POLICY + eval cdm_action $DIR "OUTPUT" $L $CDM_POLICY SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L ) fi done From noreply at svn.ci.uchicago.edu Fri Feb 5 11:34:48 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Feb 2010 11:34:48 -0600 (CST) Subject: [Swift-commit] r3229 - trunk/libexec Message-ID: <20100205173448.7C93D9CCB1@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-05 11:34:48 -0600 (Fri, 05 Feb 2010) New Revision: 3229 Modified: trunk/libexec/_swiftwrap Log: Output cleanup. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-02-04 23:47:12 UTC (rev 3228) +++ trunk/libexec/_swiftwrap 2010-02-05 17:34:48 UTC (rev 3229) @@ -341,9 +341,8 @@ logstate "LINK_INPUTS" for L in $INF ; do - log "L: $L" CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE ) - log "CDM_POLICY: $CDM_POLICY" + log "CDM_POLICY: $L -> $CDM_POLICY" if [ $CDM_POLICY != "DEFAULT" ]; then eval cdm_action $DIR "INPUT" $L $CDM_POLICY continue @@ -365,7 +364,7 @@ SKIPPED_OUTPUT=() for L in $OUTF ; do CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE ) - log "CDM_POLICY: $L $CDM_POLICY" + log "CDM_POLICY: $L -> $CDM_POLICY" if [ $CDM_POLICY != "DEFAULT" ]; then eval cdm_action $DIR "OUTPUT" $L $CDM_POLICY SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L ) From noreply at svn.ci.uchicago.edu Fri Feb 5 16:58:42 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Feb 2010 16:58:42 -0600 (CST) Subject: [Swift-commit] r3230 - trunk/libexec Message-ID: <20100205225842.4B7899CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-05 16:58:42 -0600 (Fri, 05 Feb 2010) New Revision: 3230 Modified: trunk/libexec/cdm.pl Log: Ensure that the CDM policy file is queried in-order. Modified: trunk/libexec/cdm.pl =================================================================== --- trunk/libexec/cdm.pl 2010-02-05 17:34:48 UTC (rev 3229) +++ trunk/libexec/cdm.pl 2010-02-05 22:58:42 UTC (rev 3230) @@ -12,11 +12,12 @@ sub cdm_lookup { - my $hash = $_[0]; - my $file = $_[1]; + my $keys = $_[0]; + my $hash = $_[1]; + my $file = $_[2]; $result = "DEFAULT"; - foreach (keys %$hash) + foreach (@$keys) { $pattern = $_; if ($file =~ /$pattern/) @@ -32,7 +33,8 @@ $file = $ARGV[0]; # Read fs.data off of stdin: -%map = (); + at keys = (); +%map = (); while () { chomp; @@ -41,7 +43,8 @@ @tokens = split(/ /, $_); $key = shift(@tokens); $rest = join(' ', @tokens); + @keys = (@keys, $key); $map{$key} = $rest; } -cdm_lookup(\%map, $file); +cdm_lookup(\@keys, \%map, $file); From noreply at svn.ci.uchicago.edu Fri Feb 5 17:02:18 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Feb 2010 17:02:18 -0600 (CST) Subject: [Swift-commit] r3231 - in trunk: . libexec src/org/globus/swift/data src/org/globus/swift/data/policy Message-ID: <20100205230218.51B679CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-05 17:02:18 -0600 (Fri, 05 Feb 2010) New Revision: 3231 Added: trunk/libexec/cdm_broadcast.sh trunk/src/org/globus/swift/data/Action.java trunk/src/org/globus/swift/data/policy/Broadcast.java Modified: trunk/.project trunk/build.xml trunk/libexec/_swiftwrap trunk/libexec/vdl-int.k trunk/libexec/vdl-lib.xml trunk/src/org/globus/swift/data/Director.java trunk/src/org/globus/swift/data/policy/Direct.java trunk/src/org/globus/swift/data/policy/Policy.java Log: CDM BROADCAST framework with implementation for workstation. Modified: trunk/.project =================================================================== --- trunk/.project 2010-02-05 22:58:42 UTC (rev 3230) +++ trunk/.project 2010-02-05 23:02:18 UTC (rev 3231) @@ -3,13 +3,30 @@ swift + abstraction-common + karajan - org.eclipse.jdt.core.javabuilder + org.eclipse.ui.externaltools.ExternalToolBuilder + full,incremental, + + LaunchConfigHandle + <project>/.externalToolBuilders/org.eclipse.jdt.core.javabuilder.launch + + + org.eclipse.ui.externaltools.ExternalToolBuilder + full,incremental, + + + LaunchConfigHandle + <project>/.externalToolBuilders/Swift_Builder.launch + + + org.eclipse.jdt.core.javanature Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2010-02-05 22:58:42 UTC (rev 3230) +++ trunk/build.xml 2010-02-05 23:02:18 UTC (rev 3231) @@ -254,6 +254,7 @@ + Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-02-05 22:58:42 UTC (rev 3230) +++ trunk/libexec/_swiftwrap 2010-02-05 23:02:18 UTC (rev 3231) @@ -165,6 +165,18 @@ fail 254 "Unknown MODE: $MODE" fi ;; + BROADCAST) + BROADCAST_DIR=${ARGS[0]} + log "CDM[BROADCAST]: Linking $JOBDIR/$FILE to $BROADCAST_DIR/$FILE" + if [ $MODE == "INPUT" ]; then + [ -f "$BROADCAST_DIR/$FILE" ] + checkError 254 "CDM[BROADCAST]: $BROADCAST_DIR/$FILE does not exist!" + ln -s $BROADCAST_DIR/$FILE $JOBDIR/$FILE + checkError 254 "CDM[BROADCAST]: Linking to $BROADCAST_DIR/$FILE failed!" + else + fail 254 "Cannot BROADCAST an output file!" + fi + ;; esac } Added: trunk/libexec/cdm_broadcast.sh =================================================================== --- trunk/libexec/cdm_broadcast.sh (rev 0) +++ trunk/libexec/cdm_broadcast.sh 2010-02-05 23:02:18 UTC (rev 3231) @@ -0,0 +1,14 @@ +#!/bin/sh + +SWIFT_HOME=$( dirname $( dirname $0 ) ) +LOG=${SWIFT_HOME}/etc/cdm_broadcast.log +{ + set -x + + FILE=$1 + DIR=$2 + DEST=$3 + + cp -v ${DIR}/${FILE} ${DEST} + +} >> ${LOG} 2>&1 Property changes on: trunk/libexec/cdm_broadcast.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-02-05 22:58:42 UTC (rev 3230) +++ trunk/libexec/vdl-int.k 2010-02-05 23:02:18 UTC (rev 3231) @@ -288,6 +288,9 @@ restartOnError(".*", 2 task:transfer(srcprovider=provider, srchost=srchost, srcfile=srcfile, srcdir=srcdir, desthost=desthost, destdir=destdir))) + policy == "BROADCAST" then( + log(LOG:DEBUG, "FILE_STAGE_IN_BROADCAST file={srcfile} policy={policy}") + cdm:broadcast(srcfile=srcfile, srcdir=srcdir)) else(log(LOG:DEBUG, "FILE_STAGE_IN_SKIP file={srcfile} policy={policy}"))) log(LOG:DEBUG, "FILE_STAGE_IN_END file={srcfile} srchost={srchost} srcdir={srcdir} srcname={srcfile} ", "desthost={desthost} destdir={destdir} provider={provider}") Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-02-05 22:58:42 UTC (rev 3230) +++ trunk/libexec/vdl-lib.xml 2010-02-05 23:02:18 UTC (rev 3231) @@ -101,6 +101,7 @@ + Added: trunk/src/org/globus/swift/data/Action.java =================================================================== --- trunk/src/org/globus/swift/data/Action.java (rev 0) +++ trunk/src/org/globus/swift/data/Action.java 2010-02-05 23:02:18 UTC (rev 3231) @@ -0,0 +1,38 @@ +package org.globus.swift.data; + +import java.io.IOException; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; +import org.globus.swift.data.policy.Broadcast; +import org.globus.swift.data.policy.Policy; + +public class Action extends FunctionsCollection { + + public static final Arg PA_FILE = new Arg.Positional("srcfile"); + public static final Arg PA_DIR = new Arg.Positional("srcdir"); + + static { + setArguments("cdm_broadcast", new Arg[]{ PA_FILE, PA_DIR }); + } + + public void cdm_broadcast(VariableStack stack) throws ExecutionException { + String srcfile = (String) PA_FILE.getValue(stack); + String srcdir = (String) PA_DIR.getValue(stack); + + Policy policy = Director.lookup(srcfile); + + if (!(policy instanceof Broadcast)) { + throw new RuntimeException("Attempting to BROADCAST the wrong file"); + } + + if (srcdir == "") { + srcdir = "."; + } + + Broadcast broadcast = (Broadcast) policy; + broadcast.action(srcfile, srcdir); + } +} Modified: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java 2010-02-05 22:58:42 UTC (rev 3230) +++ trunk/src/org/globus/swift/data/Director.java 2010-02-05 23:02:18 UTC (rev 3231) @@ -3,10 +3,13 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -31,12 +34,13 @@ /** Maps from Patterns to Policies */ - static Map map; + static Map map = new LinkedHashMap(); - static { - map = new LinkedHashMap(); - } - + /** + Remember the files we have broadcasted + */ + static Set broadcasted = new HashSet(); + public static void load(File policyFile) throws IOException { logger.info("loading: " + policyFile); Director.policyFile = policyFile; @@ -52,6 +56,8 @@ String[] tokens = LineReader.tokenize(s); Pattern pattern = Pattern.compile(tokens[0]); Policy policy = Policy.valueOf(tokens[1]); + List tokenList = Arrays.asList(tokens); + policy.settings(tokenList.subList(2,tokenList.size())); map.put(pattern, policy); } @@ -65,6 +71,14 @@ return Policy.DEFAULT; } + public static boolean broadcasted(String file, String dir) { + return broadcasted.contains(dir+"/"+file); + } + + public static void broadcast(String file, String dir) { + broadcasted.add(dir+"/"+file); + } + /** * Check the policy effect of name with respect to policy_file * @param args {name, policy_file} Added: trunk/src/org/globus/swift/data/policy/Broadcast.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Broadcast.java (rev 0) +++ trunk/src/org/globus/swift/data/policy/Broadcast.java 2010-02-05 23:02:18 UTC (rev 3231) @@ -0,0 +1,50 @@ +package org.globus.swift.data.policy; + +import java.util.List; + +import org.globus.swift.data.Director; + +public class Broadcast extends Policy { + + String destination = null; + + @Override + public void settings(List settings) { + try { + destination = settings.get(0); + } + catch (Exception e) { + throw new RuntimeException("Incorrect settings for BROADCAST"); + } + } + + public void action(String srcfile, String srcdir) { + if (! Director.broadcasted(srcfile, srcdir)) + callScript(srcfile, srcdir, destination); + } + + void callScript(String srcfile, String srcdir, String destination) { + String home = System.getProperties().getProperty("swift.home"); + try { + String[] line = new String[4]; + line[0] = home+"/libexec/cdm_broadcast.sh"; + line[1] = srcfile; + line[2] = srcdir; + line[3] = destination; + Process process = Runtime.getRuntime().exec(line); + process.waitFor(); + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("Could not launch external broadcast"); + } + } + + public String getDestination() { + return destination; + } + + public String toString() { + return "BROADCAST"; + } +} Modified: trunk/src/org/globus/swift/data/policy/Direct.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Direct.java 2010-02-05 22:58:42 UTC (rev 3230) +++ trunk/src/org/globus/swift/data/policy/Direct.java 2010-02-05 23:02:18 UTC (rev 3231) @@ -1,8 +1,14 @@ package org.globus.swift.data.policy; +import java.util.List; + public class Direct extends Policy { public String toString() { return "DIRECT"; } + + @Override + public void settings(List tokens) { + } } Modified: trunk/src/org/globus/swift/data/policy/Policy.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Policy.java 2010-02-05 22:58:42 UTC (rev 3230) +++ trunk/src/org/globus/swift/data/policy/Policy.java 2010-02-05 23:02:18 UTC (rev 3231) @@ -3,19 +3,22 @@ import java.util.List; -public class Policy { +public abstract class Policy { public static Policy DEFAULT = valueOf("default"); public Policy() {} + public abstract void settings(List tokens); + public static Policy valueOf(String token) { if (token.compareToIgnoreCase("default") == 0) return new Default(); else if (token.compareToIgnoreCase("direct") == 0) return new Direct(); - + else if (token.compareToIgnoreCase("broadcast") == 0) + return new Broadcast(); return null; } } From noreply at svn.ci.uchicago.edu Tue Feb 9 15:40:26 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 9 Feb 2010 15:40:26 -0600 (CST) Subject: [Swift-commit] r3232 - in trunk/src/org/globus/swift/data: . policy Message-ID: <20100209214026.258789CCA4@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-09 15:40:25 -0600 (Tue, 09 Feb 2010) New Revision: 3232 Added: trunk/src/org/globus/swift/data/policy/Gather.java Modified: trunk/src/org/globus/swift/data/Director.java trunk/src/org/globus/swift/data/policy/Policy.java Log: Support for CDM properties and the GATHER policy. Modified: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java 2010-02-05 23:02:18 UTC (rev 3231) +++ trunk/src/org/globus/swift/data/Director.java 2010-02-09 21:40:25 UTC (rev 3232) @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -32,9 +33,14 @@ static File policyFile; /** - Maps from Patterns to Policies + Maps from Patterns to Policies for fs.data rules */ - static Map map = new LinkedHashMap(); + static Map map = new LinkedHashMap(); + + /** + Maps from String names to String values for fs.data properties + */ + static Map properties = new HashMap(); /** Remember the files we have broadcasted @@ -54,19 +60,43 @@ static void addLine(String s) { String[] tokens = LineReader.tokenize(s); - Pattern pattern = Pattern.compile(tokens[0]); - Policy policy = Policy.valueOf(tokens[1]); + String type = tokens[0]; + if (type.equals("rule")) { + addRule(tokens); + } + else if (type.equals("property")) { + addProperty(tokens); + } + + } + + static void addRule(String[] tokens) { + Pattern pattern = Pattern.compile(tokens[1]); + Policy policy = Policy.valueOf(tokens[2]); List tokenList = Arrays.asList(tokens); - policy.settings(tokenList.subList(2,tokenList.size())); + policy.settings(tokenList.subList(3,tokenList.size())); map.put(pattern, policy); } - + + static void addProperty(String[] tokens) { + String name = tokens[1]; + String value = concat(tokens, 2); + properties.put(name, value); + } + + static String concat(String[] tokens, int start) { + StringBuilder result = new StringBuilder(); + for (int i = start; i < tokens.length; i++) { + result.append(tokens[i]); + } + return result.toString(); + } + public static Policy lookup(String file) { - for (Iterator it = map.keySet().iterator(); it.hasNext(); ) { - Pattern pattern = (Pattern) it.next(); + for (Pattern pattern : map.keySet()) { Matcher matcher = pattern.matcher(file); if (matcher.matches()) - return (Policy) map.get(pattern); + return map.get(pattern); } return Policy.DEFAULT; } Added: trunk/src/org/globus/swift/data/policy/Gather.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Gather.java (rev 0) +++ trunk/src/org/globus/swift/data/policy/Gather.java 2010-02-09 21:40:25 UTC (rev 3232) @@ -0,0 +1,14 @@ +package org.globus.swift.data.policy; + +import java.util.List; + +public class Gather extends Policy { + + public String toString() { + return "GATHER"; + } + + @Override + public void settings(List tokens) { + } +} Modified: trunk/src/org/globus/swift/data/policy/Policy.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Policy.java 2010-02-05 23:02:18 UTC (rev 3231) +++ trunk/src/org/globus/swift/data/policy/Policy.java 2010-02-09 21:40:25 UTC (rev 3232) @@ -19,6 +19,8 @@ return new Direct(); else if (token.compareToIgnoreCase("broadcast") == 0) return new Broadcast(); + else if (token.compareToIgnoreCase("gather") == 0) + return new Gather(); return null; } } From noreply at svn.ci.uchicago.edu Tue Feb 9 17:28:09 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 9 Feb 2010 17:28:09 -0600 (CST) Subject: [Swift-commit] r3233 - trunk/src/org/globus/swift/data Message-ID: <20100209232809.B470F9CC93@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-09 17:28:09 -0600 (Tue, 09 Feb 2010) New Revision: 3233 Modified: trunk/src/org/globus/swift/data/Director.java trunk/src/org/globus/swift/data/Query.java Log: Allow Karajan to query CDM properties. Modified: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java 2010-02-09 21:40:25 UTC (rev 3232) +++ trunk/src/org/globus/swift/data/Director.java 2010-02-09 23:28:09 UTC (rev 3233) @@ -101,6 +101,13 @@ return Policy.DEFAULT; } + public static String property(String name) { + String result = properties.get(name); + if (result == null) + result = "UNSET"; + return result; + } + public static boolean broadcasted(String file, String dir) { return broadcasted.contains(dir+"/"+file); } @@ -134,5 +141,5 @@ e.printStackTrace(); System.exit(2); } - } + } } Modified: trunk/src/org/globus/swift/data/Query.java =================================================================== --- trunk/src/org/globus/swift/data/Query.java 2010-02-09 21:40:25 UTC (rev 3232) +++ trunk/src/org/globus/swift/data/Query.java 2010-02-09 23:28:09 UTC (rev 3233) @@ -15,9 +15,11 @@ public class Query extends FunctionsCollection { public static final Arg PA_QUERY = new Arg.Positional("query"); + public static final Arg PA_NAME = new Arg.Positional("name"); static { setArguments("cdm_query", new Arg[]{ PA_QUERY }); + setArguments("cdm_name", new Arg[]{ PA_NAME }); setArguments("cdm_file", new Arg[]{}); } @@ -27,6 +29,12 @@ return policy.toString(); } + public String cdm_property(VariableStack stack) throws ExecutionException { + String name = (String) PA_NAME.getValue(stack); + String value = Director.property(name); + return value; + } + public String cdm_file(VariableStack stack) throws ExecutionException { String file = ""; if (Director.policyFile != null) From noreply at svn.ci.uchicago.edu Tue Feb 9 17:28:33 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 9 Feb 2010 17:28:33 -0600 (CST) Subject: [Swift-commit] r3234 - trunk/libexec Message-ID: <20100209232833.AB4AA9CC93@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-09 17:28:33 -0600 (Tue, 09 Feb 2010) New Revision: 3234 Modified: trunk/libexec/vdl-lib.xml Log: Karajan wrapper for CDM properties. Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-02-09 23:28:09 UTC (rev 3233) +++ trunk/libexec/vdl-lib.xml 2010-02-09 23:28:33 UTC (rev 3234) @@ -100,6 +100,7 @@ + From noreply at svn.ci.uchicago.edu Tue Feb 9 17:41:41 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 9 Feb 2010 17:41:41 -0600 (CST) Subject: [Swift-commit] r3235 - trunk/src/org/globus/swift/data Message-ID: <20100209234141.5374B9CC93@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-09 17:41:41 -0600 (Tue, 09 Feb 2010) New Revision: 3235 Modified: trunk/src/org/globus/swift/data/Query.java Log: Avoid name collision. Modified: trunk/src/org/globus/swift/data/Query.java =================================================================== --- trunk/src/org/globus/swift/data/Query.java 2010-02-09 23:28:33 UTC (rev 3234) +++ trunk/src/org/globus/swift/data/Query.java 2010-02-09 23:41:41 UTC (rev 3235) @@ -19,7 +19,7 @@ static { setArguments("cdm_query", new Arg[]{ PA_QUERY }); - setArguments("cdm_name", new Arg[]{ PA_NAME }); + setArguments("cdm_get", new Arg[]{ PA_NAME }); setArguments("cdm_file", new Arg[]{}); } @@ -29,7 +29,10 @@ return policy.toString(); } - public String cdm_property(VariableStack stack) throws ExecutionException { + /** + Get a CDM property + */ + public String cdm_get(VariableStack stack) throws ExecutionException { String name = (String) PA_NAME.getValue(stack); String value = Director.property(name); return value; From noreply at svn.ci.uchicago.edu Wed Feb 10 12:34:11 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 10 Feb 2010 12:34:11 -0600 (CST) Subject: [Swift-commit] r3236 - trunk/src/org/globus/swift/data/policy Message-ID: <20100210183411.665499CC90@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-10 12:34:11 -0600 (Wed, 10 Feb 2010) New Revision: 3236 Modified: trunk/src/org/globus/swift/data/policy/Default.java Log: Added abstract method. Modified: trunk/src/org/globus/swift/data/policy/Default.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Default.java 2010-02-09 23:41:41 UTC (rev 3235) +++ trunk/src/org/globus/swift/data/policy/Default.java 2010-02-10 18:34:11 UTC (rev 3236) @@ -1,9 +1,15 @@ package org.globus.swift.data.policy; +import java.util.List; + public class Default extends Policy { public String toString() { return "DEFAULT"; } + + @Override + public void settings(List tokens) { + } } From noreply at svn.ci.uchicago.edu Fri Feb 12 14:32:49 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 12 Feb 2010 14:32:49 -0600 (CST) Subject: [Swift-commit] r3237 - trunk/libexec Message-ID: <20100212203249.695309CC8A@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-12 14:32:49 -0600 (Fri, 12 Feb 2010) New Revision: 3237 Modified: trunk/libexec/_swiftwrap trunk/libexec/cdm.pl trunk/libexec/vdl-int.k trunk/libexec/vdl-lib.xml Log: Basic CDM GATHER policy. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-02-10 18:34:11 UTC (rev 3236) +++ trunk/libexec/_swiftwrap 2010-02-12 20:32:49 UTC (rev 3237) @@ -1,4 +1,5 @@ # this script must be invoked inside of bash, not plain sh +# note that this script modifies $IFS infosection() { echo >& "$INFO" @@ -132,7 +133,7 @@ RESULT="DEFAULT" if [ -f shared/cdm.pl ]; then - RESULT=$( perl shared/cdm.pl $FILE < $CDM_FILE ) + RESULT=$( perl shared/cdm.pl lookup $FILE < $CDM_FILE ) fi echo $RESULT } @@ -177,9 +178,23 @@ fail 254 "Cannot BROADCAST an output file!" fi ;; + GATHER) + if [ $MODE == "INPUT" ]; then + fail 254 "Cannot GATHER an input file!" + fi esac } +cdm_gather() +{ + GATHER_OUTPUT=${*} + if [[ $CDM_FILE == "" ]]; then + return + fi + + cdm_gather_action $GATHER_MAX $GATHER_OUTPUT +} + COMMANDLINE=$@ # get the parent directory of the directory containing _swiftwrap, to use @@ -263,7 +278,10 @@ shift $SHIFTCOUNT getarg "-cdmfile" "$@" -CDM_FILE=$VALUE +CDM_FILE= +if [ "X$VALUE" != "X" ]; then + CDM_FILE=shared/$VALUE +fi shift $SHIFTCOUNT getarg "-status" "$@" @@ -293,6 +311,12 @@ mkdir -p $WFDIR/status/$JOBDIR fi +if [ "X$CDM_FILE" != "X" ]; then + logstate "SOURCE_CDM_LIB $WFDIR/shared/cdm_lib.sh" + source $WFDIR/shared/cdm_lib.sh + checkError 254 "Could not source: $WFDIR/shared/cdm_lib.sh" +fi + if [ "X$SCRATCH" != "X" ]; then log "Job directory mode is: local copy" DIR=$SCRATCH/$JOBDIR/$ID @@ -348,14 +372,11 @@ log "Created output directory: $DIR/$D" done -log "PWD: $( pwd )" -log "FIND: $( find . )" - logstate "LINK_INPUTS" for L in $INF ; do - CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE ) - log "CDM_POLICY: $L -> $CDM_POLICY" + CDM_POLICY=$( cdm_lookup $L $CDM_FILE ) if [ $CDM_POLICY != "DEFAULT" ]; then + log "CDM_POLICY: $L -> $CDM_POLICY" eval cdm_action $DIR "INPUT" $L $CDM_POLICY continue fi @@ -374,13 +395,17 @@ logstate "LINK_CDM_OUTPUTS" SKIPPED_OUTPUT=() +GATHER_OUTPUT=() for L in $OUTF ; do - CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE ) - log "CDM_POLICY: $L -> $CDM_POLICY" + CDM_POLICY=$( cdm_lookup $L $CDM_FILE ) if [ $CDM_POLICY != "DEFAULT" ]; then + log "CDM_POLICY: $L -> $CDM_POLICY" eval cdm_action $DIR "OUTPUT" $L $CDM_POLICY SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L ) fi + if [ $CDM_POLICY == "GATHER" ]; then + GATHER_OUTPUT=( $GATHER_OUTPUT $L ) + fi done logstate "EXECUTE" @@ -456,23 +481,21 @@ logstate "MOVING_OUTPUTS $OUTF" for O in $OUTF ; do - log "output: $O" if ! contains SKIPPED_OUTPUT $O ; then mv -v "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >& "$INFO" checkError 254 "Failed to move output file $O to shared directory" - else - log "CDM: SKIPPED_OUTPUT: $O" fi done +cdm_gather $GATHER_OUTPUT + logstate "RM_JOBDIR" -# rm -rf "$DIR" 2>&1 >& "$INFO" +rm -rf "$DIR" 2>&1 >& "$INFO" checkError 254 "Failed to remove job directory $DIR" if [ "$STATUSMODE" = "files" ]; then logstate "TOUCH_SUCCESS" touch $WFDIR/status/${JOBDIR}/${ID}-success - log "TOUCH CODE: $WFDIR/status/${JOBDIR}/${ID}-success $?" fi logstate "END" Modified: trunk/libexec/cdm.pl =================================================================== --- trunk/libexec/cdm.pl 2010-02-10 18:34:11 UTC (rev 3236) +++ trunk/libexec/cdm.pl 2010-02-12 20:32:49 UTC (rev 3237) @@ -29,22 +29,74 @@ print "$result\n"; } +sub cdm_property +{ + my $hash = $_[0]; + my $name = $_[1]; + + $result = "UNSET"; + foreach (keys %$hash) + { + $property = $_; + if ($name eq $property) + { + $result = "$$hash{$property}\n"; + last; + } + } + print "$result\n"; +} + # Command-line arguments: -$file = $ARGV[0]; +$task = $ARGV[0]; +$arg = $ARGV[1]; # Read fs.data off of stdin: @keys = (); %map = (); +%properties = (); while () { chomp; + + my $comment = index $_, "#"; + if ($comment >= 0) { + $_ = substr($_, 0, $comment); + } + next if $_ eq ""; @tokens = split(/ /, $_); - $key = shift(@tokens); - $rest = join(' ', @tokens); - @keys = (@keys, $key); - $map{$key} = $rest; + $type = shift(@tokens); + if ($type eq "rule") + { + $key = shift(@tokens); + $rest = join(' ', @tokens); + @keys = (@keys, $key); + $map{$key} = $rest; + } + elsif ($type eq "property") + { + $property = shift(@tokens); + $rest = join(' ', @tokens); + $properties{$property} = $rest; + } + else + { + die "Unknown directive: $type"; + } } -cdm_lookup(\@keys, \%map, $file); +# Do the user task: +if ($task eq "lookup") +{ + cdm_lookup(\@keys, \%map, $arg); +} +elsif ($task eq "property") +{ + cdm_property(\%properties, $arg); +} +else +{ + die "Unknown task: $task"; +} Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-02-10 18:34:11 UTC (rev 3236) +++ trunk/libexec/vdl-int.k 2010-02-12 20:32:49 UTC (rev 3237) @@ -191,6 +191,25 @@ element(cleanup, [dir, host] log(LOG:INFO, "START dir={dir} host={host}") + cdmfile := cdm:file() + log(LOG:INFO, "cdmfile {cdmfile}") + if(cdmfile != "" then( + log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}") + task:transfer(srcfile="cdm_cleanup.sh", + srcdir="{swift.home}/libexec", + desthost=host, destdir=dir) + task:transfer(srcfile="cdm_lib.sh", + srcdir="{swift.home}/libexec", + desthost=host, destdir=dir) + log(LOG:INFO, "execute: cdm_cleanup.sh") + task:execute( + executable="/bin/bash", + arguments=list("{dir}/cdm_cleanup.sh", + cdm:get("GATHER_DIR"), cdm:get("GATHER_TARGET") + sys:uid() ) + host=host, batch=true, tcprofile(host)) + ) + ) if(vdl:configProperty("sitedir.keep") == "false" task:execute( vdl:siteprofile(host, "swift:cleanupCommand"), @@ -259,6 +278,9 @@ doStageinFile(provider="file", srchost="localhost", srcfile="cdm.pl", srcdir=libexec, desthost=host, destdir=dir, size=file:size("{libexec}/cdm.pl}"), policy="DEFAULT") + doStageinFile(provider="file", srchost="localhost", srcfile="cdm_lib.sh", + srcdir=libexec, desthost=host, destdir=dir, + size=file:size("{libexec}/cdm_lib.sh}"), policy="DEFAULT") )) uParallelFor(file, files Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-02-10 18:34:11 UTC (rev 3236) +++ trunk/libexec/vdl-lib.xml 2010-02-12 20:32:49 UTC (rev 3237) @@ -100,7 +100,7 @@ - + From noreply at svn.ci.uchicago.edu Mon Feb 15 22:44:01 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 15 Feb 2010 22:44:01 -0600 (CST) Subject: [Swift-commit] r3238 - branches/1.0/src/org/griphyn/vdl/karajan/lib Message-ID: <20100216044401.D59368D603@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-15 22:44:01 -0600 (Mon, 15 Feb 2010) New Revision: 3238 Modified: branches/1.0/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: throw a nicer exception when trying to invoke filename() and related functions on a primitive (or otherwise lacking a mapper) piece of data Modified: branches/1.0/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- branches/1.0/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-02-12 20:32:49 UTC (rev 3237) +++ branches/1.0/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-02-16 04:44:01 UTC (rev 3238) @@ -283,6 +283,9 @@ return relativize(String.valueOf(var.getValue())); } else { + if (var.getMapper() == null) { + throw new ExecutionException("Cannot invoke filename() on data without a mapper: " + var); + } PhysicalFormat f = var.getMapper().map(var.getPathFromRoot()); if (f instanceof GeneralizedFileFormat) { String filename = ((GeneralizedFileFormat) f).getURIAsString(); From noreply at svn.ci.uchicago.edu Wed Feb 17 14:30:17 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Feb 2010 14:30:17 -0600 (CST) Subject: [Swift-commit] r3239 - trunk/src/org/griphyn/vdl/engine Message-ID: <20100217203017.A10B49CC84@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-17 14:30:17 -0600 (Wed, 17 Feb 2010) New Revision: 3239 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: attempt to fix multiple writers error with iterate Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-02-16 04:44:01 UTC (rev 3238) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-02-17 20:30:17 UTC (rev 3239) @@ -711,7 +711,7 @@ Iterator scopeIterator = innerScope.getVariableIterator(); while(scopeIterator.hasNext()) { String v=(String) scopeIterator.next(); - scope.addWriter(v, statementID, false); + scope.addWriter(v, statementID, true); } scope.appendStatement(iterateST); } From noreply at svn.ci.uchicago.edu Wed Feb 17 15:56:59 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Feb 2010 15:56:59 -0600 (CST) Subject: [Swift-commit] r3240 - trunk/resources Message-ID: <20100217215659.E4D069CC84@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-17 15:56:59 -0600 (Wed, 17 Feb 2010) New Revision: 3240 Modified: trunk/resources/Karajan.stg Log: added selfclose to foreach Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2010-02-17 20:30:17 UTC (rev 3239) +++ trunk/resources/Karajan.stg 2010-02-17 21:56:59 UTC (rev 3240) @@ -202,8 +202,8 @@ >> -foreach(var,in,indexVar,declarations,statements,line) ::= << - +foreach(var,in,indexVar,declarations,statements,line,selfClose) ::= << + $in$ From noreply at svn.ci.uchicago.edu Wed Feb 17 15:58:11 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Feb 2010 15:58:11 -0600 (CST) Subject: [Swift-commit] r3241 - trunk/src/org/griphyn/vdl/engine Message-ID: <20100217215811.57B889CC84@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-17 15:58:11 -0600 (Wed, 17 Feb 2010) New Revision: 3241 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: set selfclose attribute if this foreach is iterating on a variable which is written to in the body of the foreach Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-02-17 21:56:59 UTC (rev 3240) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-02-17 21:58:11 UTC (rev 3241) @@ -743,11 +743,15 @@ statementsForSymbols(foreach.getBody(), innerScope); statements(foreach.getBody(), innerScope); + String inVar = (String) inST.getAttribute("var"); Object statementID = new Integer(callID++); Iterator scopeIterator = innerScope.getVariableIterator(); while(scopeIterator.hasNext()) { String v=(String) scopeIterator.next(); scope.addWriter(v, statementID, true); + if (v.equals(inVar) && !innerScope.isVariableLocallyDefined(v)) { + foreachST.setAttribute("selfClose", "true"); + } } scope.appendStatement(foreachST); } catch(CompilationException re) { From noreply at svn.ci.uchicago.edu Wed Feb 17 15:58:52 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Feb 2010 15:58:52 -0600 (CST) Subject: [Swift-commit] r3242 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20100217215852.353059CC84@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-17 15:58:52 -0600 (Wed, 17 Feb 2010) New Revision: 3242 Modified: trunk/src/org/griphyn/vdl/mapping/Path.java Log: added getLast method to path (getFirst was lonely) Modified: trunk/src/org/griphyn/vdl/mapping/Path.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/Path.java 2010-02-17 21:58:11 UTC (rev 3241) +++ trunk/src/org/griphyn/vdl/mapping/Path.java 2010-02-17 21:58:52 UTC (rev 3242) @@ -177,6 +177,10 @@ public String getFirst() { return ((Entry) elements.get(0)).name; } + + public String getLast() { + return ((Entry) elements.get(elements.size() - 1)).name; + } public boolean isEmpty() { return elements == null || elements.size() == 0; From noreply at svn.ci.uchicago.edu Wed Feb 17 15:59:57 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Feb 2010 15:59:57 -0600 (CST) Subject: [Swift-commit] r3243 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100217215957.BF5AD9CC84@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-17 15:59:57 -0600 (Wed, 17 Feb 2010) New Revision: 3243 Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java Log: terminate iterations if selfclose is set and if there are no current ongoing iterators and the iterator is empty Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2010-02-17 21:58:52 UTC (rev 3242) +++ trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2010-02-17 21:59:57 UTC (rev 3243) @@ -29,6 +29,7 @@ import org.globus.cog.karajan.workflow.events.EventTargetPair; import org.globus.cog.karajan.workflow.events.FutureNotificationEvent; import org.globus.cog.karajan.workflow.futures.FutureEvaluationException; +import org.globus.cog.karajan.workflow.futures.FutureFault; import org.globus.cog.karajan.workflow.futures.FutureIterator; import org.globus.cog.karajan.workflow.futures.FutureIteratorIncomplete; import org.globus.cog.karajan.workflow.nodes.AbstractParallelIterator; @@ -42,16 +43,23 @@ public static final Arg A_NAME = new Arg.Positional("name"); public static final Arg A_IN = new Arg.Positional("in"); + public static final Arg A_SELF_CLOSE = new Arg.Optional("selfclose", Boolean.FALSE); static { - setArguments(ThrottledParallelFor.class, new Arg[] { A_NAME, A_IN }); + setArguments(ThrottledParallelFor.class, new Arg[] { A_NAME, A_IN, A_SELF_CLOSE }); } public static final String THREAD_COUNT = "#threadcount"; private int maxThreadCount = -1; - public void iterate(VariableStack stack, Identifier var, KarajanIterator i) + protected void partialArgumentsEvaluated(VariableStack stack) + throws ExecutionException { + stack.setVar("selfclose", A_SELF_CLOSE.getValue(stack)); + super.partialArgumentsEvaluated(stack); + } + + public void iterate(VariableStack stack, Identifier var, KarajanIterator i) throws ExecutionException { if (elementCount() > 0) { if (logger.isDebugEnabled()) { @@ -61,7 +69,8 @@ setChildFailed(stack, false); stack.setCaller(this); initializeChannelBuffers(stack); - initThreadCount(stack); + initThreadCount(stack, TypeUtil.toBoolean(stack.currentFrame().getVar("selfclose")), i); + stack.currentFrame().deleteVar("selfclose"); citerate(stack, var, i); } else { @@ -74,7 +83,7 @@ ThreadCount tc = getThreadCount(stack); try { while (i.hasNext()) { - Object value = tc.tryIncrement(i); + Object value = tc.tryIncrement(); VariableStack copy = stack.copy(); copy.enter(); ThreadingContext.set(copy, ThreadingContext.get(copy).split( @@ -86,7 +95,12 @@ startElement(getArgCount(), copy); } - int left = tc.decrement(); + int left; + synchronized(tc) { + // can only have closed and running = 0 in one place + tc.close(); + left = tc.current(); + } if (left == 0) { complete(stack); } @@ -105,13 +119,22 @@ closeBuffers(stack); stack.leave(); ThreadCount tc = getThreadCount(stack); - int running = tc.decrement(); + int running; + boolean closed; + boolean iteratorHasValues; + synchronized(tc) { + closed = tc.isClosed(); + running = tc.decrement(); + iteratorHasValues = !tc.selfClose || tc.iteratorHasValues(); + } if (running == 0) { - complete(stack); + if (closed || !iteratorHasValues) { + complete(stack); + } } } - private void initThreadCount(VariableStack stack) { + private void initThreadCount(VariableStack stack, boolean selfClose, KarajanIterator i) { if (maxThreadCount < 0) { try { maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig() @@ -121,7 +144,7 @@ maxThreadCount = DEFAULT_MAX_THREADS; } } - stack.setVar(THREAD_COUNT, new ThreadCount(maxThreadCount)); + stack.setVar(THREAD_COUNT, new ThreadCount(maxThreadCount, selfClose, i)); } private ThreadCount getThreadCount(VariableStack stack) @@ -132,14 +155,32 @@ private static class ThreadCount implements FutureIterator { private int maxThreadCount; private int crt; + private boolean selfClose, closed; private List listeners; + private KarajanIterator i; - public ThreadCount(int maxThreadCount) { + public ThreadCount(int maxThreadCount, boolean selfClose, KarajanIterator i) { this.maxThreadCount = maxThreadCount; - crt = 1; + this.selfClose = selfClose; + this.i = i; + crt = 0; } + + public boolean iteratorHasValues() { + try { + return i.hasNext(); + } + catch (FutureFault e) { + return false; + } + } - public synchronized Object tryIncrement(KarajanIterator i) { + public boolean isSelfClose() { + return selfClose; + } + + public synchronized Object tryIncrement() { + // there is no way that both crt == 0 and i has no values outside this critical section if (crt < maxThreadCount) { Object o = i.next(); crt++; @@ -175,8 +216,8 @@ return 0; } - public int current() { - return 0; + public synchronized int current() { + return crt; } public Object peek() { @@ -205,7 +246,8 @@ } } - public void close() { + public synchronized void close() { + this.closed = true; } public void fail(FutureEvaluationException e) { @@ -215,8 +257,8 @@ return null; } - public boolean isClosed() { - return false; + public synchronized boolean isClosed() { + return closed; } } } \ No newline at end of file From noreply at svn.ci.uchicago.edu Wed Feb 17 16:00:19 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Feb 2010 16:00:19 -0600 (CST) Subject: [Swift-commit] r3244 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100217220019.ECDFD9CC84@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-17 16:00:19 -0600 (Wed, 17 Feb 2010) New Revision: 3244 Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: made method protected Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-02-17 21:59:57 UTC (rev 3243) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-02-17 22:00:19 UTC (rev 3244) @@ -487,7 +487,7 @@ return getFutureWrapperMap(stack).addFutureListListener(handle, value).futureIterator(stack); } - private void markAsAvailable(VariableStack stack, DSHandle handle, Object key) + protected void markAsAvailable(VariableStack stack, DSHandle handle, Object key) throws ExecutionException { getFutureWrapperMap(stack).markAsAvailable(handle, key); } From noreply at svn.ci.uchicago.edu Wed Feb 17 16:02:14 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Feb 2010 16:02:14 -0600 (CST) Subject: [Swift-commit] r3245 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100217220214.77AF69CC84@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-17 16:02:14 -0600 (Wed, 17 Feb 2010) New Revision: 3245 Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java Log: if the value of an array item is set then mark it such that iterations waiting on it can continue (it is a bit surprising that this wasn't properly working; or maybe I'm missing something) Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2010-02-17 22:00:19 UTC (rev 3244) +++ trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2010-02-17 22:02:14 UTC (rev 3245) @@ -29,7 +29,7 @@ public Object function(VariableStack stack) throws ExecutionException { DSHandle var = (DSHandle) PA_VAR.getValue(stack); try { - Path path = parsePath(OA_PATH.getValue(stack), stack); + Path path = parsePath(OA_PATH.getValue(stack), stack); DSHandle leaf = var.getField(path); DSHandle value = (DSHandle) PA_VALUE.getValue(stack); if (logger.isInfoEnabled()) { @@ -47,6 +47,9 @@ } deepCopy(leaf, value, stack); } + if (var.getParent().getType().isArray()) { + markAsAvailable(stack, leaf.getParent(), leaf.getPathFromRoot().getLast()); + } } return null; } From noreply at svn.ci.uchicago.edu Wed Feb 17 16:03:52 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Feb 2010 16:03:52 -0600 (CST) Subject: [Swift-commit] r3246 - trunk Message-ID: <20100217220352.400A49CC84@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-02-17 16:03:52 -0600 (Wed, 17 Feb 2010) New Revision: 3246 Modified: trunk/CHANGES.txt Log: changes Modified: trunk/CHANGES.txt =================================================================== --- trunk/CHANGES.txt 2010-02-17 22:02:14 UTC (rev 3245) +++ trunk/CHANGES.txt 2010-02-17 22:03:52 UTC (rev 3246) @@ -1,3 +1,8 @@ +(02/17/10) + +*** Fixed deadlock when iterating on an array that is being written to + in a foreach loop + (07/03/09) *** New import keyword which allows other SwiftScript source files to be imported. From noreply at svn.ci.uchicago.edu Mon Feb 22 13:37:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Feb 2010 13:37:24 -0600 (CST) Subject: [Swift-commit] r3247 - trunk/libexec Message-ID: <20100222193724.BBE329CCA9@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-02-22 13:37:24 -0600 (Mon, 22 Feb 2010) New Revision: 3247 Modified: trunk/libexec/_swiftwrap Log: Bug fix: support CDM jobs w/o CDM_GATHER. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-02-17 22:03:52 UTC (rev 3246) +++ trunk/libexec/_swiftwrap 2010-02-22 19:37:24 UTC (rev 3247) @@ -191,6 +191,9 @@ if [[ $CDM_FILE == "" ]]; then return fi + if [[ $GATHER_OUTPUT == "" ]]; then + return + fi cdm_gather_action $GATHER_MAX $GATHER_OUTPUT } From noreply at svn.ci.uchicago.edu Mon Feb 22 20:26:49 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Feb 2010 20:26:49 -0600 (CST) Subject: [Swift-commit] r3248 - SwiftApps Message-ID: <20100223022649.41A899CC96@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-02-22 20:26:49 -0600 (Mon, 22 Feb 2010) New Revision: 3248 Added: SwiftApps/RunR.sh SwiftApps/RunSwiftScript.sh SwiftApps/Swift.R SwiftApps/TestSwift.R SwiftApps/bootstrapdemo.R SwiftApps/pboot.R SwiftApps/swiftapply.swift Log: Initial revision of Swift R interface. Added: SwiftApps/RunR.sh =================================================================== --- SwiftApps/RunR.sh (rev 0) +++ SwiftApps/RunR.sh 2010-02-23 02:26:49 UTC (rev 3248) @@ -0,0 +1,13 @@ +#! /usr/bin/env Rscript + +argv = commandArgs(TRUE) + +load(argv[1]); + +result=list() +for(c in 1:length(rcall$arglistbatch)) { + # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) + result[[c]] = do.call( rcall$func, rcall$arglistbatch[[c]] ) +} + +save(result,file=argv[2]) Property changes on: SwiftApps/RunR.sh ___________________________________________________________________ Name: svn:executable + Added: SwiftApps/RunSwiftScript.sh =================================================================== --- SwiftApps/RunSwiftScript.sh (rev 0) +++ SwiftApps/RunSwiftScript.sh 2010-02-23 02:26:49 UTC (rev 3248) @@ -0,0 +1,32 @@ +rundir=$1 +site=$2 + +cd $rundir + +cat >tc <sites.xml < + + + 10000 + .03 + + $(pwd) + + + 00:00:10 + 1800 + + 8 + 10000 + .00 + + $(pwd) + + +EOF + +swift -tc.file tc -sites.file sites.xml ../swiftapply.swift Property changes on: SwiftApps/RunSwiftScript.sh ___________________________________________________________________ Name: svn:executable + Added: SwiftApps/Swift.R =================================================================== --- SwiftApps/Swift.R (rev 0) +++ SwiftApps/Swift.R 2010-02-23 02:26:49 UTC (rev 3248) @@ -0,0 +1,76 @@ +swiftapply <- function( func, arglists, site="local", callsperbatch=1 ) +{ + rundir = system("mktemp -d SwiftR.run.XXX",intern=TRUE) + cat("Running in ",rundir,"\n") + narglists = length(arglists) # number of arglists to process + batch=1 # Next arglist batch number to fill + arglist=1 # Next arglist number to insert + while(arglist <= narglists) { + arglistsleft = narglists - arglist + 1 + if(arglistsleft >= callsperbatch) { + batchsize = callsperbatch + } + else { + batchsize = arglistsleft + } + arglistbatch = list() + for(i in 1 : batchsize) { + arglistbatch[[i]] = arglists[[arglist]] + arglist = arglist +1 + } + rcall = list(func=func,arglistbatch=arglistbatch) + save(rcall,file=paste(rundir,"/cbatch.",as.character(batch),".Rdata",sep="")) + batch = batch + 1; + } + nbatches = batch - 1 + system(paste("./RunSwiftScript.sh",rundir,"local",sep=" ")) + + rno = 1 + rlist = list() + for(batch in 1:nbatches) { + load(paste(rundir,"/rbatch.",as.character(batch),".Rdata",sep="")) + nresults = length(result) + for(r in 1:nresults) { + rlist[[rno]] = result[[r]] + rno = rno + 1 + } + } + return(rlist) +} + +TODO=" + +x n args +x batch + into svn + unique dirs + select sites and swift args (throttles etc) + R docs + R package (SwiftR) + Swift docs + async exec + clean up boot: fix all calls to statistics + error handling and null and missing values: ensure res#s correspond to arg#s + status + specify swift scripts + run async and grab status (track 'runs' in R) + increm result collect + pass the func as val + pass extra funcs and packages required + pass extra vals + pass extra files + specifiy unique swift scritps ala Dirk's tools + setup the R envs + coasters for persistent R Servers + test suites + use littleR + args as alists vs args as list + runids, output logging + select exec sites and swift params etc + make polymorphic to *apply and snow + stream results back to R (so use can inspect as they arrive) + (pull them in with a Swift.poll() func) + handle discontiguous results + return good error messages including messages from R eval and from Swift + +END" Added: SwiftApps/TestSwift.R =================================================================== --- SwiftApps/TestSwift.R (rev 0) +++ SwiftApps/TestSwift.R 2010-02-23 02:26:49 UTC (rev 3248) @@ -0,0 +1,31 @@ +require(boot) +source("Swift.R") + +args=list(ducks,dogs) +sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } +res = do.call(sumcrits,args) +cat("Test of do.call(sumcrits)\n") +print(res) + +arglist = rep(list(args),9) + +cat("\nTest of swiftapply(sumcrits,arglist)\n") +res = swiftapply(sumcrits,arglist) +print(res) + +cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") +res = swiftapply(sumcrits,arglist,callsperbatch=10) +print(res) + +cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=2)\n") +res = swiftapply(sumcrits,arglist,callsperbatch=2) +print(res) + +cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=3)\n") +res = swiftapply(sumcrits,arglist,callsperbatch=3) +print(res) + +cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=20)\n") +res = swiftapply(sumcrits,arglist,callsperbatch=20) +print(res) + Added: SwiftApps/bootstrapdemo.R =================================================================== --- SwiftApps/bootstrapdemo.R (rev 0) +++ SwiftApps/bootstrapdemo.R 2010-02-23 02:26:49 UTC (rev 3248) @@ -0,0 +1,71 @@ +# +# OpenMx Script to demonstrate use of R's boot package for bootstrapping +# +# Author: M.C. Neale 1 September 2009 +# + +# Load required libraries +require(OpenMx) +require(boot) + +# Define a function called mles which will return maximum likelihood estimates +# It uses the demoOneFactor dataset and one factor model on the OpenMx homepage +# http://openmx.psyc.virginia.edu + + +mles<-function(dataset,wt){ +cat("in mles="); +require(OpenMx) + manifests <- names(dataset) + latents <- c("G") + covwt <- cov.wt(dataset,wt) + mlevals <- mxRun(mxModel("One Factor", type="RAM", + manifestVars = manifests, + latentVars = latents, + mxPath(from=latents, to=manifests), + mxPath(from=manifests, arrows=2), + mxPath(from=latents, arrows=2, + free=F, values=1.0), + mxData(covwt$cov, type="cov", + numObs=500))) + return(as.vector(mlevals at output$estimate))} + +# Run 100 bootstraps (a smallish number) + +boot.out=list() + +boot.out[[1]] = pboot(demoOneFactor,mles,R=100) +#boot.out[[2]] = boot(demoOneFactor,mles,R=8) +#boot.out[[3]] = boot(demoOneFactor,mles,R=9) + +print("done booting - boot.out is:") +print(boot.out) +print("end of boot.out") + +# For comparison, take a look at the SE output from running the homepage job once +data(demoOneFactor) +manifests <- names(demoOneFactor) +latents <- c("G") +factorModel <- mxModel("One Factor", type="RAM", + manifestVars = manifests, + latentVars = latents, + mxPath(from=latents, to=manifests), + mxPath(from=manifests, arrows=2), + mxPath(from=latents, arrows=2, + free=F, values=1.0), + mxData(cov(demoOneFactor), type="cov", + numObs=500)) +facrun<-mxRun(factorModel) +summary(facrun) + +# the estimates and standard errors should match up pretty well, though the number of replicates R above might be increased +# therefore, only the factorModel estimates are compared: + +loadings<-facrun at matrices$A at values[1:5,6] +errors<-diag(facrun at matrices$S at values[1:5,1:5]) +estimates<-as.vector(c(loadings,errors)) +omxCheckCloseEnough(as.vector(c(0.3971525,0.5036615,0.5772418,0.7027743,0.7962506,0.04081422,0.03802001,0.04082720,0.03938708,0.03628711)),estimates,.001) + +# The above should indicate that the results are close enough. + + Added: SwiftApps/pboot.R =================================================================== --- SwiftApps/pboot.R (rev 0) +++ SwiftApps/pboot.R 2010-02-23 02:26:49 UTC (rev 3248) @@ -0,0 +1,118 @@ +pboot = +function (data, statistic, R, sim = "ordinary", stype = "i", + strata = rep(1, n), L = NULL, m = 0, weights = NULL, + ran.gen = function(d, p) d, mle = NULL, simple = FALSE, ...) +{ + call <- match.call() + if (simple && (sim != "ordinary" || stype != "i" || sum(m))) { + warning("'simple=TRUE' is only valid for 'sim=\"ordinary\", stype=\"i\", n=0, so ignored") + simple <- FALSE + } + if (!exists(".Random.seed", envir = .GlobalEnv, inherits = FALSE)) + runif(1) + seed <- get(".Random.seed", envir = .GlobalEnv, inherits = FALSE) + if (isMatrix(data)) + n <- nrow(data) + else n <- length(data) + temp.str <- strata + strata <- tapply(1L:n, as.numeric(strata)) + if ((n == 0) || is.null(n)) + stop("no data in call to boot") + if (sim != "parametric") { + if ((sim == "antithetic") && is.null(L)) + L <- empinf(data = data, statistic = statistic, stype = stype, + strata = strata, ...) + if (sim != "ordinary") + m <- 0 + else if (any(m < 0)) + stop("negative value of m supplied") + if ((length(m) != 1L) && (length(m) != length(table(strata)))) + stop("length of m incompatible with strata") + if ((sim == "ordinary") || (sim == "balanced")) { + if (isMatrix(weights) && (nrow(weights) != length(R))) + stop("dimensions of R and weights do not match") + } + else weights <- NULL + if (!is.null(weights)) + weights <- t(apply(matrix(weights, n, length(R), + byrow = TRUE), 2, normalize, strata)) + if (!simple) + i <- index.array(n, R, sim, strata, m, L, weights) + if (stype == "f") + original <- rep(1, n) + else if (stype == "w") { + ns <- tabulate(strata)[strata] + original <- 1/ns + } + else original <- 1L:n + if (sum(m) > 0) { + t0 <- statistic(data, original, rep(1, sum(m)), ...) + lt0 <- length(t0) + } + else { + t0 <- statistic(data, original, ...) + lt0 <- length(t0) + } + } + else { + t0 <- statistic(data, ...) + lt0 <- length(t0) + } + t.star <- matrix(NA, sum(R), lt0) + pred.i <- NULL + if (sim == "parametric") { + for (r in 1L:R) { + t.star[r, ] <- statistic(ran.gen(data, mle), ...) + } + } + else { + if (!simple && ncol(i) > n) { + pred.i <- as.matrix(i[, (n + 1L):ncol(i)]) + i <- i[, 1L:n] + } + if (stype == "f") { +print("CASE 1") + f <- freq.array(i) + if (sum(m) == 0) + for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + f[r, ], ...) + else for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + f[r, ], pred.i[r, ], ...) + } + else if (stype == "w") { +print("CASE 2") + f <- freq.array(i) + if (sum(m) == 0) + for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + f[r, ]/ns, ...) + else for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + f[r, ]/ns, pred.i[r, ], ...) + } + else if (sum(m) > 0) { +print("CASE 3") + for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + i[r, ], pred.i[r, ], ...) + } + else if (simple) { +print("CASE 4") + for (r in 1L:sum(R)) { + inds <- index.array(n, 1, sim, strata, m, L, + weights) + t.star[r, ] <- statistic(data, inds, ...) + } + } + else { +cat("CASE 5 - sum(R)=",sum(R)) + # for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, i[r, ], ...) + alists = list() + for (r in 1L:sum(R)) alists[[r]] <- list(data,i[r,],...) + reslist = swiftapplyb(statistic,alists,callsperbatch=25) + for (r in 1L:sum(R)) t.star[r, ] <- reslist[[r]] + } + } + dimnames(t.star) <- NULL + if (is.null(weights)) + weights <- 1/tabulate(strata)[strata] + boot.return(sim, t0, t.star, temp.str, R, data, statistic, + stype, call, seed, L, m, pred.i, weights, ran.gen, mle) +} Added: SwiftApps/swiftapply.swift =================================================================== --- SwiftApps/swiftapply.swift (rev 0) +++ SwiftApps/swiftapply.swift 2010-02-23 02:26:49 UTC (rev 3248) @@ -0,0 +1,13 @@ +type RFile; + +app (RFile result) RunR (RFile rcall) +{ + RunR @rcall @result; +} + +RFile rcalls[] ; +RFile results[] ; + +foreach c, i in rcalls { + results[i] = RunR(c); +} From noreply at svn.ci.uchicago.edu Mon Feb 22 20:49:50 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Feb 2010 20:49:50 -0600 (CST) Subject: [Swift-commit] r3249 - SwiftApps Message-ID: <20100223024950.30FB19CC96@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-02-22 20:49:50 -0600 (Mon, 22 Feb 2010) New Revision: 3249 Removed: SwiftApps/Swift.R Log: Comitted to wrong dir. Deleted: SwiftApps/Swift.R =================================================================== --- SwiftApps/Swift.R 2010-02-23 02:26:49 UTC (rev 3248) +++ SwiftApps/Swift.R 2010-02-23 02:49:50 UTC (rev 3249) @@ -1,76 +0,0 @@ -swiftapply <- function( func, arglists, site="local", callsperbatch=1 ) -{ - rundir = system("mktemp -d SwiftR.run.XXX",intern=TRUE) - cat("Running in ",rundir,"\n") - narglists = length(arglists) # number of arglists to process - batch=1 # Next arglist batch number to fill - arglist=1 # Next arglist number to insert - while(arglist <= narglists) { - arglistsleft = narglists - arglist + 1 - if(arglistsleft >= callsperbatch) { - batchsize = callsperbatch - } - else { - batchsize = arglistsleft - } - arglistbatch = list() - for(i in 1 : batchsize) { - arglistbatch[[i]] = arglists[[arglist]] - arglist = arglist +1 - } - rcall = list(func=func,arglistbatch=arglistbatch) - save(rcall,file=paste(rundir,"/cbatch.",as.character(batch),".Rdata",sep="")) - batch = batch + 1; - } - nbatches = batch - 1 - system(paste("./RunSwiftScript.sh",rundir,"local",sep=" ")) - - rno = 1 - rlist = list() - for(batch in 1:nbatches) { - load(paste(rundir,"/rbatch.",as.character(batch),".Rdata",sep="")) - nresults = length(result) - for(r in 1:nresults) { - rlist[[rno]] = result[[r]] - rno = rno + 1 - } - } - return(rlist) -} - -TODO=" - -x n args -x batch - into svn - unique dirs - select sites and swift args (throttles etc) - R docs - R package (SwiftR) - Swift docs - async exec - clean up boot: fix all calls to statistics - error handling and null and missing values: ensure res#s correspond to arg#s - status - specify swift scripts - run async and grab status (track 'runs' in R) - increm result collect - pass the func as val - pass extra funcs and packages required - pass extra vals - pass extra files - specifiy unique swift scritps ala Dirk's tools - setup the R envs - coasters for persistent R Servers - test suites - use littleR - args as alists vs args as list - runids, output logging - select exec sites and swift params etc - make polymorphic to *apply and snow - stream results back to R (so use can inspect as they arrive) - (pull them in with a Swift.poll() func) - handle discontiguous results - return good error messages including messages from R eval and from Swift - -END" From noreply at svn.ci.uchicago.edu Mon Feb 22 20:55:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Feb 2010 20:55:25 -0600 (CST) Subject: [Swift-commit] r3250 - SwiftApps Message-ID: <20100223025525.D37B99CC9A@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-02-22 20:55:25 -0600 (Mon, 22 Feb 2010) New Revision: 3250 Removed: SwiftApps/RunR.sh SwiftApps/RunSwiftScript.sh SwiftApps/TestSwift.R SwiftApps/bootstrapdemo.R SwiftApps/pboot.R SwiftApps/swiftapply.swift Log: Added to wrong dir. Deleted: SwiftApps/RunR.sh =================================================================== --- SwiftApps/RunR.sh 2010-02-23 02:49:50 UTC (rev 3249) +++ SwiftApps/RunR.sh 2010-02-23 02:55:25 UTC (rev 3250) @@ -1,13 +0,0 @@ -#! /usr/bin/env Rscript - -argv = commandArgs(TRUE) - -load(argv[1]); - -result=list() -for(c in 1:length(rcall$arglistbatch)) { - # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) - result[[c]] = do.call( rcall$func, rcall$arglistbatch[[c]] ) -} - -save(result,file=argv[2]) Deleted: SwiftApps/RunSwiftScript.sh =================================================================== --- SwiftApps/RunSwiftScript.sh 2010-02-23 02:49:50 UTC (rev 3249) +++ SwiftApps/RunSwiftScript.sh 2010-02-23 02:55:25 UTC (rev 3250) @@ -1,32 +0,0 @@ -rundir=$1 -site=$2 - -cd $rundir - -cat >tc <sites.xml < - - - 10000 - .03 - - $(pwd) - - - 00:00:10 - 1800 - - 8 - 10000 - .00 - - $(pwd) - - -EOF - -swift -tc.file tc -sites.file sites.xml ../swiftapply.swift Deleted: SwiftApps/TestSwift.R =================================================================== --- SwiftApps/TestSwift.R 2010-02-23 02:49:50 UTC (rev 3249) +++ SwiftApps/TestSwift.R 2010-02-23 02:55:25 UTC (rev 3250) @@ -1,31 +0,0 @@ -require(boot) -source("Swift.R") - -args=list(ducks,dogs) -sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } -res = do.call(sumcrits,args) -cat("Test of do.call(sumcrits)\n") -print(res) - -arglist = rep(list(args),9) - -cat("\nTest of swiftapply(sumcrits,arglist)\n") -res = swiftapply(sumcrits,arglist) -print(res) - -cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") -res = swiftapply(sumcrits,arglist,callsperbatch=10) -print(res) - -cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=2)\n") -res = swiftapply(sumcrits,arglist,callsperbatch=2) -print(res) - -cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=3)\n") -res = swiftapply(sumcrits,arglist,callsperbatch=3) -print(res) - -cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=20)\n") -res = swiftapply(sumcrits,arglist,callsperbatch=20) -print(res) - Deleted: SwiftApps/bootstrapdemo.R =================================================================== --- SwiftApps/bootstrapdemo.R 2010-02-23 02:49:50 UTC (rev 3249) +++ SwiftApps/bootstrapdemo.R 2010-02-23 02:55:25 UTC (rev 3250) @@ -1,71 +0,0 @@ -# -# OpenMx Script to demonstrate use of R's boot package for bootstrapping -# -# Author: M.C. Neale 1 September 2009 -# - -# Load required libraries -require(OpenMx) -require(boot) - -# Define a function called mles which will return maximum likelihood estimates -# It uses the demoOneFactor dataset and one factor model on the OpenMx homepage -# http://openmx.psyc.virginia.edu - - -mles<-function(dataset,wt){ -cat("in mles="); -require(OpenMx) - manifests <- names(dataset) - latents <- c("G") - covwt <- cov.wt(dataset,wt) - mlevals <- mxRun(mxModel("One Factor", type="RAM", - manifestVars = manifests, - latentVars = latents, - mxPath(from=latents, to=manifests), - mxPath(from=manifests, arrows=2), - mxPath(from=latents, arrows=2, - free=F, values=1.0), - mxData(covwt$cov, type="cov", - numObs=500))) - return(as.vector(mlevals at output$estimate))} - -# Run 100 bootstraps (a smallish number) - -boot.out=list() - -boot.out[[1]] = pboot(demoOneFactor,mles,R=100) -#boot.out[[2]] = boot(demoOneFactor,mles,R=8) -#boot.out[[3]] = boot(demoOneFactor,mles,R=9) - -print("done booting - boot.out is:") -print(boot.out) -print("end of boot.out") - -# For comparison, take a look at the SE output from running the homepage job once -data(demoOneFactor) -manifests <- names(demoOneFactor) -latents <- c("G") -factorModel <- mxModel("One Factor", type="RAM", - manifestVars = manifests, - latentVars = latents, - mxPath(from=latents, to=manifests), - mxPath(from=manifests, arrows=2), - mxPath(from=latents, arrows=2, - free=F, values=1.0), - mxData(cov(demoOneFactor), type="cov", - numObs=500)) -facrun<-mxRun(factorModel) -summary(facrun) - -# the estimates and standard errors should match up pretty well, though the number of replicates R above might be increased -# therefore, only the factorModel estimates are compared: - -loadings<-facrun at matrices$A at values[1:5,6] -errors<-diag(facrun at matrices$S at values[1:5,1:5]) -estimates<-as.vector(c(loadings,errors)) -omxCheckCloseEnough(as.vector(c(0.3971525,0.5036615,0.5772418,0.7027743,0.7962506,0.04081422,0.03802001,0.04082720,0.03938708,0.03628711)),estimates,.001) - -# The above should indicate that the results are close enough. - - Deleted: SwiftApps/pboot.R =================================================================== --- SwiftApps/pboot.R 2010-02-23 02:49:50 UTC (rev 3249) +++ SwiftApps/pboot.R 2010-02-23 02:55:25 UTC (rev 3250) @@ -1,118 +0,0 @@ -pboot = -function (data, statistic, R, sim = "ordinary", stype = "i", - strata = rep(1, n), L = NULL, m = 0, weights = NULL, - ran.gen = function(d, p) d, mle = NULL, simple = FALSE, ...) -{ - call <- match.call() - if (simple && (sim != "ordinary" || stype != "i" || sum(m))) { - warning("'simple=TRUE' is only valid for 'sim=\"ordinary\", stype=\"i\", n=0, so ignored") - simple <- FALSE - } - if (!exists(".Random.seed", envir = .GlobalEnv, inherits = FALSE)) - runif(1) - seed <- get(".Random.seed", envir = .GlobalEnv, inherits = FALSE) - if (isMatrix(data)) - n <- nrow(data) - else n <- length(data) - temp.str <- strata - strata <- tapply(1L:n, as.numeric(strata)) - if ((n == 0) || is.null(n)) - stop("no data in call to boot") - if (sim != "parametric") { - if ((sim == "antithetic") && is.null(L)) - L <- empinf(data = data, statistic = statistic, stype = stype, - strata = strata, ...) - if (sim != "ordinary") - m <- 0 - else if (any(m < 0)) - stop("negative value of m supplied") - if ((length(m) != 1L) && (length(m) != length(table(strata)))) - stop("length of m incompatible with strata") - if ((sim == "ordinary") || (sim == "balanced")) { - if (isMatrix(weights) && (nrow(weights) != length(R))) - stop("dimensions of R and weights do not match") - } - else weights <- NULL - if (!is.null(weights)) - weights <- t(apply(matrix(weights, n, length(R), - byrow = TRUE), 2, normalize, strata)) - if (!simple) - i <- index.array(n, R, sim, strata, m, L, weights) - if (stype == "f") - original <- rep(1, n) - else if (stype == "w") { - ns <- tabulate(strata)[strata] - original <- 1/ns - } - else original <- 1L:n - if (sum(m) > 0) { - t0 <- statistic(data, original, rep(1, sum(m)), ...) - lt0 <- length(t0) - } - else { - t0 <- statistic(data, original, ...) - lt0 <- length(t0) - } - } - else { - t0 <- statistic(data, ...) - lt0 <- length(t0) - } - t.star <- matrix(NA, sum(R), lt0) - pred.i <- NULL - if (sim == "parametric") { - for (r in 1L:R) { - t.star[r, ] <- statistic(ran.gen(data, mle), ...) - } - } - else { - if (!simple && ncol(i) > n) { - pred.i <- as.matrix(i[, (n + 1L):ncol(i)]) - i <- i[, 1L:n] - } - if (stype == "f") { -print("CASE 1") - f <- freq.array(i) - if (sum(m) == 0) - for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, - f[r, ], ...) - else for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, - f[r, ], pred.i[r, ], ...) - } - else if (stype == "w") { -print("CASE 2") - f <- freq.array(i) - if (sum(m) == 0) - for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, - f[r, ]/ns, ...) - else for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, - f[r, ]/ns, pred.i[r, ], ...) - } - else if (sum(m) > 0) { -print("CASE 3") - for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, - i[r, ], pred.i[r, ], ...) - } - else if (simple) { -print("CASE 4") - for (r in 1L:sum(R)) { - inds <- index.array(n, 1, sim, strata, m, L, - weights) - t.star[r, ] <- statistic(data, inds, ...) - } - } - else { -cat("CASE 5 - sum(R)=",sum(R)) - # for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, i[r, ], ...) - alists = list() - for (r in 1L:sum(R)) alists[[r]] <- list(data,i[r,],...) - reslist = swiftapplyb(statistic,alists,callsperbatch=25) - for (r in 1L:sum(R)) t.star[r, ] <- reslist[[r]] - } - } - dimnames(t.star) <- NULL - if (is.null(weights)) - weights <- 1/tabulate(strata)[strata] - boot.return(sim, t0, t.star, temp.str, R, data, statistic, - stype, call, seed, L, m, pred.i, weights, ran.gen, mle) -} Deleted: SwiftApps/swiftapply.swift =================================================================== --- SwiftApps/swiftapply.swift 2010-02-23 02:49:50 UTC (rev 3249) +++ SwiftApps/swiftapply.swift 2010-02-23 02:55:25 UTC (rev 3250) @@ -1,13 +0,0 @@ -type RFile; - -app (RFile result) RunR (RFile rcall) -{ - RunR @rcall @result; -} - -RFile rcalls[] ; -RFile results[] ; - -foreach c, i in rcalls { - results[i] = RunR(c); -} From noreply at svn.ci.uchicago.edu Mon Feb 22 21:06:07 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Feb 2010 21:06:07 -0600 (CST) Subject: [Swift-commit] r3251 - in SwiftApps: . SwiftR Message-ID: <20100223030607.CB7499CC9A@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-02-22 21:06:07 -0600 (Mon, 22 Feb 2010) New Revision: 3251 Added: SwiftApps/SwiftR/ SwiftApps/SwiftR/RunR.sh SwiftApps/SwiftR/RunSwiftScript.sh SwiftApps/SwiftR/Swift.R SwiftApps/SwiftR/TestSwift.R SwiftApps/SwiftR/bootstrapdemo.R SwiftApps/SwiftR/pboot.R SwiftApps/SwiftR/swiftapply.swift Log: Initial version of Swift R interface. Added: SwiftApps/SwiftR/RunR.sh =================================================================== --- SwiftApps/SwiftR/RunR.sh (rev 0) +++ SwiftApps/SwiftR/RunR.sh 2010-02-23 03:06:07 UTC (rev 3251) @@ -0,0 +1,13 @@ +#! /usr/bin/env Rscript + +argv = commandArgs(TRUE) + +load(argv[1]); + +result=list() +for(c in 1:length(rcall$arglistbatch)) { + # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) + result[[c]] = do.call( rcall$func, rcall$arglistbatch[[c]] ) +} + +save(result,file=argv[2]) Property changes on: SwiftApps/SwiftR/RunR.sh ___________________________________________________________________ Name: svn:executable + Added: SwiftApps/SwiftR/RunSwiftScript.sh =================================================================== --- SwiftApps/SwiftR/RunSwiftScript.sh (rev 0) +++ SwiftApps/SwiftR/RunSwiftScript.sh 2010-02-23 03:06:07 UTC (rev 3251) @@ -0,0 +1,32 @@ +rundir=$1 +site=$2 + +cd $rundir + +cat >tc <sites.xml < + + + 10000 + .03 + + $(pwd) + + + 00:00:10 + 1800 + + 8 + 10000 + .00 + + $(pwd) + + +EOF + +swift -tc.file tc -sites.file sites.xml ../swiftapply.swift Property changes on: SwiftApps/SwiftR/RunSwiftScript.sh ___________________________________________________________________ Name: svn:executable + Added: SwiftApps/SwiftR/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift.R (rev 0) +++ SwiftApps/SwiftR/Swift.R 2010-02-23 03:06:07 UTC (rev 3251) @@ -0,0 +1,76 @@ +swiftapply <- function( func, arglists, site="local", callsperbatch=1 ) +{ + rundir = system("mktemp -d SwiftR.run.XXX",intern=TRUE) + cat("Running in ",rundir,"\n") + narglists = length(arglists) # number of arglists to process + batch=1 # Next arglist batch number to fill + arglist=1 # Next arglist number to insert + while(arglist <= narglists) { + arglistsleft = narglists - arglist + 1 + if(arglistsleft >= callsperbatch) { + batchsize = callsperbatch + } + else { + batchsize = arglistsleft + } + arglistbatch = list() + for(i in 1 : batchsize) { + arglistbatch[[i]] = arglists[[arglist]] + arglist = arglist +1 + } + rcall = list(func=func,arglistbatch=arglistbatch) + save(rcall,file=paste(rundir,"/cbatch.",as.character(batch),".Rdata",sep="")) + batch = batch + 1; + } + nbatches = batch - 1 + system(paste("./RunSwiftScript.sh",rundir,"local",sep=" ")) + + rno = 1 + rlist = list() + for(batch in 1:nbatches) { + load(paste(rundir,"/rbatch.",as.character(batch),".Rdata",sep="")) + nresults = length(result) + for(r in 1:nresults) { + rlist[[rno]] = result[[r]] + rno = rno + 1 + } + } + return(rlist) +} + +TODO=" + +x n args +x batch + into svn + unique dirs + select sites and swift args (throttles etc) + R docs + R package (SwiftR) + Swift docs + async exec + clean up boot: fix all calls to statistics + error handling and null and missing values: ensure res#s correspond to arg#s + status + specify swift scripts + run async and grab status (track 'runs' in R) + increm result collect + pass the func as val + pass extra funcs and packages required + pass extra vals + pass extra files + specifiy unique swift scritps ala Dirk's tools + setup the R envs + coasters for persistent R Servers + test suites + use littleR + args as alists vs args as list + runids, output logging + select exec sites and swift params etc + make polymorphic to *apply and snow + stream results back to R (so use can inspect as they arrive) + (pull them in with a Swift.poll() func) + handle discontiguous results + return good error messages including messages from R eval and from Swift + +END" Added: SwiftApps/SwiftR/TestSwift.R =================================================================== --- SwiftApps/SwiftR/TestSwift.R (rev 0) +++ SwiftApps/SwiftR/TestSwift.R 2010-02-23 03:06:07 UTC (rev 3251) @@ -0,0 +1,31 @@ +require(boot) +source("Swift.R") + +args=list(ducks,dogs) +sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } +res = do.call(sumcrits,args) +cat("Test of do.call(sumcrits)\n") +print(res) + +arglist = rep(list(args),9) + +cat("\nTest of swiftapply(sumcrits,arglist)\n") +res = swiftapply(sumcrits,arglist) +print(res) + +cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") +res = swiftapply(sumcrits,arglist,callsperbatch=10) +print(res) + +cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=2)\n") +res = swiftapply(sumcrits,arglist,callsperbatch=2) +print(res) + +cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=3)\n") +res = swiftapply(sumcrits,arglist,callsperbatch=3) +print(res) + +cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=20)\n") +res = swiftapply(sumcrits,arglist,callsperbatch=20) +print(res) + Added: SwiftApps/SwiftR/bootstrapdemo.R =================================================================== --- SwiftApps/SwiftR/bootstrapdemo.R (rev 0) +++ SwiftApps/SwiftR/bootstrapdemo.R 2010-02-23 03:06:07 UTC (rev 3251) @@ -0,0 +1,71 @@ +# +# OpenMx Script to demonstrate use of R's boot package for bootstrapping +# +# Author: M.C. Neale 1 September 2009 +# + +# Load required libraries +require(OpenMx) +require(boot) + +# Define a function called mles which will return maximum likelihood estimates +# It uses the demoOneFactor dataset and one factor model on the OpenMx homepage +# http://openmx.psyc.virginia.edu + + +mles<-function(dataset,wt){ +cat("in mles="); +require(OpenMx) + manifests <- names(dataset) + latents <- c("G") + covwt <- cov.wt(dataset,wt) + mlevals <- mxRun(mxModel("One Factor", type="RAM", + manifestVars = manifests, + latentVars = latents, + mxPath(from=latents, to=manifests), + mxPath(from=manifests, arrows=2), + mxPath(from=latents, arrows=2, + free=F, values=1.0), + mxData(covwt$cov, type="cov", + numObs=500))) + return(as.vector(mlevals at output$estimate))} + +# Run 100 bootstraps (a smallish number) + +boot.out=list() + +boot.out[[1]] = pboot(demoOneFactor,mles,R=100) +#boot.out[[2]] = boot(demoOneFactor,mles,R=8) +#boot.out[[3]] = boot(demoOneFactor,mles,R=9) + +print("done booting - boot.out is:") +print(boot.out) +print("end of boot.out") + +# For comparison, take a look at the SE output from running the homepage job once +data(demoOneFactor) +manifests <- names(demoOneFactor) +latents <- c("G") +factorModel <- mxModel("One Factor", type="RAM", + manifestVars = manifests, + latentVars = latents, + mxPath(from=latents, to=manifests), + mxPath(from=manifests, arrows=2), + mxPath(from=latents, arrows=2, + free=F, values=1.0), + mxData(cov(demoOneFactor), type="cov", + numObs=500)) +facrun<-mxRun(factorModel) +summary(facrun) + +# the estimates and standard errors should match up pretty well, though the number of replicates R above might be increased +# therefore, only the factorModel estimates are compared: + +loadings<-facrun at matrices$A at values[1:5,6] +errors<-diag(facrun at matrices$S at values[1:5,1:5]) +estimates<-as.vector(c(loadings,errors)) +omxCheckCloseEnough(as.vector(c(0.3971525,0.5036615,0.5772418,0.7027743,0.7962506,0.04081422,0.03802001,0.04082720,0.03938708,0.03628711)),estimates,.001) + +# The above should indicate that the results are close enough. + + Added: SwiftApps/SwiftR/pboot.R =================================================================== --- SwiftApps/SwiftR/pboot.R (rev 0) +++ SwiftApps/SwiftR/pboot.R 2010-02-23 03:06:07 UTC (rev 3251) @@ -0,0 +1,118 @@ +pboot = +function (data, statistic, R, sim = "ordinary", stype = "i", + strata = rep(1, n), L = NULL, m = 0, weights = NULL, + ran.gen = function(d, p) d, mle = NULL, simple = FALSE, ...) +{ + call <- match.call() + if (simple && (sim != "ordinary" || stype != "i" || sum(m))) { + warning("'simple=TRUE' is only valid for 'sim=\"ordinary\", stype=\"i\", n=0, so ignored") + simple <- FALSE + } + if (!exists(".Random.seed", envir = .GlobalEnv, inherits = FALSE)) + runif(1) + seed <- get(".Random.seed", envir = .GlobalEnv, inherits = FALSE) + if (isMatrix(data)) + n <- nrow(data) + else n <- length(data) + temp.str <- strata + strata <- tapply(1L:n, as.numeric(strata)) + if ((n == 0) || is.null(n)) + stop("no data in call to boot") + if (sim != "parametric") { + if ((sim == "antithetic") && is.null(L)) + L <- empinf(data = data, statistic = statistic, stype = stype, + strata = strata, ...) + if (sim != "ordinary") + m <- 0 + else if (any(m < 0)) + stop("negative value of m supplied") + if ((length(m) != 1L) && (length(m) != length(table(strata)))) + stop("length of m incompatible with strata") + if ((sim == "ordinary") || (sim == "balanced")) { + if (isMatrix(weights) && (nrow(weights) != length(R))) + stop("dimensions of R and weights do not match") + } + else weights <- NULL + if (!is.null(weights)) + weights <- t(apply(matrix(weights, n, length(R), + byrow = TRUE), 2, normalize, strata)) + if (!simple) + i <- index.array(n, R, sim, strata, m, L, weights) + if (stype == "f") + original <- rep(1, n) + else if (stype == "w") { + ns <- tabulate(strata)[strata] + original <- 1/ns + } + else original <- 1L:n + if (sum(m) > 0) { + t0 <- statistic(data, original, rep(1, sum(m)), ...) + lt0 <- length(t0) + } + else { + t0 <- statistic(data, original, ...) + lt0 <- length(t0) + } + } + else { + t0 <- statistic(data, ...) + lt0 <- length(t0) + } + t.star <- matrix(NA, sum(R), lt0) + pred.i <- NULL + if (sim == "parametric") { + for (r in 1L:R) { + t.star[r, ] <- statistic(ran.gen(data, mle), ...) + } + } + else { + if (!simple && ncol(i) > n) { + pred.i <- as.matrix(i[, (n + 1L):ncol(i)]) + i <- i[, 1L:n] + } + if (stype == "f") { +print("CASE 1") + f <- freq.array(i) + if (sum(m) == 0) + for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + f[r, ], ...) + else for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + f[r, ], pred.i[r, ], ...) + } + else if (stype == "w") { +print("CASE 2") + f <- freq.array(i) + if (sum(m) == 0) + for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + f[r, ]/ns, ...) + else for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + f[r, ]/ns, pred.i[r, ], ...) + } + else if (sum(m) > 0) { +print("CASE 3") + for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, + i[r, ], pred.i[r, ], ...) + } + else if (simple) { +print("CASE 4") + for (r in 1L:sum(R)) { + inds <- index.array(n, 1, sim, strata, m, L, + weights) + t.star[r, ] <- statistic(data, inds, ...) + } + } + else { +cat("CASE 5 - sum(R)=",sum(R)) + # for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, i[r, ], ...) + alists = list() + for (r in 1L:sum(R)) alists[[r]] <- list(data,i[r,],...) + reslist = swiftapplyb(statistic,alists,callsperbatch=25) + for (r in 1L:sum(R)) t.star[r, ] <- reslist[[r]] + } + } + dimnames(t.star) <- NULL + if (is.null(weights)) + weights <- 1/tabulate(strata)[strata] + boot.return(sim, t0, t.star, temp.str, R, data, statistic, + stype, call, seed, L, m, pred.i, weights, ran.gen, mle) +} Added: SwiftApps/SwiftR/swiftapply.swift =================================================================== --- SwiftApps/SwiftR/swiftapply.swift (rev 0) +++ SwiftApps/SwiftR/swiftapply.swift 2010-02-23 03:06:07 UTC (rev 3251) @@ -0,0 +1,13 @@ +type RFile; + +app (RFile result) RunR (RFile rcall) +{ + RunR @rcall @result; +} + +RFile rcalls[] ; +RFile results[] ; + +foreach c, i in rcalls { + results[i] = RunR(c); +} From noreply at svn.ci.uchicago.edu Mon Feb 22 22:36:05 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Feb 2010 22:36:05 -0600 (CST) Subject: [Swift-commit] r3252 - SwiftApps/SwiftR Message-ID: <20100223043605.A6E799CC96@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-02-22 22:36:05 -0600 (Mon, 22 Feb 2010) New Revision: 3252 Modified: SwiftApps/SwiftR/Swift.R SwiftApps/SwiftR/TestSwift.R Log: Set site correctly; add tests. Modified: SwiftApps/SwiftR/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift.R 2010-02-23 03:06:07 UTC (rev 3251) +++ SwiftApps/SwiftR/Swift.R 2010-02-23 04:36:05 UTC (rev 3252) @@ -23,7 +23,7 @@ batch = batch + 1; } nbatches = batch - 1 - system(paste("./RunSwiftScript.sh",rundir,"local",sep=" ")) + system(paste("./RunSwiftScript.sh",rundir,site,sep=" ")) rno = 1 rlist = list() Modified: SwiftApps/SwiftR/TestSwift.R =================================================================== --- SwiftApps/SwiftR/TestSwift.R 2010-02-23 03:06:07 UTC (rev 3251) +++ SwiftApps/SwiftR/TestSwift.R 2010-02-23 04:36:05 UTC (rev 3252) @@ -1,14 +1,18 @@ require(boot) source("Swift.R") +SKIP=FALSE +sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } + args=list(ducks,dogs) -sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } +arglist = rep(list(args),9) + +if(SKIP) { + res = do.call(sumcrits,args) cat("Test of do.call(sumcrits)\n") print(res) -arglist = rep(list(args),9) - cat("\nTest of swiftapply(sumcrits,arglist)\n") res = swiftapply(sumcrits,arglist) print(res) @@ -29,3 +33,10 @@ res = swiftapply(sumcrits,arglist,callsperbatch=20) print(res) +} # END SKIP + +cat("\nTest of swiftapply(sumcrits,arglist[1000],callsperbatch=1)\n") +arglist = rep(list(args),1000) +res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs") +print(res[[1]]) +print(res[[1000]]) From noreply at svn.ci.uchicago.edu Tue Feb 23 12:32:38 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 23 Feb 2010 12:32:38 -0600 (CST) Subject: [Swift-commit] r3253 - SwiftApps/SwiftR Message-ID: <20100223183238.65B7F9CCA9@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-02-23 12:32:38 -0600 (Tue, 23 Feb 2010) New Revision: 3253 Modified: SwiftApps/SwiftR/RunSwiftScript.sh SwiftApps/SwiftR/Swift.R SwiftApps/SwiftR/TestSwift.R SwiftApps/SwiftR/bootstrapdemo.R SwiftApps/SwiftR/pboot.R Log: Incremental commit: Added swiftopts; adjusted boot() demo; changed = to <- to slowly get R-ified. Modified: SwiftApps/SwiftR/RunSwiftScript.sh =================================================================== --- SwiftApps/SwiftR/RunSwiftScript.sh 2010-02-23 04:36:05 UTC (rev 3252) +++ SwiftApps/SwiftR/RunSwiftScript.sh 2010-02-23 18:32:38 UTC (rev 3253) @@ -22,7 +22,7 @@ 8 10000 - .00 + 5.99 $(pwd) Modified: SwiftApps/SwiftR/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift.R 2010-02-23 04:36:05 UTC (rev 3252) +++ SwiftApps/SwiftR/Swift.R 2010-02-23 18:32:38 UTC (rev 3253) @@ -1,38 +1,63 @@ -swiftapply <- function( func, arglists, site="local", callsperbatch=1 ) +if(is.null(swiftprops)) { + swiftprops.site <- "local" + swiftprops$callsperbatch <- 1 +} + +swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL ) { - rundir = system("mktemp -d SwiftR.run.XXX",intern=TRUE) - cat("Running in ",rundir,"\n") - narglists = length(arglists) # number of arglists to process - batch=1 # Next arglist batch number to fill - arglist=1 # Next arglist number to insert + # Set Swift properties + + if(is.null(site)) + if( is.null(swiftprops) || is.null(swiftprops$site)) + site <- "local" + else + site <- swiftprops$site + if(is.null(callsperbatch)) + if( is.null(swiftprops) || is.null(swiftprops$callsperbatch)) + callsperbatch <- 1 + else + callsperbatch <- swiftprops$callsperbatch + cat("\nSwift R properties:\n") + cat(" site =",site,"\n"); + cat(" callsperbatch =",callsperbatch,"\n") + + # Execute the calls in batches + + rundir <- system("mktemp -d SwiftR.run.XXX",intern=TRUE) + cat("Swift running in",rundir,"\n") + narglists <- length(arglists) # number of arglists to process + batch <- 1 # Next arglist batch number to fill + arglist <- 1 # Next arglist number to insert while(arglist <= narglists) { - arglistsleft = narglists - arglist + 1 + arglistsleft <- narglists - arglist + 1 if(arglistsleft >= callsperbatch) { - batchsize = callsperbatch + batchsize <- callsperbatch } else { - batchsize = arglistsleft + batchsize <- arglistsleft } - arglistbatch = list() + arglistbatch <- list() for(i in 1 : batchsize) { - arglistbatch[[i]] = arglists[[arglist]] - arglist = arglist +1 + arglistbatch[[i]] <- arglists[[arglist]] + arglist <- arglist +1 } - rcall = list(func=func,arglistbatch=arglistbatch) + rcall <- list(func=func,arglistbatch=arglistbatch) save(rcall,file=paste(rundir,"/cbatch.",as.character(batch),".Rdata",sep="")) - batch = batch + 1; + batch <- batch + 1; } - nbatches = batch - 1 + nbatches <- batch - 1 system(paste("./RunSwiftScript.sh",rundir,site,sep=" ")) - rno = 1 - rlist = list() + # Fetch the batch results + + rno <- 1 + rlist <- list() for(batch in 1:nbatches) { load(paste(rundir,"/rbatch.",as.character(batch),".Rdata",sep="")) - nresults = length(result) + nresults <- length(result) for(r in 1:nresults) { - rlist[[rno]] = result[[r]] - rno = rno + 1 + rlist[[rno]] <- result[[r]] + rno <- rno + 1 } } return(rlist) @@ -42,14 +67,14 @@ x n args x batch - into svn - unique dirs - select sites and swift args (throttles etc) +x into svn +x unique dirs +_ select sites and swift args (throttles etc) R docs R package (SwiftR) Swift docs async exec - clean up boot: fix all calls to statistics + clean up boot: fix all calls to statistics; update boot package w/ pboot() or swiftboot() error handling and null and missing values: ensure res#s correspond to arg#s status specify swift scripts @@ -60,7 +85,7 @@ pass extra vals pass extra files specifiy unique swift scritps ala Dirk's tools - setup the R envs + setup the R envs (???) coasters for persistent R Servers test suites use littleR Modified: SwiftApps/SwiftR/TestSwift.R =================================================================== --- SwiftApps/SwiftR/TestSwift.R 2010-02-23 04:36:05 UTC (rev 3252) +++ SwiftApps/SwiftR/TestSwift.R 2010-02-23 18:32:38 UTC (rev 3253) @@ -1,42 +1,48 @@ require(boot) source("Swift.R") -SKIP=FALSE sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } args=list(ducks,dogs) arglist = rep(list(args),9) -if(SKIP) { +if(TRUE) { # Basic tests -res = do.call(sumcrits,args) -cat("Test of do.call(sumcrits)\n") -print(res) + res = do.call(sumcrits,args) + cat("Test of do.call(sumcrits)\n") + print(res) -cat("\nTest of swiftapply(sumcrits,arglist)\n") -res = swiftapply(sumcrits,arglist) -print(res) + cat("\nTest of swiftapply(sumcrits,arglist)\n") + res = swiftapply(sumcrits,arglist) + print(res) +} -cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") -res = swiftapply(sumcrits,arglist,callsperbatch=10) -print(res) +if(FALSE) { # Test various batch sizes -cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=2)\n") -res = swiftapply(sumcrits,arglist,callsperbatch=2) -print(res) + cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") + res = swiftapply(sumcrits,arglist,callsperbatch=10) + print(res) -cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=3)\n") -res = swiftapply(sumcrits,arglist,callsperbatch=3) -print(res) + cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=2)\n") + res = swiftapply(sumcrits,arglist,callsperbatch=2) + print(res) -cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=20)\n") -res = swiftapply(sumcrits,arglist,callsperbatch=20) -print(res) + cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=3)\n") + res = swiftapply(sumcrits,arglist,callsperbatch=3) + print(res) -} # END SKIP + cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=20)\n") + res = swiftapply(sumcrits,arglist,callsperbatch=20) + print(res) +} -cat("\nTest of swiftapply(sumcrits,arglist[1000],callsperbatch=1)\n") -arglist = rep(list(args),1000) -res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs") -print(res[[1]]) -print(res[[1000]]) +if(FALSE) { # Larger-scale tests + + cat("\nTest of swiftapply(sumcrits,arglist[1000],callsperbatch=1)\n") + + arglist = rep(list(args),1000) + res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs") + + print(res[[1]]) + print(res[[1000]]) +} Modified: SwiftApps/SwiftR/bootstrapdemo.R =================================================================== --- SwiftApps/SwiftR/bootstrapdemo.R 2010-02-23 04:36:05 UTC (rev 3252) +++ SwiftApps/SwiftR/bootstrapdemo.R 2010-02-23 18:32:38 UTC (rev 3253) @@ -8,6 +8,13 @@ require(OpenMx) require(boot) +# Update the boot() function to call swiftapply() for parallel execution. + +if(FALSE) { + source("pboot.R") + assignInNamespace("boot",boot,"boot") +} + # Define a function called mles which will return maximum likelihood estimates # It uses the demoOneFactor dataset and one factor model on the OpenMx homepage # http://openmx.psyc.virginia.edu @@ -32,12 +39,8 @@ # Run 100 bootstraps (a smallish number) -boot.out=list() +system.time(boot(demoOneFactor,mles,R=10000)) -boot.out[[1]] = pboot(demoOneFactor,mles,R=100) -#boot.out[[2]] = boot(demoOneFactor,mles,R=8) -#boot.out[[3]] = boot(demoOneFactor,mles,R=9) - print("done booting - boot.out is:") print(boot.out) print("end of boot.out") Modified: SwiftApps/SwiftR/pboot.R =================================================================== --- SwiftApps/SwiftR/pboot.R 2010-02-23 04:36:05 UTC (rev 3252) +++ SwiftApps/SwiftR/pboot.R 2010-02-23 18:32:38 UTC (rev 3253) @@ -1,4 +1,4 @@ -pboot = +boot = function (data, statistic, R, sim = "ordinary", stype = "i", strata = rep(1, n), L = NULL, m = 0, weights = NULL, ran.gen = function(d, p) d, mle = NULL, simple = FALSE, ...) @@ -106,7 +106,7 @@ # for (r in 1L:sum(R)) t.star[r, ] <- statistic(data, i[r, ], ...) alists = list() for (r in 1L:sum(R)) alists[[r]] <- list(data,i[r,],...) - reslist = swiftapplyb(statistic,alists,callsperbatch=25) + reslist = swiftapply(statistic,alists,callsperbatch=25) for (r in 1L:sum(R)) t.star[r, ] <- reslist[[r]] } } From noreply at svn.ci.uchicago.edu Thu Feb 25 22:16:47 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 25 Feb 2010 22:16:47 -0600 (CST) Subject: [Swift-commit] r3254 - SwiftApps/SwiftR Message-ID: <20100226041647.5BC1F9CCA4@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-02-25 22:16:47 -0600 (Thu, 25 Feb 2010) New Revision: 3254 Modified: SwiftApps/SwiftR/RunSwiftScript.sh SwiftApps/SwiftR/Swift.R SwiftApps/SwiftR/bootstrapdemo.R Log: Adjust Swift.R for packaging; adjust bootstrapdemo for updates to boot package. Modified: SwiftApps/SwiftR/RunSwiftScript.sh =================================================================== --- SwiftApps/SwiftR/RunSwiftScript.sh 2010-02-23 18:32:38 UTC (rev 3253) +++ SwiftApps/SwiftR/RunSwiftScript.sh 2010-02-26 04:16:47 UTC (rev 3254) @@ -12,7 +12,7 @@ 10000 - .03 + .11 $(pwd) @@ -20,7 +20,7 @@ 00:00:10 1800 - 8 + 1 10000 5.99 Modified: SwiftApps/SwiftR/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift.R 2010-02-23 18:32:38 UTC (rev 3253) +++ SwiftApps/SwiftR/Swift.R 2010-02-26 04:16:47 UTC (rev 3254) @@ -1,10 +1,13 @@ -if(is.null(swiftprops)) { - swiftprops.site <- "local" - swiftprops$callsperbatch <- 1 -} - swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL ) { + # Move swiftprops into a Swift namespace + + if(!exists("swiftprops")) { + swiftprops <<-list() + swiftprops$site <<- "local" + swiftprops$callsperbatch <<- 1 + } + # Set Swift properties if(is.null(site)) @@ -62,40 +65,3 @@ } return(rlist) } - -TODO=" - -x n args -x batch -x into svn -x unique dirs -_ select sites and swift args (throttles etc) - R docs - R package (SwiftR) - Swift docs - async exec - clean up boot: fix all calls to statistics; update boot package w/ pboot() or swiftboot() - error handling and null and missing values: ensure res#s correspond to arg#s - status - specify swift scripts - run async and grab status (track 'runs' in R) - increm result collect - pass the func as val - pass extra funcs and packages required - pass extra vals - pass extra files - specifiy unique swift scritps ala Dirk's tools - setup the R envs (???) - coasters for persistent R Servers - test suites - use littleR - args as alists vs args as list - runids, output logging - select exec sites and swift params etc - make polymorphic to *apply and snow - stream results back to R (so use can inspect as they arrive) - (pull them in with a Swift.poll() func) - handle discontiguous results - return good error messages including messages from R eval and from Swift - -END" Modified: SwiftApps/SwiftR/bootstrapdemo.R =================================================================== --- SwiftApps/SwiftR/bootstrapdemo.R 2010-02-23 18:32:38 UTC (rev 3253) +++ SwiftApps/SwiftR/bootstrapdemo.R 2010-02-26 04:16:47 UTC (rev 3254) @@ -8,20 +8,14 @@ require(OpenMx) require(boot) -# Update the boot() function to call swiftapply() for parallel execution. +bootdemo <- function(swift=FALSE,R=100) { -if(FALSE) { - source("pboot.R") - assignInNamespace("boot",boot,"boot") -} - # Define a function called mles which will return maximum likelihood estimates # It uses the demoOneFactor dataset and one factor model on the OpenMx homepage # http://openmx.psyc.virginia.edu mles<-function(dataset,wt){ -cat("in mles="); require(OpenMx) manifests <- names(dataset) latents <- c("G") @@ -37,15 +31,14 @@ numObs=500))) return(as.vector(mlevals at output$estimate))} -# Run 100 bootstraps (a smallish number) +# Run R bootstraps -system.time(boot(demoOneFactor,mles,R=10000)) +boottime = system.time(boot(demoOneFactor,mles,R=R,swift=swift)) +cat("done booting - system.time is:\n") +print(boottime) -print("done booting - boot.out is:") -print(boot.out) -print("end of boot.out") - # For comparison, take a look at the SE output from running the homepage job once + data(demoOneFactor) manifests <- names(demoOneFactor) latents <- c("G") @@ -71,4 +64,6 @@ # The above should indicate that the results are close enough. +} # function bootdemo + From noreply at svn.ci.uchicago.edu Thu Feb 25 22:24:59 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 25 Feb 2010 22:24:59 -0600 (CST) Subject: [Swift-commit] r3255 - SwiftApps/SwiftR Message-ID: <20100226042459.AB2669CCA4@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-02-25 22:24:59 -0600 (Thu, 25 Feb 2010) New Revision: 3255 Added: SwiftApps/SwiftR/TODO SwiftApps/SwiftR/TestOpenMx1.R Log: Add TODO list and temporary OpenMx text. Added: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO (rev 0) +++ SwiftApps/SwiftR/TODO 2010-02-26 04:24:59 UTC (rev 3255) @@ -0,0 +1,35 @@ + +x n args +x batch +x into svn +x unique dirs +_ select sites and swift args (throttles etc) + R docs + R package (SwiftR) + Swift docs + async exec + clean up boot: fix all calls to statistics; update boot package w/ pboot() or swiftboot() + error handling and null and missing values: ensure res#s correspond to arg#s + status + specify swift scripts + run async and grab status (track 'runs' in R) + increm result collect + pass the func as val + pass extra funcs and packages required + pass extra vals + pass extra files + specifiy unique swift scritps ala Dirk's tools + setup the R envs (???) + coasters for persistent R Servers + test suites + use littleR + args as alists vs args as list + runids, output logging + select exec sites and swift params etc + make polymorphic to *apply and snow + stream results back to R (so use can inspect as they arrive) + (pull them in with a Swift.poll() func) + handle discontiguous results + return good error messages including messages from R eval and from Swift + TUI + Name pbs jobs mnemonically Added: SwiftApps/SwiftR/TestOpenMx1.R =================================================================== --- SwiftApps/SwiftR/TestOpenMx1.R (rev 0) +++ SwiftApps/SwiftR/TestOpenMx1.R 2010-02-26 04:24:59 UTC (rev 3255) @@ -0,0 +1,16 @@ +require(OpenMx) + + data(demoOneFactor) + manifests <- names(demoOneFactor) + latents <- c("G") + + factorModel <- mxModel("One Factor", + type="RAM", + manifestVars = manifests, + latentVars = latents, + mxPath(from=latents, to=manifests), + mxPath(from=manifests, arrows=2), + mxPath(from=latents, arrows=2, free=F, values=1.0), + mxData(observed=cov(demoOneFactor), type="cov", numObs=500)) + + summary(mxRun(factorModel))