[Swift-commit] r4242 - in branches/release-0.92: . docs docs/formatting docs/historical etc libexec src/org/griphyn/vdl/karajan src/org/griphyn/vdl/karajan/lib src/org/griphyn/vdl/karajan/lib/cache src/org/griphyn/vdl/karajan/lib/swiftscript src/org/griphyn/vdl/mapping src/org/griphyn/vdl/mapping/file tests tests/groups tests/mpi
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Sat Apr 2 14:01:11 CDT 2011
Author: hategan
Date: 2011-04-02 14:01:11 -0500 (Sat, 02 Apr 2011)
New Revision: 4242
Removed:
branches/release-0.92/src/org/griphyn/vdl/karajan/HangChecker.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Stagein.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Stageout.java
branches/release-0.92/tests/groups/group-mpi.sh
branches/release-0.92/tests/groups/group-pc.sh
branches/release-0.92/tests/persistent-coasters/
Modified:
branches/release-0.92/.classpath
branches/release-0.92/build.xml
branches/release-0.92/docs/
branches/release-0.92/docs/formatting/
branches/release-0.92/docs/historical/
branches/release-0.92/etc/log4j.properties
branches/release-0.92/libexec/_swiftwrap
branches/release-0.92/libexec/_swiftwrap.staging
branches/release-0.92/libexec/execute-default.k
branches/release-0.92/libexec/vdl-int-staging.k
branches/release-0.92/libexec/vdl-int.k
branches/release-0.92/libexec/vdl-sc.k
branches/release-0.92/libexec/vdl.k
branches/release-0.92/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java
branches/release-0.92/src/org/griphyn/vdl/karajan/DSHandleFutureWrapper.java
branches/release-0.92/src/org/griphyn/vdl/karajan/FuturePairIterator.java
branches/release-0.92/src/org/griphyn/vdl/karajan/Loader.java
branches/release-0.92/src/org/griphyn/vdl/karajan/Monitor.java
branches/release-0.92/src/org/griphyn/vdl/karajan/ScalabilityTest.java
branches/release-0.92/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java
branches/release-0.92/src/org/griphyn/vdl/karajan/VDL2FutureException.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/CacheAddAndLockFile.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/FileCopier.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Log.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/cache/File.java
branches/release-0.92/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java
branches/release-0.92/src/org/griphyn/vdl/mapping/AbstractDataNode.java
branches/release-0.92/src/org/griphyn/vdl/mapping/DSHandle.java
branches/release-0.92/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java
branches/release-0.92/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java
branches/release-0.92/tests/groups/group-all-local.sh
branches/release-0.92/tests/meta.sh
branches/release-0.92/tests/mpi/
branches/release-0.92/tests/nightly.sh
Log:
reverted to r4200
Modified: branches/release-0.92/.classpath
===================================================================
--- branches/release-0.92/.classpath 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/.classpath 2011-04-02 19:01:11 UTC (rev 4242)
@@ -18,6 +18,7 @@
<classpathentry combineaccessrules="false" kind="src" path="/util"/>
<classpathentry combineaccessrules="false" kind="src" path="/grapheditor"/>
<classpathentry combineaccessrules="false" kind="src" path="/abstraction-provider-coaster"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/provider-wonky"/>
<classpathentry kind="lib" path="lib/jline-0.9.94.jar"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="lib" path="/util/lib/log4j-1.2.8.jar"/>
Modified: branches/release-0.92/build.xml
===================================================================
--- branches/release-0.92/build.xml 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/build.xml 2011-04-02 19:01:11 UTC (rev 4242)
@@ -366,7 +366,7 @@
<ant antfile="${main.buildfile}" target="module.package"/>
</target>
- <target name="generateVersion" unless="no.version">
+ <target name="generateVersion">
<echo file="libexec/version.txt">Swift ${version} </echo>
<exec os="Linux, Mac OS X" executable="libexec/svn-revision" append="true" output="libexec/version.txt"/>
<exec os="Linux, Mac OS X" executable="libexec/generate-buildid" append="false" output="libexec/buildid.txt"/>
Property changes on: branches/release-0.92/docs
___________________________________________________________________
Name: svn:ignore
- docbook
userguide
*.php
index.html
Property changes on: branches/release-0.92/docs/formatting
___________________________________________________________________
Name: svn:ignore
- docbook
Property changes on: branches/release-0.92/docs/historical
___________________________________________________________________
Name: svn:ignore
- *.php
Modified: branches/release-0.92/etc/log4j.properties
===================================================================
--- branches/release-0.92/etc/log4j.properties 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/etc/log4j.properties 2011-04-02 19:01:11 UTC (rev 4242)
@@ -28,4 +28,5 @@
log4j.logger.org.griphyn.vdl.engine.Karajan=INFO
log4j.logger.org.globus.cog.abstraction.coaster.rlog=INFO
+# log4j.logger.org.globus.swift.data.Director=DEBUG
log4j.logger.org.griphyn.vdl.karajan.lib=INFO
Modified: branches/release-0.92/libexec/_swiftwrap
===================================================================
--- branches/release-0.92/libexec/_swiftwrap 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/libexec/_swiftwrap 2011-04-02 19:01:11 UTC (rev 4242)
@@ -238,7 +238,7 @@
openinfo "$INFOFILE"
logstate "LOG_START"
-infosection "Wrapper (_swiftwrap)"
+infosection "Wrapper"
getarg "-e" "$@"
EXEC=$VALUE
Modified: branches/release-0.92/libexec/_swiftwrap.staging
===================================================================
--- branches/release-0.92/libexec/_swiftwrap.staging 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/libexec/_swiftwrap.staging 2011-04-02 19:01:11 UTC (rev 4242)
@@ -1,20 +1,5 @@
# this script must be invoked inside of bash, not plain sh
-openinfo() {
- exec 3<> $1
- INFO=3
-}
-
-closeinfo() {
- exec 3>&-
-}
-
-# Clobbered by SWIFTWRAP_FAST
-log() {
- echo "$@" >& "$INFO"
-}
-
-# Clobbered by SWIFTWRAP_FAST
infosection() {
echo >& "$INFO"
echo "_____________________________________________________________________________" >& "$INFO"
@@ -53,11 +38,14 @@
fi
}
-# Clobbered by SWIFTWRAP_FAST
logstate() {
echo "Progress " `date +"%Y-%m-%d %H:%M:%S.%N%z"` " $@" >& "$INFO"
}
+log() {
+ echo "$@" >& "$INFO"
+}
+
fail() {
EC=$1
shift
@@ -89,32 +77,13 @@
fi
}
-# Clobbered by SWIFTWRAP_FAST
-checkMissing() {
- MISSING=
- for O in $OUTF ; do
- if [ ! -f "$O" ]; then
- if [ "$MISSING" == "" ]; then
- MISSING=$O
- else
- MISSING="$MISSING, $O"
- fi
- fi
- done
- if [ "$MISSING" != "" ]; then
- fail 254 "The following output files were not created by the application: $MISSING"
- fi
-}
-
checkparamfile() {
log "checking for paramfile"
if [ "$1" == "-p" ]; then
JOBDIR=$2
PARAMFILE=${WFDIR}/parameters/${JOBDIR}/param-${ID}
- log "paramfile is: $PARAMFILE"
- else
- log "no paramfile: using command line arguments"
fi
+ log "paramfile is: $PARAMFILE"
}
getarg() {
@@ -138,6 +107,15 @@
fi
}
+openinfo() {
+ exec 3<> $1
+ INFO=3
+}
+
+closeinfo() {
+ exec 3>&-
+}
+
COMMANDLINE=$@
# make the WFDIR absolute
@@ -147,24 +125,13 @@
checkparamfile "$@"
-# # SWIFTWRAP_FAST: Turn things off for speed
-if [[ $SWIFTWRAP_FAST == "1" ]]; then
- shopt -s expand_aliases
- alias infosection=:
- alias logstate=:
- alias log=:
- alias checkMissing=:
-fi
-
if [ "X$INFODIR" == "X" ]; then
INFODIR="."
fi
logstate "LOG_START"
-infosection "Wrapper (_swiftwrap.staging)"
+infosection "Wrapper"
-log $COMMANDLINE
-
getarg "-e" "$@"
EXEC=$VALUE
shift $SHIFTCOUNT
@@ -286,6 +253,7 @@
logstate "EXECUTE"
+#ls >>$WRAPPERLOG
if [ ! -f "$EXEC" ]; then
fail 254 "The executable $EXEC does not exist"
fi
@@ -322,7 +290,19 @@
logstate "EXECUTE_DONE"
log "Job ran successfully"
-checkMissing
+MISSING=
+for O in $OUTF ; do
+ if [ ! -f "$O" ]; then
+ if [ "$MISSING" == "" ]; then
+ MISSING=$O
+ else
+ MISSING="$MISSING, $O"
+ fi
+ fi
+done
+if [ "$MISSING" != "" ]; then
+ fail 254 "The following output files were not created by the application: $MISSING"
+fi
logstate "END"
Modified: branches/release-0.92/libexec/execute-default.k
===================================================================
--- branches/release-0.92/libexec/execute-default.k 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/libexec/execute-default.k 2011-04-02 19:01:11 UTC (rev 4242)
@@ -1,6 +1,7 @@
element([tr, optional(arguments, stdin, stdout, stderr, deperror, mdeperror), channel(stagein), channel(stageout), channel(restartout)]
vdl:initprogressstate()
+ vdl:setprogress("Initializing")
done := isDone(restartout)
derr := try(deperror, false)
@@ -8,26 +9,31 @@
if(
derr == false then(
+ if(done
+ vdl:setprogress("Finished in previous run")
+ )
if(
sys:not(done) try(
sequential(
- log(LOG:INFO, "START thread={#thread} tr={tr}")
+ log(LOG:INFO, "START thread={#thread} tr={tr}")
vdl:setprogress("Selecting site")
restartOnError(".*", vdl:configProperty("execution.retries"),
replicationChannel := channel()
discard(append(replicationChannel, true)) //trigger the first job
replicationGroup := uid()
parallelFor(i, replicationChannel
- ignoreErrors(match = "^Abort$"
+ try(
execute2(
tr, maybe(arguments=arguments),
maybe(stdin=stdin), maybe(stdout=stdout), maybe(stderr=stderr),
stagein, stageout, restartout, replicationGroup, replicationChannel
)
+ catch("^Abort$")
)
)
)
mark(restartout, err=false, mapping=false)
+ graphStuff(tr, stagein, stageout, err=false, maybe(args=arguments))
log(LOG:INFO, "END_SUCCESS thread={#thread} tr={tr}")
vdl:setprogress("Finished successfully")
)
@@ -48,9 +54,6 @@
)
)
)
- else (
- vdl:setprogress("Finished in previous run")
- )
)
)
else (
Modified: branches/release-0.92/libexec/vdl-int-staging.k
===================================================================
--- branches/release-0.92/libexec/vdl-int-staging.k 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/libexec/vdl-int-staging.k 2011-04-02 19:01:11 UTC (rev 4242)
@@ -88,6 +88,7 @@
stdout := try(stdout, "stdout.txt")
stderr := try(stderr, "stderr.txt")
+ vdl:setprogress("Stage in")
wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}"
tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid)
cdmfile := cdm:file()
@@ -168,7 +169,7 @@
"Host: {rhost}", nl(),
"Directory: {tmpdir}",
"TODO: outs", nl(),
- "- - -", nl()
+ "----", nl()
)
exception
)
Modified: branches/release-0.92/libexec/vdl-int.k
===================================================================
--- branches/release-0.92/libexec/vdl-int.k 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/libexec/vdl-int.k 2011-04-02 19:01:11 UTC (rev 4242)
@@ -256,8 +256,7 @@
if (policy == "DEFAULT" then(
restartOnError(".*", 2
task:transfer(srcprovider=provider, srchost=srchost, srcfile=srcfile,
- srcdir=srcdir, desthost=desthost, destdir=destdir,
- transferParams(srchost, desthost, provider))))
+ srcdir=srcdir, desthost=desthost, destdir=destdir)))
policy == "BROADCAST" then(
log(LOG:DEBUG, "FILE_STAGE_IN_BROADCAST file={srcfile} policy={policy}")
cdm:broadcast(srcfile=srcfile, srcdir=srcdir))
@@ -297,8 +296,7 @@
then(
restartOnError(".*", 2
task:transfer(srchost=host, srcfile=bname,srcdir=rdir,
- destdir=ldir, desthost=dhost, destprovider=provider,
- transferParams(host, dhost, provider))))
+ destdir=ldir, desthost=dhost, destprovider=provider)))
else(log(LOG:DEBUG, "FILE_STAGE_OUT_SKIP srcname={bname}"))
)
log(LOG:DEBUG, "FILE_STAGE_OUT_END srcname={bname} srcdir={rdir} srchost={host} ",
Modified: branches/release-0.92/libexec/vdl-sc.k
===================================================================
--- branches/release-0.92/libexec/vdl-sc.k 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/libexec/vdl-sc.k 2011-04-02 19:01:11 UTC (rev 4242)
@@ -32,7 +32,7 @@
major == "4" "GT4"
major == "2" "GT2"
throw("Unknown job manager version: major = {major}, minor = {minor}, patch = {patch}, url = {url}")
- )
+ )
service(type="execution", provider=provider, url=url)
)
@@ -89,7 +89,7 @@
)
element(surveyorCoasterParams, [optional(averageJobTime, blockSize)]
- print("Setting Surveyor coaster params:")
+ print("Setting Surveyor coaster params:")
p("scratch", "/scratch")
p("workersPerNode", "4")
p("kernelprofile", "zeptoos")
Modified: branches/release-0.92/libexec/vdl.k
===================================================================
--- branches/release-0.92/libexec/vdl.k 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/libexec/vdl.k 2011-04-02 19:01:11 UTC (rev 4242)
@@ -23,9 +23,6 @@
log("info","RUNID id=run:{VDL:RUNID}")
echo("RunID: {VDL:RUNID}")
)
-
- export("stagein", elementDef(classname="org.griphyn.vdl.karajan.lib.Stagein"))
- export("stageout", elementDef(classname="org.griphyn.vdl.karajan.lib.Stageout"))
export(
@@ -59,7 +56,72 @@
)
)
)
-
+
+ element(stagein, [var]
+ if(
+ vdl:isFileBound(var) try(
+ sequential(
+ fp := vdl:fringePaths(var)
+ try (
+ for(path, fp
+ vdl:waitFieldValue(path=path, var)
+ )
+// NOTE this string-based catch is messy, and needs to match with the string used in DataDependentException and VDLFunction
+ catch(".*errors in data dependencies.*"
+ log(LOG:DEBUG, exception)
+ deperror = true
+ )
+ )
+ channel:to(stagein,
+ for(path, fp
+ vdl:absFileName(vdl:getfield(var, path = path))
+ )
+ )
+ )
+ catch(".*not mapped.*"
+ log(LOG:DEBUG, exception)
+ mdeperror = true
+ deperror = true
+ )
+ )
+ else(
+ //we still wait until the primitive value is there
+ vdl:waitFieldValue(var)
+ )
+ )
+ )
+
+ element(stageout, [var]
+ try(
+ if(vdl:isFileBound(var)
+ channel:to(stageout,
+ for(path, vdl:fringePaths(var)
+ list(path, var)
+ )
+ )
+ )
+ catch(".*not mapped.*"
+ log(LOG:DEBUG, exception)
+ mdeperror = true
+ deperror = true
+ )
+ )
+ try(
+ if(vdl:isRestartable(var)
+ channel:to(restartout,
+ for(path, vdl:fringePaths(var)
+ list(path, var)
+ )
+ )
+ )
+ catch(".*not mapped.*"
+ log(LOG:DEBUG, exception)
+ mdeperror = true
+ deperror = true
+ )
+ )
+ )
+
element(mapping, [descriptor, ...]
mapping=map(map:entry("descriptor", descriptor), each(...))
)
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -12,13 +12,13 @@
import org.globus.cog.karajan.stack.VariableNotFoundException;
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.EventListener;
import org.globus.cog.karajan.workflow.events.EventTargetPair;
import org.globus.cog.karajan.workflow.futures.FutureEvaluationException;
import org.globus.cog.karajan.workflow.futures.FutureIterator;
import org.globus.cog.karajan.workflow.futures.FutureList;
-import org.globus.cog.karajan.workflow.futures.FutureListener;
import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
-import org.globus.cog.karajan.workflow.futures.ListenerStackPair;
import org.globus.cog.util.CopyOnWriteArrayList;
import org.griphyn.vdl.mapping.DSHandle;
import org.griphyn.vdl.mapping.DSHandleListener;
@@ -27,7 +27,7 @@
private ArrayList<Object> keys;
private Map values;
private boolean closed;
- private CopyOnWriteArrayList<ListenerStackPair> listeners;
+ private CopyOnWriteArrayList<EventTargetPair> listeners;
private FutureEvaluationException exception;
public ArrayIndexFutureList(DSHandle handle, Map values) {
@@ -90,13 +90,13 @@
return this;
}
- public synchronized void addModificationAction(FutureListener target,
- VariableStack stack) {
+ public synchronized void addModificationAction(EventListener target,
+ Event event) {
if (listeners == null) {
- listeners = new CopyOnWriteArrayList<ListenerStackPair>();
+ listeners = new CopyOnWriteArrayList<EventTargetPair>();
}
- listeners.add(new ListenerStackPair(target, stack));
+ listeners.add(new EventTargetPair(event, target));
if (closed) {
notifyListeners();
}
@@ -107,12 +107,17 @@
return;
}
- Iterator<ListenerStackPair> i = listeners.iterator();
+ Iterator<EventTargetPair> i = listeners.iterator();
try {
while (i.hasNext()) {
- ListenerStackPair etp = i.next();
- i.remove();
- etp.listener.futureModified(this, etp.stack);
+ try {
+ EventTargetPair etp = i.next();
+ i.remove();
+ etp.getTarget().event(etp.getEvent());
+ }
+ catch (ExecutionException e) {
+ e.printStackTrace();
+ }
}
}
finally {
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/DSHandleFutureWrapper.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/DSHandleFutureWrapper.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/DSHandleFutureWrapper.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -6,19 +6,18 @@
import java.util.LinkedList;
import org.globus.cog.karajan.stack.VariableNotFoundException;
-import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.workflow.events.Event;
import org.globus.cog.karajan.workflow.events.EventBus;
+import org.globus.cog.karajan.workflow.events.EventListener;
import org.globus.cog.karajan.workflow.events.EventTargetPair;
import org.globus.cog.karajan.workflow.futures.Future;
import org.globus.cog.karajan.workflow.futures.FutureEvaluationException;
-import org.globus.cog.karajan.workflow.futures.FutureListener;
-import org.globus.cog.karajan.workflow.futures.ListenerStackPair;
import org.griphyn.vdl.mapping.DSHandle;
import org.griphyn.vdl.mapping.DSHandleListener;
public class DSHandleFutureWrapper implements Future, DSHandleListener {
private DSHandle handle;
- private LinkedList<ListenerStackPair> listeners;
+ private LinkedList listeners;
public DSHandleFutureWrapper(DSHandle handle) {
this.handle = handle;
@@ -43,7 +42,7 @@
}
}
- public synchronized void addModificationAction(FutureListener target, VariableStack stack) {
+ public synchronized void addModificationAction(EventListener target, Event event) {
/**
* TODO So, the strategy is the following: getValue() or something else
* throws a future exception; then some entity catches that and calls
@@ -53,10 +52,10 @@
* this method and call notifyListeners().
*/
if (listeners == null) {
- listeners = new LinkedList<ListenerStackPair>();
+ listeners = new LinkedList();
}
- listeners.add(new ListenerStackPair(target, stack));
- WaitingThreadsMonitor.addThread(stack);
+ listeners.add(new EventTargetPair(event, target));
+ WaitingThreadsMonitor.addThread(event.getStack());
if (handle.isClosed()) {
notifyListeners();
}
@@ -67,13 +66,9 @@
return;
}
while (!listeners.isEmpty()) {
- final ListenerStackPair etp = listeners.removeFirst();
- WaitingThreadsMonitor.removeThread(etp.stack);
- EventBus.post(new Runnable() {
- public void run() {
- etp.listener.futureModified(DSHandleFutureWrapper.this, etp.stack);
- }
- });
+ EventTargetPair etp = (EventTargetPair) listeners.removeFirst();
+ WaitingThreadsMonitor.removeThread(etp.getEvent().getStack());
+ EventBus.post(etp.getTarget(), etp.getEvent());
}
listeners = null;
}
@@ -97,19 +92,7 @@
}
public String toString() {
- String l;
- if (listeners == null) {
- l = "no listeners";
- }
- else {
- l = listeners.size() + " listeners";
- }
- if (!isClosed()) {
- return "Open, " + l;
- }
- else {
- return "Closed, " + l;
- }
+ return "F/" + handle;
}
public void fail(FutureEvaluationException e) {
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/FuturePairIterator.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/FuturePairIterator.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/FuturePairIterator.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -5,10 +5,11 @@
import org.globus.cog.karajan.stack.VariableNotFoundException;
import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.EventListener;
import org.globus.cog.karajan.workflow.futures.FutureEvaluationException;
import org.globus.cog.karajan.workflow.futures.FutureIterator;
import org.globus.cog.karajan.workflow.futures.FutureIteratorIncomplete;
-import org.globus.cog.karajan.workflow.futures.FutureListener;
import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
public class FuturePairIterator implements FutureIterator {
@@ -93,9 +94,9 @@
return this;
}
- public synchronized void addModificationAction(FutureListener target, VariableStack stack) {
- WaitingThreadsMonitor.addThread(stack);
- array.addModificationAction(target, stack);
+ public synchronized void addModificationAction(EventListener target, Event event) {
+ WaitingThreadsMonitor.addThread(event.getStack());
+ array.addModificationAction(target, event);
}
private static volatile int cnt = 0;
Deleted: branches/release-0.92/src/org/griphyn/vdl/karajan/HangChecker.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/HangChecker.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/HangChecker.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -1,66 +0,0 @@
-//----------------------------------------------------------------------
-//This code is developed as part of the Java CoG Kit project
-//The terms of the license can be found at http://www.cogkit.org/license
-//This message may not be removed or altered.
-//----------------------------------------------------------------------
-
-/*
- * Created on Mar 6, 2011
- */
-package org.griphyn.vdl.karajan;
-
-import java.io.ByteArrayOutputStream;
-import java.io.PrintStream;
-import java.util.Timer;
-import java.util.TimerTask;
-
-import org.apache.log4j.Logger;
-import org.globus.cog.karajan.scheduler.LateBindingScheduler;
-import org.globus.cog.karajan.stack.VariableStack;
-import org.globus.cog.karajan.workflow.ExecutionException;
-import org.globus.cog.karajan.workflow.events.EventBus;
-import org.globus.cog.karajan.workflow.nodes.grid.SchedulerNode;
-import org.griphyn.vdl.karajan.lib.VDLFunction;
-
-public class HangChecker extends TimerTask {
- public static final Logger logger = Logger.getLogger(HangChecker.class);
-
- public static final int CHECK_INTERVAL = 10000;
- private Timer timer;
- private long lastEventCount;
- private WrapperMap map;
- private VariableStack stack;
-
- public HangChecker(VariableStack stack) throws ExecutionException {
- this.stack = stack;
- map = VDLFunction.getFutureWrapperMap(stack);
- }
-
- public void start() {
- timer = new Timer("Hang checker");
- timer.scheduleAtFixedRate(this, CHECK_INTERVAL, CHECK_INTERVAL);
- }
-
- public void run() {
- try {
- LateBindingScheduler s = (LateBindingScheduler) stack.getGlobal(SchedulerNode.SCHEDULER);
- if (s != null) {
- int running = s.getRunning();
-
- if (running == 0 && EventBus.eventCount == lastEventCount) {
- logger.warn("No events in " + (CHECK_INTERVAL / 1000) + "s.");
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- PrintStream ps = new PrintStream(os);
- Monitor.dumpVariables(map, ps);
- Monitor.dumpThreads(ps);
- logger.warn(os.toString());
- ps.close();
- }
- }
- lastEventCount = EventBus.eventCount;
- }
- catch (Exception e) {
- logger.warn("Exception caught during hang check", e);
- }
- }
-}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/Loader.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/Loader.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/Loader.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -128,10 +128,9 @@
}
try {
- //Thread.sleep(20000);
setupLogging(ap, projectName, runID);
logger.debug("Max heap: " + Runtime.getRuntime().maxMemory());
-
+
if (ap.isPresent(ARG_CDMFILE)) {
loadCDM(ap);
}
@@ -193,8 +192,7 @@
arguments.add("-rlog:resume=" + ap.getStringValue(ARG_RESUME));
}
ec.setArguments(arguments);
- long start = System.currentTimeMillis();
- new HangChecker(stack).start();
+
ec.start(stack);
ec.waitFor();
if (ec.isFailed()) {
@@ -207,10 +205,10 @@
}
if (runerror) {
- logger.info("Swift finished with errors");
+ logger.debug("Swift finished with errors");
}
else {
- logger.info("Swift finished with no errors");
+ logger.debug("Swift finished with no errors");
}
if (ap.isPresent(ARG_TUI)) {
ma.close();
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/Monitor.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/Monitor.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/Monitor.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -40,7 +40,6 @@
import org.globus.cog.karajan.workflow.events.EventTargetPair;
import org.globus.cog.karajan.workflow.futures.Future;
import org.griphyn.vdl.karajan.WrapperMap.FutureWrappers;
-import org.griphyn.vdl.mapping.AbstractDataNode;
import org.griphyn.vdl.mapping.ArrayDataNode;
import org.griphyn.vdl.mapping.DSHandle;
import org.griphyn.vdl.mapping.DependentException;
@@ -209,12 +208,8 @@
public void dumpVariables() {
dumpVariables(System.out);
}
-
- public void dumpVariables(PrintStream ps) {
- dumpVariables(map, ps);
- }
- public static void dumpVariables(WrapperMap map, PrintStream ps) {
+ public void dumpVariables(PrintStream ps) {
ps.println("\nRegistered futures:");
synchronized (map) {
Iterator i = map.entrySet().iterator();
@@ -228,7 +223,7 @@
else if (fw.arrayWrapper != null) {
f = fw.arrayWrapper;
}
- AbstractDataNode handle = (AbstractDataNode) en.getKey();
+ DSHandle handle = (DSHandle) en.getKey();
String value = "-";
try {
if (handle.getValue() != null) {
@@ -238,7 +233,8 @@
catch (DependentException e) {
value = "Dependent exception";
}
- ps.println(handle.getType() + " " + handle.getDisplayableName() + " " + value + " " + f);
+ ps.println(handle.getType() + " " + handle + " " + value + " " + f + " "
+ + (handle.isClosed() ? "Closed" : "Open"));
}
ps.println("----");
}
@@ -248,7 +244,7 @@
dumpThreads(System.out);
}
- public static void dumpThreads(PrintStream pw) {
+ public void dumpThreads(PrintStream pw) {
pw.println("\nWaiting threads:");
Collection c = WaitingThreadsMonitor.getAllThreads();
Iterator i = c.iterator();
@@ -294,10 +290,10 @@
else {
EventTargetPair[] l = Monitor.this.getListeners(rowIndex);
if (l != null) {
- ArrayList<Object> a = new ArrayList<Object>();
+ ArrayList a = new ArrayList();
for (int i = 0; i < l.length; i++) {
try {
- a.add(ThreadingContext.get(l[i].getEvent()));
+ a.add(ThreadingContext.get(l[i].getEvent().getStack()));
}
catch (VariableNotFoundException e) {
a.add("unknown");
@@ -371,7 +367,7 @@
try {
for (int i = 0; i < l.length; i++) {
displayPopup("Stack trace for " + t.getValueAt(row, 1),
- Trace.get(l[i].getEvent()));
+ Trace.get(l[i].getEvent().getStack()));
}
}
catch (NullPointerException ex) {
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/ScalabilityTest.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/ScalabilityTest.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/ScalabilityTest.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -15,11 +15,14 @@
import org.globus.cog.karajan.Loader;
import org.globus.cog.karajan.arguments.AbstractWriteOnlyVariableArguments;
import org.globus.cog.karajan.arguments.VariableArguments;
-import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ElementTree;
import org.globus.cog.karajan.workflow.ExecutionContext;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.EventClass;
import org.globus.cog.karajan.workflow.events.EventListener;
+import org.globus.cog.karajan.workflow.events.NotificationEvent;
+import org.globus.cog.karajan.workflow.events.NotificationEventType;
public class ScalabilityTest {
private static volatile int jobsSubmitted, jobsFailed, jobsCompleted, workflowsStarted,
@@ -108,14 +111,17 @@
}
public static class Listener implements EventListener {
- public void completed(VariableStack stack) throws ExecutionException {
- workflowsCompleted++;
- }
-
- public void failed(VariableStack stack, ExecutionException e)
- throws ExecutionException {
- workflowsFailed++;
- }
+ public void event(Event e) throws ExecutionException {
+ if (e.getEventClass().equals(EventClass.NOTIFICATION_EVENT)) {
+ NotificationEvent ne = (NotificationEvent) e;
+ if (ne.getType().equals(NotificationEventType.EXECUTION_COMPLETED)) {
+ workflowsCompleted++;
+ }
+ else if (ne.getType().equals(NotificationEventType.EXECUTION_FAILED)) {
+ workflowsFailed++;
+ }
+ }
+ }
}
public static class Monitor {
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -7,7 +7,7 @@
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ElementTree;
import org.globus.cog.karajan.workflow.ExecutionContext;
-import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.events.FailureNotificationEvent;
import org.griphyn.vdl.karajan.functions.ProcessBulkErrors;
public class VDL2ExecutionContext extends ExecutionContext {
@@ -24,14 +24,14 @@
this.scriptName = scriptName;
}
- protected void printFailure(ExecutionException e) {
+ protected void printFailure(FailureNotificationEvent e) {
if (logger.isDebugEnabled()) {
- logger.debug(e.getMessage(), e);
+ logger.debug(e.getFlowElement() + ": " + e.getMessage(), e.getException());
}
String msg = e.getMessage();
if (!"Execution completed with errors".equals(msg)) {
- if (msg == null) {
- msg = getMeaningfulMessage(e);
+ if (msg == null && e.getException() != null) {
+ msg = getMeaningfulMessage(e.getException());
}
getStderr().append("Execution failed:\n\t");
String translation = VDL2ErrorTranslator.getDefault().translate(msg);
@@ -39,7 +39,7 @@
getStderr().append(translation);
}
else {
- getStderr().append(ProcessBulkErrors.getMessageChain(e));
+ getStderr().append(ProcessBulkErrors.getMessageChain(e.getException()));
}
getStderr().append("\n");
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/VDL2FutureException.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/VDL2FutureException.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/VDL2FutureException.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -3,19 +3,19 @@
*/
package org.griphyn.vdl.karajan;
-import org.globus.cog.karajan.stack.VariableStack;
-import org.globus.cog.karajan.workflow.nodes.FlowElement;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.EventListener;
import org.griphyn.vdl.mapping.DSHandle;
public class VDL2FutureException extends RuntimeException {
private final DSHandle handle;
- private FlowElement listener;
- private VariableStack stack;
+ private EventListener listener;
+ private Event event;
- public VDL2FutureException(DSHandle handle, FlowElement listener, VariableStack stack) {
+ public VDL2FutureException(DSHandle handle, EventListener listener, Event event) {
this.handle = handle;
this.listener = listener;
- this.stack = stack;
+ this.event = event;
}
public VDL2FutureException(DSHandle handle) {
@@ -26,19 +26,19 @@
return handle;
}
- public VariableStack getStack() {
- return stack;
+ public Event getEvent() {
+ return event;
}
- public FlowElement getListener() {
+ public EventListener getListener() {
return listener;
}
- public void setStack(VariableStack stack) {
- this.stack = stack;
+ public void setEvent(Event event) {
+ this.event = event;
}
- public void setListener(FlowElement listener) {
+ public void setListener(EventListener listener) {
this.listener = listener;
}
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/CacheAddAndLockFile.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/CacheAddAndLockFile.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/CacheAddAndLockFile.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -7,6 +7,9 @@
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.util.TypeUtil;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.NotificationEvent;
+import org.globus.cog.karajan.workflow.events.NotificationEventType;
import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
import org.griphyn.vdl.karajan.lib.cache.CacheReturn;
import org.griphyn.vdl.karajan.lib.cache.File;
@@ -59,12 +62,18 @@
cache.unlockFromProcessing(f);
super.post(stack);
}
-
-
- public void failed(VariableStack stack, ExecutionException e)
- throws ExecutionException {
- VDLFileCache cache = CacheFunction.getCache(stack);
- cache.entryRemoved((File) stack.currentFrame().getVar(PFILE));
- super.failed(stack, e);
- }
+
+ public void event(Event e) throws ExecutionException {
+ super.event(e);
+ }
+
+ protected void notificationEvent(NotificationEvent e) throws ExecutionException {
+ VariableStack stack = e.getStack();
+ if (e.getType().equals(NotificationEventType.EXECUTION_FAILED)
+ && stack.currentFrame().isDefined(PFILE)) {
+ VDLFileCache cache = CacheFunction.getCache(stack);
+ cache.entryRemoved((File) stack.currentFrame().getVar(PFILE));
+ }
+ super.notificationEvent(e);
+ }
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/FileCopier.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/FileCopier.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/FileCopier.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -27,13 +27,14 @@
import org.globus.cog.abstraction.interfaces.Status;
import org.globus.cog.abstraction.interfaces.StatusListener;
import org.globus.cog.abstraction.interfaces.TaskHandler;
-import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.EventBus;
+import org.globus.cog.karajan.workflow.events.EventListener;
+import org.globus.cog.karajan.workflow.events.EventTargetPair;
import org.globus.cog.karajan.workflow.futures.Future;
import org.globus.cog.karajan.workflow.futures.FutureEvaluationException;
-import org.globus.cog.karajan.workflow.futures.FutureListener;
import org.globus.cog.karajan.workflow.futures.FuturesMonitor;
-import org.globus.cog.karajan.workflow.futures.ListenerStackPair;
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.PhysicalFormat;
@@ -41,7 +42,7 @@
private static final TaskHandler fth = new FileTransferTaskHandler();
private FileTransferTask task;
- private List<ListenerStackPair> actions;
+ private List actions;
private Exception exception;
private boolean closed;
@@ -64,12 +65,12 @@
task.addStatusListener(this);
}
- public synchronized void addModificationAction(FutureListener target,
- VariableStack stack) {
+ public synchronized void addModificationAction(EventListener target,
+ Event event) {
if (actions == null) {
- actions = new LinkedList<ListenerStackPair>();
+ actions = new LinkedList();
}
- ListenerStackPair etp = new ListenerStackPair(target, stack);
+ EventTargetPair etp = new EventTargetPair(event, target);
if (FuturesMonitor.debug) {
FuturesMonitor.monitor.add(etp, this);
}
@@ -81,21 +82,21 @@
}
}
- public List<ListenerStackPair> getModificationActions() {
+ public List getModificationActions() {
return actions;
}
private void actions() {
if (actions != null) {
synchronized (actions) {
- java.util.Iterator<ListenerStackPair> i = actions.iterator();
+ java.util.Iterator i = actions.iterator();
while (i.hasNext()) {
- ListenerStackPair etp = i.next();
+ EventTargetPair etp = (EventTargetPair) i.next();
if (FuturesMonitor.debug) {
FuturesMonitor.monitor.remove(etp);
}
i.remove();
- etp.listener.futureModified(this, etp.stack);
+ EventBus.post(etp.getTarget(), etp.getEvent());
}
}
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -9,14 +9,14 @@
import java.util.Arrays;
import java.util.List;
+import org.globus.cog.karajan.util.ThreadingContext;
+import org.globus.cog.karajan.workflow.nodes.*;
import org.globus.cog.karajan.stack.VariableStack;
-import org.globus.cog.karajan.util.ThreadingContext;
import org.globus.cog.karajan.util.TypeUtil;
import org.globus.cog.karajan.workflow.Condition;
import org.globus.cog.karajan.workflow.ExecutionException;
-import org.globus.cog.karajan.workflow.nodes.FlowElement;
-import org.globus.cog.karajan.workflow.nodes.Sequential;
-import org.globus.cog.karajan.workflow.nodes.While;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.LoopNotificationEvent;
public class InfiniteCountingWhile extends Sequential {
public static final String VAR = "##infinitecountingwhile:var";
@@ -59,10 +59,10 @@
if (index >= elementCount()) {
// starting new iteration
setIndex(stack, 1);
- fn = getElement(0);
+ fn = (FlowElement) getElement(0);
String counterName = (String) stack.getVar(VAR);
- List l = (List) stack.getVar(counterName);
+ List l = (List)stack.getVar(counterName);
Integer wrappedi = (Integer)l.get(0);
int i = wrappedi.intValue();
i++;
@@ -71,22 +71,30 @@
stack.setVar(counterName, Arrays.asList(new Integer[] {new Integer(i)}));
}
else {
- fn = getElement(index++);
+ fn = (FlowElement) getElement(index++);
setIndex(stack, index);
}
startElement(fn, stack);
}
-
- public void failed(VariableStack stack, ExecutionException e)
- throws ExecutionException {
- if (e instanceof While.Break) {
- complete(stack);
- return;
- }
- if (e instanceof While.Continue) {
- setIndex(e.getStack(), 0);
- startNext(e.getStack());
- return;
- }
- }
+
+ public void event(Event e) throws ExecutionException {
+ if (e instanceof LoopNotificationEvent) {
+ loopNotificationEvent((LoopNotificationEvent) e);
+ }
+ else {
+ super.event(e);
+ }
+ }
+
+ protected void loopNotificationEvent(LoopNotificationEvent e) throws ExecutionException {
+ if (e.getType() == LoopNotificationEvent.BREAK) {
+ complete(e.getStack());
+ return;
+ }
+ else if (e.getType() == LoopNotificationEvent.CONTINUE) {
+ setIndex(e.getStack(), 0);
+ startNext(e.getStack());
+ return;
+ }
+ }
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Log.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Log.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Log.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -69,7 +69,7 @@
else {
cls = "unknown";
}
- Level lvl = getLevel((String) LEVEL.getValue(stack));
+ Level lvl = getLevel(TypeUtil.toString(LEVEL.getValue(stack)));
Logger logger = getLogger(cls);
if (logger.isEnabledFor(lvl)) {
Object smsg = MESSAGE.getValue(stack);
@@ -78,7 +78,7 @@
}
else {
Object[] msg = Arg.VARGS.asArray(stack);
- StringBuilder sb = new StringBuilder();
+ StringBuffer sb = new StringBuffer();
for (int i = 0; i < msg.length; i++) {
sb.append(TypeUtil.toString(msg[i]));
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -1,8 +1,10 @@
package org.griphyn.vdl.karajan.lib;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import org.globus.cog.karajan.arguments.Arg;
@@ -11,7 +13,6 @@
import org.globus.cog.karajan.util.TypeUtil;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection;
-import org.globus.cog.util.CopyOnWriteArrayList;
import org.griphyn.vdl.util.VDL2Config;
/** this is an icky class that does too much with globals, but is for
@@ -88,9 +89,10 @@
public Object vdl_initprogressstate(VariableStack stack) throws ExecutionException {
RuntimeProgress rp = new RuntimeProgress();
ProgressTicker p = getTicker(stack);
- p.states.add(rp);
+ synchronized (p.states) {
+ p.states.add(rp);
+ }
setProgress(stack, rp);
- rp.status = "Initializing";
p.dumpState();
return null;
}
@@ -105,10 +107,8 @@
static public class ProgressTicker extends Thread {
- CopyOnWriteArrayList<RuntimeProgress> states =
- new CopyOnWriteArrayList<RuntimeProgress>();
+ List states = new ArrayList();
- long start;
long lastDumpTime = 0;
private boolean disabled;
private boolean shutdown;
@@ -124,7 +124,6 @@
catch (IOException e) {
logger.debug("Could not read swift properties", e);
}
- start = System.currentTimeMillis();
}
public void run() {
@@ -163,51 +162,48 @@
printStates("Final status:");
}
- public Map<String, Integer> getSummary() {
- Map<String, Integer> summary = new HashMap<String, Integer>();
- Iterator<RuntimeProgress> stateIterator = states.iterator();
- try {
+ public Map getSummary() {
+ Map summary = new HashMap();
+ synchronized(states) {
+ Iterator stateIterator = states.iterator();
+
// summarize details of known states into summary, with
// one entry per state type, storing the number of
// jobs in that state.
while(stateIterator.hasNext()) {
- String key = stateIterator.next().status;
- Integer count = summary.get(key);
- if (count == null) {
- summary.put(key, 1);
- }
- else {
- summary.put(key, count + 1);
+ String key = ((RuntimeProgress)stateIterator.next()).status;
+ Integer count = (Integer) summary.get(key);
+ if(count == null) {
+ summary.put(key,new Integer(1));
+ } else {
+ summary.put(key,new Integer(count.intValue()+1));
}
}
}
- finally {
- states.release();
- }
return summary;
}
void printStates(String header) {
- Map<String, Integer> summary = getSummary();
+ Map summary = getSummary();
// output the results of summarization, in a relatively
// pretty form - first the preferred order listed elements,
// and then anything remaining
System.err.print(header);
- System.err.print(" time:" + (System.currentTimeMillis() - start));
- for (int pos = 0; pos < preferredOutputOrder.length; pos++) {
+ for(int pos = 0; pos < preferredOutputOrder.length; pos++) {
String key = preferredOutputOrder[pos];
Object value = summary.get(key);
- if(value != null) {
- System.err.print(" " + key + ":" + value);
- }
+ if(value != null)
+ System.err.print(" "+key+":"+value);
summary.remove(key);
}
- for (Map.Entry<String, Integer> s : summary.entrySet()) {
- System.err.print(" " + s.getKey() + ":" + s.getValue());
+ Iterator summaryIterator = summary.keySet().iterator();
+ while(summaryIterator.hasNext()) {
+ Object key = summaryIterator.next();
+ System.err.print(" "+key+":"+summary.get(key));
}
- System.err.println();
+ System.err.println("");
}
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -13,7 +13,7 @@
public class SequentialWithID extends Sequential {
protected void executeChildren(VariableStack stack) throws ExecutionException {
- ThreadingContext tc = (ThreadingContext) stack.getVar("#thread");
+ ThreadingContext tc = (ThreadingContext)stack.getVar("#thread");
stack.setVar("#thread", tc.split(1));
super.executeChildren(stack);
}
Deleted: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Stagein.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Stagein.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Stagein.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -1,105 +0,0 @@
-/*
- * Created on Jan 5, 2007
- */
-package org.griphyn.vdl.karajan.lib;
-
-import java.util.Collection;
-
-import org.apache.log4j.Logger;
-import org.globus.cog.karajan.arguments.Arg;
-import org.globus.cog.karajan.arguments.ArgUtil;
-import org.globus.cog.karajan.arguments.NamedArguments;
-import org.globus.cog.karajan.arguments.Arg.Channel;
-import org.globus.cog.karajan.stack.VariableStack;
-import org.globus.cog.karajan.workflow.ExecutionException;
-import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
-import org.griphyn.vdl.mapping.AbstractDataNode;
-import org.griphyn.vdl.mapping.DSHandle;
-import org.griphyn.vdl.mapping.DependentException;
-import org.griphyn.vdl.mapping.HandleOpenException;
-import org.griphyn.vdl.mapping.InvalidPathException;
-import org.griphyn.vdl.mapping.MappingDependentException;
-import org.griphyn.vdl.mapping.Path;
-
-public class Stagein extends VDLFunction {
- public static final Logger logger = Logger.getLogger(Stagein.class);
-
- public static final Arg VAR = new Arg.Positional("var");
-
- public static final Channel STAGEIN = new Channel("stagein");
-
- static {
- setArguments(Stagein.class, new Arg[] { VAR });
- }
-
- private boolean isPrimitive(DSHandle var) {
- return (var instanceof AbstractDataNode && ((AbstractDataNode) var)
- .isPrimitive());
- }
-
- private void waitFor(DSHandle var, VariableStack stack)
- throws ExecutionException {
- synchronized (var) {
- if (!var.isClosed()) {
- if (logger.isDebugEnabled()) {
- logger.debug("Waiting for " + var);
- }
- throw new FutureNotYetAvailable(addFutureListener(stack, var));
- }
- else {
- Object v = var.getValue();
- if (logger.isDebugEnabled()) {
- logger.debug("Do not need to wait for " +
- var + " as it is closed and has value " + v +
- (v != null ? " with class " + v.getClass() : ""));
- }
- if (v != null && v instanceof RuntimeException) {
- throw (RuntimeException) v;
- }
- }
- }
- }
-
- protected Object function(VariableStack stack) throws ExecutionException {
- DSHandle var = (DSHandle) VAR.getValue(stack);
- if (!isPrimitive(var)) {
- boolean deperr = false;
- boolean mdeperr = false;
- try {
- Collection<Path> fp = var.getFringePaths();
- try {
- for (Path p : fp) {
- waitFor(var.getField(p), stack);
- }
- }
- catch (DependentException e) {
- deperr = true;
- }
- for (Path p : fp) {
- STAGEIN.ret(stack, filename(stack, var.getField(p))[0]);
- }
- }
- catch (MappingDependentException e) {
- logger.debug(e);
- deperr = true;
- mdeperr = true;
- }
- catch (HandleOpenException e) {
- throw new FutureNotYetAvailable(addFutureListener(stack, e.getSource()));
- }
- catch (InvalidPathException e) {
- throw new ExecutionException(e);
- }
- if (deperr || mdeperr) {
- NamedArguments named = ArgUtil.getNamedReturn(stack);
- named.add("deperror", deperr);
- named.add("mdeperror", mdeperr);
- }
- }
- else {
- // we still wait until the primitive value is there
- waitFor(var, stack);
- }
- return null;
- }
-}
Deleted: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Stageout.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Stageout.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/Stageout.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -1,90 +0,0 @@
-/*
- * Created on Jan 5, 2007
- */
-package org.griphyn.vdl.karajan.lib;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.log4j.Logger;
-import org.globus.cog.karajan.arguments.Arg;
-import org.globus.cog.karajan.arguments.ArgUtil;
-import org.globus.cog.karajan.arguments.NamedArguments;
-import org.globus.cog.karajan.arguments.Arg.Channel;
-import org.globus.cog.karajan.stack.VariableNotFoundException;
-import org.globus.cog.karajan.stack.VariableStack;
-import org.globus.cog.karajan.workflow.ExecutionException;
-import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
-import org.griphyn.vdl.mapping.AbstractDataNode;
-import org.griphyn.vdl.mapping.DSHandle;
-import org.griphyn.vdl.mapping.HandleOpenException;
-import org.griphyn.vdl.mapping.InvalidPathException;
-import org.griphyn.vdl.mapping.MappingDependentException;
-import org.griphyn.vdl.mapping.Path;
-
-public class Stageout extends VDLFunction {
- public static final Logger logger = Logger.getLogger(Stageout.class);
-
- public static final Arg VAR = new Arg.Positional("var");
-
- public static final Channel STAGEOUT = new Channel("stageout");
- public static final Channel RESTARTOUT = new Channel("restartout");
-
- static {
- setArguments(Stageout.class, new Arg[] { VAR });
- }
-
- private boolean isPrimitive(DSHandle var) {
- return (var instanceof AbstractDataNode && ((AbstractDataNode) var)
- .isPrimitive());
- }
-
- private List list(Path p, DSHandle var) {
- ArrayList l = new ArrayList(2);
- l.add(p);
- l.add(var);
- return l;
- }
-
- protected Object function(VariableStack stack) throws ExecutionException {
- DSHandle var = (DSHandle) VAR.getValue(stack);
- boolean deperr = false;
- boolean mdeperr = false;
- try {
- if (!isPrimitive(var)) {
- retPaths(STAGEOUT, stack, var);
- }
- else if (var.isRestartable()) {
- retPaths(RESTARTOUT, stack, var);
- }
- }
- catch (MappingDependentException e) {
- logger.debug(e);
- deperr = true;
- mdeperr = true;
- }
- if (deperr || mdeperr) {
- NamedArguments named = ArgUtil.getNamedReturn(stack);
- named.add("deperror", deperr);
- named.add("mdeperror", mdeperr);
- }
- return null;
- }
-
- private void retPaths(Channel channel, VariableStack stack, DSHandle var) throws ExecutionException {
- try {
- Collection<Path> fp = var.getFringePaths();
- for (Path p : fp) {
- channel.ret(stack, list(p, var.getField(p)));
- }
- }
- catch (HandleOpenException e) {
- throw new FutureNotYetAvailable(addFutureListener(stack, e
- .getSource()));
- }
- catch (InvalidPathException e) {
- throw new ExecutionException(e);
- }
- }
-}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -23,12 +23,15 @@
import org.globus.cog.karajan.util.ThreadingContext;
import org.globus.cog.karajan.util.TypeUtil;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.EventBus;
+import org.globus.cog.karajan.workflow.events.EventListener;
+import org.globus.cog.karajan.workflow.events.EventTargetPair;
+import org.globus.cog.karajan.workflow.events.FutureNotificationEvent;
import org.globus.cog.karajan.workflow.futures.FutureEvaluationException;
import org.globus.cog.karajan.workflow.futures.FutureFault;
import org.globus.cog.karajan.workflow.futures.FutureIterator;
import org.globus.cog.karajan.workflow.futures.FutureIteratorIncomplete;
-import org.globus.cog.karajan.workflow.futures.FutureListener;
-import org.globus.cog.karajan.workflow.futures.ListenerStackPair;
import org.globus.cog.karajan.workflow.nodes.AbstractParallelIterator;
import org.griphyn.vdl.util.VDL2Config;
@@ -65,6 +68,7 @@
stack.setVar(VAR, var);
setChildFailed(stack, false);
stack.setCaller(this);
+ initializeChannelBuffers(stack);
initThreadCount(stack, TypeUtil.toBoolean(stack.currentFrame().getVar("selfclose")), i);
stack.currentFrame().deleteVar("selfclose");
citerate(stack, var, i);
@@ -77,28 +81,7 @@
protected void citerate(VariableStack stack, Identifier var,
KarajanIterator i) throws ExecutionException {
ThreadCount tc = getThreadCount(stack);
-
- // we can bulk operations at the start to avoid contention
- // on the counter since at least as many
- // threads as reported by available() are available
- int available = tc.available();
try {
- int j = 0;
- try {
- for (; j < available && i.hasNext(); j++) {
- VariableStack copy = stack.copy();
- copy.enter();
- ThreadingContext.set(copy, ThreadingContext.get(copy).split(
- i.current()));
- setIndex(copy, getArgCount());
- setArgsDone(copy);
- copy.setVar(var.getName(), i.next());
- startElement(getArgCount(), copy);
- }
- }
- finally {
- tc.add(j);
- }
while (i.hasNext()) {
Object value = tc.tryIncrement();
VariableStack copy = stack.copy();
@@ -108,6 +91,7 @@
setIndex(copy, getArgCount());
setArgsDone(copy);
copy.setVar(var.getName(), value);
+ addChannelBuffers(copy);
startElement(getArgCount(), copy);
}
@@ -122,24 +106,17 @@
}
}
catch (FutureIteratorIncomplete fii) {
- synchronized (stack.currentFrame()) {
- stack.setVar(ITERATOR, i);
- }
- fii.getFutureIterator().addModificationAction(this, stack);
+ stack.setVar(ITERATOR, i);
+ fii.getFutureIterator().addModificationAction(
+ this,
+ new FutureNotificationEvent(ITERATE, this, fii
+ .getFutureIterator(), stack));
}
}
-
- public void failed(VariableStack stack, ExecutionException e) throws ExecutionException {
- if (!testAndSetChildFailed(stack)) {
- if (stack.parentFrame().isDefined(VAR)) {
- stack.leave();
- }
- failImmediately(stack, e);
- }
- }
protected void iterationCompleted(VariableStack stack)
throws ExecutionException {
+ closeBuffers(stack);
stack.leave();
ThreadCount tc = getThreadCount(stack);
int running;
@@ -179,7 +156,7 @@
private int maxThreadCount;
private int crt;
private boolean selfClose, closed;
- private List<ListenerStackPair> listeners;
+ private List listeners;
private KarajanIterator i;
public ThreadCount(int maxThreadCount, boolean selfClose, KarajanIterator i) {
@@ -201,14 +178,6 @@
public boolean isSelfClose() {
return selfClose;
}
-
- public synchronized int available() {
- return maxThreadCount - crt;
- }
-
- public synchronized void add(int count) {
- crt += count;
- }
public synchronized Object tryIncrement() {
// there is no way that both crt == 0 and i has no values outside this critical section
@@ -230,12 +199,11 @@
private void notifyListeners() {
if (listeners != null) {
- Iterator<ListenerStackPair> i = listeners.iterator();
- listeners = null;
+ Iterator i = listeners.iterator();
while (i.hasNext()) {
- ListenerStackPair etp = i.next();
+ EventTargetPair etp = (EventTargetPair) i.next();
i.remove();
- etp.listener.futureModified(this, etp.stack);
+ EventBus.post(etp.getTarget(), etp.getEvent());
}
}
}
@@ -267,12 +235,12 @@
public void remove() {
}
- public synchronized void addModificationAction(FutureListener target,
- VariableStack stack) {
+ public synchronized void addModificationAction(EventListener target,
+ Event event) {
if (listeners == null) {
- listeners = new ArrayList<ListenerStackPair>();
+ listeners = new ArrayList();
}
- listeners.add(new ListenerStackPair(target, stack));
+ listeners.add(new EventTargetPair(event, target));
if (crt < maxThreadCount) {
notifyListeners();
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -90,9 +90,9 @@
}
else {
Object[] array = (Object[]) value;
- for (int i = 0; i < array.length; i++) {
- vret.append(array[i]);
- }
+ for (int i = 0; i < array.length; i++) {
+ vret.append(array[i]);
+ }
}
}
else {
@@ -376,7 +376,7 @@
public static final String VDL_FUTURE_WRAPPER_MAP = "#vdl:futureWrapperMap";
- public static WrapperMap getFutureWrapperMap(VariableStack stack) throws ExecutionException {
+ protected static WrapperMap getFutureWrapperMap(VariableStack stack) throws ExecutionException {
synchronized (stack.getExecutionContext()) {
WrapperMap hash = (WrapperMap) stack.firstFrame().getVar(VDL_FUTURE_WRAPPER_MAP);
if (hash == null) {
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/cache/File.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/cache/File.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/cache/File.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -7,12 +7,13 @@
import java.util.LinkedList;
import java.util.List;
-import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.events.Event;
+import org.globus.cog.karajan.workflow.events.EventBus;
+import org.globus.cog.karajan.workflow.events.EventListener;
+import org.globus.cog.karajan.workflow.events.EventTargetPair;
import org.globus.cog.karajan.workflow.futures.Future;
import org.globus.cog.karajan.workflow.futures.FutureEvaluationException;
-import org.globus.cog.karajan.workflow.futures.FutureListener;
-import org.globus.cog.karajan.workflow.futures.ListenerStackPair;
public class File implements Future {
private String path;
@@ -20,7 +21,7 @@
private long size, lastAccess;
private int locked;
private boolean processingLock;
- private List<ListenerStackPair> listeners;
+ private List listeners;
public File(String file, String dir, Object host, long size) {
if (dir.endsWith("/")) {
@@ -150,20 +151,20 @@
public void notifyListeners() {
if (listeners != null) {
- Iterator<ListenerStackPair> i = listeners.iterator();
+ Iterator i = listeners.iterator();
while (i.hasNext()) {
- ListenerStackPair etp = i.next();
+ EventTargetPair etp = (EventTargetPair) i.next();
i.remove();
- etp.listener.futureModified(this, etp.stack);
+ EventBus.post(etp.getTarget(), etp.getEvent());
}
}
}
- public synchronized void addModificationAction(FutureListener target, VariableStack stack) {
+ public synchronized void addModificationAction(EventListener target, Event event) {
if (listeners == null) {
- listeners = new LinkedList<ListenerStackPair>();
+ listeners = new LinkedList();
}
- listeners.add(new ListenerStackPair(target, stack));
+ listeners.add(new EventTargetPair(event, target));
if (isClosed()) {
notifyListeners();
}
Modified: branches/release-0.92/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -20,8 +20,8 @@
String s = argList(filename(stack), true);
DSHandle result = RootDataNode.newNode(Types.STRING, s);
int provid = VDLFunction.nextProvenanceID();
- //VDLFunction.logProvenanceParameter(provid, (DSHandle) PA_VAR.getValue(stack), "input");
- //VDLFunction.logProvenanceResult(provid, result, "filename");
+ VDLFunction.logProvenanceParameter(provid, (DSHandle) PA_VAR.getValue(stack), "input");
+ VDLFunction.logProvenanceResult(provid, result, "filename");
return result;
} catch(VDL2FutureException ve) {
synchronized(ve.getHandle().getRoot()) {
Modified: branches/release-0.92/src/org/griphyn/vdl/mapping/AbstractDataNode.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -535,7 +535,7 @@
}
}
- public Path getPathFromRoot() {
+ public synchronized Path getPathFromRoot() {
if (pathFromRoot == null) {
AbstractDataNode parent = (AbstractDataNode) this.getParent();
Path myPath;
Modified: branches/release-0.92/src/org/griphyn/vdl/mapping/DSHandle.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/mapping/DSHandle.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/mapping/DSHandle.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -65,7 +65,7 @@
public void closeDeep();
- public Collection<Path> getFringePaths() throws HandleOpenException;
+ public Collection getFringePaths() throws HandleOpenException;
public Map getArrayValue();
Modified: branches/release-0.92/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -9,8 +9,8 @@
import java.util.Map;
import org.apache.log4j.Logger;
+
import org.griphyn.vdl.mapping.AbsFile;
-import org.griphyn.vdl.mapping.AbstractDataNode;
import org.griphyn.vdl.mapping.AbstractMapper;
import org.griphyn.vdl.mapping.InvalidMappingParameterException;
import org.griphyn.vdl.mapping.MappingParam;
@@ -200,16 +200,10 @@
if(logger.isDebugEnabled()) {
logger.debug("Finish list existing paths for mapper "+this.hashCode()+" list="+l);
}
- System.out.println(getVarName() + " (input): found " + l.size() + " files");
return l;
}
- private String getVarName() {
- AbstractDataNode var = (AbstractDataNode) getParam("handle");
- return var == null ? "" : var.getDisplayableName();
- }
-
- /** Returns the SwiftScript path for a supplied filename.
+ /** Returns the SwiftScript path for a supplied filename.
*
* Splits the filename into components using the separator
* supplied by the relevant FileNameElementMapper.
Modified: branches/release-0.92/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java
===================================================================
--- branches/release-0.92/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2011-04-02 19:01:11 UTC (rev 4242)
@@ -60,6 +60,6 @@
}
public boolean isStatic() {
- return true;
+ return false;
}
}
Modified: branches/release-0.92/tests/groups/group-all-local.sh
===================================================================
--- branches/release-0.92/tests/groups/group-all-local.sh 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/tests/groups/group-all-local.sh 2011-04-02 19:01:11 UTC (rev 4242)
@@ -11,5 +11,3 @@
$TESTDIR/cdm/ps/pinned
# $TESTDIR/site/intrepid
)
-
-checkvars WORK
Deleted: branches/release-0.92/tests/groups/group-mpi.sh
===================================================================
--- branches/release-0.92/tests/groups/group-mpi.sh 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/tests/groups/group-mpi.sh 2011-04-02 19:01:11 UTC (rev 4242)
@@ -1,6 +0,0 @@
-
-# GROUPLIST definition to run local MPI test
-
-GROUPLIST=( $TESTDIR/mpi )
-
-checkvars WORK
Deleted: branches/release-0.92/tests/groups/group-pc.sh
===================================================================
--- branches/release-0.92/tests/groups/group-pc.sh 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/tests/groups/group-pc.sh 2011-04-02 19:01:11 UTC (rev 4242)
@@ -1,6 +0,0 @@
-
-# GROUPLIST definition to run persistent coasters tests
-
-GROUPLIST=( $TESTDIR/persistent-coasters )
-
-checkvars WORK
Modified: branches/release-0.92/tests/meta.sh
===================================================================
--- branches/release-0.92/tests/meta.sh 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/tests/meta.sh 2011-04-02 19:01:11 UTC (rev 4242)
@@ -14,5 +14,3 @@
ssh $SITE_LOGIN $DIR/run-nightly.sh $DIR/$TEST
scp -r $SITE_LOGIN:$RUNDIR .
-
-exit 0
Property changes on: branches/release-0.92/tests/mpi
___________________________________________________________________
Name: svn:ignore
- mpi-cp
Modified: branches/release-0.92/tests/nightly.sh
===================================================================
--- branches/release-0.92/tests/nightly.sh 2011-04-02 13:31:23 UTC (rev 4241)
+++ branches/release-0.92/tests/nightly.sh 2011-04-02 19:01:11 UTC (rev 4242)
@@ -79,19 +79,6 @@
# via make_sites_sed() -> group_sites_xml()
# Note that some schedulers restrict your choice of RUNDIR
-# NAMING
-# Site-specific test groups are in providers/ .
-# These are named:
-# providers/<provider description>/
-# or:
-# providers/<provider description>/<site>
-# E.g., providers/local-pbs/PADS
-
-# WARNINGS
-# nightly.sh uses shopt
-
-shopt -s nullglob
-
printhelp() {
echo "nightly.sh <options> <output>"
echo ""
@@ -170,7 +157,7 @@
HTML_COMMENTS=1
fi
-# Iterations per test (may want to run each test multiple times?)
+# Iterations
ITERS_LOCAL=1
LOGCOUNT=0
@@ -192,7 +179,7 @@
SCRIPTDIR=$( cd $( dirname $0 ) ; /bin/pwd )
-TESTCOUNT=0
+SWIFTCOUNT=0
echo "RUNNING_IN: $RUNDIR"
echo "HTML_OUTPUT: $HTML"
@@ -477,7 +464,7 @@
start_row() {
html_tr testline
html_td align right width 50
- html "<b>$TESTCOUNT</b>"
+ html "<b>$SWIFTCOUNT</b>"
html " "
html_~td
html_td align right
@@ -658,7 +645,7 @@
TIMEOUT=$2 # seconds
OUTPUT=$3
- V=$TESTCOUNT
+ V=$SWIFTCOUNT
# Use background so kill/trap is immediate
sleep $TIMEOUT > /dev/null 2>&1 &
@@ -763,7 +750,7 @@
CDM=
[ -r fs.data ] && CDM="-cdm.file fs.data"
- (( TESTCOUNT++ ))
+ (( SWIFTCOUNT++ ))
TIMEOUT=$( gettimeout $GROUP/$TIMEOUTFILE )
@@ -786,40 +773,6 @@
fi
}
-# Execute shell test case w/ setup, check, clean
-script_test_case() {
- SHELLSCRIPT=$1
- SETUPSCRIPT=${SHELLSCRIPT%.test.sh}.setup.sh
- CHECKSCRIPT=${SHELLSCRIPT%.test.sh}.check.sh
- CLEANSCRIPT=${SHELLSCRIPT%.test.sh}.clean.sh
- TIMEOUTFILE=${SHELLSCRIPT%.test.sh}.timeout
-
- TEST_SHOULD_FAIL=0
- if [ -x $GROUP/$SETUPSCRIPT ]; then
- script_exec $GROUP/$SETUPSCRIPT "S"
- fi
-
- (( TESTCOUNT++ ))
-
- # Not using background for script tests yet
- # TIMEOUT=$( gettimeout $GROUP/$TIMEOUTFILE )
-
- if [ -x $GROUP/$SETUPSCRIPT ]; then
- script_exec $GROUP/$SETUPSCRIPT "S"
- fi
-
- if [ -x $GROUP/$SHELLSCRIPT ]; then
- script_exec $SHELLSCRIPT "X"
- fi
-
- if [ -x $GROUP/$CHECKSCRIPT ]; then
- script_exec $GROUP/$CHECKSCRIPT "√"
- fi
- if [ -x $GROUP/$CLEANSCRIPT ]; then
- script_exec $GROUP/$CLEANSCRIPT "C"
- fi
-}
-
# All timeouts in this script are in seconds
gettimeout() {
FILE=$1
@@ -873,12 +826,12 @@
}
# Setup coasters variables
-if which ifconfig > /dev/null 2>&1; then
+if which ifconfig > /dev/null; then
IFCONFIG=ifconfig
else
IFCONFIG=/sbin/ifconfig
fi
-$IFCONFIG > /dev/null 2>&1 || crash "Cannot run ifconfig!"
+$IFCONFIG > /dev/null || crash "Cannot run ifconfig!"
GLOBUS_HOSTNAME=$( $IFCONFIG | grep inet | head -1 | cut -d ':' -f 2 | \
awk '{print $1}' )
[ $? != 0 ] && crash "Could not obtain GLOBUS_HOSTNAME!"
@@ -948,49 +901,26 @@
group_fs_data
group_swift_properties
- SWIFTS=$( echo $GROUP/*.swift )
- checkfail "Could not list: $GROUP"
+ SWIFTS=$( ls $GROUP/*.swift )
+ checkfail "Could not ls: $GROUP"
for TEST in $SWIFTS; do
(( SKIP_COUNTER++ < SKIP_TESTS )) && continue
- TESTNAME=$( basename $TEST )
+ TESTNAME=$( basename $TEST)
cp -v $GROUP/$TESTNAME .
TESTLINK=$TESTNAME
start_row
- for (( i=0; $i<$ITERS_LOCAL; i=$i+1 )); do
+ for ((i=0; $i<$ITERS_LOCAL; i=$i+1)); do
swift_test_case $TESTNAME
- (( $TESTCOUNT >= $NUMBER_OF_TESTS )) && return
+ (( $SWIFTCOUNT >= $NUMBER_OF_TESTS )) && return
done
end_row
done
-
- SCRIPTS=$( echo $GROUP/*.test.sh )
- checkfail "Could not list: $GROUP"
- for TEST in $SCRIPTS; do
-
- (( SKIP_COUNTER++ < SKIP_TESTS )) && continue
-
- TESTNAME=$( basename $TEST )
- cp -v $GROUP/$TESTNAME .
- TESTLINK=$TESTNAME
-
- start_row
- for ((i=0; $i<$ITERS_LOCAL; i=$i+1)); do
- script_test_case $TESTNAME
- (( $TESTCOUNT >= $NUMBER_OF_TESTS )) && return
- done
- end_row
- done
}
-if [[ $WORK == "" ]]
-then
- WORK=$TOPDIR/work
-fi
-
checkvars GROUPLISTFILE
echo "GROUPLISTFILE: $GROUPLISTFILE"
source $GROUPLISTFILE || exit 1
@@ -1063,7 +993,7 @@
start_part "Part $GROUPCOUNT: $TITLE"
test_group
(( GROUPCOUNT++ ))
- (( $TESTCOUNT >= $NUMBER_OF_TESTS )) && break
+ (( $SWIFTCOUNT >= $NUMBER_OF_TESTS )) && break
done
if [ $GRID_TESTS == "0" ]; then
More information about the Swift-commit
mailing list