[Swift-commit] r2095 - in trunk: src/org/griphyn/vdl/karajan/lib src/org/griphyn/vdl/mapping src/org/griphyn/vdl/type tests/language-behaviour tests/misc
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Tue Jul 8 01:55:32 CDT 2008
Author: benc
Date: 2008-07-08 01:55:32 -0500 (Tue, 08 Jul 2008)
New Revision: 2095
Added:
trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java
trunk/tests/language-behaviour/087-external-dataset.swift
trunk/tests/misc/ordering-extern.sh
trunk/tests/misc/restart-extern.sh
trunk/tests/misc/restart-extern.swift
trunk/tests/misc/restart-external.in
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/New.java
trunk/src/org/griphyn/vdl/type/Types.java
trunk/tests/misc/run
Log:
externally stored datasets
Modified: trunk/src/org/griphyn/vdl/karajan/lib/New.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/New.java 2008-07-07 19:37:13 UTC (rev 2094)
+++ trunk/src/org/griphyn/vdl/karajan/lib/New.java 2008-07-08 06:55:32 UTC (rev 2095)
@@ -18,6 +18,7 @@
import org.griphyn.vdl.mapping.Path;
import org.griphyn.vdl.mapping.RootArrayDataNode;
import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.mapping.ExternalDataNode;
import org.griphyn.vdl.mapping.file.ConcurrentMapper;
import org.griphyn.vdl.type.Type;
import org.griphyn.vdl.type.Types;
@@ -79,7 +80,9 @@
type = Types.getType(typename);
}
DSHandle handle;
- if (type.isArray()) {
+ if(typename.equals("external")) {
+ handle = new ExternalDataNode();
+ } else if (type.isArray()) {
// dealing with array variable
handle = new RootArrayDataNode(type);
if (value != null) {
Added: trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java (rev 0)
+++ trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java 2008-07-08 06:55:32 UTC (rev 2095)
@@ -0,0 +1,299 @@
+package org.griphyn.vdl.mapping;
+
+import org.griphyn.vdl.karajan.Loader;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.griphyn.vdl.type.Field;
+import org.griphyn.vdl.type.NoSuchTypeException;
+import org.griphyn.vdl.type.Type;
+import org.griphyn.vdl.type.Types;
+
+public class ExternalDataNode implements DSHandle {
+
+ private Map params;
+
+ public void init(Map params) {
+ this.params = params;
+ }
+
+ static final String DATASET_URI_PREFIX = "tag:benc at ci.uchicago.edu,2008:swift:dataset:external:";
+
+ public static final Logger logger = Logger.getLogger(ExternalDataNode.class);
+
+ public static final MappingParam PARAM_PREFIX = new MappingParam("prefix", null);
+
+ /** Datasets are identified within a run by this sequence number and the
+ partial ID field.
+ The initial value is chosen to aid human recognition of sequence
+ numbers in the wild. There is no requirement that it start at this
+ (or any other) particular value. Note that this introduces a
+ maximum on the number of datasets which can be dealt with in any
+ run to be about 2^62. */
+ private static long datasetIDCounter = 720000000000l;
+
+ /** This is used to provide a (hopefully) globally unique identifier for
+ each time the datasetIDCounter is reset (whenever this class is
+ loaded, which will usually happen once per JVM). No meaning should be
+ inferred from this value - it exists purely for making unique URIs. */
+ private static final String datasetIDPartialID = Loader.getUUID();
+
+ private Map handles;
+ private Object value;
+ private boolean closed;
+ private List listeners;
+ final String identifierURI = makeIdentifierURIString();
+
+ public ExternalDataNode() {
+ }
+
+ public Type getType() {
+ try {
+ return Types.getType("external");
+ } catch(NoSuchTypeException te) {
+ throw new RuntimeException(te);
+ }
+ }
+
+ public boolean isPrimitive() {
+ return false;
+ }
+
+ public boolean isRestartable() {
+ return true;
+ }
+
+ /**
+ * create a String representation of this node. If the node has a value,
+ * then uses the String representation of that value. Otherwise, generates a
+ * text description.
+ */
+ public String toString() {
+ if (this.value != null && !(this.value instanceof Exception)) {
+ // special handling for ints...
+ if (this.getType().equals(Types.INT)) {
+ try {
+ Number n = (Number) this.getValue();
+ return String.valueOf(n.intValue());
+ }
+ catch (ClassCastException e) {
+ throw new RuntimeException("Internal type error. Value is not a Number for "
+ + getDisplayableName() + getPathFromRoot());
+ }
+ }
+ else {
+ return this.value.toString();
+ }
+ }
+
+ String prefix = this.getClass().getName();
+
+ prefix = prefix + " identifier "+this.getIdentifier();
+
+ prefix = prefix + " with no value at dataset=";
+
+ prefix = prefix + getDisplayableName();
+
+ if (!Path.EMPTY_PATH.equals(getPathFromRoot())) {
+ prefix = prefix + " path="+ getPathFromRoot().toString();
+ }
+
+ if(closed) {
+ prefix = prefix + " (closed)";
+ }
+ else {
+ prefix = prefix + " (not closed)";
+ }
+
+ return prefix;
+ }
+
+ public DSHandle getRoot() {
+ return this;
+ }
+
+ protected String getDisplayableName() {
+ String prefix = getRoot().getParam("dbgname");
+ if (prefix == null) {
+ prefix = getRoot().getParam("prefix");
+ }
+ if (prefix == null) {
+ prefix = "unnamed SwiftScript value";
+ }
+ return prefix;
+ }
+
+ public DSHandle getField(Path path) throws InvalidPathException {
+ if (path.isEmpty()) {
+ return this;
+ } else {
+ throw new InvalidPathException(path, this);
+ }
+ }
+
+ public Collection getFields(Path path) throws InvalidPathException, HandleOpenException {
+ List fields = new ArrayList();
+ return fields;
+ }
+
+ public void set(DSHandle handle) {
+ throw new IllegalArgumentException(this.getDisplayableName() + " is an external dataset and cannot be set");
+ }
+
+ protected void setField(String name, DSHandle handle) {
+ synchronized (handles) {
+ handles.put(name, handle);
+ }
+ }
+
+ protected DSHandle getHandle(String name) {
+ synchronized (handles) {
+ return (DSHandle) handles.get(name);
+ }
+ }
+
+ protected boolean isHandlesEmpty() {
+ synchronized (handles) {
+ return handles.isEmpty();
+ }
+ }
+
+ protected void checkDataException() {
+ if (value instanceof DependentException) {
+ throw (DependentException) value;
+ }
+ }
+
+ protected void checkMappingException() {
+ if (value instanceof MappingDependentException) {
+ throw (MappingDependentException) value;
+ }
+ }
+
+ public Object getValue() {
+logger.warn("getValue called in an external dataset");
+return value;
+// throw new RuntimeException("cannot get value of external dataset");
+ }
+
+ public Map getArrayValue() {
+throw new RuntimeException("cannot get value of external dataset");
+ }
+
+ public boolean isArray() {
+ return false;
+ }
+
+ public void setValue(Object value) {
+ if (this.closed) {
+ throw new IllegalArgumentException(this.getDisplayableName()
+ + " is closed with a value of "+this.value);
+ }
+ if (this.value != null) {
+ throw new IllegalArgumentException(this.getDisplayableName()
+ + " is already assigned with a value of " + this.value);
+ }
+ this.value = value;
+ }
+
+ public Collection getFringePaths() throws HandleOpenException {
+ ArrayList list = new ArrayList();
+ list.add(Path.EMPTY_PATH);
+ return list;
+ }
+
+ public synchronized void closeShallow() {
+ this.closed = true;
+ notifyListeners();
+ logger.info("closed "+this.getIdentifier());
+ }
+
+ public boolean isClosed() {
+ return closed;
+ }
+
+ public void closeDeep() {
+ if (!this.closed) {
+ closeShallow();
+ }
+ synchronized (handles) {
+ Iterator i = handles.entrySet().iterator();
+ while (i.hasNext()) {
+ Map.Entry e = (Map.Entry) i.next();
+ AbstractDataNode mapper = (AbstractDataNode) e.getValue();
+ mapper.closeDeep();
+ }
+ }
+ }
+
+ public Path getPathFromRoot() {
+ return Path.EMPTY_PATH;
+ }
+
+ public Mapper getMapper() {
+ return null;
+ }
+
+ protected Map getHandles() {
+ return handles;
+ }
+
+ public synchronized void addListener(DSHandleListener listener) {
+ if (logger.isInfoEnabled()) {
+ logger.info("Adding handle listener \"" + listener + "\" to \"" + this + "\"");
+ }
+ if (listeners == null) {
+ listeners = new LinkedList();
+ }
+ listeners.add(listener);
+ if (closed) {
+ notifyListeners();
+ }
+ }
+
+ protected synchronized void notifyListeners() {
+ if (listeners != null) {
+ Iterator i = listeners.iterator();
+ while (i.hasNext()) {
+ DSHandleListener listener = (DSHandleListener) i.next();
+ i.remove();
+ if (logger.isInfoEnabled()) {
+ logger.info("Notifying listener \"" + listener + "\" about \"" + this + "\"");
+ }
+ listener.handleClosed(this);
+ }
+ listeners = null;
+ }
+ }
+
+ public String getIdentifier() {
+ return identifierURI;
+ }
+
+ String makeIdentifierURIString() {
+ datasetIDCounter++;
+ return DATASET_URI_PREFIX + datasetIDPartialID + ":" + datasetIDCounter;
+ }
+
+ public String getParam(String name) {
+ if (params == null) {
+ return null;
+ }
+ return (String) params.get(name);
+ }
+
+ public DSHandle createDSHandle(String fieldName) {
+throw new RuntimeException("cannot create new field in external dataset");
+ }
+
+ public DSHandle getParent() {
+ return null;
+ }
+}
Modified: trunk/src/org/griphyn/vdl/type/Types.java
===================================================================
--- trunk/src/org/griphyn/vdl/type/Types.java 2008-07-07 19:37:13 UTC (rev 2094)
+++ trunk/src/org/griphyn/vdl/type/Types.java 2008-07-08 06:55:32 UTC (rev 2095)
@@ -55,7 +55,7 @@
return type;
}
- public static final Type INT, STRING, FLOAT, BOOLEAN, ANY;
+ public static final Type INT, STRING, FLOAT, BOOLEAN, ANY, EXTERNAL;
// add built-in primitive types
static {
@@ -64,6 +64,7 @@
FLOAT = addPrimitiveType("float");
BOOLEAN = addPrimitiveType("boolean");
ANY = addPrimitiveType("any");
+ EXTERNAL = addPrimitiveType("external");
}
public synchronized static void resolveTypes() throws NoSuchTypeException {
Added: trunk/tests/language-behaviour/087-external-dataset.swift
===================================================================
--- trunk/tests/language-behaviour/087-external-dataset.swift (rev 0)
+++ trunk/tests/language-behaviour/087-external-dataset.swift 2008-07-08 06:55:32 UTC (rev 2095)
@@ -0,0 +1,19 @@
+type file;
+
+(external o) a() {
+ app {
+ touch "foo";
+ }
+}
+
+b(external o) {
+ app {
+ touch "bar";
+ }
+}
+
+external sync;
+
+sync=a();
+b(sync);
+
Added: trunk/tests/misc/ordering-extern.sh
===================================================================
--- trunk/tests/misc/ordering-extern.sh (rev 0)
+++ trunk/tests/misc/ordering-extern.sh 2008-07-08 06:55:32 UTC (rev 2095)
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+export CF=swift.properties.restart-extern
+cat $(dirname $(which swift))/../etc/swift.properties | grep --invert-match -E '^lazy.errors=' > $CF
+echo lazy.errors=true >> $CF
+
+rm -f *.rlog restart-*.out restart-extern.kml restart-extern.xml restart-*.out
+
+rm -rf _concurrent
+
+echo "localhost helperA $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" > tmp.restartOK.tc.data
+echo "localhost helperB $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartOK.tc.data
+echo "localhost helperC $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartOK.tc.data
+
+swift -config $CF -tc.file tmp.restartOK.tc.data restart-extern.swift -dir=`pwd`
+
+PRECHECKEXIT=$?
+
+
+if [ "$PRECHECKEXIT" != 0 ]; then
+ echo Failed - attempt to run workflow without ordering configuration failed
+ exit 1
+fi
+
+rm -f *.rlog restart-*.out restart-extern.kml restart-extern.xml
+rm -rf _concurrent
+
+# make A fail and B succeed.
+# extern dependency ordering should mean that B does not run.
+
+echo "localhost helperA $(pwd)/restart5-helper-fail INSTALLED INTEL32::LINUX null" > tmp.restartB.tc.data
+echo "localhost helperB $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartB.tc.data
+echo "localhost helperC $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartB.tc.data
+
+swift -config $CF -tc.file tmp.restartB.tc.data restart-extern.swift -dir=`pwd`
+
+SECONDEXIT=$?
+
+if [ "$SECONDEXIT" = "0" ]; then
+ echo Failed - broken apps succeeded
+ exit 2
+fi
+
+if [ -f restart-extern.2.out ] || [ -f restart-extern.1.out ]; then
+ echo Failed - output files came into existence that indicate external dependency was ignored
+ exit 3
+fi
+
+echo restart-exterm success
+exit 0
+
Added: trunk/tests/misc/restart-extern.sh
===================================================================
--- trunk/tests/misc/restart-extern.sh (rev 0)
+++ trunk/tests/misc/restart-extern.sh 2008-07-08 06:55:32 UTC (rev 2095)
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+export CF=swift.properties.restart5
+cat $(dirname $(which swift))/../etc/swift.properties | grep --invert-match -E '^lazy.errors=' > $CF
+echo lazy.errors=true >> $CF
+
+rm -f *.rlog restart-*.out restart5.kml restart5.xml restart5.*.out restart5.*.out.single-run
+
+rm -rf _concurrent
+
+echo "localhost helperA $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" > tmp.restartOK.tc.data
+echo "localhost helperB $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartOK.tc.data
+echo "localhost helperC $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartOK.tc.data
+
+swift -config $CF -tc.file tmp.restartOK.tc.data restart-extern.swift -dir=`pwd`
+
+PRECHECKEXIT=$?
+
+
+if [ "$PRECHECKEXIT" != 0 ]; then
+ echo Failed - attempt to run workflow without restart configuration failed
+ exit 1
+fi
+
+rm -f *.rlog restart-*.out restart-extern.kml restart-extern.xml
+rm -rf _concurrent
+
+echo "localhost helperA $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" > tmp.restartA.tc.data
+echo "localhost helperB $(pwd)/restart5-helper-fail INSTALLED INTEL32::LINUX null" >> tmp.restartA.tc.data
+echo "localhost helperC $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartA.tc.data
+
+swift -config $CF -tc.file tmp.restartA.tc.data restart-extern.swift -dir=`pwd`
+
+FIRSTEXIT=$?
+
+# this invocation should fail, with restart-1.out in existence but
+# not the others
+
+if [ "$FIRSTEXIT" == 0 ]; then
+ echo Failed - workflow was indicated as successfully completed the first time round.
+ exit 2
+fi
+
+# now make A fail - we should have run it already, and so we want to make
+# sure it does not run again; and make B succeed this time round.
+
+echo "localhost helperA $(pwd)/restart5-helper-fail INSTALLED INTEL32::LINUX null" > tmp.restartB.tc.data
+echo "localhost helperB $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartB.tc.data
+echo "localhost helperC $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartB.tc.data
+
+# there should be only a single rlog here, because we deleted them all
+# at the start of this script.
+swift -config $CF -resume *.rlog -tc.file tmp.restartB.tc.data restart-extern.swift -dir=`pwd`
+
+SECONDEXIT=$?
+
+if [ "$SECONDEXIT" != "0" ]; then
+ echo Failed - second round failed
+ exit 3
+fi
+
+echo restart-exterm success
+exit 0
+
Added: trunk/tests/misc/restart-extern.swift
===================================================================
--- trunk/tests/misc/restart-extern.swift (rev 0)
+++ trunk/tests/misc/restart-extern.swift 2008-07-08 06:55:32 UTC (rev 2095)
@@ -0,0 +1,18 @@
+type file;
+
+(external o) a() {
+ app {
+ helperA @strcat(@arg("dir"),"/restart-extern.1.out") "/etc/group" "qux";
+ }
+}
+
+b(external o) {
+ app {
+ helperB @strcat(@arg("dir"),"/restart-extern.2.out") "/etc/group" "baz";
+ }
+}
+
+external sync;
+
+sync=a();
+b(sync);
Added: trunk/tests/misc/restart-external.in
===================================================================
--- trunk/tests/misc/restart-external.in (rev 0)
+++ trunk/tests/misc/restart-external.in 2008-07-08 06:55:32 UTC (rev 2095)
@@ -0,0 +1 @@
+foo
Modified: trunk/tests/misc/run
===================================================================
--- trunk/tests/misc/run 2008-07-07 19:37:13 UTC (rev 2094)
+++ trunk/tests/misc/run 2008-07-08 06:55:32 UTC (rev 2095)
@@ -1,5 +1,7 @@
#!/bin/sh
-for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 restart-iterate workernode-local; do
+for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 restart-iterate workernode-local \
+ordering-extern-notlazy restart-extern ordering-extern \
+; do
./${a}.sh
R=$?
echo test $a ended with return value $R
More information about the Swift-commit
mailing list