[Swift-commit] r6077 - in trunk/src/org/griphyn/vdl: engine karajan karajan/lib mapping
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Fri Nov 23 02:56:37 CST 2012
Author: hategan
Date: 2012-11-23 02:56:28 -0600 (Fri, 23 Nov 2012)
New Revision: 6077
Added:
trunk/src/org/griphyn/vdl/karajan/lib/SetWaitCount.java
Modified:
trunk/src/org/griphyn/vdl/engine/CompilerUtils.java
trunk/src/org/griphyn/vdl/engine/Karajan.java
trunk/src/org/griphyn/vdl/engine/VariableScope.java
trunk/src/org/griphyn/vdl/karajan/Loader.java
trunk/src/org/griphyn/vdl/karajan/lib/New.java
trunk/src/org/griphyn/vdl/karajan/lib/PartialCloseDataset.java
trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java
trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
trunk/src/org/griphyn/vdl/mapping/DSHandle.java
trunk/src/org/griphyn/vdl/mapping/MappingParam.java
trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java
trunk/src/org/griphyn/vdl/mapping/RootDataNode.java
Log:
revamped variable closing stuff using reference counting and improved static analysis
Modified: trunk/src/org/griphyn/vdl/engine/CompilerUtils.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/CompilerUtils.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/engine/CompilerUtils.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -14,8 +14,8 @@
public class CompilerUtils {
public static String getLine(Node n) {
- if (n == null) {
- return "line unknown";
+ if (n == null || n.getAttributes() == null) {
+ return "unknown";
}
Node src = n.getAttributes().getNamedItem("src");
if (src == null) {
@@ -38,4 +38,8 @@
}
return getLine(src.getDomNode());
}
+
+ public static String info(XmlObject src) {
+ return src.getDomNode().getLocalName() + ", line " + getLine(src);
+ }
}
Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/Karajan.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -17,6 +17,8 @@
package org.griphyn.vdl.engine;
+import static org.griphyn.vdl.engine.CompilerUtils.getLine;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -58,14 +60,15 @@
import org.globus.swift.language.TypesDocument.Types.Type;
import org.globus.swift.language.Variable.Mapping;
import org.globus.swift.language.Variable.Mapping.Param;
+import org.griphyn.vdl.engine.VariableScope.EnclosureType;
+import org.griphyn.vdl.engine.VariableScope.WriteType;
import org.griphyn.vdl.karajan.CompilationException;
import org.griphyn.vdl.karajan.Loader;
import org.griphyn.vdl.toolkit.VDLt2VDLx;
+import org.griphyn.vdl.type.NoSuchTypeException;
import org.safehaus.uuid.UUIDGenerator;
import org.w3c.dom.Node;
-import static org.griphyn.vdl.engine.CompilerUtils.*;
-
public class Karajan {
public static final Logger logger = Logger.getLogger(Karajan.class);
@@ -79,7 +82,6 @@
Map<String,Type> typesMap = new HashMap<String,Type>();
List<StringTemplate> variables = new ArrayList<StringTemplate>();
- Set<String> usedVariables = new HashSet<String>();
public static final String TEMPLATE_FILE_NAME = "Karajan.stg";
public static final String TEMPLATE_FILE_NAME_NO_PROVENANCE = "Karajan-no-provenance.stg";
@@ -92,7 +94,6 @@
/** an arbitrary statement identifier. Start at some high number to
aid visual distinction in logs, but the actual value doesn't
matter. */
- int callID = 88000;
StringTemplateGroup m_templates;
@@ -126,8 +127,9 @@
ProgramDocument programDoc;
try {
programDoc = parseProgramXML(in);
- } catch(Exception e) {
- throw new CompilationException("Unable to parse intermediate XML",e);
+ }
+ catch(Exception e) {
+ throw new CompilationException("Unable to parse intermediate XML", e);
}
Program prog = programDoc.getProgram();
@@ -293,7 +295,7 @@
public StringTemplate program(Program prog) throws CompilationException {
- VariableScope scope = new VariableScope(this, null);
+ VariableScope scope = new VariableScope(this, null, prog);
scope.bodyTemplate = template("program");
scope.bodyTemplate.setAttribute("buildversion",Loader.buildVersion);
@@ -310,37 +312,17 @@
for (Program program : importList)
statements(program, scope);
- checkUninitializedVariables();
generateInternedConstants(scope.bodyTemplate);
+ scope.analyzeWriters();
scope.bodyTemplate.setAttribute("cleanups", scope.getCleanups());
return scope.bodyTemplate;
}
-
- private void checkUninitializedVariables() throws CompilationException {
- for (StringTemplate var : variables) {
- String name = (String) var.getAttribute("name");
- if (var.getAttribute("waitfor") == null) {
- if (usedVariables.contains(name)) {
- if (org.griphyn.vdl.type.Types.isPrimitive((String) var.getAttribute("type"))) {
- throw new CompilationException("Uninitalized variable: " + name);
- }
- }
- else {
- logger.info("Unused variable " + name);
- }
- }
- }
- }
- private void setVariableUsed(String s) {
- usedVariables.add(s);
- }
-
public void procedure(Procedure proc, VariableScope containingScope) throws CompilationException {
- VariableScope outerScope = new VariableScope(this, containingScope, VariableScope.ENCLOSURE_PROCEDURE);
- VariableScope innerScope = new VariableScope(this, outerScope, VariableScope.ENCLOSURE_NONE);
+ VariableScope outerScope = new VariableScope(this, containingScope, EnclosureType.PROCEDURE, proc);
+ VariableScope innerScope = new VariableScope(this, outerScope, EnclosureType.NONE, proc);
StringTemplate procST = template("procedure");
containingScope.bodyTemplate.setAttribute("procedures", procST);
procST.setAttribute("line", getLine(proc));
@@ -349,25 +331,14 @@
FormalParameter param = proc.getOutputArray(i);
StringTemplate paramST = parameter(param, innerScope);
procST.setAttribute("outputs", paramST);
- if (!param.isNil())
- procST.setAttribute("optargs", paramST);
- else
- procST.setAttribute("arguments", paramST);
- String type = normalize(param.getType().getLocalPart());
- checkIsTypeDefined(type);
- innerScope.addVariable(param.getName(), type, "Return parameter", param);
+ addArg(procST, param, paramST, true, innerScope);
}
for (int i = 0; i < proc.sizeOfInputArray(); i++) {
FormalParameter param = proc.getInputArray(i);
StringTemplate paramST = parameter(param, outerScope);
procST.setAttribute("inputs", paramST);
- if (!param.isNil())
- procST.setAttribute("optargs", paramST);
- else
- procST.setAttribute("arguments", paramST);
- String type = normalize(param.getType().getLocalPart());
- checkIsTypeDefined(type);
- outerScope.addVariable(param.getName(), type, "Parameter", param);
+ addArg(procST, param, paramST, false, outerScope);
+ outerScope.addWriter(param.getName(), WriteType.FULL, proc, procST);
}
Binding bind;
@@ -375,7 +346,7 @@
binding(bind, procST, innerScope);
}
else {
- VariableScope compoundScope = new VariableScope(this, innerScope);
+ VariableScope compoundScope = new VariableScope(this, innerScope, proc);
compoundScope.bodyTemplate = procST;
statementsForSymbols(proc, compoundScope);
statements(proc, compoundScope);
@@ -383,6 +354,26 @@
}
}
+ private void addArg(StringTemplate procST, FormalParameter param,
+ StringTemplate paramST, boolean returnArg, VariableScope scope) throws CompilationException {
+ if (!param.isNil()) {
+ procST.setAttribute("optargs", paramST);
+ }
+ else {
+ procST.setAttribute("arguments", paramST);
+ }
+ String type = normalize(param.getType().getLocalPart());
+ checkIsTypeDefined(type);
+ scope.addVariable(param.getName(), type, returnArg ? "Return value" : "Parameter", false, param);
+
+ if (returnArg) {
+ StringTemplate initWaitCountST = template("setWaitCount");
+ initWaitCountST.setAttribute("name", param.getName());
+ procST.setAttribute("initWaitCounts", initWaitCountST);
+ }
+ }
+
+
/**
* Convert to a case-insensitive representation by
* pre-pending a '_' to upper case letters. If the
@@ -435,8 +426,9 @@
typeST.setAttribute("name", normalize(param.getType().getLocalPart()));
typeST.setAttribute("namespace", param.getType().getNamespaceURI());
paramST.setAttribute("type", typeST);
- if(!param.isNil())
+ if(!param.isNil()) {
paramST.setAttribute("default",expressionToKarajan(param.getAbstractExpression(), scope));
+ }
return paramST;
}
@@ -452,9 +444,16 @@
variableST.setAttribute("isGlobal", Boolean.valueOf(var.getIsGlobal()));
variableST.setAttribute("line", getLine(var));
variables.add(variableST);
+
+
+ /*
+ * possibly an input; mark it as such here, and if
+ * writers are detected, remove the input attribute (that is
+ * done in VariableScope).
+ */
+ variableST.setAttribute("input", "true");
- if(!var.isNil()) {
-
+ if (!var.isNil()) {
if (var.getFile() != null) {
StringTemplate fileST = new StringTemplate("file");
fileST.setAttribute("name", escapeQuotes(var.getFile().getName()));
@@ -462,8 +461,6 @@
variableST.setAttribute("file", fileST);
}
-
-
Mapping mapping = var.getMapping();
if (mapping != null) {
@@ -471,56 +468,14 @@
mappingST.setAttribute("descriptor", mapping.getDescriptor());
for (int i = 0; i < mapping.sizeOfParamArray(); i++) {
Param param = mapping.getParamArray(i);
- StringTemplate paramST = template("vdl_parameter");
- paramST.setAttribute("name", param.getName());
- Node expressionDOM = param.getAbstractExpression().getDomNode();
- String namespaceURI = expressionDOM.getNamespaceURI();
- String localName = expressionDOM.getLocalName();
- QName expressionQName = new QName(namespaceURI, localName);
- if(expressionQName.equals(VARIABLE_REFERENCE_EXPR)) {
- paramST.setAttribute("expr",expressionToKarajan(param.getAbstractExpression(),scope));
- } else {
- String parameterVariableName="swift#mapper#"+(internedIDCounter++);
- // make template for variable declaration (need to compute type of this variable too?)
- StringTemplate variableDeclarationST = template("variable");
- variableDeclarationST.setAttribute("waitfor","");
- // TODO factorise this and other code in variable()?
- StringTemplate pmappingST = new StringTemplate("mapping");
- pmappingST.setAttribute("descriptor", "concurrent_mapper");
- StringTemplate pparamST = template("vdl_parameter");
- pparamST.setAttribute("name", "prefix");
- pparamST.setAttribute("expr", parameterVariableName + "-" + UUIDGenerator.getInstance().generateRandomBasedUUID().toString());
- pmappingST.setAttribute("params", pparamST);
- //variableDeclarationST.setAttribute("mapping", pmappingST);
- variableDeclarationST.setAttribute("nil", Boolean.TRUE);
- variableDeclarationST.setAttribute("name", parameterVariableName);
- scope.bodyTemplate.setAttribute("declarations",variableDeclarationST);
- StringTemplate paramValueST=expressionToKarajan(param.getAbstractExpression(),scope);
- String paramValueType = datatype(paramValueST);
- scope.addVariable(parameterVariableName, paramValueType, "Variable", param);
- variableDeclarationST.setAttribute("type", paramValueType);
- StringTemplate variableReferenceST = template("id");
- variableReferenceST.setAttribute("var",parameterVariableName);
- StringTemplate variableAssignmentST = template("assign");
- variableAssignmentST.setAttribute("var",variableReferenceST);
- variableAssignmentST.setAttribute("value",paramValueST);
- scope.appendStatement(variableAssignmentST);
- if (param.getAbstractExpression().getDomNode().getNodeName().equals("stringConstant")) {
- StringTemplate valueST = template("sConst");
- valueST.setAttribute("innervalue", param.getAbstractExpression().getDomNode().getFirstChild().getNodeValue());
- paramST.setAttribute("expr",valueST);
- }
- else {
- paramST.setAttribute("expr",variableReferenceST);
- }
- }
- mappingST.setAttribute("params", paramST);
+ mappingST.setAttribute("params", mappingParameter(param, scope));
}
variableST.setAttribute("mapping", mappingST);
}
- } else {
- // add temporary mapping info
- if (!org.griphyn.vdl.type.Types.isPrimitive(var.getType().getLocalPart())) {
+ }
+ else {
+ // add temporary mapping info in not primitive or array of primitive
+ if (!isPrimitiveOrArrayOfPrimitive(var.getType().getLocalPart())) {
StringTemplate mappingST = new StringTemplate("mapping");
mappingST.setAttribute("descriptor", "concurrent_mapper");
StringTemplate paramST = template("vdl_parameter");
@@ -531,21 +486,79 @@
variableST.setAttribute("mapping", mappingST);
variableST.setAttribute("nil", Boolean.TRUE);
}
- }
+ }
scope.bodyTemplate.setAttribute("declarations", variableST);
}
+ private StringTemplate mappingParameter(Param param, VariableScope scope) throws CompilationException {
+ StringTemplate paramST = template("vdl_parameter");
+ paramST.setAttribute("name", param.getName());
+ Node expressionDOM = param.getAbstractExpression().getDomNode();
+ String namespaceURI = expressionDOM.getNamespaceURI();
+ String localName = expressionDOM.getLocalName();
+ QName expressionQName = new QName(namespaceURI, localName);
+ if (expressionQName.equals(VARIABLE_REFERENCE_EXPR)) {
+ paramST.setAttribute("expr", expressionToKarajan(param.getAbstractExpression(), scope));
+ }
+ else {
+ String parameterVariableName="swift#mapper#"+(internedIDCounter++);
+ // make template for variable declaration (need to compute type of this variable too?)
+ StringTemplate variableDeclarationST = template("variable");
+ // TODO factorise this and other code in variable()?
+ StringTemplate pmappingST = new StringTemplate("mapping");
+ pmappingST.setAttribute("descriptor", "concurrent_mapper");
+ StringTemplate pparamST = template("vdl_parameter");
+ pparamST.setAttribute("name", "prefix");
+ pparamST.setAttribute("expr", parameterVariableName + "-" +
+ UUIDGenerator.getInstance().generateRandomBasedUUID().toString());
+ pmappingST.setAttribute("params", pparamST);
+ variableDeclarationST.setAttribute("nil", Boolean.TRUE);
+ variableDeclarationST.setAttribute("name", parameterVariableName);
+ scope.bodyTemplate.setAttribute("declarations", variableDeclarationST);
+ StringTemplate paramValueST=expressionToKarajan(param.getAbstractExpression(),scope);
+ String paramValueType = datatype(paramValueST);
+ scope.addVariable(parameterVariableName, paramValueType, "Variable", param);
+ variableDeclarationST.setAttribute("type", paramValueType);
+ StringTemplate variableReferenceST = template("id");
+ variableReferenceST.setAttribute("var",parameterVariableName);
+ StringTemplate variableAssignmentST = template("assign");
+ variableAssignmentST.setAttribute("var",variableReferenceST);
+ variableAssignmentST.setAttribute("value",paramValueST);
+ scope.appendStatement(variableAssignmentST);
+ if (param.getAbstractExpression().getDomNode().getNodeName().equals("stringConstant")) {
+ StringTemplate valueST = template("sConst");
+ valueST.setAttribute("innervalue", param.getAbstractExpression().getDomNode().getFirstChild().getNodeValue());
+ paramST.setAttribute("expr",valueST);
+ }
+ else {
+ paramST.setAttribute("expr",variableReferenceST);
+ }
+ }
+ return paramST;
+ }
+
void checkIsTypeDefined(String type) throws CompilationException {
if (!org.griphyn.vdl.type.Types.isValidType(type, typesMap.keySet())) {
throw new CompilationException("Type " + type + " is not defined.");
}
}
+
+ private boolean isPrimitiveOrArrayOfPrimitive(String type) {
+ org.griphyn.vdl.type.Type t;
+ try {
+ t = org.griphyn.vdl.type.Types.getType(type);
+ return t.isPrimitive() || (t.isArray() && t.itemType().isPrimitive());
+ }
+ catch (NoSuchTypeException e) {
+ return false;
+ }
+ }
public void assign(Assign assign, VariableScope scope) throws CompilationException {
try {
StringTemplate assignST = template("assign");
- StringTemplate varST = expressionToKarajan(assign.getAbstractExpressionArray(0),scope);
+ StringTemplate varST = expressionToKarajan(assign.getAbstractExpressionArray(0), scope, true);
StringTemplate valueST = expressionToKarajan(assign.getAbstractExpressionArray(1),scope);
if (! (datatype(varST).equals(datatype(valueST)) ||
datatype(valueST).equals("java")))
@@ -555,7 +568,7 @@
assignST.setAttribute("value", valueST);
assignST.setAttribute("line", getLine(assign));
String rootvar = abstractExpressionToRootVariable(assign.getAbstractExpressionArray(0));
- scope.addWriter(rootvar, new Integer(callID++), rootVariableIsPartial(assign.getAbstractExpressionArray(0)));
+ scope.addWriter(rootvar, getRootVariableWriteType(assign.getAbstractExpressionArray(0)), assign, assignST);
scope.appendStatement(assignST);
} catch(CompilationException re) {
throw new CompilationException("Compile error in assignment at "+assign.getSrc()+": "+re.getMessage(),re);
@@ -580,7 +593,7 @@
appendST.setAttribute("value", value);
String rootvar = abstractExpressionToRootVariable(append.getAbstractExpressionArray(0));
// an append is always a partial write
- scope.addWriter(rootvar, new Integer(callID++), true);
+ scope.addWriter(rootvar, WriteType.PARTIAL, append, appendST);
scope.appendStatement(appendST);
} catch(CompilationException re) {
throw new CompilationException("Compile error in assignment at "+append.getSrc()+": "+re.getMessage(),re);
@@ -797,7 +810,7 @@
ActualParameter output = call.getOutputArray(i);
StringTemplate argST = actualParameter(output, scope);
callST.setAttribute("outputs", argST);
- addWriterToScope(scope, call.getOutputArray(i).getAbstractExpression());
+ addWriterToScope(scope, call.getOutputArray(i).getAbstractExpression(), call, callST);
}
}
if (keywordArgsOutput) {
@@ -816,12 +829,12 @@
throw new CompilationException("Wrong type for output parameter number " + i +
", expected " + formalType + ", got " + actualType);
- addWriterToScope(scope, call.getOutputArray(i).getAbstractExpression());
+ addWriterToScope(scope, call.getOutputArray(i).getAbstractExpression(), call, callST);
}
} else { /* Positional arguments */
for (int i = 0; i < call.sizeOfOutputArray(); i++) {
ActualParameter output = call.getOutputArray(i);
- StringTemplate argST = actualParameter(output, scope);
+ StringTemplate argST = actualParameter(output, scope, true);
callST.setAttribute("outputs", argST);
FormalArgumentSignature formalArg =proceduresMap.get(procName).getOutputArray(i);
@@ -833,7 +846,7 @@
throw new CompilationException("Wrong type for parameter number " + i +
", expected " + formalType + ", got " + actualType);
- addWriterToScope(scope, call.getOutputArray(i).getAbstractExpression());
+ addWriterToScope(scope, call.getOutputArray(i).getAbstractExpression(), call, callST);
}
}
@@ -844,8 +857,9 @@
callST.setAttribute("serialize", Boolean.TRUE);
}
return callST;
- } catch(CompilationException ce) {
- throw new CompilationException("Compile error in procedure invocation at "+call.getSrc()+": "+ce.getMessage(),ce);
+ }
+ catch (CompilationException ce) {
+ throw new CompilationException("Compile error in procedure invocation at " + call.getSrc(), ce);
}
}
@@ -883,19 +897,19 @@
return true;
}
- private void addWriterToScope(VariableScope scope, XmlObject var) throws CompilationException {
+ private void addWriterToScope(VariableScope scope, XmlObject var, XmlObject src, StringTemplate out) throws CompilationException {
String rootvar = abstractExpressionToRootVariable(var);
- boolean partial = rootVariableIsPartial(var);
- if (!partial) {
+ WriteType writeType = getRootVariableWriteType(var);
+ if (writeType == WriteType.FULL) {
// don't close variables that are already closed by the function itself
scope.inhibitClosing(rootvar);
}
- scope.addWriter(rootvar, new Integer(callID++), partial);
+ scope.addWriter(rootvar, writeType, src, out);
}
public void iterateStat(Iterate iterate, VariableScope scope) throws CompilationException {
- VariableScope loopScope = new VariableScope(this, scope, VariableScope.ENCLOSURE_LOOP);
- VariableScope innerScope = new VariableScope(this, loopScope, VariableScope.ENCLOSURE_LOOP);
+ VariableScope loopScope = new VariableScope(this, scope, EnclosureType.LOOP, iterate);
+ VariableScope innerScope = new VariableScope(this, loopScope, EnclosureType.LOOP, iterate);
loopScope.addVariable(iterate.getVar(), "int", "Iteration variable", iterate);
@@ -904,6 +918,8 @@
iterateST.setAttribute("var", iterate.getVar());
innerScope.bodyTemplate = iterateST;
+
+ loopScope.addWriter(iterate.getVar(), WriteType.FULL, iterate, iterateST);
statementsForSymbols(iterate.getBody(), innerScope);
statements(iterate.getBody(), innerScope);
@@ -912,16 +928,13 @@
StringTemplate condST = expressionToKarajan(cond, innerScope);
iterateST.setAttribute("cond", condST);
- Object statementID = new Integer(callID++);
- for (String v : innerScope.getVariables())
- scope.addWriter(v, statementID, true);
scope.appendStatement(iterateST);
iterateST.setAttribute("cleanups", innerScope.getCleanups());
}
public void foreachStat(Foreach foreach, VariableScope scope) throws CompilationException {
try {
- VariableScope innerScope = new VariableScope(this, scope, VariableScope.ENCLOSURE_LOOP);
+ VariableScope innerScope = new VariableScope(this, scope, EnclosureType.LOOP, foreach);
StringTemplate foreachST = template("foreach");
foreachST.setAttribute("var", foreach.getVar());
@@ -930,6 +943,10 @@
XmlObject in = foreach.getIn().getAbstractExpression();
StringTemplate inST = expressionToKarajan(in, scope);
foreachST.setAttribute("in", inST);
+
+ if ("id".equals(inST.getName())) {
+ innerScope.setForeachSourceVar((String) inST.getAttribute("var"), foreach);
+ }
String inType = datatype(inST);
String itemType = org.griphyn.vdl.type.Types.getArrayInnerItemTypeName(inType);
@@ -938,30 +955,26 @@
throw new CompilationException("You can iterate through an array structure only");
}
innerScope.addVariable(foreach.getVar(), itemType, "Iteration variable", foreach);
+ innerScope.addWriter(foreach.getVar(), WriteType.FULL, foreach, foreachST);
foreachST.setAttribute("indexVar", foreach.getIndexVar());
foreachST.setAttribute("indexVarType", keyType);
- if(foreach.getIndexVar() != null) {
+ if (foreach.getIndexVar() != null) {
innerScope.addVariable(foreach.getIndexVar(), keyType, "Iteration variable", foreach);
+ innerScope.addWriter(foreach.getIndexVar(), WriteType.FULL, foreach, foreachST);
}
innerScope.bodyTemplate = foreachST;
statementsForSymbols(foreach.getBody(), innerScope);
statements(foreach.getBody(), innerScope);
-
- String inVar = (String) inST.getAttribute("var");
- Object statementID = new Integer(callID++);
- for (String v : innerScope.getVariables()) {
- scope.addWriter(v, statementID, true);
- if (v.equals(inVar) && !innerScope.isVariableLocallyDefined(v))
- foreachST.setAttribute("selfClose", "true");
- }
+
scope.appendStatement(foreachST);
Collection<String> cleanups = innerScope.getCleanups();
cleanups.remove(foreach.getVar());
foreachST.setAttribute("cleanups", cleanups);
- } catch(CompilationException re) {
- throw new CompilationException("Compile error in foreach statement at "+foreach.getSrc()+": "+re.getMessage(),re);
+ }
+ catch (CompilationException re) {
+ throw new CompilationException("Compile error in foreach statement at " + foreach.getSrc(), re);
}
}
@@ -977,7 +990,8 @@
Then thenstat = ifstat.getThen();
Else elsestat = ifstat.getElse();
- VariableScope innerThenScope = new VariableScope(this, scope, VariableScope.ENCLOSURE_CONDITION);
+ VariableScope innerThenScope = new VariableScope(this, scope,
+ EnclosureType.CONDITION, thenstat);
innerThenScope.bodyTemplate = template("sub_comp");
ifST.setAttribute("vthen", innerThenScope.bodyTemplate);
@@ -985,31 +999,32 @@
statements(thenstat, innerThenScope);
innerThenScope.bodyTemplate.setAttribute("cleanups", innerThenScope.getCleanups());
- Object statementID = new Integer(callID++);
-
- for (String v : innerThenScope.getVariables())
- scope.addWriter(v, statementID, true);
-
if (elsestat != null) {
-
- VariableScope innerElseScope = new VariableScope(this, scope);
+ VariableScope innerElseScope = new VariableScope(this, scope,
+ EnclosureType.CONDITION, elsestat);
innerElseScope.bodyTemplate = template("sub_comp");
+ innerElseScope.setThen(innerThenScope);
ifST.setAttribute("velse", innerElseScope.bodyTemplate);
statementsForSymbols(elsestat, innerElseScope);
statements(elsestat, innerElseScope);
-
- for (String v : innerElseScope.getVariables())
- scope.addWriter(v, statementID, true);
innerElseScope.bodyTemplate.setAttribute("cleanups", innerElseScope.getCleanups());
}
+ else if (innerThenScope.hasPartialClosing()) {
+ // must do matching partial closing somewhere, so need
+ // else branch even though not specified in the swift source
+ VariableScope innerElseScope = new VariableScope(this, scope,
+ EnclosureType.CONDITION, elsestat);
+ innerElseScope.bodyTemplate = template("sub_comp");
+ innerElseScope.setThen(innerThenScope);
+ ifST.setAttribute("velse", innerElseScope.bodyTemplate);
+ }
scope.appendStatement(ifST);
}
public void switchStat(Switch switchstat, VariableScope scope) throws CompilationException {
StringTemplate switchST = template("switch");
- Object statementID = new Integer(callID++);
scope.bodyTemplate.setAttribute("statements", switchST);
StringTemplate conditionST = expressionToKarajan(switchstat.getAbstractExpression(), scope);
switchST.setAttribute("condition", conditionST.toString());
@@ -1020,24 +1035,19 @@
for (int i=0; i< switchstat.sizeOfCaseArray(); i++) {
Case casestat = switchstat.getCaseArray(i);
- VariableScope caseScope = new VariableScope(this, scope);
+ VariableScope caseScope = new VariableScope(this, scope, casestat);
caseScope.bodyTemplate = new StringTemplate("case");
switchST.setAttribute("cases", caseScope.bodyTemplate);
caseStat(casestat, caseScope);
-
- for (String v : caseScope.getVariables())
- scope.addWriter(v, statementID, true);
}
Default defaultstat = switchstat.getDefault();
if (defaultstat != null) {
- VariableScope defaultScope = new VariableScope(this, scope);
+ VariableScope defaultScope = new VariableScope(this, scope, defaultstat);
defaultScope.bodyTemplate = template("sub_comp");
switchST.setAttribute("sdefault", defaultScope.bodyTemplate);
statementsForSymbols(defaultstat, defaultScope);
statements(defaultstat, defaultScope);
- for (String v : defaultScope.getVariables())
- scope.addWriter(v, statementID, true);
}
}
@@ -1047,10 +1057,14 @@
statementsForSymbols(casestat.getStatements(), scope);
statements(casestat.getStatements(), scope);
}
+
+ public StringTemplate actualParameter(ActualParameter arg, VariableScope scope) throws CompilationException {
+ return actualParameter(arg, scope, false);
+ }
- public StringTemplate actualParameter(ActualParameter arg, VariableScope scope) throws CompilationException {
+ public StringTemplate actualParameter(ActualParameter arg, VariableScope scope, boolean lvalue) throws CompilationException {
StringTemplate argST = template("call_arg");
- StringTemplate expST = expressionToKarajan(arg.getAbstractExpression(), scope);
+ StringTemplate expST = expressionToKarajan(arg.getAbstractExpression(), scope, lvalue);
argST.setAttribute("bind", arg.getBind());
argST.setAttribute("expr", expST);
argST.setAttribute("datatype", datatype(expST));
@@ -1188,11 +1202,15 @@
static final QName RANGE_EXPR = new QName(SWIFTSCRIPT_NS, "range");
static final QName FUNCTION_EXPR = new QName(SWIFTSCRIPT_NS, "function");
static final QName CALL_EXPR = new QName(SWIFTSCRIPT_NS, "call");
+
+ public StringTemplate expressionToKarajan(XmlObject expression, VariableScope scope) throws CompilationException {
+ return expressionToKarajan(expression, scope, false);
+ }
/** converts an XML intermediate form expression into a
* Karajan expression.
*/
- public StringTemplate expressionToKarajan(XmlObject expression, VariableScope scope) throws CompilationException
+ public StringTemplate expressionToKarajan(XmlObject expression, VariableScope scope, boolean lvalue) throws CompilationException
{
Node expressionDOM = expression.getDomNode();
String namespaceURI = expressionDOM.getNamespaceURI();
@@ -1322,10 +1340,12 @@
XmlString xmlString = (XmlString) expression;
String s = xmlString.getStringValue();
if(!scope.isVariableDefined(s)) {
- throw new CompilationException("Variable " + s + " is undefined.");
+ throw new CompilationException("Variable " + s + " was not declared in this scope.");
}
-
- setVariableUsed(s);
+
+ if (!lvalue) {
+ scope.addReader(s, false, expression);
+ }
StringTemplate st = template("id");
st.setAttribute("var", s);
String actualType;
@@ -1336,7 +1356,7 @@
} else if (expressionQName.equals(ARRAY_SUBSCRIPT_EXPR)) {
BinaryOperator op = (BinaryOperator) expression;
StringTemplate arrayST = expressionToKarajan(op.getAbstractExpressionArray(1), scope);
- StringTemplate parentST = expressionToKarajan(op.getAbstractExpressionArray(0), scope);
+ StringTemplate parentST = expressionToKarajan(op.getAbstractExpressionArray(0), scope, true);
String indexType = datatype(arrayST);
String declaredIndexType = org.griphyn.vdl.type.Types.getArrayOuterIndexTypeName(datatype(parentST));
@@ -1359,6 +1379,8 @@
+ indexType + ") does not match the declared index type (" + declaredIndexType + ")");
}
+ scope.addReader((String) parentST.getAttribute("var"), true, expression);
+
StringTemplate newst = template("extractarrayelement");
newst.setAttribute("arraychild", arrayST);
newst.setAttribute("parent", parentST);
@@ -1367,7 +1389,7 @@
return newst;
} else if (expressionQName.equals(STRUCTURE_MEMBER_EXPR)) {
StructureMember sm = (StructureMember) expression;
- StringTemplate parentST = expressionToKarajan(sm.getAbstractExpression(), scope);
+ StringTemplate parentST = expressionToKarajan(sm.getAbstractExpression(), scope, true);
String parentType = datatype(parentST);
@@ -1414,6 +1436,8 @@
else {
newst = template("extractstructelement");
}
+
+ scope.addReader((String) parentST.getAttribute("var"), true, expression);
newst.setAttribute("parent", parentST);
newst.setAttribute("memberchild", sm.getMemberName());
newst.setAttribute("datatype", actualType);
@@ -1492,7 +1516,7 @@
} else if (expressionQName.equals(CALL_EXPR)) {
Call c = (Call) expression;
c.addNewOutput();
- VariableScope subscope = new VariableScope(this, scope);
+ VariableScope subscope = new VariableScope(this, scope, c);
VariableReferenceDocument ref = VariableReferenceDocument.Factory.newInstance();
ref.setVariableReference("swift#callintermediate");
c.getOutputArray(0).set(ref);
@@ -1517,7 +1541,6 @@
call.setAttribute("datatype", type);
call.setAttribute("call", call(c, subscope, true));
- call.setAttribute("callID", new Integer(callID++));
call.setAttribute("prefix", UUIDGenerator.getInstance().generateRandomBasedUUID().toString());
return call;
} else {
@@ -1585,17 +1608,17 @@
}
}
- public boolean rootVariableIsPartial(XmlObject expression) {
+ public WriteType getRootVariableWriteType(XmlObject expression) {
Node expressionDOM = expression.getDomNode();
String namespaceURI = expressionDOM.getNamespaceURI();
String localName = expressionDOM.getLocalName();
QName expressionQName = new QName(namespaceURI, localName);
if (expressionQName.equals(VARIABLE_REFERENCE_EXPR)) {
- return false;
+ return WriteType.FULL;
} else if (expressionQName.equals(ARRAY_SUBSCRIPT_EXPR)) {
- return true;
+ return WriteType.PARTIAL;
} else if (expressionQName.equals(STRUCTURE_MEMBER_EXPR)) {
- return true;
+ return WriteType.PARTIAL;
} else {
throw new RuntimeException("Could not find root for abstract expression.");
}
Modified: trunk/src/org/griphyn/vdl/engine/VariableScope.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/VariableScope.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/engine/VariableScope.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -18,12 +18,16 @@
package org.griphyn.vdl.engine;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.Stack;
import org.antlr.stringtemplate.StringTemplate;
import org.apache.log4j.Logger;
@@ -32,25 +36,26 @@
public class VariableScope {
-
- /** permit array up-assignment, but not entire variables */
- public final static int ENCLOSURE_LOOP = 301923;
-
- /** permit all upwards assignments */
- public static final int ENCLOSURE_ALL = 301924;
-
- /** permit no upwards assignments */
- public static final int ENCLOSURE_NONE = 301925;
-
- /** permit no access to the containing scope except for finding
- global variables */
- public static final int ENCLOSURE_PROCEDURE = 301926;
+ public enum EnclosureType {
+ /** permit array up-assignment, but not entire variables */
+ LOOP,
+ /** permit all upwards assignments */
+ ALL,
+ /** permit no upwards assignments */
+ NONE,
+ /** permit no access to the containing scope except for finding
+ global variables */
+ PROCEDURE,
+ /** Override ENCLOSURE_LOOP to allow assignment inside a loop
+ * based on some condition
+ */
+ CONDITION;
+ }
+
+ public enum WriteType {
+ FULL, PARTIAL
+ }
- /** Override ENCLOSURE_LOOP to allow assignment inside a loop
- * based on some condition
- */
- public static final int ENCLOSURE_CONDITION = 301927;
-
public static final Logger logger = Logger.getLogger(VariableScope.class);
/** need this for the program as a whole. probably should factor
@@ -64,17 +69,114 @@
this will be the same as parentScope or this. */
VariableScope rootScope;
- int enclosureType;
+ private EnclosureType enclosureType;
/** The string template in which we will store statements
outputted into this scope. */
public StringTemplate bodyTemplate;
private List<String> outputs = new ArrayList<String>();
- private Set<String> inhibitClosing = new HashSet<String>();
+ private Set<String> inhibitClosing;
+ private List<VariableScope> children = new LinkedList<VariableScope>();
+ private XmlObject src;
+
+ /** List of templates to be executed in sequence after the present
+ in-preparation statement is outputted. */
+ List<StringTemplate> presentStatementPostStatements = new ArrayList<StringTemplate>();
+
+ /** Set of variables (String token names) that are declared in
+ this scope - not variables in parent scopes, though. */
+ Map<String, Variable> variables;
+
+ /**
+ * Usage (such as writing or reading) in this scope
+ */
+ Map<String, VariableUsage> variableUsage;
+
+ /**
+ * If this is the scope of an 'else', then store a reference
+ * to the corresponding 'then'
+ */
+ private VariableScope thenScope;
+
+ private static class Variable {
+ public final String name, type;
+ public final XmlObject src;
+ public final boolean isGlobal;
+ private XmlObject foreachSrc;
+
+ public Variable(String name, String type, boolean isGlobal, XmlObject src) {
+ this.name = name;
+ this.type = type;
+ this.src = src;
+ this.isGlobal = isGlobal;
+ }
+ }
+
+ /** Stores information about a variable that is referred to in this
+ scope. Should probably get used for dataset marking eventually. */
+ private static class VariableUsage {
+ private final String name;
+ private XmlObject fullWritingLoc, foreachSourceVar;
+ private List<XmlObject> partialWritingLoc;
+ private List<XmlObject> fullReadingLoc;
+ private int referenceCount;
+
+ public VariableUsage(String name) {
+ this.name = name;
+ }
+
+ public boolean addWritingStatement(XmlObject loc, WriteType writeType) throws CompilationException {
+ if (writeType == WriteType.PARTIAL) {
+ if (partialWritingLoc == null) {
+ partialWritingLoc = new LinkedList<XmlObject>();
+ }
+ if (!partialWritingLoc.contains(loc)) {
+ partialWritingLoc.add(loc);
+ return true;
+ }
+ else {
+ return false;
+ }
+ }
+ else {
+ if (fullWritingLoc != null) {
+ throw new CompilationException("Variable " + name + " is written to in two different locations:\n\t1. "
+ + CompilerUtils.info(fullWritingLoc) + "\n\t2. " + CompilerUtils.info(loc));
+ }
+ else {
+ fullWritingLoc = loc;
+ return true;
+ }
+ }
+ }
- public VariableScope(Karajan c, VariableScope parent) {
- this(c, parent, ENCLOSURE_ALL);
+ public boolean hasWriters() {
+ return fullWritingLoc != null || (partialWritingLoc != null && partialWritingLoc.size() > 0);
+ }
+
+ public void addFullReadingStatement(XmlObject where) {
+ if (fullReadingLoc == null) {
+ fullReadingLoc = new LinkedList<XmlObject>();
+ }
+ fullReadingLoc.add(where);
+ }
+
+ public boolean hasReaders() {
+ return fullReadingLoc != null && fullReadingLoc.size() > 0;
+ }
+
+ public XmlObject getForeachSourceVar() {
+ return foreachSourceVar;
+ }
+
+ public void setForeachSourceVar(XmlObject foreachSourceVar) {
+ this.foreachSourceVar = foreachSourceVar;
+ }
+ }
+
+ public VariableScope(Karajan c, VariableScope parent, XmlObject src) {
+ this(c, parent, EnclosureType.ALL, src);
}
/** Creates a new variable scope.
@@ -84,7 +186,7 @@
@param a specifies how assignments to variables made in enclosing
scopes will be handled.
*/
- public VariableScope(Karajan c, VariableScope parent, int a) {
+ public VariableScope(Karajan c, VariableScope parent, EnclosureType a, XmlObject src) {
if (logger.isDebugEnabled()) {
if (parentScope != null) {
logger.debug("New scope " + hashCode() + " with parent scope " + parentScope.hashCode());
@@ -96,24 +198,30 @@
compiler = c;
parentScope = parent;
enclosureType = a;
- if(parentScope == null) {
+ if (parentScope == null) {
rootScope = this;
- } else {
+ }
+ else {
rootScope = parentScope.getRootScope();
+ parentScope.addChild(this);
}
+ this.src = src;
}
- /** Set of variables (String token names) that are declared in
- this scope - not variables in parent scopes, though. */
- Map<String, XmlObject> variables = new HashMap<String, XmlObject>();
- Map<String, String> varTypes = new HashMap<String, String>();
+ public void addChild(VariableScope scope) {
+ children.add(scope);
+ }
+
+ public List<VariableScope> getChildren() {
+ return children;
+ }
+
+ public void setThen(VariableScope thenScope) {
+ this.thenScope = thenScope;
+ updateBranchReferenceCounts();
+ }
- /** Set of variables (String token names) which are global and
- declared in this scope (which must be a root scope). Variables
- in this set must also appear in the variables set. */
- Map<String, XmlObject> globals = new HashMap<String, XmlObject>();
-
/** Asserts that a named variable is declared in this scope.
Might also eventually contain more information about the
variable. Need to define behaviour here when the
@@ -124,7 +232,81 @@
addVariable(name, type, context, false, src);
}
+ private Collection<VariableUsage> getVariableUsageValues() {
+ if (variableUsage == null) {
+ return Collections.emptyList();
+ }
+ else {
+ return variableUsage.values();
+ }
+ }
+
+ private Map<String, VariableUsage> getVariableUsage() {
+ if (variableUsage == null) {
+ variableUsage = new HashMap<String, VariableUsage>();
+ }
+ return variableUsage;
+ }
+
+ /**
+ * Get variable usage if it exists, otherwise return null
+ */
+ protected VariableUsage getExistingUsage(String name) {
+ if (variableUsage == null) {
+ return null;
+ }
+ else {
+ return variableUsage.get(name);
+ }
+ }
+
+ private Collection<String> getUsedVariableNames() {
+ if (variableUsage == null) {
+ return Collections.emptyList();
+ }
+ else {
+ return variableUsage.keySet();
+ }
+ }
+
+ private Collection<String> getLocallyDeclaredVariableNames() {
+ if (variables == null) {
+ return Collections.emptyList();
+ }
+ else {
+ return variables.keySet();
+ }
+ }
+
+
+ /**
+ * Returns variable usage in this scope for the given name.
+ * If no previous usage exists, create it. This method
+ * guarantees a non-null return
+ */
+ protected VariableUsage getUsageForUpdate(String name) {
+ Map<String, VariableUsage> usage = getVariableUsage();
+ VariableUsage u = usage.get(name);
+ if (u == null) {
+ u = new VariableUsage(name);
+ usage.put(name, u);
+ }
+ return u;
+ }
+
+ public void setForeachSourceVar(String name, XmlObject src) {
+ getUsageForUpdate(name).setForeachSourceVar(src);
+ }
+
+ public boolean isForeachSourceVar(String name) {
+ VariableUsage u = getExistingUsage(name);
+ return u != null && u.getForeachSourceVar() != null;
+ }
+
public void inhibitClosing(String name) {
+ if (inhibitClosing == null) {
+ inhibitClosing = new HashSet<String>();
+ }
inhibitClosing.add(name);
}
@@ -135,7 +317,7 @@
if(isVariableDefined(name)) {
throw new CompilationException("Variable " + name + ", on line "
- + CompilerUtils.getLine(src) + ", was already defined on line " + getDefinitionLine(name));
+ + CompilerUtils.getLine(src) + ", was already defined on line " + getDeclarationLine(name));
}
// TODO does this if() ever fire? or is it always taken
@@ -143,22 +325,24 @@
// be replaced by is locally defined test.
if(parentScope != null && parentScope.isVariableDefined(name)) {
Warnings.warn(context + " " + name + ", on line " + CompilerUtils.getLine(src)
- + ", shadows variable of same name on line " + parentScope.getDefinitionLine(name));
+ + ", shadows variable of same name on line " + parentScope.getDeclarationLine(name));
}
- if(global && this != rootScope) {
+ if (global && this != rootScope) {
throw new CompilationException("Global " + name + " can only be declared in the root scope of a program.");
}
-
- variables.put(name, src);
- varTypes.put(name, type);
- if (global) {
- globals.put(name, src);
- }
+ getVariablesMap().put(name, new Variable(name, type, global, src));
}
- /**
+ private Map<String, Variable> getVariablesMap() {
+ if (variables == null) {
+ variables = new HashMap<String, Variable>();
+ }
+ return variables;
+ }
+
+ /**
* Does pretty much the same as addVariable() except it doesn't throw
* an exception if the variable is defined in a parent scope
*/
@@ -169,16 +353,18 @@
if(isVariableLocallyDefined(name)) {
throw new CompilationException("Variable " + name + ", on line "
- + CompilerUtils.getLine(src) + ", was already defined on line " + getDefinitionLine(name));
+ + CompilerUtils.getLine(src) + ", was already defined on line " + getDeclarationLine(name));
}
- variables.put(name, src);
- varTypes.put(name, type);
+ getVariablesMap().put(name, new Variable(name, type, false, src));
}
- public String getDefinitionLine(String name) {
- XmlObject src = rootScope.getGlobalSrc(name);
- if (src == null) {
+ public String getDeclarationLine(String name) {
+ XmlObject src;
+ if (rootScope.isGlobalDefined(name)) {
+ src = rootScope.getGlobalSrc(name);
+ }
+ else {
src = getSrcRecursive(name);
}
if (src != null) {
@@ -191,155 +377,609 @@
public boolean isVariableDefined(String name) {
if(rootScope.isGlobalDefined(name)) return true;
- return isVariableDefinedRecursive(name);
+ return isVariableVisible(name);
}
- private XmlObject getSrcRecursive(String name) {
- XmlObject src = variables.get(name);
- if (src == null) {
- if (enclosureType != ENCLOSURE_PROCEDURE && parentScope != null) {
- src = parentScope.getSrcRecursive(name);
+ /**
+ * Recursively find information about a variable visible in this scope.
+ * Also, look for globals.
+ */
+ public Variable lookup(String name) {
+ if (variables != null) {
+ Variable var = variables.get(name);
+ if (var != null) {
+ return var;
}
}
- return src;
+ if (enclosureType != EnclosureType.PROCEDURE && parentScope != null) {
+ return parentScope.lookup(name);
+ }
+ else {
+ // if the search fails, see if there is a global with this name
+ if (rootScope.variables != null) {
+ Variable v = rootScope.variables.get(name);
+ if (v != null && v.isGlobal) {
+ return v;
+ }
+ else {
+ return null;
+ }
+ }
+ else {
+ return null;
+ }
+ }
}
+
+ private XmlObject getSrcRecursive(String name) {
+ Variable var = lookup(name);
+ if (var == null) {
+ throw new IllegalArgumentException("Variable " + name + " is not visible in this scope");
+ }
+ return var.src;
+ }
- private boolean isVariableDefinedRecursive(String name) {
- if(isVariableLocallyDefined(name)) return true;
- if(enclosureType != ENCLOSURE_PROCEDURE && parentScope != null && parentScope.isVariableDefined(name)) return true;
- return false;
+ private boolean isVariableVisible(String name) {
+ if (isVariableLocallyDefined(name)) {
+ return true;
+ }
+ if (enclosureType != EnclosureType.PROCEDURE && parentScope != null) {
+ return parentScope.isVariableVisible(name);
+ }
+ else {
+ return false;
+ }
}
+
+ private StringTemplate getExistingDeclaration(String name) {
+ if (isVariableLocallyDefined(name)) {
+ return getLocalDeclaration(name);
+ }
+ else if (parentScope != null) {
+ return parentScope.getExistingDeclaration(name);
+ }
+ else {
+ return null;
+ }
+ }
public boolean isGlobalDefined(String name) {
- return globals.containsKey(name);
+ Variable var = rootScope.lookup(name);
+ return var != null && var.isGlobal;
}
public XmlObject getGlobalSrc(String name) {
- return globals.get(name);
+ Variable var = rootScope.lookup(name);
+ if (var != null && var.isGlobal) {
+ return var.src;
+ }
+ else {
+ throw new IllegalArgumentException("'" + name + "' is not a global variable");
+ }
}
- /** note that this will return variable types for variables in
- containing scopes even if they are not accessible. non-null
- return value from this method should not be treated as an
- indication that the variable is in any way accessible from
- this scope. The isVariableDefined and isGlobalDefined methods
- should be used to check that. */
public String getVariableType(String name) {
- if(isVariableLocallyDefined(name))
- return varTypes.get(name).toString();
- if(parentScope != null && parentScope.isVariableDefined(name))
- return parentScope.getVariableType(name);
- return null;
+ Variable var = lookup(name);
+ if (var != null) {
+ return var.type;
+ }
+ else {
+ throw new IllegalArgumentException("Variable " + name + " is not visible in this scope");
+ }
}
-// TODO could also mark variable as written and catch duplicate assignments
-// in the same scope here
- public boolean isVariableWriteable(String name, boolean partialWriter) {
- if(isVariableLocallyDefined(name)) return true;
- if(parentScope != null && parentScope.isVariableWriteable(name, true) && enclosureType == ENCLOSURE_CONDITION) {
- Warnings.warn("Variable " + name + ", defined on line " + getDefinitionLine(name) + ", might have multiple writers");
+ public boolean isVariableWriteable(String name, WriteType writeType) {
+ if (isVariableLocallyDefined(name)) {
return true;
}
- if(parentScope != null && parentScope.isVariableWriteable(name, partialWriter) && enclosureType == ENCLOSURE_ALL) return true;
- if(parentScope != null && parentScope.isVariableWriteable(name, partialWriter) && enclosureType == ENCLOSURE_LOOP && partialWriter) return true;
- return false;
+ if (parentScope == null) {
+ // if we are the root scope and the variable is not declared here,
+ // then there is no other place to look
+ return false;
+ }
+ switch (enclosureType) {
+ case CONDITION:
+ if (parentScope.isVariableWriteable(name, WriteType.FULL)) {
+ if (getTopmostLoopToDeclaration(name) != null) {
+ Warnings.warn("Variable " + name + ", defined on line " +
+ getDeclarationLine(name) + ", might have multiple conflicting writers");
+ }
+ return true;
+ }
+ else {
+ return false;
+ }
+ case ALL:
+ return parentScope.isVariableWriteable(name, writeType);
+ case LOOP:
+ return parentScope.isVariableWriteable(name, writeType) && writeType == WriteType.PARTIAL;
+ default:
+ return false;
+ }
}
- public boolean isVariableLocallyDefined(String name) {
- return variables.containsKey(name);
+ /**
+ * Assumes that the variable is defined in some parent scope.
+ * Returns the top-most loop between this scope and the declaration
+ * or null if no such loop is found
+ */
+ private VariableScope getTopmostLoopToDeclaration(String name) {
+ if (isVariableLocallyDefined(name)) {
+ return null;
+ }
+ else if (enclosureType == EnclosureType.LOOP) {
+ VariableScope parentLoop = parentScope.getTopmostLoopToDeclaration(name);
+ if (parentLoop != null) {
+ return parentLoop;
+ }
+ else {
+ return this;
+ }
+ }
+ else {
+ return parentScope.getTopmostLoopToDeclaration(name);
+ }
}
+
+ /**
+ * This deals with cases like this:
+ * a[]; a[0] = 1;
+ * foreach v, k in a {
+ * if (c) {
+ * a[k + 1] = 1;
+ * }
+ * }
+ * ...
+ * foreach v, k in a {
+ * if (c) {
+ * a[k + 100] = 1;
+ * }
+ * }
+ *
+ * a[] can be considered closed when all of the following are true:
+ * 1. the write reference count is equal to the number of loops
+ * that both iterate on and write to a
+ * 2. none of those loops have any iterations running
+ * 3. there are no elements in a[] left to iterate on
+ * These three conditions guarantee that no more writing
+ * can happen to a[].
+ *
+ * Unfortunately, it seems horribly complicated to deal with multiple
+ * loops that do this. Simple reference counting doesn't work, since
+ * any of the conditions above can become false at any point and it is
+ * hard to have an absolute time for all of the loops.
+ *
+ * So engineering compromise: this is allowed for a single loop (per
+ * array), and that's been shown to work (the self close scheme).
+ * If used in more than one array, throw an error.
+ * @throws CompilationException
+ */
+ private void markInVariableUsedInsideLoop(String name) throws CompilationException {
+ if (isVariableLocallyDefined(name)) {
+ return;
+ }
+ switch (enclosureType) {
+ case PROCEDURE:
+ case NONE:
+ return;
+ case LOOP:
+ if (isForeachSourceVar(name)) {
+ // mark as selfClosing
+ Variable var = lookup(name);
+ if (var.foreachSrc != null) {
+ throw new CompilationException("Updating of an array (" +
+ name + ", line " + getDeclarationLine(name) + ") inside a " +
+ "foreach loop that iterates over it is limited to a single " +
+ "loop, but it is used in both " + CompilerUtils.info(var.foreachSrc) +
+ " and " + CompilerUtils.info(src));
+ }
+ else {
+ setSelfClose();
+ }
+ }
+ // also add partial close
+
+ default:
+ if (parentScope != null) {
+ parentScope.markInVariableUsedInsideLoop(name);
+ }
+ }
+ }
- /** List of templates to be executed in sequence after the present
- in-preparation statement is outputted. */
- List<StringTemplate> presentStatementPostStatements = new ArrayList<StringTemplate>();
+ private void setSelfClose() {
+ bodyTemplate.setAttribute("selfClose", "true");
+ }
- Map<String, Variable> variableUsage = new HashMap<String, Variable>();
+ public boolean isVariableLocallyDefined(String name) {
+ return variables != null && variables.containsKey(name);
+ }
/** indicates that the present in-preparation statement writes to the
named variable. If the variable is declared in the local scope,
register a closing statement; otherwise, record that this scope
writes to the variable so that the scope-embedding code (such as
the foreach compiler) can handle appropriately. */
- public void addWriter(String variableName, Object closeID, boolean partialWriter) throws CompilationException
- {
- if(!isVariableDefined(variableName)) {
+ public void addWriter(String variableName, WriteType writeType,
+ XmlObject where, StringTemplate out) throws CompilationException {
+ if (!isVariableDefined(variableName)) {
throw new CompilationException("Variable " + variableName + " is not defined");
}
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Writer " + variableName + " " + CompilerUtils.info(where));
+ }
- if(isVariableLocallyDefined(variableName)) {
- StringTemplate ld = getLocalDeclaration(variableName);
- if (ld != null) {
- if(!partialWriter && ld.getAttribute("waitfor") != null) {
- throw new CompilationException("variable " + variableName + ", defined on line "
- + getDefinitionLine(variableName) + ", has multiple writers.");
- }
- ld.setAttribute("waitfor", closeID);
- }
- else {
- if (logger.isDebugEnabled()) {
- logger.debug("Variable " + variableName + " is local but has no template.");
- }
- }
- outputs.add(variableName);
- if (!inhibitClosing.contains(variableName)) {
- StringTemplate postST = compiler.template("partialclose");
- postST.setAttribute("var", variableName);
- postST.setAttribute("closeID", closeID);
-
- presentStatementPostStatements.add(postST);
- }
- } else {
+ if (isVariableWriteable(variableName, writeType)) {
+ setVariableWrittenToInThisScope(variableName, writeType, where, out);
+ }
+ else {
+ isVariableWriteable(variableName, writeType);
+ throw new CompilationException("variable " + variableName + " is not writeable in this scope");
+ }
+ }
+
+ public void partialClose(String name, XmlObject src, StringTemplate out) {
+ if (inhibitClosing == null || !inhibitClosing.contains(name)) {
+ StringTemplate decl = getExistingDeclaration(name);
+ if (decl != null) {
+ // a function return value will result
+ // in a missing declaration
+ // TODO fix that
+ incWaitCount(name);
+
+ StringTemplate postST = compiler.template("partialclose");
+ postST.setAttribute("var", name);
+
+ out.setAttribute("partialClose", postST);
+ }
+ }
+ }
+
+ public boolean hasPartialClosing() {
+ if (enclosureType != EnclosureType.CONDITION || thenScope != null) {
+ throw new IllegalStateException("hasPartialClosing should only be called in a 'then' condition scope");
+ }
+ for (String name : getUsedVariableNames()) {
+ VariableUsage v = getExistingUsage(name);
+ if (v != null && v.referenceCount > 0) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void setPreClose(String name, int count) {
+ setCount("preClose", name, count,
+ new RefCountSetter<StringTemplate>() {
+ @Override
+ public StringTemplate create(String name, int count) {
+ StringTemplate t = compiler.template("partialclose");
+ t.setAttribute("var", name);
+ if (count != 1) {
+ t.setAttribute("count", count);
+ }
+ return t;
+ }
+
+ @Override
+ public void set(StringTemplate t, int count) {
+ t.removeAttribute("count");
+ if (count != 1) {
+ // 1 is the default count, so no need to have it explicitly
+ t.setAttribute("count", count);
+ }
+ }
-// TODO now we have to walk up the scopes until either we find the
-// variable or we find an upwards assignment prohibition or we run
-// out of scopes
+ @Override
+ public boolean matches(StringTemplate t, String name) {
+ return t.getAttribute("var").equals(name);
+ }
+ });
+ }
+
+ private void setLoopReferenceCount(String name, int count) {
+ setCount("refs", name, count, new RefCountSetter<SimpleRefCountRep>() {
+ @Override
+ public SimpleRefCountRep create(String name, int count) {
+ return new SimpleRefCountRep(name, count);
+ }
-// TODO so far this should find undeclared variables at compile time
-// so perhaps worth making this into a separate patch if it actually
-// works
+ @Override
+ public boolean matches(SimpleRefCountRep t, String name) {
+ return t.name.equals(name);
+ }
- if(isVariableWriteable(variableName, partialWriter)) {
- Variable variable = variableUsage.get(variableName);
- if(variable == null) {
- variable = new Variable();
- variableUsage.put(variableName, variable);
- }
+ @Override
+ public void set(SimpleRefCountRep t, int count) {
+ t.count = count;
+ }
+ });
+ }
+
+ private static class SimpleRefCountRep {
+ public final String name;
+ public int count;
+
+ public SimpleRefCountRep(String name, int count) {
+ this.name = name;
+ this.count = count;
+ }
+
+ public String toString() {
+ return name + " " + count;
+ }
+ }
+
+ private static interface RefCountSetter<T> {
+ T create(String name, int count);
+ void set(T o, int count);
+ boolean matches(T t, String name);
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> void setCount(String attrName, String name, int count, RefCountSetter<T> r) {
+ Object o = bodyTemplate.getAttribute(attrName);
+ List<T> pcs;
+ T matching = null;
+ if (o instanceof List) {
+ pcs = (List<T>) o;
+ Iterator<T> i = pcs.iterator();
+ while (i.hasNext()) {
+ T st = i.next();
+ if (r.matches(st, name)) {
+ if (count == 0) {
+ i.remove();
+ }
+ else {
+ matching = st;
+ break;
+ }
+ }
+ }
+ }
+ else if (o != null) {
+ T st = (T) o;
+ if (r.matches(st, name)) {
+ if (count == 0) {
+ bodyTemplate.removeAttribute("preClose");
+ }
+ else {
+ matching = st;
+ }
+ }
+ }
+ if (count != 0 ) {
+ // add
+ if (matching == null) {
+ bodyTemplate.setAttribute(attrName, r.create(name, count));
+ }
+ else {
+ r.set(matching, count);
+ }
+ }
+ }
+
+ private void incWaitCountLocal(String name) {
+ StringTemplate decl = getLocalDeclaration(name);
+ Integer i = (Integer) decl.getAttribute("waitCount");
+ if (i == null) {
+ i = 1;
+ }
+ else {
+ i = i + 1;
+ decl.removeAttribute("waitCount");
+ }
+ decl.setAttribute("waitCount", i);
+ }
- List<Object> statementList = variable.writingStatements;
- if(!statementList.contains(closeID)) {
- statementList.add(closeID);
- }
- if (logger.isDebugEnabled()) {
- logger.debug("added " + closeID + " to variable "
- + variableName + " in scope " + hashCode());
- }
- } else {
- isVariableWriteable(variableName, partialWriter);
- throw new CompilationException("variable " + variableName + " is not writeable in this scope");
- }
- }
+ private void incWaitCount(String name) {
+ if (this.isVariableLocallyDefined(name)) {
+ incWaitCountLocal(name);
+ }
+ else {
+ boolean propagateToParent = true;
+ if (enclosureType == EnclosureType.CONDITION) {
+ // due to the parsing order, the
+ // then scopes are done before the else scopes,
+ // so no need to update counts in subling else scopes
+ // if this is a then scope
+ if (thenScope == null) {
+ // we are then 'then' scope
+ incVariableReferenceCount(name);
+ }
+ else {
+ // else scope
+ int myCount = incVariableReferenceCount(name);
+ int thenCount = thenScope.getVariableReferenceCount(name);
+ if (myCount > thenCount) {
+ thenScope.setPreClose(name, myCount - thenCount);
+ this.setPreClose(name, 0);
+ }
+ else {
+ this.setPreClose(name, thenCount - myCount);
+ thenScope.setPreClose(name, 0);
+ // this has already been accounted for in the then branch
+ // so skip propagating it up
+ propagateToParent = false;
+ }
+ }
+ }
+ else if (enclosureType == EnclosureType.LOOP) {
+ // issue statements to dynamically increment wait
+ // count on each iteration
+ addLoopReferenceCount(name);
+ }
+ // keep going until we hit the declaration
+ if (propagateToParent && parentScope != null) {
+ parentScope.incWaitCount(name);
+ }
+ }
+ }
+
+ /**
+ * This is only called by setThen(scope), so guaranteed to
+ * be an 'else' scope
+ */
+ private void updateBranchReferenceCounts() {
+ for (VariableUsage v : thenScope.getVariableUsageValues()) {
+ if (v.referenceCount != 0) {
+ setPreClose(v.name, v.referenceCount);
+ }
+ }
+ }
+
+ public void addReader(String name, boolean partial, XmlObject where) throws CompilationException {
+ setVariableReadInThisScope(name, partial, where);
+ if (!partial) {
+ if (logger.isDebugEnabled()) {
+ logger.debug(name + " fully read in " + CompilerUtils.info(where));
+ }
+ }
}
+
+ private void setVariableWrittenToInThisScope(String name, WriteType writeType, XmlObject where, StringTemplate out)
+ throws CompilationException {
+ VariableUsage variable = getUsageForUpdate(name);
+ boolean added = variable.addWritingStatement(where, writeType);
+ if (added) {
+ partialClose(name, where, out);
+ setNonInput(name);
+ }
+ markInVariableUsedInsideLoop(name);
+ }
+
+ private void setNonInput(String name) {
+ StringTemplate decl = getExistingDeclaration(name);
+ if (decl != null) {
+ decl.removeAttribute("input");
+ }
+ }
- /** looks up the template that declared the named variable in the
+ private void setVariableReadInThisScope(String name, boolean partial, XmlObject where)
+ throws CompilationException {
+ // only care about full reads, since they can only happen in
+ // a scope higher than the closing
+ if (partial) {
+ return;
+ }
+ getUsageForUpdate(name).addFullReadingStatement(where);
+ }
+
+ private boolean isVariableWrittenToInThisScope(String name) {
+ VariableUsage v = getExistingUsage(name);
+ return v != null && v.hasWriters();
+ }
+
+ private int incVariableReferenceCount(String name) {
+ return ++getUsageForUpdate(name).referenceCount;
+ }
+
+ private void addLoopReferenceCount(String name) {
+ int count = incVariableReferenceCount(name);
+ setLoopReferenceCount(name, count);
+ }
+
+ private int getVariableReferenceCount(String name) {
+ VariableUsage v = getExistingUsage(name);
+ if (v == null) {
+ return 0;
+ }
+ else {
+ return v.referenceCount;
+ }
+ }
+
+ private boolean isVariableFullyReadInThisScope(String name) {
+ VariableUsage v = getExistingUsage(name);
+ return v != null && v.hasReaders();
+ }
+
+ private XmlObject getFirstFullRead(String name) {
+ VariableUsage v = getExistingUsage(name);
+ if (v == null) {
+ return null;
+ }
+ if (v.fullReadingLoc == null) {
+ return null;
+ }
+ if (v.fullReadingLoc.isEmpty()) {
+ return null;
+ }
+ return v.fullReadingLoc.get(0);
+ }
+
+ private XmlObject getFirstWrite(String name) {
+ VariableUsage v = getExistingUsage(name);
+ if (v == null) {
+ return null;
+ }
+ if (v.fullWritingLoc != null) {
+ return v.fullWritingLoc;
+ }
+ if (v.partialWritingLoc == null) {
+ return null;
+ }
+ if (v.partialWritingLoc.isEmpty()) {
+ return null;
+ }
+ return v.partialWritingLoc.get(0);
+ }
+
+ private List<XmlObject> getAllWriters(String name) {
+ VariableUsage v = getExistingUsage(name);
+ if (v == null) {
+ throw new IllegalArgumentException("Variable " + name + " is not written to in this scope");
+ }
+ return v.partialWritingLoc;
+ }
+
+ StringTemplate getLocalDeclaration(String name) {
+ StringTemplate st = getLocalDeclaration(name, "declarations");
+ /*
+ * procedure scopes go like this:
+ * outerScope - contains all arguments - PROCEDURE
+ * innerScope - contains all returns - NONE
+ * compoundScope - this is where the proc template lives - ALL
+ * this prevents writing to arguments, since writing upwards of NONE is prohibited,
+ * but allows writing to return values
+ */
+
+ if (st == null && parentScope != null && parentScope.enclosureType == EnclosureType.PROCEDURE) {
+ if (children.size() != 1) {
+ throw new IllegalStateException("Procedure scope with more than one child found");
+ }
+ return children.get(0).getLocalDeclaration(name, "initWaitCounts");
+ }
+ else {
+ return st;
+ }
+ }
+
+ /** looks up the template that declared the named variable in the
present scope. If such a template does not exist, returns null.
TODO this should probably merge into general variable structure
rather then walking the template.
*/
- StringTemplate getLocalDeclaration(String name) {
+ StringTemplate getLocalDeclaration(String name, String type) {
if (bodyTemplate == null) {
return null;
}
- Object decls = bodyTemplate.getAttribute("declarations");
- if(decls == null) return null;
+ Object decls = bodyTemplate.getAttribute(type);
+ if (decls == null) {
+ return null;
+ }
- if(decls instanceof StringTemplate) {
+ if (decls instanceof StringTemplate) {
StringTemplate declST = (StringTemplate) decls;
- if(declST.getAttribute("name").equals(name)) {
- logger.debug("thats the declaration for "+name);
+ if (declST.getAttribute("name").equals(name)) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Found declaration for " + name);
+ }
return declST;
}
- } else { // assume its a List of StringTemplate
+ }
+ else { // assume its a List of StringTemplate
@SuppressWarnings("unchecked")
List<StringTemplate> list = (List<StringTemplate>) decls;
for (StringTemplate declST : list) {
@@ -347,9 +987,9 @@
logger.debug("looking at declaration " + declST);
}
try {
- if(declST.getAttribute("name").equals(name)) {
+ if (declST.getAttribute("name").equals(name)) {
if (logger.isDebugEnabled()) {
- logger.debug("thats the declaration for " + name);
+ logger.debug("Found declaration for " + name);
}
return declST;
}
@@ -361,7 +1001,9 @@
}
}
- logger.info("UH OH - couldn't find local definition for "+name);
+ if (logger.isInfoEnabled()) {
+ logger.info("Couldn't find local definition for " + name);
+ }
return null;
}
@@ -393,7 +1035,7 @@
}
presentStatementPostStatements.clear();
outputs.clear();
- inhibitClosing.clear();
+ inhibitClosing = null;
}
private String join(List<String> l) {
@@ -407,20 +1049,6 @@
}
return sb.toString();
}
-
- /** Stores information about a variable that is referred to in this
- scope. Should probably get used for dataset marking eventually. */
- class Variable {
- List<Object> writingStatements = new ArrayList<Object>();
- }
-
- Iterator<String> getVariableIterator() {
- return variableUsage.keySet().iterator();
- }
-
- Set<String> getVariables() {
- return variableUsage.keySet();
- }
public VariableScope getRootScope() {
return rootScope;
@@ -428,7 +1056,7 @@
public List<String> getCleanups() {
List<String> cleanups = new ArrayList<String>();
- for (String var : variables.keySet()) {
+ for (String var : getLocallyDeclaredVariableNames()) {
String type = getVariableType(var);
if (!org.griphyn.vdl.type.Types.isPrimitive(type)) {
cleanups.add(var);
@@ -438,7 +1066,78 @@
}
public String toString() {
- return variables.toString();
+ return CompilerUtils.info(src);
}
+
+ /**
+ * @throws CompilationException
+ */
+ public void analyzeWriters() throws CompilationException {
+ // find variables that are read but not written to
+
+ // keep a stack to account for shadowing
+ Map<String, Stack<Usage>> access = new HashMap<String, Stack<Usage>>();
+
+ analyzeWriters(access);
+ }
+
+ private static class Usage {
+ public VariableScope lastWriter, lastReader;
+ }
+
+ private void analyzeWriters(Map<String, Stack<Usage>> access) throws CompilationException {
+ for (String name : getLocallyDeclaredVariableNames()) {
+ declare(access, name);
+ }
+
+ // parent first, which ensures that the topmost writer
+ // is always first
+ for (Map.Entry<String, Stack<Usage>> e : access.entrySet()) {
+ String name = e.getKey();
+ Stack<Usage> stack = e.getValue();
+
+ boolean fullyRead = isVariableFullyReadInThisScope(name);
+
+ if (isVariableWrittenToInThisScope(name)) {
+ stack.peek().lastWriter = this;
+ }
+
+ if (fullyRead && org.griphyn.vdl.type.Types.isPrimitive(getVariableType(name))) {
+ stack.peek().lastReader = this;
+ }
+ }
+
+ for (VariableScope child : children) {
+ child.analyzeWriters(access);
+ }
+
+ for (String name : getLocallyDeclaredVariableNames()) {
+ undeclare(access, name);
+ }
+ }
+
+ private void declare(Map<String, Stack<Usage>> access, String name) {
+ Stack<Usage> s = access.get(name);
+ if (s == null) {
+ s = new Stack<Usage>();
+ access.put(name, s);
+ }
+ Usage u = new Usage();
+ s.push(u);
+ }
+
+ private void undeclare(Map<String, Stack<Usage>> access, String name) throws CompilationException {
+ Stack<Usage> s = access.get(name);
+ if (s == null || s.isEmpty()) {
+ throw new RuntimeException("Mistmatched undeclare for " + name + " in scope " + this);
+ }
+ Usage u = s.pop();
+ if (s.isEmpty()) {
+ access.remove(name);
+ }
+ if (u.lastReader != null && u.lastWriter == null) {
+ throw new CompilationException("Uninitalized variable: " + name);
+ }
+ }
}
Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/Loader.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -248,9 +248,17 @@
private static String getMessages(Throwable e) {
StringBuilder sb = new StringBuilder();
+ String lastMessage = null;
while (e != null) {
- sb.append(":\n\t");
- sb.append(e.getMessage());
+ String msg = e.getMessage();
+ if (msg == null) {
+ msg = e.toString();
+ }
+ sb.append("\n\t");
+ if (lastMessage == null || !lastMessage.contains(msg)) {
+ sb.append(msg);
+ lastMessage = msg;
+ }
e = e.getCause();
}
return sb.toString();
@@ -336,7 +344,6 @@
if (recompile) {
VDLt2VDLx.compile(new FileInputStream(swiftscript),
new PrintStream(new FileOutputStream(xml)));
-
try {
FileOutputStream f = new FileOutputStream(kml);
Karajan.compile(xml.getAbsolutePath(), new PrintStream(f), provenanceEnabled);
Modified: trunk/src/org/griphyn/vdl/karajan/lib/New.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/New.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/karajan/lib/New.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -51,11 +51,12 @@
public static final Arg OA_MAPPING = new Arg.Optional("mapping", null);
public static final Arg OA_VALUE = new Arg.Optional("value", null);
public static final Arg OA_DBGNAME = new Arg.Optional("dbgname", null);
- public static final Arg OA_WAITFOR = new Arg.Optional("waitfor", null);
+ public static final Arg OA_WAITCOUNT = new Arg.Optional("waitcount", null);
+ public static final Arg OA_INPUT = new Arg.Optional("input", Boolean.FALSE);
static {
setArguments(New.class,
- new Arg[] { OA_TYPE, OA_MAPPING, OA_VALUE, OA_DBGNAME, OA_WAITFOR});
+ new Arg[] { OA_TYPE, OA_MAPPING, OA_VALUE, OA_DBGNAME, OA_WAITCOUNT, OA_INPUT});
}
private Tracer tracer;
@@ -73,7 +74,8 @@
Map<String,Object> mapping =
(Map<String,Object>) OA_MAPPING.getValue(stack);
String dbgname = TypeUtil.toString(OA_DBGNAME.getValue(stack));
- String waitfor = (String) OA_WAITFOR.getValue(stack);
+ String waitCount = (String) OA_WAITCOUNT.getValue(stack);
+ boolean input = TypeUtil.toBoolean(OA_INPUT.getValue(stack));
String line = (String) getProperty("_defline");
MappingParamSet mps = new MappingParamSet();
@@ -83,6 +85,10 @@
mps.set(MappingParam.SWIFT_DBGNAME, dbgname);
}
+ if (input) {
+ mps.set(MappingParam.SWIFT_INPUT, true);
+ }
+
if (line != null) {
mps.set(MappingParam.SWIFT_LINE, line);
}
@@ -91,9 +97,15 @@
mps.set(MappingParam.SWIFT_RESTARTID, threadPrefix + ":" + dbgname);
- if (waitfor != null) {
- mps.set(MappingParam.SWIFT_WAITFOR, waitfor);
+ // input means never written to, but read at least once
+ int initialWriteRefCount;
+ boolean noWriters = input;
+ if (waitCount != null) {
+ initialWriteRefCount = Integer.parseInt(waitCount);
}
+ else {
+ initialWriteRefCount = 0;
+ }
if (typename == null && value == null) {
throw new ExecutionException("You must specify either a type or a value");
@@ -157,7 +169,7 @@
}
handle.closeShallow();
}
- else {
+ else {
if (tracer.isEnabled()) {
tracer.trace(threadPrefix, dbgname);
}
@@ -191,6 +203,7 @@
if (AbstractDataNode.provenance && logger.isDebugEnabled()) {
logger.debug("NEW id=" + handle.getIdentifier());
}
+ handle.setWriteRefCount(initialWriteRefCount);
return handle;
}
catch (Exception e) {
Modified: trunk/src/org/griphyn/vdl/karajan/lib/PartialCloseDataset.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/PartialCloseDataset.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/karajan/lib/PartialCloseDataset.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -20,92 +20,32 @@
import org.apache.log4j.Logger;
import org.globus.cog.karajan.arguments.Arg;
import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.util.TypeUtil;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.griphyn.vdl.mapping.DSHandle;
-import org.griphyn.vdl.mapping.MappingParam;
-import java.util.StringTokenizer;
-import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.ArrayList;
-
public class PartialCloseDataset extends VDLFunction {
public static final Logger logger = Logger.getLogger(CloseDataset.class);
- public static final Arg OA_CLOSE_ID = new Arg.Optional("closeID", null);
+ public static final Arg OA_COUNT = new Arg.Optional("count", 1);
static {
- setArguments(PartialCloseDataset.class, new Arg[] { PA_VAR, OA_CLOSE_ID });
+ setArguments(PartialCloseDataset.class, new Arg[] { PA_VAR, OA_COUNT });
}
-
- /** Map from DSHandles (as keys) to lists of what we have seen
- already. TODO this may end up growing too much when a program
- has lots of objects. Consider alternative ways of doing this. */
- static Map<DSHandle,List<String>> pendingDatasets =
- new HashMap<DSHandle,List<String>>();
-
public Object function(VariableStack stack) throws ExecutionException {
- boolean hasUnseenToken = false;
DSHandle var = (DSHandle) PA_VAR.getValue(stack);
- String closeID = (String) OA_CLOSE_ID.getValue(stack);
if (logger.isDebugEnabled()) {
- logger.debug("Partially closing " + var +
- " hash: " + var.hashCode() +
- " for statement " + closeID);
+ logger.debug("Partially closing " + var);
}
- if(var.isClosed()) {
+ if (var.isClosed()) {
logger.debug("variable already closed - skipping partial close processing");
return null;
}
-
- synchronized(pendingDatasets) {
-
- List<String> c = pendingDatasets.get(var);
- if (c == null) {
- c = new ArrayList<String>();
- pendingDatasets.put(var, c);
- }
-
- c.add(closeID);
- if (logger.isDebugEnabled()) {
- logger.debug("Adding token "+closeID+" with hash "+closeID.hashCode());
- }
-
- String needToWaitFor = var.getParam(MappingParam.SWIFT_WAITFOR);
- logger.debug("need to wait for " + needToWaitFor);
- StringTokenizer stok = new StringTokenizer(needToWaitFor, " ");
- while(stok.hasMoreTokens()) {
- String s = stok.nextToken();
- // do we have this token in our list of already seen
- // statement IDs?
- if(! c.contains(s)) {
- // then we have a required element that we have not
- // seen yet, so...
- hasUnseenToken = true;
- if (logger.isDebugEnabled()) {
- logger.debug("Container does not contain token " + s);
- }
- } else {
- if (logger.isDebugEnabled()) {
- logger.debug("Container does contain token " + s);
- }
- }
- }
- }
- if (logger.isDebugEnabled()) {
- logger.debug("hasUnseenToken = "+hasUnseenToken);
- }
- if(!hasUnseenToken) {
- if(logger.isInfoEnabled()) {
- logger.info("All partial closes for " + var +
- " have happened. Closing fully.");
- }
- var.closeDeep();
- pendingDatasets.remove(var);
- }
+
+ int count = TypeUtil.toInt(OA_COUNT.getValue(stack));
+ var.updateWriteRefCount(-count);
return null;
}
}
Added: trunk/src/org/griphyn/vdl/karajan/lib/SetWaitCount.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/SetWaitCount.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/SetWaitCount.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2012 University of Chicago
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.griphyn.vdl.karajan.lib;
+
+import org.apache.log4j.Logger;
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.util.TypeUtil;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.griphyn.vdl.mapping.DSHandle;
+
+public class SetWaitCount extends VDLFunction {
+ public static final Logger logger = Logger.getLogger(CloseDataset.class);
+
+ public static final Arg OA_COUNT = new Arg.Optional("count", 1);
+
+ static {
+ setArguments(SetWaitCount.class, new Arg[] { PA_VAR, OA_COUNT });
+ }
+
+ public Object function(VariableStack stack) throws ExecutionException {
+ DSHandle var = (DSHandle) PA_VAR.getValue(stack);
+
+ if (var.isClosed()) {
+ throw new ExecutionException("Attempted to set a wait count for a closed variable " + var);
+ }
+
+ int count = TypeUtil.toInt(OA_COUNT.getValue(stack));
+ var.setWriteRefCount(count);
+ return null;
+ }
+}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -21,6 +21,7 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.StringTokenizer;
import org.apache.log4j.Logger;
import org.globus.cog.karajan.arguments.Arg;
@@ -43,17 +44,17 @@
import org.griphyn.vdl.util.VDL2Config;
public class ThrottledParallelFor extends AbstractParallelIterator {
- public static final Logger logger = Logger
- .getLogger(ThrottledParallelFor.class);
+ public static final Logger logger = Logger.getLogger(ThrottledParallelFor.class);
public static final int DEFAULT_MAX_THREADS = 1024;
public static final Arg A_NAME = new Arg.Positional("name");
public static final Arg A_IN = new Arg.Positional("in");
- public static final Arg A_SELF_CLOSE = new Arg.Optional("selfclose", Boolean.FALSE);
+ public static final Arg O_SELFCLOSE = new Arg.Optional("selfclose", Boolean.FALSE);
+ public static final Arg O_REFS = new Arg.Optional("refs", null);
static {
- setArguments(ThrottledParallelFor.class, new Arg[] { A_NAME, A_IN, A_SELF_CLOSE });
+ setArguments(ThrottledParallelFor.class, new Arg[] { A_NAME, A_IN, O_SELFCLOSE, O_REFS });
}
public static final String THREAD_COUNT = "#threadcount";
@@ -61,7 +62,36 @@
private int maxThreadCount = -1;
private Tracer forTracer, iterationTracer;
private String kvar, vvar;
+ private List<StaticRefCount> srefs;
+ private static class StaticRefCount {
+ public final String name;
+ public final int count;
+
+ public StaticRefCount(String name, int count) {
+ this.name = name;
+ this.count = count;
+ }
+ }
+
+ private static class RefCount {
+ public final DSHandle var;
+ public final int count;
+
+ public RefCount(DSHandle var, int count) {
+ this.var = var;
+ this.count = count;
+ }
+
+ public void inc() {
+
+ }
+
+ public void dec() {
+
+ }
+ }
+
@Override
protected void initializeStatic() {
super.initializeStatic();
@@ -69,11 +99,44 @@
iterationTracer = Tracer.getTracer(this, "ITERATION");
kvar = (String) getProperty("_kvar");
vvar = (String) getProperty("_vvar");
+ srefs = buildStaticRefs();
}
+ private List<StaticRefCount> buildStaticRefs() {
+ String refs = (String) O_REFS.getStatic(this);
+ if (refs == null) {
+ return null;
+ }
+ List<StaticRefCount> l = new ArrayList<StaticRefCount>();
+ String name = null;
+ boolean flip = true;
+ StringTokenizer st = new StringTokenizer(refs);
+ while (st.hasMoreTokens()) {
+ if (flip) {
+ name = st.nextToken();
+ }
+ else {
+ int count = Integer.parseInt(st.nextToken());
+ l.add(new StaticRefCount(name, count));
+ }
+ flip = !flip;
+ }
+ return l;
+ }
+
+ private List<RefCount> buildRefs(VariableStack stack) throws VariableNotFoundException {
+ if (srefs == null) {
+ return null;
+ }
+ List<RefCount> l = new ArrayList<RefCount>(srefs.size());
+ for (StaticRefCount s : srefs) {
+ l.add(new RefCount((DSHandle) stack.getVar(s.name), s.count));
+ }
+ return l;
+ }
+
protected void partialArgumentsEvaluated(VariableStack stack)
throws ExecutionException {
- stack.setVar("selfclose", A_SELF_CLOSE.getValue(stack));
if (forTracer.isEnabled()) {
forTracer.trace(ThreadingContext.get(stack).toString());
}
@@ -89,8 +152,7 @@
stack.setVar(VAR, var);
setChildFailed(stack, false);
stack.setCaller(this);
- initThreadCount(stack, TypeUtil.toBoolean(stack.currentFrame().getVar("selfclose")), i);
- stack.currentFrame().deleteVar("selfclose");
+ initThreadCount(stack, TypeUtil.toBoolean(O_SELFCLOSE.getStatic(this)), i);
citerate(stack, var, i);
}
else {
@@ -110,35 +172,18 @@
int j = 0;
try {
for (; j < available && i.hasNext(); j++) {
- VariableStack copy = stack.copy();
- copy.enter();
- ThreadingContext ntc = ThreadingContext.get(copy).split(i.current());
- ThreadingContext.set(copy, ntc);
- setIndex(copy, getArgCount());
- setArgsDone(copy);
- Object value = i.next();
- if (iterationTracer.isEnabled()) {
- iterationTracer.trace(ntc.toString(), unwrap(value));
- }
- copy.setVar(var.getName(), value);
- startElement(getArgCount(), copy);
+ startIteration(tc, var, i.current(), i.next(), stack);
}
}
finally {
tc.add(j);
}
while (i.hasNext()) {
- Object value = tc.tryIncrement();
- VariableStack copy = stack.copy();
- copy.enter();
- ThreadingContext.set(copy, ThreadingContext.get(copy).split(
- i.current()));
- setIndex(copy, getArgCount());
- setArgsDone(copy);
- copy.setVar(var.getName(), value);
- startElement(getArgCount(), copy);
+ startIteration(tc, var, i.current(), tc.tryIncrement(), stack);
}
+ decRefs(tc.rc);
+
int left;
synchronized(tc) {
// can only have closed and running = 0 in one place
@@ -157,7 +202,38 @@
}
}
- private Object unwrap(Object value) {
+ private void startIteration(ThreadCount tc, Identifier var, int id, Object value,
+ VariableStack stack) throws ExecutionException {
+ incRefs(tc.rc);
+ VariableStack copy = stack.copy();
+ copy.enter();
+ ThreadingContext ntc = ThreadingContext.get(copy).split(id);
+ ThreadingContext.set(copy, ntc);
+ setIndex(copy, 2);
+ if (iterationTracer.isEnabled()) {
+ iterationTracer.trace(ntc.toString(), unwrap(value));
+ }
+ copy.setVar(var.getName(), value);
+ startElement(1, copy);
+ }
+
+ private void decRefs(List<RefCount> rcs) throws ExecutionException {
+ if (rcs != null) {
+ for (RefCount rc : rcs) {
+ rc.var.updateWriteRefCount(-rc.count);
+ }
+ }
+ }
+
+ private void incRefs(List<RefCount> rcs) throws ExecutionException {
+ if (rcs != null) {
+ for (RefCount rc : rcs) {
+ rc.var.updateWriteRefCount(rc.count);
+ }
+ }
+ }
+
+ private Object unwrap(Object value) {
if (value instanceof Pair) {
Pair p = (Pair) value;
if (kvar != null) {
@@ -171,6 +247,28 @@
return "!";
}
}
+
+ @Override
+ public void completed(VariableStack stack) throws ExecutionException {
+ int index = preIncIndex(stack) - 1;
+ if (index == 1) {
+ // iterator
+ stack.currentFrame().deleteVar(QUOTED);
+ processArguments(stack);
+ try {
+ partialArgumentsEvaluated(stack);
+ }
+ catch (FutureFault e) {
+ e.getFuture().addModificationAction(new PartialResume(), stack);
+ }
+ }
+ else if (index == elementCount()) {
+ iterationCompleted(stack);
+ }
+ else {
+ startElement(index, stack);
+ }
+ }
public void failed(VariableStack stack, ExecutionException e) throws ExecutionException {
if (!testAndSetChildFailed(stack)) {
@@ -191,16 +289,21 @@
synchronized(tc) {
closed = tc.isClosed();
running = tc.decrement();
- iteratorHasValues = !tc.selfClose || tc.iteratorHasValues();
+ iteratorHasValues = tc.iteratorHasValues();
}
+ boolean done = false;
if (running == 0) {
- if (closed || !iteratorHasValues) {
+ if (closed) {
complete(stack);
}
+ if (tc.selfClose && !iteratorHasValues) {
+ decRefs(tc.rc);
+ complete(stack);
+ }
}
}
- private void initThreadCount(VariableStack stack, boolean selfClose, KarajanIterator i) {
+ private void initThreadCount(VariableStack stack, boolean selfClose, KarajanIterator i) throws VariableNotFoundException {
if (maxThreadCount < 0) {
try {
maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig()
@@ -210,7 +313,7 @@
maxThreadCount = DEFAULT_MAX_THREADS;
}
}
- stack.setVar(THREAD_COUNT, new ThreadCount(maxThreadCount, selfClose, i));
+ stack.setVar(THREAD_COUNT, new ThreadCount(maxThreadCount, selfClose, i, buildRefs(stack)));
}
private ThreadCount getThreadCount(VariableStack stack)
@@ -224,20 +327,27 @@
}
private static class ThreadCount implements FutureIterator {
- private int maxThreadCount;
+ public boolean selfClose;
+ private int maxThreadCount;
private int crt;
- private boolean selfClose, closed;
+ private boolean closed;
private List<ListenerStackPair> listeners;
private KarajanIterator i;
+ private final List<RefCount> rc;
- public ThreadCount(int maxThreadCount, boolean selfClose, KarajanIterator i) {
+ public ThreadCount(int maxThreadCount, boolean selfClose, KarajanIterator i, List<RefCount> rc) {
this.maxThreadCount = maxThreadCount;
- this.selfClose = selfClose;
this.i = i;
crt = 0;
+ this.selfClose = selfClose;
+ this.rc = rc;
}
- public boolean iteratorHasValues() {
+ public boolean raiseWaiting() {
+ return false;
+ }
+
+ public boolean iteratorHasValues() {
try {
return i.hasNext();
}
@@ -245,10 +355,6 @@
return false;
}
}
-
- public boolean isSelfClose() {
- return selfClose;
- }
public synchronized int available() {
return maxThreadCount - crt;
Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -29,6 +29,7 @@
import java.util.Map;
import org.apache.log4j.Logger;
+import org.globus.cog.karajan.workflow.ExecutionException;
import org.globus.cog.karajan.workflow.futures.Future;
import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
import org.griphyn.vdl.karajan.DSHandleFutureWrapper;
@@ -84,6 +85,7 @@
private Path pathFromRoot;
protected FutureWrapper wrapper;
+ private int writeRefCount;
protected AbstractDataNode(Field field) {
this.field = field;
@@ -681,4 +683,30 @@
value = null;
pathFromRoot = null;
}
+
+ @Override
+ public synchronized void setWriteRefCount(int count) {
+ this.writeRefCount = count;
+ }
+
+ @Override
+ public synchronized int updateWriteRefCount(int delta) {
+ this.writeRefCount += delta;
+
+ if (this.writeRefCount < 0) {
+ throw new IllegalArgumentException("Reference count mismatch for " + this + ". Count is " + this.writeRefCount);
+ }
+
+ if (logger.isDebugEnabled()) {
+ logger.debug(this + " writeRefCount " + this.writeRefCount);
+ }
+ if (this.writeRefCount == 0) {
+ if(logger.isInfoEnabled()) {
+ logger.info("All partial closes for " + this +
+ " have happened. Closing fully.");
+ }
+ closeDeep();
+ }
+ return this.writeRefCount;
+ }
}
Modified: trunk/src/org/griphyn/vdl/mapping/DSHandle.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/DSHandle.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/mapping/DSHandle.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -103,4 +103,8 @@
public String getIdentifyingString();
public boolean isRestartable();
+
+ public void setWriteRefCount(int count);
+
+ public int updateWriteRefCount(int delta);
}
Modified: trunk/src/org/griphyn/vdl/mapping/MappingParam.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/MappingParam.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/mapping/MappingParam.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -29,7 +29,6 @@
public static final MappingParam SWIFT_LINE = new MappingParam("swift#line");
public static final MappingParam SWIFT_DBGNAME = new MappingParam("swift#dbgname");
public static final MappingParam SWIFT_RESTARTID = new MappingParam("swift#restartid");
- public static final MappingParam SWIFT_WAITFOR = new MappingParam("swift#waitfor");
public static final MappingParam SWIFT_BASEDIR = new MappingParam("swift#basedir");
public static final MappingParam SWIFT_DESCRIPTOR = new MappingParam("swift#descriptor");
public static final MappingParam SWIFT_INPUT = new MappingParam("swift#input");
Modified: trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -77,7 +77,12 @@
String desc = (String) params.get(MappingParam.SWIFT_DESCRIPTOR);
if (desc == null) {
- initialized();
+ initialized();
+ Boolean input = (Boolean) params.get(MappingParam.SWIFT_INPUT);
+ if (input != null && input.booleanValue()) {
+ // Empty array. Clearly only in test cases.
+ closeDeep();
+ }
return;
}
try {
Modified: trunk/src/org/griphyn/vdl/mapping/RootDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2012-11-23 08:44:21 UTC (rev 6076)
+++ trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2012-11-23 08:56:28 UTC (rev 6077)
@@ -120,8 +120,8 @@
static protected void checkInputs(MappingParamSet params, Mapper mapper, AbstractDataNode root,
DuplicateMappingChecker dmc) {
- String input = (String) params.get(MappingParam.SWIFT_INPUT);
- if (input != null && Boolean.valueOf(input.trim()).booleanValue()) {
+ Boolean input = (Boolean) params.get(MappingParam.SWIFT_INPUT);
+ if (input != null && input.booleanValue()) {
addExisting(mapper, root);
checkConsistency(root, true, mapper, dmc);
}
More information about the Swift-commit
mailing list