[Swift-commit] r6214 - in trunk/src/org/griphyn/vdl: karajan mapping util
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Sat Feb 2 03:05:56 CST 2013
Author: hategan
Date: 2013-02-02 03:05:55 -0600 (Sat, 02 Feb 2013)
New Revision: 6214
Modified:
trunk/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java
trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java
trunk/src/org/griphyn/vdl/util/VDL2Config.java
trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java
Log:
don't store full references to the handles in the mapping checker; only store minimum necessary to provide useful information - this allows the actual swift data to be garbage-collected properly; also add configuration option to disable the mapping checker since, in theory, it is an organized memory leak
Modified: trunk/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java 2013-02-02 09:02:23 UTC (rev 6213)
+++ trunk/src/org/griphyn/vdl/karajan/VDL2ExecutionContext.java 2013-02-02 09:05:55 UTC (rev 6214)
@@ -28,8 +28,10 @@
import org.globus.cog.karajan.workflow.ElementTree;
import org.globus.cog.karajan.workflow.ExecutionContext;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.griphyn.vdl.karajan.functions.ConfigProperty;
import org.griphyn.vdl.karajan.functions.ProcessBulkErrors;
import org.griphyn.vdl.mapping.DuplicateMappingChecker;
+import org.griphyn.vdl.util.VDL2Config;
public class VDL2ExecutionContext extends ExecutionContext {
public static final Logger logger = Logger.getLogger(VDL2ExecutionContext.class);
@@ -100,7 +102,9 @@
super.setGlobals(stack);
stack.setGlobal(RUN_ID, runID);
stack.setGlobal(SCRIPT_NAME, scriptName);
- stack.setGlobal(DM_CHECKER, new DuplicateMappingChecker());
+
+ VDL2Config conf = (VDL2Config) stack.getGlobal(ConfigProperty.INSTANCE_CONFIG);
+ stack.setGlobal(DM_CHECKER, new DuplicateMappingChecker(conf));
}
public String getRunID() {
Modified: trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2013-02-02 09:02:23 UTC (rev 6213)
+++ trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2013-02-02 09:05:55 UTC (rev 6214)
@@ -15,22 +15,44 @@
import java.util.Map;
import org.apache.log4j.Logger;
+import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.VDL2ConfigProperties;
public class DuplicateMappingChecker {
public static final Logger logger = Logger.getLogger(DuplicateMappingChecker.class);
+ public boolean enabled = true;
+
private final Map<PhysicalFormat, Entry> map;
- public DuplicateMappingChecker() {
+ public DuplicateMappingChecker(VDL2Config conf) {
+ enabled = !"off".equals(conf.getProperty(VDL2ConfigProperties.DM_CHECKER));
map = new HashMap<PhysicalFormat, Entry>();
}
- private static class Entry {
- private DSHandle write;
- private List<DSHandle> read;
+ private static class Info {
+ private final String name, line;
+ public Info(String name, String line) {
+ this.name = name;
+ this.line = line;
+ }
+
+ public String toString() {
+ if (line == null) {
+ return name;
+ }
+ else {
+ return name + " (line " + line + ")";
+ }
+ }
}
+ private static class Entry {
+ private Info write;
+ private List<Info> read;
+ }
+
private Entry getEntry(PhysicalFormat f) {
Entry e = map.get(f);
if (e == null) {
@@ -41,30 +63,46 @@
}
public synchronized void addRead(PhysicalFormat f, DSHandle h) {
+ if (!enabled) {
+ return;
+ }
Entry e = getEntry(f);
if (e.write != null) {
warn("Duplicate mapping found:\n\t" +
- getVarInfo(h) + " is used to read from " + f + "\n\t" +
- getVarInfo(e.write) + " is used to write to " + f);
+ formatInfo(getInfo(h)) + " is used to read from " + f + "\n\t" +
+ formatInfo(e.write) + " is used to write to " + f);
}
if (e.read == null) {
- e.read = new LinkedList<DSHandle>();
+ e.read = new LinkedList<Info>();
}
- e.read.add(h);
+ e.read.add(getInfo(h));
}
+ private Info getInfo(DSHandle h) {
+ if (h instanceof AbstractDataNode) {
+ AbstractDataNode a = (AbstractDataNode) h;
+ return new Info(a.getDisplayableName(), a.getDeclarationLine());
+ }
+ else {
+ return new Info(String.valueOf(h), null);
+ }
+ }
+
public synchronized void addWrite(PhysicalFormat f, DSHandle h) {
+ if (!enabled) {
+ return;
+ }
Entry e = getEntry(f);
if (e.write != null) {
warn("Duplicate mapping found:\n\t" +
- getVarInfo(h) + " and " + getVarInfo(e.write) + " are both used to write to " + f);
+ formatInfo(getInfo(h)) + " and " + formatInfo(e.write) + " are both used to write to " + f);
}
if (e.read != null) {
warn("Duplicate mapping found:\n\t" +
- getVarInfo(e.write) + " is used to write to " + f + "\n\t" +
- "The following variables(s) are also used to read from " + f + ":" + getVarInfos(e.read));
+ formatInfo(e.write) + " is used to write to " + f + "\n\t" +
+ "The following variables(s) are also used to read from " + f + ":" + formatInfos(e.read));
}
- e.write = h;
+ e.write = getInfo(h);
}
private void warn(String s) {
@@ -74,22 +112,16 @@
System.err.println(s);
}
- private String getVarInfos(List<DSHandle> l) {
+ private String formatInfos(List<Info> l) {
StringBuilder sb = new StringBuilder();
- for (DSHandle h : l) {
+ for (Info h : l) {
sb.append("\n\t\t");
- sb.append(getVarInfo(h));
+ sb.append(formatInfo(h));
}
return sb.toString();
}
- private String getVarInfo(DSHandle h) {
- if (h instanceof AbstractDataNode) {
- AbstractDataNode a = (AbstractDataNode) h;
- return a.getDisplayableName() + " (line " + a.getDeclarationLine() + ")";
- }
- else {
- return String.valueOf(h);
- }
+ private String formatInfo(Info i) {
+ return i.toString();
}
}
Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2013-02-02 09:02:23 UTC (rev 6213)
+++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2013-02-02 09:05:55 UTC (rev 6214)
@@ -120,6 +120,7 @@
put("ticker.prefix", "Progress: time:");
put(VDL2ConfigProperties.FILE_GC_ENABLED, "true");
+ put(VDL2ConfigProperties.DM_CHECKER, "on");
}
private VDL2Config(VDL2Config other) {
Modified: trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2013-02-02 09:02:23 UTC (rev 6213)
+++ trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2013-02-02 09:05:55 UTC (rev 6214)
@@ -44,6 +44,7 @@
public static final String PROVENANCE_LOG = "provenance.log";
public static final String FILE_GC_ENABLED = "file.gc.enabled";
public static final String TRACING_ENABLED = "tracing.enabled";
+ public static final String DM_CHECKER = "mapping.checker";
public static final Map<String, PropInfo> PROPERTIES;
static {
@@ -143,6 +144,11 @@
"operations within swift such as iterations, invocations, assignments, and declarations, as well " +
"as data dependencies will be logged. This comes at a cost in performance. It is therefore " +
"disabled by default."));
+
+ PROPERTIES.put(DM_CHECKER, new PropInfo("<on|off>", "Controls the run-time duplicate mapping checker " +
+ "(which indetifies mapping conflicts). If set to 'on', the checker is enabled. When enabled, a record " +
+ "of all mapped data is kept, so this comes at the expense of a slight memory leak. If set 'off', the" +
+ " mapping checker is disabled. Enabled by default."));
}
public static Map<String, PropInfo> getPropertyDescriptions() {
More information about the Swift-commit
mailing list