[Swift-commit] r4781 - in trunk: resources src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib src/org/griphyn/vdl/mapping src/org/griphyn/vdl/mapping/file
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Tue Jul 5 17:20:13 CDT 2011
Author: hategan
Date: 2011-07-05 17:20:12 -0500 (Tue, 05 Jul 2011)
New Revision: 4781
Modified:
trunk/resources/Karajan.stg
trunk/src/org/griphyn/vdl/engine/Karajan.java
trunk/src/org/griphyn/vdl/karajan/lib/CleanDataset.java
trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java
trunk/src/org/griphyn/vdl/mapping/Mapper.java
trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java
trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java
trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java
Log:
and now with proper support for the main scope and composite data
Modified: trunk/resources/Karajan.stg
===================================================================
--- trunk/resources/Karajan.stg 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/resources/Karajan.stg 2011-07-05 22:20:12 UTC (rev 4781)
@@ -3,7 +3,7 @@
// TODO can move progress ticker start into vdl:mains so karajan files
// are smaller
-program(types,procedures,declarations,statements,constants,buildversion) ::= <<
+program(types,procedures,declarations,statements,constants,buildversion,cleanups) ::= <<
<project><!-- CACHE ID $buildversion$ -->
<import file="sys.xml"/>
<import file="scheduler.xml"/>
@@ -28,8 +28,10 @@
</vdl:mainp>
<vdl:stopprogressticker />
</vdl:mains>
- </restartLog>
+ </restartLog>
$endif$
+ $cleanups:vdl_cleandataset();separator="\n"$
+ <vdl:cleandataset shutdown="true"/>
</project>
>>
Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/Karajan.java 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2011-07-05 22:20:12 UTC (rev 4781)
@@ -287,6 +287,7 @@
checkUninitializedVariables();
generateInternedConstants(scope.bodyTemplate);
+ scope.bodyTemplate.setAttribute("cleanups", scope.getCleanups());
return scope.bodyTemplate;
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/CleanDataset.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/CleanDataset.java 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/src/org/griphyn/vdl/karajan/lib/CleanDataset.java 2011-07-05 22:20:12 UTC (rev 4781)
@@ -8,18 +8,33 @@
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.griphyn.vdl.mapping.AbstractDataNode;
+import org.griphyn.vdl.mapping.file.FileGarbageCollector;
public class CleanDataset extends VDLFunction {
public static final Logger logger = Logger.getLogger(CleanDataset.class);
+ public static final Arg.Optional OA_SHUTDOWN = new Arg.Optional("shutdown");
+
static {
- setArguments(CleanDataset.class, new Arg[] { PA_VAR });
+ setArguments(CleanDataset.class, new Arg[] { PA_VAR, OA_SHUTDOWN });
}
public Object function(VariableStack stack) throws ExecutionException {
- AbstractDataNode var = (AbstractDataNode) PA_VAR.getValue(stack);
- logger.info("Cleaning " + var);
- var.clean();
- return null;
+ if (OA_SHUTDOWN.isPresent(stack)) {
+ // signals that everything is done and the main program should wait for the
+ // garbage collector to finish everything
+ try {
+ FileGarbageCollector.getDefault().waitFor();
+ }
+ catch (InterruptedException e) {
+ // good time to quit now
+ }
+ }
+ else {
+ AbstractDataNode var = (AbstractDataNode) PA_VAR.getValue(stack);
+ logger.info("Cleaning " + var);
+ var.clean();
+ }
+ return null;
}
}
Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2011-07-05 22:20:12 UTC (rev 4781)
@@ -632,7 +632,12 @@
public synchronized void clean() {
Mapper mapper = getMapper();
if (mapper != null) {
- mapper.clean(getPathFromRoot());
+ try {
+ mapper.clean(getFringePaths());
+ }
+ catch (HandleOpenException e) {
+ logger.warn("Unexpected exception", e);
+ }
}
field = null;
handles = null;
Modified: trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2011-07-05 22:20:12 UTC (rev 4781)
@@ -1,5 +1,6 @@
package org.griphyn.vdl.mapping;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@@ -61,7 +62,7 @@
}
@Override
- public void clean(Path path) {
+ public void clean(Collection<Path> path) {
// no cleaning by default
}
Modified: trunk/src/org/griphyn/vdl/mapping/Mapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/Mapper.java 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/src/org/griphyn/vdl/mapping/Mapper.java 2011-07-05 22:20:12 UTC (rev 4781)
@@ -60,7 +60,7 @@
* Clean the specified path. A temporary mapper may remove the
* corresponding file after this call.
*/
- void clean(Path path);
+ void clean(Collection<Path> paths);
boolean isPersistent(Path path);
Modified: trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java 2011-07-05 22:20:12 UTC (rev 4781)
@@ -71,10 +71,12 @@
}
@Override
- public void clean(Path path) {
- PhysicalFormat pf = map(path);
- logger.info("Cleaning file " + pf);
- FileGarbageCollector.getDefault().decreaseUsageCount(pf);
+ public void clean(Collection<Path> paths) {
+ for (Path path : paths) {
+ PhysicalFormat pf = map(path);
+ logger.info("Cleaning file " + pf);
+ FileGarbageCollector.getDefault().decreaseUsageCount(pf);
+ }
}
@Override
Modified: trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java 2011-07-05 22:20:12 UTC (rev 4781)
@@ -35,6 +35,7 @@
private Thread thread;
private Map<PhysicalFormat, Integer> usageCount;
private Set<PhysicalFormat> persistent;
+ private boolean shutdown, done;
public FileGarbageCollector() {
queue = new LinkedList<PhysicalFormat>();
@@ -93,9 +94,13 @@
while (true) {
PhysicalFormat pf;
synchronized(this) {
- while (queue.isEmpty()) {
+ while (queue.isEmpty() && !shutdown) {
this.wait();
}
+ if (shutdown) {
+ done = true;
+ break;
+ }
pf = queue.remove();
}
try {
@@ -109,4 +114,14 @@
catch (InterruptedException e) {
}
}
+
+ public void waitFor() throws InterruptedException {
+ shutdown = true;
+ while (!done) {
+ synchronized(this) {
+ notify();
+ }
+ Thread.sleep(1);
+ }
+ }
}
Modified: trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java 2011-07-05 21:50:52 UTC (rev 4780)
+++ trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java 2011-07-05 22:20:12 UTC (rev 4781)
@@ -46,15 +46,17 @@
}
@Override
- public void clean(Path path) {
- PhysicalFormat pf = map(path);
- if (PARAM_TEMP.getBooleanValue(this)) {
- System.out.println("Cleaning file " + pf);
- FileGarbageCollector.getDefault().decreaseUsageCount(pf);
+ public void clean(Collection<Path> paths) {
+ for (Path path : paths) {
+ PhysicalFormat pf = map(path);
+ if (PARAM_TEMP.getBooleanValue(this)) {
+ System.out.println("Cleaning file " + pf);
+ FileGarbageCollector.getDefault().decreaseUsageCount(pf);
+ }
+ else {
+ System.out.println("Not cleaning " + pf + " (not temporary)");
+ }
}
- else {
- System.out.println("Not cleaning " + pf + " (not temporary)");
- }
}
@Override
More information about the Swift-commit
mailing list