[Swift-commit] r3468 - trunk/src/org/griphyn/vdl/karajan/lib
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Fri Jul 23 23:27:28 CDT 2010
Author: hategan
Date: 2010-07-23 23:27:28 -0500 (Fri, 23 Jul 2010)
New Revision: 3468
Added:
trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java
trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java
trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java
trunk/src/org/griphyn/vdl/karajan/lib/Mark.java
trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java
trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java
trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java
Log:
added java implementations for some functions in vdl-int
Added: trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java 2010-07-24 04:27:28 UTC (rev 3468)
@@ -0,0 +1,51 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.List;
+
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.arguments.VariableArguments;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.util.TypeUtil;
+import org.globus.cog.karajan.workflow.ExecutionException;
+
+public class Flatten extends VDLFunction {
+
+ static {
+ setArguments(Flatten.class, new Arg[] { Arg.VARGS });
+ }
+
+ @Override
+ protected Object function(VariableStack stack) throws ExecutionException {
+ VariableArguments v = Arg.VARGS.get(stack);
+ if (v.isEmpty()) {
+ return "";
+ }
+ else {
+ StringBuilder sb = new StringBuilder();
+ flatten(sb, v.getAll());
+ sb.deleteCharAt(sb.length() - 1);
+ return sb.toString();
+ }
+ }
+
+ private void flatten(StringBuilder sb, List l) {
+ for (Object o : l) {
+ if (o instanceof List) {
+ flatten(sb, (List) o);
+ }
+ else {
+ sb.append(TypeUtil.toString(o));
+ sb.append('|');
+ }
+ }
+ }
+}
Added: trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2010-07-24 04:27:28 UTC (rev 3468)
@@ -0,0 +1,46 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.List;
+
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.arguments.ArgUtil;
+import org.globus.cog.karajan.arguments.VariableArguments;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.util.TypeUtil;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments;
+import org.griphyn.vdl.mapping.AbsFile;
+
+public class InFileDirs extends AbstractSequentialWithArguments {
+ public static final Arg STAGEINS = new Arg.Positional("stageins");
+
+ static {
+ setArguments(InFileDirs.class, new Arg[] { STAGEINS });
+ }
+
+ @Override
+ protected void post(VariableStack stack) throws ExecutionException {
+ List files = TypeUtil.toList(STAGEINS.getValue(stack));
+ VariableArguments ret = ArgUtil.getVariableReturn(stack);
+ for (Object f : files) {
+ String path = (String) f;
+ String dir = new AbsFile(path).getDir();
+ if (dir.startsWith("/")) {
+ ret.append(dir.substring(1));
+ }
+ else {
+ ret.append(dir);
+ }
+ }
+ super.post(stack);
+ }
+}
Added: trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java 2010-07-24 04:27:28 UTC (rev 3468)
@@ -0,0 +1,46 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.List;
+
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.util.TypeUtil;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.HandleOpenException;
+import org.griphyn.vdl.mapping.Path;
+
+public class IsDone extends VDLFunction {
+ public static final Arg STAGEOUT = new Arg.Positional("stageout");
+
+ static {
+ setArguments(IsDone.class, new Arg[] { STAGEOUT });
+ }
+
+ @Override
+ protected Object function(VariableStack stack) throws ExecutionException,
+ HandleOpenException {
+ List files = TypeUtil.toList(STAGEOUT.getValue(stack));
+ for (Object f : files) {
+ List pv = TypeUtil.toList(f);
+ Path p = Path.parse(TypeUtil.toString(pv.get(0)));
+ DSHandle handle = (DSHandle) pv.get(1);
+ if (!IsLogged.isLogged(stack, handle, p)) {
+ return Boolean.FALSE;
+ }
+ }
+ if (files.isEmpty()) {
+ return Boolean.FALSE;
+ }
+ return Boolean.TRUE;
+ }
+}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java 2010-07-24 04:22:18 UTC (rev 3467)
+++ trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java 2010-07-24 04:27:28 UTC (rev 3468)
@@ -29,16 +29,20 @@
else {
path = Path.parse(TypeUtil.toString(p));
}
- path = var.getPathFromRoot().append(path);
- LogEntry entry = LogEntry.build(var.getRoot().getParam("swift#restartid") + "." + path.stringForm());
- Map map = getLogData(stack);
- boolean found = false;
- synchronized (map) {
- List files = (List) map.get(entry);
- if (files != null && !files.isEmpty()) {
- found = true;
- }
- }
- return Boolean.valueOf(found);
+ return Boolean.valueOf(isLogged(stack, var, path));
}
+
+ public static boolean isLogged(VariableStack stack, DSHandle var, Path path) throws ExecutionException {
+ path = var.getPathFromRoot().append(path);
+ LogEntry entry = LogEntry.build(var.getRoot().getParam("swift#restartid") + "." + path.stringForm());
+ Map map = getLogData(stack);
+ boolean found = false;
+ synchronized (map) {
+ List files = (List) map.get(entry);
+ if (files != null && !files.isEmpty()) {
+ found = true;
+ }
+ }
+ return found;
+ }
}
Added: trunk/src/org/griphyn/vdl/karajan/lib/Mark.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Mark.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Mark.java 2010-07-24 04:27:28 UTC (rev 3468)
@@ -0,0 +1,62 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.List;
+
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.util.TypeUtil;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.HandleOpenException;
+import org.griphyn.vdl.mapping.MappingDependentException;
+import org.griphyn.vdl.mapping.Path;
+
+public class Mark extends VDLFunction {
+ public static final Arg RESTARTS = new Arg.Positional("restarts");
+ public static final Arg ERR = new Arg.Positional("err");
+ public static final Arg MAPPING = new Arg.Optional("mapping", Boolean.FALSE);
+
+ static {
+ setArguments(Mark.class, new Arg[] { RESTARTS, ERR, MAPPING });
+ }
+
+ @Override
+ protected Object function(VariableStack stack) throws ExecutionException,
+ HandleOpenException {
+ try {
+ if (TypeUtil.toBoolean(ERR.getValue(stack))) {
+ boolean mapping = TypeUtil.toBoolean(MAPPING.getValue(stack));
+ List files = TypeUtil.toList(RESTARTS.getValue(stack));
+ for (Object f : files) {
+ List pv = TypeUtil.toList(f);
+ Path p = Path.parse(TypeUtil.toString(pv.get(0)));
+ DSHandle handle = (DSHandle) pv.get(1);
+ DSHandle leaf = handle.getField(p);
+ synchronized (leaf) {
+ if (mapping) {
+ leaf.setValue(new MappingDependentException(leaf, null));
+ }
+ else {
+ leaf.setValue(new DataDependentException(leaf, null));
+ }
+ closeShallow(stack, leaf);
+ }
+ }
+ }
+ }
+ catch (Exception e) {
+ throw new ExecutionException(e);
+ }
+ return null;
+ }
+}
Added: trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java 2010-07-24 04:27:28 UTC (rev 3468)
@@ -0,0 +1,52 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.List;
+
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.arguments.ArgUtil;
+import org.globus.cog.karajan.arguments.VariableArguments;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.util.TypeUtil;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments;
+import org.griphyn.vdl.mapping.AbsFile;
+import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.Path;
+
+public class OutFileDirs extends AbstractSequentialWithArguments {
+ public static final Arg STAGEOUTS = new Arg.Positional("stageouts");
+
+ static {
+ setArguments(OutFileDirs.class, new Arg[] { STAGEOUTS });
+ }
+
+ @Override
+ protected void post(VariableStack stack) throws ExecutionException {
+ List files = TypeUtil.toList(STAGEOUTS.getValue(stack));
+ VariableArguments ret = ArgUtil.getVariableReturn(stack);
+ try {
+ for (Object f : files) {
+ List pv = TypeUtil.toList(f);
+ Path p = Path.parse(TypeUtil.toString(pv.get(0)));
+ DSHandle handle = (DSHandle) pv.get(1);
+ DSHandle leaf = handle.getField(p);
+ String fname = VDLFunction.filename(leaf)[0];
+ String dir = new AbsFile(fname).getDir();
+ ret.append(dir);
+ }
+ }
+ catch (Exception e) {
+ throw new ExecutionException(e);
+ }
+ super.post(stack);
+ }
+}
Added: trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java 2010-07-24 04:27:28 UTC (rev 3468)
@@ -0,0 +1,51 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.List;
+
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.arguments.ArgUtil;
+import org.globus.cog.karajan.arguments.VariableArguments;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.util.TypeUtil;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments;
+import org.griphyn.vdl.mapping.AbsFile;
+import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.Path;
+
+public class OutFiles extends AbstractSequentialWithArguments {
+ public static final Arg STAGEOUTS = new Arg.Positional("stageouts");
+
+ static {
+ setArguments(OutFiles.class, new Arg[] { STAGEOUTS });
+ }
+
+ @Override
+ protected void post(VariableStack stack) throws ExecutionException {
+ List files = TypeUtil.toList(STAGEOUTS.getValue(stack));
+ VariableArguments ret = ArgUtil.getVariableReturn(stack);
+ try {
+ for (Object f : files) {
+ List pv = TypeUtil.toList(f);
+ Path p = Path.parse(TypeUtil.toString(pv.get(0)));
+ DSHandle handle = (DSHandle) pv.get(1);
+ DSHandle leaf = handle.getField(p);
+ String fname = VDLFunction.filename(leaf)[0];
+ ret.append(fname);
+ }
+ }
+ catch (Exception e) {
+ throw new ExecutionException(e);
+ }
+ super.post(stack);
+ }
+}
Added: trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2010-07-24 04:27:28 UTC (rev 3468)
@@ -0,0 +1,60 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import org.apache.log4j.Logger;
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments;
+import org.griphyn.vdl.karajan.functions.ConfigProperty;
+
+public class Parameterlog extends AbstractSequentialWithArguments {
+ public static final Logger logger = Logger.getLogger(Parameterlog.class);
+
+ public static final Arg DIRECTION = new Arg.Positional("direction");
+ public static final Arg VAR = new Arg.Positional("variable");
+ public static final Arg ID = new Arg.Positional("id");
+ public static final Arg THREAD = new Arg.Positional("thread");
+
+ static {
+ setArguments(Parameterlog.class, new Arg[] { DIRECTION, VAR, ID, THREAD });
+ }
+
+ private Boolean enabled;
+
+ @Override
+ public void pre(VariableStack stack) throws ExecutionException {
+ if (enabled == null) {
+ enabled = "true".equals(ConfigProperty.getProperty("provenance.log", true, stack));
+ }
+ }
+
+
+
+ @Override
+ protected void executeChildren(VariableStack stack) throws ExecutionException {
+ if (enabled) {
+ super.executeChildren(stack);
+ }
+ else {
+ complete(stack);
+ }
+ }
+
+ @Override
+ protected void post(VariableStack stack) throws ExecutionException {
+ if (enabled) {
+ logger.info("PARAM thread=" + THREAD.getValue(stack) + " direction="
+ + DIRECTION.getValue(stack) + " variable=" + VAR.getValue(stack)
+ + " provenanceid=" + ID.getValue(stack));
+ }
+ }
+}
More information about the Swift-commit
mailing list