From noreply at svn.ci.uchicago.edu Mon Mar 1 14:17:04 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 1 Mar 2010 14:17:04 -0600 (CST) Subject: [Swift-commit] r3256 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100301201704.08DF39CC9C@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-01 14:17:03 -0600 (Mon, 01 Mar 2010) New Revision: 3256 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: Whitespace corrections. Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-02-26 04:24:59 UTC (rev 3255) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-03-01 20:17:03 UTC (rev 3256) @@ -58,121 +58,122 @@ return null; } - /** - Formatted trace output.
- Example: tracef("\t%s\n", "hello");
- Differences from trace(): 1) respects \t, \n and \\; - 2) allows for typechecked format specifiers; 3) allows for - consumption of variables without display (%k); 4) does not - impose any formatting (commas, etc.).

- Format specifiers:
+ /** + Formatted trace output.
+ Example: tracef("\t%s\n", "hello");
+ Differences from trace(): + 1) respects \t, \n and \\; + 2) allows for typechecked format specifiers; + 3) allows for consumption of variables without display (%k); + 4) does not impose any formatting (commas, etc.).

+ Format specifiers:
%%: % sign.
%p: Not typechecked, output as in trace().
%i: Typechecked int output.
%s: Typechecked string output.
%k: Variable sKipped, no output. - */ - public DSHandle swiftscript_tracef(VariableStack stack) throws ExecutionException, NoSuchTypeException, - InvalidPathException { + */ + public DSHandle swiftscript_tracef(VariableStack stack) + throws ExecutionException, NoSuchTypeException, + InvalidPathException { + DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); - DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); - StringBuffer buf = new StringBuffer(); for (int i = 0; i < args.length; i++) { DSHandle handle = args[i]; VDLFunction.waitFor(stack, handle); } - String msg = format(args); - buf.append(msg); - traceLogger.warn(buf); + String msg = format(args); + buf.append(msg); + traceLogger.warn(buf); return null; } - /** - Helper for {@link #swiftscript_tracef}. - */ - private String format(DSHandle[] args) throws ExecutionException { + /** + Helper for {@link #swiftscript_tracef}. + */ + private String format(DSHandle[] args) throws ExecutionException { if (! (args[0].getType() == Types.STRING)) - throw new ExecutionException("First argument to tracef() must be a string!"); - + throw new ExecutionException("First argument to tracef() must be a string!"); + String spec = args[0].toString(); StringBuffer output = new StringBuffer(); int i = 0; int a = 1; while (i < spec.length()) { - char c = spec.charAt(i); - if (c == '%') { - char d = spec.charAt(++i); - a = append(d, a, args, output); - } - else if (c == '\\') { - char d = spec.charAt(++i); - escape(d, output); - } - else { - output.append(c); - } - i++; + char c = spec.charAt(i); + if (c == '%') { + char d = spec.charAt(++i); + a = append(d, a, args, output); + } + else if (c == '\\') { + char d = spec.charAt(++i); + escape(d, output); + } + else { + output.append(c); + } + i++; } String result = output.toString(); return result; } - - /** - Helper for {@link #swiftscript_tracef}. - */ - private int append(char c, int arg, DSHandle[] args, StringBuffer output) throws ExecutionException { - if (c == '%') { - output.append('%'); - return arg; - } - if (arg >= args.length) { - throw new ExecutionException("tracef(): too many specifiers!"); - } + + /** + Helper for {@link #swiftscript_tracef}. + */ + private int append(char c, int arg, DSHandle[] args, StringBuffer output) throws ExecutionException { + if (c == '%') { + output.append('%'); + return arg; + } + if (arg >= args.length) { + throw new ExecutionException("tracef(): too many specifiers!"); + } if (c == 'p') { - output.append(args[arg].toString()); - } - else if (c == 's') { - if (args[arg].getType() == Types.STRING) { - output.append(args[arg]).toString(); - } - else { - throw new ExecutionException("tracef(): %s requires a string!"); - } - } + output.append(args[arg].toString()); + } + else if (c == 's') { + if (args[arg].getType() == Types.STRING) { + output.append(args[arg]).toString(); + } + else { + throw new ExecutionException("tracef(): %s requires a string!"); + } + } else if (c == 'i') { - if (args[arg].getType() == Types.INT) { - output.append(args[arg]).toString(); - } - else { - throw new ExecutionException("tracef(): %i requires an int!"); - } - } - else if (c == 'k') { - ; - } + if (args[arg].getType() == Types.INT) { + output.append(args[arg]).toString(); + } + else { + throw new ExecutionException("tracef(): %i requires an int!"); + } + } + else if (c == 'k') { + ; + } else { - throw new ExecutionException("tracef(): Unknown format: %" + c); - } + throw new ExecutionException("tracef(): Unknown format: %" + c); + } return arg+1; } - - /** - Helper for {@link #swiftscript_tracef}. - */ - private void escape(char c, StringBuffer output) throws ExecutionException { - if (c == '\\') { - output.append('\\'); - } - else if (c == 'n') { - output.append('\n'); - } - else if (c == 't') { - output.append('\t'); - } - else { - throw new ExecutionException("tracef(): unknown backslash escape sequence!"); - } + + /** + Helper for {@link #swiftscript_tracef}. + */ + private void escape(char c, StringBuffer output) throws ExecutionException { + if (c == '\\') { + output.append('\\'); + } + else if (c == 'n') { + output.append('\n'); + } + else if (c == 't') { + output.append('\t'); + } + else { + throw new ExecutionException("tracef(): unknown backslash escape sequence!"); + } } public DSHandle swiftscript_strcat(VariableStack stack) throws ExecutionException, NoSuchTypeException, From noreply at svn.ci.uchicago.edu Mon Mar 1 14:22:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 1 Mar 2010 14:22:24 -0600 (CST) Subject: [Swift-commit] r3257 - in trunk: libexec src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100301202224.689449CC9C@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-01 14:22:24 -0600 (Mon, 01 Mar 2010) New Revision: 3257 Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java Modified: trunk/libexec/vdl-lib.xml trunk/src/org/griphyn/vdl/engine/Karajan.java trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java Log: New @java(). Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-03-01 20:17:03 UTC (rev 3256) +++ trunk/libexec/vdl-lib.xml 2010-03-01 20:22:24 UTC (rev 3257) @@ -13,6 +13,7 @@ + Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-03-01 20:17:03 UTC (rev 3256) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-03-01 20:22:24 UTC (rev 3257) @@ -430,7 +430,8 @@ StringTemplate assignST = template("assign"); StringTemplate varST = expressionToKarajan(assign.getAbstractExpressionArray(0),scope); StringTemplate valueST = expressionToKarajan(assign.getAbstractExpressionArray(1),scope); - if (!datatype(varST).equals(datatype(valueST))) + if (! (datatype(varST).equals(datatype(valueST)) || + datatype(valueST).equals("java"))) throw new CompilationException("You cannot assign value of type " + datatype(valueST) + " to a variable of type " + datatype(varST)); assignST.setAttribute("var", varST); @@ -1395,7 +1396,13 @@ } String datatype(StringTemplate st) { - return st.getAttribute("datatype").toString(); + String result = null; + try { + result = st.getAttribute("datatype").toString(); + } + catch (Exception e) { + throw new RuntimeException("Not typed properly: " + st); + } + return result; } - } Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-03-01 20:17:03 UTC (rev 3256) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-03-01 20:22:24 UTC (rev 3257) @@ -139,10 +139,10 @@ proceduresMap.put("trace", trace); ProcedureSignature tracef = new ProcedureSignature("tracef"); - trace.setAnyNumOfInputArgs(); - trace.setInvocationMode(INVOCATION_INTERNAL); - proceduresMap.put("tracef", trace); - + tracef.setAnyNumOfInputArgs(); + tracef.setInvocationMode(INVOCATION_INTERNAL); + proceduresMap.put("tracef", tracef); + ProcedureSignature writeData = new ProcedureSignature("writeData"); FormalArgumentSignature wdInputArg = new FormalArgumentSignature(true); writeData.addInputArg(wdInputArg); @@ -230,6 +230,12 @@ toint.addOutputArg(toOut1); functionsMap.put(toint.getName(), toint); + ProcedureSignature java = new ProcedureSignature("java"); + java.setAnyNumOfInputArgs(); + FormalArgumentSignature output = new FormalArgumentSignature("java"); + java.addOutputArg(output); + functionsMap.put(java.getName(), java); + return functionsMap; } Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java 2010-03-01 20:22:24 UTC (rev 3257) @@ -0,0 +1,147 @@ +package org.griphyn.vdl.karajan.lib.swiftscript; + +import java.lang.reflect.Method; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.karajan.lib.SwiftArg; +import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.type.NoSuchTypeException; +import org.griphyn.vdl.type.Type; +import org.griphyn.vdl.type.Types; + +public class Java extends VDLFunction { + + static { + setArguments(Java.class, new Arg[] { Arg.VARGS }); + } + + protected Object function(VariableStack stack) throws ExecutionException, + HandleOpenException { + DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); + + for (int i = 0; i < args.length; i++) { + DSHandle handle = args[i]; + VDLFunction.waitFor(stack, handle); + } + + Method method = getMethod(args); + Object[] p = convertInputs(method, args); + Type type = returnType(method); + Object value = invoke(method, p); + DSHandle result = swiftResult(type, value); + + return result; + } + + /** + Given the user args, locate the Java Method. + */ + Method getMethod(DSHandle[] args) { + Method result = null; + Class clazz; + + String lib = "unset"; + String name = "unset"; + + if (args.length < 2) + throw new RuntimeException + ("@java() requires at least two arguments"); + + try { + lib = args[0].toString(); + name = args[1].toString(); + clazz = Class.forName(lib); + Method[] methods = clazz.getMethods(); + result = null; + for (Method m : methods) { + if (m.getName().equals(name)) { + result = m; + break; + } + } + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException + ("@java(): Error attempting to use: " + args[0]); + } + + if (result == null) + throw new RuntimeException + ("No method: " + name + " in " + lib); + return result; + } + + /** + Convert the user args to a Java Object array. + */ + Object[] convertInputs(Method method, DSHandle[] args) { + Object[] result = new Object[args.length-2]; + Object a = null; + try { + for (int i = 2; i < args.length; i++) { + Type t = args[i].getType(); + Object v = args[i].getValue(); + if (t.equals(Types.FLOAT)) + a = (Double) v; + else if (t.equals(Types.INT)) + a = (Integer) v; + else if (t.equals(Types.BOOLEAN)) + a = (Boolean) v; + else if (t.equals(Types.STRING)) + a = (String) v; + result[i-2] = a; + } + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException + ("Error converting input arguments: \n" + + " to: " + method.getDeclaringClass() + + "." + method + " \n argument: " + a); + } + return result; + } + + Type returnType(Method method) { + Type result = null; + + Class rt = method.getReturnType(); + if (rt.equals(Double.TYPE)) + result = Types.FLOAT; + else if (rt.equals(Integer.TYPE)) + result = Types.INT; + else if (rt.equals(Boolean.TYPE)) + result = Types.BOOLEAN; + else if (rt.equals(String.class)) + result = Types.STRING; + return result; + } + + Object invoke(Method method, Object[] p) { + Object result = null; + try { + result = method.invoke(null, p); + } + catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException + ("Error attempting to invoke: " + method.getDeclaringClass() + + "." + method); + } + return result; + } + + DSHandle swiftResult(Type type, Object value) { + DSHandle result = new RootDataNode(type); + result.setValue(value); + result.closeShallow(); + return result; + } +} From noreply at svn.ci.uchicago.edu Mon Mar 1 14:59:19 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 1 Mar 2010 14:59:19 -0600 (CST) Subject: [Swift-commit] r3258 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100301205919.6869E9CC8B@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-03-01 14:59:19 -0600 (Mon, 01 Mar 2010) New Revision: 3258 Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java Log: that breaks with root data Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2010-03-01 20:22:24 UTC (rev 3257) +++ trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2010-03-01 20:59:19 UTC (rev 3258) @@ -47,7 +47,7 @@ } deepCopy(leaf, value, stack); } - if (var.getParent().getType().isArray()) { + if (var.getParent() != null && var.getParent().getType().isArray()) { markAsAvailable(stack, leaf.getParent(), leaf.getPathFromRoot().getLast()); } } From noreply at svn.ci.uchicago.edu Thu Mar 11 15:19:33 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 11 Mar 2010 15:19:33 -0600 (CST) Subject: [Swift-commit] r3259 - in trunk/src/org/griphyn/vdl/karajan/lib: . swiftscript Message-ID: <20100311211933.878DA9CC9B@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-11 15:19:33 -0600 (Thu, 11 Mar 2010) New Revision: 3259 Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: New %M specifier for tracef(). Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-03-01 20:59:19 UTC (rev 3258) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-03-11 21:19:33 UTC (rev 3259) @@ -191,7 +191,7 @@ } /** The caller is expected to have synchronized on the root of var. */ - public String[] filename(DSHandle var) throws ExecutionException, HandleOpenException { + public static String[] filename(DSHandle var) throws ExecutionException, HandleOpenException { assert Thread.holdsLock(var.getRoot()); try { if (var.getType().isArray()) { @@ -207,7 +207,7 @@ } } - private String[] leavesFileNames(DSHandle var) throws ExecutionException, HandleOpenException { + private static String[] leavesFileNames(DSHandle var) throws ExecutionException, HandleOpenException { List l = new ArrayList(); Iterator i; try { @@ -278,7 +278,7 @@ } } - private String leafFileName(DSHandle var) throws ExecutionException { + private static String leafFileName(DSHandle var) throws ExecutionException { if (Types.STRING.equals(var.getType())) { return relativize(String.valueOf(var.getValue())); } @@ -311,7 +311,7 @@ } } - protected String pathOnly(String file) { + protected static String pathOnly(String file) { return new AbsFile(file).getPath(); } @@ -347,7 +347,7 @@ * removes leading / character from a supplied filename if present, so that * the path can be used as a relative path. */ - public String relativize(String name) { + public static String relativize(String name) { name = pathOnly(name); if (name != null && name.length() > 0 && name.charAt(0) == '/') { return name.substring(1); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-03-01 20:59:19 UTC (rev 3258) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-03-11 21:19:33 UTC (rev 3259) @@ -35,6 +35,7 @@ setArguments("swiftscript_tracef", new Arg[] { Arg.VARGS }); setArguments("swiftscript_strcat", new Arg[] { Arg.VARGS }); setArguments("swiftscript_strcut", new Arg[] { PA_INPUT, PA_PATTERN }); + setArguments("swiftscript_strstr", new Arg[] { PA_INPUT, PA_PATTERN }); setArguments("swiftscript_strsplit", new Arg[] { PA_INPUT, PA_PATTERN }); setArguments("swiftscript_regexp", new Arg[] { PA_INPUT, PA_PATTERN, PA_TRANSFORM }); setArguments("swiftscript_toint", new Arg[] { PA_INPUT }); @@ -133,6 +134,20 @@ if (c == 'p') { output.append(args[arg].toString()); } + else if (c == 'M') { + try { + synchronized (args[arg].getRoot()) { + String[] names = VDLFunction.filename(args[arg]); + if (names.length > 1) + output.append(names); + else + output.append(names[0]); + } + } + catch (Exception e) { + throw new ExecutionException("tracef(%M): Could not lookup: " + args[arg]); + } + } else if (c == 's') { if (args[arg].getType() == Types.STRING) { output.append(args[arg]).toString(); @@ -235,6 +250,22 @@ return handle; } + public DSHandle swiftscript_strstr(VariableStack stack) throws ExecutionException, NoSuchTypeException, + InvalidPathException { + + String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); + String pattern = TypeUtil.toString(PA_PATTERN.getValue(stack)); + if (logger.isDebugEnabled()) { + logger.debug("strstr will search '" + inputString + + "' for pattern '" + pattern + "'"); + } + int result = inputString.indexOf(pattern); + DSHandle handle = new RootDataNode(Types.INT); + handle.setValue(new Double(result)); + handle.closeShallow(); + return handle; + } + public DSHandle swiftscript_strsplit(VariableStack stack) throws ExecutionException, NoSuchTypeException, InvalidPathException { String str = TypeUtil.toString(PA_INPUT.getValue(stack)); From noreply at svn.ci.uchicago.edu Thu Mar 11 16:14:07 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 11 Mar 2010 16:14:07 -0600 (CST) Subject: [Swift-commit] r3260 - in trunk: etc libexec Message-ID: <20100311221407.DC2DB9CC86@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-11 16:14:07 -0600 (Thu, 11 Mar 2010) New Revision: 3260 Added: trunk/libexec/cdm_cleanup.sh trunk/libexec/cdm_lib.sh Modified: trunk/etc/log4j.properties Log: New CDM features. Modified: trunk/etc/log4j.properties =================================================================== --- trunk/etc/log4j.properties 2010-03-11 21:19:33 UTC (rev 3259) +++ trunk/etc/log4j.properties 2010-03-11 22:14:07 UTC (rev 3260) @@ -29,3 +29,4 @@ log4j.logger.org.globus.cog.abstraction.coaster.rlog=INFO log4j.logger.org.globus.swift.data.Director=DEBUG +log4j.logger.org.griphyn.vdl.karajan.lib=INFO Added: trunk/libexec/cdm_cleanup.sh =================================================================== --- trunk/libexec/cdm_cleanup.sh (rev 0) +++ trunk/libexec/cdm_cleanup.sh 2010-03-11 22:14:07 UTC (rev 3260) @@ -0,0 +1,47 @@ + +# Used by vdl-int.k after workflow completion +# Flushes remaining GATHER output by using cdm_lib + +# Must define some _swiftwrap functions so they can be called by cdm_lib + +logstate() { + log "Progress " `date +"%Y-%m-%d %H:%M:%S.%N%z"` ${*} +} + +log() { + echo ${*} +} + +checkError() { + if [ "$?" != "0" ]; then + fail $@ + fi +} + +fail() { + EC=$1 + shift + log ${*} + exit $EC +} + +{ + set -x + GATHER_DIR=$1 + GATHER_TARGET=$2 + ID=cleanup-$3 + + if [ "X$GATHER_DIR" == "X" ]; then + fail 254 "Not specified: GATHER_DIR" + fi + + DIR=$( dirname $0 ) + source ${DIR}/cdm_lib.sh + + cdm_gather_cleanup +} >> /sandbox/cdm_cleanup.txt + +# Local Variables: +# mode: sh +# sh-basic-offset: 8 +# End: Property changes on: trunk/libexec/cdm_cleanup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/libexec/cdm_lib.sh =================================================================== --- trunk/libexec/cdm_lib.sh (rev 0) +++ trunk/libexec/cdm_lib.sh 2010-03-11 22:14:07 UTC (rev 3260) @@ -0,0 +1,134 @@ + +# Source this for CDM shell functions + +# Setup GATHER_DIR and some variables +cdm_gather_setup() { + GATHER_DIR=$1 + + logstate "GATHER_DIR $GATHER_DIR" + mkdir -p $GATHER_DIR + checkError 254 "Could not create: $GATHER_DIR" + + GATHER_LOCKFILE=$GATHER_DIR/.cdm.lock + GATHER_MY_FILE=$GATHER_DIR/.cdm-$ID.lock + GATHER_MY_OUTBOX=$GATHER_DIR/.cdm-outgoing-$ID +} + +# Acquire the GATHER_LOCKFILE +cdm_gather_lock_acquire() { + TRYING=1 + COUNT=0 + + touch $GATHER_MY_FILE + checkError 254 "Could not touch my file: $GATHER_MY_FILE" + logstate "LOCK_ACQUIRE $GATHER_MY_FILE" + while (( TRYING )); do + ln $GATHER_MY_FILE $GATHER_LOCKFILE + CODE=$? + TRYING=$CODE + if (( TRYING )); then + logstate "LOCK_DELAY $GATHER_MY_FILE : $CODE" + if (( COUNT++ > 10 )); then + fail 254 "Could not acquire lock!" + exit 1 + fi + sleep $(( RANDOM % 10 )) + fi + done +} + +# Move files from JOBDIR to GATHER_DIR +cdm_gather_import() { + pushd jobs/$JOBDIR/$ID + logstate "GATHER_IMPORT $GATHER_OUTPUT" + mv $GATHER_OUTPUT $GATHER_DIR + checkError 254 "Could not move output to $GATHER_DIR" + popd +} + +# Move files from GATHER_DIR to OUTBOX +# Note that this function modifies $IFS +cdm_gather_export() { + RESULT=1 + pushd ${GATHER_DIR} + IFS=" +" + FILES=( $( ls -A -I ".cdm*.lock" -I ".cdm-outgoing-*" ) ) + log "FILES: ${#FILES[@]}" + mkdir -p $GATHER_MY_OUTBOX + checkError 254 "Could not mkdir ${GATHER_MY_OUTBOX}" + if (( ${#FILES} > 0 )); then + mv ${FILES[@]} $GATHER_MY_OUTBOX + checkError 254 "Could not move ${FILES[@]} to $GATHER_MY_OUTBOX" + RESULT=0 + fi + popd + return $RESULT +} + +# Release the GATHER_LOCKFILE +cdm_gather_lock_release() { + logstate "LOCK_RELEASE $GATHER_LOCKFILE" + unlink $GATHER_LOCKFILE +} + +# Move files from (LFS) OUTBOX to (GFS) GATHER_TARGET +cdm_gather_flush() { + pushd ${GATHER_MY_OUTBOX} + logstate "GATHER_TARGET $GATHER_TARGET" + mkdir -p ${GATHER_TARGET} + logstate "GATHER_FLUSH_START ${GATHER_TARGET}/cdm-gather-$ID.tar" + TARBALL=${GATHER_TARGET}/cdm-gather-$ID.tar + tar cf ${TARBALL} * + checkError 254 "CDM[GATHER]: error writing: ${TARBALL}" + logstate "GATHER_FLUSH_DONE" + popd +} + +# Called by _swiftwrap at the end of each job +cdm_gather_action() { + GATHER_OUTPUT=${*} + + GATHER_DIR=$( perl shared/cdm.pl property GATHER_DIR < $CDM_FILE ) + GATHER_MAX=$( perl shared/cdm.pl property GATHER_LIMIT < $CDM_FILE ) + GATHER_TARGET=$( perl shared/cdm.pl property GATHER_TARGET < $CDM_FILE ) + + cdm_gather_setup $GATHER_DIR + cdm_gather_lock_acquire + + cdm_gather_import + + USAGE=$( du -s --exclude=".cdm*" -B 1 $GATHER_DIR ) + USAGE=${USAGE% *} # Chop off filename in output + + logstate "USAGE_CHECK $USAGE / $GATHER_MAX" + + FLUSH="no" + if (( USAGE > GATHER_MAX )); then + FLUSH="yes" + cdm_gather_export + fi + + cdm_gather_lock_release + if [ $FLUSH == "yes" ]; then + cdm_gather_flush + fi +} + +# Called by cdm_cleanup.sh at the end of the workflow +cdm_gather_cleanup() { + declare -p PWD DIR GATHER_DIR GATHER_TARGET ID + cdm_gather_setup $GATHER_DIR + cdm_gather_lock_acquire + cdm_gather_export + EXPORT_RESULT=$? + cdm_gather_lock_release + if (( EXPORT_RESULT == 0)); then + cdm_gather_flush + fi +} + +# Local Variables: +# mode: sh +# sh-basic-offset: 8 +# End: From noreply at svn.ci.uchicago.edu Thu Mar 11 16:37:05 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 11 Mar 2010 16:37:05 -0600 (CST) Subject: [Swift-commit] r3261 - trunk/libexec Message-ID: <20100311223705.9BD649CC86@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-11 16:37:05 -0600 (Thu, 11 Mar 2010) New Revision: 3261 Modified: trunk/libexec/_swiftwrap Log: CDM corrections for DIRECT files inside directories. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-03-11 22:14:07 UTC (rev 3260) +++ trunk/libexec/_swiftwrap 2010-03-11 22:37:05 UTC (rev 3261) @@ -158,6 +158,8 @@ ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!" elif [ $MODE == "OUTPUT" ]; then + mkdir -p $DIRECT_DIR + checkError 254 "CDM[DIRECT]: mkdir -p $DIRECT_DIR failed!" touch $DIRECT_DIR/$FILE checkError 254 "CDM[DIRECT]: Touching $DIRECT_DIR/$FILE failed!" ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE From noreply at svn.ci.uchicago.edu Thu Mar 18 11:15:19 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 18 Mar 2010 11:15:19 -0500 (CDT) Subject: [Swift-commit] r3262 - in trunk: libexec src/org/griphyn/vdl/engine Message-ID: <20100318161519.5011C9CD21@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-18 11:15:19 -0500 (Thu, 18 Mar 2010) New Revision: 3262 Modified: trunk/libexec/vdl-lib.xml trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java Log: Turn on @strstr(). Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-03-11 22:37:05 UTC (rev 3261) +++ trunk/libexec/vdl-lib.xml 2010-03-18 16:15:19 UTC (rev 3262) @@ -8,6 +8,7 @@ + Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-03-11 22:37:05 UTC (rev 3261) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-03-18 16:15:19 UTC (rev 3262) @@ -213,15 +213,24 @@ FormalArgumentSignature strcutOut1 = new FormalArgumentSignature("string"); strcut.addOutputArg(strcutOut1); functionsMap.put(strcut.getName(), strcut); - + + ProcedureSignature strstr = new ProcedureSignature("strstr"); + FormalArgumentSignature strstrIn1 = new FormalArgumentSignature("string"); + strstr.addInputArg(strstrIn1); + FormalArgumentSignature strstrIn2 = new FormalArgumentSignature("string"); + strstr.addInputArg(strstrIn2); + FormalArgumentSignature strstrOut1 = new FormalArgumentSignature("int"); + strstr.addOutputArg(strstrOut1); + functionsMap.put(strstr.getName(), strstr); + ProcedureSignature strsplit = new ProcedureSignature("strsplit"); - FormalArgumentSignature strsplitIn1 = new FormalArgumentSignature("string"); - strsplit.addInputArg(strsplitIn1); - FormalArgumentSignature strsplitIn2 = new FormalArgumentSignature("string"); - strsplit.addInputArg(strsplitIn2); - FormalArgumentSignature strsplitOut1 = new FormalArgumentSignature("string[]"); - strsplit.addOutputArg(strsplitOut1); - functionsMap.put(strsplit.getName(), strsplit); + FormalArgumentSignature strsplitIn1 = new FormalArgumentSignature("string"); + strsplit.addInputArg(strsplitIn1); + FormalArgumentSignature strsplitIn2 = new FormalArgumentSignature("string"); + strsplit.addInputArg(strsplitIn2); + FormalArgumentSignature strsplitOut1 = new FormalArgumentSignature("string[]"); + strsplit.addOutputArg(strsplitOut1); + functionsMap.put(strsplit.getName(), strsplit); ProcedureSignature toint = new ProcedureSignature("toint"); FormalArgumentSignature tointIn1 = new FormalArgumentSignature("string"); From noreply at svn.ci.uchicago.edu Thu Mar 18 13:21:11 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 18 Mar 2010 13:21:11 -0500 (CDT) Subject: [Swift-commit] r3263 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100318182111.EB9F59CCA8@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-18 13:21:11 -0500 (Thu, 18 Mar 2010) New Revision: 3263 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: New tracef() %q specifier for arrays. Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-03-18 16:15:19 UTC (rev 3262) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-03-18 18:21:11 UTC (rev 3263) @@ -2,6 +2,9 @@ import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -14,8 +17,10 @@ import org.griphyn.vdl.karajan.lib.SwiftArg; import org.griphyn.vdl.karajan.lib.VDLFunction; import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.RootArrayDataNode; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.NoSuchTypeException; @@ -164,6 +169,31 @@ throw new ExecutionException("tracef(): %i requires an int!"); } } + else if (c == 'q') { + if (args[arg] instanceof ArrayDataNode) { + ArrayDataNode node = (ArrayDataNode) args[arg]; + output.append("["); + try { + Stack stack = new Stack(); + Collection children = args[arg].getFields(Path.CHILDREN); + for (Object o : children) + stack.push(o); + while (!stack.empty()) { + DSHandle child = (DSHandle) stack.pop(); + output.append(child); + if (!stack.empty()) + output.append(","); + } + } + catch (Exception e) { + throw new ExecutionException("trace(%q): Could not get children of: " + args[arg]); + } + output.append("]"); + } + else { + throw new ExecutionException("tracef(): %q requires an array!"); + } + } else if (c == 'k') { ; } From noreply at svn.ci.uchicago.edu Thu Mar 18 14:05:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 18 Mar 2010 14:05:32 -0500 (CDT) Subject: [Swift-commit] r3264 - in trunk: libexec src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100318190532.B85109CCA8@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-18 14:05:32 -0500 (Thu, 18 Mar 2010) New Revision: 3264 Modified: trunk/libexec/vdl-lib.xml trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: New @tostring(). Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-03-18 18:21:11 UTC (rev 3263) +++ trunk/libexec/vdl-lib.xml 2010-03-18 19:05:32 UTC (rev 3264) @@ -12,6 +12,7 @@ + Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-03-18 18:21:11 UTC (rev 3263) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-03-18 19:05:32 UTC (rev 3264) @@ -214,7 +214,7 @@ strcut.addOutputArg(strcutOut1); functionsMap.put(strcut.getName(), strcut); - ProcedureSignature strstr = new ProcedureSignature("strstr"); + ProcedureSignature strstr = new ProcedureSignature("strstr"); FormalArgumentSignature strstrIn1 = new FormalArgumentSignature("string"); strstr.addInputArg(strstrIn1); FormalArgumentSignature strstrIn2 = new FormalArgumentSignature("string"); @@ -224,13 +224,13 @@ functionsMap.put(strstr.getName(), strstr); ProcedureSignature strsplit = new ProcedureSignature("strsplit"); - FormalArgumentSignature strsplitIn1 = new FormalArgumentSignature("string"); - strsplit.addInputArg(strsplitIn1); - FormalArgumentSignature strsplitIn2 = new FormalArgumentSignature("string"); - strsplit.addInputArg(strsplitIn2); - FormalArgumentSignature strsplitOut1 = new FormalArgumentSignature("string[]"); - strsplit.addOutputArg(strsplitOut1); - functionsMap.put(strsplit.getName(), strsplit); + FormalArgumentSignature strsplitIn1 = new FormalArgumentSignature("string"); + strsplit.addInputArg(strsplitIn1); + FormalArgumentSignature strsplitIn2 = new FormalArgumentSignature("string"); + strsplit.addInputArg(strsplitIn2); + FormalArgumentSignature strsplitOut1 = new FormalArgumentSignature("string[]"); + strsplit.addOutputArg(strsplitOut1); + functionsMap.put(strsplit.getName(), strsplit); ProcedureSignature toint = new ProcedureSignature("toint"); FormalArgumentSignature tointIn1 = new FormalArgumentSignature("string"); @@ -239,6 +239,13 @@ toint.addOutputArg(toOut1); functionsMap.put(toint.getName(), toint); + ProcedureSignature tostring = new ProcedureSignature("tostring"); + FormalArgumentSignature tostringIn1 = new FormalArgumentSignature(true); + tostring.addInputArg(tostringIn1); + FormalArgumentSignature tostringOut1 = new FormalArgumentSignature("string"); + tostring.addOutputArg(tostringOut1); + functionsMap.put(tostring.getName(), tostring); + ProcedureSignature java = new ProcedureSignature("java"); java.setAnyNumOfInputArgs(); FormalArgumentSignature output = new FormalArgumentSignature("java"); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-03-18 18:21:11 UTC (rev 3263) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-03-18 19:05:32 UTC (rev 3264) @@ -2,9 +2,6 @@ import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -44,6 +41,7 @@ setArguments("swiftscript_strsplit", new Arg[] { PA_INPUT, PA_PATTERN }); setArguments("swiftscript_regexp", new Arg[] { PA_INPUT, PA_PATTERN, PA_TRANSFORM }); setArguments("swiftscript_toint", new Arg[] { PA_INPUT }); + setArguments("swiftscript_tostring", new Arg[] { PA_INPUT }); } private static final Logger traceLogger = Logger.getLogger("org.globus.swift.trace"); @@ -174,20 +172,19 @@ ArrayDataNode node = (ArrayDataNode) args[arg]; output.append("["); try { - Stack stack = new Stack(); - Collection children = args[arg].getFields(Path.CHILDREN); - for (Object o : children) - stack.push(o); - while (!stack.empty()) { - DSHandle child = (DSHandle) stack.pop(); - output.append(child); - if (!stack.empty()) - output.append(","); - } + int size = node.size(); + for (int i = 0; i < size; i++) { + String entry = ""+i; + DSHandle handle = node.getField(Path.parse(entry)); + output.append(handle); + if (i < size-1) + output.append(","); + } + } + catch (Exception e) { + e.printStackTrace(); + throw new ExecutionException("trace(%q): Could not get children of: " + args[arg]); } - catch (Exception e) { - throw new ExecutionException("trace(%q): Could not get children of: " + args[arg]); - } output.append("]"); } else { @@ -364,4 +361,13 @@ VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "string"); return handle; } + + public DSHandle swiftscript_tostring(VariableStack stack) throws ExecutionException, NoSuchTypeException, + InvalidPathException { + Object input = PA_INPUT.getValue(stack); + DSHandle handle = new RootDataNode(Types.STRING); + handle.setValue(""+input); + handle.closeShallow(); + return handle; + } } From noreply at svn.ci.uchicago.edu Wed Mar 24 16:58:27 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Mar 2010 16:58:27 -0500 (CDT) Subject: [Swift-commit] r3265 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20100324215827.8F4639CC90@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-03-24 16:58:27 -0500 (Wed, 24 Mar 2010) New Revision: 3265 Modified: trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java Log: noauto parameter Modified: trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2010-03-18 19:05:32 UTC (rev 3264) +++ trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2010-03-24 21:58:27 UTC (rev 3265) @@ -12,6 +12,7 @@ import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractMapper; +import org.griphyn.vdl.mapping.InvalidMappingParameterException; import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; @@ -36,6 +37,8 @@ * this string. If suffix does not begin with a '.' * character, then a '.' will be added automatically to * separate the rest of the filename from the suffix + *
  • noauto - if specified as "true", then the suffix auto addition of a'.' + * will be disabled. Default value is "false".
  • *
  • pattern - if specified, then filenames will be selected from * the location directory when they match the unix glob * pattern supplied in this parameter.
  • @@ -47,6 +50,7 @@ public static final MappingParam PARAM_SUFFIX = new MappingParam("suffix", null); public static final MappingParam PARAM_PATTERN = new MappingParam("pattern", null); public static final MappingParam PARAM_LOCATION = new MappingParam("location", null); + public static final MappingParam PARAM_NOAUTO = new MappingParam("noauto", "false"); public static final Logger logger = Logger.getLogger(AbstractFileMapper.class); @@ -79,7 +83,12 @@ super.setParams(params); if (PARAM_SUFFIX.isPresent(this)) { String suffix = PARAM_SUFFIX.getStringValue(this); - if (!suffix.startsWith(".")) { + String noauto = PARAM_NOAUTO.getStringValue(this); + if (!noauto.equals("true") && !noauto.equals("false")) { + throw new InvalidMappingParameterException("noauto parameter value should be 'true' or 'false'" + + ". Value set was '" + noauto + "'"); + } + if (!suffix.startsWith(".") && noauto.equals("false")) { PARAM_SUFFIX.setValue(this, "." + suffix); } } From noreply at svn.ci.uchicago.edu Wed Mar 24 16:59:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Mar 2010 16:59:25 -0500 (CDT) Subject: [Swift-commit] r3266 - trunk/tests/language-behaviour Message-ID: <20100324215925.5C9649CC90@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-03-24 16:59:25 -0500 (Wed, 24 Mar 2010) New Revision: 3266 Added: trunk/tests/language-behaviour/T078-simplemapper-nosuffix.swift trunk/tests/language-behaviour/T078-simplemapper-nosuffix_nodot.out.expected trunk/tests/language-behaviour/T079-simplemapper-nosuffix_exception.swift Log: tests for noauto patch Added: trunk/tests/language-behaviour/T078-simplemapper-nosuffix.swift =================================================================== --- trunk/tests/language-behaviour/T078-simplemapper-nosuffix.swift (rev 0) +++ trunk/tests/language-behaviour/T078-simplemapper-nosuffix.swift 2010-03-24 21:59:25 UTC (rev 3266) @@ -0,0 +1,17 @@ +// this one is like test 073-simplemapper, but doesn't +// add the missing dot in front of the suffix + +type messagefile; + +(messagefile t) write() { + app { + echo @filename(t) stdout=@filename(t); + } +} + +messagefile outfile ; + +outfile = write(); + Added: trunk/tests/language-behaviour/T078-simplemapper-nosuffix_nodot.out.expected =================================================================== --- trunk/tests/language-behaviour/T078-simplemapper-nosuffix_nodot.out.expected (rev 0) +++ trunk/tests/language-behaviour/T078-simplemapper-nosuffix_nodot.out.expected 2010-03-24 21:59:25 UTC (rev 3266) @@ -0,0 +1 @@ +T078-simplemapper-nosuffix_nodot.out Added: trunk/tests/language-behaviour/T079-simplemapper-nosuffix_exception.swift =================================================================== --- trunk/tests/language-behaviour/T079-simplemapper-nosuffix_exception.swift (rev 0) +++ trunk/tests/language-behaviour/T079-simplemapper-nosuffix_exception.swift 2010-03-24 21:59:25 UTC (rev 3266) @@ -0,0 +1,16 @@ +// this one is like test T078-simplemapper-nosuffix, but demonstrates the +// mapping exception on invalid values +type messagefile; + +(messagefile t) write() { + app { + echo @filename(t) stdout=@filename(t); + } +} + +messagefile outfile ; + +outfile = write(); + From noreply at svn.ci.uchicago.edu Wed Mar 24 19:07:31 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 24 Mar 2010 19:07:31 -0500 (CDT) Subject: [Swift-commit] r3267 - branches/1.0/src/org/griphyn/vdl/mapping/file Message-ID: <20100325000731.4B38E9CCA2@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-03-24 19:07:30 -0500 (Wed, 24 Mar 2010) New Revision: 3267 Modified: branches/1.0/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java Log: a more user friendly exception Modified: branches/1.0/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java =================================================================== --- branches/1.0/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2010-03-24 21:59:25 UTC (rev 3266) +++ branches/1.0/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2010-03-25 00:07:30 UTC (rev 3267) @@ -21,6 +21,9 @@ public Collection existing() { List l = new ArrayList(); DSHandle dn = (DSHandle) PARAM_FILES.getRawValue(this); + if (dn == null) { + throw new RuntimeException("Missing 'files' mapper attribute"); + } Map m = dn.getArrayValue(); Set s = m.keySet(); Iterator i = s.iterator(); From noreply at svn.ci.uchicago.edu Thu Mar 25 11:58:16 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 25 Mar 2010 11:58:16 -0500 (CDT) Subject: [Swift-commit] r3268 - in trunk: libexec src/org/globus/swift/data/policy Message-ID: <20100325165816.704BA9CC96@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-03-25 11:58:15 -0500 (Thu, 25 Mar 2010) New Revision: 3268 Added: trunk/src/org/globus/swift/data/policy/Local.java Modified: trunk/libexec/_swiftwrap trunk/src/org/globus/swift/data/policy/Policy.java Log: Draft of CDM LOCAL method. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-03-25 00:07:30 UTC (rev 3267) +++ trunk/libexec/_swiftwrap 2010-03-25 16:58:15 UTC (rev 3268) @@ -127,6 +127,7 @@ return 1 } +# Output: cdm_lookup() { FILE=$1 CDM_FILE=$2 @@ -151,7 +152,7 @@ case $POLICY in DIRECT) DIRECT_DIR=${ARGS[0]} - log "CDM[DIRECT]: Linking $JOBDIR/$FILE to $DIRECT_DIR/$FILE" + log "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE via $JOBDIR/$FILE" if [ $MODE == "INPUT" ]; then [ -f "$DIRECT_DIR/$FILE" ] checkError 254 "CDM[DIRECT]: $DIRECT_DIR/$FILE does not exist!" @@ -168,6 +169,22 @@ fail 254 "Unknown MODE: $MODE" fi ;; + LOCAL) + TOOL=${ARGS[0]} + REMOTE_DIR=${ARGS[1]} + FLAGS=${ARGS[3]} + log "CDM[LOCAL]: Copying $DIRECT_DIR/$FILE to $JOBDIR/$FILE" + if [ $MODE == "INPUT" ]; then + [ -f "$DIRECT_DIR/$FILE" ] + checkError 254 "CDM[DIRECT]: $REMOTE_DIR/$FILE does not exist!" + $TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE + checkError 254 "CDM[DIRECT]: Tool failed!" + elif [ $MODE == "OUTPUT" ]; then + log "CDM[LOCAL]..." + else + fail 254 "Unknown MODE: $MODE" + fi + ;; BROADCAST) BROADCAST_DIR=${ARGS[0]} log "CDM[BROADCAST]: Linking $JOBDIR/$FILE to $BROADCAST_DIR/$FILE" @@ -187,6 +204,31 @@ esac } +cdm_local_output() +{ + L=$1 + + if [[ $CDM_FILE == "" ]]; then + return + fi + + CDM_POLICY=$( cdm_lookup $L $CDM_FILE ) + cdm_local_output_perform $L $CDM_POLICY +} + +cdm_local_output_perform() +{ + L=$1 + TOOL=$2 + REMOTE_DIR=$3 + FLAGS=$3 + log "Copying $REMOTE_DIR/$FILE to $JOBDIR/$FILE" + mkdir -p $REMOTE_DIR + checkError 254 "CDM[LOCAL]: mkdir -p $REMOTE_DIR failed!" + $TOOL $FLAGS $JOBDIR/$FILE $REMOTE_DIR/$FILE + checkError 254 "CDM[LOCAL]: Tool failed!" +} + cdm_gather() { GATHER_OUTPUT=${*} @@ -202,6 +244,8 @@ COMMANDLINE=$@ +echo $0 $COMMANDLINE >> /tmp/swiftwrap.out + # get the parent directory of the directory containing _swiftwrap, to use # as the run directory # this assumes that _swiftwrap is being executed from the top level of @@ -410,6 +454,8 @@ fi if [ $CDM_POLICY == "GATHER" ]; then GATHER_OUTPUT=( $GATHER_OUTPUT $L ) + elif [ $CDM_POLICY == "LOCAL" ]; then + CDM_LOCAL_OUTPUT=( $CDM_LOCAL_OUTPUT $L ) fi done @@ -492,6 +538,7 @@ fi done +cdm_local_output $CDM_LOCAL_OUTPUT cdm_gather $GATHER_OUTPUT logstate "RM_JOBDIR" Added: trunk/src/org/globus/swift/data/policy/Local.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Local.java (rev 0) +++ trunk/src/org/globus/swift/data/policy/Local.java 2010-03-25 16:58:15 UTC (rev 3268) @@ -0,0 +1,14 @@ +package org.globus.swift.data.policy; + +import java.util.List; + +public class Local extends Policy { + + public String toString() { + return "PULL"; + } + + @Override + public void settings(List tokens) { + } +} Modified: trunk/src/org/globus/swift/data/policy/Policy.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Policy.java 2010-03-25 00:07:30 UTC (rev 3267) +++ trunk/src/org/globus/swift/data/policy/Policy.java 2010-03-25 16:58:15 UTC (rev 3268) @@ -17,6 +17,8 @@ return new Default(); else if (token.compareToIgnoreCase("direct") == 0) return new Direct(); + else if (token.compareToIgnoreCase("local") == 0) + return new Local(); else if (token.compareToIgnoreCase("broadcast") == 0) return new Broadcast(); else if (token.compareToIgnoreCase("gather") == 0)