From noreply at svn.ci.uchicago.edu Mon Dec 1 20:56:12 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 1 Dec 2008 20:56:12 -0600 (CST)
Subject: [Swift-commit] r2352 - trunk/src/org/griphyn/vdl/karajan
Message-ID: <20081202025612.79DF1228193@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-01 20:56:11 -0600 (Mon, 01 Dec 2008)
New Revision: 2352
Modified:
trunk/src/org/griphyn/vdl/karajan/InHook.java
trunk/src/org/griphyn/vdl/karajan/Monitor.java
Log:
rearrange console debugger so that it will take more than one cmmand; previously it would enter an infinite loop unable to process the end-of-line character at the end of a command line
Modified: trunk/src/org/griphyn/vdl/karajan/InHook.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/InHook.java 2008-11-28 06:52:55 UTC (rev 2351)
+++ trunk/src/org/griphyn/vdl/karajan/InHook.java 2008-12-02 02:56:11 UTC (rev 2352)
@@ -7,7 +7,12 @@
import java.io.IOException;
import java.io.InputStream;
+import org.apache.log4j.Logger;
+
public class InHook extends InputStream implements Runnable {
+
+ public static final Logger logger = Logger.getLogger(InHook.class);
+
public synchronized static void install(Monitor m) {
if (!(System.in instanceof InHook)) {
System.setIn(new InHook(System.in, m));
@@ -21,9 +26,11 @@
if (is instanceof BufferedInputStream) {
this.is = (BufferedInputStream) is;
this.m = m;
- Thread t = new Thread(this, "stdin debugger");
+ Thread t = new Thread(this, "Swift console debugger");
t.setDaemon(true);
t.start();
+ } else {
+ logger.error("Attempt to start console debugger with stdin not an instance of BufferedInputStream. InputStream class is "+is.getClass());
}
}
@@ -32,29 +39,29 @@
}
public void run() {
+ logger.debug("Starting console debugger thread");
while (true) {
+ logger.debug("Console debugger outer loop");
try {
- while (is.available() > 0) {
- is.mark(1);
- int c = is.read();
- if (c == 'd') {
- m.toggle();
- }
- else if (c == 'v') {
- m.dumpVariables();
- }
- else if (c == 't') {
- m.dumpThreads();
- }
- else {
- is.reset();
- }
+ int c = is.read();
+ logger.debug("Command: "+c); // TODO display as char?
+ if (c == 'd') {
+ m.toggle();
}
- if (is.available() == 0) {
- Thread.sleep(250);
+ else if (c == 'v') {
+ m.dumpVariables();
}
+ else if (c == 't') {
+ m.dumpThreads();
+ } else if (c == 10) {
+ logger.debug("Ignoring LF");
+ }
+ else {
+ logger.warn("Unknown console debugger command "+c);
+ }
}
catch (IOException e) {
+ logger.debug("Console debugger encountered IOException",e);
return;
}
catch (Exception e) {
Modified: trunk/src/org/griphyn/vdl/karajan/Monitor.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/Monitor.java 2008-11-28 06:52:55 UTC (rev 2351)
+++ trunk/src/org/griphyn/vdl/karajan/Monitor.java 2008-12-02 02:56:11 UTC (rev 2352)
@@ -64,7 +64,7 @@
private synchronized void init() {
frame = new JFrame();
- frame.setTitle("VDS Debugga'");
+ frame.setTitle("VDS Debugger");
buttons = new JPanel();
frame.getContentPane().setLayout(new BorderLayout());
frame.getContentPane().add(buttons, BorderLayout.NORTH);
From noreply at svn.ci.uchicago.edu Mon Dec 1 21:08:34 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 1 Dec 2008 21:08:34 -0600 (CST)
Subject: [Swift-commit] r2353 - in trunk: libexec
src/org/griphyn/vdl/karajan/lib
Message-ID: <20081202030834.6BBC6228035@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-01 21:08:32 -0600 (Mon, 01 Dec 2008)
New Revision: 2353
Modified:
trunk/libexec/vdl.k
trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
Log:
more expressive text around ExecutionException that wraps a DependentException
Modified: trunk/libexec/vdl.k
===================================================================
--- trunk/libexec/vdl.k 2008-12-02 02:56:11 UTC (rev 2352)
+++ trunk/libexec/vdl.k 2008-12-02 03:08:32 UTC (rev 2353)
@@ -61,6 +61,7 @@
for(path, fp
vdl:waitFieldValue(path=path, var)
)
+// NOTE this string-based catch is messy, and needs to match with the string used in DataDependentException and VDLFunction
catch(".*errors in data dependencies.*"
log(LOG:DEBUG, exception)
deperror = true
Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2008-12-02 02:56:11 UTC (rev 2352)
+++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2008-12-02 03:08:32 UTC (rev 2353)
@@ -74,7 +74,7 @@
catch (DependentException e) {
// This would not be the primal fault so in non-lazy errors mode it
// should not matter
- throw new ExecutionException(e);
+ throw new ExecutionException("Wrapping a dependent exception in VDLFunction.post() - errors in data dependencies",e);
}
}
From noreply at svn.ci.uchicago.edu Mon Dec 1 21:22:01 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 1 Dec 2008 21:22:01 -0600 (CST)
Subject: [Swift-commit] r2354 - trunk/src/org/griphyn/vdl/karajan/lib
Message-ID: <20081202032201.4AB3422819B@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-01 21:22:00 -0600 (Mon, 01 Dec 2008)
New Revision: 2354
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
Log:
descriptive text for ExecutionExceptions
Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2008-12-02 03:08:32 UTC (rev 2353)
+++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2008-12-02 03:22:00 UTC (rev 2354)
@@ -418,7 +418,7 @@
}
}
catch (HandleOpenException e) {
- throw new ExecutionException(e);
+ throw new ExecutionException("Handle open in closeChildren",e);
}
if (!closed) {
@@ -463,7 +463,7 @@
}
}
catch (HandleOpenException e) {
- throw new ExecutionException(e);
+ throw new ExecutionException("HandleOpen during closeDeep",e);
}
markToRoot(stack, handle);
}
From noreply at svn.ci.uchicago.edu Tue Dec 2 10:21:41 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 2 Dec 2008 10:21:41 -0600 (CST)
Subject: [Swift-commit] r2355 - log-processing/bin
Message-ID: <20081202162141.33D6922810E@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-02 10:21:40 -0600 (Tue, 02 Dec 2008)
New Revision: 2355
Added:
log-processing/bin/swift2cedps
Log:
commandline utility to drive CEDPS format log conversion
Added: log-processing/bin/swift2cedps
===================================================================
--- log-processing/bin/swift2cedps (rev 0)
+++ log-processing/bin/swift2cedps 2008-12-02 16:21:40 UTC (rev 2355)
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+ORIGDIR=$(pwd)
+WORKINGDIR=$(mktemp -d /tmp/swift-plot-log-XXXXXXXXXXXXXXXX)
+
+
+# $1 should be the pathname of the log file to plot
+
+LOG_CODE_HOME="`dirname $0`/../libexec/"
+export SWIFT_PLOT_HOME=$LOG_CODE_HOME
+
+export PATH=${LOG_CODE_HOME}:${PATH}
+
+FC="${1:0:1}"
+
+if [ "$FC" = "/" ]; then # absolute path
+ LOG_FILE_PATH=$1
+else
+ LOG_FILE_PATH="`pwd`/$1"
+fi
+
+LOG_DIRECTORY=`dirname $LOG_FILE_PATH`
+
+LOG_FILE_BASE=`basename $LOG_FILE_PATH .log`
+
+echo Log file path is $LOG_FILE_PATH
+echo Log is in directory $LOG_DIRECTORY
+echo Log basename is $LOG_FILE_BASE
+
+cd $WORKINGDIR
+echo Now in directory $(pwd)
+
+MAKEENV="-f ${LOG_CODE_HOME}/makefile -I ${LOG_CODE_HOME}"
+MAKETARGETS=""
+
+INFODIR=${LOG_DIRECTORY}/${LOG_FILE_BASE}.d
+
+if [ -d $INFODIR ]; then
+ MAKEENV="$MAKEENV IDIR=${LOG_DIRECTORY}/${LOG_FILE_BASE}.d/"
+ MAKETARGETS="webpage.info"
+fi
+
+make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean execute.cedps execute2.cedps karatasks.cedps
+
+cp -v *.cedps $ORIGDIR
+
+# webpage.kara webpage.weights karatasks.JOB_SUBMISSION.Queue.transitions karatasks.JOB_SUBMISSION.Queue.event karatasks.JOB_SUBMISSION.eip $MAKETARGETS webpage distribute
+
+# mv report-$LOG_FILE_BASE $LOG_DIRECTORY/
+
+# rm -r $WORKINGDIR
+echo not removing $WORKINGDIR
Property changes on: log-processing/bin/swift2cedps
___________________________________________________________________
Name: svn:executable
+ *
From noreply at svn.ci.uchicago.edu Thu Dec 4 09:45:10 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 4 Dec 2008 09:45:10 -0600 (CST)
Subject: [Swift-commit] r2356 - trunk/src/org/griphyn/vdl/karajan
Message-ID: <20081204154510.8BA00228193@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-04 09:45:08 -0600 (Thu, 04 Dec 2008)
New Revision: 2356
Modified:
trunk/src/org/griphyn/vdl/karajan/InHook.java
Log:
console debugger needs to deal with stdin being closed; this case occurs in the NMI Build/Test system, and if stdin is redirected from /dev/null on the commandline
Modified: trunk/src/org/griphyn/vdl/karajan/InHook.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/InHook.java 2008-12-02 16:21:40 UTC (rev 2355)
+++ trunk/src/org/griphyn/vdl/karajan/InHook.java 2008-12-04 15:45:08 UTC (rev 2356)
@@ -55,6 +55,9 @@
m.dumpThreads();
} else if (c == 10) {
logger.debug("Ignoring LF");
+ } else if (c == -1) {
+ logger.debug("End of stdin - exiting debugger");
+ return;
}
else {
logger.warn("Unknown console debugger command "+c);
From noreply at svn.ci.uchicago.edu Thu Dec 4 11:11:56 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 4 Dec 2008 11:11:56 -0600 (CST)
Subject: [Swift-commit] r2357 - trunk/src/org/griphyn/vdl/karajan/lib
Message-ID: <20081204171156.90BF922815A@www.ci.uchicago.edu>
Author: hategan
Date: 2008-12-04 11:11:54 -0600 (Thu, 04 Dec 2008)
New Revision: 2357
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/JobConstraints.java
Log:
pass list of required files
Modified: trunk/src/org/griphyn/vdl/karajan/lib/JobConstraints.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/JobConstraints.java 2008-12-04 15:45:08 UTC (rev 2356)
+++ trunk/src/org/griphyn/vdl/karajan/lib/JobConstraints.java 2008-12-04 17:11:54 UTC (rev 2357)
@@ -9,19 +9,32 @@
import org.globus.cog.karajan.util.TypeUtil;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.griphyn.vdl.util.FQN;
+import java.util.Collection;
+import org.griphyn.vdl.karajan.lib.cache.CacheMapAdapter;
public class JobConstraints extends VDLFunction {
public static final Arg A_TR = new Arg.Positional("tr");
-
+ public static final Arg STAGE_IN = new Arg.Optional("stagein");
+
static {
- setArguments(JobConstraints.class, new Arg[] { A_TR });
+ setArguments(JobConstraints.class, new Arg[] { A_TR, STAGE_IN });
}
+
+ private static final String[] STRING_ARRAY = new String[0];
public Object function(VariableStack stack) throws ExecutionException {
String tr = TypeUtil.toString(A_TR.getValue(stack));
+ String[] filenames = null;
+ if (STAGE_IN.isPresent(stack)) {
+ filenames = (String[]) ((Collection) STAGE_IN.getValue(stack)).toArray(STRING_ARRAY);
+ }
TaskConstraints tc = new TaskConstraints();
tc.addConstraint("tr", tr);
tc.addConstraint("trfqn", new FQN(tr));
+ if (filenames != null) {
+ tc.addConstraint("filenames", filenames);
+ tc.addConstraint("filecache", new CacheMapAdapter(CacheFunction.getCache(stack)));
+ }
return tc;
}
}
From noreply at svn.ci.uchicago.edu Thu Dec 4 11:15:07 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 4 Dec 2008 11:15:07 -0600 (CST)
Subject: [Swift-commit] r2358 - trunk/libexec
Message-ID: <20081204171507.7FA8122810E@www.ci.uchicago.edu>
Author: hategan
Date: 2008-12-04 11:15:06 -0600 (Thu, 04 Dec 2008)
New Revision: 2358
Modified:
trunk/libexec/vdl-int.k
Log:
pass stagein files to job constraints
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2008-12-04 17:11:54 UTC (rev 2357)
+++ trunk/libexec/vdl-int.k 2008-12-04 17:15:06 UTC (rev 2358)
@@ -362,7 +362,7 @@
replicationGroup, replicationChannel]
stagein := list(unique(each(stagein)))
stageout := list(unique(each(stageout)))
- allocateHost(rhost, constraints=vdl:jobConstraints(tr)
+ allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein)
[wfdir, sharedDir] := try(
initSharedDir(rhost)
From noreply at svn.ci.uchicago.edu Sat Dec 6 21:15:38 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sat, 6 Dec 2008 21:15:38 -0600 (CST)
Subject: [Swift-commit] r2359 - trunk/libexec
Message-ID: <20081207031538.6813B2281AD@www.ci.uchicago.edu>
Author: hategan
Date: 2008-12-06 21:15:37 -0600 (Sat, 06 Dec 2008)
New Revision: 2359
Modified:
trunk/libexec/vdl-int.k
Log:
fixed error
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2008-12-04 17:15:06 UTC (rev 2358)
+++ trunk/libexec/vdl-int.k 2008-12-07 03:15:37 UTC (rev 2359)
@@ -362,7 +362,7 @@
replicationGroup, replicationChannel]
stagein := list(unique(each(stagein)))
stageout := list(unique(each(stageout)))
- allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein)
+ allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein)
[wfdir, sharedDir] := try(
initSharedDir(rhost)
From noreply at svn.ci.uchicago.edu Mon Dec 8 14:50:54 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 8 Dec 2008 14:50:54 -0600 (CST)
Subject: [Swift-commit] r2360 - trunk/src/org/griphyn/vdl/engine
Message-ID: <20081208205054.485AC22810E@www.ci.uchicago.edu>
Author: hategan
Date: 2008-12-08 14:50:53 -0600 (Mon, 08 Dec 2008)
New Revision: 2360
Modified:
trunk/src/org/griphyn/vdl/engine/Karajan.java
Log:
fixed string index out of bounds exception when typechecking an expression involving and array of nonexistent type
Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/Karajan.java 2008-12-07 03:15:37 UTC (rev 2359)
+++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2008-12-08 20:50:53 UTC (rev 2360)
@@ -1008,6 +1008,9 @@
break;
}
}
+ if ("".equals(actualType)) {
+ throw new CompilationException("Type " + parentType + " is not defined.");
+ }
newst.setAttribute("datatype", actualType);
return newst;
// TODO the template layout for this and ARRAY_SUBSCRIPT are
From noreply at svn.ci.uchicago.edu Mon Dec 8 14:52:51 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 8 Dec 2008 14:52:51 -0600 (CST)
Subject: [Swift-commit] r2361 - trunk/src/org/griphyn/vdl/engine
Message-ID: <20081208205251.B34AA22810E@www.ci.uchicago.edu>
Author: hategan
Date: 2008-12-08 14:52:51 -0600 (Mon, 08 Dec 2008)
New Revision: 2361
Modified:
trunk/src/org/griphyn/vdl/engine/Karajan.java
Log:
initializing to null seems cleaner
Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/Karajan.java 2008-12-08 20:50:53 UTC (rev 2360)
+++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2008-12-08 20:52:51 UTC (rev 2361)
@@ -992,7 +992,7 @@
newst.setAttribute("memberchild", sm.getMemberName());
String parentType = datatype(parentST);
- String actualType = "";
+ String actualType = null;
for (int i = 0; i < types.sizeOfTypeArray(); i++) {
if (types.getTypeArray(i).getTypename().equals(parentType)) {
TypeStructure ts = types.getTypeArray(i).getTypestructure();
@@ -1008,7 +1008,7 @@
break;
}
}
- if ("".equals(actualType)) {
+ if (actualType == null) {
throw new CompilationException("Type " + parentType + " is not defined.");
}
newst.setAttribute("datatype", actualType);
From noreply at svn.ci.uchicago.edu Tue Dec 9 13:13:02 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 9 Dec 2008 13:13:02 -0600 (CST)
Subject: [Swift-commit] r2362 - trunk/docs
Message-ID: <20081209191302.D7D20228115@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-09 13:13:01 -0600 (Tue, 09 Dec 2008)
New Revision: 2362
Modified:
trunk/docs/tutorial.xml
Log:
extra \ needed in regexp tutorial (syntax changed several versions ago...)
Modified: trunk/docs/tutorial.xml
===================================================================
--- trunk/docs/tutorial.xml 2008-12-08 20:52:51 UTC (rev 2361)
+++ trunk/docs/tutorial.xml 2008-12-09 19:13:01 UTC (rev 2362)
@@ -540,7 +540,7 @@
countfile c <regexp_mapper;
source=@inputfile,
match="(.*)txt",
- transform="\1count"
+ transform="\\1count"
>;
@@ -591,7 +591,7 @@
countfile c <regexp_mapper;
source=@f,
match="(.*)txt",
- transform="\1count">;
+ transform="\\1count">;
c = countwords(f);
}
From noreply at svn.ci.uchicago.edu Wed Dec 10 09:49:41 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 10 Dec 2008 09:49:41 -0600 (CST)
Subject: [Swift-commit] r2363 - trunk/docs
Message-ID: <20081210154941.3503A228115@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-10 09:49:40 -0600 (Wed, 10 Dec 2008)
New Revision: 2363
Modified:
trunk/docs/log-processing.xml
Log:
update log-processing doc to present swift-plot-log command
Modified: trunk/docs/log-processing.xml
===================================================================
--- trunk/docs/log-processing.xml 2008-12-09 19:13:01 UTC (rev 2362)
+++ trunk/docs/log-processing.xml 2008-12-10 15:49:40 UTC (rev 2363)
@@ -26,50 +26,15 @@
Web page about a run
-make LOG=/path/to/readData-20080304-0903-xgqf5nhe.log clean webpage
+cd log-processing/bin
+export PATH=$(pwd):$PATH
+swift-plot-log /path/to/readData-20080304-0903-xgqf5nhe.log
-This will create a web page in the log-processing directory giving
-information about the run that is extracted from the log file. If the
-above command is used before a run is completed, the web page will report
-information about the workflow progress so far.
+This will create a web page, report-readData-20080304-0903-xgqf5nhe
+If the above command is used before a run is completed, the web page will
+report information about the workflow progress so far.
-
-Additional more expensive to compute information can added to this using
-the following makefile targets, which should be added to the command line
-between the clean and makefile targets.
-
-
-webpage.kara - more detailed information about internal Karajan-level
-execution
-
-
-webpage.clusters - details of job clustering. This will fail if no job
-clustering occurred.
-
-
-webpage.falkon - details of falkon workers. This will fail if no Falkon
-log file is present.
-
-
-webpage.weights - details of site scoring
-
-
-webpage.info - details of execute-site wrapper logs. For versions of Swift
-prior to r1700, you will need to stage the *-info logs back to the
-same place as the Swift log manually. AFter r1700, they are staged back
-automatically under the control of the wrapperlog.always.transfer property.
-The IDIR variable must be set to point to the directory containg the logs:
-
-make LOG=/path/fmri-20080304-0901-h8h78lnf.log \
- IDIR=/path/130-fmri-20080304-0901-h8h78lnf.d/ clean webpage.info webpage
-
-
-
-webpage.kickstart - similar to the webpage.info target, this gives execute-site
-kickstart record information.
-
-
CEDPS logs
From noreply at svn.ci.uchicago.edu Wed Dec 10 10:30:41 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 10 Dec 2008 10:30:41 -0600 (CST)
Subject: [Swift-commit] r2364 - log-processing/bin
Message-ID: <20081210163041.1A1FF228154@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-10 10:30:40 -0600 (Wed, 10 Dec 2008)
New Revision: 2364
Modified:
log-processing/bin/swift-plot-log
Log:
allow a non-default target to be specified to swift-plot-log to override default webpage generation target
Modified: log-processing/bin/swift-plot-log
===================================================================
--- log-processing/bin/swift-plot-log 2008-12-10 15:49:40 UTC (rev 2363)
+++ log-processing/bin/swift-plot-log 2008-12-10 16:30:40 UTC (rev 2364)
@@ -40,7 +40,11 @@
MAKETARGETS="webpage.info"
fi
+if [ "X$2" = "X" ]; then
make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean webpage.kara webpage.weights karatasks.JOB_SUBMISSION.Queue.transitions karatasks.JOB_SUBMISSION.Queue.event karatasks.JOB_SUBMISSION.eip $MAKETARGETS webpage distribute
+else
+make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean $2
+fi
# mv report-$LOG_FILE_BASE $LOG_DIRECTORY/
From noreply at svn.ci.uchicago.edu Wed Dec 10 10:33:00 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 10 Dec 2008 10:33:00 -0600 (CST)
Subject: [Swift-commit] r2365 - log-processing/libexec
Message-ID: <20081210163301.05B35228154@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-10 10:32:56 -0600 (Wed, 10 Dec 2008)
New Revision: 2365
Modified:
log-processing/libexec/info-to-transitions
Log:
extract application base name from info file names to end up in its own column in .transitions and .execute files
Modified: log-processing/libexec/info-to-transitions
===================================================================
--- log-processing/libexec/info-to-transitions 2008-12-10 16:30:40 UTC (rev 2364)
+++ log-processing/libexec/info-to-transitions 2008-12-10 16:32:56 UTC (rev 2365)
@@ -5,6 +5,7 @@
for infofn in $1/*-info; do
jobid=$(echo $infofn | sed 's%^.*/\([^/]*\)-info$%\1%' )
echo Processing info file for job $jobid in file $infofn >&2
+ jobbase=$(echo $jobid | sed 's/^\(.*\)-........$/\1/')
# Progress 2007-10-29 10:05:50+0000 RM_JOBDIR
- grep -E '^Progress ' $infofn | sed "s/^Progress *\([^ ]* [^ ]*\) *\([^ ]*\).*/\1 $jobid \2/"
+ grep -E '^Progress ' $infofn | sed "s/^Progress *\([^ ]* [^ ]*\) *\([^ ]*\).*/\1 $jobid \2 $jobbase/"
done | iso-to-secs | swap-and-sort-and-swap
From noreply at svn.ci.uchicago.edu Thu Dec 11 10:27:07 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 11 Dec 2008 10:27:07 -0600 (CST)
Subject: [Swift-commit] r2366 - log-processing/libexec
Message-ID: <20081211162707.C84CC228198@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-11 10:27:06 -0600 (Thu, 11 Dec 2008)
New Revision: 2366
Added:
log-processing/libexec/error-summary
log-processing/libexec/makefile.errors
Modified:
log-processing/libexec/makefile
Log:
file summarizing APPLICATION_EXCEPTION messages
Added: log-processing/libexec/error-summary
===================================================================
--- log-processing/libexec/error-summary (rev 0)
+++ log-processing/libexec/error-summary 2008-12-11 16:27:06 UTC (rev 2366)
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+TFN=$(mktemp /tmp/error-summary-XXXXXXXXX)
+
+grep APPLICATION_EXC $1 | sed 's/^.*DEBUG vdl:execute2 APPLICATION_EXCEPTION jobid=\([^ ]*\) - \(.*\)$/\1 \2/' > $TFN
+
+echo === overall summary ===
+echo count / message
+cat $TFN | cut -d ' ' -f 2- | sort | uniq -c | sort -n
+
Property changes on: log-processing/libexec/error-summary
___________________________________________________________________
Name: svn:executable
+ *
Modified: log-processing/libexec/makefile
===================================================================
--- log-processing/libexec/makefile 2008-12-10 16:32:56 UTC (rev 2365)
+++ log-processing/libexec/makefile 2008-12-11 16:27:06 UTC (rev 2366)
@@ -7,6 +7,7 @@
include makefile.karatasks
include makefile.webpage
include makefile.kickstart
+include makefile.errors
distributable:
rm -f *.tmp log *.transitions tmp-*
Added: log-processing/libexec/makefile.errors
===================================================================
--- log-processing/libexec/makefile.errors (rev 0)
+++ log-processing/libexec/makefile.errors 2008-12-11 16:27:06 UTC (rev 2366)
@@ -0,0 +1,3 @@
+
+error-summary.txt: $(LOG)
+ error-summary $< > $@
From noreply at svn.ci.uchicago.edu Fri Dec 12 11:57:32 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 12 Dec 2008 11:57:32 -0600 (CST)
Subject: [Swift-commit] r2367 - trunk/libexec
Message-ID: <20081212175732.B8D1A22811D@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-12 11:57:31 -0600 (Fri, 12 Dec 2008)
New Revision: 2367
Modified:
trunk/libexec/wrapper.sh
Log:
Better error message when unsuccessful in looking up applications on remote site $PATH
Modified: trunk/libexec/wrapper.sh
===================================================================
--- trunk/libexec/wrapper.sh 2008-12-11 16:27:06 UTC (rev 2366)
+++ trunk/libexec/wrapper.sh 2008-12-12 17:57:31 UTC (rev 2367)
@@ -164,8 +164,12 @@
fi
if [ "X${EXEC:0:1}" != "X/" ] ; then
+export ORIGEXEC=$EXEC
export EXEC=$(which $EXEC)
+if [ "X$EXEC" = "X" ] ; then
+fail 254 "Cannot find executable $ORIGEXEC on site system path"
fi
+fi
log "DIR=$DIR"
log "EXEC=$EXEC"
From noreply at svn.ci.uchicago.edu Fri Dec 12 15:57:02 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 12 Dec 2008 15:57:02 -0600 (CST)
Subject: [Swift-commit] r2368 - trunk/docs
Message-ID: <20081212215702.C0C7922811D@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-12 15:57:01 -0600 (Fri, 12 Dec 2008)
New Revision: 2368
Modified:
trunk/docs/userguide.xml
Log:
typo in section id
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2008-12-12 17:57:31 UTC (rev 2367)
+++ trunk/docs/userguide.xml 2008-12-12 21:57:01 UTC (rev 2368)
@@ -2276,7 +2276,7 @@
Profiles
- Karajan namespace
+ Karajan namespace
maxSubmitRate - limits the maximum rate of job submission, in jobs per second.
For example:
From noreply at svn.ci.uchicago.edu Sun Dec 14 12:25:28 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 14 Dec 2008 12:25:28 -0600 (CST)
Subject: [Swift-commit] r2369 - trunk/libexec
Message-ID: <20081214182528.5C4F322810E@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-14 12:25:27 -0600 (Sun, 14 Dec 2008)
New Revision: 2369
Modified:
trunk/libexec/vdl-int.k
Log:
fix bug 162 - slightly confusing error message format in wrapper log errors
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2008-12-12 21:57:01 UTC (rev 2368)
+++ trunk/libexec/vdl-int.k 2008-12-14 18:25:27 UTC (rev 2369)
@@ -335,7 +335,7 @@
task:transfer(srchost=rhost, srcdir=srcdir, srcfile=recfile, destdir="{VDL:SCRIPTNAME}-{VDL:RUNID}.d/")
(
maybe(file:remove(recfile))
- log(LOG:WARN, "Failed to transfer kickstart records from {srcdir}/{rhost}", exception)
+ log(LOG:WARN, "Failed to transfer kickstart records from {srcdir} on {rhost}", exception)
)
)
recfile
@@ -351,8 +351,8 @@
task:transfer(srchost=rhost, srcdir=srcdir, srcfile=recfile, destdir="{VDL:SCRIPTNAME}-{VDL:RUNID}.d/")
(
maybe(file:remove(recfile))
- log(LOG:WARN, "Failed to transfer wrapper log from {srcdir}/{rhost}")
- log(LOG:DEBUG, "Exception for wrapper log failure from {srcdir}/{rhost}: ", exception)
+ log(LOG:WARN, "Failed to transfer wrapper log from {srcdir} on {rhost}")
+ log(LOG:DEBUG, "Exception for wrapper log failure from {srcdir} on {rhost}: ", exception)
)
)
recfile
From noreply at svn.ci.uchicago.edu Sun Dec 14 12:32:33 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 14 Dec 2008 12:32:33 -0600 (CST)
Subject: [Swift-commit] r2370 - trunk/docs
Message-ID: <20081214183233.6F0C122810E@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-14 12:32:32 -0600 (Sun, 14 Dec 2008)
New Revision: 2370
Modified:
trunk/docs/Makefile
Log:
makefile target to make all htmls
Modified: trunk/docs/Makefile
===================================================================
--- trunk/docs/Makefile 2008-12-14 18:25:27 UTC (rev 2369)
+++ trunk/docs/Makefile 2008-12-14 18:32:32 UTC (rev 2370)
@@ -4,6 +4,8 @@
phps: userguide.php tutorial.php tutorial-live.php quickstartguide.php reallyquickstartguide.php languagespec.php languagespec-0.6.php log-processing.php plot-tour.php
+htmls: userguide.html tutorial.html tutorial-live.html quickstartguide.html reallyquickstartguide.html languagespec.html languagespec-0.6.html log-processing.html plot-tour.html
+
pdfs: userguide.pdf tutorial.pdf tutorial-live.pdf quickstartguide.pdf reallyquickstartguide.pdf languagespec.pdf languagespec-0.6.pdf log-processing.pdf
chunked-userguide: userguide.xml
From noreply at svn.ci.uchicago.edu Sun Dec 14 14:04:53 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 14 Dec 2008 14:04:53 -0600 (CST)
Subject: [Swift-commit] r2371 - in trunk/src/org/griphyn/vdl/karajan/lib: .
swiftscript
Message-ID: <20081214200454.0E4F3228115@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-14 14:04:53 -0600 (Sun, 14 Dec 2008)
New Revision: 2371
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java
Log:
synchronization around filename calls - this is needed later on when there is more parallelism
Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2008-12-14 18:32:32 UTC (rev 2370)
+++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2008-12-14 20:04:53 UTC (rev 2371)
@@ -175,14 +175,17 @@
public String[] filename(VariableStack stack) throws ExecutionException {
DSHandle ovar = (DSHandle)PA_VAR.getValue(stack);
- try {
- return filename(ovar);
+ synchronized(ovar.getRoot()) {
+ try {
+ return filename(ovar);
+ }
+ catch (HandleOpenException e) {
+ throw new FutureNotYetAvailable(addFutureListener(stack, e.getSource()));
+ }
}
- catch (HandleOpenException e) {
- throw new FutureNotYetAvailable(addFutureListener(stack, e.getSource()));
- }
}
+ /** The caller is expected to have synchronized on the root of var. */
public String[] filename(DSHandle var) throws ExecutionException, HandleOpenException {
try {
if (var.getType().isArray()) {
Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2008-12-14 18:32:32 UTC (rev 2370)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2008-12-14 20:04:53 UTC (rev 2371)
@@ -23,13 +23,15 @@
DSHandle handle = null;
try {
handle = (DSHandle) PA_VAR.getValue(stack);
- String fn = argList(filename(handle), true);
- Reader freader = new FileReader(fn);
- BufferedReader breader = new BufferedReader(freader);
- String str = breader.readLine();
- freader.close();
- Double i = new Double(str);
- return RootDataNode.newNode(Types.FLOAT, i);
+ synchronized(handle.getRoot()) {
+ String fn = argList(filename(handle), true);
+ Reader freader = new FileReader(fn);
+ BufferedReader breader = new BufferedReader(freader);
+ String str = breader.readLine();
+ freader.close();
+ Double i = new Double(str);
+ return RootDataNode.newNode(Types.FLOAT, i);
+ }
}
catch (IOException ioe) {
throw new ExecutionException("Reading integer content of file", ioe);
From noreply at svn.ci.uchicago.edu Sun Dec 14 15:14:17 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 14 Dec 2008 15:14:17 -0600 (CST)
Subject: [Swift-commit] r2372 - in trunk:
src/org/griphyn/vdl/karajan/lib/swiftscript tests/misc
Message-ID: <20081214211417.E14C822819E@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-14 15:14:16 -0600 (Sun, 14 Dec 2008)
New Revision: 2372
Added:
trunk/tests/misc/extract-int-delayed-slowint.sh
trunk/tests/misc/extract-int-delayed.sh
trunk/tests/misc/extract-int-delayed.swift
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java
trunk/tests/misc/run
Log:
make @extractint work with files that are constructed during a run - previously @extractint did not properly defer its execution when passed an unclosed file
Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2008-12-14 20:04:53 UTC (rev 2371)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java 2008-12-14 21:14:16 UTC (rev 2372)
@@ -8,12 +8,14 @@
import org.globus.cog.karajan.arguments.Arg;
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
import org.griphyn.vdl.karajan.lib.VDLFunction;
import org.griphyn.vdl.mapping.DSHandle;
import org.griphyn.vdl.mapping.HandleOpenException;
import org.griphyn.vdl.mapping.RootDataNode;
import org.griphyn.vdl.type.Types;
+
public class ExtractInt extends VDLFunction {
static {
setArguments(ExtractInt.class, new Arg[] { PA_VAR });
@@ -24,6 +26,9 @@
try {
handle = (DSHandle) PA_VAR.getValue(stack);
synchronized(handle.getRoot()) {
+ if (!handle.isClosed()) {
+ throw new FutureNotYetAvailable(addFutureListener(stack, handle));
+ }
String fn = argList(filename(handle), true);
Reader freader = new FileReader(fn);
BufferedReader breader = new BufferedReader(freader);
Added: trunk/tests/misc/extract-int-delayed-slowint.sh
===================================================================
--- trunk/tests/misc/extract-int-delayed-slowint.sh (rev 0)
+++ trunk/tests/misc/extract-int-delayed-slowint.sh 2008-12-14 21:14:16 UTC (rev 2372)
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# helper for extractint test - output something suitable for
+# extractint, but after substantial delay
+
+sleep 10s
+echo 121
Property changes on: trunk/tests/misc/extract-int-delayed-slowint.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/misc/extract-int-delayed.sh
===================================================================
--- trunk/tests/misc/extract-int-delayed.sh (rev 0)
+++ trunk/tests/misc/extract-int-delayed.sh 2008-12-14 21:14:16 UTC (rev 2372)
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# generate an integer in a file, where that file is generated by a slow
+# executing procedure invocation
+
+# this is intended to test how well extractint works with
+
+# TODO should also make tests for readData and readData2
+
+rm -f *.kml
+
+echo "localhost slowint $(pwd)/extract-int-delayed-slowint.sh INSTALLED INTEL32::LINUX null" > ./extract-int-delayed-tc.data
+
+swift -tc.file ./extract-int-delayed-tc.data extract-int-delayed.swift
Property changes on: trunk/tests/misc/extract-int-delayed.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/misc/extract-int-delayed.swift
===================================================================
--- trunk/tests/misc/extract-int-delayed.swift (rev 0)
+++ trunk/tests/misc/extract-int-delayed.swift 2008-12-14 21:14:16 UTC (rev 2372)
@@ -0,0 +1,23 @@
+
+type file;
+
+app (file intermediate) p() {
+ slowint stdout=@intermediate;
+}
+
+file f = p();
+
+
+// TODO need two cases (at least) - one with assignment in declaration
+// and one with assignment decoupled from declaration
+
+// the decoupled case gets further along that the single-statement
+// case, at time of writing.
+
+// int i = @extractint(f);
+
+int i;
+i = @extractint(f);
+
+trace(i);
+
Modified: trunk/tests/misc/run
===================================================================
--- trunk/tests/misc/run 2008-12-14 20:04:53 UTC (rev 2371)
+++ trunk/tests/misc/run 2008-12-14 21:14:16 UTC (rev 2372)
@@ -1,7 +1,7 @@
#!/bin/sh
for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 restart-iterate workernode-local \
ordering-extern-notlazy restart-extern ordering-extern \
-external-mapper-args \
+external-mapper-args extract-int-delayed \
; do
./${a}.sh
R=$?
From noreply at svn.ci.uchicago.edu Sun Dec 14 15:22:30 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 14 Dec 2008 15:22:30 -0600 (CST)
Subject: [Swift-commit] r2373 - trunk/tests/language-behaviour
Message-ID: <20081214212230.EAB39228193@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-14 15:22:30 -0600 (Sun, 14 Dec 2008)
New Revision: 2373
Added:
trunk/tests/language-behaviour/delayedcat.sh
Modified:
trunk/tests/language-behaviour/generate-tc.data
Log:
delayedcat test helper script for use in future tests
Added: trunk/tests/language-behaviour/delayedcat.sh
===================================================================
--- trunk/tests/language-behaviour/delayedcat.sh (rev 0)
+++ trunk/tests/language-behaviour/delayedcat.sh 2008-12-14 21:22:30 UTC (rev 2373)
@@ -0,0 +1,3 @@
+#!/bin/bash
+sleep 10s
+cat $@
Modified: trunk/tests/language-behaviour/generate-tc.data
===================================================================
--- trunk/tests/language-behaviour/generate-tc.data 2008-12-14 21:14:16 UTC (rev 2372)
+++ trunk/tests/language-behaviour/generate-tc.data 2008-12-14 21:22:30 UTC (rev 2373)
@@ -3,3 +3,8 @@
for cmd in wc touch sleep; do
echo "localhost $cmd $(which $cmd) INSTALLED INTEL32::LINUX null" >> ./tc.data
done
+
+for cmd in delayedcat; do
+ echo "localhost $cmd $(pwd)/${cmd}.sh INSTALLED INTEL32::LINUX null" >> ./tc.data
+done
+
From noreply at svn.ci.uchicago.edu Mon Dec 15 17:55:15 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 15 Dec 2008 17:55:15 -0600 (CST)
Subject: [Swift-commit] r2374 - in trunk: libexec resources
src/org/griphyn/vdl/karajan/lib
Message-ID: <20081215235515.31A02228114@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-15 17:55:12 -0600 (Mon, 15 Dec 2008)
New Revision: 2374
Added:
trunk/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java
Modified:
trunk/libexec/vdl-lib.xml
trunk/resources/Karajan.stg
trunk/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java
Log:
separate out arrayiterator construction from getting a field value
Modified: trunk/libexec/vdl-lib.xml
===================================================================
--- trunk/libexec/vdl-lib.xml 2008-12-14 21:22:30 UTC (rev 2373)
+++ trunk/libexec/vdl-lib.xml 2008-12-15 23:55:12 UTC (rev 2374)
@@ -45,6 +45,7 @@
+
Modified: trunk/resources/Karajan.stg
===================================================================
--- trunk/resources/Karajan.stg 2008-12-14 21:22:30 UTC (rev 2373)
+++ trunk/resources/Karajan.stg 2008-12-15 23:55:12 UTC (rev 2374)
@@ -201,7 +201,7 @@
foreach(var,in,indexVar,declarations,statements) ::= <<
- $in$
+ $in$
Added: trunk/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/GetArrayIterator.java 2008-12-15 23:55:12 UTC (rev 2374)
@@ -0,0 +1,57 @@
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
+import org.griphyn.vdl.karajan.PairIterator;
+import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.HandleOpenException;
+import org.griphyn.vdl.mapping.InvalidPathException;
+import org.griphyn.vdl.mapping.Path;
+
+public class GetArrayIterator extends VDLFunction {
+ public static final Logger logger = Logger.getLogger(GetArrayIterator.class);
+
+ static {
+ setArguments(GetArrayIterator.class, new Arg[] { PA_VAR, OA_PATH });
+ }
+
+ /**
+ * Takes a supplied variable and path, and returns an array iterator.
+ */
+ public Object function(VariableStack stack) throws ExecutionException {
+ Object var1 = PA_VAR.getValue(stack);
+ if (!(var1 instanceof DSHandle)) {
+ return var1;
+ }
+ DSHandle var = (DSHandle) var1;
+ try {
+ Path path = parsePath(OA_PATH.getValue(stack), stack);
+ if (path.hasWildcards()) {
+ throw new RuntimeException("Wildcards not supported");
+ }
+ else {
+ var = var.getField(path);
+ if (var.getType().isArray()) {
+ Map value = var.getArrayValue();
+ if (var.isClosed()) {
+ return new PairIterator(value);
+ }
+ else {
+ return addFutureListListener(stack, var, value);
+ }
+ } else {
+ throw new RuntimeException("Cannot get array iterator for non-array");
+ }
+ }
+ }
+ catch (InvalidPathException e) {
+ throw new ExecutionException(e);
+ }
+ }
+
+}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java 2008-12-14 21:22:30 UTC (rev 2373)
+++ trunk/src/org/griphyn/vdl/karajan/lib/GetFieldValue.java 2008-12-15 23:55:12 UTC (rev 2374)
@@ -10,7 +10,6 @@
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
-import org.griphyn.vdl.karajan.PairIterator;
import org.griphyn.vdl.mapping.DSHandle;
import org.griphyn.vdl.mapping.HandleOpenException;
import org.griphyn.vdl.mapping.InvalidPathException;
@@ -49,14 +48,7 @@
else {
var = var.getField(path);
if (var.getType().isArray()) {
- // this bit from GetArrayFieldValue
- Map value = var.getArrayValue();
- if (var.isClosed()) {
- return new PairIterator(value);
- }
- else {
- return addFutureListListener(stack, var, value);
- }
+ throw new RuntimeException("Getting value for array "+var+" which is not permitted.");
}
synchronized (var) {
if (!var.isClosed()) {
From noreply at svn.ci.uchicago.edu Mon Dec 15 18:02:44 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 15 Dec 2008 18:02:44 -0600 (CST)
Subject: [Swift-commit] r2375 - trunk/src/org/griphyn/vdl/mapping
Message-ID: <20081216000244.28CE9228193@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-15 18:02:43 -0600 (Mon, 15 Dec 2008)
New Revision: 2375
Modified:
trunk/src/org/griphyn/vdl/mapping/InvalidPathException.java
Log:
more detail in InvalidPath exception message
Modified: trunk/src/org/griphyn/vdl/mapping/InvalidPathException.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/InvalidPathException.java 2008-12-15 23:55:12 UTC (rev 2374)
+++ trunk/src/org/griphyn/vdl/mapping/InvalidPathException.java 2008-12-16 00:02:43 UTC (rev 2375)
@@ -5,8 +5,8 @@
public class InvalidPathException extends Exception {
public InvalidPathException(String path, DSHandle source) {
- super("Invalid path (" + path + ") for type "
- + source.getType());
+ super("Invalid path (" + path + ") for "
+ + source.toString());
}
public InvalidPathException(Path path, DSHandle source) {
From noreply at svn.ci.uchicago.edu Mon Dec 15 20:46:27 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 15 Dec 2008 20:46:27 -0600 (CST)
Subject: [Swift-commit] r2376 - in trunk: resources
src/org/griphyn/vdl/karajan/lib
Message-ID: <20081216024627.79D1B22810E@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-15 20:46:26 -0600 (Mon, 15 Dec 2008)
New Revision: 2376
Modified:
trunk/resources/Karajan.stg
trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
Log:
move field value retrieval into setfieldvalue implementation
Modified: trunk/resources/Karajan.stg
===================================================================
--- trunk/resources/Karajan.stg 2008-12-16 00:02:43 UTC (rev 2375)
+++ trunk/resources/Karajan.stg 2008-12-16 02:46:26 UTC (rev 2376)
@@ -289,11 +289,7 @@
$var$
-
-
$value$
-
>>
Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2008-12-16 00:02:43 UTC (rev 2375)
+++ trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2008-12-16 02:46:26 UTC (rev 2376)
@@ -26,7 +26,7 @@
try {
Path path = parsePath(OA_PATH.getValue(stack), stack);
DSHandle leaf = var.getField(path);
- Object value = PA_VALUE.getValue(stack);
+ DSHandle value = (DSHandle)PA_VALUE.getValue(stack);
if (logger.isInfoEnabled()) {
logger.info("Setting " + leaf + " to " + value);
}
@@ -39,10 +39,16 @@
// leaf.setValue(internalValue(leaf.getType(), value));
if( (value instanceof DSHandle && ((DSHandle)value).getType().isArray()) || (value instanceof PairIterator)) {
logger.warn("Warning: array assignment outside of initialisation does not work correctly.");
+ } else {
+ synchronized(value.getRoot()) {
+ if (!value.isClosed()) {
+ throw new FutureNotYetAvailable(addFutureListener(stack,value));
+ } else {
+ leaf.setValue(value.getValue());
+ closeShallow(stack, leaf);
+ }
+ }
}
-
- leaf.setValue(value);
- closeShallow(stack, leaf);
}
return null;
}
From noreply at svn.ci.uchicago.edu Tue Dec 16 17:31:37 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 16 Dec 2008 17:31:37 -0600 (CST)
Subject: [Swift-commit] r2377 - in trunk: src/org/griphyn/vdl/karajan/lib
src/org/griphyn/vdl/mapping tests/language-behaviour
Message-ID: <20081216233137.7A19A228193@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-16 17:31:36 -0600 (Tue, 16 Dec 2008)
New Revision: 2377
Added:
trunk/tests/language-behaviour/027-array-assignment.out.expected
trunk/tests/language-behaviour/027-array-assignment.swift
trunk/tests/language-behaviour/028-array-assignment.swift
trunk/tests/language-behaviour/array_multidimensional_assign.swift
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
Log:
makes it so you can assign arrays in SwiftScript in a similar way to assigning primitive values
Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2008-12-16 02:46:26 UTC (rev 2376)
+++ trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2008-12-16 23:31:36 UTC (rev 2377)
@@ -4,12 +4,16 @@
package org.griphyn.vdl.karajan.lib;
import org.apache.log4j.Logger;
+import org.griphyn.vdl.karajan.Pair;
+import org.griphyn.vdl.karajan.FuturePairIterator;
import org.griphyn.vdl.karajan.PairIterator;
+import org.griphyn.vdl.karajan.VDL2FutureException;
import org.globus.cog.karajan.arguments.Arg;
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.InvalidPathException;
import org.griphyn.vdl.mapping.Path;
public class SetFieldValue extends VDLFunction {
@@ -30,24 +34,17 @@
if (logger.isInfoEnabled()) {
logger.info("Setting " + leaf + " to " + value);
}
- synchronized (leaf) {
+ synchronized (var.getRoot()) {
// TODO want to do a type check here, for runtime type checking
// and pull out the appropriate internal value from value if it
// is a DSHandle. There is no need (I think? maybe numerical casting?)
// for type conversion here; but would be useful to have
// type checking.
- // leaf.setValue(internalValue(leaf.getType(), value));
- if( (value instanceof DSHandle && ((DSHandle)value).getType().isArray()) || (value instanceof PairIterator)) {
- logger.warn("Warning: array assignment outside of initialisation does not work correctly.");
- } else {
- synchronized(value.getRoot()) {
- if (!value.isClosed()) {
- throw new FutureNotYetAvailable(addFutureListener(stack,value));
- } else {
- leaf.setValue(value.getValue());
- closeShallow(stack, leaf);
- }
+ synchronized(value.getRoot()) {
+ if(!value.isClosed()) {
+ throw new FutureNotYetAvailable(addFutureListener(stack, value));
}
+ deepCopy(leaf,value,stack);
}
}
return null;
@@ -60,4 +57,31 @@
}
}
+ /** make dest look like source - if its a simple value, copy that
+ and if its an array then recursively copy */
+ void deepCopy(DSHandle dest, DSHandle source, VariableStack stack) throws ExecutionException {
+ if(source.getType().isPrimitive()) {
+ dest.setValue(source.getValue());
+ } else if(source.getType().isArray()) {
+ PairIterator it = new PairIterator(source.getArrayValue());
+ while(it.hasNext()) {
+ Pair pair = (Pair) it.next();
+ Object lhs = pair.get(0);
+ DSHandle rhs = (DSHandle) pair.get(1);
+ Path memberPath = Path.EMPTY_PATH.addLast(String.valueOf(lhs),true);
+ DSHandle field;
+ try {
+ field = dest.getField(memberPath);
+ } catch(InvalidPathException ipe) {
+ throw new ExecutionException("Could not get destination field",ipe);
+ }
+ deepCopy(field,rhs,stack);
+ }
+ closeShallow(stack, dest);
+ } else {
+ // TODO implement this
+ throw new RuntimeException("Deep non-array structure copying not implemented");
+ }
+ }
+
}
Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2008-12-16 02:46:26 UTC (rev 2376)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2008-12-16 23:31:36 UTC (rev 2377)
@@ -303,6 +303,7 @@
+ " is already assigned with a value of " + this.value);
}
this.value = value;
+ closeShallow();
}
public Collection getFringePaths() throws HandleOpenException {
Added: trunk/tests/language-behaviour/027-array-assignment.out.expected
===================================================================
--- trunk/tests/language-behaviour/027-array-assignment.out.expected (rev 0)
+++ trunk/tests/language-behaviour/027-array-assignment.out.expected 2008-12-16 23:31:36 UTC (rev 2377)
@@ -0,0 +1 @@
+two
Added: trunk/tests/language-behaviour/027-array-assignment.swift
===================================================================
--- trunk/tests/language-behaviour/027-array-assignment.swift (rev 0)
+++ trunk/tests/language-behaviour/027-array-assignment.swift 2008-12-16 23:31:36 UTC (rev 2377)
@@ -0,0 +1,15 @@
+type messagefile {}
+
+(messagefile t) greeting(string m[]) {
+ app {
+ echo m[1] stdout=@filename(t);
+ }
+}
+
+messagefile outfile <"027-array-assignment.out">;
+
+string msg[];
+msg = [ "one", "two" ];
+
+outfile = greeting(msg);
+
Added: trunk/tests/language-behaviour/028-array-assignment.swift
===================================================================
--- trunk/tests/language-behaviour/028-array-assignment.swift (rev 0)
+++ trunk/tests/language-behaviour/028-array-assignment.swift 2008-12-16 23:31:36 UTC (rev 2377)
@@ -0,0 +1,18 @@
+type messagefile {}
+
+(messagefile t) greeting(string m[]) {
+ app {
+ echo m[1] stdout=@filename(t);
+ }
+}
+
+messagefile outfile <"027-array-assignment.out">;
+
+string msg[];
+string baz[];
+
+baz = msg;
+msg = [ "one", "two" ];
+
+outfile = greeting(baz);
+
Added: trunk/tests/language-behaviour/array_multidimensional_assign.swift
===================================================================
--- trunk/tests/language-behaviour/array_multidimensional_assign.swift (rev 0)
+++ trunk/tests/language-behaviour/array_multidimensional_assign.swift 2008-12-16 23:31:36 UTC (rev 2377)
@@ -0,0 +1,16 @@
+type file {}
+
+(file t) echo_array (string s[][]) {
+ app {
+ echo s[0][0] s[1][0] s[1][1] stdout=@filename(t);
+ }
+}
+
+string greetings[][];
+
+greetings = [ [ "left", "right" ], ["up", "down"]];
+
+file hw <"array_multidimensional_assign.out">;
+
+hw = echo_array(greetings);
+
From noreply at svn.ci.uchicago.edu Fri Dec 19 10:33:32 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 19 Dec 2008 10:33:32 -0600 (CST)
Subject: [Swift-commit] r2378 - in trunk/src/org/griphyn/vdl: karajan/lib
karajan/lib/swiftscript mapping
Message-ID: <20081219163332.D4A7322810E@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-19 10:33:30 -0600 (Fri, 19 Dec 2008)
New Revision: 2378
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/New.java
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java
trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java
trunk/src/org/griphyn/vdl/mapping/RootDataNode.java
Log:
mapper initialization can now be deferred, instead of happening at dataset creation time; this means mappers can take future values as parameters
Modified: trunk/src/org/griphyn/vdl/karajan/lib/New.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/New.java 2008-12-16 23:31:36 UTC (rev 2377)
+++ trunk/src/org/griphyn/vdl/karajan/lib/New.java 2008-12-19 16:33:30 UTC (rev 2378)
@@ -123,9 +123,7 @@
}
else if (type != null) {
handle = new RootDataNode(type);
- if (mapping != null) {
- handle.init(mapping);
- }
+ handle.init(mapping);
if (value != null) {
handle.setValue(internalValue(type, value));
closeShallow(stack, handle);
Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2008-12-16 23:31:36 UTC (rev 2377)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java 2008-12-19 16:33:30 UTC (rev 2378)
@@ -3,6 +3,8 @@
import org.globus.cog.karajan.arguments.Arg;
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
+import org.griphyn.vdl.karajan.VDL2FutureException;
import org.griphyn.vdl.karajan.lib.VDLFunction;
import org.griphyn.vdl.mapping.RootDataNode;
import org.griphyn.vdl.type.Types;
@@ -13,7 +15,11 @@
}
public Object function(VariableStack stack) throws ExecutionException {
- String s = argList(filename(stack), true);
- return RootDataNode.newNode(Types.STRING, s);
+ try {
+ String s = argList(filename(stack), true);
+ return RootDataNode.newNode(Types.STRING, s);
+ } catch(VDL2FutureException ve) {
+ throw new FutureNotYetAvailable(addFutureListener(stack, ve.getHandle()));
+ }
}
}
Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2008-12-16 23:31:36 UTC (rev 2377)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2008-12-19 16:33:30 UTC (rev 2378)
@@ -18,6 +18,8 @@
import org.griphyn.vdl.type.Type;
import org.griphyn.vdl.type.Types;
+import org.griphyn.vdl.karajan.VDL2FutureException;
+
public abstract class AbstractDataNode implements DSHandle {
static final String DATASET_URI_PREFIX = "tag:benc at ci.uchicago.edu,2008:swift:dataset:";
@@ -348,7 +350,11 @@
notifyListeners();
logger.info("closed "+this.getIdentifier());
// so because its closed, we can dump the contents
- logContent();
+ try {
+ logContent();
+ } catch(Exception e) {
+ logger.warn("Exception whilst logging dataset content for "+this,e);
+ }
// TODO record retrospective provenance information for this dataset here
// we should do it at closing time because that's the point at which we
// know the dataset has its values (all the way down the tree) assigned.
@@ -368,15 +374,23 @@
if(this.getType().isPrimitive()) {
logger.info("VALUE dataset="+this.getIdentifier()+" VALUE="+this.toString());
}
- if(this.getMapper() != null) {
+
+ Mapper m;
+
+ try {
+ m = this.getMapper();
+ } catch(VDL2FutureException fe) {
+ m = null; // no mapping info if mapper isn't initialised yet
+ }
+ if(m != null) {
// TODO proper type here
// Not sure catching exception here is really the right thing to do here
// anyway - should perhaps only be trying to map leafnodes? Mapping
// non-leaf stuff is giving wierd paths anyway
try {
- Object path=this.getMapper().map(this.getPathFromRoot());
+ Object path=m.map(this.getPathFromRoot());
logger.info("FILENAME dataset="+this.getIdentifier()+" filename="+
- this.getMapper().map(this.getPathFromRoot()));
+ m.map(this.getPathFromRoot()));
} catch(Exception e) {
logger.info("dataset "+this.getIdentifier()+" exception while mapping path from root",e);
}
@@ -435,7 +449,8 @@
public synchronized void addListener(DSHandleListener listener) {
if (logger.isInfoEnabled()) {
- logger.info("Adding handle listener \"" + listener + "\" to \"" + this + "\"");
+Exception e = new Exception("To get stack trace");
+ logger.info("Adding handle listener \"" + listener + "\" to \"" + this + "\"", e);
}
if (listeners == null) {
listeners = new LinkedList();
Modified: trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2008-12-16 23:31:36 UTC (rev 2377)
+++ trunk/src/org/griphyn/vdl/mapping/RootArrayDataNode.java 2008-12-19 16:33:30 UTC (rev 2378)
@@ -1,6 +1,7 @@
package org.griphyn.vdl.mapping;
import java.util.Map;
+import java.util.Iterator;
import org.griphyn.vdl.karajan.VDL2FutureException;
import org.griphyn.vdl.type.Field;
@@ -8,6 +9,7 @@
public class RootArrayDataNode extends ArrayDataNode implements DSHandleListener {
+ private boolean initialized=false;
private Mapper mapper;
private Map params;
@@ -21,18 +23,40 @@
public void init(Map params) {
this.params = params;
+ if(this.params == null) {
+ initialized();
+ } else {
+ innerInit();
+ }
+ }
+
+ private void innerInit() {
+ Iterator i = params.entrySet().iterator();
+ while(i.hasNext()) {
+ Map.Entry entry = (Map.Entry) i.next();
+ Object k = entry.getKey();
+ Object v = entry.getValue();
+ if(v instanceof DSHandle && !( (DSHandle)v).isClosed()) {
+ DSHandle dh = (DSHandle)v;
+ dh.addListener(this);
+ return;
+ }
+ }
String desc = (String) params.get("descriptor");
if (desc == null) {
+ initialized();
return;
}
try {
mapper = MapperFactory.getMapper(desc, params);
checkInputs();
getField().setName(PARAM_PREFIX.getStringValue(mapper));
+ initialized();
}
catch (InvalidMapperException e) {
throw new RuntimeException(e);
}
+ notifyListeners();
}
private void checkInputs() {
@@ -46,10 +70,11 @@
setValue(new MappingDependentException(this, e));
closeShallow();
}
+ initialized();
}
public void handleClosed(DSHandle handle) {
- checkInputs();
+ innerInit();
}
public String getParam(String name) {
@@ -68,11 +93,24 @@
}
public Mapper getMapper() {
- return mapper;
+ if(initialized) {
+ return mapper;
+ } else {
+ throw new VDL2FutureException(this);
+ }
}
public boolean isArray() {
return true;
}
+ public void setValue(Object value) {
+ super.setValue(value);
+ initialized();
+ }
+
+ private void initialized() {
+ initialized=true;
+ }
+
}
Modified: trunk/src/org/griphyn/vdl/mapping/RootDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2008-12-16 23:31:36 UTC (rev 2377)
+++ trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2008-12-19 16:33:30 UTC (rev 2378)
@@ -12,6 +12,7 @@
public class RootDataNode extends AbstractDataNode implements DSHandleListener {
+ private boolean initialized=false;
private Mapper mapper;
private Map params;
@@ -29,18 +30,43 @@
public void init(Map params) {
this.params = params;
+ if(this.params == null) {
+ initialized();
+ } else {
+ innerInit();
+ }
+ }
+ /** must have this.params set to the appropriate parameters before
+ being called. */
+ private void innerInit() {
+
+ Iterator i = params.entrySet().iterator();
+ while(i.hasNext()) {
+ Map.Entry entry = (Map.Entry) i.next();
+ Object k = entry.getKey();
+ Object v = entry.getValue();
+ if(v instanceof DSHandle && !( (DSHandle)v).isClosed()) {
+ DSHandle dh = (DSHandle)v;
+ dh.addListener(this);
+ return;
+ }
+ }
String desc = (String) params.get("descriptor");
- if (desc == null)
+ if (desc == null) {
+ initialized();
return;
+ }
try {
mapper = MapperFactory.getMapper(desc, params);
checkInputs();
getField().setName(PARAM_PREFIX.getStringValue(mapper));
+ initialized();
}
catch (InvalidMapperException e) {
- throw new RuntimeException(e);
+ throw new RuntimeException("InvalidMapperException caught in mapper initialization", e);
}
+ notifyListeners();
}
private void checkInputs() {
@@ -48,20 +74,23 @@
checkInputs(params, mapper, this);
}
catch (VDL2FutureException e) {
- e.printStackTrace();
- e.getHandle().addListener(this);
+ logger.warn("Unexpected VDL2FutureException checking inputs");
+ throw new RuntimeException("Got a VDL2FutureException but all parameters should have their values",e);
}
catch (DependentException e) {
setValue(new MappingDependentException(this, e));
closeShallow();
+ return;
}
+ initialized();
}
public void handleClosed(DSHandle handle) {
- checkInputs();
+ innerInit();
}
- public static void checkInputs(Map params, Mapper mapper, AbstractDataNode root) {
+
+ static protected void checkInputs(Map params, Mapper mapper, AbstractDataNode root) {
String input = (String) params.get("input");
if (input != null && Boolean.valueOf(input.trim()).booleanValue()) {
Iterator i = mapper.existing().iterator();
@@ -167,11 +196,26 @@
return null;
}
+
+
public Mapper getMapper() {
- return mapper;
+ if(initialized) {
+ return mapper;
+ } else {
+ throw new VDL2FutureException(this);
+ }
}
public boolean isArray() {
return false;
}
+
+ public void setValue(Object value) {
+ super.setValue(value);
+ initialized();
+ }
+
+ private void initialized() {
+ initialized=true;
+ }
}
From noreply at svn.ci.uchicago.edu Fri Dec 19 13:48:05 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 19 Dec 2008 13:48:05 -0600 (CST)
Subject: [Swift-commit] r2379 -
provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky
Message-ID: <20081219194805.833DA22810E@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-19 13:48:03 -0600 (Fri, 19 Dec 2008)
New Revision: 2379
Modified:
provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java
Log:
tidy up source a bit
Modified: provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java
===================================================================
--- provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java 2008-12-19 16:33:30 UTC (rev 2378)
+++ provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java 2008-12-19 19:48:03 UTC (rev 2379)
@@ -74,7 +74,7 @@
throw new TaskSubmissionException("Task is not in unsubmitted state");
}
else {
-logger.info("Submitting wonky job "+jobNumber);
+ logger.info("Submitting wonky job "+jobNumber);
this.task = task;
@@ -82,7 +82,7 @@
.getServiceContact();
String server = serviceContact.getContact();
-logger.debug("Contact is: "+server);
+ logger.debug("Contact is: "+server);
String[] splitServer = server.split("/");
siteName = splitServer[0];
siteOptions = Arrays.asList(splitServer);
@@ -146,17 +146,17 @@
public void suspend() throws InvalidSecurityContextException,
TaskSubmissionException {
-logger.info("Suspend called on wonky job "+jobNumber);
+ logger.info("Suspend called on wonky job "+jobNumber);
}
public void resume() throws InvalidSecurityContextException,
TaskSubmissionException {
-logger.info("Resume called on wonky job "+jobNumber);
+ logger.info("Resume called on wonky job "+jobNumber);
}
public void cancel() throws InvalidSecurityContextException,
TaskSubmissionException {
-logger.info("Cancel called on wonky job "+jobNumber);
+ logger.info("Cancel called on wonky job "+jobNumber);
synchronized(this) {
killed = true;
process.destroy();
@@ -176,41 +176,39 @@
if (spec.getDirectory() != null) {
dir = new File(spec.getDirectory());
}
-logger.info("Wonky job in queue, job number "+jobNumber);
-String[] cmdarray = buildCmdArray(spec, jobNumber);
-try {
-logger.info("wonky site is "+siteName);
- if(siteOptions.contains("firstfast") && !firstJobList.contains(siteName)) {
-logger.info("not sleeping - this is the first job on site "+siteName+", "+jobNumber);
-firstJobList.add(siteName);
-} else
-{
-double qmean=0;
-double qstddev=0;
+ logger.info("Wonky job in queue, job number "+jobNumber);
+ String[] cmdarray = buildCmdArray(spec, jobNumber);
+ try {
+ logger.info("wonky site is "+siteName);
+ if(siteOptions.contains("firstfast") && !firstJobList.contains(siteName)) {
+ logger.info("not sleeping - this is the first job on site "+siteName+", "+jobNumber);
+ firstJobList.add(siteName);
+ } else {
+ double qmean=0;
+ double qstddev=0;
-Iterator it = siteOptions.iterator();
-while(it.hasNext()) {
-String op = (String) it.next();
-logger.debug("Checking: "+op);
-if(op.startsWith("qmean=")) {
-qmean = Double.parseDouble(op.substring(6));
-logger.debug("qmean = "+qmean);
-} else if(op.startsWith("qstddev=")) {
-qstddev = Double.parseDouble(op.substring(8));
-logger.debug("qstddev = "+qstddev);
-}
-}
+ Iterator it = siteOptions.iterator();
+ while(it.hasNext()) {
+ String op = (String) it.next();
+ logger.debug("Checking: "+op);
+ if(op.startsWith("qmean=")) {
+ qmean = Double.parseDouble(op.substring(6));
+ logger.debug("qmean = "+qmean);
+ } else if(op.startsWith("qstddev=")) {
+ qstddev = Double.parseDouble(op.substring(8));
+ logger.debug("qstddev = "+qstddev);
+ }
+ }
-double qdelay = qmean + qstddev * r.nextGaussian();
+ double qdelay = qmean + qstddev * r.nextGaussian();
-Thread.sleep((long)(qdelay*1000));
-}
-} catch(InterruptedException ie) {
-System.out.println("fake queue delay interrupted for job "+jobNumber);
-}
-System.out.println("Wonky job running now");
- process = Runtime.getRuntime().exec(cmdarray,
- buildEnvp(spec), dir);
+ Thread.sleep((long)(qdelay*1000));
+ }
+ } catch(InterruptedException ie) {
+ logger.warn("fake queue delay interrupted for job "+jobNumber);
+ }
+ logger.info("Wonky job running now");
+ process = Runtime.getRuntime().exec(cmdarray, buildEnvp(spec), dir);
if(!failDelay("active")) {
this.task.setStatus(Status.FAILED);
return;
@@ -389,7 +387,7 @@
double failFirstCount = getWonkyParam(name+"failfirstcount");
if(failFirstCount > 0) {
- System.out.println("State "+name+" will fail first "+failFirstCount+" times");
+ logger.info("State "+name+" will fail first "+failFirstCount+" times");
String key = getSiteName() + ":" + name + "failfirstcountsofar";
Integer iObj = (Integer)siteStates.get(key);
if(iObj == null) {
@@ -400,19 +398,19 @@
int i = iObj.intValue();
i++;
siteStates.put(key,new Integer(i));
- System.out.println("new value is "+i);
+ logger.info("new value is "+i);
if(i<=failFirstCount) { // not failed enough
- System.out.println("Failing because early job");
+ logger.info("Failing because early job");
return false;
}
}
- System.out.print("State "+name+" with probability of failure "+failRate+": ");
+ logger.info("State "+name+" with probability of failure "+failRate+": ");
if(Math.random()
Author: benc
Date: 2008-12-26 10:00:42 -0600 (Fri, 26 Dec 2008)
New Revision: 2380
Modified:
trunk/libexec/vdl-int.k
trunk/libexec/wrapper.sh
trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java
Log:
work around job execution systems that do not set job initial working directory as previously expected
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2008-12-19 19:48:03 UTC (rev 2379)
+++ trunk/libexec/vdl-int.k 2008-12-26 16:00:42 UTC (rev 2380)
@@ -393,7 +393,7 @@
vdl:setprogress("Submitting")
vdl:execute("/bin/bash",
- list("shared/wrapper.sh", jobid,
+ list("{sharedDir}/wrapper.sh", jobid,
"-jobdir", jobdir,
"-e", vdl:executable(tr, rhost),
"-out", stdout,
Modified: trunk/libexec/wrapper.sh
===================================================================
--- trunk/libexec/wrapper.sh 2008-12-19 19:48:03 UTC (rev 2379)
+++ trunk/libexec/wrapper.sh 2008-12-26 16:00:42 UTC (rev 2380)
@@ -87,7 +87,15 @@
}
COMMANDLINE=$@
-WFDIR=$PWD
+
+# get the parent directory of the directory containing wrapper.sh, to use
+# as the run directory
+# this assumes that wrapper.sh is being executed from the top level of
+# the shared directory, and that shared directory is in the top level
+# of the workflow run directory
+WFDIR=$(dirname $(dirname $0))
+
+cd $WFDIR
openinfo "wrapper.log"
ID=$1
checkEmpty "$ID" "Missing job ID"
Modified: trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2008-12-19 19:48:03 UTC (rev 2379)
+++ trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2008-12-26 16:00:42 UTC (rev 2380)
@@ -108,9 +108,9 @@
private void applyJobWorkDirectory(Task task, Contact[] contacts) {
JobSpecification spec = (JobSpecification) task.getSpecification();
String dir = spec.getDirectory();
+ BoundContact bc = (BoundContact) contacts[0];
+ String workdir = (String) bc.getProperty("workdir");
if (dir == null || !dir.startsWith("/")) {
- BoundContact bc = (BoundContact) contacts[0];
- String workdir = (String) bc.getProperty("workdir");
if (workdir != null) {
if (dir == null) {
spec.setDirectory(workdir);
@@ -120,6 +120,13 @@
}
}
}
+ List l = spec.getArgumentsAsList();
+ // perhaps should check for /bin/bash in the executable, or some other way of detecting we need to do a substitution here... or equally could assume that the second parameter always needs to undergo this substitution...
+ if(((String)l.get(0)).endsWith("shared/wrapper.sh")) {
+ String s = workdir+"/"+l.get(0);
+ l.set(0,s);
+ }
+
}
protected abstract void applyTCEntry(Task task, Contact[] contacts);
From noreply at svn.ci.uchicago.edu Tue Dec 30 00:45:13 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 30 Dec 2008 00:45:13 -0600 (CST)
Subject: [Swift-commit] r2382 - text/hpdc09submission
Message-ID: <20081230064513.8D2992281AD@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-30 00:45:11 -0600 (Tue, 30 Dec 2008)
New Revision: 2382
Modified:
text/hpdc09submission/paper.latex
Log:
id in draft paper for when people throw PDFs around
Modified: text/hpdc09submission/paper.latex
===================================================================
--- text/hpdc09submission/paper.latex 2008-12-30 06:32:10 UTC (rev 2381)
+++ text/hpdc09submission/paper.latex 2008-12-30 06:45:11 UTC (rev 2382)
@@ -26,6 +26,8 @@
\maketitle
+\verb|$Id$|
+
\begin{abstract}
SwiftScript is a scripting language used to describe the composition of
Property changes on: text/hpdc09submission/paper.latex
___________________________________________________________________
Name: svn:keywords
+ Id Revision
From noreply at svn.ci.uchicago.edu Tue Dec 30 07:46:26 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 30 Dec 2008 07:46:26 -0600 (CST)
Subject: [Swift-commit] r2383 - text/hpdc09submission
Message-ID: <20081230134626.1B38D2281E3@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-30 07:46:25 -0600 (Tue, 30 Dec 2008)
New Revision: 2383
Modified:
text/hpdc09submission/paper.latex
Log:
merge procedure and function table
Modified: text/hpdc09submission/paper.latex
===================================================================
--- text/hpdc09submission/paper.latex 2008-12-30 06:45:11 UTC (rev 2382)
+++ text/hpdc09submission/paper.latex 2008-12-30 13:46:25 UTC (rev 2383)
@@ -319,7 +319,7 @@
\subsection{Operators and built-in procedures and functions}
SwiftScript has a number of built in operators, procedures and functions
-are very briefly described here:
+are very briefly described in tables \ref{optable} and \ref{proctable}.
\begin{table}[htb]
\begin{tabular}{|c|p{2in}|}
@@ -339,6 +339,7 @@
\hline
\end{tabular}
\caption{SwiftScript operators}
+\label{optable}
\end{table}
% ops: @ [ ] .
@@ -351,13 +352,6 @@
\hline
trace & output trace debugging information \\
\hline
-\end{tabular}
-\caption{SwiftScript built-in procedures}
-\end{table}
-
-\begin{table}[htb]
-\begin{tabular}{|r|p{2in}|}
-\hline
@arg & returns a named commandline argument \\
\hline
@extractint & reads an integer from a file \\
@@ -374,7 +368,8 @@
@toint \\
\hline
\end{tabular}
-\caption{SwiftScript functions}
+\caption{SwiftScript procedures and functions}
+\label{proctable}
\end{table}
\subsection{The execution environment for component programs}
From noreply at svn.ci.uchicago.edu Tue Dec 30 08:00:22 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 30 Dec 2008 08:00:22 -0600 (CST)
Subject: [Swift-commit] r2384 - text/hpdc09submission
Message-ID: <20081230140022.C75E72281AD@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-30 08:00:22 -0600 (Tue, 30 Dec 2008)
New Revision: 2384
Modified:
text/hpdc09submission/paper.latex
Log:
abstract as submitted
Modified: text/hpdc09submission/paper.latex
===================================================================
--- text/hpdc09submission/paper.latex 2008-12-30 13:46:25 UTC (rev 2383)
+++ text/hpdc09submission/paper.latex 2008-12-30 14:00:22 UTC (rev 2384)
@@ -30,18 +30,31 @@
\begin{abstract}
-SwiftScript is a scripting language used to describe the composition of
-tens of thousands of serial component programs into distributed,
-parallelised, data-oriented applications. Drawing on experience from
-both scientific workflow and programming language domains, the language
-provides a high level representation of collections of data and how
-those collections are to be processed by component programs. Underlying
-this is an implementation to execute scripts on grid and other
-platforms, providing inbuillt site selection, data management and
-reliability.
+Scientists, engineers and business analysts often pursue their work by
+applying application programs to massive collections of file-based data.
+Distributed and parallel computing resources provide a powerful way to
+get more of this type of work done faster, but using such resources
+greatly increases the complexity of the computing task.
+Swift addresses this problem with a scripting language for composing
+ordinary application programs (serial or parallel) into more powerful
+distributed, parallelized applications. Applications expressed in Swift
+are location-independent and automatically parallelized.
+
+Swift can execute scripts that perform tens to hundreds of thousands of
+program invocations on highly parallel resources, and its conceptual
+design is intended to scale to applications that perform many millions
+of invocations, and beyond.
+
+The language provides a high level representation of collections of data
+and a specification of how those collections are to be mapped to an
+abstract representation and processed by component programs. Underlying
+this is an implementation that executes component applications on grids
+and other parallel platforms, providing automated site selection, data
+management and reliability.
+
We present the language, details of the implementation, application
-use-cases and ongoing research.
+examples, measurements and ongoing research.
\end{abstract}
From noreply at svn.ci.uchicago.edu Tue Dec 30 09:43:22 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 30 Dec 2008 09:43:22 -0600 (CST)
Subject: [Swift-commit] r2385 - text/hpdc09submission
Message-ID: <20081230154322.A21D2228198@www.ci.uchicago.edu>
Author: benc
Date: 2008-12-30 09:43:21 -0600 (Tue, 30 Dec 2008)
New Revision: 2385
Modified:
text/hpdc09submission/paper.latex
Log:
more stuff mostly about the language
Modified: text/hpdc09submission/paper.latex
===================================================================
--- text/hpdc09submission/paper.latex 2008-12-30 14:00:22 UTC (rev 2384)
+++ text/hpdc09submission/paper.latex 2008-12-30 15:43:21 UTC (rev 2385)
@@ -75,7 +75,10 @@
Data is represented in a script by strongly-typed single-assignment
variables, using a syntax familiar to many programmers.
- Data stored in POSIX-like files can be \emph{mapped} to a variable.
+ A variable may store data of \emph{primitive type} such as an
+integer or a string. However, a variable may also be \emph{mapper}
+to one or more POSIX-like files, allowing treatment of those files using
+the same syntax as other variables.
In that case, the variable declaration is annotated with a
\emph{mapping} describing the file(s) that make up that \emph{dataset}.
For example, this line declares a variable named \verb|photo| with
@@ -159,8 +162,10 @@
\subsection{Working with arrays}
-Arrays of values can be declared, with a mapper describing which files
-are to go into that array. For example, the \verb|filesys_mapper| maps
+Arrays of values can be declared using the \verb|[]| suffix. An array
+can be mapped to a collection of files, one element per file, by using
+a different form of mapping expression. For example, the
+\verb|filesys_mapper| maps
all files matching a particular unix glob pattern into an array:
\begin{verbatim}
@@ -250,6 +255,48 @@
\verb|a| will be regarded as closed since no other statements in the
script make an assignment to \verb|a|.
+\subsection{Compound procedures}
+
+As with many other programming languages, procedures consisting of SwiftScript
+code can be defined. These differ from the previously mentioned procedures
+declared with the \verb|app| keyword, as they invoke other SwiftScript
+procedures rather than a component program.
+
+ \begin{verbatim}
+ (file output) process (file input) {
+ file intermediate;
+ intermediate = first(input);
+ output = second(intermediate);
+ }
+
+ file x <"x.txt">;
+ file y <"y.txt">;
+ y = process(x);
+ \end{verbatim}
+
+This will invoke two procedures, with an intermediate data file named
+anonymously connecting the \verb|first| and \verb|second| procedures.
+
+Ordering of execution is generally determined by execution of \verb|app|
+procedures, not by any containing procedures. In this code block:
+
+ \begin{verbatim}
+ (file a, file b) A() {
+ a = A1();
+ b = A2();
+ }
+ file x, y, s, t;
+ (x,y) = A();
+ s = S(x);
+ t = S(y);
+ \end{verbatim}
+
+then a valid execution order is: \verb|A1 S(x) A2 S(y)|. The
+compound procedure \verb|A| does not have to have fully completed
+for its return values to be used by subsequent statements.
+
+TODO: talk about anonymous mapping somewhere - a mappers section...
+
\subsection{More about types}
Each variable and procedure parameter in SwiftScript is strongly typed.
@@ -286,6 +333,15 @@
o = p(brain.h);
\end{verbatim}
+Collections of files can be mapped to complex types using mappers, like
+for arrays. For example, the simple mapper used in this expression will
+map the files \verb|data.h| and \verb|data.v| to the variable members
+\verb|m.h| and \verb|m.v| respectively:
+
+ \begin{verbatim}
+ volume m ;
+ \end{verbatim}
+
Sometimes data may be stored in a form that does not fit with Swift's
file-and-site model; for example, data might be stored in an RDBMS on some
database server. In that case, a variable can be declared to have
@@ -329,10 +385,11 @@
TODO mappings may be to URLs, not only to local filesystem files; and more
explicit description of what mapping is.
-\subsection{Operators and built-in procedures and functions}
+\subsection{Operators, built-in procedures, functions and mappers}
-SwiftScript has a number of built in operators, procedures and functions
-are very briefly described in tables \ref{optable} and \ref{proctable}.
+SwiftScript has a number of built in operators, procedures, functions
+are very briefly described in tables \ref{optable}, \ref{proctable}
+and \ref{mappertable}.
\begin{table}[htb]
\begin{tabular}{|c|p{2in}|}
@@ -385,6 +442,20 @@
\label{proctable}
\end{table}
+\begin{table}[htb]
+\begin{tabular}{|r|p{2in}|}
+\hline
+\verb|single_file_mapper| & maps a single explicitly named file \\
+\hline
+\verb|filesys_mapper| & maps files matching a pattern into an array \\
+\hline
+\verb|simple_mapper| & maps files to arbitrarily nested data types based on
+ components of the file name \\
+\hline
+\end{tabular}
+\caption{SwiftScript built-in mappers}
+\end{table}
+
\subsection{The execution environment for component programs}
A SwiftScript \verb|app| declaration describes how a component
@@ -784,14 +855,19 @@
active development group; releases roughly every 2 months.
+\section{Acknowledgements}
+TODO: authors beyond number 3 go here according to ACM style guide, rather
+than in header
+
+TODO: NSF/DOE grant acknowledgements
+
\section{TODO}
Reference Swift as a follow-on project to VDL in VDS; how does XDTM fit
into this? Is it of any interest other than as part of the
project history?
- NSF/DOE grant acknowledgements
Acknowledgement of all developers names?