From swift at ci.uchicago.edu  Thu May  3 11:15:28 2012
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Thu,  3 May 2012 11:15:28 -0500 (CDT)
Subject: [Swift-commit] Cog update
Message-ID: <20120503161528.77E138D00D56@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3339 | davidkelly999 | 2011-12-23 16:30:53 -0600 (Fri, 23 Dec 2011) | 2 lines

Add optional job submission delay

------------------------------------------------------------------------
Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java
===================================================================
--- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java	(revision 3338)
+++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java	(working copy)
@@ -17,10 +17,11 @@
 	public static final String PROPERTIES = "provider-sge.properties";
 	public static final String POLL_INTERVAL = "poll.interval";
 	public static final String QSUB = "qsub";
-	public static final String QSTAT = "qstat -xml";
+	public static final String QSTAT = "qstat";
 	public static final String QDEL = "qdel";
 	public static final String QCONF = "qconf";
 	public static final String DEFAULT_PE = "parallel.environment"; 
+	public static final String SUBMISSION_DELAY = "submission.delay";
 	private static Properties properties;
 
 	/**
@@ -67,6 +68,15 @@
 	}
 
 	/**
+	 * getSubmissionDelay - Get length to sleep before submitting a job
+	 * Value as a string representing milliseconds
+	 * @return Submission delay as String in milliseconds
+	 */
+	public String getSubmissionDelay() {
+		return getProperty(SUBMISSION_DELAY);
+	}
+	
+	/**
 	 * getSubmitCommandName - Get submit command
 	 * @return String of submit command
 	 */
@@ -100,5 +110,14 @@
 		setRemoveCommand("qdel");
 		setDefaultPE("threaded");
 		setConfigCommand("qconf");
+		setSubmissionDelay("0");
 	}
+
+	/**
+	 * setSubmissionDelay - set the submission delay
+	 * @param delay String representing milliseconds to sleep 
+	 */
+	private void setSubmissionDelay(String delay) {
+		setProperty(SUBMISSION_DELAY, delay);
+	}
 }
Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java
===================================================================
--- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java	(revision 3338)
+++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java	(working copy)
@@ -8,6 +8,7 @@
 
 import java.io.BufferedReader;
 import java.io.CharArrayReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.Writer;
 import java.text.DecimalFormat;
@@ -24,11 +25,14 @@
 import org.globus.cog.abstraction.impl.scheduler.common.AbstractProperties;
 import org.globus.cog.abstraction.impl.scheduler.common.AbstractQueuePoller;
 import org.globus.cog.abstraction.impl.scheduler.common.Job;
+import org.globus.cog.abstraction.impl.scheduler.common.ProcessException;
 import org.globus.cog.abstraction.impl.scheduler.common.ProcessListener;
 import org.globus.cog.abstraction.impl.scheduler.pbs.PBSExecutor;
 import org.globus.cog.abstraction.interfaces.FileLocation;
 import org.globus.cog.abstraction.interfaces.JobSpecification;
 import org.globus.cog.abstraction.interfaces.Task;
+import org.globus.gsi.gssapi.auth.AuthorizationException;
+import org.ietf.jgss.GSSException;
 
 /**
  * Java CoG interface for Sun/Oracle Grid Engine
@@ -179,6 +183,20 @@
     }
 
     /**
+     * @see AbstractExecutor#start()
+     */
+    public void start() throws AuthorizationException, GSSException,
+           IOException, ProcessException {
+    	try {
+    		Thread.sleep(Integer.valueOf(getSGEProperties().getSubmissionDelay()));
+    	}
+    	catch (InterruptedException e) {
+    		logger.error(e.getStackTrace());
+    	}
+    	super.start();
+    }
+    
+    /**
      * Check that job specification values are valid for this system
      * @throws IllegalArgumentException
      */
Index: modules/provider-localscheduler/etc/provider-sge.properties
===================================================================
--- modules/provider-localscheduler/etc/provider-sge.properties	(revision 3338)
+++ modules/provider-localscheduler/etc/provider-sge.properties	(working copy)
@@ -13,14 +13,13 @@
 #
 # The path to qstat. The default assumes that qstat is in PATH
 #
-qstat=qstat
+qstat=qstat -xml
 
 #
 # The path to qdel. The default assumes that qdel is in PATH
 #
 qdel=qdel
 
-
 #
 # If the jobType attribute is specified, then the SGE provider
 # will look for a property named "wrapper.<jobType>" and prepend
@@ -36,6 +35,10 @@
 # a default parallel environment. It can be overriden using
 # the "pe" job attribute
 #
-
 parallel.environment=1way
 
+# Some systems (notably Ranger) may not adequately handle
+# a rapid submission of jobs. Use this setting to introduce
+# a delay. The value is in milliseconds (1000ms = 1s)
+#
+submission.delay=1000


From swift at ci.uchicago.edu  Thu May  3 11:21:32 2012
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Thu,  3 May 2012 11:21:32 -0500 (CDT)
Subject: [Swift-commit] Cog update
Message-ID: <20120503162132.4A6808D00D56@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3339 | davidkelly999 | 2011-12-23 16:30:53 -0600 (Fri, 23 Dec 2011) | 2 lines

Add optional job submission delay

------------------------------------------------------------------------
Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java
===================================================================
--- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java	(revision 3338)
+++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java	(working copy)
@@ -17,10 +17,11 @@
 	public static final String PROPERTIES = "provider-sge.properties";
 	public static final String POLL_INTERVAL = "poll.interval";
 	public static final String QSUB = "qsub";
-	public static final String QSTAT = "qstat -xml";
+	public static final String QSTAT = "qstat";
 	public static final String QDEL = "qdel";
 	public static final String QCONF = "qconf";
 	public static final String DEFAULT_PE = "parallel.environment"; 
+	public static final String SUBMISSION_DELAY = "submission.delay";
 	private static Properties properties;
 
 	/**
@@ -67,6 +68,15 @@
 	}
 
 	/**
+	 * getSubmissionDelay - Get length to sleep before submitting a job
+	 * Value as a string representing milliseconds
+	 * @return Submission delay as String in milliseconds
+	 */
+	public String getSubmissionDelay() {
+		return getProperty(SUBMISSION_DELAY);
+	}
+	
+	/**
 	 * getSubmitCommandName - Get submit command
 	 * @return String of submit command
 	 */
@@ -100,5 +110,14 @@
 		setRemoveCommand("qdel");
 		setDefaultPE("threaded");
 		setConfigCommand("qconf");
+		setSubmissionDelay("0");
 	}
+
+	/**
+	 * setSubmissionDelay - set the submission delay
+	 * @param delay String representing milliseconds to sleep 
+	 */
+	private void setSubmissionDelay(String delay) {
+		setProperty(SUBMISSION_DELAY, delay);
+	}
 }
Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java
===================================================================
--- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java	(revision 3338)
+++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java	(working copy)
@@ -8,6 +8,7 @@
 
 import java.io.BufferedReader;
 import java.io.CharArrayReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.Writer;
 import java.text.DecimalFormat;
@@ -24,11 +25,14 @@
 import org.globus.cog.abstraction.impl.scheduler.common.AbstractProperties;
 import org.globus.cog.abstraction.impl.scheduler.common.AbstractQueuePoller;
 import org.globus.cog.abstraction.impl.scheduler.common.Job;
+import org.globus.cog.abstraction.impl.scheduler.common.ProcessException;
 import org.globus.cog.abstraction.impl.scheduler.common.ProcessListener;
 import org.globus.cog.abstraction.impl.scheduler.pbs.PBSExecutor;
 import org.globus.cog.abstraction.interfaces.FileLocation;
 import org.globus.cog.abstraction.interfaces.JobSpecification;
 import org.globus.cog.abstraction.interfaces.Task;
+import org.globus.gsi.gssapi.auth.AuthorizationException;
+import org.ietf.jgss.GSSException;
 
 /**
  * Java CoG interface for Sun/Oracle Grid Engine
@@ -179,6 +183,20 @@
     }
 
     /**
+     * @see AbstractExecutor#start()
+     */
+    public void start() throws AuthorizationException, GSSException,
+           IOException, ProcessException {
+    	try {
+    		Thread.sleep(Integer.valueOf(getSGEProperties().getSubmissionDelay()));
+    	}
+    	catch (InterruptedException e) {
+    		logger.error(e.getStackTrace());
+    	}
+    	super.start();
+    }
+    
+    /**
      * Check that job specification values are valid for this system
      * @throws IllegalArgumentException
      */
Index: modules/provider-localscheduler/etc/provider-sge.properties
===================================================================
--- modules/provider-localscheduler/etc/provider-sge.properties	(revision 3338)
+++ modules/provider-localscheduler/etc/provider-sge.properties	(working copy)
@@ -13,14 +13,13 @@
 #
 # The path to qstat. The default assumes that qstat is in PATH
 #
-qstat=qstat
+qstat=qstat -xml
 
 #
 # The path to qdel. The default assumes that qdel is in PATH
 #
 qdel=qdel
 
-
 #
 # If the jobType attribute is specified, then the SGE provider
 # will look for a property named "wrapper.<jobType>" and prepend
@@ -36,6 +35,10 @@
 # a default parallel environment. It can be overriden using
 # the "pe" job attribute
 #
-
 parallel.environment=1way
 
+# Some systems (notably Ranger) may not adequately handle
+# a rapid submission of jobs. Use this setting to introduce
+# a delay. The value is in milliseconds (1000ms = 1s)
+#
+submission.delay=1000


From swift at ci.uchicago.edu  Thu May  3 11:21:36 2012
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Thu,  3 May 2012 11:21:36 -0500 (CDT)
Subject: [Swift-commit] Cog update
Message-ID: <20120503162136.AD9248D00D56@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3339 | davidkelly999 | 2011-12-23 16:30:53 -0600 (Fri, 23 Dec 2011) | 2 lines

Add optional job submission delay

------------------------------------------------------------------------
Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java
===================================================================
--- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java	(revision 3338)
+++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/Properties.java	(working copy)
@@ -17,10 +17,11 @@
 	public static final String PROPERTIES = "provider-sge.properties";
 	public static final String POLL_INTERVAL = "poll.interval";
 	public static final String QSUB = "qsub";
-	public static final String QSTAT = "qstat -xml";
+	public static final String QSTAT = "qstat";
 	public static final String QDEL = "qdel";
 	public static final String QCONF = "qconf";
 	public static final String DEFAULT_PE = "parallel.environment"; 
+	public static final String SUBMISSION_DELAY = "submission.delay";
 	private static Properties properties;
 
 	/**
@@ -67,6 +68,15 @@
 	}
 
 	/**
+	 * getSubmissionDelay - Get length to sleep before submitting a job
+	 * Value as a string representing milliseconds
+	 * @return Submission delay as String in milliseconds
+	 */
+	public String getSubmissionDelay() {
+		return getProperty(SUBMISSION_DELAY);
+	}
+	
+	/**
 	 * getSubmitCommandName - Get submit command
 	 * @return String of submit command
 	 */
@@ -100,5 +110,14 @@
 		setRemoveCommand("qdel");
 		setDefaultPE("threaded");
 		setConfigCommand("qconf");
+		setSubmissionDelay("0");
 	}
+
+	/**
+	 * setSubmissionDelay - set the submission delay
+	 * @param delay String representing milliseconds to sleep 
+	 */
+	private void setSubmissionDelay(String delay) {
+		setProperty(SUBMISSION_DELAY, delay);
+	}
 }
Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java
===================================================================
--- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java	(revision 3338)
+++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/sge/SGEExecutor.java	(working copy)
@@ -8,6 +8,7 @@
 
 import java.io.BufferedReader;
 import java.io.CharArrayReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.Writer;
 import java.text.DecimalFormat;
@@ -24,11 +25,14 @@
 import org.globus.cog.abstraction.impl.scheduler.common.AbstractProperties;
 import org.globus.cog.abstraction.impl.scheduler.common.AbstractQueuePoller;
 import org.globus.cog.abstraction.impl.scheduler.common.Job;
+import org.globus.cog.abstraction.impl.scheduler.common.ProcessException;
 import org.globus.cog.abstraction.impl.scheduler.common.ProcessListener;
 import org.globus.cog.abstraction.impl.scheduler.pbs.PBSExecutor;
 import org.globus.cog.abstraction.interfaces.FileLocation;
 import org.globus.cog.abstraction.interfaces.JobSpecification;
 import org.globus.cog.abstraction.interfaces.Task;
+import org.globus.gsi.gssapi.auth.AuthorizationException;
+import org.ietf.jgss.GSSException;
 
 /**
  * Java CoG interface for Sun/Oracle Grid Engine
@@ -179,6 +183,20 @@
     }
 
     /**
+     * @see AbstractExecutor#start()
+     */
+    public void start() throws AuthorizationException, GSSException,
+           IOException, ProcessException {
+    	try {
+    		Thread.sleep(Integer.valueOf(getSGEProperties().getSubmissionDelay()));
+    	}
+    	catch (InterruptedException e) {
+    		logger.error(e.getStackTrace());
+    	}
+    	super.start();
+    }
+    
+    /**
      * Check that job specification values are valid for this system
      * @throws IllegalArgumentException
      */
Index: modules/provider-localscheduler/etc/provider-sge.properties
===================================================================
--- modules/provider-localscheduler/etc/provider-sge.properties	(revision 3338)
+++ modules/provider-localscheduler/etc/provider-sge.properties	(working copy)
@@ -13,14 +13,13 @@
 #
 # The path to qstat. The default assumes that qstat is in PATH
 #
-qstat=qstat
+qstat=qstat -xml
 
 #
 # The path to qdel. The default assumes that qdel is in PATH
 #
 qdel=qdel
 
-
 #
 # If the jobType attribute is specified, then the SGE provider
 # will look for a property named "wrapper.<jobType>" and prepend
@@ -36,6 +35,10 @@
 # a default parallel environment. It can be overriden using
 # the "pe" job attribute
 #
-
 parallel.environment=1way
 
+# Some systems (notably Ranger) may not adequately handle
+# a rapid submission of jobs. Use this setting to introduce
+# a delay. The value is in milliseconds (1000ms = 1s)
+#
+submission.delay=1000


From swift at ci.uchicago.edu  Thu May  3 11:22:20 2012
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Thu,  3 May 2012 11:22:20 -0500 (CDT)
Subject: [Swift-commit] Cog update
Message-ID: <20120503162220.1661F8D00D56@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3400 | hategan | 2012-04-30 22:51:26 -0500 (Mon, 30 Apr 2012) | 1 line

slightly faster and cleaner variable frame cachin code
------------------------------------------------------------------------
Index: modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java
===================================================================
--- modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java	(revision 3399)
+++ modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java	(working copy)
@@ -40,44 +40,35 @@
 
 	public Object function(VariableStack stack) throws ExecutionException {
 		synchronized (this) {
-			if (name == null) {
-				name = TypeUtil.toString(A_NAME.getStatic(this)).toLowerCase();
-				if (getStaticArguments().size() == 1) {
-					java.util.Map<String, Object> m = Collections.emptyMap();
-					setStaticArguments(m);
+			if (frame == UNINITIALIZED || frame == VariableStack.NO_FRAME) {
+				if (name == null) {
+					name = TypeUtil.toString(A_NAME.getStatic(this)).toLowerCase();
+					if (getStaticArguments().size() == 1) {
+						java.util.Map<String, Object> m = Collections.emptyMap();
+						setStaticArguments(m);
+					}
 				}
-			}
-		}
-
-		switch (frame) {
-			case UNINITIALIZED:
-			case VariableStack.NO_FRAME:
+			
 				if (stack.parentFrame().isDefined("#quoted")) {
 					Object value = new Identifier(name);
 					setSimple(value);
 					return value;
 				}
-				else {
-					frame = stack.getVarFrameFromTop(name);
-					switch (frame) {
-					    case VariableStack.NO_FRAME:
-					        throw new VariableNotFoundException(name);
-					    case VariableStack.FIRST_FRAME:
-					        Object value = stack.firstFrame().getVar(name);
-					        setSimple(value);
-					        return value;
-					    case VariableStack.DYNAMIC_FRAME:
-					        return stack.getVar(name);
-					    default:
-					        return stack.getFrameFromTop(frame).getVar(name);
-					}
-				}
+				frame = stack.getVarFrameFromTop(name);
+			}
+		}
+		
+		switch (frame) {
+			case VariableStack.NO_FRAME:
+				throw new VariableNotFoundException(name);
 			case VariableStack.FIRST_FRAME:
-			    return stack.firstFrame().getVar(name);
+				Object value = stack.firstFrame().getVar(name);
+				setSimple(value);
+				return value;
 			case VariableStack.DYNAMIC_FRAME:
-			    return stack.getVar(name);
+				return stack.getVar(name);
 			default:
-			    return stack.getFrameFromTop(frame).getVar(name);
+				return stack.getFrameFromTop(frame).getVar(name);
 		}
 	}
 }
Index: modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java
===================================================================
--- modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java	(revision 3399)
+++ modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java	(working copy)
@@ -130,24 +130,17 @@
 			this.name = ident.getName();
 		}
 
-		public synchronized Object getValue(VariableStack stack) throws VariableNotFoundException {
+		public Object getValue(VariableStack stack) throws VariableNotFoundException {
+			synchronized(this) {
+				if (frame == UNINITIALIZED || frame == VariableStack.NO_FRAME) {
+					frame = stack.getVarFrameFromTop(name);
+				}
+			}
 			switch (frame) {
-				case UNINITIALIZED:
 				case VariableStack.NO_FRAME:
-					frame = stack.getVarFrameFromTop(name);
-					switch (frame) {
-						case VariableStack.NO_FRAME:
-							throw new VariableNotFoundException(name);
-						case VariableStack.FIRST_FRAME:
-							value = stack.firstFrame().getVar(name);
-							return value;
-						case VariableStack.DYNAMIC_FRAME:
-							return stack.getVar(name);
-						default:
-							return stack.getFrameFromTop(frame).getVar(name);
-					}
+					throw new VariableNotFoundException(name);
 				case VariableStack.FIRST_FRAME:
-					return value;
+					return stack.firstFrame().getVar(name);
 				case VariableStack.DYNAMIC_FRAME:
 					return stack.getVar(name);
 				default:


From davidk at ci.uchicago.edu  Thu May  3 11:24:15 2012
From: davidk at ci.uchicago.edu (David Kelly)
Date: Thu, 3 May 2012 11:24:15 -0500 (CDT)
Subject: [Swift-commit] Cog update
In-Reply-To: <20120503162220.1661F8D00D56@bridled.ci.uchicago.edu>
Message-ID: <808170027.12470.1336062255358.JavaMail.root@zimbra-mb2.anl.gov>

Thank you buggy svn script for sending cog updates from months ago. Very useful. Not. Who would write such a bugg.. oh, wait, nevermind.

----- Original Message -----
> From: swift at ci.uchicago.edu
> To: swift-commit at ci.uchicago.edu
> Sent: Thursday, May 3, 2012 11:22:20 AM
> Subject: [Swift-commit] Cog update
> ------------------------------------------------------------------------
> r3400 | hategan | 2012-04-30 22:51:26 -0500 (Mon, 30 Apr 2012) | 1
> line
> 
> slightly faster and cleaner variable frame cachin code
> ------------------------------------------------------------------------
> Index:
> modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java
> ===================================================================
> ---
> modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java
> (revision 3399)
> +++
> modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java
> (working copy)
> @@ -40,44 +40,35 @@
> 
> public Object function(VariableStack stack) throws ExecutionException
> {
> synchronized (this) {
> - if (name == null) {
> - name = TypeUtil.toString(A_NAME.getStatic(this)).toLowerCase();
> - if (getStaticArguments().size() == 1) {
> - java.util.Map<String, Object> m = Collections.emptyMap();
> - setStaticArguments(m);
> + if (frame == UNINITIALIZED || frame == VariableStack.NO_FRAME) {
> + if (name == null) {
> + name = TypeUtil.toString(A_NAME.getStatic(this)).toLowerCase();
> + if (getStaticArguments().size() == 1) {
> + java.util.Map<String, Object> m = Collections.emptyMap();
> + setStaticArguments(m);
> + }
> }
> - }
> - }
> -
> - switch (frame) {
> - case UNINITIALIZED:
> - case VariableStack.NO_FRAME:
> +
> if (stack.parentFrame().isDefined("#quoted")) {
> Object value = new Identifier(name);
> setSimple(value);
> return value;
> }
> - else {
> - frame = stack.getVarFrameFromTop(name);
> - switch (frame) {
> - case VariableStack.NO_FRAME:
> - throw new VariableNotFoundException(name);
> - case VariableStack.FIRST_FRAME:
> - Object value = stack.firstFrame().getVar(name);
> - setSimple(value);
> - return value;
> - case VariableStack.DYNAMIC_FRAME:
> - return stack.getVar(name);
> - default:
> - return stack.getFrameFromTop(frame).getVar(name);
> - }
> - }
> + frame = stack.getVarFrameFromTop(name);
> + }
> + }
> +
> + switch (frame) {
> + case VariableStack.NO_FRAME:
> + throw new VariableNotFoundException(name);
> case VariableStack.FIRST_FRAME:
> - return stack.firstFrame().getVar(name);
> + Object value = stack.firstFrame().getVar(name);
> + setSimple(value);
> + return value;
> case VariableStack.DYNAMIC_FRAME:
> - return stack.getVar(name);
> + return stack.getVar(name);
> default:
> - return stack.getFrameFromTop(frame).getVar(name);
> + return stack.getFrameFromTop(frame).getVar(name);
> }
> }
> }
> Index:
> modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java
> ===================================================================
> ---
> modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java
> (revision 3399)
> +++
> modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java
> (working copy)
> @@ -130,24 +130,17 @@
> this.name = ident.getName();
> }
> 
> - public synchronized Object getValue(VariableStack stack) throws
> VariableNotFoundException {
> + public Object getValue(VariableStack stack) throws
> VariableNotFoundException {
> + synchronized(this) {
> + if (frame == UNINITIALIZED || frame == VariableStack.NO_FRAME) {
> + frame = stack.getVarFrameFromTop(name);
> + }
> + }
> switch (frame) {
> - case UNINITIALIZED:
> case VariableStack.NO_FRAME:
> - frame = stack.getVarFrameFromTop(name);
> - switch (frame) {
> - case VariableStack.NO_FRAME:
> - throw new VariableNotFoundException(name);
> - case VariableStack.FIRST_FRAME:
> - value = stack.firstFrame().getVar(name);
> - return value;
> - case VariableStack.DYNAMIC_FRAME:
> - return stack.getVar(name);
> - default:
> - return stack.getFrameFromTop(frame).getVar(name);
> - }
> + throw new VariableNotFoundException(name);
> case VariableStack.FIRST_FRAME:
> - return value;
> + return stack.firstFrame().getVar(name);
> case VariableStack.DYNAMIC_FRAME:
> return stack.getVar(name);
> default:
> _______________________________________________
> Swift-commit mailing list
> Swift-commit at ci.uchicago.edu
> https://lists.ci.uchicago.edu/cgi-bin/mailman/listinfo/swift-commit


From swift at ci.uchicago.edu  Thu May  3 12:08:45 2012
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Thu,  3 May 2012 12:08:45 -0500 (CDT)
Subject: [Swift-commit] Cog update
Message-ID: <20120503170845.56C1A8D00D56@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3339 | davidkelly999 | 2011-12-23 16:30:53 -0600 (Fri, 23 Dec 2011) | 2 lines

Add optional job submission delay

------------------------------------------------------------------------


From swift at ci.uchicago.edu  Thu May  3 12:09:46 2012
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Thu,  3 May 2012 12:09:46 -0500 (CDT)
Subject: [Swift-commit] Cog update
Message-ID: <20120503170946.DB75F8D00D56@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3400 | hategan | 2012-04-30 22:51:26 -0500 (Mon, 30 Apr 2012) | 1 line

slightly faster and cleaner variable frame cachin code
------------------------------------------------------------------------
Index: modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java
===================================================================
--- modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java	(revision 3399)
+++ modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java	(working copy)
@@ -40,44 +40,35 @@
 
 	public Object function(VariableStack stack) throws ExecutionException {
 		synchronized (this) {
-			if (name == null) {
-				name = TypeUtil.toString(A_NAME.getStatic(this)).toLowerCase();
-				if (getStaticArguments().size() == 1) {
-					java.util.Map<String, Object> m = Collections.emptyMap();
-					setStaticArguments(m);
+			if (frame == UNINITIALIZED || frame == VariableStack.NO_FRAME) {
+				if (name == null) {
+					name = TypeUtil.toString(A_NAME.getStatic(this)).toLowerCase();
+					if (getStaticArguments().size() == 1) {
+						java.util.Map<String, Object> m = Collections.emptyMap();
+						setStaticArguments(m);
+					}
 				}
-			}
-		}
-
-		switch (frame) {
-			case UNINITIALIZED:
-			case VariableStack.NO_FRAME:
+			
 				if (stack.parentFrame().isDefined("#quoted")) {
 					Object value = new Identifier(name);
 					setSimple(value);
 					return value;
 				}
-				else {
-					frame = stack.getVarFrameFromTop(name);
-					switch (frame) {
-					    case VariableStack.NO_FRAME:
-					        throw new VariableNotFoundException(name);
-					    case VariableStack.FIRST_FRAME:
-					        Object value = stack.firstFrame().getVar(name);
-					        setSimple(value);
-					        return value;
-					    case VariableStack.DYNAMIC_FRAME:
-					        return stack.getVar(name);
-					    default:
-					        return stack.getFrameFromTop(frame).getVar(name);
-					}
-				}
+				frame = stack.getVarFrameFromTop(name);
+			}
+		}
+		
+		switch (frame) {
+			case VariableStack.NO_FRAME:
+				throw new VariableNotFoundException(name);
 			case VariableStack.FIRST_FRAME:
-			    return stack.firstFrame().getVar(name);
+				Object value = stack.firstFrame().getVar(name);
+				setSimple(value);
+				return value;
 			case VariableStack.DYNAMIC_FRAME:
-			    return stack.getVar(name);
+				return stack.getVar(name);
 			default:
-			    return stack.getFrameFromTop(frame).getVar(name);
+				return stack.getFrameFromTop(frame).getVar(name);
 		}
 	}
 }
Index: modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java
===================================================================
--- modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java	(revision 3399)
+++ modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java	(working copy)
@@ -130,24 +130,17 @@
 			this.name = ident.getName();
 		}
 
-		public synchronized Object getValue(VariableStack stack) throws VariableNotFoundException {
+		public Object getValue(VariableStack stack) throws VariableNotFoundException {
+			synchronized(this) {
+				if (frame == UNINITIALIZED || frame == VariableStack.NO_FRAME) {
+					frame = stack.getVarFrameFromTop(name);
+				}
+			}
 			switch (frame) {
-				case UNINITIALIZED:
 				case VariableStack.NO_FRAME:
-					frame = stack.getVarFrameFromTop(name);
-					switch (frame) {
-						case VariableStack.NO_FRAME:
-							throw new VariableNotFoundException(name);
-						case VariableStack.FIRST_FRAME:
-							value = stack.firstFrame().getVar(name);
-							return value;
-						case VariableStack.DYNAMIC_FRAME:
-							return stack.getVar(name);
-						default:
-							return stack.getFrameFromTop(frame).getVar(name);
-					}
+					throw new VariableNotFoundException(name);
 				case VariableStack.FIRST_FRAME:
-					return value;
+					return stack.firstFrame().getVar(name);
 				case VariableStack.DYNAMIC_FRAME:
 					return stack.getVar(name);
 				default:


From swift at ci.uchicago.edu  Thu May  3 12:10:26 2012
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Thu,  3 May 2012 12:10:26 -0500 (CDT)
Subject: [Swift-commit] Cog update
Message-ID: <20120503171027.E62E88D00D56@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3400 | hategan | 2012-04-30 22:51:26 -0500 (Mon, 30 Apr 2012) | 1 line

slightly faster and cleaner variable frame cachin code
------------------------------------------------------------------------
Index: modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java
===================================================================
--- modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java	(revision 3399)
+++ modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/Variable.java	(working copy)
@@ -40,44 +40,35 @@
 
 	public Object function(VariableStack stack) throws ExecutionException {
 		synchronized (this) {
-			if (name == null) {
-				name = TypeUtil.toString(A_NAME.getStatic(this)).toLowerCase();
-				if (getStaticArguments().size() == 1) {
-					java.util.Map<String, Object> m = Collections.emptyMap();
-					setStaticArguments(m);
+			if (frame == UNINITIALIZED || frame == VariableStack.NO_FRAME) {
+				if (name == null) {
+					name = TypeUtil.toString(A_NAME.getStatic(this)).toLowerCase();
+					if (getStaticArguments().size() == 1) {
+						java.util.Map<String, Object> m = Collections.emptyMap();
+						setStaticArguments(m);
+					}
 				}
-			}
-		}
-
-		switch (frame) {
-			case UNINITIALIZED:
-			case VariableStack.NO_FRAME:
+			
 				if (stack.parentFrame().isDefined("#quoted")) {
 					Object value = new Identifier(name);
 					setSimple(value);
 					return value;
 				}
-				else {
-					frame = stack.getVarFrameFromTop(name);
-					switch (frame) {
-					    case VariableStack.NO_FRAME:
-					        throw new VariableNotFoundException(name);
-					    case VariableStack.FIRST_FRAME:
-					        Object value = stack.firstFrame().getVar(name);
-					        setSimple(value);
-					        return value;
-					    case VariableStack.DYNAMIC_FRAME:
-					        return stack.getVar(name);
-					    default:
-					        return stack.getFrameFromTop(frame).getVar(name);
-					}
-				}
+				frame = stack.getVarFrameFromTop(name);
+			}
+		}
+		
+		switch (frame) {
+			case VariableStack.NO_FRAME:
+				throw new VariableNotFoundException(name);
 			case VariableStack.FIRST_FRAME:
-			    return stack.firstFrame().getVar(name);
+				Object value = stack.firstFrame().getVar(name);
+				setSimple(value);
+				return value;
 			case VariableStack.DYNAMIC_FRAME:
-			    return stack.getVar(name);
+				return stack.getVar(name);
 			default:
-			    return stack.getFrameFromTop(frame).getVar(name);
+				return stack.getFrameFromTop(frame).getVar(name);
 		}
 	}
 }
Index: modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java
===================================================================
--- modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java	(revision 3399)
+++ modules/karajan/src/org/globus/cog/karajan/util/ElementProperty.java	(working copy)
@@ -130,24 +130,17 @@
 			this.name = ident.getName();
 		}
 
-		public synchronized Object getValue(VariableStack stack) throws VariableNotFoundException {
+		public Object getValue(VariableStack stack) throws VariableNotFoundException {
+			synchronized(this) {
+				if (frame == UNINITIALIZED || frame == VariableStack.NO_FRAME) {
+					frame = stack.getVarFrameFromTop(name);
+				}
+			}
 			switch (frame) {
-				case UNINITIALIZED:
 				case VariableStack.NO_FRAME:
-					frame = stack.getVarFrameFromTop(name);
-					switch (frame) {
-						case VariableStack.NO_FRAME:
-							throw new VariableNotFoundException(name);
-						case VariableStack.FIRST_FRAME:
-							value = stack.firstFrame().getVar(name);
-							return value;
-						case VariableStack.DYNAMIC_FRAME:
-							return stack.getVar(name);
-						default:
-							return stack.getFrameFromTop(frame).getVar(name);
-					}
+					throw new VariableNotFoundException(name);
 				case VariableStack.FIRST_FRAME:
-					return value;
+					return stack.firstFrame().getVar(name);
 				case VariableStack.DYNAMIC_FRAME:
 					return stack.getVar(name);
 				default:


From wozniak at ci.uchicago.edu  Fri May  4 14:51:43 2012
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri,  4 May 2012 14:51:43 -0500 (CDT)
Subject: [Swift-commit] r5784 - usertools/plotter/src/plotter
Message-ID: <20120504195143.0DDF99CD1C@svn.ci.uchicago.edu>

Author: wozniak
Date: 2012-05-04 14:51:42 -0500 (Fri, 04 May 2012)
New Revision: 5784

Modified:
   usertools/plotter/src/plotter/Lines.java
   usertools/plotter/src/plotter/Util.java
Log:
Support annotations


Modified: usertools/plotter/src/plotter/Lines.java
===================================================================
--- usertools/plotter/src/plotter/Lines.java	2012-05-01 03:53:59 UTC (rev 5783)
+++ usertools/plotter/src/plotter/Lines.java	2012-05-04 19:51:42 UTC (rev 5784)
@@ -13,6 +13,7 @@
 import org.apache.xmlgraphics.java2d.ps.EPSDocumentGraphics2D;
 import org.jfree.chart.ChartFactory;
 import org.jfree.chart.JFreeChart;
+import org.jfree.chart.annotations.XYTextAnnotation;
 import org.jfree.chart.axis.LogarithmicAxis;
 import org.jfree.chart.axis.NumberAxis;
 import org.jfree.chart.plot.PlotOrientation;
@@ -51,6 +52,9 @@
 
   static boolean withLegend = true;
 
+  static List<XYTextAnnotation> notes =
+      new ArrayList<XYTextAnnotation>();
+
   /**
        Args: Lines <properties> <output file> <data file>*
        Reads settings from properties: see scanProperties()
@@ -165,6 +169,7 @@
        PlotOrientation.VERTICAL, withLegend, false, false);
 
     setupPlot(chart, collection);
+
     chart.draw(g2d, rectangle);
 
     try
@@ -235,6 +240,9 @@
     setAxes(plot);
     plot.setRenderer(renderer);
     plot.setBackgroundPaint(Color.WHITE);
+
+    for (XYTextAnnotation note : notes)
+      plot.addAnnotation(note);
   }
 
   static void setAxes(XYPlot plot)
@@ -345,8 +353,28 @@
     tmp = properties.getProperty("axis.y");
     if (tmp != null)
       axis_y_type = tmp;
+    tmp = properties.getProperty("notes");
+    if (tmp != null)
+      loadNotes();
   }
 
+  static void loadNotes()
+  {
+    String s = properties.getProperty("notes");
+    int count = Integer.parseInt(s);
+    for (int i = 0; i < count; i++)
+    {
+      String p = "note."+i;
+      String d = properties.getProperty(p);
+      String[] tokens = d.split("\\s");
+      double x = Double.parseDouble(tokens[0]);
+      double y = Double.parseDouble(tokens[1]);
+      String text = Util.concat(tokens, 2);
+      XYTextAnnotation note = new XYTextAnnotation(text, x, y);
+      notes.add(note);
+    }
+  }
+
   static void load(String propFile)
   {
     try

Modified: usertools/plotter/src/plotter/Util.java
===================================================================
--- usertools/plotter/src/plotter/Util.java	2012-05-01 03:53:59 UTC (rev 5783)
+++ usertools/plotter/src/plotter/Util.java	2012-05-04 19:51:42 UTC (rev 5784)
@@ -1,5 +1,7 @@
 package plotter;
 
+import java.util.List;
+
 /**
  * Plot data helpers.
  * */
@@ -24,6 +26,52 @@
     return sb.toString();
   }
 
+  public static String concat(List<String> tokens)
+  {
+    String[] array = new String[tokens.size()];
+    tokens.toArray(array);
+    return concat(array, 0, " ");
+  }
+
+  public static String concat(String... strings)
+  {
+    return concat(' ', strings);
+  }
+
+  public static String concat(String[] tokens, int start)
+  {
+    return concat(tokens, start, " ");
+  }
+
+  public static String concat(String[] tokens, int start,
+                              String separator)
+  {
+    if (tokens == null)
+      return "null";
+    StringBuilder sb = new StringBuilder();
+    for (int i = start; i < tokens.length; i++)
+    {
+      sb.append(tokens[i]);
+      if (i < tokens.length - 1)
+        sb.append(separator);
+    }
+    return sb.toString();
+  }
+
+  public static String concat(char c, String... strings)
+  {
+    if (strings == null)
+      return "null";
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0; i < strings.length; i++)
+    {
+      sb.append(strings[i]);
+      if (i < strings.length - 1)
+        sb.append(c);
+    }
+    return sb.toString();
+  }
+
   public static void fatal(String s)
   {
     System.err.println(s);



From davidk at ci.uchicago.edu  Wed May  9 17:18:29 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed,  9 May 2012 17:18:29 -0500 (CDT)
Subject: [Swift-commit] r5785 - branches/release-0.93/tests/sites/mcs
Message-ID: <20120509221829.095C59CC90@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-09 17:18:28 -0500 (Wed, 09 May 2012)
New Revision: 5785

Modified:
   branches/release-0.93/tests/sites/mcs/coaster-service.conf
Log:
Update to mcs coaster-service.conf for documentation purposes


Modified: branches/release-0.93/tests/sites/mcs/coaster-service.conf
===================================================================
--- branches/release-0.93/tests/sites/mcs/coaster-service.conf	2012-05-04 19:51:42 UTC (rev 5784)
+++ branches/release-0.93/tests/sites/mcs/coaster-service.conf	2012-05-09 22:18:28 UTC (rev 5785)
@@ -1,48 +1,45 @@
-# Keep all interesting settings in one place
-# User should modify this to fit environment
-
 # Location of SWIFT. If empty, PATH is referenced
 export SWIFT=
 
 # Where to place/launch worker.pl on the remote machine for sites.xml
-export WORKER_LOCATION=/sandbox/${USER}
+export WORKER_LOCATION=/home/${USER}/work
 
 # How to launch workers: local, ssh, or cobalt
 export WORKER_MODE=ssh
 
 # Worker logging setting passed to worker.pl for sites.xml
-export WORKER_LOGGING=INFO
+export WORKER_LOGGING_LEVEL=INFO
 
 # User name to use for all systems
 export WORKER_USERNAME=$USER
 
 # Worker host names for ssh
-export WORKER_HOSTS="crush.mcs.anl.gov thwomp.mcs.anl.gov stomp.mcs.anl.gov crank.mcs.anl.gov
-steamroller.mcs.anl.gov grind.mcs.anl.gov churn.mcs.anl.gov trounce.mcs.anl.gov
-thrash.mcs.anl.gov vanquish.mcs.anl.gov"
+export WORKER_HOSTS="crush thwomp stomp crank steamroller grind churn trounce thrash vanquish"
 
 # Directory to keep log files, relative to working directory when launching start-coaster-service
 export LOG_DIR=logs
-export WORKER_LOG_DIR=/home/ketan/worker.log
+export WORKER_LOG_DIR=/home/${USER}/work
 
 # Manually define ports. If not specified, ports will be automatically generated
 export LOCAL_PORT=
 export SERVICE_PORT=
 
-# Set shared filesystem to no since work will be done in local /sandbox directory
-export SHARED_FILESYSTEM=no
+# Set shared filesystem to no if work will be done in local /sandbox directory
+# Then specify -config cf in command line
+export SHARED_FILESYSTEM=yes
 
 # start-coaster-service tries to automatically detect IP address. 
 # Specify here if auto detection is not working correctly
 export IPADDR=
 
 # Below are various settings to give information about how to create sites.xml
- export WORK=/sandbox/${USER}
- export JOBS_PER_NODE=8
- export JOB_THROTTLE=$( echo "scale=5; ($JOBS_PER_NODE * $( echo $WORKER_HOSTS | wc -w ))/100 - 0.00001"|bc )
+export WORK=/home/${USER}/work
+export JOBS_PER_NODE=4
 
+# Try to determine throttle automatically based on the number of nodes and jobs per node
+export JOB_THROTTLE=$( echo "scale=5; ($JOBS_PER_NODE * $( echo $WORKER_HOSTS | wc -w ))/100 - 0.00001"|bc )
+
 # Swift applications
 #app cat=/bin/cat
 #app bash=/bin/bash
 #app echo=/bin/echo
-



From davidk at ci.uchicago.edu  Thu May 10 23:23:58 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 10 May 2012 23:23:58 -0500 (CDT)
Subject: [Swift-commit] r5787 - trunk/bin
Message-ID: <20120511042358.B8E259CFD9@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-10 23:23:58 -0500 (Thu, 10 May 2012)
New Revision: 5787

Modified:
   trunk/bin/start-coaster-service
   trunk/bin/stop-coaster-service
Log:
Let start-coaster scripts work with non-OSG condor providers (UC3)


Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service	2012-05-10 12:27:54 UTC (rev 5786)
+++ trunk/bin/start-coaster-service	2012-05-11 04:23:58 UTC (rev 5787)
@@ -257,6 +257,35 @@
    return 0
 }
 
+# Start condor workers
+start-workers-condor()
+{
+   PORT=$1
+   EXECUTION_URL=http://$IPADDR:$PORT
+   if [ -z "$PORT" ]; then
+      crash "start-workers-local: Port number not specified, giving up"
+   fi
+   echo Starting worker on local machine
+
+   if [ -z "$CONDOR_WORKERS" ]; then
+      crash "CONDOR_WORKERS undefined - please check coaster-service.conf"
+   fi
+
+cat <<EOF > start_workers.submit
+output = workers.stdout
+error = workers.stderr
+initialdir = $PWD 
+executable = $SWIFT_BIN/$WORKER
+arguments = $EXECUTION_URL node .
+notification = Never
+leave_in_queue = TRUE
+queue $CONDOR_WORKERS
+EOF
+
+   condor_submit start_workers.submit
+   return 0
+}
+
 # Start cobalt workers
 start-workers-cobalt()
 {
@@ -445,6 +474,9 @@
    ec2)
       start-workers-ec2 $LOCAL_PORT
       ;;
+   condor)
+      start-workers-condor $LOCAL_PORT
+      ;;
    *)
       crash "Unknown WORKER_MODE. Please modify coaster-service.conf"
       ;;

Modified: trunk/bin/stop-coaster-service
===================================================================
--- trunk/bin/stop-coaster-service	2012-05-10 12:27:54 UTC (rev 5786)
+++ trunk/bin/stop-coaster-service	2012-05-11 04:23:58 UTC (rev 5787)
@@ -60,6 +60,10 @@
    fi
 fi
 
+if [ "$WORKER_MODE" == "condor" ]; then
+   condor_rm $USER
+fi
+
 if [ -f "$HOME/.swift/machines" ]; then
    rm "$HOME/.swift/machines"
 fi



From davidk at ci.uchicago.edu  Thu May 10 23:28:38 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 10 May 2012 23:28:38 -0500 (CDT)
Subject: [Swift-commit] r5788 - in trunk/tests/sites: . uc3
Message-ID: <20120511042838.DD96C9CFD9@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-10 23:28:38 -0500 (Thu, 10 May 2012)
New Revision: 5788

Added:
   trunk/tests/sites/uc3/
   trunk/tests/sites/uc3/001-catsn.check.sh
   trunk/tests/sites/uc3/001-catsn.clean.sh
   trunk/tests/sites/uc3/001-catsn.setup.sh
   trunk/tests/sites/uc3/001-catsn.swift
   trunk/tests/sites/uc3/001-catsn.timeout
   trunk/tests/sites/uc3/catsn.0001.out.expected
   trunk/tests/sites/uc3/catsn.0002.out.expected
   trunk/tests/sites/uc3/catsn.0003.out.expected
   trunk/tests/sites/uc3/catsn.0004.out.expected
   trunk/tests/sites/uc3/catsn.0005.out.expected
   trunk/tests/sites/uc3/catsn.0006.out.expected
   trunk/tests/sites/uc3/catsn.0007.out.expected
   trunk/tests/sites/uc3/catsn.0008.out.expected
   trunk/tests/sites/uc3/catsn.0009.out.expected
   trunk/tests/sites/uc3/catsn.0010.out.expected
   trunk/tests/sites/uc3/coaster-service.conf
   trunk/tests/sites/uc3/data.txt
   trunk/tests/sites/uc3/title.txt
Log:
Site test scripts for uc3


Added: trunk/tests/sites/uc3/001-catsn.check.sh
===================================================================
--- trunk/tests/sites/uc3/001-catsn.check.sh	                        (rev 0)
+++ trunk/tests/sites/uc3/001-catsn.check.sh	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -x
+
+for count in `seq --format "%04.f" 1 1 10`
+do
+	[ -f catsn.$count.out ] || exit 1
+	CONTENTS1=$( cat catsn.$count.out.expected )
+	CONTENTS2=$( cat catsn.$count.out )
+	[[ $CONTENTS1 == $CONTENTS2 ]] || exit 1
+done
+exit 0


Property changes on: trunk/tests/sites/uc3/001-catsn.check.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: trunk/tests/sites/uc3/001-catsn.clean.sh
===================================================================
--- trunk/tests/sites/uc3/001-catsn.clean.sh	                        (rev 0)
+++ trunk/tests/sites/uc3/001-catsn.clean.sh	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+stop-coaster-service


Property changes on: trunk/tests/sites/uc3/001-catsn.clean.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: trunk/tests/sites/uc3/001-catsn.setup.sh
===================================================================
--- trunk/tests/sites/uc3/001-catsn.setup.sh	                        (rev 0)
+++ trunk/tests/sites/uc3/001-catsn.setup.sh	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+mkdir -p ~/work
+cp -v $GROUP/data.txt . || exit 1
+cp -v $GROUP/*expected . || exit 1
+export WORK=$HOME/work
+start-coaster-service || exit 1


Property changes on: trunk/tests/sites/uc3/001-catsn.setup.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: trunk/tests/sites/uc3/001-catsn.swift
===================================================================
--- trunk/tests/sites/uc3/001-catsn.swift	                        (rev 0)
+++ trunk/tests/sites/uc3/001-catsn.swift	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1,12 @@
+type file;
+
+app (file o) cat (file i)
+{
+  cat @i stdout=@o;
+}
+
+file out[]<simple_mapper; location=".", prefix="catsn.",suffix=".out">;
+foreach j in [1:@toint(@arg("n","10"))] {
+  file data<"data.txt">;
+  out[j] = cat(data);
+}

Added: trunk/tests/sites/uc3/001-catsn.timeout
===================================================================
--- trunk/tests/sites/uc3/001-catsn.timeout	                        (rev 0)
+++ trunk/tests/sites/uc3/001-catsn.timeout	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+180

Added: trunk/tests/sites/uc3/catsn.0001.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0001.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0001.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0002.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0002.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0002.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0003.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0003.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0003.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0004.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0004.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0004.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0005.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0005.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0005.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0006.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0006.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0006.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0007.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0007.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0007.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0008.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0008.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0008.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0009.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0009.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0009.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/catsn.0010.out.expected
===================================================================
--- trunk/tests/sites/uc3/catsn.0010.out.expected	                        (rev 0)
+++ trunk/tests/sites/uc3/catsn.0010.out.expected	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/coaster-service.conf
===================================================================
--- trunk/tests/sites/uc3/coaster-service.conf	                        (rev 0)
+++ trunk/tests/sites/uc3/coaster-service.conf	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1,37 @@
+# Location of SWIFT. If empty, PATH is searched
+export SWIFT=
+
+# Where to copy worker.pl on the remote machine for sites.xml
+export WORKER_LOCATION=$HOME/swiftwork
+
+# How to launch workers: local, ssh, cobalt, or futuregrid
+export WORKER_MODE=condor
+export CONDOR_WORKERS="240"
+
+# Do all the worker nodes you're using have a shared filesystem? (yes/no)
+export SHARED_FILESYSTEM=yes
+
+# Enable SSH tunneling? (yes/no)
+export SSH_TUNNELING=no
+
+# Directory to keep log files, relative to working directory when launching start-coaster-service
+export LOG_DIR=logs
+
+# Manually define ports. If not specified, an available port will be used
+export LOCAL_PORT=
+export SERVICE_PORT=
+
+# Swift information for creating sites.xml
+export WORK=$HOME/swiftwork
+
+# Jobs per node is usually equal to the number of CPUs per node
+export JOBS_PER_NODE=1
+
+# To calculate job throttle:
+# jobs per node * nodes - 0.1 / 100
+export JOB_THROTTLE=$( echo "scale=5; ($JOBS_PER_NODE * $CONDOR_WORKERS)/100 - 0.00001"|bc )
+
+# Swift applications
+#app cat=/bin/cat
+#app bash=/bin/bash
+#app echo=/bin/echo

Added: trunk/tests/sites/uc3/data.txt
===================================================================
--- trunk/tests/sites/uc3/data.txt	                        (rev 0)
+++ trunk/tests/sites/uc3/data.txt	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+Hello world

Added: trunk/tests/sites/uc3/title.txt
===================================================================
--- trunk/tests/sites/uc3/title.txt	                        (rev 0)
+++ trunk/tests/sites/uc3/title.txt	2012-05-11 04:28:38 UTC (rev 5788)
@@ -0,0 +1 @@
+UC3 with Persistent Coasters and Condor



From davidk at ci.uchicago.edu  Fri May 11 12:47:43 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri, 11 May 2012 12:47:43 -0500 (CDT)
Subject: [Swift-commit] r5789 - trunk/docs/siteguide
Message-ID: <20120511174743.0A22B9CD1B@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-11 12:47:42 -0500 (Fri, 11 May 2012)
New Revision: 5789

Added:
   trunk/docs/siteguide/uc3
Modified:
   trunk/docs/siteguide/siteguide.txt
Log:
siteguide documentation for uc3


Modified: trunk/docs/siteguide/siteguide.txt
===================================================================
--- trunk/docs/siteguide/siteguide.txt	2012-05-11 04:28:38 UTC (rev 5788)
+++ trunk/docs/siteguide/siteguide.txt	2012-05-11 17:47:42 UTC (rev 5789)
@@ -23,3 +23,5 @@
 include::mcs[]
 
 include::grid[]
+
+include::uc3[]

Added: trunk/docs/siteguide/uc3
===================================================================
--- trunk/docs/siteguide/uc3	                        (rev 0)
+++ trunk/docs/siteguide/uc3	2012-05-11 17:47:42 UTC (rev 5789)
@@ -0,0 +1,53 @@
+x86 Cluster: UC3
+----------------
+
+Create a coaster-service.conf
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+To begin, copy the text below and paste it into your Swift distribution's etc
+directory. Name the file coaster-service.conf.
+
+-----
+include::../../tests/providers/uc3/coaster-service.conf[]
+-----
+
+Starting the Coaster Service
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Change directories to the location you would like to run a
+Swift script and start the coaster service with this
+command:
+
+-----
+start-coaster-service
+-----
+
+This will create a configuration file that Swift needs
+called sites.xml.
+
+WARNING: Any existing sites.xml files in this directory
+will be overwritten. Be sure to make a copy of any
+custom configuration files you may have.
+
+Run Swift
+~~~~~~~~~
+
+Next, run Swift. If you do not have a particular script
+in mind, you can test Swift by using a Swift script in
+the examples/ directory.
+
+Run the following command to run the script:
+-----
+swift -sites.file sites.xml -tc.file tc.data yourscript.swift
+-----
+
+Stopping the Coaster Service
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The coaster service will run indefinitely. The stop-coaster-service
+script will terminate the coaster service.
+
+-----
+$ stop-coaster-service
+-----
+
+This will kill the coaster service and kill the worker scripts on remote systems.
+



From davidk at ci.uchicago.edu  Tue May 22 07:27:09 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 22 May 2012 07:27:09 -0500 (CDT)
Subject: [Swift-commit] r5790 - in trunk: bin tests/sites tests/sites/osg
Message-ID: <20120522122709.5A2789CD2D@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-22 07:27:09 -0500 (Tue, 22 May 2012)
New Revision: 5790

Added:
   trunk/tests/sites/osg/
   trunk/tests/sites/osg/coaster-service.conf
Modified:
   trunk/bin/start-coaster-service
   trunk/bin/stop-coaster-service
Log:
Integrate gwms scripts with coaster-service scripts
Added a test configuration for osg which uses the worker.pl init command to symlink work directory


Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service	2012-05-11 17:47:42 UTC (rev 5789)
+++ trunk/bin/start-coaster-service	2012-05-22 12:27:09 UTC (rev 5790)
@@ -265,7 +265,7 @@
    if [ -z "$PORT" ]; then
       crash "start-workers-local: Port number not specified, giving up"
    fi
-   echo Starting worker on local machine
+   echo Starting workers
 
    if [ -z "$CONDOR_WORKERS" ]; then
       crash "CONDOR_WORKERS undefined - please check coaster-service.conf"
@@ -286,6 +286,25 @@
    return 0
 }
 
+# Start GWMS workers
+start-workers-gwms()
+{
+   PORT=$1
+   EXECUTION_URL=http://$IPADDR:$PORT
+
+   if [ -z "$PORT" ]; then
+      crash "start-workers-local: Port number not specified, giving up"
+   fi
+
+   if [ -z "$CONDOR_WORKERS" ]; then
+      crash "CONDOR_WORKERS undefined - please check coaster-service.conf"
+   fi
+
+   echo Starting workers   
+   mkdir -p condor
+   run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS > gwms-workers.log 2>&1 &
+}
+
 # Start cobalt workers
 start-workers-cobalt()
 {
@@ -477,6 +496,9 @@
    condor)
       start-workers-condor $LOCAL_PORT
       ;;
+   gwms)
+      start-workers-gwms $LOCAL_PORT
+      ;;
    *)
       crash "Unknown WORKER_MODE. Please modify coaster-service.conf"
       ;;

Modified: trunk/bin/stop-coaster-service
===================================================================
--- trunk/bin/stop-coaster-service	2012-05-11 17:47:42 UTC (rev 5789)
+++ trunk/bin/stop-coaster-service	2012-05-22 12:27:09 UTC (rev 5790)
@@ -60,7 +60,7 @@
    fi
 fi
 
-if [ "$WORKER_MODE" == "condor" ]; then
+if [ "$WORKER_MODE" == "condor" ] || [ "$WORKER_MODE" == "gwms" ]; then
    condor_rm $USER
 fi
 

Added: trunk/tests/sites/osg/coaster-service.conf
===================================================================
--- trunk/tests/sites/osg/coaster-service.conf	                        (rev 0)
+++ trunk/tests/sites/osg/coaster-service.conf	2012-05-22 12:27:09 UTC (rev 5790)
@@ -0,0 +1,40 @@
+# Location of SWIFT. If empty, PATH is searched
+export SWIFT=
+
+# Where to copy worker.pl on the remote machine for sites.xml
+export IPADDR=engage-submit3.renci.org
+
+# Make sure we always use $OSG_WN_TMP as our work directory
+export WORKER_INIT_CMD="ln -s \$OSG_WN_TMP /tmp/osg_wn_tmp"
+
+# How to launch workers: local, ssh, cobalt, or futuregrid
+export WORKER_MODE=gwms
+export CONDOR_WORKERS=1
+
+# Do all the worker nodes you're using have a shared filesystem? (yes/no)
+export SHARED_FILESYSTEM=no
+
+# Enable SSH tunneling? (yes/no)
+export SSH_TUNNELING=no
+
+# Directory to keep log files, relative to working directory when launching start-coaster-service
+export LOG_DIR=logs
+
+# Manually define ports. If not specified, an available port will be used
+export LOCAL_PORT=
+export SERVICE_PORT=
+
+# Swift information for creating sites.xml
+export WORK=/tmp/osg_wn_tmp
+
+# Jobs per node is usually equal to the number of CPUs per node
+export JOBS_PER_NODE=1
+
+# To calculate job throttle:
+# jobs per node * nodes - 0.1 / 100
+export JOB_THROTTLE=$( echo "scale=5; ($JOBS_PER_NODE * $CONDOR_WORKERS)/100 - 0.00001"|bc )
+
+# Swift applications
+#app bash=/bin/bash
+#app cat=/bin/cat
+#app echo=/bin/echo



From davidk at ci.uchicago.edu  Tue May 22 10:25:28 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 22 May 2012 10:25:28 -0500 (CDT)
Subject: [Swift-commit] r5791 - in trunk/bin: . grid
Message-ID: <20120522152528.D1BDD9CD2D@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-22 10:25:28 -0500 (Tue, 22 May 2012)
New Revision: 5791

Modified:
   trunk/bin/grid/run-worker.sh
   trunk/bin/start-coaster-service
Log:
A few updates for better logging


Modified: trunk/bin/grid/run-worker.sh
===================================================================
--- trunk/bin/grid/run-worker.sh	2012-05-22 12:27:09 UTC (rev 5790)
+++ trunk/bin/grid/run-worker.sh	2012-05-22 15:25:28 UTC (rev 5791)
@@ -4,14 +4,13 @@
 workername=$2
 origlogdir=$3
 echo OSG_WN_TMP=$OSG_WN_TMP
-if [ _$OSG_WN_TMP = _ ]; then
-  OSG_WN_TMP=/tmp
-fi
 
+#if [ _$OSG_WN_TMP = _ ]; then
+#  OSG_WN_TMP=/tmp
+#fi
 mkdir -p $OSG_WN_TMP
+
 logdir=$(mktemp -d $OSG_WN_TMP/${workername}.workerdir.XXXXXX)
-nlines=1000
-
 export PATH=${OSG_APP}/engage/scec:${OSG_APP}/engage/scec/JBSim3d/bin:${OSG_APP}/engage/scec/SpectralAcceleration/p2utils:${OSG_APP}/extenci/swift/DSSAT/bin:${OSG_APP}/extenci/aashish/terfix/bin:$PATH
 #export PATH=${OSG_PATH}:$PATH
 
@@ -19,6 +18,8 @@
 echo "=== name:    $workername Running in dir $(pwd)"
 echo "=== cwd:     $(pwd)"
 echo "=== logdir:  $logdir"
+echo "=== disk space: "
+df -h
 echo "=== path:    $PATH"
 echo "==============================================="
 
@@ -31,9 +32,14 @@
 
 echo "=== exit: worker.pl exited with code=$exitcode"
 
-echo "=== worker log - last $nlines lines:"
+echo "=== worker logs:"
 
 echo
 
-tail -v -n $nlines $logdir/*
+for file in $logdir/*
+do
+   echo $file
+   echo -------
+   cat $file
+done
 

Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service	2012-05-22 12:27:09 UTC (rev 5790)
+++ trunk/bin/start-coaster-service	2012-05-22 15:25:28 UTC (rev 5791)
@@ -302,7 +302,7 @@
 
    echo Starting workers   
    mkdir -p condor
-   run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS > gwms-workers.log 2>&1 &
+   run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS 
 }
 
 # Start cobalt workers
@@ -513,7 +513,7 @@
 cat > $RUN_DIR/cf << EOF
 use.provider.staging=true
 wrapperlog.always.transfer=false
-execution.retries=0
+execution.retries=10
 provider.staging.pin.swiftfiles=false
 sitedir.keep=false
 EOF



From davidk at ci.uchicago.edu  Tue May 22 10:30:19 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 22 May 2012 10:30:19 -0500 (CDT)
Subject: [Swift-commit] r5792 - trunk/tests/sites/osg
Message-ID: <20120522153019.097839CD2D@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-22 10:30:18 -0500 (Tue, 22 May 2012)
New Revision: 5792

Modified:
   trunk/tests/sites/osg/coaster-service.conf
Log:
Updates for osg config file


Modified: trunk/tests/sites/osg/coaster-service.conf
===================================================================
--- trunk/tests/sites/osg/coaster-service.conf	2012-05-22 15:25:28 UTC (rev 5791)
+++ trunk/tests/sites/osg/coaster-service.conf	2012-05-22 15:30:18 UTC (rev 5792)
@@ -5,7 +5,7 @@
 export IPADDR=engage-submit3.renci.org
 
 # Make sure we always use $OSG_WN_TMP as our work directory
-export WORKER_INIT_CMD="ln -s \$OSG_WN_TMP /tmp/osg_wn_tmp"
+export WORKER_INIT_CMD="if [ ! -h /tmp/osg_wn_tmp ]; then ln -s \$OSG_WN_TMP /tmp/osg_wn_tmp; fi"
 
 # How to launch workers: local, ssh, cobalt, or futuregrid
 export WORKER_MODE=gwms



From davidk at ci.uchicago.edu  Wed May 23 08:37:56 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 23 May 2012 08:37:56 -0500 (CDT)
Subject: [Swift-commit] r5793 - in trunk: bin
	src/org/griphyn/vdl/mapping/file
Message-ID: <20120523133756.09D079CCAF@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-23 08:37:55 -0500 (Wed, 23 May 2012)
New Revision: 5793

Modified:
   trunk/bin/gensites
   trunk/bin/start-coaster-service
   trunk/bin/stop-coaster-service
   trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java
Log:
AbstractFileMapper currently prints a message that says "(input): N number of files". This can get very spammy on some workflows, to the point where it's nearly impossible to read the ticker bar. Disabled this message for now
start-coaster-service: Ability to remotely start workers for gwms and condor sites
gensites: When generating tc, currently assumes that the application is on the local filesystem. Add a test for this to avoid errors


Modified: trunk/bin/gensites
===================================================================
--- trunk/bin/gensites	2012-05-22 15:30:18 UTC (rev 5792)
+++ trunk/bin/gensites	2012-05-23 13:37:55 UTC (rev 5793)
@@ -199,8 +199,13 @@
                   NAME=`echo $line |cut -d'=' -f1|awk '{print $2}'`
                   COMMAND=`echo $line |cut -d'=' -f2`
                   COMMAND=`eval echo $COMMAND`
-                  COMMAND_PATH=$(cd $(dirname $COMMAND); pwd)/$(basename $COMMAND)
-                  echo $HOST $NAME $COMMAND_PATH null null null >> tc.data
+                  COMMAND_DIRNAME=$(dirname $COMMAND)
+                  if [ -d "$COMMAND_DIRNAME" ]; then
+                     COMMAND_PATH=$(cd $COMMAND_DIRNAME; pwd)/$(basename $COMMAND)
+                     echo $HOST $NAME $COMMAND_PATH null null null >> tc.data
+                  else
+                     echo $HOST $NAME $COMMAND null null null >> tc.data
+                  fi
               done
            fi
 

Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service	2012-05-22 15:30:18 UTC (rev 5792)
+++ trunk/bin/start-coaster-service	2012-05-23 13:37:55 UTC (rev 5793)
@@ -274,7 +274,7 @@
 cat <<EOF > start_workers.submit
 output = workers.stdout
 error = workers.stderr
-initialdir = $PWD 
+#initialdir = \$PWD 
 executable = $SWIFT_BIN/$WORKER
 arguments = $EXECUTION_URL node .
 notification = Never
@@ -282,8 +282,12 @@
 queue $CONDOR_WORKERS
 EOF
 
-   condor_submit start_workers.submit
-   return 0
+   if [ -n "$WORKER_RELAY_HOST" ]; then
+      scp start_workers.submit $WORKER_USERNAME@$WORKER_RELAY_HOST:
+      ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "condor_submit start_workers.submit"
+   else
+      condor_submit start_workers.submit
+   fi
 }
 
 # Start GWMS workers
@@ -301,8 +305,13 @@
    fi
 
    echo Starting workers   
-   mkdir -p condor
-   run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS 
+   if [ -n "$WORKER_RELAY_HOST" ]; then
+      ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "mkdir -p condor"
+      ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS" 
+   else
+      mkdir -p condor
+      run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS
+   fi
 }
 
 # Start cobalt workers

Modified: trunk/bin/stop-coaster-service
===================================================================
--- trunk/bin/stop-coaster-service	2012-05-22 15:30:18 UTC (rev 5792)
+++ trunk/bin/stop-coaster-service	2012-05-23 13:37:55 UTC (rev 5793)
@@ -61,7 +61,11 @@
 fi
 
 if [ "$WORKER_MODE" == "condor" ] || [ "$WORKER_MODE" == "gwms" ]; then
-   condor_rm $USER
+   if [ -n "$WORKER_RELAY_HOST" ]; then
+      ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "condor_rm $USER"    
+   else
+     condor_rm $USER
+   fi
 fi
 
 if [ -f "$HOME/.swift/machines" ]; then

Modified: trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java	2012-05-22 15:30:18 UTC (rev 5792)
+++ trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java	2012-05-23 13:37:55 UTC (rev 5793)
@@ -257,7 +257,7 @@
 		if(logger.isDebugEnabled()) {
 			logger.debug("Finish list existing paths for mapper "+this.hashCode()+" list="+result);
 		}
-		System.out.println(getVarName() + " (input): found " + result.size() + " files");
+		// System.out.println(getVarName() + " (input): found " + result.size() + " files");
 		return result;
 	}
 



From davidk at ci.uchicago.edu  Wed May 23 09:49:07 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 23 May 2012 09:49:07 -0500 (CDT)
Subject: [Swift-commit] r5794 - SwiftApps
Message-ID: <20120523144907.7A55A9CCA5@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-23 09:49:07 -0500 (Wed, 23 May 2012)
New Revision: 5794

Added:
   SwiftApps/cray-benchmarks/
Log:
Creating a place to store the scripts used for the cray benchmarks




From davidk at ci.uchicago.edu  Fri May 25 04:56:26 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri, 25 May 2012 04:56:26 -0500 (CDT)
Subject: [Swift-commit] r5796 - trunk/bin/grid
Message-ID: <20120525095626.961F59CC9C@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-25 04:56:26 -0500 (Fri, 25 May 2012)
New Revision: 5796

Modified:
   trunk/bin/grid/run-gwms-workers
Log:
Make sure important environment variables get used when starting worker.pl


Modified: trunk/bin/grid/run-gwms-workers
===================================================================
--- trunk/bin/grid/run-gwms-workers	2012-05-23 14:53:20 UTC (rev 5795)
+++ trunk/bin/grid/run-gwms-workers	2012-05-25 09:56:26 UTC (rev 5796)
@@ -11,6 +11,10 @@
 x509userproxy=$( voms-proxy-info 2>&1 | grep path | awk '{print $NF}' )
 wallTimeMinutes=60
 n=$2
+if [ -z "$WORKER_LOGGING_LEVEL" ];  then
+   WORKER_LOGGING_LEVEL=INFO
+fi
+echo WORKER_INIT_CMD in run-gwms-workers: $WORKER_INIT_CMD
 
 [ "$#" -eq 2 ] || die "2 arguments required, $# provided"
 
@@ -32,7 +36,7 @@
 
 executable = $workerWrapper
 arguments = $workerContact scec /tmp
-environment = WORKER_LOGGING_LEVEL=DEBUG
+environment = "WORKER_LOGGING_LEVEL='$WORKER_LOGGING_LEVEL' WORKER_INIT_CMD='$WORKER_INIT_CMD'"
 Input   = $workerExecutable
 Error   = condor/job.\$(Cluster).\$(Process).err
 Output  = condor/job.\$(Cluster).\$(Process).out



From hategan at ci.uchicago.edu  Sat May 26 18:14:21 2012
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Sat, 26 May 2012 18:14:21 -0500 (CDT)
Subject: [Swift-commit] r5797 - trunk/libexec
Message-ID: <20120526231421.26E819CCAF@svn.ci.uchicago.edu>

Author: hategan
Date: 2012-05-26 18:14:20 -0500 (Sat, 26 May 2012)
New Revision: 5797

Added:
   trunk/libexec/_swiftwrap.wrapperstaging
   trunk/libexec/vdl-int-wrapper-staging.k
Modified:
   trunk/libexec/vdl.k
Log:
added wrapper staging libraries and wrapper

Added: trunk/libexec/_swiftwrap.wrapperstaging
===================================================================
--- trunk/libexec/_swiftwrap.wrapperstaging	                        (rev 0)
+++ trunk/libexec/_swiftwrap.wrapperstaging	2012-05-26 23:14:20 UTC (rev 5797)
@@ -0,0 +1,544 @@
+#!/bin/bash
+# this script must be invoked inside of bash, not plain sh
+# note that this script modifies $IFS
+
+# Toggle debugging output from debug()
+DEBUG=0
+
+infosection() {
+	echo >& "$INFO"
+	echo "_____________________________________________________________________________" >& "$INFO"
+	echo >& "$INFO"
+	echo "        $1" >& "$INFO"
+	echo "_____________________________________________________________________________" >& "$INFO"
+	echo >& "$INFO"
+}
+
+info() {
+	infosection "command line"
+	echo $COMMANDLINE 2>&1 >& "$INFO"
+	infosection "uname -a"
+	uname -a 2>&1 >& "$INFO"
+	infosection "id"
+	id 2>&1 >& "$INFO"
+	infosection "env"
+	env 2>&1 >& "$INFO"
+	infosection "df"
+	df 2>&1 >& "$INFO"
+        if [ -e "/proc/cpuinfo" ]; then
+		infosection "/proc/cpuinfo"
+		cat /proc/cpuinfo 2>&1 >& "$INFO"
+	fi
+	if [ -e "/proc/meminfo" ]; then
+		infosection "/proc/meminfo"
+		cat /proc/meminfo 2>&1 >& "$INFO"
+	fi
+	if [ -f "$STDOUT" ] ; then
+		infosection "stdout"
+		cat $STDOUT >& "$INFO"
+	fi
+	if [ -f "$STDERR" ] ; then
+		infosection "stderr"
+		cat $STDERR >& "$INFO"
+	fi
+}
+
+logstate() {
+	echo "Progress " `date +"%Y-%m-%d %H:%M:%S.%N%z"` " $@" >& "$INFO"
+}
+
+log() {
+	echo "$@" >& "$INFO"
+}
+
+debug() {
+	[[ $DEBUG == 1 ]] && echo "$@" >& "$INFO"
+}
+
+stagein() {
+	SRC=$1
+	DST=$2
+	
+	log "Staging in $URLPREFIX/$SRC to $DST"
+	
+	case $URLPREFIX in
+		file://*)
+			SRC2="${URLPREFIX#file://}/$SRC"
+			log "Copying $SRC2 to $DST"
+			cp $SRC2 $DST
+			;;
+		*://*)
+			fail 254 "Cannot handle protocol ${URLPREFIX%%://*}"
+			;;
+		*)
+			fail 254 "Invalid URL prefix: $URLPREFIX"
+			;;
+	esac
+}
+
+stageout() {
+	SRC=$1
+	DST=$2
+	
+	log "Staging out $SRC to $URLPREFIX/$DST"
+	
+	case $URLPREFIX in
+		file://*)
+			DST2="${URLPREFIX#file://}/$DST"
+			DIR=`dirname $DST2`
+			mkdir -p $DIR
+			log "Copying $SRC to $DST2"
+			cp $SRC $DST2
+			;;
+		*://*)
+			fail 254 "Cannot handle protocol ${URLPREFIX%%://*}"
+			;;
+		*)
+			fail 254 "Invalid URL prefix: $URLPREFIX"
+			;;
+	esac
+}
+
+
+fail() {
+	EC=$1
+	shift
+	
+	if [ "X$DIR" != "X" ]; then
+		echo $@ >"$DIR/_error"
+		stageout "$DIR/_error" "$JOBDIR/error"
+	else
+		echo $@
+	fi
+		
+	log $@
+	info
+	if [ "$STATUSMODE" = "files" ]; then
+		exit 0
+	else
+		exit $EC
+	fi
+}
+
+checkError() {
+	if [ "$?" != "0" ]; then
+		fail $@
+	fi
+}
+
+checkEmpty() {
+	if [ "$1" == "" ]; then
+		shift
+		fail 254 $@
+	fi
+}
+
+checkparamfile() {
+	log "checking for paramfile"
+	if [ "$1" == "-p" ]; then
+		PARAMFILE="$SCRATCH/_paramfile"
+		stagein "$URLPREFIX/$2" "$PARAMFILE"
+	fi
+	log "paramfile is: $PARAMFILE"
+}
+
+getarg() {
+	NAME=$1
+	shift
+	VALUE=""
+	SHIFTCOUNT=0
+	if [ "$PARAMFILE" == "" ] && [ "$1" == "$NAME" ]; then
+		shift
+		let "SHIFTCOUNT=$SHIFTCOUNT+1"
+		while [ "${1:0:1}" != "-" ] && [ "$#" != "0" ]; do
+			VALUE="$VALUE $1"
+			shift
+			let "SHIFTCOUNT=$SHIFTCOUNT+1"
+		done
+		VALUE="${VALUE:1}"
+	elif [ "$PARAMFILE" != "" ] && grep -E "^$NAME " $PARAMFILE ; then
+		VALUE=$(grep -E "^$NAME " $PARAMFILE | cut -d ' ' -f 2-)
+	else
+		fail 254 "Missing $NAME argument"
+	fi
+}
+
+openinfo() {
+	exec 3<> $1
+	INFO=3
+}
+
+closeinfo() {
+	exec 3>&-
+}
+
+contains() {
+	ARRAY=$1
+	X=$2
+
+	for a in ${!ARRAY}
+	do
+		if [[ ${a} == ${X} ]]; then
+			return 0
+		fi
+	done
+	return 1
+}
+
+genScripts() {
+	echo "#!/bin/bash" > run.sh
+	echo -n "\"$EXEC\" " >> run.sh
+	for CMDARG in "${CMDARGS[@]}"; do
+    	echo -n "\"$CMDARG\" " >> run.sh
+	done
+	echo "1>\"$STDOUT\" 2>\"$STDERR\"" >> run.sh
+	chmod +x run.sh
+}
+
+cdm_local_output()
+{
+ 	L=$1
+
+	if [[ $CDM_FILE == "" ]]; then
+		return
+	fi
+
+ 	CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+	if [[ $CDM_POLICY == "LOCAL" ]]; then
+		cdm_local_output_perform $L $CDM_POLICY
+	fi
+}
+
+cdm_local_output_perform()
+{
+	L=$1
+	TOOL=$2
+	REMOTE_DIR=$3
+	FLAGS=$3
+	log "Copying $REMOTE_DIR/$FILE to $JOBDIR/$FILE"
+	mkdir -p $REMOTE_DIR
+	checkError 254 "CDM[LOCAL]: mkdir -p $REMOTE_DIR failed!"
+	$TOOL $FLAGS $JOBDIR/$FILE $REMOTE_DIR/$FILE
+	checkError 254 "CDM[LOCAL]: Tool failed!"
+}
+
+cdm_gather()
+{
+	GATHER_OUTPUT=${*}
+	if [[ $CDM_FILE == "" ]]; then
+		return
+	fi
+	if [[ $GATHER_OUTPUT == "" ]]; then
+		return
+	fi
+
+	cdm_gather_action $GATHER_MAX $GATHER_OUTPUT
+}
+
+COMMANDLINE=$@
+
+PARAMFILE=
+
+openinfo "wrapper.log"
+ID=$1
+checkEmpty "$ID" "Missing job ID"
+
+shift
+
+getarg "-urlprefix" "$@"
+URLPREFIX=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-jobdir" "$@"
+JOBDIR=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-scratch" "$@"
+SCRATCH=$VALUE/$ID
+shift $SHIFTCOUNT
+
+checkparamfile "$@"
+
+INFODIR=$SCRATCH
+
+checkEmpty "$JOBDIR" "Missing job directory prefix"
+mkdir -p $INFODIR
+closeinfo
+
+if [ -z $MPI_RANK ]; then
+	INFOFILE="$INFODIR/_info"
+else
+	# Rename info file for each rank
+	INFOFILE="$INFODIR/_info-${PMI_RANK}"
+	# Build list of per-rank info files
+	echo $INFOFILE >> $INFODIR/_info
+fi
+rm -f $INFOFILE
+openinfo "$INFOFILE"
+
+logstate "LOG_START"
+infosection "Wrapper (_swiftwrap)"
+
+getarg "-e" "$@"
+EXEC=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-out" "$@"
+STDOUT=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-err" "$@"
+STDERR=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-i" "$@"
+STDIN=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-d" "$@"
+DIRS=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-if" "$@"
+INF=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-of" "$@"
+OUTF=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-wt" "$@"
+WRAPPERLOG_ALWAYS_TRANSFER=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-sk" "$@"
+SITEDIR_KEEP=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-cdmfile" "$@"
+CDM_FILE=
+if [ "X$VALUE" != "X" ]; then
+	CDM_FILE=shared/$VALUE
+fi
+shift $SHIFTCOUNT
+
+getarg "-status" "$@"
+STATUSMODE=$VALUE
+shift $SHIFTCOUNT
+
+declare -a CMDARGS
+if [ "$PARAMFILE" == "" ] && [ "$1" == "-a" ] ; then
+	shift
+	CMDARGS=("$@")
+elif [ "$PARAMFILE" != "" ] ; then
+	CMDARGS=()
+	FIRST=1
+	while read line ; do
+		if [ "$FIRST" == "1" ] ; then
+			CMDARGS=("$line")
+			FIRST=0
+		else
+			CMDARGS=("${CMDARGS[@]}" "$line")
+		fi
+	done < <(grep -E "^-a " $PARAMFILE | cut -d " " -f 2-)
+else
+	fail 254 "Missing arguments (-a option)"
+fi
+
+if [ "X$CDM_FILE" != "X" ]; then
+	# TODO
+	logstate "SOURCE_CDM_LIB $WFDIR/shared/cdm_lib.sh"
+	source $WFDIR/shared/cdm_lib.sh
+	checkError 254 "Could not source: $WFDIR/shared/cdm_lib.sh"
+fi
+
+if [ "X$SCRATCH" == "X" ]; then
+	error "Wrapper staging requires a scratch directory to be specified"
+fi
+
+DIR="$SCRATCH/$JOBID"
+
+PATH=$PATH:/bin:/usr/bin
+
+if [ "$PATHPREFIX" != "" ]; then
+	export PATH=$PATHPREFIX:$PATH
+fi
+
+if [ "$SWIFT_EXTRA_INFO" != "" ]; then
+	log "EXTRAINFO=$($SWIFT_EXTRA_INFO)"
+fi
+
+if [ "X${EXEC:0:1}" != "X/" ] ; then
+	export ORIGEXEC=$EXEC
+	export EXEC=$(which $EXEC)
+	if [ "X$EXEC" = "X" ] ; then
+		fail 254 "Cannot find executable $ORIGEXEC on site system path"
+	fi
+fi
+
+log "PID=$$"
+log "HOST=$HOST"
+log "PWD=$PWD"
+log "DIR=$DIR"
+log "EXEC=$EXEC"
+log "STDIN=$STDIN"
+log "STDOUT=$STDOUT"
+log "STDERR=$STDERR"
+log "DIRS=$DIRS"
+log "INF=$INF"
+log "OUTF=$OUTF"
+log "WRAPPERLOG_ALWAYS_TRANSFER=$WRAPPER_LOG_ALWAYS_TRANSFER"
+log "SITEDIR_KEEP=$SITEDIR_KEEP"
+log "CDM_FILE=$CDM_FILE"
+log "ARGS=$@"
+log "ARGC=$#"
+[ -n $MPI_RANK ] && log "MPI_RANK=$MPI_RANK" && log "PMI_RANK=$PMI_RANK"
+IFS="|"
+
+logstate "CREATE_JOBDIR"
+mkdir -p $DIR
+checkError 254 "Failed to create job directory $DIR"
+log "Created job directory: $DIR"
+
+if [[ $PMI_RANK == "" || $PMI_RANK == 0 ]]; then
+
+	logstate "CREATE_INPUTDIR"
+	for D in $DIRS ; do
+		mkdir -p "$DIR/$D" 2>&1 >>"$INFO"
+		checkError 254 "Failed to create input directory $D"
+		log "Created output directory: $DIR/$D"
+	done
+	
+	logstate "LINK_INPUTS"
+	for L in $INF ; do
+	    CDM_POLICY="DEFAULT"
+		if [[ $CDM_FILE != "" ]]; then
+			CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+		fi
+		if [[ $CDM_POLICY != "DEFAULT" && $CDM_POLICY != "EXTERNAL"* ]]; then
+			log "CDM_POLICY: $L -> $CDM_POLICY"
+			eval cdm_action $DIR "INPUT" $L $CDM_POLICY
+			continue
+		fi
+		
+		stagein $L "$DIR/$L"
+	done
+	
+	if [[ $CDM_FILE != "" ]]; then
+	    logstate "LINK_CDM_OUTPUTS"
+	    SKIPPED_OUTPUT=()
+		GATHER_OUTPUT=()
+		for L in $OUTF ; do
+			CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+			log "CDM_POLICY: $L -> $CDM_POLICY"
+			if [[ $CDM_POLICY != "DEFAULT" &&
+				  $CDM_POLICY != "BROADCAST"* ]]; then
+	    	    eval cdm_action $DIR "OUTPUT" $L $CDM_POLICY
+				SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L )
+			fi
+			if [ $CDM_POLICY == "GATHER" ]; then
+				GATHER_OUTPUT=( $GATHER_OUTPUT $L )
+			elif [ $CDM_POLICY == "LOCAL" ]; then
+				CDM_LOCAL_OUTPUT=( $CDM_LOCAL_OUTPUT $L )
+			fi
+		done
+	fi
+
+fi # PMI_RANK==0
+
+debug "Moving to jobdir: $DIR"
+cd $DIR
+if [ $? != 0 ]; then
+	log "PWD: $PWD"
+	log $( find . )
+	fail 254 "Could not cd to: $DIR"
+fi
+logstate "EXECUTE"
+
+debug "Command line: $EXEC ${CMDARGS[@]}"
+
+if [ ! -f "$EXEC" ]; then
+	fail 254 "The executable $EXEC does not exist"
+fi
+if [ ! -x "$EXEC" ]; then
+	fail 254 "The executable $EXEC does not have the executable bit set"
+fi
+
+if [ "$STDIN" == "" ]; then
+	if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then
+		genScripts
+	fi
+	"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR"
+else
+	if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then
+		genScripts
+	fi
+	"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN"
+fi
+checkError $? "Application $EXEC failed with an exit code of $?"
+
+logstate "EXECUTE_DONE"
+log "Job ran successfully"
+
+if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then
+
+	MISSING=
+	for O in $OUTF ; do
+		if [ ! -f "$DIR/$O" ]; then
+			if [ "$MISSING" == "" ]; then
+				MISSING=$O
+			else
+				MISSING="$MISSING, $O"
+			fi
+		fi
+	done
+	if [ "$MISSING" != "" ]; then
+		log $( find . )
+		fail 254 "The following output files were not created by the application: $MISSING"
+	fi
+	
+	logstate "MOVING_OUTPUTS $OUTF"
+	for O in $OUTF ; do
+		if ! contains SKIPPED_OUTPUT $O ; then
+			stageout "$DIR/$O" "$O" 
+		fi
+	done
+	
+	cdm_local_output $CDM_LOCAL_OUTPUT
+	cdm_gather $GATHER_OUTPUT
+	
+	if [ "$STATUSMODE" = "files" ]; then
+		logstate "TOUCH_SUCCESS"
+		touch ${ID}-success
+		stageout "$DIR/${ID}-success" "$JOBDIR/success"  
+	fi
+	
+	log "Moving back to workflow directory $WFDIR"
+	cd $WFDIR
+	if [ $? != 0 ]; then
+		fail 254 "Could not cd to workflow directory: $WFDIR"
+	fi
+	
+	if [ "$WRAPPERLOG_ALWAYS_TRANSFER" == "true" ]; then
+		stageout "$INFOFILE" "$JOBDIR/info"
+	fi
+	if [ "$SITEDIR_KEEP" != "true" ]; then
+		logstate "RM_JOBDIR"
+		rm -rf "$DIR" 2>&1 >& "$INFO"
+		checkError 254 "Failed to remove job directory $DIR"
+	fi
+else
+	# Allow rank 0 to write output
+	sleep 1
+fi # MPI_RANK==0
+
+logstate "END"
+
+closeinfo
+
+if [ "$WRAPPER_LOG_ALWAYS_TRANSFER" == "true" ]; then
+	stageout "$DIR/${ID}-info" "$JOBID/${ID}-info"
+fi
+
+# ensure we exit with a 0 after a successful execution
+exit 0


Property changes on: trunk/libexec/_swiftwrap.wrapperstaging
___________________________________________________________________
Added: svn:executable
   + *

Added: trunk/libexec/vdl-int-wrapper-staging.k
===================================================================
--- trunk/libexec/vdl-int-wrapper-staging.k	                        (rev 0)
+++ trunk/libexec/vdl-int-wrapper-staging.k	2012-05-26 23:14:20 UTC (rev 5797)
@@ -0,0 +1,465 @@
+import("sys.k")
+import("task.k")
+import("vdl-lib.xml")
+/*
+ * Things that are not exposed to the translated file
+ */
+
+global(LOG:DEBUG, "debug")
+global(LOG:INFO, "info")
+global(LOG:WARN, "warn")
+global(LOG:ERROR, "error")
+global(LOG:FATAL, "fatal")
+
+global(URL_PREFIX, 
+	elementDef(getURLPrefix, className="org.griphyn.vdl.karajan.lib.GetURLPrefix")
+	getURLPrefix() 
+)
+
+global(WRAPPERLOG_ALWAYS_TRANSFER, vdl:configProperty("wrapperlog.always.transfer"))
+global(SITEDIR_KEEP, vdl:configProperty("sitedir.keep"))
+
+
+namespace("vdl"
+	export(
+		element(rmdir, [dir, host]
+			parallelFor(entry, file:list(dir, host=host)
+				epath := "{dir}/{entry}"
+				if(
+					file:isDirectory(epath, host=host) rmdir(epath, host)
+					file:remove(epath, host=host)
+				)
+			)
+			dir:remove(dir, host=host)
+		)
+
+		element(createdirs, [path, dir, host]
+			dc := dircat(dir, path)
+			log(LOG:INFO, "START path={path} dir={dir} - Creating directory structure")
+
+			dir:make(dc, host=host)
+		)
+
+		element(checkJobStatus, [wfdir, jobid, tr]
+			log(LOG:DEBUG, "START jobid={jobid}")
+			try(
+				sequential(
+					/*
+					 * This is a bit of optimization, but I'm not completely
+					 * sure of its correctness. The goal is to both detect
+					 * the presence of the success file and remove it, all
+					 * in one operation. It relies on file:remove() throwing
+					 * an exception if the file is not there.
+					 */
+					file:remove("{wfdir}/status/{jobdir}/{jobid}-success", host=rhost)
+					log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found")
+				)
+				sequential(
+					try (
+						msg = checkErrorFile(rhost, wfdir, jobid, jobdir)
+						sequential (
+							log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing")
+							throw("No status file was found. Check the shared filesystem on {rhost}")
+						)
+					)
+				)
+				throw(checkErrorFile(rhost, wfdir, jobid, jobdir))
+			)
+		)
+		
+		element(checkErrorFile, [wfdir, jobid]
+			if (
+				file:exists("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost) then(
+					log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found")
+					task:transfer(srchost=rhost, srcdir="{wfdir}/status/{jobdir}", srcfile="{jobid}-error")
+					error := parallel(
+						file:remove("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost)
+						sequential(
+							str:strip(file:read("{jobid}-error"))
+							file:remove("{jobid}-error")
+						)
+					)
+					error
+				)
+				else (
+					log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Error file missing")
+					throw("No status file was found. Check the shared filesystem on {rhost}")
+				)
+			)
+		)
+
+		element(initSharedDir, [rhost]
+			once(list(rhost, "shared")
+				vdl:setprogress("Initializing site shared directory")
+
+				log(LOG:INFO, "START host={rhost} - Initializing shared directory")
+
+				wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}"
+				dir:make(wfdir, host=rhost)
+				transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftwrap.wrapperstaging", destdir=wfdir, desthost=rhost)
+
+				wfdir
+				//we send the cleanup data to vdl:main()
+				to(cleanup, list(wfdir, rhost))
+				log(LOG:INFO, "END host={rhost} - Done initializing shared directory")
+			)
+		)
+
+		element(initDDir, []
+			ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d"
+			once(ddir
+				if(sys:not(file:exists(ddir))
+					task:dir:make(ddir)
+				)
+			)
+			ddir
+		)
+
+		element(inFiles, [stageins]
+			pathnames(stageins)
+		)
+
+		element(fileDirs, [stageins, stageouts]
+			list(
+				unique(
+					inFileDirs(stageins)
+					outFileDirs(stageouts)
+				)
+			)
+		)
+
+		element(createDirSet, [jobid, destdir, host, dirs]
+			/*
+			 * Ideally this would be done by creating a tree of the directories
+			 * to be created and (eventually) exploiting the concurrency in that.
+			 */
+			log(LOG:INFO, "START jobid={jobid} host={host} - Initializing directory structure")
+			for(u, dirs
+				cacheOn(list(u, destdir, host)
+					createdirs(u, destdir, host)
+				)
+			)
+			log(LOG:INFO, "END jobid={jobid} - Done initializing directory structure")
+		)
+
+		element(cleanup, [dir, host]
+			log(LOG:INFO, "START dir={dir} host={host}")
+			cdmfile := cdm:file()
+			log(LOG:DEBUG, "cdmfile {cdmfile}")
+			if(cdmfile != "" &
+                           cdm:get("GATHER_DIR") != "UNSET" then(
+				log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}")
+				task:transfer(srcfile="cdm_cleanup.sh",
+					      srcdir="{swift.home}/libexec",
+					      desthost=host, destdir=dir)
+				task:transfer(srcfile="cdm_lib.sh",
+					      srcdir="{swift.home}/libexec",
+					      desthost=host, destdir=dir)
+			        log(LOG:INFO, "execute: cdm_cleanup.sh")
+				task:execute(
+					     executable="/bin/bash",
+					     arguments=list("{dir}/cdm_cleanup.sh",
+					                     cdm:get("GATHER_DIR"), cdm:get("GATHER_TARGET")
+							     sys:uid() )
+					     host=host, batch=true, tcprofile(host))
+				)
+			)
+			if(vdl:configProperty("sitedir.keep") == "false"
+				task:execute(
+					vdl:siteprofile(host, "swift:cleanupCommand"),
+					arguments=list(
+						vdl:siteprofile(host, "swift:cleanupCommandOptions"),
+						dir
+					)
+					host=host, batch=true, tcprofile(host))
+			)
+			log(LOG:INFO, "END dir={dir} host={host}")
+		)
+
+		element(cleanups, [cleanup]
+			log(LOG:INFO, "START cleanups={cleanup}")
+			parallelFor(i, cleanup
+				[dir, host] := each(i)
+				try(
+					vdl:cleanup(dir, host)
+					catch(".*",
+						log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception)
+						to(warnings, exception("Cleanup on {host} failed", exception))
+					)
+				)
+			)
+			log(LOG:INFO, "END cleanups={cleanup}")
+		)
+
+		element(cleanupFiles, [files, host]
+			uParallelFor(r, files
+				log(LOG:INFO, "Purging ", r, " on ", host)
+				file:remove(r, host=host)
+				vdl:cacheFileRemoved(r, host)
+			)
+		)
+
+		element(stageWrapperParams, [jobid, wrapfile, dir, host]
+			log(LOG:INFO, "START jobid={jobid} - staging in wrapper params")
+			provider := provider(wrapfile)
+			srchost := hostname(wrapfile)
+			srcdir := vdl:dirname(wrapfile)
+			destdir := dir
+			filename := basename(wrapfile)
+
+			cacheOn(list(destdir, host)
+				dir:make(destdir, host=host, provider=provider)
+			)
+
+			log(LOG:INFO, "END jobid={jobid}")
+		)
+
+		element(graphStuff, [tr, stagein, stageout, err, optional(args)]
+			if(
+				vdl:configProperty("pgraph") != "false" then(
+					errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1")
+					tp := vdl:threadPrefix()
+					to(graph,
+						concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]")
+					)
+					for(si, stagein
+						si := basename(si)
+						to(graph
+							concat(str:quote(si), " [shape=parallelogram]")
+							concat(str:quote(si), " -> ", str:quote(tp))
+						)
+					)
+					for(pv, stageout
+						[path, var] := each(pv)
+						file := vdl:fileName(vdl:getfield(var, path=path))
+						file := basename(file)
+						label := vdl:niceName(var, path = path)
+						to(graph
+							concat(str:quote(file), " [shape=parallelogram,label=",
+								str:quote(label), "]")
+							concat(str:quote(tp), " -> ", str:quote(file))
+						)
+					)
+				)
+			)
+		)
+
+		element(fileSizes, [files]
+			math:sum(
+				for(f, files, file:size(file))
+			)
+		)
+		
+		element(readStandardFiles, [jobdir, stdout, stderr]
+			concat(
+				for(f, list(list("stderr.txt", stderr), list("stdout.txt", stdout))
+					[name, file] := each(f)
+					destfile := "{jobdir}/{file}"
+					nl()
+					"{name}: "
+					try(
+						file:read(destfile)
+						nl()
+					)
+				)
+			)
+		)
+		
+
+		element(execute2, [tr, optional(arguments, stdin, stdout, stderr, attributes), stagein, stageout,  restartout,
+			replicationGroup, replicationChannel]
+			stagein := list(unique(each(stagein)))
+			stageout := list(unique(each(stageout)))
+
+			allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein)
+
+				ddir := initDDir()
+				wfdir := try(
+					initSharedDir(rhost)
+					throw(exception("Could not initialize shared directory on {rhost}", exception))
+				)
+
+				uid := uid()
+				jobid := "{tr}-{uid}"
+				
+				jobdir := concat(ddir, "/jobs/", substring(uid, from=0, to=1), "/{jobid}/")
+
+				log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}")
+
+				statusMode := configProperty("status.mode",host=rhost)
+				wrapperMode := configProperty("wrapper.parameter.mode",host=rhost)
+
+				wrapfile := "{jobdir}/_paramfile"
+
+				stdout := try(getFieldValue(stdout), "stdout.txt")
+				stderr := try(getFieldValue(stderr), "stderr.txt")
+				fileDirs := fileDirs(stagein, stageout)
+				os := vdl:siteprofile(rhost, "SYSINFO:OS")
+				
+				scratch := vdl:siteprofile(rhost, "scratch")
+
+				if(wrapperMode == "files"
+					sequential(
+						sys:file:write(wrapfile,
+							"-e ",vdl:executable(tr, rhost), nl(),
+							"-out ", stdout, nl(),
+							"-err ", stderr, nl(),
+							"-i ", maybe(getFieldValue(stdin)), nl(),
+							"-d ", flatten(each(fileDirs)), nl(),
+							"-if ", flatten(infiles(stagein)), nl(),
+							"-of ", flatten(outfiles(stageout)), nl(),
+							"-wt", WRAPPERLOG_ALWAYS_TRANSFER,
+							"-sk", SITEDIR_KEEP,
+							"-cdmfile ", cdm:file(), nl(),
+							"-status ", statusMode, nl(),
+							for(a, arguments, "-a ", a, nl())
+						)
+					)
+				)
+
+				vdl:setprogress("Stage in")
+
+				try(
+					sequential(
+
+						if(wrapperMode == "files"
+							stageWrapperParams(jobid, wrapfile, wfdir, rhost)
+						)
+
+						log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " host={rhost}")
+
+						vdl:setprogress("Submitting")
+
+						if(wrapperMode == "files"
+							vdl:execute(
+								vdl:siteprofile(rhost, "swift:wrapperInterpreter"),
+								list(
+									vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"),
+									"_swiftwrap.wrapperstaging",
+									jobid,
+									"-urlprefix", URL_PREFIX,
+									"-jobdir", jobdir,
+									"-scratch", scratch, 
+									"-p", wrapfile
+								)
+								directory=wfdir
+								redirect=false
+								host=rhost
+								vdl:tcprofile(rhost, maybe(attributes=attributes), tr=tr) //this gets various app params from the tc, such as environment, walltime, etc
+								replicationGroup=replicationGroup
+								replicationChannel=replicationChannel
+								jobid=jobid
+							)
+						)
+						if(wrapperMode == "args"
+							vdl:execute(
+								vdl:siteprofile(rhost, "swift:wrapperInterpreter"),
+								list(
+									vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"),
+									"_swiftwrap.wrapperstaging",
+									jobid,
+									"-urlprefix", URL_PREFIX,
+									"-jobdir", jobdir,
+									"-scratch", scratch,
+									"-e", vdl:executable(tr, rhost),
+									"-out", stdout,
+									"-err", stderr,
+									"-i", maybe(getFieldValue(stdin)),
+									"-d", flatten(each(fileDirs)),
+									"-if", flatten(infiles(stagein)),
+									"-of", flatten(outfiles(stageout)),
+									"-wt", WRAPPERLOG_ALWAYS_TRANSFER,
+									"-sk", SITEDIR_KEEP,
+									"-cdmfile", cdm:file(),
+									"-status", statusMode,
+									"-a", maybe(each(arguments))
+								)
+								directory=wfdir
+								redirect=false
+								host=rhost
+								vdl:tcprofile(rhost, maybe(attributes=attributes), tr=tr)
+								replicationGroup=replicationGroup
+								replicationChannel=replicationChannel
+								jobid=jobid
+							)
+						)
+
+						vdl:setprogress("Checking status")
+						if(statusMode == "files"
+							checkJobStatus(rhost, wfdir, jobid, tr)
+						)
+
+						if(wrapperMode == "files"
+							file:remove(wrapfile)
+						)
+
+						log(LOG:DEBUG, "STAGING_OUT jobid={jobid}")
+
+
+						/* need to stage the files to upper scratch area in case they are not transfered to another site
+						   before all the files get cleaned out */
+
+
+						vdl:setprogress("Stage out")
+						doRestartlog(restartout)
+						
+						log(LOG:DEBUG, "JOB_END jobid={jobid}")
+					)
+					catch("^Abort$"
+						log(LOG:DEBUG, "JOB_CANCELED jobid={jobid}")
+						throw(exception)
+					)
+					catch("^(?!Abort$).*"
+						vdl:setprogress("Failed but can retry")
+						prev := exception
+						exception := try(exception(checkErrorFile(rhost, wfdir, jobid)), prev)
+						
+						log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception)
+
+						if(matches(exception,".*executable bit.*")
+							generateError(exception)
+						)
+						
+						outs := readStandardFiles(jobdir, stdout, stderr)
+
+						throw(
+							exception(
+								concat(
+									"Exception in {tr}:", nl(),
+									maybe("Arguments: ", arguments, nl()),
+									"Host: {rhost}", nl(),
+									"Directory: {scratch}/{jobid}",
+									"{outs}", nl(),
+									"----", nl()
+								)
+								exception
+							)
+						)
+					)
+				)
+			)
+		)
+
+		element(generateProvenanceGraph, [gdata]
+			pgraph := vdl:configProperty("pgraph")
+			gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph)
+			file:write(gname
+				"digraph SwiftProvenance {{", nl()
+				"	graph [", vdl:configProperty("pgraph.graph.options"), "];", nl()
+				"	node [", vdl:configProperty("pgraph.node.options"), "];", nl()
+
+				for(i, gdata
+					"	", i, nl()
+				)
+				"}", nl()
+			)
+			log(LOG:INFO, "Provenance graph saved in ", gname)
+		)
+	)
+)
+
+// Local variables:
+// mode: scheme
+// tab-width: 4
+// indent-tabs-mode: t
+// End:

Modified: trunk/libexec/vdl.k
===================================================================
--- trunk/libexec/vdl.k	2012-05-25 09:56:26 UTC (rev 5796)
+++ trunk/libexec/vdl.k	2012-05-26 23:14:20 UTC (rev 5797)
@@ -11,7 +11,12 @@
 	import("vdl-lib.xml", export = true)
 
 	pstaging := configProperty("use.provider.staging")
-	int := if (pstaging == "true", "vdl-int-staging.k", "vdl-int.k")
+	wstaging := configProperty("use.wrapper.staging")
+	int := if (
+		pstaging == "true", "vdl-int-staging.k",
+		wstaging == "true", "vdl-int-wrapper-staging.k", 
+		"vdl-int.k"
+	)
 
 	import(int)
 	import("java.k")



From hategan at ci.uchicago.edu  Sat May 26 18:15:16 2012
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Sat, 26 May 2012 18:15:16 -0500 (CDT)
Subject: [Swift-commit] r5798 - trunk/src/org/griphyn/vdl/karajan/lib
Message-ID: <20120526231516.4DAE99CCAF@svn.ci.uchicago.edu>

Author: hategan
Date: 2012-05-26 18:15:16 -0500 (Sat, 26 May 2012)
New Revision: 5798

Added:
   trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java
Log:
added class to get URL prefix for wrapper staging

Added: trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java	                        (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java	2012-05-26 23:15:16 UTC (rev 5798)
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2012 University of Chicago
+ * 
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.List;
+
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments;
+import org.griphyn.vdl.karajan.functions.ConfigProperty;
+
+public class GetURLPrefix extends AbstractSequentialWithArguments {
+    
+    @Override
+    protected void post(VariableStack stack) throws ExecutionException {
+        
+        String localServerBase = ConfigProperty.getProperty("wrapper.staging.local.server", stack);
+        
+        String cwd = stack.getExecutionContext().getCwd();
+        if (cwd.endsWith("/.")) {
+            cwd = cwd.substring(0, cwd.length() - 2);
+        }
+        
+        ret(stack, localServerBase + cwd);
+        
+        super.post(stack);
+    }
+}



From hategan at ci.uchicago.edu  Sat May 26 18:20:50 2012
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Sat, 26 May 2012 18:20:50 -0500 (CDT)
Subject: [Swift-commit] r5799 - trunk/etc
Message-ID: <20120526232050.116619CCAF@svn.ci.uchicago.edu>

Author: hategan
Date: 2012-05-26 18:20:49 -0500 (Sat, 26 May 2012)
New Revision: 5799

Modified:
   trunk/etc/swift.properties
Log:
updated swift.properties with wrapper staging properties

Modified: trunk/etc/swift.properties
===================================================================
--- trunk/etc/swift.properties	2012-05-26 23:15:16 UTC (rev 5798)
+++ trunk/etc/swift.properties	2012-05-26 23:20:49 UTC (rev 5799)
@@ -351,3 +351,17 @@
 
 use.provider.staging=false
 provider.staging.pin.swiftfiles=false
+
+# Controls whether wrapper staging is enabled
+# With wrapper staging, files are staged in and out by the 
+# swift wrapper, on the worker node.
+
+use.wrapper.staging=true
+
+# The URL prefix used to access local files in wrapper
+# staging mode. Full URLs are created by concatenating:
+#	* the value of this property
+#	* the current swift work directory (as an absolute path)
+#	* the file in question (relative to the current work directory)
+#
+wrapper.staging.local.server=file://



From davidk at ci.uchicago.edu  Mon May 28 09:30:26 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Mon, 28 May 2012 09:30:26 -0500 (CDT)
Subject: [Swift-commit] r5800 - trunk
Message-ID: <20120528143026.98F1F9CCAA@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-28 09:30:26 -0500 (Mon, 28 May 2012)
New Revision: 5800

Modified:
   trunk/build.xml
Log:
Make run-gwms-workers executable


Modified: trunk/build.xml
===================================================================
--- trunk/build.xml	2012-05-26 23:20:49 UTC (rev 5799)
+++ trunk/build.xml	2012-05-28 14:30:26 UTC (rev 5800)
@@ -94,7 +94,7 @@
 				<include name="stop-coaster-service"/>
 				<include name="run-worker.sh"/>
 				<include name="start-grid-service"/>
-
+                                <include name="run-gwms-workers"/>
 				<include name="foreachsite"/>
 				<include name="gen_gridsites"/>
 				<include name="get_greensites"/>



From davidk at ci.uchicago.edu  Mon May 28 09:43:25 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Mon, 28 May 2012 09:43:25 -0500 (CDT)
Subject: [Swift-commit] r5801 - trunk/bin
Message-ID: <20120528144325.9129A9CCAA@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-28 09:43:25 -0500 (Mon, 28 May 2012)
New Revision: 5801

Modified:
   trunk/bin/start-coaster-service
Log:
Some fixes for remote gwms


Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service	2012-05-28 14:30:26 UTC (rev 5800)
+++ trunk/bin/start-coaster-service	2012-05-28 14:43:25 UTC (rev 5801)
@@ -304,10 +304,14 @@
       crash "CONDOR_WORKERS undefined - please check coaster-service.conf"
    fi
 
+   if [ -z "$WORKER_LOGGING_LEVEL" ]; then
+      WORKER_LOGGING_LEVEL=INFO
+   fi
+
    echo Starting workers   
    if [ -n "$WORKER_RELAY_HOST" ]; then
       ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "mkdir -p condor"
-      ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS" 
+      ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "WORKER_INIT_CMD='$WORKER_INIT_CMD' WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS" 
    else
       mkdir -p condor
       run-gwms-workers $EXECUTION_URL $CONDOR_WORKERS



From hategan at ci.uchicago.edu  Wed May 30 13:58:21 2012
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Wed, 30 May 2012 13:58:21 -0500 (CDT)
Subject: [Swift-commit] r5802 - trunk/libexec
Message-ID: <20120530185821.E64989CF82@svn.ci.uchicago.edu>

Author: hategan
Date: 2012-05-30 13:58:21 -0500 (Wed, 30 May 2012)
New Revision: 5802

Modified:
   trunk/libexec/vdl-int-wrapper-staging.k
Log:
fixed "extra argument to checkJobStatus"

Modified: trunk/libexec/vdl-int-wrapper-staging.k
===================================================================
--- trunk/libexec/vdl-int-wrapper-staging.k	2012-05-28 14:43:25 UTC (rev 5801)
+++ trunk/libexec/vdl-int-wrapper-staging.k	2012-05-30 18:58:21 UTC (rev 5802)
@@ -386,7 +386,7 @@
 
 						vdl:setprogress("Checking status")
 						if(statusMode == "files"
-							checkJobStatus(rhost, wfdir, jobid, tr)
+							checkJobStatus(wfdir, jobid, tr)
 						)
 
 						if(wrapperMode == "files"



From davidk at ci.uchicago.edu  Wed May 23 09:53:21 2012
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 23 May 2012 14:53:21 -0000
Subject: [Swift-commit] r5795 - in SwiftApps/cray-benchmarks: . dssat-eff
	dssat-eff/ddnap dssat-eff/plots eff.hera eff.raven plots plots2
Message-ID: <20120523145320.A00809CCA5@svn.ci.uchicago.edu>

Author: davidk
Date: 2012-05-23 09:53:20 -0500 (Wed, 23 May 2012)
New Revision: 5795

Added:
   SwiftApps/cray-benchmarks/dssat-eff/
   SwiftApps/cray-benchmarks/dssat-eff/ddnap/
   SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.sh
   SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.swift
   SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.xml
   SwiftApps/cray-benchmarks/dssat-eff/ddnap/input.test
   SwiftApps/cray-benchmarks/dssat-eff/ddnap/sites.xml
   SwiftApps/cray-benchmarks/dssat-eff/ddnap/tc.data
   SwiftApps/cray-benchmarks/dssat-eff/do_eff.sh
   SwiftApps/cray-benchmarks/dssat-eff/dssat
   SwiftApps/cray-benchmarks/dssat-eff/plots/
   SwiftApps/cray-benchmarks/dssat-eff/plots/extract4plots
   SwiftApps/cray-benchmarks/dssat-eff/plots/plotit.gp
   SwiftApps/cray-benchmarks/dssat-eff/plots/taskrate.sh
   SwiftApps/cray-benchmarks/dssat-eff/plots/timetocomplete.sh
   SwiftApps/cray-benchmarks/eff.hera/
   SwiftApps/cray-benchmarks/eff.hera/eff.gp
   SwiftApps/cray-benchmarks/eff.hera/eff100.txt
   SwiftApps/cray-benchmarks/eff.hera/eff150.txt
   SwiftApps/cray-benchmarks/eff.hera/eff200.txt
   SwiftApps/cray-benchmarks/eff.hera/eff400.txt
   SwiftApps/cray-benchmarks/eff.hera/eff90.pdf
   SwiftApps/cray-benchmarks/eff.hera/eff90.png
   SwiftApps/cray-benchmarks/eff.hera/eff90.ps
   SwiftApps/cray-benchmarks/eff.hera/eff90.svg
   SwiftApps/cray-benchmarks/eff.hera/exm.pdf
   SwiftApps/cray-benchmarks/eff.hera/expectedtime.gp
   SwiftApps/cray-benchmarks/eff.hera/mike.gp
   SwiftApps/cray-benchmarks/eff.hera/mike.txt
   SwiftApps/cray-benchmarks/eff.hera/mnotes
   SwiftApps/cray-benchmarks/eff.hera/plotit.gp
   SwiftApps/cray-benchmarks/eff.hera/timetocomplete.sh
   SwiftApps/cray-benchmarks/eff.raven/
   SwiftApps/cray-benchmarks/eff.raven/eff.gp
   SwiftApps/cray-benchmarks/eff.raven/eff100.txt
   SwiftApps/cray-benchmarks/eff.raven/eff200.txt
   SwiftApps/cray-benchmarks/eff.raven/eff400.txt
   SwiftApps/cray-benchmarks/eff.raven/eff90-raven.pdf
   SwiftApps/cray-benchmarks/eff.raven/eff90.pdf
   SwiftApps/cray-benchmarks/eff.raven/expectedtime.gp
   SwiftApps/cray-benchmarks/eff.raven/plotit.gp
   SwiftApps/cray-benchmarks/plots/
   SwiftApps/cray-benchmarks/plots/T.data
   SwiftApps/cray-benchmarks/plots/active.txt
   SwiftApps/cray-benchmarks/plots/activeplot.png
   SwiftApps/cray-benchmarks/plots/activetics.txt
   SwiftApps/cray-benchmarks/plots/anneal.data
   SwiftApps/cray-benchmarks/plots/cumulative.txt
   SwiftApps/cray-benchmarks/plots/cumulativeplot-openmp.png
   SwiftApps/cray-benchmarks/plots/cumulativeplot.png
   SwiftApps/cray-benchmarks/plots/cumultics.txt
   SwiftApps/cray-benchmarks/plots/extract4plots
   SwiftApps/cray-benchmarks/plots/multiloss.txt
   SwiftApps/cray-benchmarks/plots/plot_active.txt
   SwiftApps/cray-benchmarks/plots/plot_cumulative.txt
   SwiftApps/cray-benchmarks/plots/plotit.gp
   SwiftApps/cray-benchmarks/plots/scs.png
   SwiftApps/cray-benchmarks/plots2/
   SwiftApps/cray-benchmarks/plots2/T.data
   SwiftApps/cray-benchmarks/plots2/active.txt
   SwiftApps/cray-benchmarks/plots2/activeplot.png
   SwiftApps/cray-benchmarks/plots2/activetics.txt
   SwiftApps/cray-benchmarks/plots2/anneal.data
   SwiftApps/cray-benchmarks/plots2/cumulative.txt
   SwiftApps/cray-benchmarks/plots2/cumulativeplot-openmp.png
   SwiftApps/cray-benchmarks/plots2/cumulativeplot.png
   SwiftApps/cray-benchmarks/plots2/cumultics.txt
   SwiftApps/cray-benchmarks/plots2/extract4plots
   SwiftApps/cray-benchmarks/plots2/multiloss.txt
   SwiftApps/cray-benchmarks/plots2/plot_active.txt
   SwiftApps/cray-benchmarks/plots2/plot_cumulative.txt
   SwiftApps/cray-benchmarks/plots2/plotit.gp
   SwiftApps/cray-benchmarks/plots2/run021.out
   SwiftApps/cray-benchmarks/plots2/scs.png
Log:
Random dump of scripts to be sorted out and organized later


Added: SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.sh
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.sh	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.sh	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+sleep $3
+dd if=$1 of=t bs=1M
+dd if=t of=$2
+


Property changes on: SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.swift
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.swift	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.swift	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,14 @@
+type file;
+
+app ddnap (string delay, string i, string o)
+{
+  ddnap i o delay;
+}
+
+string delay=@arg("s","1");
+string infile=@arg("input");
+string outfile=@arg("output");
+
+foreach j in [1:@toint(@arg("n","1"))] {
+  ddnap(delay, infile, outfile);
+}

Added: SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.xml
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.xml	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/ddnap/ddnap.xml	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,66 @@
+<program xmlns="http://ci.uchicago.edu/swift/2009/02/swiftscript"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns:xs="http://www.w3.org/2001/XMLSchema">
+  <types>
+     	<type>
+     		<typename>file</typename>
+     		<typealias>string</typealias>
+     		<typestructure></typestructure>
+     	</type>
+  </types>
+  <procedure name="ddnap" src="line 3">
+
+      <input
+    name="delay" type="string"
+
+     xsi:nil="true" />
+
+      <input
+    name="i" type="string"
+
+     xsi:nil="true" />
+
+      <input
+    name="o" type="string"
+
+     xsi:nil="true" />
+    <binding>
+      <application src="line 3">
+        <executable>ddnap</executable>
+        <variableReference>i</variableReference><variableReference>o</variableReference><variableReference>delay</variableReference>  </application>
+    </binding>
+  </procedure>
+  <variable name="delay" type="string" isGlobal="false" xsi:nil="true"/>
+  <assign src="line 9">
+   <variableReference>delay</variableReference>
+   <function name="arg">
+     <stringConstant>s</stringConstant><stringConstant>1</stringConstant></function>
+  </assign>
+  <variable name="infile" type="string" isGlobal="false" xsi:nil="true"/>
+  <assign src="line 10">
+   <variableReference>infile</variableReference>
+   <function name="arg">
+     <stringConstant>input</stringConstant></function>
+  </assign>
+  <variable name="outfile" type="string" isGlobal="false" xsi:nil="true"/>
+  <assign src="line 12">
+   <variableReference>outfile</variableReference>
+   <function name="arg">
+     <stringConstant>output</stringConstant></function>
+  </assign>
+  <foreach var="j"  src="line 12">
+  <in>
+    <range>
+      <integerConstant>1</integerConstant>
+      <function name="toint">
+        <function name="arg">
+          <stringConstant>n</stringConstant><stringConstant>1</stringConstant></function></function>
+    </range>
+  </in>
+  <body><call proc="ddnap" src="line 13">
+      <input><variableReference>delay</variableReference></input>
+      <input><variableReference>infile</variableReference></input>
+      <input><variableReference>outfile</variableReference></input>
+    </call></body>
+  </foreach>
+</program>

Added: SwiftApps/cray-benchmarks/dssat-eff/ddnap/input.test
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/ddnap/input.test	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/ddnap/input.test	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,100 @@
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass
+dfjdklass


Property changes on: SwiftApps/cray-benchmarks/dssat-eff/ddnap/input.test
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/dssat-eff/ddnap/sites.xml
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/ddnap/sites.xml	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/ddnap/sites.xml	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,14 @@
+<config>
+  <pool handle="localhost">
+    <filesystem provider="local" />
+    <execution provider="coaster" jobmanager="local:local"/>
+    <profile namespace="karajan"  key="jobthrottle">2.55</profile>
+    <profile namespace="karajan"  key="initialScore">10000</profile>
+    <profile namespace="globus"   key="jobsPerNode">4</profile>
+    <profile namespace="globus"   key="slots">8</profile>
+    <profile namespace="globus"   key="maxTime">1000</profile>
+    <profile namespace="globus"   key="nodeGranularity">1</profile>
+    <profile namespace="globus"   key="maxNodes">4</profile>
+    <workdirectory>/tmp</workdirectory>
+  </pool>
+</config>

Added: SwiftApps/cray-benchmarks/dssat-eff/ddnap/tc.data
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/ddnap/tc.data	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/ddnap/tc.data	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1 @@
+localhost ddnap /home/users/p01537/ddnap/ddnap.sh null null null

Added: SwiftApps/cray-benchmarks/dssat-eff/do_eff.sh
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/do_eff.sh	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/do_eff.sh	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+export PATH=$PWD/swift-tickerfix/bin:$PATH
+export STORAGE=$HOME
+
+# tar and remove
+movelogs()
+{
+    dir=`ls -1rtd run* | tail -1`
+    tar cvfz $dir.tar.gz $dir
+    cp $dir.tar.gz $STORAGE && rm $dir.tar.gz && rm -rf $dir/*
+}
+
+./dssat hera 100 3600 3200 $PWD 100 && movelogs
+./dssat hera 300 3600 6400 $PWD 100 && movelogs
+./dssat hera 500 3600 12800 $PWD 100 && movelogs
+./dssat hera 588 3600 12800 $PWD 100 && movelogs
+
+./dssat hera 100 3600 3200 $PWD 200 && movelogs
+./dssat hera 300 3600 6400 $PWD 200 && movelogs
+./dssat hera 500 3600 12800 $PWD 200 && movelogs
+./dssat hera 588 3600 12800 $PWD 200 && movelogs
+
+./dssat hera 100 3600 3200 $PWD 150 && movelogs
+./dssat hera 300 3600 6400 $PWD 150 && movelogs
+./dssat hera 500 3600 12800 $PWD 150 && movelogs
+./dssat hera 588 3600 12800 $PWD 150 && movelogs
+


Property changes on: SwiftApps/cray-benchmarks/dssat-eff/do_eff.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/dssat-eff/dssat
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/dssat	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/dssat	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ $# -lt 5 ]; then
+    echo "Usage: $0 site #nodes maxtime #cells dssat_home <sleepvalue> <datasize>
+
+             maxtime must be in seconds
+             dssat_home is where all the DSSAT file reside(normally PWD)"
+    exit 1
+fi
+
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+echo "Running in $rundir"
+
+mkdir $rundir
+
+cd $rundir
+mkdir output
+
+export WORK=/dev/shm
+execsite=$1
+export NODES=$2
+export MAXTIME=$3
+
+cells=$4
+export dssat_home=$5
+if [ ! -z "$6" ]; then
+   export sleepvalue=$6
+fi
+
+if [ ! -z "$7" ]; then
+   export datasize=$7
+   dd if=../rand.data of=data.txt bs=1K count=$datasize  
+fi
+
+
+gensites -p ../conf/$execsite.cf ../conf/sites.$execsite.xml > sites.xml
+
+cp ../conf/$execsite.cf cf
+cp ../RunDSSAT.swift .
+head -$cells ../gridLists/gridList100000.txt > gridList.txt
+
+date +%s > swift.out
+if [ ! -z "$sleepvalue" ]; then
+   if [ ! -z "$datasize" ]; then
+   SWIFT_HEAP_MAX=7000M swift -config cf -tc.file tc.data -sites.file sites.xml \
+      RunDSSAT.swift \
+      -cmppath="$dssat_home/campaigns/dummy100000" \
+      -refdata="$dssat_home/common" \
+      -campaign="$dssat_home/campaigns/dummy100000" \
+      -bindata="$dssat_home/bin" \
+      -sleeponly="yes" \
+      -sleepvalue="$sleepvalue" \
+      -usedata="yes" \
+      -datasize="$datasize" 2>&1 | tee -a swift.out
+   else
+      SWIFT_HEAP_MAX=7000M swift -config cf -tc.file tc.data -sites.file sites.xml \
+         RunDSSAT.swift \
+         -cmppath="$dssat_home/campaigns/dummy100000" \
+         -refdata="$dssat_home/common" \
+         -campaign="$dssat_home/campaigns/dummy100000" \
+         -bindata="$dssat_home/bin" \
+         -sleeponly="yes" \
+         -sleepvalue="$sleepvalue" 2>&1 | tee -a swift.out 
+   fi
+else
+   SWIFT_HEAP_MAX=7000M swift -config cf -tc.file tc.data -sites.file sites.xml \
+      RunDSSAT.swift \
+      -cmppath="$dssat_home/campaigns/dummy100000" \
+      -refdata="$dssat_home/common" \
+      -campaign="$dssat_home/campaigns/dummy100000" \
+      -bindata="$dssat_home/bin" 2>&1 | tee -a swift.out
+fi
+date +%s >> swift.out
+../plots/timetocomplete.sh swift.out
+


Property changes on: SwiftApps/cray-benchmarks/dssat-eff/dssat
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/dssat-eff/plots/extract4plots
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/plots/extract4plots	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/plots/extract4plots	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+#usage: ./extract4plots <swift.outfile>
+
+SWIFTOUTFILE=$1
+
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE | head -n 1 | cut -f4-9 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE | tail -n 1 | cut -f4-9 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+
+#extract active runs in a file
+grep -o -i "Active:[0-9]*" $SWIFTOUTFILE | awk -F: '{print $2}' > active.txt
+
+#extract successful completions in a file
+grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE | awk -F: '{print $2}' > cumulative.txt
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+grep  "T =" $SWIFTOUTFILE | awk '{print $6}' | cut -c8- | sed 's/....$//' > T.data
+
+grep multi_annealing $SWIFTOUTFILE | grep "1;30" | awk '{print $3}' | cut -c11- | sed 's/....$//' > anneal.data
+
+grep returning $SWIFTOUTFILE | awk '{print $3, $4, $5, $6}' | sed -e 's/'ci='//' -e 's/'cj='//' -e 's/'r.loss='//' -e 's/'r.sdev='//' | sort -t' ' -k 1,2n > multiloss.txt


Property changes on: SwiftApps/cray-benchmarks/dssat-eff/plots/extract4plots
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/dssat-eff/plots/plotit.gp
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/plots/plotit.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/plots/plotit.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,48 @@
+#To be run as a gnuplot script as follows:
+
+# $ gnuplot plotit.gp
+
+set terminal png enhanced 
+#set term postscript eps enhanced 
+#set terminal svg enhanced size 1000 1000
+#set style line 1 linecolor rgb "blue"
+set output "activeplot.png"
+set nokey
+set xlabel "Time in sec"
+set ylabel "number of active jobs"
+set title "Active jobs"
+plot "plot_active.txt" using 1:2 with line
+
+#multiplies the active jobs by 24 to take into account the number of openmp
+#procs running on Beagle
+set output "cumulativeplot-openmp.png"
+set xlabel "Time in seconds"
+set ylabel "number of completed jobs"
+set title "Cumulative SciColSim-openMP jobs"
+plot "plot_cumulative.txt" using 1:($2*24) with lines
+
+set output "cumulativeplot.png"
+set xlabel "Time in seconds"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+
+set output "scs.png"
+set xlabel "Evolution"
+set ylabel "Value of T"
+set title "SciColSim evolution Results"
+plot "T.data" using 1 with lines 
+
+set output "scs_loss.png"
+set title "SciColSim evolution loss Results"
+set xlabel "Evolution"
+set ylabel "Value of loss(AR)"
+plot "anneal.data" using 1 with lines 
+
+set output "multiloss.png"
+set title "SciColSim evolution loss Results"
+set key auto
+set yrange [0:200]
+set xlabel "Evolution"
+set ylabel "loss"
+plot "multiloss.txt" using 3 with lines title "multiloss mean val",  "multiloss.txt" using ($3+$4) with lines title "+stddev", "multiloss.txt" using ($3-$4) with lines title "-stddev"

Added: SwiftApps/cray-benchmarks/dssat-eff/plots/taskrate.sh
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/plots/taskrate.sh	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/plots/taskrate.sh	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+for log in $@
+do
+   stime=`head -1 $log`
+   etime=`tail -1 $log`
+   python -c "print 10000 / ( $etime - $stime )"
+done


Property changes on: SwiftApps/cray-benchmarks/dssat-eff/plots/taskrate.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/dssat-eff/plots/timetocomplete.sh
===================================================================
--- SwiftApps/cray-benchmarks/dssat-eff/plots/timetocomplete.sh	                        (rev 0)
+++ SwiftApps/cray-benchmarks/dssat-eff/plots/timetocomplete.sh	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+for log in $@
+do
+   stime=`head -1 $log`
+   etime=`tail -1 $log`
+   expr $etime - $stime
+done


Property changes on: SwiftApps/cray-benchmarks/dssat-eff/plots/timetocomplete.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/eff.hera/eff.gp
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/eff.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/eff.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,24 @@
+#set terminal png font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf" 12 size 4000,2800
+#set term postscript eps enhanced 
+#set terminal svg enhanced size 2000 800
+#set terminal postscript enhanced
+set terminal pdf
+
+#set output "eff90.ps"
+#set output "eff90.png"
+set output "eff90.pdf"
+
+set decimal locale
+#set format x "%'g"
+set xrange [2000:17000]
+
+set style line 1 lt 1 lw 8 pt 3 lc rgb "red"
+set style line 2 lt 1 lw 8 pt 3 lc rgb "red"
+set style line 3 lt 1 lw 8 pt 3 lc rgb "blue"
+set style line 4 lt 1 lw 8 pt 3 lc rgb "blue"
+
+set key above
+set xlabel "Number of Processors" 
+set ylabel "Efficiency" 
+set yrange [50:100]
+plot "eff100.txt" using 1:5 with linespoints pt 5 lw 5 ps 0.5 lc rgb "red" title "100 second tasks", "eff200.txt" using 1:5 with linespoints pt 7 ps 0.5 lw 5 lc rgb "orange" title "200 second tasks", "eff400.txt" using 1:5 with linespoint pt 9 ps 0.5 lw 5 lc rgb "#1E90FF" title "400 seconds"

Added: SwiftApps/cray-benchmarks/eff.hera/eff100.txt
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/eff100.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/eff100.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,3 @@
+16000	300	448	107.14	66.96
+9600	300	388	74.23	77.32
+3200	300	338	28.40	88.76

Added: SwiftApps/cray-benchmarks/eff.hera/eff150.txt
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/eff150.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/eff150.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,3 @@
+3200	450	499	19.24	90.18
+9600	450	570	50.53	78.95
+16000	450	626	76.68	71.88

Added: SwiftApps/cray-benchmarks/eff.hera/eff200.txt
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/eff200.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/eff200.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,3 @@
+16000	600	741	64.78	80.97
+9600	600	693	41.56	86.58
+3200	600	639	15.02	93.90

Added: SwiftApps/cray-benchmarks/eff.hera/eff400.txt
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/eff400.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/eff400.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,3 @@
+3200	1200	1245	7.71	96.39
+9600	1200	1294	22.26	92.74
+16000	1200	1345	35.69	89.22

Added: SwiftApps/cray-benchmarks/eff.hera/eff90.pdf
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/eff.hera/eff90.pdf
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/eff.hera/eff90.png
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/eff.hera/eff90.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/eff.hera/eff90.ps
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/eff90.ps	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/eff90.ps	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,696 @@
+%!PS-Adobe-2.0
+%%Title: eff90.ps
+%%Creator: gnuplot 4.4 patchlevel 3
+%%CreationDate: Wed May  2 17:00:42 2012
+%%DocumentFonts: (atend)
+%%BoundingBox: 50 50 554 770
+%%Orientation: Landscape
+%%Pages: (atend)
+%%EndComments
+%%BeginProlog
+/gnudict 256 dict def
+gnudict begin
+%
+% The following true/false flags may be edited by hand if desired.
+% The unit line width and grayscale image gamma correction may also be changed.
+%
+/Color false def
+/Blacktext false def
+/Solid false def
+/Dashlength 1 def
+/Landscape true def
+/Level1 false def
+/Rounded false def
+/ClipToBoundingBox false def
+/TransparentPatterns false def
+/gnulinewidth 5.000 def
+/userlinewidth gnulinewidth def
+/Gamma 1.0 def
+%
+/vshift -46 def
+/dl1 {
+  10.0 Dashlength mul mul
+  Rounded { currentlinewidth 0.75 mul sub dup 0 le { pop 0.01 } if } if
+} def
+/dl2 {
+  10.0 Dashlength mul mul
+  Rounded { currentlinewidth 0.75 mul add } if
+} def
+/hpt_ 31.5 def
+/vpt_ 31.5 def
+/hpt hpt_ def
+/vpt vpt_ def
+Level1 {} {
+/SDict 10 dict def
+systemdict /pdfmark known not {
+  userdict /pdfmark systemdict /cleartomark get put
+} if
+SDict begin [
+  /Title (eff90.ps)
+  /Subject (gnuplot plot)
+  /Creator (gnuplot 4.4 patchlevel 3)
+  /Author (david)
+%  /Producer (gnuplot)
+%  /Keywords ()
+  /CreationDate (Wed May  2 17:00:42 2012)
+  /DOCINFO pdfmark
+end
+} ifelse
+/doclip {
+  ClipToBoundingBox {
+    newpath 50 50 moveto 554 50 lineto 554 770 lineto 50 770 lineto closepath
+    clip
+  } if
+} def
+%
+% Gnuplot Prolog Version 4.4 (August 2010)
+%
+%/SuppressPDFMark true def
+%
+/M {moveto} bind def
+/L {lineto} bind def
+/R {rmoveto} bind def
+/V {rlineto} bind def
+/N {newpath moveto} bind def
+/Z {closepath} bind def
+/C {setrgbcolor} bind def
+/f {rlineto fill} bind def
+/g {setgray} bind def
+/Gshow {show} def   % May be redefined later in the file to support UTF-8
+/vpt2 vpt 2 mul def
+/hpt2 hpt 2 mul def
+/Lshow {currentpoint stroke M 0 vshift R 
+	Blacktext {gsave 0 setgray show grestore} {show} ifelse} def
+/Rshow {currentpoint stroke M dup stringwidth pop neg vshift R
+	Blacktext {gsave 0 setgray show grestore} {show} ifelse} def
+/Cshow {currentpoint stroke M dup stringwidth pop -2 div vshift R 
+	Blacktext {gsave 0 setgray show grestore} {show} ifelse} def
+/UP {dup vpt_ mul /vpt exch def hpt_ mul /hpt exch def
+  /hpt2 hpt 2 mul def /vpt2 vpt 2 mul def} def
+/DL {Color {setrgbcolor Solid {pop []} if 0 setdash}
+ {pop pop pop 0 setgray Solid {pop []} if 0 setdash} ifelse} def
+/BL {stroke userlinewidth 2 mul setlinewidth
+	Rounded {1 setlinejoin 1 setlinecap} if} def
+/AL {stroke userlinewidth 2 div setlinewidth
+	Rounded {1 setlinejoin 1 setlinecap} if} def
+/UL {dup gnulinewidth mul /userlinewidth exch def
+	dup 1 lt {pop 1} if 10 mul /udl exch def} def
+/PL {stroke userlinewidth setlinewidth
+	Rounded {1 setlinejoin 1 setlinecap} if} def
+3.8 setmiterlimit
+% Default Line colors
+/LCw {1 1 1} def
+/LCb {0 0 0} def
+/LCa {0 0 0} def
+/LC0 {1 0 0} def
+/LC1 {0 1 0} def
+/LC2 {0 0 1} def
+/LC3 {1 0 1} def
+/LC4 {0 1 1} def
+/LC5 {1 1 0} def
+/LC6 {0 0 0} def
+/LC7 {1 0.3 0} def
+/LC8 {0.5 0.5 0.5} def
+% Default Line Types
+/LTw {PL [] 1 setgray} def
+/LTb {BL [] LCb DL} def
+/LTa {AL [1 udl mul 2 udl mul] 0 setdash LCa setrgbcolor} def
+/LT0 {PL [] LC0 DL} def
+/LT1 {PL [4 dl1 2 dl2] LC1 DL} def
+/LT2 {PL [2 dl1 3 dl2] LC2 DL} def
+/LT3 {PL [1 dl1 1.5 dl2] LC3 DL} def
+/LT4 {PL [6 dl1 2 dl2 1 dl1 2 dl2] LC4 DL} def
+/LT5 {PL [3 dl1 3 dl2 1 dl1 3 dl2] LC5 DL} def
+/LT6 {PL [2 dl1 2 dl2 2 dl1 6 dl2] LC6 DL} def
+/LT7 {PL [1 dl1 2 dl2 6 dl1 2 dl2 1 dl1 2 dl2] LC7 DL} def
+/LT8 {PL [2 dl1 2 dl2 2 dl1 2 dl2 2 dl1 2 dl2 2 dl1 4 dl2] LC8 DL} def
+/Pnt {stroke [] 0 setdash gsave 1 setlinecap M 0 0 V stroke grestore} def
+/Dia {stroke [] 0 setdash 2 copy vpt add M
+  hpt neg vpt neg V hpt vpt neg V
+  hpt vpt V hpt neg vpt V closepath stroke
+  Pnt} def
+/Pls {stroke [] 0 setdash vpt sub M 0 vpt2 V
+  currentpoint stroke M
+  hpt neg vpt neg R hpt2 0 V stroke
+ } def
+/Box {stroke [] 0 setdash 2 copy exch hpt sub exch vpt add M
+  0 vpt2 neg V hpt2 0 V 0 vpt2 V
+  hpt2 neg 0 V closepath stroke
+  Pnt} def
+/Crs {stroke [] 0 setdash exch hpt sub exch vpt add M
+  hpt2 vpt2 neg V currentpoint stroke M
+  hpt2 neg 0 R hpt2 vpt2 V stroke} def
+/TriU {stroke [] 0 setdash 2 copy vpt 1.12 mul add M
+  hpt neg vpt -1.62 mul V
+  hpt 2 mul 0 V
+  hpt neg vpt 1.62 mul V closepath stroke
+  Pnt} def
+/Star {2 copy Pls Crs} def
+/BoxF {stroke [] 0 setdash exch hpt sub exch vpt add M
+  0 vpt2 neg V hpt2 0 V 0 vpt2 V
+  hpt2 neg 0 V closepath fill} def
+/TriUF {stroke [] 0 setdash vpt 1.12 mul add M
+  hpt neg vpt -1.62 mul V
+  hpt 2 mul 0 V
+  hpt neg vpt 1.62 mul V closepath fill} def
+/TriD {stroke [] 0 setdash 2 copy vpt 1.12 mul sub M
+  hpt neg vpt 1.62 mul V
+  hpt 2 mul 0 V
+  hpt neg vpt -1.62 mul V closepath stroke
+  Pnt} def
+/TriDF {stroke [] 0 setdash vpt 1.12 mul sub M
+  hpt neg vpt 1.62 mul V
+  hpt 2 mul 0 V
+  hpt neg vpt -1.62 mul V closepath fill} def
+/DiaF {stroke [] 0 setdash vpt add M
+  hpt neg vpt neg V hpt vpt neg V
+  hpt vpt V hpt neg vpt V closepath fill} def
+/Pent {stroke [] 0 setdash 2 copy gsave
+  translate 0 hpt M 4 {72 rotate 0 hpt L} repeat
+  closepath stroke grestore Pnt} def
+/PentF {stroke [] 0 setdash gsave
+  translate 0 hpt M 4 {72 rotate 0 hpt L} repeat
+  closepath fill grestore} def
+/Circle {stroke [] 0 setdash 2 copy
+  hpt 0 360 arc stroke Pnt} def
+/CircleF {stroke [] 0 setdash hpt 0 360 arc fill} def
+/C0 {BL [] 0 setdash 2 copy moveto vpt 90 450 arc} bind def
+/C1 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 0 90 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C2 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 90 180 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C3 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 0 180 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C4 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 180 270 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C5 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 0 90 arc
+	2 copy moveto
+	2 copy vpt 180 270 arc closepath fill
+	vpt 0 360 arc} bind def
+/C6 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 90 270 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C7 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 0 270 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C8 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 270 360 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C9 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 270 450 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C10 {BL [] 0 setdash 2 copy 2 copy moveto vpt 270 360 arc closepath fill
+	2 copy moveto
+	2 copy vpt 90 180 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C11 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 0 180 arc closepath fill
+	2 copy moveto
+	2 copy vpt 270 360 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C12 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 180 360 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C13 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 0 90 arc closepath fill
+	2 copy moveto
+	2 copy vpt 180 360 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/C14 {BL [] 0 setdash 2 copy moveto
+	2 copy vpt 90 360 arc closepath fill
+	vpt 0 360 arc} bind def
+/C15 {BL [] 0 setdash 2 copy vpt 0 360 arc closepath fill
+	vpt 0 360 arc closepath} bind def
+/Rec {newpath 4 2 roll moveto 1 index 0 rlineto 0 exch rlineto
+	neg 0 rlineto closepath} bind def
+/Square {dup Rec} bind def
+/Bsquare {vpt sub exch vpt sub exch vpt2 Square} bind def
+/S0 {BL [] 0 setdash 2 copy moveto 0 vpt rlineto BL Bsquare} bind def
+/S1 {BL [] 0 setdash 2 copy vpt Square fill Bsquare} bind def
+/S2 {BL [] 0 setdash 2 copy exch vpt sub exch vpt Square fill Bsquare} bind def
+/S3 {BL [] 0 setdash 2 copy exch vpt sub exch vpt2 vpt Rec fill Bsquare} bind def
+/S4 {BL [] 0 setdash 2 copy exch vpt sub exch vpt sub vpt Square fill Bsquare} bind def
+/S5 {BL [] 0 setdash 2 copy 2 copy vpt Square fill
+	exch vpt sub exch vpt sub vpt Square fill Bsquare} bind def
+/S6 {BL [] 0 setdash 2 copy exch vpt sub exch vpt sub vpt vpt2 Rec fill Bsquare} bind def
+/S7 {BL [] 0 setdash 2 copy exch vpt sub exch vpt sub vpt vpt2 Rec fill
+	2 copy vpt Square fill Bsquare} bind def
+/S8 {BL [] 0 setdash 2 copy vpt sub vpt Square fill Bsquare} bind def
+/S9 {BL [] 0 setdash 2 copy vpt sub vpt vpt2 Rec fill Bsquare} bind def
+/S10 {BL [] 0 setdash 2 copy vpt sub vpt Square fill 2 copy exch vpt sub exch vpt Square fill
+	Bsquare} bind def
+/S11 {BL [] 0 setdash 2 copy vpt sub vpt Square fill 2 copy exch vpt sub exch vpt2 vpt Rec fill
+	Bsquare} bind def
+/S12 {BL [] 0 setdash 2 copy exch vpt sub exch vpt sub vpt2 vpt Rec fill Bsquare} bind def
+/S13 {BL [] 0 setdash 2 copy exch vpt sub exch vpt sub vpt2 vpt Rec fill
+	2 copy vpt Square fill Bsquare} bind def
+/S14 {BL [] 0 setdash 2 copy exch vpt sub exch vpt sub vpt2 vpt Rec fill
+	2 copy exch vpt sub exch vpt Square fill Bsquare} bind def
+/S15 {BL [] 0 setdash 2 copy Bsquare fill Bsquare} bind def
+/D0 {gsave translate 45 rotate 0 0 S0 stroke grestore} bind def
+/D1 {gsave translate 45 rotate 0 0 S1 stroke grestore} bind def
+/D2 {gsave translate 45 rotate 0 0 S2 stroke grestore} bind def
+/D3 {gsave translate 45 rotate 0 0 S3 stroke grestore} bind def
+/D4 {gsave translate 45 rotate 0 0 S4 stroke grestore} bind def
+/D5 {gsave translate 45 rotate 0 0 S5 stroke grestore} bind def
+/D6 {gsave translate 45 rotate 0 0 S6 stroke grestore} bind def
+/D7 {gsave translate 45 rotate 0 0 S7 stroke grestore} bind def
+/D8 {gsave translate 45 rotate 0 0 S8 stroke grestore} bind def
+/D9 {gsave translate 45 rotate 0 0 S9 stroke grestore} bind def
+/D10 {gsave translate 45 rotate 0 0 S10 stroke grestore} bind def
+/D11 {gsave translate 45 rotate 0 0 S11 stroke grestore} bind def
+/D12 {gsave translate 45 rotate 0 0 S12 stroke grestore} bind def
+/D13 {gsave translate 45 rotate 0 0 S13 stroke grestore} bind def
+/D14 {gsave translate 45 rotate 0 0 S14 stroke grestore} bind def
+/D15 {gsave translate 45 rotate 0 0 S15 stroke grestore} bind def
+/DiaE {stroke [] 0 setdash vpt add M
+  hpt neg vpt neg V hpt vpt neg V
+  hpt vpt V hpt neg vpt V closepath stroke} def
+/BoxE {stroke [] 0 setdash exch hpt sub exch vpt add M
+  0 vpt2 neg V hpt2 0 V 0 vpt2 V
+  hpt2 neg 0 V closepath stroke} def
+/TriUE {stroke [] 0 setdash vpt 1.12 mul add M
+  hpt neg vpt -1.62 mul V
+  hpt 2 mul 0 V
+  hpt neg vpt 1.62 mul V closepath stroke} def
+/TriDE {stroke [] 0 setdash vpt 1.12 mul sub M
+  hpt neg vpt 1.62 mul V
+  hpt 2 mul 0 V
+  hpt neg vpt -1.62 mul V closepath stroke} def
+/PentE {stroke [] 0 setdash gsave
+  translate 0 hpt M 4 {72 rotate 0 hpt L} repeat
+  closepath stroke grestore} def
+/CircE {stroke [] 0 setdash 
+  hpt 0 360 arc stroke} def
+/Opaque {gsave closepath 1 setgray fill grestore 0 setgray closepath} def
+/DiaW {stroke [] 0 setdash vpt add M
+  hpt neg vpt neg V hpt vpt neg V
+  hpt vpt V hpt neg vpt V Opaque stroke} def
+/BoxW {stroke [] 0 setdash exch hpt sub exch vpt add M
+  0 vpt2 neg V hpt2 0 V 0 vpt2 V
+  hpt2 neg 0 V Opaque stroke} def
+/TriUW {stroke [] 0 setdash vpt 1.12 mul add M
+  hpt neg vpt -1.62 mul V
+  hpt 2 mul 0 V
+  hpt neg vpt 1.62 mul V Opaque stroke} def
+/TriDW {stroke [] 0 setdash vpt 1.12 mul sub M
+  hpt neg vpt 1.62 mul V
+  hpt 2 mul 0 V
+  hpt neg vpt -1.62 mul V Opaque stroke} def
+/PentW {stroke [] 0 setdash gsave
+  translate 0 hpt M 4 {72 rotate 0 hpt L} repeat
+  Opaque stroke grestore} def
+/CircW {stroke [] 0 setdash 
+  hpt 0 360 arc Opaque stroke} def
+/BoxFill {gsave Rec 1 setgray fill grestore} def
+/Density {
+  /Fillden exch def
+  currentrgbcolor
+  /ColB exch def /ColG exch def /ColR exch def
+  /ColR ColR Fillden mul Fillden sub 1 add def
+  /ColG ColG Fillden mul Fillden sub 1 add def
+  /ColB ColB Fillden mul Fillden sub 1 add def
+  ColR ColG ColB setrgbcolor} def
+/BoxColFill {gsave Rec PolyFill} def
+/PolyFill {gsave Density fill grestore grestore} def
+/h {rlineto rlineto rlineto gsave closepath fill grestore} bind def
+%
+% PostScript Level 1 Pattern Fill routine for rectangles
+% Usage: x y w h s a XX PatternFill
+%	x,y = lower left corner of box to be filled
+%	w,h = width and height of box
+%	  a = angle in degrees between lines and x-axis
+%	 XX = 0/1 for no/yes cross-hatch
+%
+/PatternFill {gsave /PFa [ 9 2 roll ] def
+  PFa 0 get PFa 2 get 2 div add PFa 1 get PFa 3 get 2 div add translate
+  PFa 2 get -2 div PFa 3 get -2 div PFa 2 get PFa 3 get Rec
+  gsave 1 setgray fill grestore clip
+  currentlinewidth 0.5 mul setlinewidth
+  /PFs PFa 2 get dup mul PFa 3 get dup mul add sqrt def
+  0 0 M PFa 5 get rotate PFs -2 div dup translate
+  0 1 PFs PFa 4 get div 1 add floor cvi
+	{PFa 4 get mul 0 M 0 PFs V} for
+  0 PFa 6 get ne {
+	0 1 PFs PFa 4 get div 1 add floor cvi
+	{PFa 4 get mul 0 2 1 roll M PFs 0 V} for
+ } if
+  stroke grestore} def
+%
+/languagelevel where
+ {pop languagelevel} {1} ifelse
+ 2 lt
+	{/InterpretLevel1 true def}
+	{/InterpretLevel1 Level1 def}
+ ifelse
+%
+% PostScript level 2 pattern fill definitions
+%
+/Level2PatternFill {
+/Tile8x8 {/PaintType 2 /PatternType 1 /TilingType 1 /BBox [0 0 8 8] /XStep 8 /YStep 8}
+	bind def
+/KeepColor {currentrgbcolor [/Pattern /DeviceRGB] setcolorspace} bind def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop 0 0 M 8 8 L 0 8 M 8 0 L stroke} 
+>> matrix makepattern
+/Pat1 exch def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop 0 0 M 8 8 L 0 8 M 8 0 L stroke
+	0 4 M 4 8 L 8 4 L 4 0 L 0 4 L stroke}
+>> matrix makepattern
+/Pat2 exch def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop 0 0 M 0 8 L
+	8 8 L 8 0 L 0 0 L fill}
+>> matrix makepattern
+/Pat3 exch def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop -4 8 M 8 -4 L
+	0 12 M 12 0 L stroke}
+>> matrix makepattern
+/Pat4 exch def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop -4 0 M 8 12 L
+	0 -4 M 12 8 L stroke}
+>> matrix makepattern
+/Pat5 exch def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop -2 8 M 4 -4 L
+	0 12 M 8 -4 L 4 12 M 10 0 L stroke}
+>> matrix makepattern
+/Pat6 exch def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop -2 0 M 4 12 L
+	0 -4 M 8 12 L 4 -4 M 10 8 L stroke}
+>> matrix makepattern
+/Pat7 exch def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop 8 -2 M -4 4 L
+	12 0 M -4 8 L 12 4 M 0 10 L stroke}
+>> matrix makepattern
+/Pat8 exch def
+<< Tile8x8
+ /PaintProc {0.5 setlinewidth pop 0 -2 M 12 4 L
+	-4 0 M 12 8 L -4 4 M 8 10 L stroke}
+>> matrix makepattern
+/Pat9 exch def
+/Pattern1 {PatternBgnd KeepColor Pat1 setpattern} bind def
+/Pattern2 {PatternBgnd KeepColor Pat2 setpattern} bind def
+/Pattern3 {PatternBgnd KeepColor Pat3 setpattern} bind def
+/Pattern4 {PatternBgnd KeepColor Landscape {Pat5} {Pat4} ifelse setpattern} bind def
+/Pattern5 {PatternBgnd KeepColor Landscape {Pat4} {Pat5} ifelse setpattern} bind def
+/Pattern6 {PatternBgnd KeepColor Landscape {Pat9} {Pat6} ifelse setpattern} bind def
+/Pattern7 {PatternBgnd KeepColor Landscape {Pat8} {Pat7} ifelse setpattern} bind def
+} def
+%
+%
+%End of PostScript Level 2 code
+%
+/PatternBgnd {
+  TransparentPatterns {} {gsave 1 setgray fill grestore} ifelse
+} def
+%
+% Substitute for Level 2 pattern fill codes with
+% grayscale if Level 2 support is not selected.
+%
+/Level1PatternFill {
+/Pattern1 {0.250 Density} bind def
+/Pattern2 {0.500 Density} bind def
+/Pattern3 {0.750 Density} bind def
+/Pattern4 {0.125 Density} bind def
+/Pattern5 {0.375 Density} bind def
+/Pattern6 {0.625 Density} bind def
+/Pattern7 {0.875 Density} bind def
+} def
+%
+% Now test for support of Level 2 code
+%
+Level1 {Level1PatternFill} {Level2PatternFill} ifelse
+%
+/Symbol-Oblique /Symbol findfont [1 0 .167 1 0 0] makefont
+dup length dict begin {1 index /FID eq {pop pop} {def} ifelse} forall
+currentdict end definefont pop
+/MFshow {
+   { dup 5 get 3 ge
+     { 5 get 3 eq {gsave} {grestore} ifelse }
+     {dup dup 0 get findfont exch 1 get scalefont setfont
+     [ currentpoint ] exch dup 2 get 0 exch R dup 5 get 2 ne {dup dup 6
+     get exch 4 get {Gshow} {stringwidth pop 0 R} ifelse }if dup 5 get 0 eq
+     {dup 3 get {2 get neg 0 exch R pop} {pop aload pop M} ifelse} {dup 5
+     get 1 eq {dup 2 get exch dup 3 get exch 6 get stringwidth pop -2 div
+     dup 0 R} {dup 6 get stringwidth pop -2 div 0 R 6 get
+     show 2 index {aload pop M neg 3 -1 roll neg R pop pop} {pop pop pop
+     pop aload pop M} ifelse }ifelse }ifelse }
+     ifelse }
+   forall} def
+/Gswidth {dup type /stringtype eq {stringwidth} {pop (n) stringwidth} ifelse} def
+/MFwidth {0 exch { dup 5 get 3 ge { 5 get 3 eq { 0 } { pop } ifelse }
+ {dup 3 get{dup dup 0 get findfont exch 1 get scalefont setfont
+     6 get Gswidth pop add} {pop} ifelse} ifelse} forall} def
+/MLshow { currentpoint stroke M
+  0 exch R
+  Blacktext {gsave 0 setgray MFshow grestore} {MFshow} ifelse } bind def
+/MRshow { currentpoint stroke M
+  exch dup MFwidth neg 3 -1 roll R
+  Blacktext {gsave 0 setgray MFshow grestore} {MFshow} ifelse } bind def
+/MCshow { currentpoint stroke M
+  exch dup MFwidth -2 div 3 -1 roll R
+  Blacktext {gsave 0 setgray MFshow grestore} {MFshow} ifelse } bind def
+/XYsave    { [( ) 1 2 true false 3 ()] } bind def
+/XYrestore { [( ) 1 2 true false 4 ()] } bind def
+end
+%%EndProlog
+%%Page: 1 1
+gnudict begin
+gsave
+doclip
+50 50 translate
+0.100 0.100 scale
+90 rotate
+0 -5040 translate
+0 setgray
+newpath
+(Helvetica) findfont 140 scalefont setfont
+1.000 UL
+LTb
+602 448 M
+63 0 V
+6282 0 R
+-63 0 V
+stroke
+518 448 M
+[ [(Helvetica) 140.0 0.0 true true 0 (50)]
+] -46.7 MRshow
+1.000 UL
+LTb
+602 1277 M
+63 0 V
+6282 0 R
+-63 0 V
+stroke
+518 1277 M
+[ [(Helvetica) 140.0 0.0 true true 0 (60)]
+] -46.7 MRshow
+1.000 UL
+LTb
+602 2105 M
+63 0 V
+6282 0 R
+-63 0 V
+stroke
+518 2105 M
+[ [(Helvetica) 140.0 0.0 true true 0 (70)]
+] -46.7 MRshow
+1.000 UL
+LTb
+602 2934 M
+63 0 V
+6282 0 R
+-63 0 V
+stroke
+518 2934 M
+[ [(Helvetica) 140.0 0.0 true true 0 (80)]
+] -46.7 MRshow
+1.000 UL
+LTb
+602 3762 M
+63 0 V
+6282 0 R
+-63 0 V
+stroke
+518 3762 M
+[ [(Helvetica) 140.0 0.0 true true 0 (90)]
+] -46.7 MRshow
+1.000 UL
+LTb
+602 4591 M
+63 0 V
+6282 0 R
+-63 0 V
+stroke
+518 4591 M
+[ [(Helvetica) 140.0 0.0 true true 0 (100)]
+] -46.7 MRshow
+1.000 UL
+LTb
+602 448 M
+0 63 V
+0 4080 R
+0 -63 V
+stroke
+602 308 M
+[ [(Helvetica) 140.0 0.0 true true 0 ( 0)]
+] -46.7 MCshow
+1.000 UL
+LTb
+1508 448 M
+0 63 V
+0 4080 R
+0 -63 V
+stroke
+1508 308 M
+[ [(Helvetica) 140.0 0.0 true true 0 ( 20000)]
+] -46.7 MCshow
+1.000 UL
+LTb
+2415 448 M
+0 63 V
+0 4080 R
+0 -63 V
+stroke
+2415 308 M
+[ [(Helvetica) 140.0 0.0 true true 0 ( 40000)]
+] -46.7 MCshow
+1.000 UL
+LTb
+3321 448 M
+0 63 V
+0 4080 R
+0 -63 V
+stroke
+3321 308 M
+[ [(Helvetica) 140.0 0.0 true true 0 ( 60000)]
+] -46.7 MCshow
+1.000 UL
+LTb
+4228 448 M
+0 63 V
+0 4080 R
+0 -63 V
+stroke
+4228 308 M
+[ [(Helvetica) 140.0 0.0 true true 0 ( 80000)]
+] -46.7 MCshow
+1.000 UL
+LTb
+5134 448 M
+0 63 V
+0 4080 R
+0 -63 V
+stroke
+5134 308 M
+[ [(Helvetica) 140.0 0.0 true true 0 ( 100000)]
+] -46.7 MCshow
+1.000 UL
+LTb
+6041 448 M
+0 63 V
+0 4080 R
+0 -63 V
+stroke
+6041 308 M
+[ [(Helvetica) 140.0 0.0 true true 0 ( 120000)]
+] -46.7 MCshow
+1.000 UL
+LTb
+6947 448 M
+0 63 V
+0 4080 R
+0 -63 V
+stroke
+6947 308 M
+[ [(Helvetica) 140.0 0.0 true true 0 ( 140000)]
+] -46.7 MCshow
+1.000 UL
+LTb
+1.000 UL
+LTb
+602 4591 N
+602 448 L
+6345 0 V
+0 4143 V
+-6345 0 V
+Z stroke
+LCb setrgbcolor
+112 2519 M
+currentpoint gsave translate -270 rotate 0 0 moveto
+[ [(Helvetica) 140.0 0.0 true true 0 (Efficiency)]
+] -46.7 MCshow
+grestore
+LTb
+LCb setrgbcolor
+3774 98 M
+[ [(Helvetica) 140.0 0.0 true true 0 (Number of Processors)]
+] -46.7 MCshow
+LTb
+1.000 UP
+1.000 UL
+LTb
+% Begin plot #1
+2.000 UP
+5.000 UL
+LT0
+1.00 0.00 0.00 C LCb setrgbcolor
+4163 4907 M
+[ [(Helvetica) 140.0 0.0 true true 0 (100 second tasks)]
+] -46.7 MRshow
+LT0
+1.00 0.00 0.00 C 4247 4907 M
+399 0 V
+608 4409 M
+6 -5 V
+11 -16 V
+23 -2 V
+47 -9 V
+93 -55 V
+185 5 V
+372 -75 V
+742 -193 V
+1485 82 V
+6542 3102 L
+608 4409 BoxF
+614 4404 BoxF
+625 4388 BoxF
+648 4386 BoxF
+695 4377 BoxF
+788 4322 BoxF
+973 4327 BoxF
+1345 4252 BoxF
+2087 4059 BoxF
+3572 4141 BoxF
+6542 3102 BoxF
+4446 4907 BoxF
+% End plot #1
+1.000 UL
+LTb
+602 4591 N
+602 448 L
+6345 0 V
+0 4143 V
+-6345 0 V
+Z stroke
+1.000 UP
+1.000 UL
+LTb
+stroke
+grestore
+end
+showpage
+%%Trailer
+%%DocumentFonts: Helvetica
+%%Pages: 1

Added: SwiftApps/cray-benchmarks/eff.hera/eff90.svg
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/eff90.svg	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/eff90.svg	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,182 @@
+<?xml version="1.0" encoding="utf-8"  standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" 
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg width="2000" height="800" viewBox="0 0 2000 800"
+ xmlns="http://www.w3.org/2000/svg"
+ xmlns:xlink="http://www.w3.org/1999/xlink"
+>
+
+<desc>Produced by GNUPLOT 4.4 patchlevel 3 </desc>
+
+<defs>
+
+	<circle id='gpDot' r='0.5' stroke-width='0.5'/>
+	<path id='gpPt0' stroke-width='0.222' stroke='currentColor' d='M-1,0 h2 M0,-1 v2'/>
+	<path id='gpPt1' stroke-width='0.222' stroke='currentColor' d='M-1,-1 L1,1 M1,-1 L-1,1'/>
+	<path id='gpPt2' stroke-width='0.222' stroke='currentColor' d='M-1,0 L1,0 M0,-1 L0,1 M-1,-1 L1,1 M-1,1 L1,-1'/>
+	<rect id='gpPt3' stroke-width='0.222' stroke='currentColor' x='-1' y='-1' width='2' height='2'/>
+	<rect id='gpPt4' stroke-width='0.222' stroke='currentColor' fill='currentColor' x='-1' y='-1' width='2' height='2'/>
+	<circle id='gpPt5' stroke-width='0.222' stroke='currentColor' cx='0' cy='0' r='1'/>
+	<use xlink:href='#gpPt5' id='gpPt6' fill='currentColor' stroke='none'/>
+	<path id='gpPt7' stroke-width='0.222' stroke='currentColor' d='M0,-1.33 L-1.33,0.67 L1.33,0.67 z'/>
+	<use xlink:href='#gpPt7' id='gpPt8' fill='currentColor' stroke='none'/>
+	<use xlink:href='#gpPt7' id='gpPt9' stroke='currentColor' transform='rotate(180)'/>
+	<use xlink:href='#gpPt9' id='gpPt10' fill='currentColor' stroke='none'/>
+	<use xlink:href='#gpPt3' id='gpPt11' stroke='currentColor' transform='rotate(45)'/>
+	<use xlink:href='#gpPt11' id='gpPt12' fill='currentColor' stroke='none'/>
+</defs>
+<g style="fill:none; color:white; stroke:currentColor; stroke-width:1.00; stroke-linecap:butt; stroke-linejoin:miter">
+</g>
+<g style="fill:none; color:black; stroke:currentColor; stroke-width:1.00; stroke-linecap:butt; stroke-linejoin:miter">
+	<path  d='M63.6,742.4 L72.6,742.4 M1975.0,742.4 L1966.0,742.4 '></path>
+	<g transform="translate(55.3,746.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>50</tspan>
+		</text>
+	</g>
+	<path  d='M63.6,604.5 L72.6,604.5 M1975.0,604.5 L1966.0,604.5 '></path>
+	<g transform="translate(55.3,609.0)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>60</tspan>
+		</text>
+	</g>
+	<path  d='M63.6,466.5 L72.6,466.5 M1975.0,466.5 L1966.0,466.5 '></path>
+	<g transform="translate(55.3,471.0)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>70</tspan>
+		</text>
+	</g>
+	<path  d='M63.6,328.6 L72.6,328.6 M1975.0,328.6 L1966.0,328.6 '></path>
+	<g transform="translate(55.3,333.1)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>80</tspan>
+		</text>
+	</g>
+	<path  d='M63.6,190.6 L72.6,190.6 M1975.0,190.6 L1966.0,190.6 '></path>
+	<g transform="translate(55.3,195.1)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>90</tspan>
+		</text>
+	</g>
+	<path  d='M63.6,52.7 L72.6,52.7 M1975.0,52.7 L1966.0,52.7 '></path>
+	<g transform="translate(55.3,57.2)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>100</tspan>
+		</text>
+	</g>
+	<path  d='M63.6,742.4 L63.6,733.4 M63.6,52.7 L63.6,61.7 '></path>
+	<g transform="translate(63.6,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 2000</tspan>
+		</text>
+	</g>
+	<path  d='M276.0,742.4 L276.0,733.4 M276.0,52.7 L276.0,61.7 '></path>
+	<g transform="translate(276.0,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 4000</tspan>
+		</text>
+	</g>
+	<path  d='M488.4,742.4 L488.4,733.4 M488.4,52.7 L488.4,61.7 '></path>
+	<g transform="translate(488.4,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 6000</tspan>
+		</text>
+	</g>
+	<path  d='M700.7,742.4 L700.7,733.4 M700.7,52.7 L700.7,61.7 '></path>
+	<g transform="translate(700.7,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 8000</tspan>
+		</text>
+	</g>
+	<path  d='M913.1,742.4 L913.1,733.4 M913.1,52.7 L913.1,61.7 '></path>
+	<g transform="translate(913.1,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 10000</tspan>
+		</text>
+	</g>
+	<path  d='M1125.5,742.4 L1125.5,733.4 M1125.5,52.7 L1125.5,61.7 '></path>
+	<g transform="translate(1125.5,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 12000</tspan>
+		</text>
+	</g>
+	<path  d='M1337.9,742.4 L1337.9,733.4 M1337.9,52.7 L1337.9,61.7 '></path>
+	<g transform="translate(1337.9,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 14000</tspan>
+		</text>
+	</g>
+	<path  d='M1550.2,742.4 L1550.2,733.4 M1550.2,52.7 L1550.2,61.7 '></path>
+	<g transform="translate(1550.2,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 16000</tspan>
+		</text>
+	</g>
+	<path  d='M1762.6,742.4 L1762.6,733.4 M1762.6,52.7 L1762.6,61.7 '></path>
+	<g transform="translate(1762.6,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 18000</tspan>
+		</text>
+	</g>
+	<path  d='M1975.0,742.4 L1975.0,733.4 M1975.0,52.7 L1975.0,61.7 '></path>
+	<g transform="translate(1975.0,764.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan> 20000</tspan>
+		</text>
+	</g>
+	<path  d='M63.6,52.7 L63.6,742.4 L1975.0,742.4 L1975.0,52.7 L63.6,52.7 Z '></path>
+	<g transform="translate(17.6,397.6) rotate(270)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan>Efficiency</tspan>
+		</text>
+	</g>
+	<g transform="translate(1019.3,791.9)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:middle">
+		<text><tspan>Number of Processors</tspan>
+		</text>
+	</g>
+</g>
+	<a xlink:title="Plot #1">
+<g style="fill:none; color:white; stroke:currentColor; stroke-width:5.00; stroke-linecap:butt; stroke-linejoin:miter">
+</g>
+<g style="fill:none; color:red; stroke:currentColor; stroke-width:5.00; stroke-linecap:butt; stroke-linejoin:miter">
+	<g transform="translate(768.9,22.5)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>100 second tasks</tspan>
+		</text>
+	</g>
+	<path stroke='rgb(255,   0,   0)'  d='M777.2,18.0 L819.4,18.0 M1849.3,599.5 L1550.2,508.5 L870.6,365.5 L191.0,207.7 '></path>
+	<use xlink:href='#gpPt4' transform='translate(1849.3,599.5) scale(9.00)' color='rgb(255,   0,   0)'/>
+	<use xlink:href='#gpPt4' transform='translate(1550.2,508.5) scale(9.00)' color='rgb(255,   0,   0)'/>
+	<use xlink:href='#gpPt4' transform='translate(870.6,365.5) scale(9.00)' color='rgb(255,   0,   0)'/>
+	<use xlink:href='#gpPt4' transform='translate(191.0,207.7) scale(9.00)' color='rgb(255,   0,   0)'/>
+	<use xlink:href='#gpPt4' transform='translate(798.3,18.0) scale(9.00)' color='rgb(255,   0,   0)'/>
+</g>
+	</a>
+	<a xlink:title="Plot #2">
+<g style="fill:none; color:green; stroke:currentColor; stroke-width:5.00; stroke-linecap:butt; stroke-linejoin:miter">
+	<g transform="translate(960.5,22.5)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>200 second tasks</tspan>
+		</text>
+	</g>
+	<path stroke='rgb(255, 165,   0)'  d='M968.8,18.0 L1011.0,18.0 M1550.2,315.2 L870.6,237.8 L191.0,136.8 '></path>
+	<use xlink:href='#gpPt6' transform='translate(1550.2,315.2) scale(9.00)' color='rgb(255, 165,   0)'/>
+	<use xlink:href='#gpPt6' transform='translate(870.6,237.8) scale(9.00)' color='rgb(255, 165,   0)'/>
+	<use xlink:href='#gpPt6' transform='translate(191.0,136.8) scale(9.00)' color='rgb(255, 165,   0)'/>
+	<use xlink:href='#gpPt6' transform='translate(989.9,18.0) scale(9.00)' color='rgb(255, 165,   0)'/>
+</g>
+	</a>
+	<a xlink:title="Plot #3">
+<g style="fill:none; color:blue; stroke:currentColor; stroke-width:5.00; stroke-linecap:butt; stroke-linejoin:miter">
+	<g transform="translate(1152.1,22.5)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>150 seconds</tspan>
+		</text>
+	</g>
+	<path stroke='rgb(135, 206, 235)'  d='M1160.4,18.0 L1202.6,18.0 M191.0,188.2 L870.6,343.1 L1550.2,440.6 '></path>
+	<use xlink:href='#gpPt6' transform='translate(191.0,188.2) scale(9.00)' color='rgb(135, 206, 235)'/>
+	<use xlink:href='#gpPt6' transform='translate(870.6,343.1) scale(9.00)' color='rgb(135, 206, 235)'/>
+	<use xlink:href='#gpPt6' transform='translate(1550.2,440.6) scale(9.00)' color='rgb(135, 206, 235)'/>
+	<use xlink:href='#gpPt6' transform='translate(1181.5,18.0) scale(9.00)' color='rgb(135, 206, 235)'/>
+</g>
+	</a>
+	<a xlink:title="Plot #4">
+<g style="fill:none; color:cyan; stroke:currentColor; stroke-width:5.00; stroke-linecap:butt; stroke-linejoin:miter">
+	<g transform="translate(1343.7,22.5)" style="stroke:none; fill:black; font-family:Arial; font-size:12.00pt; text-anchor:end">
+		<text><tspan>400 seconds</tspan>
+		</text>
+	</g>
+	<path stroke='rgb(165,  42,  42)'  d='M1352.0,18.0 L1394.2,18.0 M191.0,102.5 L870.6,152.8 L1550.2,201.4 '></path>
+	<use xlink:href='#gpPt6' transform='translate(191.0,102.5) scale(9.00)' color='rgb(165,  42,  42)'/>
+	<use xlink:href='#gpPt6' transform='translate(870.6,152.8) scale(9.00)' color='rgb(165,  42,  42)'/>
+	<use xlink:href='#gpPt6' transform='translate(1550.2,201.4) scale(9.00)' color='rgb(165,  42,  42)'/>
+	<use xlink:href='#gpPt6' transform='translate(1373.1,18.0) scale(9.00)' color='rgb(165,  42,  42)'/>
+</g>
+	</a>
+<g style="fill:none; color:white; stroke:rgb(165,  42,  42); stroke-width:1.00; stroke-linecap:butt; stroke-linejoin:miter">
+</g>
+<g style="fill:none; color:black; stroke:currentColor; stroke-width:1.00; stroke-linecap:butt; stroke-linejoin:miter">
+	<path  d='M63.6,52.7 L63.6,742.4 L1975.0,742.4 L1975.0,52.7 L63.6,52.7 Z '></path>
+</g>
+</svg>
+

Added: SwiftApps/cray-benchmarks/eff.hera/exm.pdf
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/eff.hera/exm.pdf
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/eff.hera/expectedtime.gp
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/expectedtime.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/expectedtime.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,8 @@
+set terminal png enhanced font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf" 16 size 1000,1000
+
+set output "expectedtime.png"
+set multiplot
+set xlabel "Number of Processors" 
+
+set ylabel "Time in Seconds" 
+plot "eff.txt" using 1:2 with lines lc rgb "green" lw 2 title "Optimal Time", "eff.txt" using 1:3 with lines lc rgb "red" lw 2 title "Real Time"

Added: SwiftApps/cray-benchmarks/eff.hera/mike.gp
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/mike.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/mike.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,22 @@
+#set terminal png size 2000,800
+#set term postscript eps enhanced 
+#set terminal svg enhanced size 2000 800
+#set terminal postscript enhanced
+set nokey
+set terminal pdf
+set output "mike.pdf"
+set decimal locale
+set format y "%'g"
+
+# enhanced font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf" 16 size 1000,400
+set style line 1 lt 1 lw 8 pt 3 lc rgb "red"
+set style line 2 lt 1 lw 8 pt 3 lc rgb "red"
+set style line 3 lt 1 lw 8 pt 3 lc rgb "blue"
+set style line 4 lt 1 lw 8 pt 3 lc rgb "blue"
+#set output "eff90.png"
+
+set xlabel "Number of Processors" 
+set ylabel "Efficiency" 
+set yrange [50:100]
+plot "mike.txt" using 1:2 with linespoints pt 5 linewidth 5 ps 0.5 lc rgb "red" 
+

Added: SwiftApps/cray-benchmarks/eff.hera/mike.txt
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/mike.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/mike.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,7 @@
+128        97.80
+8192        96.81 # 488481
+16384        95.91 # 488491
+32768        93.58 # 488717
+65536        94.57 # 488881
+131072  82.03 # 489239
+

Added: SwiftApps/cray-benchmarks/eff.hera/mnotes
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/mnotes	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/mnotes	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,11 @@
+solid lines
+triangles
+dodger blue
+commas in processors if possible
+slightly larger font size
+
+later:
+tasks per second with same conventions
+500, 100 seconds
+
+

Added: SwiftApps/cray-benchmarks/eff.hera/plotit.gp
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/plotit.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/plotit.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,22 @@
+#To be run as a gnuplot script as follows:
+# $ gnuplot plotit.gp
+
+set terminal png enhanced font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf" 18 size 1000,1000
+#set term postscript eps enhanced 
+#set terminal svg enhanced size 1000 1000
+
+#set linestyle 3 lt 3 lw 1 
+set output "activeplot.png"
+set nokey
+set xlabel "Time in Seconds" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,20"
+set ylabel "Active Jobs" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,20"
+set title "Active jobs" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,22"
+plot "plot_active.txt" using 1:2 with lines lc rgb "orange" lw 2 title "Active Tasks"
+# "plot_available.txt" using 1:2 with filledcurves lc rgb "skyblue" lw 2 title "Available Cores"
+#plot "plot_available.txt" using 1:2 with filledcurves lc rgb "skyblue" lw 2 title "Available Cores", "plot_active.txt" using 1:2 with filledcurves lc rgb "orange" lw 2 title "Active Tasks"
+set output "cumulativeplot.png"
+set xlabel "Time in seconds"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+

Added: SwiftApps/cray-benchmarks/eff.hera/timetocomplete.sh
===================================================================
--- SwiftApps/cray-benchmarks/eff.hera/timetocomplete.sh	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.hera/timetocomplete.sh	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+for log in $@
+do
+   stime=`head -1 $log`
+   etime=`tail -1 $log`
+   expr $etime - $stime
+done


Property changes on: SwiftApps/cray-benchmarks/eff.hera/timetocomplete.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/eff.raven/eff.gp
===================================================================
--- SwiftApps/cray-benchmarks/eff.raven/eff.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.raven/eff.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,23 @@
+#set terminal png font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf" 12 size 4000,2800
+#set term postscript eps enhanced 
+#set terminal svg enhanced size 2000 800
+#set terminal postscript enhanced
+set terminal pdf
+
+#set output "eff90.ps"
+#set output "eff90.png"
+set output "eff90.pdf"
+
+set decimal locale
+#set format x "%'g"
+
+set style line 1 lt 1 lw 8 pt 3 lc rgb "red"
+set style line 2 lt 1 lw 8 pt 3 lc rgb "red"
+set style line 3 lt 1 lw 8 pt 3 lc rgb "blue"
+set style line 4 lt 1 lw 8 pt 3 lc rgb "blue"
+set yrange [90:100]
+
+set key above
+set xlabel "Number of Processors" 
+set ylabel "Efficiency" 
+plot "eff100.txt" using 1:5 with linespoints pt 5 lw 5 ps 0.5 lc rgb "red" title "100 second tasks", "eff200.txt" using 1:5 with linespoints pt 7 ps 0.5 lw 5 lc rgb "orange" title "200 second tasks", "eff400.txt" using 1:5 with linespoint pt 9 ps 0.5 lw 5 lc rgb "#1E90FF" title "400 seconds"

Added: SwiftApps/cray-benchmarks/eff.raven/eff100.txt
===================================================================
--- SwiftApps/cray-benchmarks/eff.raven/eff100.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.raven/eff100.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,5 @@
+24	300	305	0.24	98.36
+48	300	305	0.47	98.36
+96	300	308	0.94	97.40
+192	300	311	1.85	96.46
+384	300	314	3.67	95.54

Added: SwiftApps/cray-benchmarks/eff.raven/eff200.txt
===================================================================
--- SwiftApps/cray-benchmarks/eff.raven/eff200.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.raven/eff200.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,5 @@
+24	600	605	0.12	99.17
+48	600	606	0.24	99.01
+96	600	607	0.47	98.85
+192	600	610	0.94	98.36
+384	600	616	1.87	97.40

Added: SwiftApps/cray-benchmarks/eff.raven/eff400.txt
===================================================================
--- SwiftApps/cray-benchmarks/eff.raven/eff400.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.raven/eff400.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,5 @@
+24	1200	1205	0.06	99.59
+48	1200	1206	0.12	99.50
+96	1200	1207	0.24	99.42
+192	1200	1210	0.48	99.17
+384	1200	1215	0.95	98.77

Added: SwiftApps/cray-benchmarks/eff.raven/eff90-raven.pdf
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/eff.raven/eff90-raven.pdf
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/eff.raven/eff90.pdf
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/eff.raven/eff90.pdf
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/eff.raven/expectedtime.gp
===================================================================
--- SwiftApps/cray-benchmarks/eff.raven/expectedtime.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.raven/expectedtime.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,8 @@
+set terminal png enhanced font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf" 16 size 1000,1000
+
+set output "expectedtime.png"
+set multiplot
+set xlabel "Number of Processors" 
+
+set ylabel "Time in Seconds" 
+plot "eff.txt" using 1:2 with lines lc rgb "green" lw 2 title "Optimal Time", "eff.txt" using 1:3 with lines lc rgb "red" lw 2 title "Real Time"

Added: SwiftApps/cray-benchmarks/eff.raven/plotit.gp
===================================================================
--- SwiftApps/cray-benchmarks/eff.raven/plotit.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/eff.raven/plotit.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,22 @@
+#To be run as a gnuplot script as follows:
+# $ gnuplot plotit.gp
+
+set terminal png enhanced font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf" 18 size 1000,1000
+#set term postscript eps enhanced 
+#set terminal svg enhanced size 1000 1000
+
+#set linestyle 3 lt 3 lw 1 
+set output "activeplot.png"
+set nokey
+set xlabel "Time in Seconds" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,20"
+set ylabel "Active Jobs" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,20"
+set title "Active jobs" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,22"
+plot "plot_active.txt" using 1:2 with lines lc rgb "orange" lw 2 title "Active Tasks"
+# "plot_available.txt" using 1:2 with filledcurves lc rgb "skyblue" lw 2 title "Available Cores"
+#plot "plot_available.txt" using 1:2 with filledcurves lc rgb "skyblue" lw 2 title "Available Cores", "plot_active.txt" using 1:2 with filledcurves lc rgb "orange" lw 2 title "Active Tasks"
+set output "cumulativeplot.png"
+set xlabel "Time in seconds"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+

Added: SwiftApps/cray-benchmarks/plots/T.data
===================================================================
Added: SwiftApps/cray-benchmarks/plots/active.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots/active.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots/active.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,326 @@
+16
+16
+16
+16
+16
+61
+61
+61
+62
+88
+88
+88
+88
+136
+136
+136
+136
+154
+154
+154
+154
+154
+154
+154
+154
+154
+154
+154
+154
+154
+154
+154
+154
+175
+174
+171
+169
+168
+166
+165
+165
+164
+164
+164
+164
+164
+279
+278
+278
+277
+276
+274
+272
+269
+265
+261
+257
+255
+255
+252
+249
+246
+242
+240
+237
+236
+234
+232
+231
+229
+228
+226
+225
+223
+218
+215
+211
+207
+205
+205
+204
+204
+203
+203
+200
+200
+200
+527
+525
+524
+522
+520
+519
+519
+518
+517
+515
+513
+513
+511
+508
+506
+503
+500
+499
+496
+491
+488
+486
+483
+481
+481
+480
+479
+476
+472
+470
+466
+463
+461
+460
+460
+458
+456
+455
+454
+454
+453
+452
+451
+450
+450
+449
+448
+445
+444
+444
+443
+442
+440
+437
+436
+435
+433
+432
+432
+430
+429
+426
+424
+423
+421
+419
+417
+414
+412
+412
+411
+408
+407
+403
+400
+396
+392
+392
+389
+384
+378
+375
+370
+365
+364
+358
+352
+345
+342
+336
+334
+333
+331
+331
+432
+432
+430
+427
+424
+422
+421
+419
+417
+412
+410
+406
+402
+400
+398
+397
+393
+389
+386
+383
+380
+376
+375
+374
+372
+370
+366
+363
+359
+356
+352
+353
+351
+351
+347
+344
+340
+339
+335
+335
+333
+330
+328
+323
+322
+317
+313
+309
+310
+307
+304
+300
+297
+296
+291
+289
+287
+285
+281
+278
+277
+272
+270
+265
+265
+265
+260
+257
+255
+254
+250
+249
+246
+246
+242
+237
+235
+231
+229
+224
+220
+221
+219
+218
+212
+210
+207
+203
+200
+199
+197
+192
+189
+189
+184
+181
+176
+177
+175
+172
+170
+166
+162
+160
+156
+157
+155
+150
+146
+143
+141
+138
+135
+132
+133
+129
+128
+125
+121
+117
+116
+114
+110
+107
+104
+101
+100
+95
+89
+83
+77
+71
+66
+61
+55
+49
+43
+37
+32
+26
+25
+20
+14
+8
+2

Added: SwiftApps/cray-benchmarks/plots/activeplot.png
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/plots/activeplot.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/plots/activetics.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots/activetics.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots/activetics.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,327 @@
+0
+1.48159
+2.96318
+4.44477
+5.92636
+7.40795
+8.88954
+10.3711
+11.8527
+13.3343
+14.8159
+16.2975
+17.7791
+19.2607
+20.7423
+22.2238
+23.7054
+25.187
+26.6686
+28.1502
+29.6318
+31.1134
+32.595
+34.0766
+35.5582
+37.0397
+38.5213
+40.0029
+41.4845
+42.9661
+44.4477
+45.9293
+47.4109
+48.8925
+50.3741
+51.8556
+53.3372
+54.8188
+56.3004
+57.782
+59.2636
+60.7452
+62.2268
+63.7084
+65.19
+66.6715
+68.1531
+69.6347
+71.1163
+72.5979
+74.0795
+75.5611
+77.0427
+78.5243
+80.0059
+81.4874
+82.969
+84.4506
+85.9322
+87.4138
+88.8954
+90.377
+91.8586
+93.3402
+94.8218
+96.3033
+97.7849
+99.2665
+100.748
+102.23
+103.711
+105.193
+106.674
+108.156
+109.638
+111.119
+112.601
+114.082
+115.564
+117.046
+118.527
+120.009
+121.49
+122.972
+124.454
+125.935
+127.417
+128.898
+130.38
+131.862
+133.343
+134.825
+136.306
+137.788
+139.269
+140.751
+142.233
+143.714
+145.196
+146.677
+148.159
+149.641
+151.122
+152.604
+154.085
+155.567
+157.049
+158.53
+160.012
+161.493
+162.975
+164.456
+165.938
+167.42
+168.901
+170.383
+171.864
+173.346
+174.828
+176.309
+177.791
+179.272
+180.754
+182.236
+183.717
+185.199
+186.68
+188.162
+189.644
+191.125
+192.607
+194.088
+195.57
+197.051
+198.533
+200.015
+201.496
+202.978
+204.459
+205.941
+207.423
+208.904
+210.386
+211.867
+213.349
+214.831
+216.312
+217.794
+219.275
+220.757
+222.238
+223.72
+225.202
+226.683
+228.165
+229.646
+231.128
+232.61
+234.091
+235.573
+237.054
+238.536
+240.018
+241.499
+242.981
+244.462
+245.944
+247.426
+248.907
+250.389
+251.87
+253.352
+254.833
+256.315
+257.797
+259.278
+260.76
+262.241
+263.723
+265.205
+266.686
+268.168
+269.649
+271.131
+272.613
+274.094
+275.576
+277.057
+278.539
+280.021
+281.502
+282.984
+284.465
+285.947
+287.428
+288.91
+290.392
+291.873
+293.355
+294.836
+296.318
+297.8
+299.281
+300.763
+302.244
+303.726
+305.208
+306.689
+308.171
+309.652
+311.134
+312.615
+314.097
+315.579
+317.06
+318.542
+320.023
+321.505
+322.987
+324.468
+325.95
+327.431
+328.913
+330.395
+331.876
+333.358
+334.839
+336.321
+337.803
+339.284
+340.766
+342.247
+343.729
+345.21
+346.692
+348.174
+349.655
+351.137
+352.618
+354.1
+355.582
+357.063
+358.545
+360.026
+361.508
+362.99
+364.471
+365.953
+367.434
+368.916
+370.397
+371.879
+373.361
+374.842
+376.324
+377.805
+379.287
+380.769
+382.25
+383.732
+385.213
+386.695
+388.177
+389.658
+391.14
+392.621
+394.103
+395.585
+397.066
+398.548
+400.029
+401.511
+402.992
+404.474
+405.956
+407.437
+408.919
+410.4
+411.882
+413.364
+414.845
+416.327
+417.808
+419.29
+420.772
+422.253
+423.735
+425.216
+426.698
+428.18
+429.661
+431.143
+432.624
+434.106
+435.587
+437.069
+438.551
+440.032
+441.514
+442.995
+444.477
+445.959
+447.44
+448.922
+450.403
+451.885
+453.367
+454.848
+456.33
+457.811
+459.293
+460.774
+462.256
+463.738
+465.219
+466.701
+468.182
+469.664
+471.146
+472.627
+474.109
+475.59
+477.072
+478.554
+480.035
+481.517
+482.998

Added: SwiftApps/cray-benchmarks/plots/anneal.data
===================================================================
Added: SwiftApps/cray-benchmarks/plots/cumulative.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots/cumulative.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots/cumulative.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,182 @@
+1
+2
+3
+3
+4
+4
+4
+4
+4
+5
+6
+81
+112
+152
+209
+219
+222
+226
+229
+230
+233
+238
+243
+247
+252
+256
+257
+263
+270
+276
+280
+285
+287
+288
+290
+292
+294
+296
+300
+306
+311
+316
+317
+323
+329
+335
+341
+346
+352
+358
+359
+363
+368
+373
+379
+384
+390
+396
+397
+398
+403
+409
+415
+421
+427
+433
+438
+439
+440
+444
+450
+455
+461
+467
+473
+477
+478
+484
+490
+496
+502
+508
+514
+520
+521
+526
+531
+536
+542
+548
+554
+560
+563
+564
+570
+576
+582
+587
+592
+598
+603
+605
+611
+616
+620
+625
+631
+636
+641
+642
+648
+654
+659
+665
+670
+676
+682
+683
+684
+689
+695
+701
+707
+713
+719
+720
+726
+732
+738
+743
+749
+754
+760
+761
+767
+772
+778
+784
+790
+796
+801
+802
+806
+812
+818
+823
+828
+834
+839
+841
+842
+847
+852
+857
+863
+868
+874
+878
+884
+889
+895
+898
+899
+905
+910
+916
+922
+928
+933
+938
+944
+950
+956
+962
+967
+973
+974
+980
+985
+991
+997
+1000

Added: SwiftApps/cray-benchmarks/plots/cumulativeplot-openmp.png
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/plots/cumulativeplot-openmp.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/plots/cumulativeplot.png
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/plots/cumulativeplot.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/plots/cumultics.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots/cumultics.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots/cumultics.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,183 @@
+0
+2.65384
+5.30768
+7.96152
+10.6154
+13.2692
+15.923
+18.5769
+21.2307
+23.8846
+26.5384
+29.1922
+31.8461
+34.4999
+37.1538
+39.8076
+42.4614
+45.1153
+47.7691
+50.423
+53.0768
+55.7306
+58.3845
+61.0383
+63.6922
+66.346
+68.9998
+71.6537
+74.3075
+76.9614
+79.6152
+82.269
+84.9229
+87.5767
+90.2306
+92.8844
+95.5382
+98.1921
+100.846
+103.5
+106.154
+108.807
+111.461
+114.115
+116.769
+119.423
+122.077
+124.73
+127.384
+130.038
+132.692
+135.346
+138
+140.654
+143.307
+145.961
+148.615
+151.269
+153.923
+156.577
+159.23
+161.884
+164.538
+167.192
+169.846
+172.5
+175.153
+177.807
+180.461
+183.115
+185.769
+188.423
+191.076
+193.73
+196.384
+199.038
+201.692
+204.346
+207
+209.653
+212.307
+214.961
+217.615
+220.269
+222.923
+225.576
+228.23
+230.884
+233.538
+236.192
+238.846
+241.499
+244.153
+246.807
+249.461
+252.115
+254.769
+257.422
+260.076
+262.73
+265.384
+268.038
+270.692
+273.346
+275.999
+278.653
+281.307
+283.961
+286.615
+289.269
+291.922
+294.576
+297.23
+299.884
+302.538
+305.192
+307.845
+310.499
+313.153
+315.807
+318.461
+321.115
+323.768
+326.422
+329.076
+331.73
+334.384
+337.038
+339.692
+342.345
+344.999
+347.653
+350.307
+352.961
+355.615
+358.268
+360.922
+363.576
+366.23
+368.884
+371.538
+374.191
+376.845
+379.499
+382.153
+384.807
+387.461
+390.114
+392.768
+395.422
+398.076
+400.73
+403.384
+406.038
+408.691
+411.345
+413.999
+416.653
+419.307
+421.961
+424.614
+427.268
+429.922
+432.576
+435.23
+437.884
+440.537
+443.191
+445.845
+448.499
+451.153
+453.807
+456.46
+459.114
+461.768
+464.422
+467.076
+469.73
+472.384
+475.037
+477.691
+480.345
+482.999

Added: SwiftApps/cray-benchmarks/plots/extract4plots
===================================================================
--- SwiftApps/cray-benchmarks/plots/extract4plots	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots/extract4plots	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+#usage: ./extract4plots <swift.outfile>
+
+SWIFTOUTFILE=$1
+
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE | head -n 1 | cut -f4-9 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE | tail -n 1 | cut -f4-9 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+
+#extract active runs in a file
+grep -o -i "Active:[0-9]*" $SWIFTOUTFILE | awk -F: '{print $2}' > active.txt
+
+#extract successful completions in a file
+grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE | awk -F: '{print $2}' > cumulative.txt
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+grep  "T =" $SWIFTOUTFILE | awk '{print $6}' | cut -c8- | sed 's/....$//' > T.data
+
+grep multi_annealing $SWIFTOUTFILE | grep "1;30" | awk '{print $3}' | cut -c11- | sed 's/....$//' > anneal.data
+
+grep returning $SWIFTOUTFILE | awk '{print $3, $4, $5, $6}' | sed -e 's/'ci='//' -e 's/'cj='//' -e 's/'r.loss='//' -e 's/'r.sdev='//' | sort -t' ' -k 1,2n > multiloss.txt


Property changes on: SwiftApps/cray-benchmarks/plots/extract4plots
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/plots/multiloss.txt
===================================================================
Added: SwiftApps/cray-benchmarks/plots/plot_active.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots/plot_active.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots/plot_active.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,327 @@
+0	16
+1.48159	16
+2.96318	16
+4.44477	16
+5.92636	16
+7.40795	61
+8.88954	61
+10.3711	61
+11.8527	62
+13.3343	88
+14.8159	88
+16.2975	88
+17.7791	88
+19.2607	136
+20.7423	136
+22.2238	136
+23.7054	136
+25.187	154
+26.6686	154
+28.1502	154
+29.6318	154
+31.1134	154
+32.595	154
+34.0766	154
+35.5582	154
+37.0397	154
+38.5213	154
+40.0029	154
+41.4845	154
+42.9661	154
+44.4477	154
+45.9293	154
+47.4109	154
+48.8925	175
+50.3741	174
+51.8556	171
+53.3372	169
+54.8188	168
+56.3004	166
+57.782	165
+59.2636	165
+60.7452	164
+62.2268	164
+63.7084	164
+65.19	164
+66.6715	164
+68.1531	279
+69.6347	278
+71.1163	278
+72.5979	277
+74.0795	276
+75.5611	274
+77.0427	272
+78.5243	269
+80.0059	265
+81.4874	261
+82.969	257
+84.4506	255
+85.9322	255
+87.4138	252
+88.8954	249
+90.377	246
+91.8586	242
+93.3402	240
+94.8218	237
+96.3033	236
+97.7849	234
+99.2665	232
+100.748	231
+102.23	229
+103.711	228
+105.193	226
+106.674	225
+108.156	223
+109.638	218
+111.119	215
+112.601	211
+114.082	207
+115.564	205
+117.046	205
+118.527	204
+120.009	204
+121.49	203
+122.972	203
+124.454	200
+125.935	200
+127.417	200
+128.898	527
+130.38	525
+131.862	524
+133.343	522
+134.825	520
+136.306	519
+137.788	519
+139.269	518
+140.751	517
+142.233	515
+143.714	513
+145.196	513
+146.677	511
+148.159	508
+149.641	506
+151.122	503
+152.604	500
+154.085	499
+155.567	496
+157.049	491
+158.53	488
+160.012	486
+161.493	483
+162.975	481
+164.456	481
+165.938	480
+167.42	479
+168.901	476
+170.383	472
+171.864	470
+173.346	466
+174.828	463
+176.309	461
+177.791	460
+179.272	460
+180.754	458
+182.236	456
+183.717	455
+185.199	454
+186.68	454
+188.162	453
+189.644	452
+191.125	451
+192.607	450
+194.088	450
+195.57	449
+197.051	448
+198.533	445
+200.015	444
+201.496	444
+202.978	443
+204.459	442
+205.941	440
+207.423	437
+208.904	436
+210.386	435
+211.867	433
+213.349	432
+214.831	432
+216.312	430
+217.794	429
+219.275	426
+220.757	424
+222.238	423
+223.72	421
+225.202	419
+226.683	417
+228.165	414
+229.646	412
+231.128	412
+232.61	411
+234.091	408
+235.573	407
+237.054	403
+238.536	400
+240.018	396
+241.499	392
+242.981	392
+244.462	389
+245.944	384
+247.426	378
+248.907	375
+250.389	370
+251.87	365
+253.352	364
+254.833	358
+256.315	352
+257.797	345
+259.278	342
+260.76	336
+262.241	334
+263.723	333
+265.205	331
+266.686	331
+268.168	432
+269.649	432
+271.131	430
+272.613	427
+274.094	424
+275.576	422
+277.057	421
+278.539	419
+280.021	417
+281.502	412
+282.984	410
+284.465	406
+285.947	402
+287.428	400
+288.91	398
+290.392	397
+291.873	393
+293.355	389
+294.836	386
+296.318	383
+297.8	380
+299.281	376
+300.763	375
+302.244	374
+303.726	372
+305.208	370
+306.689	366
+308.171	363
+309.652	359
+311.134	356
+312.615	352
+314.097	353
+315.579	351
+317.06	351
+318.542	347
+320.023	344
+321.505	340
+322.987	339
+324.468	335
+325.95	335
+327.431	333
+328.913	330
+330.395	328
+331.876	323
+333.358	322
+334.839	317
+336.321	313
+337.803	309
+339.284	310
+340.766	307
+342.247	304
+343.729	300
+345.21	297
+346.692	296
+348.174	291
+349.655	289
+351.137	287
+352.618	285
+354.1	281
+355.582	278
+357.063	277
+358.545	272
+360.026	270
+361.508	265
+362.99	265
+364.471	265
+365.953	260
+367.434	257
+368.916	255
+370.397	254
+371.879	250
+373.361	249
+374.842	246
+376.324	246
+377.805	242
+379.287	237
+380.769	235
+382.25	231
+383.732	229
+385.213	224
+386.695	220
+388.177	221
+389.658	219
+391.14	218
+392.621	212
+394.103	210
+395.585	207
+397.066	203
+398.548	200
+400.029	199
+401.511	197
+402.992	192
+404.474	189
+405.956	189
+407.437	184
+408.919	181
+410.4	176
+411.882	177
+413.364	175
+414.845	172
+416.327	170
+417.808	166
+419.29	162
+420.772	160
+422.253	156
+423.735	157
+425.216	155
+426.698	150
+428.18	146
+429.661	143
+431.143	141
+432.624	138
+434.106	135
+435.587	132
+437.069	133
+438.551	129
+440.032	128
+441.514	125
+442.995	121
+444.477	117
+445.959	116
+447.44	114
+448.922	110
+450.403	107
+451.885	104
+453.367	101
+454.848	100
+456.33	95
+457.811	89
+459.293	83
+460.774	77
+462.256	71
+463.738	66
+465.219	61
+466.701	55
+468.182	49
+469.664	43
+471.146	37
+472.627	32
+474.109	26
+475.59	25
+477.072	20
+478.554	14
+480.035	8
+481.517	2
+482.998	

Added: SwiftApps/cray-benchmarks/plots/plot_cumulative.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots/plot_cumulative.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots/plot_cumulative.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,183 @@
+0	1
+2.65384	2
+5.30768	3
+7.96152	3
+10.6154	4
+13.2692	4
+15.923	4
+18.5769	4
+21.2307	4
+23.8846	5
+26.5384	6
+29.1922	81
+31.8461	112
+34.4999	152
+37.1538	209
+39.8076	219
+42.4614	222
+45.1153	226
+47.7691	229
+50.423	230
+53.0768	233
+55.7306	238
+58.3845	243
+61.0383	247
+63.6922	252
+66.346	256
+68.9998	257
+71.6537	263
+74.3075	270
+76.9614	276
+79.6152	280
+82.269	285
+84.9229	287
+87.5767	288
+90.2306	290
+92.8844	292
+95.5382	294
+98.1921	296
+100.846	300
+103.5	306
+106.154	311
+108.807	316
+111.461	317
+114.115	323
+116.769	329
+119.423	335
+122.077	341
+124.73	346
+127.384	352
+130.038	358
+132.692	359
+135.346	363
+138	368
+140.654	373
+143.307	379
+145.961	384
+148.615	390
+151.269	396
+153.923	397
+156.577	398
+159.23	403
+161.884	409
+164.538	415
+167.192	421
+169.846	427
+172.5	433
+175.153	438
+177.807	439
+180.461	440
+183.115	444
+185.769	450
+188.423	455
+191.076	461
+193.73	467
+196.384	473
+199.038	477
+201.692	478
+204.346	484
+207	490
+209.653	496
+212.307	502
+214.961	508
+217.615	514
+220.269	520
+222.923	521
+225.576	526
+228.23	531
+230.884	536
+233.538	542
+236.192	548
+238.846	554
+241.499	560
+244.153	563
+246.807	564
+249.461	570
+252.115	576
+254.769	582
+257.422	587
+260.076	592
+262.73	598
+265.384	603
+268.038	605
+270.692	611
+273.346	616
+275.999	620
+278.653	625
+281.307	631
+283.961	636
+286.615	641
+289.269	642
+291.922	648
+294.576	654
+297.23	659
+299.884	665
+302.538	670
+305.192	676
+307.845	682
+310.499	683
+313.153	684
+315.807	689
+318.461	695
+321.115	701
+323.768	707
+326.422	713
+329.076	719
+331.73	720
+334.384	726
+337.038	732
+339.692	738
+342.345	743
+344.999	749
+347.653	754
+350.307	760
+352.961	761
+355.615	767
+358.268	772
+360.922	778
+363.576	784
+366.23	790
+368.884	796
+371.538	801
+374.191	802
+376.845	806
+379.499	812
+382.153	818
+384.807	823
+387.461	828
+390.114	834
+392.768	839
+395.422	841
+398.076	842
+400.73	847
+403.384	852
+406.038	857
+408.691	863
+411.345	868
+413.999	874
+416.653	878
+419.307	884
+421.961	889
+424.614	895
+427.268	898
+429.922	899
+432.576	905
+435.23	910
+437.884	916
+440.537	922
+443.191	928
+445.845	933
+448.499	938
+451.153	944
+453.807	950
+456.46	956
+459.114	962
+461.768	967
+464.422	973
+467.076	974
+469.73	980
+472.384	985
+475.037	991
+477.691	997
+480.345	1000
+482.999	

Added: SwiftApps/cray-benchmarks/plots/plotit.gp
===================================================================
--- SwiftApps/cray-benchmarks/plots/plotit.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots/plotit.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,21 @@
+#To be run as a gnuplot script as follows:
+# $ gnuplot plotit.gp
+
+set terminal png enhanced size font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf" 18 1000,1000
+#set term postscript eps enhanced 
+#set terminal svg enhanced size 1000 1000
+
+set style line 2 linecolor rgb "blue"
+set output "activeplot.png"
+set nokey
+set xlabel "Time in sec" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,20"
+set ylabel "number of active jobs" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,20"
+set title "Active jobs" font "/usr/share/fonts/truetype/msttcorefonts/verdana.ttf,22"
+plot "plot_active.txt" using 1:2 with lines lc rgb "skyblue" lw 4 notitle;
+
+set output "cumulativeplot.png"
+set xlabel "Time in seconds"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+

Added: SwiftApps/cray-benchmarks/plots/scs.png
===================================================================
Added: SwiftApps/cray-benchmarks/plots2/T.data
===================================================================
Added: SwiftApps/cray-benchmarks/plots2/active.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots2/active.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/active.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,331 @@
+5
+5
+5
+5
+5
+46
+46
+46
+46
+46
+95
+95
+95
+118
+118
+118
+118
+118
+158
+158
+158
+158
+185
+185
+185
+185
+185
+233
+233
+233
+233
+258
+258
+258
+258
+258
+258
+258
+258
+258
+273
+272
+269
+265
+261
+259
+259
+259
+258
+258
+258
+258
+258
+258
+359
+358
+355
+353
+350
+347
+346
+344
+344
+343
+342
+341
+341
+340
+339
+337
+335
+334
+333
+333
+332
+331
+329
+325
+321
+321
+320
+317
+315
+312
+309
+306
+304
+303
+303
+300
+296
+294
+292
+291
+290
+286
+283
+279
+278
+277
+275
+275
+274
+273
+273
+273
+273
+273
+273
+683
+681
+680
+677
+676
+673
+671
+669
+668
+666
+663
+661
+659
+656
+653
+652
+647
+646
+644
+643
+641
+640
+639
+639
+638
+637
+634
+632
+630
+629
+627
+624
+624
+624
+624
+623
+621
+616
+614
+611
+608
+607
+604
+602
+599
+595
+592
+590
+589
+588
+584
+582
+576
+570
+570
+566
+565
+565
+563
+557
+555
+551
+548
+543
+544
+541
+540
+537
+532
+531
+529
+526
+523
+520
+517
+515
+515
+514
+512
+507
+504
+501
+502
+496
+490
+484
+480
+475
+469
+468
+464
+457
+451
+445
+441
+435
+429
+429
+424
+417
+412
+411
+410
+407
+403
+399
+394
+389
+387
+382
+379
+374
+369
+367
+362
+357
+353
+352
+351
+347
+344
+342
+336
+334
+329
+328
+324
+318
+317
+313
+308
+304
+297
+296
+294
+288
+284
+279
+275
+271
+268
+262
+260
+256
+252
+248
+245
+241
+239
+234
+235
+233
+232
+227
+224
+222
+217
+212
+211
+209
+206
+203
+198
+195
+189
+184
+181
+178
+176
+171
+168
+164
+158
+154
+151
+151
+149
+145
+141
+139
+133
+129
+127
+123
+124
+119
+116
+110
+106
+103
+98
+92
+89
+86
+82
+77
+74
+69
+67
+63
+61
+59
+56
+53
+50
+45
+41
+38
+36
+37
+33
+29
+27
+28
+25
+22
+17
+14
+11
+7
+4
+1

Added: SwiftApps/cray-benchmarks/plots2/activeplot.png
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/plots2/activeplot.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/plots2/activetics.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots2/activetics.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/activetics.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,332 @@
+0.00000
+1.48036
+2.96072
+4.44108
+5.92144
+7.40180
+8.88216
+10.36252
+11.84288
+13.32324
+14.80360
+16.28396
+17.76432
+19.24468
+20.72504
+22.20540
+23.68576
+25.16612
+26.64648
+28.12684
+29.60720
+31.08756
+32.56792
+34.04828
+35.52864
+37.00900
+38.48936
+39.96972
+41.45008
+42.93044
+44.41080
+45.89116
+47.37152
+48.85188
+50.33224
+51.81260
+53.29296
+54.77332
+56.25368
+57.73404
+59.21440
+60.69476
+62.17512
+63.65548
+65.13584
+66.61620
+68.09656
+69.57692
+71.05728
+72.53764
+74.01800
+75.49836
+76.97872
+78.45908
+79.93944
+81.41980
+82.90016
+84.38052
+85.86088
+87.34124
+88.82160
+90.30196
+91.78232
+93.26268
+94.74304
+96.22340
+97.70376
+99.18412
+100.66448
+102.14484
+103.62520
+105.10556
+106.58592
+108.06628
+109.54664
+111.02700
+112.50736
+113.98772
+115.46808
+116.94844
+118.42880
+119.90916
+121.38952
+122.86988
+124.35024
+125.83060
+127.31096
+128.79132
+130.27168
+131.75204
+133.23240
+134.71276
+136.19312
+137.67348
+139.15384
+140.63420
+142.11456
+143.59492
+145.07528
+146.55564
+148.03600
+149.51636
+150.99672
+152.47708
+153.95744
+155.43780
+156.91816
+158.39852
+159.87888
+161.35924
+162.83960
+164.31996
+165.80032
+167.28068
+168.76104
+170.24140
+171.72176
+173.20212
+174.68248
+176.16284
+177.64320
+179.12356
+180.60392
+182.08428
+183.56464
+185.04500
+186.52536
+188.00572
+189.48608
+190.96644
+192.44680
+193.92716
+195.40752
+196.88788
+198.36824
+199.84860
+201.32896
+202.80932
+204.28968
+205.77004
+207.25040
+208.73076
+210.21112
+211.69148
+213.17184
+214.65220
+216.13256
+217.61292
+219.09328
+220.57364
+222.05400
+223.53436
+225.01472
+226.49508
+227.97544
+229.45580
+230.93616
+232.41652
+233.89688
+235.37724
+236.85760
+238.33796
+239.81832
+241.29868
+242.77904
+244.25940
+245.73976
+247.22012
+248.70048
+250.18084
+251.66120
+253.14156
+254.62192
+256.10228
+257.58264
+259.06300
+260.54336
+262.02372
+263.50408
+264.98444
+266.46480
+267.94516
+269.42552
+270.90588
+272.38624
+273.86660
+275.34696
+276.82732
+278.30768
+279.78804
+281.26840
+282.74876
+284.22912
+285.70948
+287.18984
+288.67020
+290.15056
+291.63092
+293.11128
+294.59164
+296.07200
+297.55236
+299.03272
+300.51308
+301.99344
+303.47380
+304.95416
+306.43452
+307.91488
+309.39524
+310.87560
+312.35596
+313.83632
+315.31668
+316.79704
+318.27740
+319.75776
+321.23812
+322.71848
+324.19884
+325.67920
+327.15956
+328.63992
+330.12028
+331.60064
+333.08100
+334.56136
+336.04172
+337.52208
+339.00244
+340.48280
+341.96316
+343.44352
+344.92388
+346.40424
+347.88460
+349.36496
+350.84532
+352.32568
+353.80604
+355.28640
+356.76676
+358.24712
+359.72748
+361.20784
+362.68820
+364.16856
+365.64892
+367.12928
+368.60964
+370.09000
+371.57036
+373.05072
+374.53108
+376.01144
+377.49180
+378.97216
+380.45252
+381.93288
+383.41324
+384.89360
+386.37396
+387.85432
+389.33468
+390.81504
+392.29540
+393.77576
+395.25612
+396.73648
+398.21684
+399.69720
+401.17756
+402.65792
+404.13828
+405.61864
+407.09900
+408.57936
+410.05972
+411.54008
+413.02044
+414.50080
+415.98116
+417.46152
+418.94188
+420.42224
+421.90260
+423.38296
+424.86332
+426.34368
+427.82404
+429.30440
+430.78476
+432.26512
+433.74548
+435.22584
+436.70620
+438.18656
+439.66692
+441.14728
+442.62764
+444.10800
+445.58836
+447.06872
+448.54908
+450.02944
+451.50980
+452.99016
+454.47052
+455.95088
+457.43124
+458.91160
+460.39196
+461.87232
+463.35268
+464.83304
+466.31340
+467.79376
+469.27412
+470.75448
+472.23484
+473.71520
+475.19556
+476.67592
+478.15628
+479.63664
+481.11700
+482.59736
+484.07772
+485.55808
+487.03844
+488.51880
+489.99916

Added: SwiftApps/cray-benchmarks/plots2/anneal.data
===================================================================
Added: SwiftApps/cray-benchmarks/plots2/cumulative.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots2/cumulative.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/cumulative.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,200 @@
+1
+1
+1
+2
+2
+2
+3
+27
+61
+84
+149
+172
+173
+174
+178
+181
+186
+190
+190
+193
+196
+198
+204
+206
+209
+210
+211
+214
+217
+224
+227
+231
+235
+235
+236
+238
+245
+248
+253
+257
+261
+262
+264
+266
+270
+274
+275
+277
+281
+283
+287
+291
+292
+293
+293
+295
+300
+305
+308
+309
+315
+321
+327
+333
+339
+345
+346
+352
+358
+364
+370
+376
+382
+387
+390
+396
+402
+408
+409
+410
+415
+420
+425
+431
+437
+443
+448
+453
+459
+464
+470
+476
+481
+487
+490
+491
+496
+501
+507
+513
+519
+525
+528
+534
+540
+546
+552
+558
+564
+570
+571
+576
+582
+588
+594
+599
+605
+611
+617
+620
+626
+632
+638
+644
+648
+654
+659
+660
+661
+666
+672
+677
+682
+688
+694
+698
+699
+705
+710
+716
+721
+727
+732
+738
+740
+744
+749
+755
+761
+767
+773
+778
+780
+781
+787
+793
+799
+805
+811
+817
+822
+823
+829
+835
+841
+847
+853
+859
+865
+868
+874
+880
+885
+890
+896
+901
+906
+907
+910
+914
+919
+924
+930
+936
+942
+943
+944
+948
+954
+957
+958
+961
+967
+972
+978
+983
+989
+993
+998
+1000

Added: SwiftApps/cray-benchmarks/plots2/cumulativeplot-openmp.png
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/plots2/cumulativeplot-openmp.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/plots2/cumulativeplot.png
===================================================================
(Binary files differ)


Property changes on: SwiftApps/cray-benchmarks/plots2/cumulativeplot.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: SwiftApps/cray-benchmarks/plots2/cumultics.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots2/cumultics.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/cumultics.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,201 @@
+0.00000
+2.45000
+4.90000
+7.35000
+9.80000
+12.25000
+14.70000
+17.15000
+19.60000
+22.05000
+24.50000
+26.95000
+29.40000
+31.85000
+34.30000
+36.75000
+39.20000
+41.65000
+44.10000
+46.55000
+49.00000
+51.45000
+53.90000
+56.35000
+58.80000
+61.25000
+63.70000
+66.15000
+68.60000
+71.05000
+73.50000
+75.95000
+78.40000
+80.85000
+83.30000
+85.75000
+88.20000
+90.65000
+93.10000
+95.55000
+98.00000
+100.45000
+102.90000
+105.35000
+107.80000
+110.25000
+112.70000
+115.15000
+117.60000
+120.05000
+122.50000
+124.95000
+127.40000
+129.85000
+132.30000
+134.75000
+137.20000
+139.65000
+142.10000
+144.55000
+147.00000
+149.45000
+151.90000
+154.35000
+156.80000
+159.25000
+161.70000
+164.15000
+166.60000
+169.05000
+171.50000
+173.95000
+176.40000
+178.85000
+181.30000
+183.75000
+186.20000
+188.65000
+191.10000
+193.55000
+196.00000
+198.45000
+200.90000
+203.35000
+205.80000
+208.25000
+210.70000
+213.15000
+215.60000
+218.05000
+220.50000
+222.95000
+225.40000
+227.85000
+230.30000
+232.75000
+235.20000
+237.65000
+240.10000
+242.55000
+245.00000
+247.45000
+249.90000
+252.35000
+254.80000
+257.25000
+259.70000
+262.15000
+264.60000
+267.05000
+269.50000
+271.95000
+274.40000
+276.85000
+279.30000
+281.75000
+284.20000
+286.65000
+289.10000
+291.55000
+294.00000
+296.45000
+298.90000
+301.35000
+303.80000
+306.25000
+308.70000
+311.15000
+313.60000
+316.05000
+318.50000
+320.95000
+323.40000
+325.85000
+328.30000
+330.75000
+333.20000
+335.65000
+338.10000
+340.55000
+343.00000
+345.45000
+347.90000
+350.35000
+352.80000
+355.25000
+357.70000
+360.15000
+362.60000
+365.05000
+367.50000
+369.95000
+372.40000
+374.85000
+377.30000
+379.75000
+382.20000
+384.65000
+387.10000
+389.55000
+392.00000
+394.45000
+396.90000
+399.35000
+401.80000
+404.25000
+406.70000
+409.15000
+411.60000
+414.05000
+416.50000
+418.95000
+421.40000
+423.85000
+426.30000
+428.75000
+431.20000
+433.65000
+436.10000
+438.55000
+441.00000
+443.45000
+445.90000
+448.35000
+450.80000
+453.25000
+455.70000
+458.15000
+460.60000
+463.05000
+465.50000
+467.95000
+470.40000
+472.85000
+475.30000
+477.75000
+480.20000
+482.65000
+485.10000
+487.55000
+490.00000

Added: SwiftApps/cray-benchmarks/plots2/extract4plots
===================================================================
--- SwiftApps/cray-benchmarks/plots2/extract4plots	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/extract4plots	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+#usage: ./extract4plots <swift.outfile>
+
+SWIFTOUTFILE=$1
+
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE | head -n 1 | cut -f4-9 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE | tail -n 1 | cut -f4-9 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+
+#extract active runs in a file
+grep -o -i "Active:[0-9]*" $SWIFTOUTFILE | awk -F: '{print $2}' > active.txt
+
+#extract successful completions in a file
+grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE | awk -F: '{print $2}' > cumulative.txt
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+grep  "T =" $SWIFTOUTFILE | awk '{print $6}' | cut -c8- | sed 's/....$//' > T.data
+
+grep multi_annealing $SWIFTOUTFILE | grep "1;30" | awk '{print $3}' | cut -c11- | sed 's/....$//' > anneal.data
+
+grep returning $SWIFTOUTFILE | awk '{print $3, $4, $5, $6}' | sed -e 's/'ci='//' -e 's/'cj='//' -e 's/'r.loss='//' -e 's/'r.sdev='//' | sort -t' ' -k 1,2n > multiloss.txt


Property changes on: SwiftApps/cray-benchmarks/plots2/extract4plots
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/cray-benchmarks/plots2/multiloss.txt
===================================================================
Added: SwiftApps/cray-benchmarks/plots2/plot_active.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots2/plot_active.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/plot_active.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,332 @@
+0.00000	5
+1.48036	5
+2.96072	5
+4.44108	5
+5.92144	5
+7.40180	46
+8.88216	46
+10.36252	46
+11.84288	46
+13.32324	46
+14.80360	95
+16.28396	95
+17.76432	95
+19.24468	118
+20.72504	118
+22.20540	118
+23.68576	118
+25.16612	118
+26.64648	158
+28.12684	158
+29.60720	158
+31.08756	158
+32.56792	185
+34.04828	185
+35.52864	185
+37.00900	185
+38.48936	185
+39.96972	233
+41.45008	233
+42.93044	233
+44.41080	233
+45.89116	258
+47.37152	258
+48.85188	258
+50.33224	258
+51.81260	258
+53.29296	258
+54.77332	258
+56.25368	258
+57.73404	258
+59.21440	273
+60.69476	272
+62.17512	269
+63.65548	265
+65.13584	261
+66.61620	259
+68.09656	259
+69.57692	259
+71.05728	258
+72.53764	258
+74.01800	258
+75.49836	258
+76.97872	258
+78.45908	258
+79.93944	359
+81.41980	358
+82.90016	355
+84.38052	353
+85.86088	350
+87.34124	347
+88.82160	346
+90.30196	344
+91.78232	344
+93.26268	343
+94.74304	342
+96.22340	341
+97.70376	341
+99.18412	340
+100.66448	339
+102.14484	337
+103.62520	335
+105.10556	334
+106.58592	333
+108.06628	333
+109.54664	332
+111.02700	331
+112.50736	329
+113.98772	325
+115.46808	321
+116.94844	321
+118.42880	320
+119.90916	317
+121.38952	315
+122.86988	312
+124.35024	309
+125.83060	306
+127.31096	304
+128.79132	303
+130.27168	303
+131.75204	300
+133.23240	296
+134.71276	294
+136.19312	292
+137.67348	291
+139.15384	290
+140.63420	286
+142.11456	283
+143.59492	279
+145.07528	278
+146.55564	277
+148.03600	275
+149.51636	275
+150.99672	274
+152.47708	273
+153.95744	273
+155.43780	273
+156.91816	273
+158.39852	273
+159.87888	273
+161.35924	683
+162.83960	681
+164.31996	680
+165.80032	677
+167.28068	676
+168.76104	673
+170.24140	671
+171.72176	669
+173.20212	668
+174.68248	666
+176.16284	663
+177.64320	661
+179.12356	659
+180.60392	656
+182.08428	653
+183.56464	652
+185.04500	647
+186.52536	646
+188.00572	644
+189.48608	643
+190.96644	641
+192.44680	640
+193.92716	639
+195.40752	639
+196.88788	638
+198.36824	637
+199.84860	634
+201.32896	632
+202.80932	630
+204.28968	629
+205.77004	627
+207.25040	624
+208.73076	624
+210.21112	624
+211.69148	624
+213.17184	623
+214.65220	621
+216.13256	616
+217.61292	614
+219.09328	611
+220.57364	608
+222.05400	607
+223.53436	604
+225.01472	602
+226.49508	599
+227.97544	595
+229.45580	592
+230.93616	590
+232.41652	589
+233.89688	588
+235.37724	584
+236.85760	582
+238.33796	576
+239.81832	570
+241.29868	570
+242.77904	566
+244.25940	565
+245.73976	565
+247.22012	563
+248.70048	557
+250.18084	555
+251.66120	551
+253.14156	548
+254.62192	543
+256.10228	544
+257.58264	541
+259.06300	540
+260.54336	537
+262.02372	532
+263.50408	531
+264.98444	529
+266.46480	526
+267.94516	523
+269.42552	520
+270.90588	517
+272.38624	515
+273.86660	515
+275.34696	514
+276.82732	512
+278.30768	507
+279.78804	504
+281.26840	501
+282.74876	502
+284.22912	496
+285.70948	490
+287.18984	484
+288.67020	480
+290.15056	475
+291.63092	469
+293.11128	468
+294.59164	464
+296.07200	457
+297.55236	451
+299.03272	445
+300.51308	441
+301.99344	435
+303.47380	429
+304.95416	429
+306.43452	424
+307.91488	417
+309.39524	412
+310.87560	411
+312.35596	410
+313.83632	407
+315.31668	403
+316.79704	399
+318.27740	394
+319.75776	389
+321.23812	387
+322.71848	382
+324.19884	379
+325.67920	374
+327.15956	369
+328.63992	367
+330.12028	362
+331.60064	357
+333.08100	353
+334.56136	352
+336.04172	351
+337.52208	347
+339.00244	344
+340.48280	342
+341.96316	336
+343.44352	334
+344.92388	329
+346.40424	328
+347.88460	324
+349.36496	318
+350.84532	317
+352.32568	313
+353.80604	308
+355.28640	304
+356.76676	297
+358.24712	296
+359.72748	294
+361.20784	288
+362.68820	284
+364.16856	279
+365.64892	275
+367.12928	271
+368.60964	268
+370.09000	262
+371.57036	260
+373.05072	256
+374.53108	252
+376.01144	248
+377.49180	245
+378.97216	241
+380.45252	239
+381.93288	234
+383.41324	235
+384.89360	233
+386.37396	232
+387.85432	227
+389.33468	224
+390.81504	222
+392.29540	217
+393.77576	212
+395.25612	211
+396.73648	209
+398.21684	206
+399.69720	203
+401.17756	198
+402.65792	195
+404.13828	189
+405.61864	184
+407.09900	181
+408.57936	178
+410.05972	176
+411.54008	171
+413.02044	168
+414.50080	164
+415.98116	158
+417.46152	154
+418.94188	151
+420.42224	151
+421.90260	149
+423.38296	145
+424.86332	141
+426.34368	139
+427.82404	133
+429.30440	129
+430.78476	127
+432.26512	123
+433.74548	124
+435.22584	119
+436.70620	116
+438.18656	110
+439.66692	106
+441.14728	103
+442.62764	98
+444.10800	92
+445.58836	89
+447.06872	86
+448.54908	82
+450.02944	77
+451.50980	74
+452.99016	69
+454.47052	67
+455.95088	63
+457.43124	61
+458.91160	59
+460.39196	56
+461.87232	53
+463.35268	50
+464.83304	45
+466.31340	41
+467.79376	38
+469.27412	36
+470.75448	37
+472.23484	33
+473.71520	29
+475.19556	27
+476.67592	28
+478.15628	25
+479.63664	22
+481.11700	17
+482.59736	14
+484.07772	11
+485.55808	7
+487.03844	4
+488.51880	1
+489.99916	

Added: SwiftApps/cray-benchmarks/plots2/plot_cumulative.txt
===================================================================
--- SwiftApps/cray-benchmarks/plots2/plot_cumulative.txt	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/plot_cumulative.txt	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,201 @@
+0.00000	1
+2.45000	1
+4.90000	1
+7.35000	2
+9.80000	2
+12.25000	2
+14.70000	3
+17.15000	27
+19.60000	61
+22.05000	84
+24.50000	149
+26.95000	172
+29.40000	173
+31.85000	174
+34.30000	178
+36.75000	181
+39.20000	186
+41.65000	190
+44.10000	190
+46.55000	193
+49.00000	196
+51.45000	198
+53.90000	204
+56.35000	206
+58.80000	209
+61.25000	210
+63.70000	211
+66.15000	214
+68.60000	217
+71.05000	224
+73.50000	227
+75.95000	231
+78.40000	235
+80.85000	235
+83.30000	236
+85.75000	238
+88.20000	245
+90.65000	248
+93.10000	253
+95.55000	257
+98.00000	261
+100.45000	262
+102.90000	264
+105.35000	266
+107.80000	270
+110.25000	274
+112.70000	275
+115.15000	277
+117.60000	281
+120.05000	283
+122.50000	287
+124.95000	291
+127.40000	292
+129.85000	293
+132.30000	293
+134.75000	295
+137.20000	300
+139.65000	305
+142.10000	308
+144.55000	309
+147.00000	315
+149.45000	321
+151.90000	327
+154.35000	333
+156.80000	339
+159.25000	345
+161.70000	346
+164.15000	352
+166.60000	358
+169.05000	364
+171.50000	370
+173.95000	376
+176.40000	382
+178.85000	387
+181.30000	390
+183.75000	396
+186.20000	402
+188.65000	408
+191.10000	409
+193.55000	410
+196.00000	415
+198.45000	420
+200.90000	425
+203.35000	431
+205.80000	437
+208.25000	443
+210.70000	448
+213.15000	453
+215.60000	459
+218.05000	464
+220.50000	470
+222.95000	476
+225.40000	481
+227.85000	487
+230.30000	490
+232.75000	491
+235.20000	496
+237.65000	501
+240.10000	507
+242.55000	513
+245.00000	519
+247.45000	525
+249.90000	528
+252.35000	534
+254.80000	540
+257.25000	546
+259.70000	552
+262.15000	558
+264.60000	564
+267.05000	570
+269.50000	571
+271.95000	576
+274.40000	582
+276.85000	588
+279.30000	594
+281.75000	599
+284.20000	605
+286.65000	611
+289.10000	617
+291.55000	620
+294.00000	626
+296.45000	632
+298.90000	638
+301.35000	644
+303.80000	648
+306.25000	654
+308.70000	659
+311.15000	660
+313.60000	661
+316.05000	666
+318.50000	672
+320.95000	677
+323.40000	682
+325.85000	688
+328.30000	694
+330.75000	698
+333.20000	699
+335.65000	705
+338.10000	710
+340.55000	716
+343.00000	721
+345.45000	727
+347.90000	732
+350.35000	738
+352.80000	740
+355.25000	744
+357.70000	749
+360.15000	755
+362.60000	761
+365.05000	767
+367.50000	773
+369.95000	778
+372.40000	780
+374.85000	781
+377.30000	787
+379.75000	793
+382.20000	799
+384.65000	805
+387.10000	811
+389.55000	817
+392.00000	822
+394.45000	823
+396.90000	829
+399.35000	835
+401.80000	841
+404.25000	847
+406.70000	853
+409.15000	859
+411.60000	865
+414.05000	868
+416.50000	874
+418.95000	880
+421.40000	885
+423.85000	890
+426.30000	896
+428.75000	901
+431.20000	906
+433.65000	907
+436.10000	910
+438.55000	914
+441.00000	919
+443.45000	924
+445.90000	930
+448.35000	936
+450.80000	942
+453.25000	943
+455.70000	944
+458.15000	948
+460.60000	954
+463.05000	957
+465.50000	958
+467.95000	961
+470.40000	967
+472.85000	972
+475.30000	978
+477.75000	983
+480.20000	989
+482.65000	993
+485.10000	998
+487.55000	1000
+490.00000	

Added: SwiftApps/cray-benchmarks/plots2/plotit.gp
===================================================================
--- SwiftApps/cray-benchmarks/plots2/plotit.gp	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/plotit.gp	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,48 @@
+#To be run as a gnuplot script as follows:
+
+# $ gnuplot plotit.gp
+
+set terminal png enhanced size 1000,1000
+#set term postscript eps enhanced 
+#set terminal svg enhanced size 1000 1000
+set style line 1 linecolor rgb "blue"
+set output "activeplot.png"
+set nokey
+set xlabel "Time in sec"
+set ylabel "number of active jobs"
+set title "Active jobs"
+plot "plot_active.txt" using 1:2 with line
+
+#multiplies the active jobs by 24 to take into account the number of openmp
+#procs running on Beagle
+set output "cumulativeplot-openmp.png"
+set xlabel "Time in seconds"
+set ylabel "number of completed jobs"
+set title "Cumulative SciColSim-openMP jobs"
+plot "plot_cumulative.txt" using 1:($2*24) with lines
+
+set output "cumulativeplot.png"
+set xlabel "Time in seconds"
+set ylabel "number of completed jobs"
+set title "Cumulative jobs"
+plot "plot_cumulative.txt" using 1:2 with lines
+
+set output "scs.png"
+set xlabel "Evolution"
+set ylabel "Value of T"
+set title "SciColSim evolution Results"
+plot "T.data" using 1 with lines 
+
+set output "scs_loss.png"
+set title "SciColSim evolution loss Results"
+set xlabel "Evolution"
+set ylabel "Value of loss(AR)"
+plot "anneal.data" using 1 with lines 
+
+set output "multiloss.png"
+set title "SciColSim evolution loss Results"
+set key auto
+set yrange [0:200]
+set xlabel "Evolution"
+set ylabel "loss"
+plot "multiloss.txt" using 3 with lines title "multiloss mean val",  "multiloss.txt" using ($3+$4) with lines title "+stddev", "multiloss.txt" using ($3-$4) with lines title "-stddev"

Added: SwiftApps/cray-benchmarks/plots2/run021.out
===================================================================
--- SwiftApps/cray-benchmarks/plots2/run021.out	                        (rev 0)
+++ SwiftApps/cray-benchmarks/plots2/run021.out	2012-05-23 14:53:20 UTC (rev 5795)
@@ -0,0 +1,363 @@
+Running in run021
+Swift 0.93 swift-r5483 cog-r3339
+
+RunID: 20120428-1656-480zag0g
+Progress:  time: Sat, 28 Apr 2012 16:56:08 -0500
+Progress:  time: Sat, 28 Apr 2012 16:56:09 -0500  Initializing:997  Selecting site:3
+Progress:Progress:  time: Sat, 28 Apr 2012 16:56:09 -0500  Initializing:997  Selecting site:3
+  time: Sat, 28 Apr 2012 16:56:09 -0500  Initializing:997  Selecting site:3
+Progress:  time: Sat, 28 Apr 2012 16:56:10 -0500  Submitting:997  Submitted:3
+Progress:  time: Sat, 28 Apr 2012 16:56:11 -0500  Submitting:981  Submitted:19
+Progress:  time: Sat, 28 Apr 2012 16:56:12 -0500  Submitting:963  Submitted:37
+Progress:  time: Sat, 28 Apr 2012 16:56:13 -0500  Submitting:947  Submitted:53
+Progress:  time: Sat, 28 Apr 2012 16:56:14 -0500  Submitting:931  Submitted:69
+Progress:  time: Sat, 28 Apr 2012 16:56:15 -0500  Submitting:919  Submitted:81
+Progress:  time: Sat, 28 Apr 2012 16:56:16 -0500  Submitting:903  Submitted:97
+Progress:  time: Sat, 28 Apr 2012 16:56:17 -0500  Submitting:887  Submitted:113
+Progress:  time: Sat, 28 Apr 2012 16:56:18 -0500  Submitting:871  Submitted:129
+Progress:  time: Sat, 28 Apr 2012 16:56:19 -0500  Submitting:855  Submitted:145
+Progress:  time: Sat, 28 Apr 2012 16:56:20 -0500  Submitting:841  Submitted:159
+Progress:  time: Sat, 28 Apr 2012 16:56:21 -0500  Submitting:825  Submitted:175
+Progress:  time: Sat, 28 Apr 2012 16:56:22 -0500  Submitting:809  Submitted:191
+Progress:  time: Sat, 28 Apr 2012 16:56:23 -0500  Submitting:793  Submitted:207
+Progress:  time: Sat, 28 Apr 2012 16:56:24 -0500  Submitting:777  Submitted:223
+Progress:  time: Sat, 28 Apr 2012 16:56:26 -0500  Submitting:775  Submitted:225
+Progress:  time: Sat, 28 Apr 2012 16:56:27 -0500  Submitting:773  Submitted:227
+Progress:  time: Sat, 28 Apr 2012 16:56:28 -0500  Submitting:769  Submitted:231
+Progress:  time: Sat, 28 Apr 2012 16:56:29 -0500  Submitting:753  Submitted:247
+Progress:  time: Sat, 28 Apr 2012 16:56:30 -0500  Submitting:740  Submitted:260
+Progress:  time: Sat, 28 Apr 2012 16:56:31 -0500  Submitting:731  Submitted:264  Active:5
+Progress:  time: Sat, 28 Apr 2012 16:56:32 -0500  Submitting:720  Submitted:275  Active:5
+Progress:  time: Sat, 28 Apr 2012 16:56:33 -0500  Submitting:708  Submitted:287  Active:5
+Progress:  time: Sat, 28 Apr 2012 16:56:34 -0500  Submitting:697  Submitted:298  Active:5
+Progress:  time: Sat, 28 Apr 2012 16:56:37 -0500  Submitting:696  Submitted:299  Active:5
+Progress:  time: Sat, 28 Apr 2012 16:56:38 -0500  Submitting:688  Submitted:266  Active:46
+Progress:  time: Sat, 28 Apr 2012 16:56:39 -0500  Submitting:677  Submitted:277  Active:46
+Progress:  time: Sat, 28 Apr 2012 16:56:40 -0500  Submitting:665  Submitted:289  Active:46
+Progress:  time: Sat, 28 Apr 2012 16:56:41 -0500  Submitting:656  Submitted:298  Active:46
+Progress:  time: Sat, 28 Apr 2012 16:56:42 -0500  Submitting:646  Submitted:308  Active:46
+Progress:  time: Sat, 28 Apr 2012 16:56:43 -0500  Submitting:636  Submitted:269  Active:95
+Progress:  time: Sat, 28 Apr 2012 16:56:44 -0500  Submitting:627  Submitted:278  Active:95
+Progress:  time: Sat, 28 Apr 2012 16:56:47 -0500  Submitting:623  Submitted:282  Active:95
+Progress:Progress:  time: Sat, 28 Apr 2012 16:56:48 -0500  time: Sat, 28 Apr 2012 16:56:48 -0500  Submitting:616  Submitting:616  Submitted:266  Active:118
+  Submitted:266  Active:118
+Progress:  time: Sat, 28 Apr 2012 16:56:49 -0500  Submitting:607  Submitted:275  Active:118
+Progress:  time: Sat, 28 Apr 2012 16:56:50 -0500  Submitting:599  Submitted:283  Active:118
+Progress:  time: Sat, 28 Apr 2012 16:56:51 -0500  Submitting:588  Submitted:294  Active:118
+Progress:  time: Sat, 28 Apr 2012 16:56:52 -0500  Submitting:576  Submitted:266  Active:158
+Progress:  time: Sat, 28 Apr 2012 16:56:54 -0500  Submitting:565  Submitted:277  Active:158
+Progress:  time: Sat, 28 Apr 2012 16:56:55 -0500  Submitting:556  Submitted:286  Active:158
+Progress:  time: Sat, 28 Apr 2012 16:56:57 -0500  Submitting:555  Submitted:287  Active:158
+Progress:  time: Sat, 28 Apr 2012 16:56:58 -0500  Submitting:548  Submitted:267  Active:185
+Progress:  time: Sat, 28 Apr 2012 16:56:59 -0500  Submitting:540  Submitted:275  Active:185
+Progress:  time: Sat, 28 Apr 2012 16:57:00 -0500  Submitting:530  Submitted:285  Active:185
+Progress:  time: Sat, 28 Apr 2012 16:57:01 -0500  Submitting:519  Submitted:296  Active:185
+Progress:  time: Sat, 28 Apr 2012 16:57:02 -0500  Submitting:509  Submitted:306  Active:185
+Progress:  time: Sat, 28 Apr 2012 16:57:03 -0500  Submitting:498  Submitted:269  Active:233
+Progress:  time: Sat, 28 Apr 2012 16:57:04 -0500  Submitting:487  Submitted:280  Active:233
+Progress:  time: Sat, 28 Apr 2012 16:57:06 -0500  Submitting:485  Submitted:282  Active:233
+Progress:  time: Sat, 28 Apr 2012 16:57:07 -0500  Submitting:481  Submitted:286  Active:233
+Progress:  time: Sat, 28 Apr 2012 16:57:08 -0500  Submitting:465  Submitted:277  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:09 -0500  Submitting:449  Submitted:293  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:10 -0500  Submitting:433  Submitted:309  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:11 -0500  Submitting:417  Submitted:325  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:13 -0500  Submitting:407  Submitted:335  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:14 -0500  Submitting:391  Submitted:351  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:15 -0500  Submitting:381  Submitted:361  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:17 -0500  Submitting:380  Submitted:362  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:19 -0500  Submitting:378  Submitted:364  Active:258
+Progress:  time: Sat, 28 Apr 2012 16:57:20 -0500  Submitting:376  Submitted:349  Active:273  Checking status:1  Stage out:1
+Progress:  time: Sat, 28 Apr 2012 16:57:21 -0500  Submitting:371  Submitted:354  Active:272  Stage out:3
+Progress:  time: Sat, 28 Apr 2012 16:57:22 -0500  Submitting:364  Submitted:361  Active:269  Stage out:6
+Progress:  time: Sat, 28 Apr 2012 16:57:23 -0500  Submitting:354  Submitted:371  Active:265  Stage out:10
+Progress:  time: Sat, 28 Apr 2012 16:57:24 -0500  Submitting:343  Submitted:382  Active:261  Stage out:14
+Progress:  time: Sat, 28 Apr 2012 16:57:25 -0500  Submitting:340  Submitted:385  Active:259  Checking status:1  Stage out:15
+Progress:  time: Sat, 28 Apr 2012 16:57:26 -0500  Submitting:338  Submitted:387  Active:259  Stage out:16
+Progress:  time: Sat, 28 Apr 2012 16:57:27 -0500  Submitting:337  Submitted:388  Active:259  Stage out:16
+Progress:  time: Sat, 28 Apr 2012 16:57:29 -0500  Submitting:335  Submitted:390  Active:258  Stage out:17
+Progress:  time: Sat, 28 Apr 2012 16:57:30 -0500  Submitting:333  Submitted:392  Active:258  Stage out:17
+Progress:  time: Sat, 28 Apr 2012 16:57:31 -0500  Submitting:330  Submitted:395  Active:258  Stage out:17
+Progress:  time: Sat, 28 Apr 2012 16:57:32 -0500  Submitting:327  Submitted:398  Active:258  Stage out:17
+Progress:  time: Sat, 28 Apr 2012 16:57:34 -0500  Submitting:323  Submitted:402  Active:258  Stage out:17
+Progress:  time: Sat, 28 Apr 2012 16:57:36 -0500  Submitting:320  Submitted:405  Active:258  Stage out:17
+Progress:  time: Sat, 28 Apr 2012 16:57:37 -0500  Submitting:319  Submitted:304  Active:359  Stage out:18
+Progress:  time: Sat, 28 Apr 2012 16:57:38 -0500  Submitting:316  Submitted:307  Active:358  Stage out:19
+Progress:  time: Sat, 28 Apr 2012 16:57:39 -0500  Submitting:309  Submitted:314  Active:355  Stage out:22
+Progress:  time: Sat, 28 Apr 2012 16:57:40 -0500  Submitting:303  Submitted:320  Active:353  Stage out:24
+Progress:  time: Sat, 28 Apr 2012 16:57:42 -0500  Submitting:297  Submitted:326  Active:350  Stage out:27
+Progress:  time: Sat, 28 Apr 2012 16:57:43 -0500  Submitting:292  Submitted:331  Active:347  Checking status:1  Stage out:29
+Progress:  time: Sat, 28 Apr 2012 16:57:44 -0500  Submitting:287  Submitted:336  Active:346  Stage out:31
+Progress:  time: Sat, 28 Apr 2012 16:57:48 -0500  Submitting:284  Submitted:339  Active:344  Checking status:1  Stage out:32
+Progress:  time: Sat, 28 Apr 2012 16:57:50 -0500  Submitting:281  Submitted:342  Active:344  Stage out:33
+Progress:  time: Sat, 28 Apr 2012 16:57:52 -0500  Submitting:279  Submitted:344  Active:343  Stage out:34
+Progress:  time: Sat, 28 Apr 2012 16:57:53 -0500  Submitting:277  Submitted:346  Active:342  Stage out:35
+Progress:  time: Sat, 28 Apr 2012 16:57:54 -0500  Submitting:275  Submitted:348  Active:341  Stage out:36
+Progress:  time: Sat, 28 Apr 2012 16:57:57 -0500  Submitting:273  Submitted:350  Active:341  Stage out:36
+Progress:  time: Sat, 28 Apr 2012 16:57:58 -0500  Submitting:271  Submitted:352  Active:340  Stage out:37
+Progress:  time: Sat, 28 Apr 2012 16:57:59 -0500  Submitting:267  Submitted:356  Active:339  Stage out:38
+Progress:  time: Sat, 28 Apr 2012 16:58:01 -0500  Submitting:263  Submitted:360  Active:337  Stage out:40
+Progress:  time: Sat, 28 Apr 2012 16:58:02 -0500  Submitting:259  Submitted:364  Active:335  Stage out:42
+Progress:  time: Sat, 28 Apr 2012 16:58:03 -0500  Submitting:257  Submitted:366  Active:334  Stage out:43
+Progress:  time: Sat, 28 Apr 2012 16:58:04 -0500  Submitting:255  Submitted:368  Active:333  Stage out:44
+Progress:  time: Sat, 28 Apr 2012 16:58:08 -0500  Submitting:254  Submitted:369  Active:333  Stage out:44
+Progress:  time: Sat, 28 Apr 2012 16:58:09 -0500  Submitting:251  Submitted:372  Active:332  Stage out:45
+Progress:  time: Sat, 28 Apr 2012 16:58:10 -0500  Submitting:249  Submitted:374  Active:331  Stage out:46
+Progress:  time: Sat, 28 Apr 2012 16:58:12 -0500  Submitting:246  Submitted:377  Active:329  Checking status:1  Stage out:47
+Progress:  time: Sat, 28 Apr 2012 16:58:13 -0500  Submitting:238  Submitted:385  Active:325  Checking status:1  Stage out:51
+Progress:  time: Sat, 28 Apr 2012 16:58:15 -0500  Submitting:230  Submitted:393  Active:321  Checking status:1  Stage out:55
+Progress:  time: Sat, 28 Apr 2012 16:58:17 -0500  Submitting:229  Submitted:394  Active:321  Stage out:56
+Progress:  time: Sat, 28 Apr 2012 16:58:18 -0500  Submitting:228  Submitted:395  Active:320  Checking status:1  Stage out:56
+Progress:  time: Sat, 28 Apr 2012 16:58:20 -0500  Submitting:220  Submitted:403  Active:317  Checking status:1  Stage out:59
+Progress:  time: Sat, 28 Apr 2012 16:58:21 -0500  Submitting:213  Submitted:410  Active:315  Stage out:62
+Progress:  time: Sat, 28 Apr 2012 16:58:22 -0500  Submitting:206  Submitted:417  Active:312  Stage out:65
+Progress:  time: Sat, 28 Apr 2012 16:58:23 -0500  Submitting:201  Submitted:422  Active:309  Checking status:1  Stage out:67
+Progress:  time: Sat, 28 Apr 2012 16:58:24 -0500  Submitting:193  Submitted:430  Active:306  Stage out:71
+Progress:  time: Sat, 28 Apr 2012 16:58:26 -0500  Submitting:189  Submitted:434  Active:304  Stage out:73
+Progress:  time: Sat, 28 Apr 2012 16:58:28 -0500  Submitting:189  Submitted:434  Active:303  Checking status:1  Stage out:73
+Progress:  time: Sat, 28 Apr 2012 16:58:29 -0500  Submitting:186  Submitted:437  Active:303  Stage out:74
+Progress:  time: Sat, 28 Apr 2012 16:58:30 -0500  Submitting:178  Submitted:445  Active:300  Stage out:77
+Progress:  time: Sat, 28 Apr 2012 16:58:31 -0500  Submitting:170  Submitted:453  Active:296  Checking status:1  Stage out:80
+Progress:  time: Sat, 28 Apr 2012 16:58:32 -0500  Submitting:163  Submitted:460  Active:294  Stage out:83
+Progress:  time: Sat, 28 Apr 2012 16:58:34 -0500  Submitting:162  Submitted:461  Active:292  Checking status:1  Stage out:84
+Progress:  time: Sat, 28 Apr 2012 16:58:36 -0500  Submitting:157  Submitted:466  Active:291  Stage out:86
+Progress:  time: Sat, 28 Apr 2012 16:58:37 -0500  Submitting:157  Submitted:466  Active:290  Checking status:1  Stage out:86
+Progress:  time: Sat, 28 Apr 2012 16:58:38 -0500  Submitting:147  Submitted:476  Active:286  Checking status:1  Stage out:90
+Progress:  time: Sat, 28 Apr 2012 16:58:40 -0500  Submitting:136  Submitted:487  Active:283  Stage out:94
+Progress:  time: Sat, 28 Apr 2012 16:58:41 -0500  Submitting:128  Submitted:495  Active:279  Stage out:98
+Progress:  time: Sat, 28 Apr 2012 16:58:42 -0500  Submitting:126  Submitted:497  Active:278  Stage out:99
+Progress:  time: Sat, 28 Apr 2012 16:58:43 -0500  Submitting:122  Submitted:501  Active:277  Stage out:100
+Progress:  time: Sat, 28 Apr 2012 16:58:47 -0500  Submitting:120  Submitted:503  Active:275  Checking status:1  Stage out:101
+Progress:  time: Sat, 28 Apr 2012 16:58:50 -0500  Submitting:117  Submitted:506  Active:275  Stage out:102
+Progress:  time: Sat, 28 Apr 2012 16:58:51 -0500  Submitting:115  Submitted:508  Active:274  Stage out:103
+Progress:  time: Sat, 28 Apr 2012 16:58:52 -0500  Submitting:115  Submitted:508  Active:273  Checking status:1  Stage out:103
+Progress:  time: Sat, 28 Apr 2012 16:58:53 -0500  Submitting:113  Submitted:510  Active:273  Stage out:104
+Progress:  time: Sat, 28 Apr 2012 16:58:58 -0500  Submitting:112  Submitted:511  Active:273  Stage out:104
+Progress:  time: Sat, 28 Apr 2012 16:58:59 -0500  Submitting:110  Submitted:513  Active:273  Stage out:104
+Progress:  time: Sat, 28 Apr 2012 16:59:01 -0500  Submitting:106  Submitted:517  Active:273  Stage out:104
+Progress:  time: Sat, 28 Apr 2012 16:59:02 -0500  Submitting:100  Submitted:523  Active:273  Stage out:104
+Progress:  time: Sat, 28 Apr 2012 16:59:03 -0500  Submitting:94  Submitted:117  Active:683  Stage out:106
+Failed to remove job 7977 from queue: 
+Failed to remove job 8014 from queue: 
+Progress:  time: Sat, 28 Apr 2012 16:59:07 -0500  Submitting:87  Submitted:123  Active:681  Stage out:109
+Progress:  time: Sat, 28 Apr 2012 16:59:08 -0500  Submitting:85  Submitted:125  Active:680  Stage out:110
+Progress:  time: Sat, 28 Apr 2012 16:59:09 -0500  Submitting:80  Submitted:130  Active:677  Stage out:113
+Progress:  time: Sat, 28 Apr 2012 16:59:10 -0500  Submitting:74  Submitted:135  Active:676  Stage out:115
+Progress:  time: Sat, 28 Apr 2012 16:59:11 -0500  Submitting:66  Submitted:143  Active:673  Stage out:118
+Progress:  time: Sat, 28 Apr 2012 16:59:13 -0500  Submitting:60  Submitted:149  Active:671  Stage out:120
+Progress:  time: Sat, 28 Apr 2012 16:59:14 -0500  Submitting:57  Submitted:152  Active:669  Checking status:1  Stage out:121
+Progress:  time: Sat, 28 Apr 2012 16:59:18 -0500  Submitting:53  Submitted:156  Active:668  Stage out:123
+Progress:  time: Sat, 28 Apr 2012 16:59:19 -0500  Submitting:50  Submitted:159  Active:666  Checking status:1  Stage out:124
+Progress:  time: Sat, 28 Apr 2012 16:59:20 -0500  Submitting:42  Submitted:167  Active:663  Checking status:1  Stage out:127
+Progress:  time: Sat, 28 Apr 2012 16:59:21 -0500  Submitting:34  Submitted:174  Active:661  Checking status:1  Stage out:130
+Progress:  time: Sat, 28 Apr 2012 16:59:22 -0500  Submitting:27  Submitted:181  Active:659  Stage out:133
+Progress:  time: Sat, 28 Apr 2012 16:59:24 -0500  Submitting:20  Submitted:188  Active:656  Stage out:136
+Progress:  time: Sat, 28 Apr 2012 16:59:27 -0500  Submitting:13  Submitted:195  Active:653  Checking status:1  Stage out:138
+Progress:  time: Sat, 28 Apr 2012 16:59:28 -0500  Submitting:11  Submitted:197  Active:652  Checking status:1  Stage out:139
+Progress:  time: Sat, 28 Apr 2012 16:59:30 -0500  Submitted:208  Active:647  Checking status:1  Stage out:144
+Progress:  time: Sat, 28 Apr 2012 16:59:32 -0500  Submitted:208  Active:646  Checking status:1  Stage out:145
+Progress:  time: Sat, 28 Apr 2012 16:59:33 -0500  Submitted:207  Active:644  Checking status:1  Stage out:148
+Progress:  time: Sat, 28 Apr 2012 16:59:38 -0500  Submitted:207  Active:643  Stage out:150
+Progress:  time: Sat, 28 Apr 2012 16:59:39 -0500  Submitted:207  Active:641  Checking status:1  Stage out:151
+Progress:  time: Sat, 28 Apr 2012 16:59:42 -0500  Submitted:206  Active:640  Checking status:1  Stage out:153
+Progress:  time: Sat, 28 Apr 2012 16:59:43 -0500  Submitted:206  Active:639  Checking status:1  Stage out:154
+Progress:  time: Sat, 28 Apr 2012 16:59:46 -0500  Submitted:206  Active:639  Stage out:154  Finished successfully:1
+Progress:  time: Sat, 28 Apr 2012 16:59:48 -0500  Submitted:206  Active:638  Checking status:1  Stage out:154  Finished successfully:1
+Progress:  time: Sat, 28 Apr 2012 16:59:50 -0500  Submitted:206  Active:637  Checking status:1  Stage out:155  Finished successfully:1
+Progress:  time: Sat, 28 Apr 2012 16:59:52 -0500  Submitted:206  Active:634  Checking status:1  Stage out:157  Finished successfully:2
+Progress:  time: Sat, 28 Apr 2012 16:59:59 -0500  Submitted:206  Active:632  Checking status:1  Stage out:159  Finished successfully:2
+Progress:  time: Sat, 28 Apr 2012 17:00:00 -0500  Submitted:206  Active:630  Checking status:1  Stage out:161  Finished successfully:2
+Progress:  time: Sat, 28 Apr 2012 17:00:01 -0500  Submitted:205  Active:629  Stage out:163  Finished successfully:3
+Progress:  time: Sat, 28 Apr 2012 17:00:02 -0500  Submitted:205  Active:627  Stage out:141  Finished successfully:27
+Progress:  time: Sat, 28 Apr 2012 17:00:03 -0500  Submitted:205  Active:624  Stage out:110  Finished successfully:61
+Progress:  time: Sat, 28 Apr 2012 17:00:05 -0500  Submitted:205  Active:624  Stage out:87  Finished successfully:84
+Progress:  time: Sat, 28 Apr 2012 17:00:06 -0500  Submitted:204  Active:624  Stage out:23  Finished successfully:149
+Progress:  time: Sat, 28 Apr 2012 17:00:08 -0500  Submitted:204  Active:624  Finished successfully:172
+Progress:  time: Sat, 28 Apr 2012 17:00:09 -0500  Submitted:204  Active:623  Finished successfully:173
+Progress:  time: Sat, 28 Apr 2012 17:00:10 -0500  Submitted:204  Active:621  Stage out:1  Finished successfully:174
+Progress:  time: Sat, 28 Apr 2012 17:00:11 -0500  Submitted:204  Active:616  Checking status:1  Stage out:1  Finished successfully:178
+Progress:  time: Sat, 28 Apr 2012 17:00:13 -0500  Submitted:203  Active:614  Stage out:2  Finished successfully:181
+Progress:  time: Sat, 28 Apr 2012 17:00:14 -0500  Submitted:203  Active:611  Finished successfully:186
+Progress:  time: Sat, 28 Apr 2012 17:00:15 -0500  Submitted:202  Active:608  Finished successfully:190
+Progress:  time: Sat, 28 Apr 2012 17:00:17 -0500  Submitted:202  Active:607  Checking status:1  Finished successfully:190
+Progress:  time: Sat, 28 Apr 2012 17:00:18 -0500  Submitted:201  Active:604  Checking status:1  Stage out:1  Finished successfully:193
+Progress:  time: Sat, 28 Apr 2012 17:00:19 -0500  Submitted:201  Active:602  Stage out:1  Finished successfully:196
+Progress:  time: Sat, 28 Apr 2012 17:00:21 -0500  Submitted:201  Active:599  Stage out:2  Finished successfully:198
+Progress:  time: Sat, 28 Apr 2012 17:00:22 -0500  Submitted:201  Active:595  Finished successfully:204
+Progress:  time: Sat, 28 Apr 2012 17:00:23 -0500  Submitted:201  Active:592  Stage out:1  Finished successfully:206
+Progress:  time: Sat, 28 Apr 2012 17:00:24 -0500  Submitted:201  Active:590  Finished successfully:209
+Progress:  time: Sat, 28 Apr 2012 17:00:25 -0500  Submitted:201  Active:589  Finished successfully:210
+Progress:  time: Sat, 28 Apr 2012 17:00:28 -0500  Submitted:200  Active:588  Checking status:1  Finished successfully:211
+Progress:  time: Sat, 28 Apr 2012 17:00:29 -0500  Submitted:200  Active:584  Checking status:1  Stage out:1  Finished successfully:214
+Progress:  time: Sat, 28 Apr 2012 17:00:30 -0500  Submitted:200  Active:582  Stage out:1  Finished successfully:217
+Progress:  time: Sat, 28 Apr 2012 17:00:31 -0500  Submitted:200  Active:576  Finished successfully:224
+Progress:  time: Sat, 28 Apr 2012 17:00:32 -0500  Submitted:200  Active:570  Checking status:1  Stage out:2  Finished successfully:227
+Progress:  time: Sat, 28 Apr 2012 17:00:33 -0500  Submitted:199  Active:570  Finished successfully:231
+Progress:  time: Sat, 28 Apr 2012 17:00:35 -0500  Submitted:199  Active:566  Finished successfully:235
+Progress:  time: Sat, 28 Apr 2012 17:00:36 -0500  Submitted:199  Active:565  Checking status:1  Finished successfully:235
+Progress:  time: Sat, 28 Apr 2012 17:00:38 -0500  Submitted:199  Active:565  Finished successfully:236
+Progress:  time: Sat, 28 Apr 2012 17:00:39 -0500  Submitted:198  Active:563  Stage out:1  Finished successfully:238
+Progress:  time: Sat, 28 Apr 2012 17:00:40 -0500  Submitted:198  Active:557  Finished successfully:245
+Progress:  time: Sat, 28 Apr 2012 17:00:41 -0500  Submitted:196  Active:555  Stage out:1  Finished successfully:248
+Progress:  time: Sat, 28 Apr 2012 17:00:42 -0500  Submitted:195  Active:551  Stage out:1  Finished successfully:253
+Failed to remove job 7783 from queue: 
+Progress:  time: Sat, 28 Apr 2012 17:00:43 -0500  Submitted:195  Active:548  Finished successfully:257
+Progress:  time: Sat, 28 Apr 2012 17:00:49 -0500  Submitted:195  Active:543  Checking status:1  Finished successfully:261
+Progress:  time: Sat, 28 Apr 2012 17:00:50 -0500  Submitted:194  Active:544  Finished successfully:262
+Progress:  time: Sat, 28 Apr 2012 17:00:52 -0500  Submitted:194  Active:541  Checking status:1  Finished successfully:264
+Progress:  time: Sat, 28 Apr 2012 17:00:53 -0500  Submitted:194  Active:540  Finished successfully:266
+Progress:  time: Sat, 28 Apr 2012 17:00:54 -0500  Submitted:193  Active:537  Finished successfully:270
+Progress:  time: Sat, 28 Apr 2012 17:00:56 -0500  Submitted:193  Active:532  Checking status:1  Finished successfully:274
+Progress:  time: Sat, 28 Apr 2012 17:00:58 -0500  Submitted:193  Active:531  Checking status:1  Finished successfully:275
+Progress:  time: Sat, 28 Apr 2012 17:00:59 -0500  Submitted:193  Active:529  Checking status:1  Finished successfully:277
+Progress:  time: Sat, 28 Apr 2012 17:01:00 -0500  Submitted:193  Active:526  Finished successfully:281
+Progress:  time: Sat, 28 Apr 2012 17:01:01 -0500  Submitted:193  Active:523  Checking status:1  Finished successfully:283
+Progress:  time: Sat, 28 Apr 2012 17:01:03 -0500  Submitted:193  Active:520  Finished successfully:287
+Progress:  time: Sat, 28 Apr 2012 17:01:04 -0500  Submitted:192  Active:517  Finished successfully:291
+Progress:  time: Sat, 28 Apr 2012 17:01:05 -0500  Submitted:192  Active:515  Checking status:1  Finished successfully:292
+Progress:  time: Sat, 28 Apr 2012 17:01:08 -0500  Submitted:192  Active:515  Finished successfully:293
+Progress:  time: Sat, 28 Apr 2012 17:01:10 -0500  Submitted:192  Active:514  Checking status:1  Finished successfully:293
+Progress:  time: Sat, 28 Apr 2012 17:01:11 -0500  Submitted:192  Active:512  Checking status:1  Finished successfully:295
+Progress:  time: Sat, 28 Apr 2012 17:01:12 -0500  Submitted:192  Active:507  Checking status:1  Finished successfully:300
+Progress:  time: Sat, 28 Apr 2012 17:01:13 -0500  Submitted:191  Active:504  Finished successfully:305
+Progress:  time: Sat, 28 Apr 2012 17:01:14 -0500  Submitted:190  Active:501  Checking status:1  Finished successfully:308
+Progress:  time: Sat, 28 Apr 2012 17:01:18 -0500  Submitted:189  Active:502  Finished successfully:309
+Progress:  time: Sat, 28 Apr 2012 17:01:19 -0500  Submitted:188  Active:496  Checking status:1  Finished successfully:315
+Progress:  time: Sat, 28 Apr 2012 17:01:20 -0500  Submitted:188  Active:490  Checking status:1  Finished successfully:321
+Progress:  time: Sat, 28 Apr 2012 17:01:21 -0500  Submitted:188  Active:484  Checking status:1  Finished successfully:327
+Progress:  time: Sat, 28 Apr 2012 17:01:22 -0500  Submitted:186  Active:480  Checking status:1  Finished successfully:333
+Progress:  time: Sat, 28 Apr 2012 17:01:23 -0500  Submitted:185  Active:475  Checking status:1  Finished successfully:339
+Progress:  time: Sat, 28 Apr 2012 17:01:24 -0500  Submitted:185  Active:469  Checking status:1  Finished successfully:345
+Progress:  time: Sat, 28 Apr 2012 17:01:27 -0500  Submitted:185  Active:468  Checking status:1  Finished successfully:346
+Progress:  time: Sat, 28 Apr 2012 17:01:28 -0500  Submitted:184  Active:464  Finished successfully:352
+Progress:  time: Sat, 28 Apr 2012 17:01:29 -0500  Submitted:184  Active:457  Checking status:1  Finished successfully:358
+Progress:  time: Sat, 28 Apr 2012 17:01:30 -0500  Submitted:184  Active:451  Checking status:1  Finished successfully:364
+Progress:  time: Sat, 28 Apr 2012 17:01:31 -0500  Submitted:184  Active:445  Checking status:1  Finished successfully:370
+Progress:  time: Sat, 28 Apr 2012 17:01:32 -0500  Submitted:183  Active:441  Finished successfully:376
+Progress:  time: Sat, 28 Apr 2012 17:01:33 -0500  Submitted:183  Active:435  Finished successfully:382
+Progress:  time: Sat, 28 Apr 2012 17:01:34 -0500  Submitted:183  Active:429  Checking status:1  Finished successfully:387
+Progress:  time: Sat, 28 Apr 2012 17:01:37 -0500  Submitted:181  Active:429  Finished successfully:390
+Progress:  time: Sat, 28 Apr 2012 17:01:38 -0500  Submitted:180  Active:424  Finished successfully:396
+Progress:  time: Sat, 28 Apr 2012 17:01:39 -0500  Submitted:180  Active:417  Checking status:1  Finished successfully:402
+Progress:  time: Sat, 28 Apr 2012 17:01:40 -0500  Submitted:179  Active:412  Checking status:1  Finished successfully:408
+Progress:  time: Sat, 28 Apr 2012 17:01:45 -0500  Submitted:179  Active:411  Checking status:1  Finished successfully:409
+Progress:  time: Sat, 28 Apr 2012 17:01:47 -0500  Submitted:179  Active:410  Checking status:1  Finished successfully:410
+Progress:  time: Sat, 28 Apr 2012 17:01:48 -0500  Submitted:177  Active:407  Checking status:1  Finished successfully:415
+Progress:  time: Sat, 28 Apr 2012 17:01:49 -0500  Submitted:176  Active:403  Checking status:1  Finished successfully:420
+Progress:  time: Sat, 28 Apr 2012 17:01:50 -0500  Submitted:175  Active:399  Checking status:1  Finished successfully:425
+Progress:  time: Sat, 28 Apr 2012 17:01:51 -0500  Submitted:174  Active:394  Checking status:1  Finished successfully:431
+Progress:  time: Sat, 28 Apr 2012 17:01:52 -0500  Submitted:173  Active:389  Checking status:1  Finished successfully:437
+Progress:  time: Sat, 28 Apr 2012 17:01:54 -0500  Submitted:170  Active:387  Finished successfully:443
+Progress:  time: Sat, 28 Apr 2012 17:01:57 -0500  Submitted:169  Active:382  Checking status:1  Finished successfully:448
+Progress:  time: Sat, 28 Apr 2012 17:01:58 -0500  Submitted:168  Active:379  Finished successfully:453
+Progress:  time: Sat, 28 Apr 2012 17:01:59 -0500  Submitted:167  Active:374  Finished successfully:459
+Progress:  time: Sat, 28 Apr 2012 17:02:00 -0500  Submitted:166  Active:369  Checking status:1  Finished successfully:464
+Progress:  time: Sat, 28 Apr 2012 17:02:01 -0500  Submitted:163  Active:367  Finished successfully:470
+Progress:  time: Sat, 28 Apr 2012 17:02:02 -0500  Submitted:162  Active:362  Finished successfully:476
+Progress:  time: Sat, 28 Apr 2012 17:02:03 -0500  Submitted:161  Active:357  Stage out:1  Finished successfully:481
+Progress:  time: Sat, 28 Apr 2012 17:02:04 -0500  Submitted:159  Active:353  Checking status:1  Finished successfully:487
+Progress:  time: Sat, 28 Apr 2012 17:02:06 -0500  Submitted:158  Active:352  Finished successfully:490
+Progress:  time: Sat, 28 Apr 2012 17:02:08 -0500  Submitted:158  Active:351  Finished successfully:491
+Progress:  time: Sat, 28 Apr 2012 17:02:09 -0500  Submitted:157  Active:347  Finished successfully:496
+Progress:  time: Sat, 28 Apr 2012 17:02:10 -0500  Submitted:154  Active:344  Checking status:1  Finished successfully:501
+Progress:  time: Sat, 28 Apr 2012 17:02:11 -0500  Submitted:151  Active:342  Finished successfully:507
+Progress:  time: Sat, 28 Apr 2012 17:02:12 -0500  Submitted:150  Active:336  Checking status:1  Finished successfully:513
+Progress:  time: Sat, 28 Apr 2012 17:02:13 -0500  Submitted:147  Active:334  Finished successfully:519
+Progress:  time: Sat, 28 Apr 2012 17:02:14 -0500  Submitted:145  Active:329  Checking status:1  Finished successfully:525
+Progress:  time: Sat, 28 Apr 2012 17:02:17 -0500  Submitted:144  Active:328  Finished successfully:528
+Progress:  time: Sat, 28 Apr 2012 17:02:18 -0500  Submitted:142  Active:324  Finished successfully:534
+Progress:  time: Sat, 28 Apr 2012 17:02:19 -0500  Submitted:141  Active:318  Checking status:1  Finished successfully:540
+Progress:  time: Sat, 28 Apr 2012 17:02:20 -0500  Submitted:137  Active:317  Finished successfully:546
+Progress:  time: Sat, 28 Apr 2012 17:02:21 -0500  Submitted:135  Active:313  Finished successfully:552
+Progress:  time: Sat, 28 Apr 2012 17:02:22 -0500  Submitted:134  Active:308  Finished successfully:558
+Progress:  time: Sat, 28 Apr 2012 17:02:23 -0500  Submitted:132  Active:304  Finished successfully:564
+Progress:  time: Sat, 28 Apr 2012 17:02:24 -0500  Submitted:132  Active:297  Checking status:1  Finished successfully:570
+Progress:  time: Sat, 28 Apr 2012 17:02:26 -0500  Submitted:132  Active:296  Checking status:1  Finished successfully:571
+Progress:  time: Sat, 28 Apr 2012 17:02:27 -0500  Submitted:130  Active:294  Finished successfully:576
+Progress:  time: Sat, 28 Apr 2012 17:02:28 -0500  Submitted:129  Active:288  Checking status:1  Finished successfully:582
+Progress:  time: Sat, 28 Apr 2012 17:02:29 -0500  Submitted:127  Active:284  Checking status:1  Finished successfully:588
+Progress:  time: Sat, 28 Apr 2012 17:02:30 -0500  Submitted:127  Active:279  Finished successfully:594
+Progress:  time: Sat, 28 Apr 2012 17:02:31 -0500  Submitted:125  Active:275  Checking status:1  Finished successfully:599
+Progress:  time: Sat, 28 Apr 2012 17:02:32 -0500  Submitted:123  Active:271  Checking status:1  Finished successfully:605
+Progress:  time: Sat, 28 Apr 2012 17:02:33 -0500  Submitted:121  Active:268  Finished successfully:611
+Progress:  time: Sat, 28 Apr 2012 17:02:34 -0500  Submitted:120  Active:262  Checking status:1  Finished successfully:617
+Progress:  time: Sat, 28 Apr 2012 17:02:37 -0500  Submitted:119  Active:260  Checking status:1  Finished successfully:620
+Progress:  time: Sat, 28 Apr 2012 17:02:38 -0500  Submitted:117  Active:256  Checking status:1  Finished successfully:626
+Progress:  time: Sat, 28 Apr 2012 17:02:39 -0500  Submitted:115  Active:252  Checking status:1  Finished successfully:632
+Progress:  time: Sat, 28 Apr 2012 17:02:40 -0500  Submitted:113  Active:248  Checking status:1  Finished successfully:638
+Progress:  time: Sat, 28 Apr 2012 17:02:41 -0500  Submitted:111  Active:245  Finished successfully:644
+Progress:  time: Sat, 28 Apr 2012 17:02:42 -0500  Submitted:110  Active:241  Stage out:1  Finished successfully:648
+Progress:  time: Sat, 28 Apr 2012 17:02:43 -0500  Submitted:107  Active:239  Finished successfully:654
+Progress:  time: Sat, 28 Apr 2012 17:02:44 -0500  Submitted:106  Active:234  Checking status:1  Finished successfully:659
+Progress:  time: Sat, 28 Apr 2012 17:02:46 -0500  Submitted:105  Active:235  Finished successfully:660
+Progress:  time: Sat, 28 Apr 2012 17:02:47 -0500  Submitted:105  Active:233  Checking status:1  Finished successfully:661
+Progress:  time: Sat, 28 Apr 2012 17:02:48 -0500  Submitted:102  Active:232  Finished successfully:666
+Progress:  time: Sat, 28 Apr 2012 17:02:49 -0500  Submitted:101  Active:227  Finished successfully:672
+Progress:  time: Sat, 28 Apr 2012 17:02:50 -0500  Submitted:99  Active:224  Finished successfully:677
+Progress:  time: Sat, 28 Apr 2012 17:02:51 -0500  Submitted:96  Active:222  Finished successfully:682
+Progress:  time: Sat, 28 Apr 2012 17:02:53 -0500  Submitted:94  Active:217  Checking status:1  Finished successfully:688
+Progress:  time: Sat, 28 Apr 2012 17:02:54 -0500  Submitted:93  Active:212  Checking status:1  Finished successfully:694
+Progress:  time: Sat, 28 Apr 2012 17:02:55 -0500  Submitted:91  Active:211  Finished successfully:698
+Progress:  time: Sat, 28 Apr 2012 17:02:57 -0500  Submitted:91  Active:209  Checking status:1  Finished successfully:699
+Progress:  time: Sat, 28 Apr 2012 17:02:58 -0500  Submitted:89  Active:206  Finished successfully:705
+Progress:  time: Sat, 28 Apr 2012 17:02:59 -0500  Submitted:87  Active:203  Finished successfully:710
+Progress:  time: Sat, 28 Apr 2012 17:03:00 -0500  Submitted:86  Active:198  Finished successfully:716
+Progress:  time: Sat, 28 Apr 2012 17:03:01 -0500  Submitted:84  Active:195  Finished successfully:721
+Progress:  time: Sat, 28 Apr 2012 17:03:02 -0500  Submitted:84  Active:189  Finished successfully:727
+Progress:  time: Sat, 28 Apr 2012 17:03:03 -0500  Submitted:83  Active:184  Checking status:1  Finished successfully:732
+Progress:  time: Sat, 28 Apr 2012 17:03:04 -0500  Submitted:81  Active:181  Finished successfully:738
+Progress:  time: Sat, 28 Apr 2012 17:03:07 -0500  Submitted:81  Active:178  Checking status:1  Finished successfully:740
+Progress:  time: Sat, 28 Apr 2012 17:03:08 -0500  Submitted:80  Active:176  Finished successfully:744
+Progress:  time: Sat, 28 Apr 2012 17:03:09 -0500  Submitted:79  Active:171  Checking status:1  Finished successfully:749
+Progress:  time: Sat, 28 Apr 2012 17:03:10 -0500  Submitted:77  Active:168  Finished successfully:755
+Progress:  time: Sat, 28 Apr 2012 17:03:11 -0500  Submitted:75  Active:164  Finished successfully:761
+Progress:  time: Sat, 28 Apr 2012 17:03:12 -0500  Submitted:74  Active:158  Checking status:1  Finished successfully:767
+Progress:  time: Sat, 28 Apr 2012 17:03:13 -0500  Submitted:72  Active:154  Checking status:1  Finished successfully:773
+Progress:  time: Sat, 28 Apr 2012 17:03:14 -0500  Submitted:70  Active:151  Checking status:1  Finished successfully:778
+Progress:  time: Sat, 28 Apr 2012 17:03:15 -0500  Submitted:69  Active:151  Finished successfully:780
+Progress:  time: Sat, 28 Apr 2012 17:03:17 -0500  Submitted:69  Active:149  Checking status:1  Finished successfully:781
+Progress:  time: Sat, 28 Apr 2012 17:03:18 -0500  Submitted:67  Active:145  Checking status:1  Finished successfully:787
+Progress:  time: Sat, 28 Apr 2012 17:03:19 -0500  Submitted:65  Active:141  Checking status:1  Finished successfully:793
+Progress:  time: Sat, 28 Apr 2012 17:03:20 -0500  Submitted:62  Active:139  Finished successfully:799
+Progress:  time: Sat, 28 Apr 2012 17:03:21 -0500  Submitted:61  Active:133  Checking status:1  Finished successfully:805
+Progress:  time: Sat, 28 Apr 2012 17:03:22 -0500  Submitted:59  Active:129  Checking status:1  Finished successfully:811
+Progress:  time: Sat, 28 Apr 2012 17:03:23 -0500  Submitted:56  Active:127  Finished successfully:817
+Progress:  time: Sat, 28 Apr 2012 17:03:25 -0500  Submitted:54  Active:123  Checking status:1  Finished successfully:822
+Progress:  time: Sat, 28 Apr 2012 17:03:26 -0500  Submitted:53  Active:124  Finished successfully:823
+Progress:  time: Sat, 28 Apr 2012 17:03:27 -0500  Submitted:51  Active:119  Checking status:1  Finished successfully:829
+Progress:  time: Sat, 28 Apr 2012 17:03:28 -0500  Submitted:49  Active:116  Finished successfully:835
+Progress:  time: Sat, 28 Apr 2012 17:03:30 -0500  Submitted:48  Active:110  Checking status:1  Finished successfully:841
+Progress:  time: Sat, 28 Apr 2012 17:03:31 -0500  Submitted:46  Active:106  Checking status:1  Finished successfully:847
+Progress:  time: Sat, 28 Apr 2012 17:03:32 -0500  Submitted:44  Active:103  Finished successfully:853
+Progress:  time: Sat, 28 Apr 2012 17:03:33 -0500  Submitted:43  Active:98  Finished successfully:859
+Progress:  time: Sat, 28 Apr 2012 17:03:34 -0500  Submitted:42  Active:92  Checking status:1  Finished successfully:865
+Progress:  time: Sat, 28 Apr 2012 17:03:37 -0500  Submitted:42  Active:89  Checking status:1  Finished successfully:868
+Progress:  time: Sat, 28 Apr 2012 17:03:38 -0500  Submitted:39  Active:86  Checking status:1  Finished successfully:874
+Progress:  time: Sat, 28 Apr 2012 17:03:39 -0500  Submitted:38  Active:82  Finished successfully:880
+Progress:  time: Sat, 28 Apr 2012 17:03:40 -0500  Submitted:37  Active:77  Checking status:1  Finished successfully:885
+Progress:  time: Sat, 28 Apr 2012 17:03:41 -0500  Submitted:36  Active:74  Finished successfully:890
+Progress:  time: Sat, 28 Apr 2012 17:03:42 -0500  Submitted:35  Active:69  Finished successfully:896
+Progress:  time: Sat, 28 Apr 2012 17:03:43 -0500  Submitted:32  Active:67  Finished successfully:901
+Progress:  time: Sat, 28 Apr 2012 17:03:44 -0500  Submitted:31  Active:63  Finished successfully:906
+Progress:  time: Sat, 28 Apr 2012 17:03:47 -0500  Submitted:31  Active:61  Checking status:1  Finished successfully:907
+Progress:  time: Sat, 28 Apr 2012 17:03:48 -0500  Submitted:31  Active:59  Finished successfully:910
+Progress:  time: Sat, 28 Apr 2012 17:03:49 -0500  Submitted:29  Active:56  Checking status:1  Finished successfully:914
+Progress:  time: Sat, 28 Apr 2012 17:03:50 -0500  Submitted:27  Active:53  Checking status:1  Finished successfully:919
+Progress:  time: Sat, 28 Apr 2012 17:03:51 -0500  Submitted:26  Active:50  Finished successfully:924
+Progress:  time: Sat, 28 Apr 2012 17:03:52 -0500  Submitted:24  Active:45  Checking status:1  Finished successfully:930
+Progress:  time: Sat, 28 Apr 2012 17:03:53 -0500  Submitted:22  Active:41  Checking status:1  Finished successfully:936
+Progress:  time: Sat, 28 Apr 2012 17:03:54 -0500  Submitted:20  Active:38  Finished successfully:942
+Progress:  time: Sat, 28 Apr 2012 17:03:55 -0500  Submitted:20  Active:36  Checking status:1  Finished successfully:943
+Progress:  time: Sat, 28 Apr 2012 17:03:57 -0500  Submitted:19  Active:37  Finished successfully:944
+Progress:  time: Sat, 28 Apr 2012 17:04:03 -0500  Submitted:18  Active:33  Checking status:1  Finished successfully:948
+Progress:  time: Sat, 28 Apr 2012 17:04:04 -0500  Submitted:16  Active:29  Checking status:1  Finished successfully:954
+Progress:  time: Sat, 28 Apr 2012 17:04:05 -0500  Submitted:15  Active:27  Checking status:1  Finished successfully:957
+Progress:  time: Sat, 28 Apr 2012 17:04:07 -0500  Submitted:14  Active:28  Finished successfully:958
+Progress:  time: Sat, 28 Apr 2012 17:04:09 -0500  Submitted:13  Active:25  Checking status:1  Finished successfully:961
+Progress:  time: Sat, 28 Apr 2012 17:04:10 -0500  Submitted:11  Active:22  Finished successfully:967
+Progress:  time: Sat, 28 Apr 2012 17:04:11 -0500  Submitted:10  Active:17  Checking status:1  Finished successfully:972
+Progress:  time: Sat, 28 Apr 2012 17:04:12 -0500  Submitted:7  Active:14  Checking status:1  Finished successfully:978
+Progress:  time: Sat, 28 Apr 2012 17:04:13 -0500  Submitted:5  Active:11  Stage out:1  Finished successfully:983
+Progress:  time: Sat, 28 Apr 2012 17:04:14 -0500  Submitted:3  Active:7  Checking status:1  Finished successfully:989
+Progress:  time: Sat, 28 Apr 2012 17:04:17 -0500  Submitted:2  Active:4  Checking status:1  Finished successfully:993
+Progress:  time: Sat, 28 Apr 2012 17:04:18 -0500  Submitted:1  Active:1  Finished successfully:998
+Final status:  time: Sat, 28 Apr 2012 17:04:18 -0500  Finished successfully:1000

Added: SwiftApps/cray-benchmarks/plots2/scs.png
===================================================================


From lgadelha at ci.uchicago.edu  Thu May 10 07:19:03 2012
From: lgadelha at ci.uchicago.edu (lgadelha at ci.uchicago.edu)
Date: Thu, 10 May 2012 12:19:03 -0000
Subject: [Swift-commit] r5786 - in provenancedb: . swift_mod
Message-ID: <20120510122754.B45789CCA7@svn.ci.uchicago.edu>

Author: lgadelha
Date: 2012-05-10 07:27:54 -0500 (Thu, 10 May 2012)
New Revision: 5786

Added:
   provenancedb/swift_mod/
   provenancedb/swift_mod/_swiftwrap_runtime_aggregate
   provenancedb/swift_mod/_swiftwrap_runtime_snapshots
   provenancedb/swift_mod/create-everylog-vs-versions-data
Modified:
   provenancedb/pql_functions.sql
   provenancedb/prepare-provenance-chart
   provenancedb/prov-init.sql
   provenancedb/prov-to-sql.sh
   provenancedb/swift-prov-import-all-logs
Log:
- Prospective provenance import scripts.
- Created swift_mod directory containing the modifications needed to gather runtime information in Swift about application executions.


Modified: provenancedb/pql_functions.sql
===================================================================
--- provenancedb/pql_functions.sql	2012-05-09 22:18:28 UTC (rev 5785)
+++ provenancedb/pql_functions.sql	2012-05-10 12:27:54 UTC (rev 5786)
@@ -12,8 +12,8 @@
 returns setof compare_run_by_parameter_type
 as $$
    select run_id, parameter, value
-   from   ds_io,fun_call,primitive
-   where  fun_call.id=ds_io.fun_call_id and ds_io.ds_id=primitive.id and parameter=$1;
+   from   dataset_io,fun_call,primitive
+   where  fun_call.id=dataset_io.function_call_id and dataset_io.dataset_id=primitive.id and parameter=$1;
 $$ language sql;
 
 -- PostgreSQL >= 9.0
@@ -26,13 +26,13 @@
 -- AS $$
 --    SELECT   fun_call.run_id, ds_out.parameter, primitive.value
 --    FROM     primitive, ds_out, fun_call
---    WHERE    primitive.id=ds_out.ds_id AND ds_out.fun_call_id=fun_call.id AND 
+--    WHERE    primitive.id=ds_out.dataset_id AND ds_out.fun_call_id=fun_call.id AND 
 --             ds_out.parameter=$1 
 --    GROUP BY fun_call.run_id, ds_out.parameter, primitive.value
 --  UNION
 --    SELECT   fun_call.run_id, ds_in.parameter, primitive.value
 --    FROM     primitive, ds_in, fun_call
---    WHERE    primitive.id=ds_in.ds_id AND ds_in.fun_call_id=fun_call.id AND 
+--    WHERE    primitive.id=ds_in.dataset_id AND ds_in.fun_call_id=fun_call.id AND 
 --             ds_in.parameter=$1 
 --    GROUP BY fun_call.run_id, ds_in.parameter, primitive.value	
 --$$ LANGUAGE SQL;
@@ -54,105 +54,105 @@
 --         USING (workflow_id); 
 --$$ LANGUAGE SQL;
 
-DROP TYPE compare_run_by_annot_num_type;
+DROP TYPE compare_run_by_annot_num_type CASCADE;
 CREATE TYPE compare_run_by_annot_num_type as (run_id VARCHAR, name VARCHAR, value NUMERIC);
 
 CREATE OR REPLACE FUNCTION compare_run_by_annot_num(name VARCHAR)
 RETURNS SETOF compare_run_by_annot_num_type
 AS $$
-    SELECT fun_call.run_id, annot_ds_num.name, annot_ds_num.value
-    FROM   annot_ds_num,ds_io,ds_cont,fun_call
-    WHERE  annot_ds_num.ds_id=ds_cont.in_id AND ds_cont.out_id=ds_io.ds_id AND
-           ds_io.fun_call_id=fun_call.id AND annot_ds_num.name=$1
+    SELECT fun_call.run_id, annot_dataset_num.name, annot_dataset_num.value
+    FROM   annot_dataset_num,dataset_io,dataset_containment,fun_call
+    WHERE  annot_dataset_num.dataset_id=dataset_containment.in_id AND dataset_containment.out_id=dataset_io.dataset_id AND
+           dataset_io.function_call_id=fun_call.id AND annot_dataset_num.name=$1
   UNION
-    SELECT fun_call.run_id, annot_ds_num.name, annot_ds_num.value 
-    FROM   fun_call, ds_io, annot_ds_num
-    WHERE  fun_call.id=ds_io.fun_call_id and ds_io.ds_id=annot_ds_num.ds_id and
-           annot_ds_num.name=$1
+    SELECT fun_call.run_id, annot_dataset_num.name, annot_dataset_num.value 
+    FROM   fun_call, dataset_io, annot_dataset_num
+    WHERE  fun_call.id=dataset_io.function_call_id and dataset_io.dataset_id=annot_dataset_num.dataset_id and
+           annot_dataset_num.name=$1
   UNION
-    SELECT fun_call.run_id, annot_fun_call_num.name, annot_fun_call_num.value 
-    FROM   fun_call, annot_fun_call_num
-    WHERE  fun_call.id=annot_fun_call_num.fun_call_id and annot_fun_call_num.name=$1
+    SELECT fun_call.run_id, annot_function_call_num.name, annot_function_call_num.value 
+    FROM   fun_call, annot_function_call_num
+    WHERE  fun_call.id=annot_function_call_num.function_call_id and annot_function_call_num.name=$1
   UNION
-    SELECT run.id as run_id, annot_run_num.name, annot_run_num.value 
-    FROM   run, annot_run_num
-    WHERE  run.id=annot_run_num.run_id and annot_run_num.name=$1
+    SELECT run.id as run_id, annot_script_run_num.name, annot_script_run_num.value 
+    FROM   run, annot_script_run_num
+    WHERE  run.id=annot_script_run_num.script_run_id and annot_script_run_num.name=$1
 $$ LANGUAGE SQL;
 
-DROP TYPE compare_run_by_key_numeric_type;
+DROP TYPE compare_run_by_key_numeric_type CASCADE;
 CREATE TYPE compare_run_by_key_numeric_type as (run_id VARCHAR, name VARCHAR, value NUMERIC);
 
 CREATE OR REPLACE FUNCTION compare_run_by_key_numeric(name VARCHAR)
 RETURNS SETOF compare_run_by_key_numeric_type
 AS $$
-    SELECT fun_call.run_id, annot_ds_n.name, annot_ds_n.value
-    FROM   annot_ds_n,ds_io,ds_cont,fun_call
-    WHERE  annot_ds_n.ds_id=ds_cont.in_id AND ds_cont.out_id=ds_io.ds_id AND
-           ds_io.fun_call_id=fun_call.id AND annot_ds_n.name=$1
+    SELECT fun_call.run_id, annot_dataset_num.name, annot_dataset_num.value
+    FROM   annot_dataset_num,dataset_io,dataset_containment,fun_call
+    WHERE  annot_dataset_num.dataset_id=dataset_containment.in_id AND dataset_containment.out_id=dataset_io.dataset_id AND
+           dataset_io.function_call_id=fun_call.id AND annot_dataset_num.name=$1
   UNION
-    SELECT fun_call.run_id, annot_ds_n.name, annot_ds_n.value 
-    FROM   fun_call, ds_io, annot_ds_n
-    WHERE  fun_call.id=ds_io.fun_call_id and ds_io.ds_id=annot_ds_n.ds_id and
-           annot_ds_n.name=$1
+    SELECT fun_call.run_id, annot_dataset_num.name, annot_dataset_num.value 
+    FROM   fun_call, dataset_io, annot_dataset_num
+    WHERE  fun_call.id=dataset_io.function_call_id and dataset_io.dataset_id=annot_dataset_num.dataset_id and
+           annot_dataset_num.name=$1
   UNION
-    SELECT fun_call.run_id, annot_fun_call_n.name, annot_fun_call_n.value 
-    FROM   fun_call, annot_fun_call_n
-    WHERE  fun_call.id=annot_fun_call_n.fun_call_id and annot_fun_call_n.name=$1
+    SELECT fun_call.run_id, annot_function_call_num.name, annot_function_call_num.value 
+    FROM   fun_call, annot_function_call_num
+    WHERE  fun_call.id=annot_function_call_num.function_call_id and annot_function_call_num.name=$1
   UNION
-    SELECT run.id as run_id, annot_run_n.name, annot_run_n.value 
-    FROM   run, annot_run_n
-    WHERE  run.id=annot_run_n.run_id and annot_run_n.name=$1
+    SELECT run.id as run_id, annot_script_run_num.name, annot_script_run_num.value 
+    FROM   run, annot_script_run_num
+    WHERE  run.id=annot_script_run_num.script_run_id and annot_script_run_num.name=$1
 $$ LANGUAGE SQL;
 
 
-DROP TYPE compare_run_by_annot_txt_type;
+DROP TYPE compare_run_by_annot_txt_type CASCADE;
 CREATE TYPE compare_run_by_annot_txt_type as (run_id VARCHAR, name VARCHAR, value VARCHAR);
 
 CREATE OR REPLACE FUNCTION compare_run_by_annot_txt(name VARCHAR)
 RETURNS SETOF compare_run_by_annot_txt_type
 AS $$
-    SELECT fun_call.run_id, annot_ds_text.name, annot_ds_text.value
-    FROM   annot_ds_text,ds_io,ds_cont,fun_call
-    WHERE  annot_ds_text.ds_id=ds_cont.in_id AND ds_cont.out_id=ds_io.ds_id AND
-           ds_io.fun_call_id=fun_call.id AND annot_ds_text.name=$1
+    SELECT fun_call.run_id, annot_dataset_text.name, annot_dataset_text.value
+    FROM   annot_dataset_text,dataset_io,dataset_containment,fun_call
+    WHERE  annot_dataset_text.dataset_id=dataset_containment.in_id AND dataset_containment.out_id=dataset_io.dataset_id AND
+           dataset_io.function_call_id=fun_call.id AND annot_dataset_text.name=$1
   UNION
-    SELECT fun_call.run_id, annot_ds_text.name, annot_ds_text.value 
-    FROM   fun_call, ds_io, annot_ds_text
-    WHERE  fun_call.id=ds_io.fun_call_id and ds_io.ds_id=annot_ds_text.ds_id and
-           annot_ds_text.name=$1
+    SELECT fun_call.run_id, annot_dataset_text.name, annot_dataset_text.value 
+    FROM   fun_call, dataset_io, annot_dataset_text
+    WHERE  fun_call.id=dataset_io.function_call_id and dataset_io.dataset_id=annot_dataset_text.dataset_id and
+           annot_dataset_text.name=$1
   UNION
-    SELECT fun_call.run_id, annot_fun_call_text.name, annot_fun_call_text.value 
-    FROM   fun_call, annot_fun_call_text
-    WHERE  fun_call.id=annot_fun_call_text.fun_call_id and annot_fun_call_text.name=$1
+    SELECT fun_call.run_id, annot_function_call_text.name, annot_function_call_text.value 
+    FROM   fun_call, annot_function_call_text
+    WHERE  fun_call.id=annot_function_call_text.function_call_id and annot_function_call_text.name=$1
   UNION
-    SELECT run.id as run_id, annot_run_text.name, annot_run_text.value 
-    FROM   run, annot_run_text
-    WHERE  run.id=annot_run_text.run_id and annot_run_text.name=$1
+    SELECT run.id as run_id, annot_script_run_text.name, annot_script_run_text.value 
+    FROM   run, annot_script_run_text
+    WHERE  run.id=annot_script_run_text.script_run_id and annot_script_run_text.name=$1
 $$ LANGUAGE SQL;
 
-DROP TYPE compare_run_by_key_text_type;
+DROP TYPE compare_run_by_key_text_type CASCADE;
 CREATE TYPE compare_run_by_key_text_type as (run_id VARCHAR, name VARCHAR, value VARCHAR);
 
 CREATE OR REPLACE FUNCTION compare_run_by_key_text(name VARCHAR)
 RETURNS SETOF compare_run_by_key_text_type
 AS $$
-    SELECT fun_call.run_id, annot_ds_text.name, annot_ds_text.value
-    FROM   annot_ds_text,ds_io,ds_cont,fun_call
-    WHERE  annot_ds_text.ds_id=ds_cont.in_id AND ds_cont.out_id=ds_io.ds_id AND
-           ds_io.fun_call_id=fun_call.id AND annot_ds_text.name=$1
+    SELECT fun_call.run_id, annot_dataset_text.name, annot_dataset_text.value
+    FROM   annot_dataset_text,dataset_io,dataset_containment,fun_call
+    WHERE  annot_dataset_text.dataset_id=dataset_containment.in_id AND dataset_containment.out_id=dataset_io.dataset_id AND
+           dataset_io.function_call_id=fun_call.id AND annot_dataset_text.name=$1
   UNION
-    SELECT fun_call.run_id, annot_ds_text.name, annot_ds_text.value 
-    FROM   fun_call, ds_io, annot_ds_text
-    WHERE  fun_call.id=ds_io.fun_call_id and ds_io.ds_id=annot_ds_text.ds_id and
-           annot_ds_text.name=$1
+    SELECT fun_call.run_id, annot_dataset_text.name, annot_dataset_text.value 
+    FROM   fun_call, dataset_io, annot_dataset_text
+    WHERE  fun_call.id=dataset_io.function_call_id and dataset_io.dataset_id=annot_dataset_text.dataset_id and
+           annot_dataset_text.name=$1
   UNION
-    SELECT fun_call.run_id, annot_fun_call_text.name, annot_fun_call_text.value 
-    FROM   fun_call, annot_fun_call_text
-    WHERE  fun_call.id=annot_fun_call_text.fun_call_id and annot_fun_call_text.name=$1
+    SELECT fun_call.run_id, annot_function_call_text.name, annot_function_call_text.value 
+    FROM   fun_call, annot_function_call_text
+    WHERE  fun_call.id=annot_function_call_text.function_call_id and annot_function_call_text.name=$1
   UNION
-    SELECT run.id as run_id, annot_run_text.name, annot_run_text.value 
-    FROM   run, annot_run_text
-    WHERE  run.id=annot_run_text.run_id and annot_run_text.name=$1
+    SELECT run.id as run_id, annot_script_run_text.name, annot_script_run_text.value 
+    FROM   run, annot_script_run_text
+    WHERE  run.id=annot_script_run_text.script_run_id and annot_script_run_text.name=$1
 $$ LANGUAGE SQL;
 
 -- CREATE OR REPLACE FUNCTION compare_run_by_annot_num(name VARCHAR)
@@ -162,23 +162,23 @@
 --   value NUMERIC
 -- )
 -- AS $$
---     SELECT fun_call.workflow_id, annot_ds_num.name, annot_ds_num.value
---     FROM   annot_ds_num,ds_usage,ds_containment,fun_call
---     WHERE  annot_ds_num.id=ds_containment.in_id AND ds_containment.out_id=ds_usage.dataset_id AND
---            ds_usage.fun_call_id=fun_call.id AND annot_ds_num.name=$1
+--     SELECT fun_call.workflow_id, annot_dataset_num.name, annot_dataset_num.value
+--     FROM   annot_dataset_num,dataset_io,dataset_containment,fun_call
+--     WHERE  annot_dataset_num.id=dataset_containment.in_id AND dataset_containment.out_id=dataset_io.dataset_id AND
+--            dataset_io.function_call_id=fun_call.id AND annot_dataset_num.name=$1
 --   UNION
---     SELECT fun_call.workflow_id, annot_ds_num.name, annot_ds_num.value 
---     FROM   fun_call, ds_usage, annot_ds_num
---     WHERE  fun_call.id=ds_usage.fun_call_id and ds_usage.dataset_id=annot_ds_num.id and
---            annot_ds_num.name=$1
+--     SELECT fun_call.workflow_id, annot_dataset_num.name, annot_dataset_num.value 
+--     FROM   fun_call, dataset_io, annot_dataset_num
+--     WHERE  fun_call.id=dataset_io.function_call_id and dataset_io.dataset_id=annot_dataset_num.id and
+--            annot_dataset_num.name=$1
 --   UNION
 --     SELECT fun_call.workflow_id, annot_p_num.name, annot_p_num.value 
 --     FROM   fun_call, annot_p_num
 --     WHERE  fun_call.id=annot_p_num.id and annot_p_num.name=$1
 --   UNION
---     SELECT workflow.id as workflow_id, annot_wf_num.name, annot_wf_num.value 
+--     SELECT script_run.id as workflow_id, annot_wf_num.name, annot_wf_num.value 
 --     FROM   workflow, annot_wf_num
---     WHERE  workflow.id=annot_wf_num.id and annot_wf_num.name=$1
+--     WHERE  script_run.id=annot_wf_num.id and annot_wf_num.name=$1
 -- $$ LANGUAGE SQL;
 
 
@@ -188,57 +188,57 @@
 --   name VARCHAR, 
 --   value VARCHAR) 
 -- AS $$
---     SELECT   fun_call.workflow_id, annot_ds_txt.name, annot_ds_txt.value 
---     FROM     fun_call, ds_usage, annot_ds_txt
---     WHERE    fun_call.id=ds_usage.fun_call_id and ds_usage.dataset_id=annot_ds_txt.id and
---              annot_ds_txt.name=$1
+--     SELECT   fun_call.workflow_id, annot_dataset_txt.name, annot_dataset_txt.value 
+--     FROM     fun_call, dataset_io, annot_dataset_txt
+--     WHERE    fun_call.id=dataset_io.function_call_id and dataset_io.dataset_id=annot_dataset_txt.id and
+--              annot_dataset_txt.name=$1
 --   UNION
 --     SELECT   fun_call.workflow_id, annot_p_txt.name, annot_p_txt.value 
 --     FROM     fun_call, annot_p_txt
 --     WHERE    fun_call.id=annot_p_txt.id and annot_p_txt.name=$1
 --   UNION
---     SELECT   workflow.id as workflow_id, annot_wf_txt.name, annot_wf_txt.value 
+--     SELECT   script_run.id as workflow_id, annot_wf_txt.name, annot_wf_txt.value 
 --     FROM     workflow, annot_wf_txt
---     WHERE    workflow.id=annot_wf_txt.id and annot_wf_txt.name=$1
+--     WHERE    script_run.id=annot_wf_txt.id and annot_wf_txt.name=$1
 -- $$ LANGUAGE SQL;
 
 
-CREATE OR REPLACE FUNCTION compare_run_by_annot_bool(name VARCHAR)
-RETURNS TABLE (
-  workflow_id VARCHAR,
-  name VARCHAR, 
-  value BOOLEAN
-) 
-AS $$
-    SELECT   fun_call.workflow_id, annot_ds_bool.name, annot_ds_bool.value 
-    FROM     fun_call, ds_usage, annot_ds_bool
-    WHERE    fun_call.id=ds_usage.fun_call_id and ds_usage.dataset_id=annot_ds_bool.id and
-             annot_ds_bool.name=$1
-  UNION
-    SELECT   fun_call.workflow_id, annot_p_bool.name, annot_p_bool.value 
-    FROM     fun_call, annot_p_bool
-    WHERE    fun_call.id=annot_p_bool.id and annot_p_bool.name=$1
-  UNION
-    SELECT   workflow.id as workflow_id, annot_wf_bool.name, annot_wf_bool.value 
-    FROM     workflow, annot_wf_bool
-    WHERE    workflow.id=annot_wf_bool.id and annot_wf_bool.name=$1
-$$ LANGUAGE SQL;
+--CREATE OR REPLACE FUNCTION compare_run_by_annot_bool(name VARCHAR)
+--RETURNS TABLE (
+--  workflow_id VARCHAR,
+--  name VARCHAR, 
+--  value BOOLEAN
+--) 
+--AS $$
+--    SELECT   fun_call.workflow_id, annot_dataset_bool.name, annot_dataset_bool.value 
+--    FROM     fun_call, dataset_io, annot_dataset_bool
+--    WHERE    fun_call.id=dataset_io.function_call_id and dataset_io.dataset_id=annot_dataset_bool.id and
+--             annot_dataset_bool.name=$1
+--  UNION
+--    SELECT   fun_call.workflow_id, annot_p_bool.name, annot_p_bool.value 
+--    FROM     fun_call, annot_p_bool
+--    WHERE    fun_call.id=annot_p_bool.id and annot_p_bool.name=$1
+--  UNION
+--    SELECT   script_run.id as workflow_id, annot_wf_bool.name, annot_wf_bool.value 
+--    FROM     workflow, annot_wf_bool
+--   WHERE    script_run.id=annot_wf_bool.id and annot_wf_bool.name=$1
+--$$ LANGUAGE SQL;
 
 
 -- correlate a parameter with workflow runtime statistics
 CREATE OR REPLACE FUNCTION correlate_parameter_runtime(parameter_name VARCHAR) 
 RETURNS TABLE (
-    workflow VARCHAR,  
-    workflow_starttime TIMESTAMP WITH TIME ZONE, 
-    workflow_duration NUMERIC, 
+    run VARCHAR,  
+    starttime TIMESTAMP WITH TIME ZONE, 
+    duration NUMERIC, 
     parameter VARCHAR, 
-    parameter_value VARCHAR
+    value VARCHAR
 ) 
 AS $$
-	SELECT workflow.id,to_timestamp(workflow.start_time),workflow.duration,ds_usage.parameter_name,variable.value
-	FROM   variable,ds_usage,fun_call,workflow
-	WHERE  variable.id=ds_usage.dataset_id AND ds_usage.fun_call_id=fun_call.id AND 
-	       fun_call.workflow_id=workflow.id AND ds_usage.param_name=$1 
+	SELECT script_run.id,script_run.start_time,script_run.duration,dataset_io.parameter,dataset.value
+	FROM   dataset,dataset_io,fun_call,script_run
+	WHERE  dataset.id=dataset_io.dataset_id AND dataset_io.function_call_id=fun_call.id AND 
+	       fun_call.run_id=script_run.id AND dataset_io.parameter=$1 
 $$ LANGUAGE SQL;
 
 -- recursive query to find ancestor entities in a provenance graph
@@ -247,13 +247,13 @@
   WITH RECURSIVE anc(ancestor,descendant) AS
   (    
        SELECT parent AS ancestor, child AS descendant 
-       FROM   parent_of 
+       FROM   provenance_graph_edge 
        WHERE child=$1
      UNION
-       SELECT parent_of.parent AS ancestor, 
+       SELECT provenance_graph_edge.parent AS ancestor, 
               anc.descendant AS descendant
-       FROM   anc,parent_of
-       WHERE  anc.ancestor=parent_of.child
+       FROM   anc, provenance_graph_edge
+       WHERE  anc.ancestor=provenance_graph_edge.child
   )
   SELECT ancestor FROM anc
 $$ LANGUAGE SQL;
@@ -280,15 +280,13 @@
       function_name := 'compare_run_by_parameter';
     WHEN 'annot_num' THEN
       function_name := 'compare_run_by_annot_num';
-    WHEN 'annot_txt' THEN
-      function_name := 'compare_run_by_annot_txt';
-    WHEN 'annot_bool' THEN
-      function_name := 'compare_run_by_annot_bool';
+    WHEN 'annot_text' THEN
+      function_name := 'compare_run_by_annot_text';
     END CASE;
     IF i = 1 THEN
       fromq := function_name || '(''' || property || ''') as t' || i;
     ELSE
-      fromq := fromq || ' INNER JOIN ' || function_name || '(''' || property || ''') as t' || i || ' USING (workflow_id)';
+      fromq := fromq || ' INNER JOIN ' || function_name || '(''' || property || ''') as t' || i || ' USING (run_id)';
     END IF;
   END LOOP;
   q := selectq || ' FROM ' || fromq;

Modified: provenancedb/prepare-provenance-chart
===================================================================
--- provenancedb/prepare-provenance-chart	2012-05-09 22:18:28 UTC (rev 5785)
+++ provenancedb/prepare-provenance-chart	2012-05-10 12:27:54 UTC (rev 5786)
@@ -1,52 +1,22 @@
 #!/bin/bash
 
-# invent an ID. doesn't need to have any meaning. needs to be unique.
-# uuidgen for now.
-# any time we ouptut a thread, prefix the WFID on the front.
-# should we do this in the logging code in swift proper? - it would be a
-# nicer way of getting the runid into the URI
-
 export RUNID=$(basename $1 .log)
-
-export WFID="execute:${RUNID}:"
-export EXECUTE2PREFIX="execute2:${RUNID}:"
-
-# will output log information about datasets from a log file passed as $1
-
+export WFID="${RUNID}:"
+export EXECUTE2PREFIX="${RUNID}:"
 cat $1 | grep ' PARAM ' | sed "s/^.* thread=\([^ ]*\).*direction=\([^ ]*\).*variable=\([^ ]*\).*provenanceid=\([^ ]*\).*\$/${WFID}\1 \2 \4 \3/" > tie-data-invocs.txt
-
-# 2007-12-13 14:29:13,967+0000 INFO  AbstractDataNode dataset 2461363 has child 938665
 cat $1 | grep ' CONTAINMENT ' | sed 's/^.*parent=\([^ ]*\) child=\([^ ]*\)$/\1 \2/' > tie-containers.txt
-
-#AbstractDataNode dataset 3814442 has filename file://localhost/q.out
 cat $1 | grep ' FILENAME ' | sed 's/^.*dataset=\([^ ]*\) filename=\([^ ]*\).*$/\1 \2/' | sort | uniq > dataset-filenames.txt
-
 cat $1 | grep ' VALUE ' | sed 's/^.*dataset=\([^ ]*\) VALUE=\(.*\)$/\1 \2/' | sort | uniq > dataset-values.txt
-
 cat $1 | grep ' PROCEDURE ' | sed "s/^.*thread=\([^ ]*\) name=\([^ ]*\)\$/${WFID}\1 \2/" > invocation-procedure-names.txt
-
-info-to-extrainfo > extrainfo.txt
-
-info-to-runtime > runtime.txt
-
 cat $1 | grep ' OPERATOR ' | sed 's/^.*thread=\([^ ]*\) operator="\([^ ]*\)" lhs=\([^ ]*\) rhs=\([^ ]*\) result=\([^ ]*\).*$/\1 \2 \3 \4 \5/' > operators.txt
-
-
-# 2009-03-19 19:15:35,244+0100 INFO  vdl:arguments FUNCTION id=88000-0-4-4 name="f ilename" result=dataset:20090319-1915-xj8flg 13:720000000060
-# 2009-03-19 19:15:35,246+0100 INFO  vdl:arguments FUNCTIONPARAMETER id=88001-0-4- 4 input=dataset:20090319-1915-xj8flg13:72000 0000058
-
 cat $1 | grep ' FUNCTION ' | sed "s/^.*id=\([^ ]*\) name=\([^ ]*\) result=\([^ ]*\).*\$/$WFID\1 \2 \3/" > functions.txt
-# the IDs in functions.txt should be unique...
-# we could even test that...
-
 cat $1 | grep ' FUNCTIONPARAMETER ' | sed "s/^.*id=\([^ ]*\) input=\([^ ]*\).*\$/$WFID\1 \2/" > function-inputs.txt
-
 cat $1 | grep ' CREATEARRAY START ' | sed 's/^.* array=\([^ ]*\).*$/\1/' > createarray.txt
-
 cat $1 | grep ' CREATEARRAY MEMBER ' | sed 's/^.* array=\([^ ]*\) index=\([^ ]*\) member=\([^ ]*\).*$/\1 \2 \3/' > createarray-members.txt
-
 cat $1 | grep ' ARRAYRANGE ' | sed 's/^.* thread=\([^ ]*\).*$/\1/' > arrayrange.txt
-
-# 2009-05-18 21:19:20,295+0200 INFO  vdl:mains SCOPE thread=0-6-5-1-5
-
 cat $1 | grep ' SCOPE ' | sed 's/^.* thread=\([^ ]*\).*/\1/' > scopes.txt
+awk '/BEGIN SWIFTSCRIPT/,/END SWIFTSCRIPT/{if (!/BEGIN SWIFTSCRIPT/&&!/END SWIFTSCRIPT/)print}' $1 > script.txt
+awk '/BEGIN SITES/,/END SITES/{if (!/BEGIN SITES/&&!/END SITES/)print}' $1 > sites.txt
+awk '/BEGIN TC/,/END TC/{if (!/BEGIN TC/&&!/END TC/)print}' $1 > tc.txt
+info-to-extrainfo > extrainfo.txt
+info-to-runtime > runtime.txt

Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql	2012-05-09 22:18:28 UTC (rev 5785)
+++ provenancedb/prov-init.sql	2012-05-10 12:27:54 UTC (rev 5786)
@@ -1,46 +1,52 @@
 -- this is the schema definition used for the main relational provenance
 -- implementation (in both sqlite3 and postgres)
 
--- base relations
-drop table app_catalog cascade;
-drop table site_catalog cascade;
-drop table script cascade;
-drop table run cascade;
-drop table fun_call cascade;
-drop table app_fun_call cascade;
-drop table app_exec cascade;
+drop view  script_run;
+drop view  function_call;
+drop view  application_execution;
+drop view  runtime_info;
+drop view  dataset;
+drop view  dataset_io;
+drop view  provenance_graph_edge;
+drop table annot_script_run_num cascade;
+drop table annot_script_run_text cascade;
+drop table annot_function_call_num cascade;
+drop table annot_function_call_text cascade;
+drop table annot_app_exec_num cascade;
+drop table annot_app_exec_text cascade;
+drop table annot_dataset_num cascade;
+drop table annot_dataset_text cascade;
 drop table rt_info cascade;
-drop table ds cascade;
+drop table app_exec cascade;
+drop table app_fun_call cascade;
+drop table dataset_in cascade;
+drop table dataset_out cascade;
+drop table fun_call cascade;
+drop table run cascade;
+drop table tc_file cascade;
+drop table sites_file cascade;
+drop table script cascade;
 drop table mapped cascade;
 drop table primitive cascade;
-drop table ds_containment cascade;
-drop table ds_in cascade;
-drop table ds_out cascade;
-drop table annot_run_num cascade;
-drop table annot_run_text cascade;
-drop table annot_fun_call_num cascade;
-drop table annot_fun_call_text cascade;
-drop table annot_app_exec_num cascade;
-drop table annot_app_exec_text cascade;
-drop table annot_ds_num cascade;
-drop table annot_ds_text cascade;
+drop table dataset_containment cascade;
+drop table ds cascade;
 
 -- application_catalog stores tc.file
-create table app_catalog (
+create table tc_file (
        hash_value		 varchar(256) primary key,
        content			 text
 );
 
 -- application_catalog stores tc.file
-create table site_catalog (
-	hash_value		varchar(256) primary key,
-	content			text
+create table sites_file (
+	hash_value      varchar(256) primary key,
+	content         text
 );
-	
+
 -- script stores Swift script source codes
 create table script (
-	hash_value		varchar(256) primary key,
-	content			text
+	hash_value      varchar(256) primary key,
+	content         text
 );
 
 -- run stores information about each script run log that has
@@ -48,122 +54,134 @@
 -- status.
 -- Might be interesting to store xml translation of the Swift script
 -- here for prospective provenance and versioning.
-create table run
-    (
-     id				varchar(256) primary key,
-     log_filename 		varchar(2048),
-     swift_version 		varchar(16),
-     cog_version   		varchar(16),
-     final_state   		varchar(32),
-     start_time    		numeric,
-     duration      		numeric,
-     script_filename		varchar(2048),
-     script_hash		varchar(256) references script (hash_value),
-     application_catalog_hash	varchar(256) references app_catalog (hash_value),
-     site_catalog_hash	   	varchar(256) references site_catalog (hash_value)
+create table run (
+     id                 varchar(256) primary key,
+     log_filename       varchar(2048),
+     swift_version      varchar(16),
+     cog_version        varchar(16),
+     final_state        varchar(32),
+     start_time         numeric,
+     duration           numeric,
+     script_filename    varchar(2048),
+     script_hash        varchar(256) references script (hash_value),
+     tc_file_hash   	varchar(256) references tc_file (hash_value),
+     sites_file_hash    varchar(256) references sites_file (hash_value)
 );
 
+create view script_run as
+  select id, log_filename, swift_version, cog_version, final_state, 
+         to_timestamp(start_time) as start_time, duration, script_filename,
+	 script_hash, tc_file_hash, sites_file_hash
+  from   run;
+
 -- process gives information about each process (in the OPM sense)
 -- it is augmented by information in other tables
 -- specifies the type of process. for any type, it
 -- must be the case that the specific type table
 -- has an entry for this process.
 -- process types: internal, rootthread, execute, function, compound, scope, operator
-create table fun_call
-    (
-     id     varchar(256), 
-     run_id varchar(256) references run (id) on delete cascade,  
-     type   varchar(16),
-     name   varchar(256), -- in the case of an execute this refers to the transformation name in tc.data
-     primary key (id, run_id)
-									   									      	 
+create table fun_call (
+     id      	    varchar(256) primary key, 
+     run_id  	    varchar(256) references run (id) on delete cascade,  
+     type    	    varchar(16),
+     name    	    varchar(256) -- in the case of an execute this refers to the transformation name in tc.data
 );
 
 -- this gives information about each execute.
 -- each execute is identified by a unique URI. other information from
 -- swift logs is also stored here. an execute is an OPM process.
-create table app_fun_call
-    (
-     id			varchar(256),  
-     run_id		varchar(256),
-     name      		varchar(256), -- name of the app procedure that invokes the transformation
-     start_time	    	numeric,
-     duration		numeric,
-     final_state	varchar(32),
-     scratch		varchar(2048),
-     foreign key (id, run_id) references fun_call,
-     primary key (id, run_id)
+create table app_fun_call (
+     id             varchar(256) primary key references fun_call (id),  
+     name      	    varchar(256), -- name of the app procedure that invokes the transformation
+     start_time     numeric,
+     duration       numeric,
+     final_state    varchar(32),
+     scratch        varchar(2048)
 );
 
+create view function_call as 
+    select fun_call.id, fun_call.name, fun_call.type, app_fun_call.name as app_catalog_name, fun_call.run_id as script_run_id,  
+           to_timestamp(app_fun_call.start_time) as start_time, app_fun_call.duration, app_fun_call.final_state, app_fun_call.scratch
+    from
+      fun_call
+    left outer join
+      app_fun_call 
+    on fun_call.id=app_fun_call.id;
+
 -- this gives information about each application execution attempt, including
 -- aggregate resource consumption. the app_exec_id is tied to per-execution-attempt
 -- information such as wrapper logs
-create table app_exec
-    (
-     id                varchar(256),
-     app_fun_call_id   varchar(256), 
-     run_id	       varchar(256),
+create table app_exec (
+     id                varchar(256) primary key,
+     app_fun_call_id   varchar(256) references app_fun_call (id), 
      start_time        numeric,
      duration          numeric,
      final_state       varchar(32),
-     site              varchar(256),
-     maxrss	       numeric,
-     walltime	       numeric,
-     systime	       numeric,
-     usertime	       numeric,
-     cpu	       numeric,
-     fsin	       numeric,
-     fsout	       numeric,
-     timesswpd         numeric,
-     socketrecv	       numeric,
-     socketsent	       numeric,
-     majpfaults        numeric,
-     minpfaults        numeric,
-     ctxswinv	       numeric,
-     ctxswvol	       numeric,
-     foreign key(app_fun_call_id, run_id) references app_fun_call,
-     primary key(id, app_fun_call_id, run_id)
+     site              varchar(256)
 );
+--     maxrss            numeric,
+--     walltime          numeric,
+--     systime           numeric,
+--     usertime          numeric,
+--     cpu               numeric,
+--     fsin              numeric,
+--     fsout             numeric,
+--     timesswpd         numeric,
+--     socketrecv        numeric,
+--     socketsent        numeric,
+--     majpfaults        numeric,
+--     minpfaults        numeric,
+--     ctxswinv          numeric,
+--     ctxswvol          numeric,
 
+create view application_execution as
+  select id, app_fun_call_id as function_call_id, to_timestamp(start_time) as start_time, duration, final_state, site
+  from   app_exec;
+
+
 -- app execution runtime info extracted from the /proc filesystem (assumes the app executed
 -- in a Linux host) 
-create table rt_info
-   ( 
-     app_exec_id        varchar(256), 
-     app_fun_call_id    varchar(256), 
-     run_id	        varchar(256),    
+create table rt_info ( 
+     app_exec_id        varchar(256) references app_exec (id), 
      timestamp		numeric,
      cpu_usage          numeric,
      max_phys_mem	numeric,
      max_virt_mem	numeric,
      io_read		numeric,
      io_write		numeric,
-     foreign key (app_exec_id, app_fun_call_id, run_id) references app_exec,
-     primary key (app_exec_id, app_fun_call_id, run_id, timestamp)
+     primary key (app_exec_id, timestamp)
 );
 
+create view runtime_info as
+  select app_exec_id, 
+	 to_timestamp(timestamp) as timestamp, 
+	 cpu_usage, 
+	 max_phys_mem, 
+	 max_virt_mem, 
+	 io_read, 
+	 io_write
+  from rt_info;
+
 -- ds stores all dataset identifiers.
-create table ds
-    (
+create table ds (
       id	 varchar(256) primary key
-    );
+);
 
 -- file stores the filename mapped to each dataset. 
-create table mapped
-    ( 
+create table mapped ( 
       id	 varchar(256) primary key references ds (id) on delete cascade,
       filename   varchar(2048)
-    );
+);
 
 -- dataset_values stores the value for each dataset which is known to have
 -- a value (which is all assigned primitive types). No attempt is made here
 -- to expose that value as an SQL type other than a string, and so (for
 -- example) SQL numerical operations should not be expected to work, even
 -- though the user knows that a particular dataset stores a numeric value.
-create table primitive
-    ( id    varchar(256) primary key references ds (id) on delete cascade,
-      value varchar(2048)
-    );
+create table primitive ( 
+       id    varchar(256) primary key references ds (id) on delete cascade,
+       value varchar(2048)
+);
 
 -- dataset_containment stores the containment hierarchy between
 -- container datasets (arrays and structs) and their contents.
@@ -172,215 +190,108 @@
 -- constructors and accessors, rather than, or in addition to,
 -- a containment hierarchy. The relationship (such as array index or
 -- structure member name) should also be stored in this table.
-create table ds_containment
-    ( out_id varchar(256) references ds (id) on delete cascade,
-      in_id  varchar(256) references ds (id) on delete cascade,
-      primary key (out_id,in_id)
-    );
+create table dataset_containment ( 
+       out_id varchar(256) references ds (id) on delete cascade,
+       in_id  varchar(256) references ds (id) on delete cascade,
+       primary key (out_id,in_id)
+);
 
+create view dataset as 
+    select mapped.id, 'mapped' as type, mapped.filename, null as value
+    from mapped
+  union all 
+    select primitive.id, 'primitive' as type, null as filename, primitive.value
+    from primitive
+  union all
+    select dataset_containment.out_id as id, 'composite' as type, null as filename, null as value 
+    from dataset_containment;
 
 -- dataset_usage records usage relationships between processes and datasets;
 -- in SwiftScript terms, the input and output parameters for each
 -- application procedure invocation; in OPM terms, the artificts which are
 -- input to and output from each process that is a Swift execution
-create table ds_in
-    (
-     fun_call_id	varchar(256), 
-     run_id		varchar(256),
-     ds_id   		varchar(256) references ds (id) on delete cascade,
-     parameter   	varchar(256), -- the name of the parameter in this execute that
-                             	      -- this dataset was bound to. sometimes this must
-                              	      -- be contrived (for example, in positional varargs)
-     foreign key (fun_call_id, run_id) references fun_call,
-     primary key (fun_call_id, run_id, ds_id, parameter)
-    );
+create table dataset_in (
+       function_call_id	varchar(256) references fun_call (id), 
+       dataset_id   	varchar(256) references ds (id) on delete cascade,
+       parameter   	varchar(256), -- the name of the parameter in this execute that
+                             	  -- this dataset was bound to. sometimes this must
+                              	  -- be contrived (for example, in positional varargs)
+       primary key (function_call_id, dataset_id, parameter)
+ );
 
-create table ds_out
-    (
-     fun_call_id	varchar(256), 
-     run_id		varchar(256),
-     ds_id   		varchar(256) references ds (id) on delete cascade,
-     parameter   	varchar(256), -- the name of the parameter in this execute that
-                              	      -- this dataset was bound to. sometimes this must
-                              	      -- be contrived (for example, in positional varargs)
-     foreign key (fun_call_id, run_id) references fun_call,
-     primary key (fun_call_id, run_id, ds_id, parameter)
-    );
+create table dataset_out (
+       function_call_id	varchar(256) references fun_call (id), 
+       dataset_id   	varchar(256) references ds (id) on delete cascade,
+       parameter   	varchar(256), -- the name of the parameter in this execute that
+                              	  -- this dataset was bound to. sometimes this must
+                              	  -- be contrived (for example, in positional varargs)
+       primary key (function_call_id, dataset_id, parameter)
+);
 
+create view dataset_io as
+ select dataset_in.function_call_id, dataset_in.dataset_id, dataset_in.parameter, 'I' as type
+ from   dataset_in
+union all 
+ select dataset_out.function_call_id, dataset_out.dataset_id, dataset_out.parameter, 'O' as type
+ from   dataset_out;
 
--- annotations 
-create table annot_ds_num
-   ( ds_id varchar(256) references ds (id) on delete cascade, 
-     name  varchar(256),
-     value numeric,
-     primary key (ds_id, name)
-   );
+create table annot_script_run_num ( 
+       script_run_id    varchar(256) references run (id) on delete cascade, 
+       name      varchar(256),
+       value     numeric,
+       primary key (script_run_id, name)
+);
 
-create table annot_ds_text
-   ( ds_id varchar(256) references ds (id) on delete cascade, 
-     name  varchar(256),
-     value varchar(2048),
-     primary key (ds_id, name)
-   );
-
-create table annot_fun_call_num
-   ( fun_call_id	varchar(256), 
-     run_id	        varchar(256),    
-     name       	varchar(256),
-     value      	numeric,
-     foreign key (fun_call_id, run_id) references fun_call,
-     primary key (fun_call_id, run_id, name)
-   );
-
-create table annot_fun_call_text
-   ( fun_call_id	varchar(256), 
-     run_id	        varchar(256),    
-     name       	varchar(256),
-     value      	varchar(2048),
-     foreign key (fun_call_id, run_id) references fun_call,
-     primary key (fun_call_id, run_id, name)
-   );
-
-create table annot_run_num
-   ( run_id    varchar(256) references run (id) on delete cascade, 
+create table annot_script_run_text ( script_run_id    varchar(256) references run (id) on delete cascade, 
      name      varchar(256),
-     value     numeric,
-     primary key (run_id, name)
-   );
-
-create table annot_run_text
-   ( run_id    varchar(256) references run (id) on delete cascade, 
-     name      varchar(256),
      value     varchar(2048),
-     primary key (run_id, name)
-   );
+     primary key (script_run_id, name)
+);
 
-create table annot_app_exec_num
-   ( id			varchar(256), 
-     app_fun_call_id    varchar(256), 
-     run_id	        varchar(256),    
-     name      		varchar(256),
-     value     		numeric,
-     foreign key (id, app_fun_call_id, run_id) references app_exec,
-     primary key (id, app_fun_call_id, run_id, name)
-   );
+create table annot_function_call_num ( 
+       function_call_id	varchar(256) references fun_call (id) on delete cascade, 
+       name       	varchar(256),
+       value      	numeric,
+       primary key (function_call_id, name)
+);
 
-create table annot_app_exec_text
-   ( id			varchar(256), 
-     app_fun_call_id    varchar(256), 
-     run_id	        varchar(256),    
-     name      		varchar(256),
-     value     		varchar(2048),
-     foreign key (id, app_fun_call_id, run_id) references app_exec,
-     primary key (id, app_fun_call_id, run_id, name)
-   );
+create table annot_function_call_text ( 
+       function_call_id	varchar(256) references fun_call (id) on delete cascade, 
+       name       	varchar(256),
+       value      	varchar(2048),
+       primary key (function_call_id, name)
+);
 
+create table annot_app_exec_num ( 
+       app_exec_id            varchar(256) references app_exec (id) on delete cascade, 
+       name      		    varchar(256),
+       value     		    numeric,
+       primary key (app_exec_id, name)
+);
 
--- create table iq
---    ( idx      serial primary key,
---     q        varchar(2048)
---   );
-drop view ds_io;
-create view ds_io as
- select ds_in.fun_call_id as function_call_id, ds_in.ds_id as variable_id, ds_in.parameter
- from   ds_in
-union all 
- select ds_out.fun_call_id as function_call_id, ds_out.ds_id as variable_id, ds_out.parameter
- from   ds_out;
+create table annot_app_exec_text ( 
+       app_exec_id            varchar(256) references app_exec (id) on delete cascade, 
+       name      		    varchar(256),
+       value     		    varchar(2048),
+       primary key (app_exec_id, name)
+);
 
-drop view pgraph_edge;
-create view pgraph_edge as 
-       select fun_call_id as parent,ds_id as child from ds_out
-       union all
-       select ds_id as parent,fun_call_id as child from ds_in
-       union all
-       select out_id as parent,in_id as child from ds_containment;
+create table annot_dataset_num ( 
+       dataset_id varchar(256) references ds (id) on delete cascade, 
+       name  varchar(256),
+       value numeric,
+       primary key (dataset_id, name)
+);
 
-drop view annot_text cascade;
-create view annot_text as
-    select *
-    from annot_run_text 
-  union all
-    select * 
-    from annot_ds_text 
-  union all 
-    select * 
-    from annot_fun_call_text;
+create table annot_dataset_text( 
+       dataset_id varchar(256) references ds (id) on delete cascade, 
+       name  varchar(256),
+       value varchar(2048),
+       primary key (dataset_id, name)
+);
 
-drop view annot_num cascade;
-create view annot_num as
-    select *
-    from annot_run_num 
-  union all
-    select * 
-    from annot_ds_num 
-  union all 
-    select * 
-    from annot_fun_call_num;
-
--- views used for queries based on the schema summary
-
-drop view function_call;
-
-create view function_call as 
-    select fun_call.id, fun_call.name as name, fun_call.type, app_fun_call.name as app_catalog_name, fun_call.run_id as script_run_id,  
-           to_timestamp(app_fun_call.start_time) as start_time, app_fun_call.duration, app_fun_call.final_state, app_fun_call.scratch 
-    from fun_call 
-    left outer join 
-    app_fun_call on fun_call.id=app_fun_call.id;
-
-drop view variable;
-
-create view variable as 
-    select mapped.id, 'mapped' as type, mapped.filename, null as value
-    from mapped
-  union all 
-    select primitive.id, 'primitive' as type, null as filename, primitive.value
-    from primitive
-  union all
-    select ds_containment.out_id as id, 'composite' as type, null as filename, null as value 
-    from ds_containment;
-
-drop view annotation;
-
-create view annotation as 
-    select annot_text.run_id as id, annot_text.name as key, annot_text.value as string_value, null as numeric_value
-    from annot_text
-  union all
-    select annot_num.run_id as id, annot_num.name as key, null as string_value, annot_num.value as numeric_value
-    from annot_num;
-   
-drop view script_run;
-
-create view script_run as
-  select id, log_filename, swift_version, cog_version, final_state, 
-         to_timestamp(start_time) as start_time, duration
-  from   run;
-
-drop view application_execution;
-
-create view application_execution as
-  select id, app_fun_call_id as function_call_id, to_timestamp(start_time) as start_time, duration, final_state, site
-  from   app_exec;
-
-drop view runtime_info;
-
-create view runtime_info as
-  select app_exec_id as application_execution_id, to_timestamp(timestamp) as timestamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write
-  from rt_info;
-
-drop view produces;
-
-create view produces as
-  select fun_call_id as function_call_id, ds_id as variable_id, parameter from ds_out;
-
-drop view consumes;
-
-create view consumes as
-  select fun_call_id as function_call_id, ds_id as variable_id, parameter from ds_in;
-
-drop view variable_containment;
-
-create view variable_containment as
-  select out_id as container, in_id as containee
-  from   ds_containment;
+create view provenance_graph_edge as 
+       select function_call_id as parent, dataset_id as child from dataset_out
+    union all
+       select dataset_id as parent, function_call_id as child from dataset_in;
+	

Modified: provenancedb/prov-to-sql.sh
===================================================================
--- provenancedb/prov-to-sql.sh	2012-05-09 22:18:28 UTC (rev 5785)
+++ provenancedb/prov-to-sql.sh	2012-05-10 12:27:54 UTC (rev 5786)
@@ -2,7 +2,7 @@
 
 export RUNID=$(basename $1 .log)
 
-export WFID="execute:${RUNID}:"
+export WFID="${RUNID}:"
 
 # TODO is there already a URI form for identifying workflows?
 export WF="${RUNID}"
@@ -16,16 +16,18 @@
 
 echo "    - Function calls."
 while read time duration thread localthread endstate tr_name scratch; do
-    echo "INSERT INTO fun_call (id, type, run_id) VALUES ('$localthread', 'execute', '$WF');"  >> /tmp/$RUNID.sql
-    echo "INSERT INTO app_fun_call (id, run_id, name, start_time, duration, final_state, scratch) VALUES ('$localthread', '$WF', '$tr_name', $time, $duration, '$endstate', '$scratch');"   >> /tmp/$RUNID.sql
+    id=$(echo "$thread" | sed "s/execute\://")
+    echo "INSERT INTO fun_call (id, type, run_id) VALUES ('$id', 'execute', '$WF');"  >> /tmp/$RUNID.sql
+    echo "INSERT INTO app_fun_call (id, name, start_time, duration, final_state, scratch) VALUES ('$id', '$tr_name', $time, $duration, '$endstate', '$scratch');"   >> /tmp/$RUNID.sql
 done < execute.global.event
 
 echo "    - Application executions."
 while read start_time duration globalid id endstate thread site scratch; do
     # cut off the last component of the thread, so that we end up at the
     # parent thread id which should correspond with the execute-level ID
-    inv_id="$(echo $thread | sed 's/-[^-]*$//')"
-    echo  "INSERT INTO app_exec (id, app_fun_call_id, run_id, start_time, duration, final_state, site) VALUES ('$id', '$inv_id', '$WF', $start_time, $duration, '$endstate', '$site');"  >> /tmp/$RUNID.sql
+    inv_id="$WFID$(echo $thread | sed 's/-[^-]*$//')"
+    eid=$(echo "$globalid" | sed "s/execute2\://")   
+    echo  "INSERT INTO app_exec (id, app_fun_call_id, start_time, duration, final_state, site) VALUES ('$eid', '$inv_id', $start_time, $duration, '$endstate', '$site');"  >> /tmp/$RUNID.sql
 done < execute2.global.event
 
 echo "    - Mapped variables."
@@ -44,10 +46,11 @@
 while read outer inner; do
     echo  "INSERT INTO ds (id) VALUES ('$outer');"  >> /tmp/$RUNID.sql
     echo  "INSERT INTO ds (id) VALUES ('$inner');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO ds_containment (out_id, in_id) VALUES ('$outer', '$inner');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO fun_call (id, type, name, run_id) VALUES ('constructor:$outer', 'constructor', 'constructor', '$WF');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO ds_in (fun_call_id, ds_id, parameter) VALUES ('constructor:$outer', '$inner', 'element');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO ds_out (fun_call_id, ds_id, parameter) VALUES ('constructor:$outer', '$outer', 'collection');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO dataset_containment (out_id, in_id) VALUES ('$outer', '$inner');"  >> /tmp/$RUNID.sql
+    cid=$(echo $outer | awk -F "-" '{print $3}')
+    echo  "INSERT INTO fun_call (id, type, name, run_id) VALUES ('${WFID}$cid', 'constructor', 'constructor', '$WF');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO dataset_in (function_call_id, dataset_id, parameter) VALUES ('${WFID}$cid', '$inner', 'element');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO dataset_out (function_call_id, dataset_id, parameter) VALUES ('${WFID}$cid', '$outer', 'collection');"  >> /tmp/$RUNID.sql
 done < tie-containers.txt
 
 echo "    - Operator calls."
@@ -58,29 +61,27 @@
     rhs=$(echo $rhs | awk 'BEGIN { FS = "=" }; {print $2}')
     result=$(echo $result | awk 'BEGIN { FS = "=" }; {print $2}')
     
-    operatorid="operator:$thread"
+    operatorid="${WFID}operator:$thread"
     
     echo  "INSERT INTO ds (id) VALUES ('$lhs');" >> /tmp/$RUNID.sql
     echo  "INSERT INTO ds (id) VALUES ('$rhs');" >> /tmp/$RUNID.sql
     echo  "INSERT INTO ds (id) VALUES ('$result');" >> /tmp/$RUNID.sql
     echo  "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$operatorid', 'operator', '$name', '$WF');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO ds_in (fun_call_id, ds_id, parameter) VALUES ('$operatorid', '$lhs', 'lhs');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO ds_in (fun_call_id, ds_id, parameter) VALUES ('$operatorid', '$rhs', 'rhs');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO ds_out (fun_call_id, ds_id, parameter) VALUES ('$operatorid', '$result', 'result');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO dataset_in (function_call_id, dataset_id, parameter) VALUES ('$operatorid', '$lhs', 'lhs');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO dataset_in (function_call_id, dataset_id, parameter) VALUES ('$operatorid', '$rhs', 'rhs');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO dataset_out (function_call_id, dataset_id, parameter) VALUES ('$operatorid', '$result', 'result');"  >> /tmp/$RUNID.sql
 done < operators.txt
 
 echo "    - Built-in function calls."
 while read id name output; do
-    fid=$(echo $id | awk -F ":" '{print $3}')
     echo  "INSERT INTO ds (id) VALUES ('$output');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$fid', 'function', '$name', '$WF');"  >> /tmp/$RUNID.sql
-    echo  "INSERT INTO ds_out (fun_call_id, ds_id, parameter) VALUES ('$fid', '$output', 'result');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$id', 'function', '$name', '$WF');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO dataset_out (function_call_id, dataset_id, parameter) VALUES ('$id', '$output', 'result');"  >> /tmp/$RUNID.sql
 done < functions.txt
 
 while read id value; do
-    fid=$(echo $id | awk -F ":" '{print $3}')
     echo  "INSERT INTO ds (id) VALUES ('$value');" >> /tmp/$RUNID.sql
-    echo  "INSERT INTO ds_in (fun_call_id, ds_id, parameter) VALUES ('$fid', '$value', 'undefined');"  >> /tmp/$RUNID.sql
+    echo  "INSERT INTO dataset_in (function_call_id, dataset_id, parameter) VALUES ('$id', '$value', 'undefined');"  >> /tmp/$RUNID.sql
 done < function-inputs.txt
 
 
@@ -95,87 +96,99 @@
 echo "    - Compound functions."
 while read start duration thread final_state procname ; do
     if [ "$duration" != "last-event-line" ]; then
-	echo "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$thread', 'compound', '$procname', '$WF');"  >> /tmp/$RUNID.sql
+	compoundid=$WFID$thread
+	echo "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$compoundid', 'compound', '$procname', '$WF');"  >> /tmp/$RUNID.sql
     fi
 done < compound.event
 
 while read start duration thread final_state procname ; do
     if [ "$duration" != "last-event-line" ]; then
-	echo "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$thread', 'internal', '$procname', '$WF');"  >> /tmp/$RUNID.sql
+	fqid=$WFID$thread
+	echo "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$fqid', 'internal', '$procname', '$WF');"  >> /tmp/$RUNID.sql
     fi	
 done < internalproc.event
 
 while read t ; do 
-    echo "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$t', 'scope', 'scope', '$WF');"  >> /tmp/$RUNID.sql
+    thread="${WFID}$t"
+    echo "INSERT INTO fun_call (id, type, name, run_id) VALUES ('$thread', 'scope', 'scope', '$WF');"  >> /tmp/$RUNID.sql
 done < scopes.txt
 
-echo "    - Variable consumption and production."
+echo "    - Dataset consumption and production."
 while read thread direction dataset variable rest; do 
-    fid=$(echo $thread | awk -F ":" '{print $3}')
     if [ "$direction" == "input" ] ; then
-	table=ds_in
+	table=dataset_in
     else
-	table=ds_out
+	table=dataset_out
     fi
     
     echo "INSERT INTO ds (id) VALUES ('$dataset');"  >> /tmp/$RUNID.sql
-    echo "INSERT INTO $table (fun_call_id, ds_id, parameter) VALUES ('$fid', '$dataset', '$variable');"  >> /tmp/$RUNID.sql
+    echo "INSERT INTO $table (function_call_id, dataset_id, parameter) VALUES ('$thread', '$dataset', '$variable');"  >> /tmp/$RUNID.sql
 done < tie-data-invocs.txt
 
 
 echo "    - Wrapper log resource consumption info."
 if [ -f runtime.txt ]; then
     while read execute2_id runtime; do
-	#timestamp=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}')
-	#cpu_usage=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
-	#max_phys_mem=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}')
-	#max_virtual_mem=$(echo $runtime | awk -F "," '{print $4}' | awk -F ":" '{print $2}')
-	#io_read_bytes=$(echo $runtime | awk -F "," '{print $5}' | awk -F ":" '{print $2}')
-	#io_write_bytes=$(echo $runtime | awk -F "," '{print $6}' | awk -F ":" '{print $2}')
-	#echo "INSERT INTO rt_info (app_exec_id, tstamp, cpu_usage, max_phys_mem, max_virt_mem, io_read, io_write) VALUES ('$execute2_id', $timestamp, $cpu_usage, $max_phys_mem, $max_virtual_mem, $io_read_bytes, $io_write_bytes);"  >> /tmp/$RUNID.sql
-	maxrss=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}')
-	walltime=$(echo $runtime | awk -F "," '{print $2}' | awk -F ":" '{print $2}')
-	systime=$(echo $runtime | awk -F "," '{print $3}' | awk -F ":" '{print $2}')
-	usertime=$(echo $runtime | awk -F "," '{print $4}' | awk -F ":" '{print $2}')
-	cpu=$(echo $runtime | awk -F "," '{print $5}' | awk -F ":" '{print $2}' | awk -F "%" '{print $1}')
-	fsin=$(echo $runtime | awk -F "," '{print $6}' | awk -F ":" '{print $2}')
-	fsout=$(echo $runtime | awk -F "," '{print $7}' | awk -F ":" '{print $2}')
-	timesswapped=$(echo $runtime | awk -F "," '{print $8}' | awk -F ":" '{print $2}')
-	socketrecv=$(echo $runtime | awk -F "," '{print $9}' | awk -F ":" '{print $2}')
-	socketsent=$(echo $runtime | awk -F "," '{print $10}' | awk -F ":" '{print $2}')
-	majorpagefaults=$(echo $runtime | awk -F "," '{print $11}' | awk -F ":" '{print $2}')
-	minorpagefaults=$(echo $runtime | awk -F "," '{print $12}' | awk -F ":" '{print $2}')
-	contextswitchesinv=$(echo $runtime | awk -F "," '{print $13}' | awk -F ":" '{print $2}')
-	contextswitchesvol=$(echo $runtime | awk -F "," '{print $14}' | awk -F ":" '{print $2}')
-	echo "UPDATE app_exec SET maxrss=$maxrss, walltime=$walltime, systime=$systime, usertime=$usertime, cpu=$cpu, fsin=$fsin, fsout=$fsout, timesswpd=$timesswapped, socketrecv=$socketrecv, socketsent=$socketsent, majpfaults=$majorpagefaults, minpfaults=$minorpagefaults, ctxswinv=$contextswitchesinv, ctxswvol=$contextswitchesvol where id='$execute2_id' and run_id='$WF';"  >> /tmp/$RUNID.sql
+	for key in $(echo maxrss walltime systime usertime cpu fsin fsout timesswapped socketrecv socketsent majorpagefaults minorpagefaults contextswitchesinv contextswitchesvol); do
+	    value=$(echo $runtime | awk -F "," '{print $1}' | awk -F ":" '{print $2}')
+	    echo "INSERT INTO annot_app_exec_num VALUES ('$execute2_id','$key',$value)"  >>  /tmp/$RUNID.sql
+	done
     done < runtime.txt
 fi
 
 echo "    - Function call names."
 while read thread appname; do
-    fid=$(echo $thread | awk -F ":" '{print $3}')
-    echo  "UPDATE fun_call SET name='$appname' WHERE id='$fid';"  >> /tmp/$RUNID.sql
+    echo  "UPDATE fun_call SET name='$appname' WHERE id='$thread';"  >> /tmp/$RUNID.sql
 done < invocation-procedure-names.txt
 
 echo "    - Wrapper log extra info."
 if [ -f extrainfo.txt ]; then
     while read execute2_id extrainfo; do
-	eid=$(echo $execute2_id | awk -F ":" '{print $3}')
 	echo $extrainfo | awk -F ";"  '{ for (i = 1; i <= NF; i++)
                                                print $i
                                          }' | awk -F "=" '{ print $1 " " $2 }' | awk -F ":" '{ print $1 " " $2 }' > fields.txt
-	fid=$($SQLCMD --tuples-only -c "select app_fun_call_id from app_exec where id='$eid' and run_id='$WF';" | awk '{print $1}')
+	id=$($SQLCMD --tuples-only -c "select app_fun_call_id from app_exec where id='$execute2_id';" | awk '{print $1}')
 	while read name type value; do
 	    if [ "$type" = "num" ]; then
-		echo "INSERT INTO annot_app_exec_num (id, fun_call_id, run_id, name, value) VALUES ('$eid', '$fid', '$name', $value);"  >> /tmp/$RUNID.sql
+		echo "INSERT INTO annot_app_exec_num (id, name, value) VALUES ('$id', '$name', $value);"  >> /tmp/$RUNID.sql
 	    fi 
 	    if [ "$type" = "txt" ]; then
-		echo "INSERT INTO annot_app_exec_text (id, fun_call_id, run_id, name, value) VALUES ('$eid', '$fid', '$name', '$value');"  >> /tmp/$RUNID.sql
+		echo "INSERT INTO annot_app_exec_text (id, name, value) VALUES ('$id', '$name', '$value');"  >> /tmp/$RUNID.sql
 	    fi
 	done < fields.txt
     done < extrainfo.txt
 fi
 
+echo "    - Prospective provenance (script, tc, sites)."
+script_hash=$(openssl dgst -sha1  script.txt | awk  '{ print $2 }')
+EXISTING=$($SQLCMD --tuples-only -c "select count(*) from script where hash_value='$script_hash';")
+if [ "$EXISTING" -eq "0" ];  then
+    content=$(cat script.txt | sed "s/'/''/g")
+    echo "INSERT INTO script VALUES ('$script_hash', '$content');" >> /tmp/$RUNID.sql
+fi
+echo "UPDATE run SET script_hash='$script_hash' WHERE id='$WF';" >> /tmp/$RUNID.sql
+
+
+    
+tc_hash=$(openssl dgst -sha1  tc.txt | awk  '{ print $2 }')
+EXISTING=$($SQLCMD --tuples-only -c "select count(*) from tc_file where hash_value='$tc_hash';")
+if [ "$EXISTING" -eq "0" ];  then
+    content=$(cat tc.txt | sed "s/'/''/g")
+    echo "INSERT INTO tc_file VALUES ('$tc_hash', '$content');" >> /tmp/$RUNID.sql
+fi
+echo "UPDATE run SET tc_file_hash='$tc_hash' WHERE id='$WF';" >> /tmp/$RUNID.sql
+
+
+sites_hash=$(openssl dgst -sha1  sites.txt | awk  '{ print $2 }')
+EXISTING=$($SQLCMD --tuples-only -c "select count(*) from sites_file where hash_value='$sites_hash';")
+if [ "$EXISTING" -eq "0" ];  then
+    content=$(cat sites.txt | sed "s/'/''/g")
+    echo "INSERT INTO sites_file VALUES ('$sites_hash', '$content');" >> /tmp/$RUNID.sql
+fi
+echo "UPDATE run SET sites_file_hash='$sites_hash' WHERE id='$WF';" >> /tmp/$RUNID.sql
+
+
+
 echo "Finished SQL generation."
 echo "Exporting provenance to database..."
 $SQLCMD -f /tmp/$RUNID.sql 1> /dev/null 2> /tmp/$RUNID-provenancedb-error.log

Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs	2012-05-09 22:18:28 UTC (rev 5785)
+++ provenancedb/swift-prov-import-all-logs	2012-05-10 12:27:54 UTC (rev 5786)
@@ -31,16 +31,16 @@
     $SQLCMD < $PROVDIR/prov-init.sql
 fi
 
-while read start version filename; do
+while read start version cogversion filename; do
     
     export IDIR=$(echo $filename | sed 's/\.log$/.d/')
     COG_VERSION=$(grep -m 1 -E 'Swift .* swift-r[0-9]*' $filename | sed 's/.*Swift .* cog-r\([0-9]*\).*/\1/')
     echo IDIR=$IDIR
     if [ $version -ge 1538 ]; then
 	echo -n "Log: $filename ... "
-	
         # TODO: does not work in sqlite
-	EXISTING=$($SQLCMD --tuples-only -c "select count(*) from run where log_filename='$filename';")
+	fname=$(echo $filename | sed -e 's:[^\`].*/::')
+	EXISTING=$($SQLCMD --tuples-only -c "select count(*) from run where log_filename='$fname';")
 	
 	if [ "$EXISTING" -eq "0" ];  then
 	    PROV_ENABLED=$(grep provenanceid $filename | wc -l)
@@ -54,11 +54,11 @@
 		fi
 		
 		export RUNID=$(basename $filename .log)
-		
+		source_file=$(echo $fname | sed "s/-[0-9]*-[0-9]*-[0-9a-z]*\.log$/\.swift/")	
 		export WF="${RUNID}"
 		
 		#echo "BEGIN TRANSACTION;" > /tmp/$WF.sql
-		echo "INSERT INTO run (id, log_filename, swift_version, cog_version, final_state) VALUES ('$WF','$filename','$version', '$COG_VERSION', '$wfstatus');" >> /tmp/$WF.sql
+		echo "INSERT INTO run (id, log_filename, script_filename, swift_version, cog_version, final_state) VALUES ('$WF','$fname','$source_file','$version', '$COG_VERSION', '$wfstatus');" >> /tmp/$WF.sql
 		
 		echo version $version in log file $filename
 		echo ============= will import =============

Added: provenancedb/swift_mod/_swiftwrap_runtime_aggregate
===================================================================
--- provenancedb/swift_mod/_swiftwrap_runtime_aggregate	                        (rev 0)
+++ provenancedb/swift_mod/_swiftwrap_runtime_aggregate	2012-05-10 12:27:54 UTC (rev 5786)
@@ -0,0 +1,605 @@
+#!/bin/bash
+# this script must be invoked inside of bash, not plain sh
+# note that this script modifies $IFS
+
+# Toggle debugging output from debug()
+DEBUG=0
+
+infosection() {
+        echo >& "$INFO"
+	echo "_____________________________________________________________________________" >& "$INFO"
+	echo >& "$INFO"
+	echo "        $1" >& "$INFO"
+	echo "_____________________________________________________________________________" >& "$INFO"
+	echo >& "$INFO"
+}
+
+gather_proc_info()
+{
+        #TIME_PID=$!
+		#sleep 0.1
+		##EXEC_PID=$#(ps -o pid --ppid $TIME_PID --no-headers)
+	    EXEC_PID=$!
+		SAMPLING_INTERVAL=0.1
+        while true
+        do
+
+                MAX_VIRTUAL_MEM=$(grep "VmPeak:" /proc/$EXEC_PID/status | awk '{print $2}')
+                if [ -z "$MAX_VIRTUAL_MEM" ]; then
+            #process finished
+                        break
+                fi
+
+                MAX_PHYS_MEM=$(grep "VmHWM:" /proc/$EXEC_PID/status | awk '{print $2}')
+                if [ -z "$MAX_PHYS_MEM" ]; then
+            #process finished
+                        break
+                fi
+
+                READ_BYTES=$(grep "read_bytes:" /proc/$EXEC_PID/io | awk '{print $2}')
+                if [ -z "$READ_BYTES" ]; then
+            #process finished
+                        break
+                fi
+
+                WRITE_BYTES=$(grep "^write_bytes:" /proc/$EXEC_PID/io | awk '{print $2}')
+                if [ -z "$WRITE_BYTES" ]; then
+            #process finished
+                        break
+                fi
+
+                STEP_DATE=$(date +%s).$(date +%N)
+                PSLINE=$(ps auxw | grep $EXEC_PID | grep -v grep)
+                if [ -z "$PSLINE" ]; then
+            #process finished
+                        break
+                fi
+                CPU_USAGE=$(echo $PSLINE | awk '{print $3}')
+                log "RUNTIME_INFO=timestamp:$STEP_DATE,cpu_usage:$CPU_USAGE,max_phys_mem:$MAX_PHYS_MEM,max_virtual_mem:$MAX_VIRTUAL_MEM,io_read_bytes:$READ_BYTES,io_write_bytes:$WRITE_BYTES"
+                INT_SAMPLING_INTERVAL=$(echo "$SAMPLING_INTERVAL/1" | bc)
+                if [ "$INT_SAMPLING_INTERVAL" -lt 60 ]; then
+                        SAMPLING_INTERVAL=$(echo "$SAMPLING_INTERVAL+0.1" | bc)
+                fi
+				sleep $SAMPLING_INTERVAL
+        done
+        wait $EXEC_PID
+}
+
+info() {
+	infosection "command line"
+	echo $COMMANDLINE 2>&1 >& "$INFO"
+	infosection "uname -a"
+	uname -a 2>&1 >& "$INFO"
+	infosection "id"
+	id 2>&1 >& "$INFO"
+	infosection "env"
+	env 2>&1 >& "$INFO"
+	infosection "df"
+	df 2>&1 >& "$INFO"
+        if [ -e "/proc/cpuinfo" ]; then
+		infosection "/proc/cpuinfo"
+		cat /proc/cpuinfo 2>&1 >& "$INFO"
+	fi
+	if [ -e "/proc/meminfo" ]; then
+		infosection "/proc/meminfo"
+		cat /proc/meminfo 2>&1 >& "$INFO"
+	fi
+	if [ -f "$STDOUT" ] ; then
+		infosection "stdout"
+		cat $STDOUT >& "$INFO"
+	fi
+	if [ -f "$STDERR" ] ; then
+		infosection "stderr"
+		cat $STDERR >& "$INFO"
+	fi
+}
+
+logstate() {
+	echo "Progress " `date +"%Y-%m-%d %H:%M:%S.%N%z"` " $@" >& "$INFO"
+}
+
+log() {
+	echo "$@" >& "$INFO"
+}
+
+debug() {
+	[[ $DEBUG == 1 ]] && echo "$@" >& "$INFO"
+}
+
+fail() {
+	EC=$1
+	shift
+	
+	if [ "$STATUSMODE" != "files" ]; then
+		mkdir -p $WFDIR/status/$JOBDIR
+	fi
+	
+	echo $@ >"$WFDIR/status/$JOBDIR/${ID}-error"
+		
+	log $@
+	info
+	if [ "$STATUSMODE" = "files" ]; then
+		exit 0
+	else
+		exit $EC
+	fi
+}
+
+checkError() {
+	if [ "$?" != "0" ]; then
+		fail $@
+	fi
+}
+
+checkEmpty() {
+	if [ "$1" == "" ]; then
+		shift
+		fail 254 $@
+	fi
+}
+
+checkparamfile() {
+	log "checking for paramfile"
+	if [ "$1" == "-p" ]; then
+		JOBDIR=$2
+		PARAMFILE=${WFDIR}/parameters/${JOBDIR}/param-${ID}
+	fi
+	log "paramfile is: $PARAMFILE"
+}
+
+getarg() {
+	NAME=$1
+	shift
+	VALUE=""
+	SHIFTCOUNT=0
+	if [ "$PARAMFILE" == "" ] && [ "$1" == "$NAME" ]; then
+		shift
+		let "SHIFTCOUNT=$SHIFTCOUNT+1"
+		while [ "${1:0:1}" != "-" ] && [ "$#" != "0" ]; do
+			VALUE="$VALUE $1"
+			shift
+			let "SHIFTCOUNT=$SHIFTCOUNT+1"
+		done
+		VALUE="${VALUE:1}"
+	elif [ "$PARAMFILE" != "" ] && grep -E "^$NAME " $PARAMFILE ; then
+		VALUE=$(grep -E "^$NAME " $PARAMFILE | cut -d ' ' -f 2-)
+	else
+		fail 254 "Missing $NAME argument"
+	fi
+}
+
+openinfo() {
+	exec 3<> $1
+	INFO=3
+}
+
+closeinfo() {
+	exec 3>&-
+}
+
+contains() {
+	ARRAY=$1
+	X=$2
+
+	for a in ${!ARRAY}
+	do
+		if [[ ${a} == ${X} ]]; then
+			return 0
+		fi
+	done
+	return 1
+}
+
+genScripts() {
+	echo "#!/bin/bash" > run.sh
+	echo -n "\"$EXEC\" " >> run.sh
+	for CMDARG in "${CMDARGS[@]}"; do
+    	echo -n "\"$CMDARG\" " >> run.sh
+	done
+	echo "1>\"$STDOUT\" 2>\"$STDERR\"" >> run.sh
+	chmod +x run.sh
+}
+
+cdm_local_output()
+{
+ 	L=$1
+
+	if [[ $CDM_FILE == "" ]]; then
+		return
+	fi
+
+ 	CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+	if [[ $CDM_POLICY == "LOCAL" ]]; then
+		cdm_local_output_perform $L $CDM_POLICY
+	fi
+}
+
+cdm_local_output_perform()
+{
+	L=$1
+	TOOL=$2
+	REMOTE_DIR=$3
+	FLAGS=$3
+	log "Copying $REMOTE_DIR/$FILE to $JOBDIR/$FILE"
+	mkdir -p $REMOTE_DIR
+	checkError 254 "CDM[LOCAL]: mkdir -p $REMOTE_DIR failed!"
+	$TOOL $FLAGS $JOBDIR/$FILE $REMOTE_DIR/$FILE
+	checkError 254 "CDM[LOCAL]: Tool failed!"
+}
+
+cdm_gather()
+{
+	GATHER_OUTPUT=${*}
+	if [[ $CDM_FILE == "" ]]; then
+		return
+	fi
+	if [[ $GATHER_OUTPUT == "" ]]; then
+		return
+	fi
+
+	cdm_gather_action $GATHER_MAX $GATHER_OUTPUT
+}
+
+COMMANDLINE=$@
+
+# get the parent directory of the directory containing _swiftwrap, to use
+# as the run directory
+# this assumes that _swiftwrap is being executed from the top level of
+# the shared directory, and that shared directory is in the top level
+# of the workflow run directory
+WFDIR=$(dirname $(dirname $0))
+
+cd $WFDIR
+
+# make the WFDIR absolute
+WFDIR=$(pwd)
+PARAMFILE=
+
+openinfo "wrapper.log"
+ID=$1
+checkEmpty "$ID" "Missing job ID"
+
+shift
+
+checkparamfile "$@"
+
+# JOBDIR might have been assigned through the -p option, or might
+# be a parameter here
+if [ "$JOBDIR" == "" ] ; then
+	getarg "-jobdir" "$@"
+	JOBDIR=$VALUE
+	shift $SHIFTCOUNT
+fi
+
+getarg "-scratch" "$@"
+SCRATCH=$VALUE
+shift $SHIFTCOUNT
+
+if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then
+	INFODIR=$SCRATCH
+else
+	INFODIR=$WFDIR/info/$JOBDIR
+fi
+checkEmpty "$JOBDIR" "Missing job directory prefix"
+mkdir -p $INFODIR
+closeinfo
+
+if [ -z $MPI_RANK ]; then
+	INFOFILE="$INFODIR/${ID}-info"
+else
+	INFOFILE="$INFODIR/${ID}-${MPI_RANK}-info"
+fi
+rm -f $INFOFILE
+openinfo "$INFOFILE"
+
+logstate "LOG_START"
+infosection "Wrapper (_swiftwrap)"
+
+getarg "-e" "$@"
+EXEC=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-out" "$@"
+STDOUT=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-err" "$@"
+STDERR=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-i" "$@"
+STDIN=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-d" "$@"
+DIRS=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-if" "$@"
+INF=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-of" "$@"
+OUTF=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-k" "$@"
+KICKSTART=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-cdmfile" "$@"
+CDM_FILE=
+if [ "X$VALUE" != "X" ]; then
+	CDM_FILE=shared/$VALUE
+fi
+shift $SHIFTCOUNT
+
+getarg "-status" "$@"
+STATUSMODE=$VALUE
+shift $SHIFTCOUNT
+
+declare -a CMDARGS
+if [ "$PARAMFILE" == "" ] && [ "$1" == "-a" ] ; then
+	shift
+	CMDARGS=("$@")
+elif [ "$PARAMFILE" != "" ] ; then
+	CMDARGS=()
+	FIRST=1
+	while read line ; do
+		if [ "$FIRST" == "1" ] ; then
+			CMDARGS=("$line")
+			FIRST=0
+		else
+			CMDARGS=("${CMDARGS[@]}" "$line")
+		fi
+	done < <(grep -E "^-a " $PARAMFILE | cut -d " " -f 2-)
+else
+	fail 254 "Missing arguments (-a option)"
+fi
+
+if [ "$STATUSMODE" = "files" ]; then
+	mkdir -p $WFDIR/status/$JOBDIR
+fi
+
+if [ "X$CDM_FILE" != "X" ]; then
+	logstate "SOURCE_CDM_LIB $WFDIR/shared/cdm_lib.sh"
+	source $WFDIR/shared/cdm_lib.sh
+	checkError 254 "Could not source: $WFDIR/shared/cdm_lib.sh"
+fi
+
+if [ "X$SCRATCH" != "X" ]; then
+	log "Job directory mode is: local copy"
+	DIR=$SCRATCH/$JOBDIR/$ID
+	COPYNOTLINK=1
+else
+	log "Job directory mode is: link on shared filesystem"
+	DIR=jobs/$JOBDIR/$ID
+	COPYNOTLINK=0
+fi
+
+PATH=$PATH:/bin:/usr/bin
+
+if [ "$PATHPREFIX" != "" ]; then
+	export PATH=$PATHPREFIX:$PATH
+fi
+
+if [ "$SWIFT_EXTRA_INFO" != "" ]; then
+	log "EXTRAINFO=$($SWIFT_EXTRA_INFO)"
+fi
+
+if [ "X${EXEC:0:1}" != "X/" ] ; then
+	export ORIGEXEC=$EXEC
+	export EXEC=$(which $EXEC)
+	if [ "X$EXEC" = "X" ] ; then
+		fail 254 "Cannot find executable $ORIGEXEC on site system path"
+	fi
+fi
+
+debug "PID=$$"
+log "PWD=$PWD"
+log "DIR=$DIR"
+log "EXEC=$EXEC"
+log "STDIN=$STDIN"
+log "STDOUT=$STDOUT"
+log "STDERR=$STDERR"
+log "DIRS=$DIRS"
+log "INF=$INF"
+log "OUTF=$OUTF"
+log "KICKSTART=$KICKSTART"
+log "CDM_FILE=$CDM_FILE"
+[ -z $MPI_RANK ] && log "MPI_RANK=$MPI_RANK"
+log "ARGS=$@"
+log "ARGC=$#"
+IFS="|"
+
+logstate "CREATE_JOBDIR"
+mkdir -p $DIR
+checkError 254 "Failed to create job directory $DIR"
+log "Created job directory: $DIR"
+
+if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then
+
+logstate "CREATE_INPUTDIR"
+for D in $DIRS ; do
+	mkdir -p "$DIR/$D" 2>&1 >>"$INFO"
+	checkError 254 "Failed to create input directory $D"
+	log "Created output directory: $DIR/$D"
+done
+
+logstate "LINK_INPUTS"
+for L in $INF ; do
+    CDM_POLICY="DEFAULT"
+	if [[ $CDM_FILE != "" ]]; then
+		CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+	fi
+	if [[ $CDM_POLICY != "DEFAULT" && $CDM_POLICY != "EXTERNAL"* ]]; then
+		log "CDM_POLICY: $L -> $CDM_POLICY"
+		eval cdm_action $DIR "INPUT" $L $CDM_POLICY
+		continue
+	fi
+	if [ $COPYNOTLINK = 1 ]; then
+		cp "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO
+		checkError 254 "Failed to copy input file $L"
+		log "Copied input: $WFDIR/shared/$L to $DIR/$L"
+	else
+		[ -f $WFDIR/shared/$L ]
+		checkError 254 "Could not locate input file: $L"
+		ln -s "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO
+		checkError 254 "Failed to link input file $L"
+		log "Linked input: $WFDIR/shared/$L to $DIR/$L"
+	fi
+done
+
+if [ ! -z $CDM_FILE ]; then
+    logstate "LINK_CDM_OUTPUTS"
+    SKIPPED_OUTPUT=()
+	GATHER_OUTPUT=()
+	for L in $OUTF ; do
+		CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+		if [[ $CDM_POLICY != "DEFAULT" &&
+			  $CDM_POLICY != "BROADCAST"* ]]; then
+			log "CDM_POLICY: $L -> $CDM_POLICY"
+    	    eval cdm_action $DIR "OUTPUT" $L $CDM_POLICY
+			SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L )
+		fi
+		if [ $CDM_POLICY == "GATHER" ]; then
+			GATHER_OUTPUT=( $GATHER_OUTPUT $L )
+		elif [ $CDM_POLICY == "LOCAL" ]; then
+			CDM_LOCAL_OUTPUT=( $CDM_LOCAL_OUTPUT $L )
+		fi
+	done
+fi
+
+fi # MPI_RANK==0
+
+debug "Moving to jobdir: $DIR"
+cd $DIR
+if [ $? != 0 ]; then
+	log "PWD: $PWD"
+	log $( find . )
+	fail 254 "Could not cd to: $DIR"
+fi
+logstate "EXECUTE"
+
+debug "Command line: $EXEC ${CMDARGS[@]}"
+
+if [ ! -f "$EXEC" ]; then
+	fail 254 "The executable $EXEC does not exist"
+fi
+if [ ! -x "$EXEC" ]; then
+	fail 254 "The executable $EXEC does not have the executable bit set"
+fi
+if [ "$KICKSTART" == "" ]; then
+	if [ "$STDIN" == "" ]; then
+		if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then
+			genScripts
+		fi
+		/usr/bin/time -f "RUNTIME_AGGR=maxrss:%M,walltime:%e,systime:%S,usertime:%U,cpu:%P,fsin:%I,fsout:%O,timesswapped:%W,socketrecv:%r,socketsent:%s,majorpagefaults:%F,minorpagefaults:%R,contextswitchesinv:%c,contextswitchesvol:%w" "$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR"
+		#"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" &
+		#gather_proc_info
+		RTAGGR=$(cat "$STDERR" | grep RUNTIME_AGGR)
+		log "$RTAGGR"
+	else
+		if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then
+			genScripts
+		fi
+		/usr/bin/time -f "RUNTIME_AGGR=maxrss:%M,walltime:%e,systime:%S,usertime:%U,cpu:%P,fsin:%I,fsout:%O,timesswapped:%W,socketrecv:%r,socketsent:%s,majorpagefaults:%F,minorpagefaults:%R,contextswitchesinv:%c,contextswitchesvol:%w" "$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN" 
+		#"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN" &
+		#gather_proc_info
+		RTAGGR=$(cat "$STDERR" | grep RUNTIME_AGGR)
+		log "$RTAGGR"
+	fi
+	checkError $? "Application $EXEC failed with an exit code of $?"
+else
+	if [ ! -f "$KICKSTART" ]; then
+		log "Kickstart executable ($KICKSTART) not found"
+		fail 254 "The Kickstart executable ($KICKSTART) was not found"
+	elif [ ! -x "$KICKSTART" ]; then
+		log "Kickstart executable ($KICKSTART) is not executable"
+		fail 254 "The Kickstart executable ($KICKSTART) does not have the executable bit set"
+	else
+		mkdir -p $WFDIR/kickstart/$JOBDIR
+		log "Using Kickstart ($KICKSTART)"
+		if [ "$STDIN" == "" ]; then
+			"$KICKSTART" -H -o "$STDOUT" -e "$STDERR" "$EXEC" "$@" 1>kickstart.xml 2>"$STDERR"
+		else
+			"$KICKSTART" -H -o "$STDOUT" -i "$STDIN" -e "$STDERR" "$EXEC" "$@" 1>kickstart.xml 2>"$STDERR"
+		fi
+		export APPEXIT=$?
+		mv -f kickstart.xml "$WFDIR/kickstart/$JOBDIR/$ID-kickstart.xml" 2>&1 >& "$INFO"
+		checkError 254 "Failed to copy Kickstart record to shared directory"
+		if [ "$APPEXIT" != "0" ]; then
+			fail $APPEXIT "Application $EXEC failed with an exit code of $APPEXIT"
+		fi
+	fi
+fi
+
+log "Moving back to workflow directory $WFDIR"
+cd $WFDIR
+if [ $? != 0 ]; then
+	fail 254 "Could not cd to workflow directory: $WFDIR"
+fi
+
+logstate "EXECUTE_DONE"
+log "Job ran successfully"
+
+if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then
+
+MISSING=
+for O in $OUTF ; do
+	if [ ! -f "$DIR/$O" ]; then
+		if [ "$MISSING" == "" ]; then
+			MISSING=$O
+		else
+			MISSING="$MISSING, $O"
+		fi
+	fi
+done
+if [ "$MISSING" != "" ]; then
+	log $( find . )
+	fail 254 "The following output files were not created by the application: $MISSING"
+fi
+
+logstate "MOVING_OUTPUTS $OUTF"
+for O in $OUTF ; do
+	if ! contains SKIPPED_OUTPUT $O ; then
+		mv "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >&	"$INFO"
+		checkError 254 "Failed to move output file $O to shared directory"
+	fi
+done
+
+cdm_local_output $CDM_LOCAL_OUTPUT
+cdm_gather $GATHER_OUTPUT
+
+logstate "RM_JOBDIR"
+rm -rf "$DIR" 2>&1 >& "$INFO"
+checkError 254 "Failed to remove job directory $DIR"
+
+if [ "$STATUSMODE" = "files" ]; then
+	logstate "TOUCH_SUCCESS"
+	touch $WFDIR/status/${JOBDIR}/${ID}-success
+fi
+
+else
+	# Allow rank 0 to write output
+	sleep 1
+fi # MPI_RANK==0
+
+logstate "END"
+
+closeinfo
+
+if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then
+	mkdir -p "$WFDIR/info/$JOBDIR"
+	mv "$INFODIR/${ID}-info" "$WFDIR/info/$JOBDIR/${ID}-info"
+fi
+
+# ensure we exit with a 0 after a successful execution
+exit 0
+
+# Local Variables:
+# mode: sh
+# sh-basic-offset: 4
+# tab-width: 4
+# indent-tabs-mode: 1
+# End:


Property changes on: provenancedb/swift_mod/_swiftwrap_runtime_aggregate
___________________________________________________________________
Added: svn:executable
   + *

Added: provenancedb/swift_mod/_swiftwrap_runtime_snapshots
===================================================================
--- provenancedb/swift_mod/_swiftwrap_runtime_snapshots	                        (rev 0)
+++ provenancedb/swift_mod/_swiftwrap_runtime_snapshots	2012-05-10 12:27:54 UTC (rev 5786)
@@ -0,0 +1,595 @@
+#!/bin/bash
+# this script must be invoked inside of bash, not plain sh
+# note that this script modifies $IFS
+
+# Toggle debugging output from debug()
+DEBUG=0
+
+infosection() {
+        echo >& "$INFO"
+	echo "_____________________________________________________________________________" >& "$INFO"
+	echo >& "$INFO"
+	echo "        $1" >& "$INFO"
+	echo "_____________________________________________________________________________" >& "$INFO"
+	echo >& "$INFO"
+}
+
+gather_proc_info()
+{
+	EXEC_PID=$!
+	SAMPLING_INTERVAL=1
+	while true
+	do
+		sleep $SAMPLING_INTERVAL
+		
+		MAX_VIRTUAL_MEM=$(grep "VmPeak:" /proc/$EXEC_PID/status | awk '{print $2}')
+		if [ -z "$MAX_VIRTUAL_MEM" ]; then
+            #process finished
+			break
+		fi 
+
+		MAX_PHYS_MEM=$(grep "VmHWM:" /proc/$EXEC_PID/status | awk '{print $2}')
+		if [ -z "$MAX_PHYS_MEM" ]; then
+            #process finished
+			break
+		fi 
+				
+		READ_BYTES=$(grep "read_bytes:" /proc/$EXEC_PID/io | awk '{print $2}') 
+		if [ -z "$READ_BYTES" ]; then
+            #process finished
+			break
+		fi
+
+		WRITE_BYTES=$(grep "^write_bytes:" /proc/$EXEC_PID/io | awk '{print $2}') 
+		if [ -z "$WRITE_BYTES" ]; then
+            #process finished
+			break
+		fi
+		
+		STEP_DATE=$(date +%s)
+		PSLINE=$(ps auxw | grep $EXEC_PID | grep -v grep)
+		if [ -z "$PSLINE" ]; then
+            #process finished
+			break
+		fi
+		CPU_USAGE=$(echo $PSLINE | awk '{print $3}')
+		log "RUNTIME_INFO=timestamp:$STEP_DATE,cpu_usage:$CPU_USAGE,max_phys_mem:$MAX_PHYS_MEM,max_virtual_mem:$MAX_VIRTUAL_MEM,io_read_bytes:$READ_BYTES,io_write_bytes:$WRITE_BYTES"
+		if [ "$SAMPLING_INTERVAL" -lt 60 ]; then
+			let "SAMPLING_INTERVAL=$SAMPLING_INTERVAL+1"
+		fi
+	done
+	wait $EXEC_PID
+}
+
+info() {
+	infosection "command line"
+	echo $COMMANDLINE 2>&1 >& "$INFO"
+	infosection "uname -a"
+	uname -a 2>&1 >& "$INFO"
+	infosection "id"
+	id 2>&1 >& "$INFO"
+	infosection "env"
+	env 2>&1 >& "$INFO"
+	infosection "df"
+	df 2>&1 >& "$INFO"
+        if [ -e "/proc/cpuinfo" ]; then
+		infosection "/proc/cpuinfo"
+		cat /proc/cpuinfo 2>&1 >& "$INFO"
+	fi
+	if [ -e "/proc/meminfo" ]; then
+		infosection "/proc/meminfo"
+		cat /proc/meminfo 2>&1 >& "$INFO"
+	fi
+	if [ -f "$STDOUT" ] ; then
+		infosection "stdout"
+		cat $STDOUT >& "$INFO"
+	fi
+	if [ -f "$STDERR" ] ; then
+		infosection "stderr"
+		cat $STDERR >& "$INFO"
+	fi
+}
+
+logstate() {
+	echo "Progress " `date +"%Y-%m-%d %H:%M:%S.%N%z"` " $@" >& "$INFO"
+}
+
+log() {
+	echo "$@" >& "$INFO"
+}
+
+debug() {
+	[[ $DEBUG == 1 ]] && echo "$@" >& "$INFO"
+}
+
+fail() {
+	EC=$1
+	shift
+	
+	if [ "$STATUSMODE" != "files" ]; then
+		mkdir -p $WFDIR/status/$JOBDIR
+	fi
+	
+	echo $@ >"$WFDIR/status/$JOBDIR/${ID}-error"
+		
+	log $@
+	info
+	if [ "$STATUSMODE" = "files" ]; then
+		exit 0
+	else
+		exit $EC
+	fi
+}
+
+checkError() {
+	if [ "$?" != "0" ]; then
+		fail $@
+	fi
+}
+
+checkEmpty() {
+	if [ "$1" == "" ]; then
+		shift
+		fail 254 $@
+	fi
+}
+
+checkparamfile() {
+	log "checking for paramfile"
+	if [ "$1" == "-p" ]; then
+		JOBDIR=$2
+		PARAMFILE=${WFDIR}/parameters/${JOBDIR}/param-${ID}
+	fi
+	log "paramfile is: $PARAMFILE"
+}
+
+getarg() {
+	NAME=$1
+	shift
+	VALUE=""
+	SHIFTCOUNT=0
+	if [ "$PARAMFILE" == "" ] && [ "$1" == "$NAME" ]; then
+		shift
+		let "SHIFTCOUNT=$SHIFTCOUNT+1"
+		while [ "${1:0:1}" != "-" ] && [ "$#" != "0" ]; do
+			VALUE="$VALUE $1"
+			shift
+			let "SHIFTCOUNT=$SHIFTCOUNT+1"
+		done
+		VALUE="${VALUE:1}"
+	elif [ "$PARAMFILE" != "" ] && grep -E "^$NAME " $PARAMFILE ; then
+		VALUE=$(grep -E "^$NAME " $PARAMFILE | cut -d ' ' -f 2-)
+	else
+		fail 254 "Missing $NAME argument"
+	fi
+}
+
+openinfo() {
+	exec 3<> $1
+	INFO=3
+}
+
+closeinfo() {
+	exec 3>&-
+}
+
+contains() {
+	ARRAY=$1
+	X=$2
+
+	for a in ${!ARRAY}
+	do
+		if [[ ${a} == ${X} ]]; then
+			return 0
+		fi
+	done
+	return 1
+}
+
+genScripts() {
+	echo "#!/bin/bash" > run.sh
+	echo -n "\"$EXEC\" " >> run.sh
+	for CMDARG in "${CMDARGS[@]}"; do
+    	echo -n "\"$CMDARG\" " >> run.sh
+	done
+	echo "1>\"$STDOUT\" 2>\"$STDERR\"" >> run.sh
+	chmod +x run.sh
+}
+
+cdm_local_output()
+{
+ 	L=$1
+
+	if [[ $CDM_FILE == "" ]]; then
+		return
+	fi
+
+ 	CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+	if [[ $CDM_POLICY == "LOCAL" ]]; then
+		cdm_local_output_perform $L $CDM_POLICY
+	fi
+}
+
+cdm_local_output_perform()
+{
+	L=$1
+	TOOL=$2
+	REMOTE_DIR=$3
+	FLAGS=$3
+	log "Copying $REMOTE_DIR/$FILE to $JOBDIR/$FILE"
+	mkdir -p $REMOTE_DIR
+	checkError 254 "CDM[LOCAL]: mkdir -p $REMOTE_DIR failed!"
+	$TOOL $FLAGS $JOBDIR/$FILE $REMOTE_DIR/$FILE
+	checkError 254 "CDM[LOCAL]: Tool failed!"
+}
+
+cdm_gather()
+{
+	GATHER_OUTPUT=${*}
+	if [[ $CDM_FILE == "" ]]; then
+		return
+	fi
+	if [[ $GATHER_OUTPUT == "" ]]; then
+		return
+	fi
+
+	cdm_gather_action $GATHER_MAX $GATHER_OUTPUT
+}
+
+COMMANDLINE=$@
+
+# get the parent directory of the directory containing _swiftwrap, to use
+# as the run directory
+# this assumes that _swiftwrap is being executed from the top level of
+# the shared directory, and that shared directory is in the top level
+# of the workflow run directory
+WFDIR=$(dirname $(dirname $0))
+
+cd $WFDIR
+
+# make the WFDIR absolute
+WFDIR=$(pwd)
+PARAMFILE=
+
+openinfo "wrapper.log"
+ID=$1
+checkEmpty "$ID" "Missing job ID"
+
+shift
+
+checkparamfile "$@"
+
+# JOBDIR might have been assigned through the -p option, or might
+# be a parameter here
+if [ "$JOBDIR" == "" ] ; then
+	getarg "-jobdir" "$@"
+	JOBDIR=$VALUE
+	shift $SHIFTCOUNT
+fi
+
+getarg "-scratch" "$@"
+SCRATCH=$VALUE
+shift $SHIFTCOUNT
+
+if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then
+	INFODIR=$SCRATCH
+else
+	INFODIR=$WFDIR/info/$JOBDIR
+fi
+checkEmpty "$JOBDIR" "Missing job directory prefix"
+mkdir -p $INFODIR
+closeinfo
+
+if [ -z $MPI_RANK ]; then
+	INFOFILE="$INFODIR/${ID}-info"
+else
+	INFOFILE="$INFODIR/${ID}-${MPI_RANK}-info"
+fi
+rm -f $INFOFILE
+openinfo "$INFOFILE"
+
+logstate "LOG_START"
+infosection "Wrapper (_swiftwrap)"
+
+getarg "-e" "$@"
+EXEC=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-out" "$@"
+STDOUT=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-err" "$@"
+STDERR=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-i" "$@"
+STDIN=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-d" "$@"
+DIRS=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-if" "$@"
+INF=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-of" "$@"
+OUTF=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-k" "$@"
+KICKSTART=$VALUE
+shift $SHIFTCOUNT
+
+getarg "-cdmfile" "$@"
+CDM_FILE=
+if [ "X$VALUE" != "X" ]; then
+	CDM_FILE=shared/$VALUE
+fi
+shift $SHIFTCOUNT
+
+getarg "-status" "$@"
+STATUSMODE=$VALUE
+shift $SHIFTCOUNT
+
+declare -a CMDARGS
+if [ "$PARAMFILE" == "" ] && [ "$1" == "-a" ] ; then
+	shift
+	CMDARGS=("$@")
+elif [ "$PARAMFILE" != "" ] ; then
+	CMDARGS=()
+	FIRST=1
+	while read line ; do
+		if [ "$FIRST" == "1" ] ; then
+			CMDARGS=("$line")
+			FIRST=0
+		else
+			CMDARGS=("${CMDARGS[@]}" "$line")
+		fi
+	done < <(grep -E "^-a " $PARAMFILE | cut -d " " -f 2-)
+else
+	fail 254 "Missing arguments (-a option)"
+fi
+
+if [ "$STATUSMODE" = "files" ]; then
+	mkdir -p $WFDIR/status/$JOBDIR
+fi
+
+if [ "X$CDM_FILE" != "X" ]; then
+	logstate "SOURCE_CDM_LIB $WFDIR/shared/cdm_lib.sh"
+	source $WFDIR/shared/cdm_lib.sh
+	checkError 254 "Could not source: $WFDIR/shared/cdm_lib.sh"
+fi
+
+if [ "X$SCRATCH" != "X" ]; then
+	log "Job directory mode is: local copy"
+	DIR=$SCRATCH/$JOBDIR/$ID
+	COPYNOTLINK=1
+else
+	log "Job directory mode is: link on shared filesystem"
+	DIR=jobs/$JOBDIR/$ID
+	COPYNOTLINK=0
+fi
+
+PATH=$PATH:/bin:/usr/bin
+
+if [ "$PATHPREFIX" != "" ]; then
+	export PATH=$PATHPREFIX:$PATH
+fi
+
+if [ "$SWIFT_EXTRA_INFO" != "" ]; then
+	log "EXTRAINFO=$($SWIFT_EXTRA_INFO)"
+fi
+
+if [ "X${EXEC:0:1}" != "X/" ] ; then
+	export ORIGEXEC=$EXEC
+	export EXEC=$(which $EXEC)
+	if [ "X$EXEC" = "X" ] ; then
+		fail 254 "Cannot find executable $ORIGEXEC on site system path"
+	fi
+fi
+
+debug "PID=$$"
+log "PWD=$PWD"
+log "DIR=$DIR"
+log "EXEC=$EXEC"
+log "STDIN=$STDIN"
+log "STDOUT=$STDOUT"
+log "STDERR=$STDERR"
+log "DIRS=$DIRS"
+log "INF=$INF"
+log "OUTF=$OUTF"
+log "KICKSTART=$KICKSTART"
+log "CDM_FILE=$CDM_FILE"
+[ -z $MPI_RANK ] && log "MPI_RANK=$MPI_RANK"
+log "ARGS=$@"
+log "ARGC=$#"
+IFS="|"
+
+logstate "CREATE_JOBDIR"
+mkdir -p $DIR
+checkError 254 "Failed to create job directory $DIR"
+log "Created job directory: $DIR"
+
+if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then
+
+logstate "CREATE_INPUTDIR"
+for D in $DIRS ; do
+	mkdir -p "$DIR/$D" 2>&1 >>"$INFO"
+	checkError 254 "Failed to create input directory $D"
+	log "Created output directory: $DIR/$D"
+done
+
+logstate "LINK_INPUTS"
+for L in $INF ; do
+    CDM_POLICY="DEFAULT"
+	if [[ $CDM_FILE != "" ]]; then
+		CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+	fi
+	if [[ $CDM_POLICY != "DEFAULT" && $CDM_POLICY != "EXTERNAL"* ]]; then
+		log "CDM_POLICY: $L -> $CDM_POLICY"
+		eval cdm_action $DIR "INPUT" $L $CDM_POLICY
+		continue
+	fi
+	if [ $COPYNOTLINK = 1 ]; then
+		cp "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO
+		checkError 254 "Failed to copy input file $L"
+		log "Copied input: $WFDIR/shared/$L to $DIR/$L"
+	else
+		[ -f $WFDIR/shared/$L ]
+		checkError 254 "Could not locate input file: $L"
+		ln -s "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO
+		checkError 254 "Failed to link input file $L"
+		log "Linked input: $WFDIR/shared/$L to $DIR/$L"
+	fi
+done
+
+if [ ! -z $CDM_FILE ]; then
+    logstate "LINK_CDM_OUTPUTS"
+    SKIPPED_OUTPUT=()
+	GATHER_OUTPUT=()
+	for L in $OUTF ; do
+		CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+		if [[ $CDM_POLICY != "DEFAULT" &&
+			  $CDM_POLICY != "BROADCAST"* ]]; then
+			log "CDM_POLICY: $L -> $CDM_POLICY"
+    	    eval cdm_action $DIR "OUTPUT" $L $CDM_POLICY
+			SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L )
+		fi
+		if [ $CDM_POLICY == "GATHER" ]; then
+			GATHER_OUTPUT=( $GATHER_OUTPUT $L )
+		elif [ $CDM_POLICY == "LOCAL" ]; then
+			CDM_LOCAL_OUTPUT=( $CDM_LOCAL_OUTPUT $L )
+		fi
+	done
+fi
+
+fi # MPI_RANK==0
+
+debug "Moving to jobdir: $DIR"
+cd $DIR
+if [ $? != 0 ]; then
+	log "PWD: $PWD"
+	log $( find . )
+	fail 254 "Could not cd to: $DIR"
+fi
+logstate "EXECUTE"
+
+debug "Command line: $EXEC ${CMDARGS[@]}"
+
+if [ ! -f "$EXEC" ]; then
+	fail 254 "The executable $EXEC does not exist"
+fi
+if [ ! -x "$EXEC" ]; then
+	fail 254 "The executable $EXEC does not have the executable bit set"
+fi
+if [ "$KICKSTART" == "" ]; then
+	if [ "$STDIN" == "" ]; then
+		if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then
+			genScripts
+		fi
+		"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" &
+		gather_proc_info
+	else
+		if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then
+			genScripts
+		fi
+		"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN" &
+		gather_proc_info
+	fi
+	checkError $? "Application $EXEC failed with an exit code of $?"
+else
+	if [ ! -f "$KICKSTART" ]; then
+		log "Kickstart executable ($KICKSTART) not found"
+		fail 254 "The Kickstart executable ($KICKSTART) was not found"
+	elif [ ! -x "$KICKSTART" ]; then
+		log "Kickstart executable ($KICKSTART) is not executable"
+		fail 254 "The Kickstart executable ($KICKSTART) does not have the executable bit set"
+	else
+		mkdir -p $WFDIR/kickstart/$JOBDIR
+		log "Using Kickstart ($KICKSTART)"
+		if [ "$STDIN" == "" ]; then
+			"$KICKSTART" -H -o "$STDOUT" -e "$STDERR" "$EXEC" "$@" 1>kickstart.xml 2>"$STDERR"
+		else
+			"$KICKSTART" -H -o "$STDOUT" -i "$STDIN" -e "$STDERR" "$EXEC" "$@" 1>kickstart.xml 2>"$STDERR"
+		fi
+		export APPEXIT=$?
+		mv -f kickstart.xml "$WFDIR/kickstart/$JOBDIR/$ID-kickstart.xml" 2>&1 >& "$INFO"
+		checkError 254 "Failed to copy Kickstart record to shared directory"
+		if [ "$APPEXIT" != "0" ]; then
+			fail $APPEXIT "Application $EXEC failed with an exit code of $APPEXIT"
+		fi
+	fi
+fi
+
+log "Moving back to workflow directory $WFDIR"
+cd $WFDIR
+if [ $? != 0 ]; then
+	fail 254 "Could not cd to workflow directory: $WFDIR"
+fi
+
+logstate "EXECUTE_DONE"
+log "Job ran successfully"
+
+if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then
+
+MISSING=
+for O in $OUTF ; do
+	if [ ! -f "$DIR/$O" ]; then
+		if [ "$MISSING" == "" ]; then
+			MISSING=$O
+		else
+			MISSING="$MISSING, $O"
+		fi
+	fi
+done
+if [ "$MISSING" != "" ]; then
+	log $( find . )
+	fail 254 "The following output files were not created by the application: $MISSING"
+fi
+
+logstate "MOVING_OUTPUTS $OUTF"
+for O in $OUTF ; do
+	if ! contains SKIPPED_OUTPUT $O ; then
+		mv "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >&	"$INFO"
+		checkError 254 "Failed to move output file $O to shared directory"
+	fi
+done
+
+cdm_local_output $CDM_LOCAL_OUTPUT
+cdm_gather $GATHER_OUTPUT
+
+logstate "RM_JOBDIR"
+rm -rf "$DIR" 2>&1 >& "$INFO"
+checkError 254 "Failed to remove job directory $DIR"
+
+if [ "$STATUSMODE" = "files" ]; then
+	logstate "TOUCH_SUCCESS"
+	touch $WFDIR/status/${JOBDIR}/${ID}-success
+fi
+
+else
+	# Allow rank 0 to write output
+	sleep 1
+fi # MPI_RANK==0
+
+logstate "END"
+
+closeinfo
+
+if [ "X$PROGRESSIVE_INFO" == "X" ] && [ "X$SCRATCH" != "X" ]; then
+	mkdir -p "$WFDIR/info/$JOBDIR"
+	mv "$INFODIR/${ID}-info" "$WFDIR/info/$JOBDIR/${ID}-info"
+fi
+
+# ensure we exit with a 0 after a successful execution
+exit 0
+
+# Local Variables:
+# mode: sh
+# sh-basic-offset: 4
+# tab-width: 4
+# indent-tabs-mode: 1
+# End:


Property changes on: provenancedb/swift_mod/_swiftwrap_runtime_snapshots
___________________________________________________________________
Added: svn:executable
   + *

Added: provenancedb/swift_mod/create-everylog-vs-versions-data
===================================================================
--- provenancedb/swift_mod/create-everylog-vs-versions-data	                        (rev 0)
+++ provenancedb/swift_mod/create-everylog-vs-versions-data	2012-05-10 12:27:54 UTC (rev 5786)
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+echo Creating log/version data file: everylog-vs-versions.data
+
+rm -f everylog-vs-versions.data
+
+
+for logfile in `find $1 -name \*.log -and -not -name cluster-\* -and -not -name swift.log $EXTRALOGRESTRICTION`; do
+    SWIFT_VERSION=$(grep -m 1 -E 'Swift .* swift-r[0-9]*' $logfile | sed 's/.*Swift .* swift-r\([0-9]*\).*/\1/')
+    COG_VERSION=$(grep -m 1 -E 'Swift .* swift-r[0-9]*' $logfile | sed 's/.*Swift .* cog-r\([0-9]*\).*/\1/')
+    START=$(head -n1 < $logfile | iso-to-secs | cut -f 1 -d ' ')
+    if [ "X$SWIFT_VERSION" == "X" ]; then
+	SWIFT_VERSION=na
+	COG_VERSION=na
+    fi
+    if [ "X$START" != "X" ]; then
+	echo $START $SWIFT_VERSION $COG_VERSION $logfile >> everylog-vs-versions.data
+    fi
+    
+done
+
+echo Finished creating log/version data file


Property changes on: provenancedb/swift_mod/create-everylog-vs-versions-data
___________________________________________________________________
Added: svn:executable
   + *