From davidk at ci.uchicago.edu  Wed Apr  2 17:04:22 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed,  2 Apr 2014 17:04:22 -0500 (CDT)
Subject: [Swift-commit] r7753 - trunk/docs/userguide
Message-ID: <20140402220422.C0A249D6FA@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-02 17:04:22 -0500 (Wed, 02 Apr 2014)
New Revision: 7753

Modified:
   trunk/docs/userguide/configuration
Log:
Fix formatting


Modified: trunk/docs/userguide/configuration
===================================================================
--- trunk/docs/userguide/configuration	2014-03-28 18:10:53 UTC (rev 7752)
+++ trunk/docs/userguide/configuration	2014-04-02 22:04:22 UTC (rev 7753)
@@ -318,9 +318,9 @@
 |true, false
 |true
 |By default, Swift will generate a run directory that contains logs, scheduler submit scripts,
-|debug directories, and other files associated with a particular Swift run. Setting this value
-|to false disables the creation of run directories and causes all logs and directories to be
-|created in the current working directory.
+debug directories, and other files associated with a particular Swift run. Setting this value
+to false disables the creation of run directories and causes all logs and directories to be
+created in the current working directory.
 
 |execution.retries
 |Positive integer



From yadunandb at ci.uchicago.edu  Thu Apr  3 16:32:19 2014
From: yadunandb at ci.uchicago.edu (yadunandb at ci.uchicago.edu)
Date: Thu,  3 Apr 2014 16:32:19 -0500 (CDT)
Subject: [Swift-commit] r7754 - in trunk: libexec src/org/griphyn/vdl/engine
	src/org/griphyn/vdl/karajan/lib/swiftscript
Message-ID: <20140403213219.3462E178884@svn.ci.uchicago.edu>

Author: yadunandb
Date: 2014-04-03 16:32:18 -0500 (Thu, 03 Apr 2014)
New Revision: 7754

Modified:
   trunk/libexec/swift-lib.k
   trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java
   trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
Log:
Committing changes for system built-in command.
Allows for easy specification of files to mappers using common unix commands.

Tested with "ls" and "find <dir> -type f"
Tests in next commit


Modified: trunk/libexec/swift-lib.k
===================================================================
--- trunk/libexec/swift-lib.k	2014-04-02 22:04:22 UTC (rev 7753)
+++ trunk/libexec/swift-lib.k	2014-04-03 21:32:18 UTC (rev 7754)
@@ -9,6 +9,7 @@
 	export(strcut, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrCut"))
 	export(strstr, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrStr"))
 	export(strsplit, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrSplit"))
+	export(system, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$ExecSystem"))
     export(strjoin, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$StrJoin"))
 	export(regexp, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Regexp"))
 	export(toInt, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$ToInt"))
@@ -21,7 +22,7 @@
 	export(trace, def("org.griphyn.vdl.karajan.lib.swiftscript.Misc$Trace"))
 	export(tracef, def("org.griphyn.vdl.karajan.lib.swiftscript.Tracef"))
 	export(fprintf, def("org.griphyn.vdl.karajan.lib.swiftscript.Fprintf"))
-	
+
 	/* included for backwards compatibility */
 	export(readdata, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadData"))
 	export(readdata2, def("org.griphyn.vdl.karajan.lib.swiftscript.ReadStructured"))

Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java	2014-04-02 22:04:22 UTC (rev 7753)
+++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java	2014-04-03 21:32:18 UTC (rev 7754)
@@ -197,6 +197,7 @@
 		add(functionsMap, "strcut", returns(STRING), args(STRING, STRING));
 		add(functionsMap, "strstr", returns(INT), args(STRING, STRING));
 		add(functionsMap, "strsplit", returns(STRING_ARRAY), args(STRING, STRING));
+		add(functionsMap, "system", returns(STRING_ARRAY), args(STRING));
 		add(functionsMap, "strjoin", returns(STRING), args(ANY, STRING));
 		add(functionsMap, "toInt", returns(INT), args(ANY));
 		add(functionsMap, "toFloat", returns(FLOAT), args(ANY));

Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-02 22:04:22 UTC (rev 7753)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-03 21:32:18 UTC (rev 7754)
@@ -18,6 +18,9 @@
 package org.griphyn.vdl.karajan.lib.swiftscript;
 
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.BufferedReader;
+
 import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -277,6 +280,51 @@
         }
     }
 	
+    public static class ExecSystem extends AbstractSingleValuedFunction {
+        private ArgRef<AbstractDataNode> input;
+        //private ArgRef<AbstractDataNode> pattern;
+	
+        @Override
+	    protected Signature getSignature() {
+            return new Signature(params("input"));
+				 }
+		
+        @Override
+	    public Object function(Stack stack) {
+		AbstractDataNode hinput = this.input.getValue(stack);
+		String input     = SwiftFunction.unwrap(this, hinput);
+
+		DSHandle handle  = new RootArrayDataNode(Types.STRING.arrayType());
+		StringBuffer out = new StringBuffer();
+		Process p;
+		int i = 0;
+		
+		try {
+		    p = Runtime.getRuntime().exec(input);
+		    p.waitFor();
+		    BufferedReader reader = new BufferedReader( new InputStreamReader(p.getInputStream()));
+		    String line = "";
+		while ( (line = reader.readLine()) != null ) {
+		    DSHandle el;
+		    el = handle.getField(i++);
+		    el.setValue(line);		    
+		    //out.append(line + "\n");
+		}
+		
+		} catch (Exception e) {
+		    e.printStackTrace();
+		}
+		handle.closeDeep();
+		
+		if (PROVENANCE_ENABLED) {
+		    int provid = SwiftFunction.nextProvenanceID();
+		    SwiftFunction.logProvenanceResult(provid, handle, "system");
+		    SwiftFunction.logProvenanceParameter(provid, hinput, "input");
+		}
+		return handle;
+	    }
+	}
+	
 	public static class StrSplit extends AbstractSingleValuedFunction {
         private ArgRef<AbstractDataNode> input;
         private ArgRef<AbstractDataNode> pattern;
@@ -601,7 +649,7 @@
             return handle;
         }
     }
-		
+
 	public static class Length extends AbstractSingleValuedFunction {
         private ArgRef<AbstractDataNode> array;
 



From davidk at ci.uchicago.edu  Thu Apr  3 23:11:45 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu,  3 Apr 2014 23:11:45 -0500 (CDT)
Subject: [Swift-commit] r7755 - in trunk: bin docs/userguide
Message-ID: <20140404041145.C62FA178884@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-03 23:11:45 -0500 (Thu, 03 Apr 2014)
New Revision: 7755

Modified:
   trunk/bin/swiftrun
   trunk/docs/userguide/configuration
Log:
Add pe to config option


Modified: trunk/bin/swiftrun
===================================================================
--- trunk/bin/swiftrun	2014-04-03 21:32:18 UTC (rev 7754)
+++ trunk/bin/swiftrun	2014-04-04 04:11:45 UTC (rev 7755)
@@ -47,6 +47,7 @@
                    'maxnodesperjob'            => 'globus.maxNodes',
                    'maxsubmitrate'             => 'karajan.maxSubmitRate',
                    'overallocationdecayfactor' => 'globus.overallocationDecayFactor',
+		   'pe'                        => 'globus.pe',
                    'providerattributes'        => 'globus.providerAttributes',
                    'remotemonitorenabled'      => 'globus.remoteMonitorEnabled',
                    'slurm'                     => 'globus.slurm',

Modified: trunk/docs/userguide/configuration
===================================================================
--- trunk/docs/userguide/configuration	2014-04-03 21:32:18 UTC (rev 7754)
+++ trunk/docs/userguide/configuration	2014-04-04 04:11:45 UTC (rev 7755)
@@ -212,15 +212,20 @@
 The maximum number of nodes to request per scheduler job.
 |site.westmere.maxNodesPerJob=2
 
+|pe
+|The parallel environment to use for SGE schedulers
+|site.sunhpc.pe=mpi
+
 |providerAttributes|
 Allows user to pass attributes through directly to scheduler submit script. Currently
 only implemented for sites that use PBS.
 |site.beagle.providerAttributes=pbs.aprun;pbs.mpp;depth=24
 
-|slurm|
-Pass parameters directly through to the submit script generated for the slurm
+|slurm
+|Pass parameters directly through to the submit script generated for the slurm
 scheduler. For example, the setting "site.midway.slurm.mail-user=username" generates
 the line "#SBATCH --mail-user=username".
+|site.midway.slurm.mail-user=username
 
 |taskDir|
 Tasks will be run from this directory. In the absence of a taskDir definition, 



From davidk at ci.uchicago.edu  Thu Apr  3 23:13:18 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu,  3 Apr 2014 23:13:18 -0500 (CDT)
Subject: [Swift-commit] r7756 - branches/release-0.95/bin
Message-ID: <20140404041318.46322178884@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-03 23:13:18 -0500 (Thu, 03 Apr 2014)
New Revision: 7756

Modified:
   branches/release-0.95/bin/swiftrun
Log:
pe config option


Modified: branches/release-0.95/bin/swiftrun
===================================================================
--- branches/release-0.95/bin/swiftrun	2014-04-04 04:11:45 UTC (rev 7755)
+++ branches/release-0.95/bin/swiftrun	2014-04-04 04:13:18 UTC (rev 7756)
@@ -47,6 +47,7 @@
                    'maxnodesperjob'            => 'globus.maxNodes',
                    'maxsubmitrate'             => 'karajan.maxSubmitRate',
                    'overallocationdecayfactor' => 'globus.overallocationDecayFactor',
+		   'pe'                        => 'globus.pe',
                    'providerattributes'        => 'globus.providerAttributes',
                    'remotemonitorenabled'      => 'globus.remoteMonitorEnabled',
                    'slurm'                     => 'globus.slurm',



From davidk at ci.uchicago.edu  Fri Apr  4 11:53:01 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri,  4 Apr 2014 11:53:01 -0500 (CDT)
Subject: [Swift-commit] r7757 - trunk/tests/functions
Message-ID: <20140404165301.327CC9CFE8@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-04 11:53:00 -0500 (Fri, 04 Apr 2014)
New Revision: 7757

Added:
   trunk/tests/functions/system_commandnotfound.setup.sh
   trunk/tests/functions/system_commandnotfound.swift
   trunk/tests/functions/system_find_arraymapper.setup.sh
   trunk/tests/functions/system_find_arraymapper.swift
   trunk/tests/functions/system_nonzero.setup.sh
   trunk/tests/functions/system_nonzero.swift
Log:
Tests for system()


Added: trunk/tests/functions/system_commandnotfound.setup.sh
===================================================================
--- trunk/tests/functions/system_commandnotfound.setup.sh	                        (rev 0)
+++ trunk/tests/functions/system_commandnotfound.setup.sh	2014-04-04 16:53:00 UTC (rev 7757)
@@ -0,0 +1 @@
+link system_find_arraymapper.setup.sh
\ No newline at end of file


Property changes on: trunk/tests/functions/system_commandnotfound.setup.sh
___________________________________________________________________
Added: svn:special
   + *

Added: trunk/tests/functions/system_commandnotfound.swift
===================================================================
--- trunk/tests/functions/system_commandnotfound.swift	                        (rev 0)
+++ trunk/tests/functions/system_commandnotfound.swift	2014-04-04 16:53:00 UTC (rev 7757)
@@ -0,0 +1,8 @@
+type file;
+
+# *_THIS-SCRIPT-SHOULD-FAIL_*
+
+string results[] = system("fdjskflsdk");
+foreach r in results {
+   tracef("%s\n", r);
+}

Added: trunk/tests/functions/system_find_arraymapper.setup.sh
===================================================================
--- trunk/tests/functions/system_find_arraymapper.setup.sh	                        (rev 0)
+++ trunk/tests/functions/system_find_arraymapper.setup.sh	2014-04-04 16:53:00 UTC (rev 7757)
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+data="fjkdlsfjlkdsjflka"
+mkdir -p data/foo_a
+mkdir -p data/foo_b
+echo $data >> data/foo_a/foo_a.txt
+echo $data >> data/foo_a/foo_b.txt
+echo $data >> data/foo_b/bar_1
+echo $data >> data/foo_b/bar_2
+echo $data >> data/data.txt


Property changes on: trunk/tests/functions/system_find_arraymapper.setup.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: trunk/tests/functions/system_find_arraymapper.swift
===================================================================
--- trunk/tests/functions/system_find_arraymapper.swift	                        (rev 0)
+++ trunk/tests/functions/system_find_arraymapper.swift	2014-04-04 16:53:00 UTC (rev 7757)
@@ -0,0 +1,13 @@
+type file;
+
+app check_files (file inputs[])
+{
+   ls "data/foo_a/foo_a.txt" "data/foo_a/foo_b.txt" "data/foo_b/bar_1" "data/foo_b/bar_2" "data/data.txt";
+}
+
+file inputs[] <array_mapper; files=system("find data -type f")>;
+foreach i in inputs {
+   tracef("%s\n", filename(i));
+}
+
+check_files(inputs);

Added: trunk/tests/functions/system_nonzero.setup.sh
===================================================================
--- trunk/tests/functions/system_nonzero.setup.sh	                        (rev 0)
+++ trunk/tests/functions/system_nonzero.setup.sh	2014-04-04 16:53:00 UTC (rev 7757)
@@ -0,0 +1 @@
+link system_find_arraymapper.setup.sh
\ No newline at end of file


Property changes on: trunk/tests/functions/system_nonzero.setup.sh
___________________________________________________________________
Added: svn:special
   + *

Added: trunk/tests/functions/system_nonzero.swift
===================================================================
--- trunk/tests/functions/system_nonzero.swift	                        (rev 0)
+++ trunk/tests/functions/system_nonzero.swift	2014-04-04 16:53:00 UTC (rev 7757)
@@ -0,0 +1,8 @@
+type file;
+
+# *_THIS-SCRIPT-SHOULD-FAIL_*
+
+string results[] = system("false");
+foreach r in results {
+   tracef("%s\n", r);
+}



From yadunandb at ci.uchicago.edu  Fri Apr  4 12:07:30 2014
From: yadunandb at ci.uchicago.edu (yadunandb at ci.uchicago.edu)
Date: Fri,  4 Apr 2014 12:07:30 -0500 (CDT)
Subject: [Swift-commit] r7758 -
	trunk/src/org/griphyn/vdl/karajan/lib/swiftscript
Message-ID: <20140404170730.BD9F29CFE8@svn.ci.uchicago.edu>

Author: yadunandb
Date: 2014-04-04 12:07:30 -0500 (Fri, 04 Apr 2014)
New Revision: 7758

Modified:
   trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
Log:
Updates for reporting exitcodes and stderr to swift stdout from system builtin


Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-04 16:53:00 UTC (rev 7757)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-04 17:07:30 UTC (rev 7758)
@@ -279,16 +279,16 @@
             return result;
         }
     }
-	
+
     public static class ExecSystem extends AbstractSingleValuedFunction {
         private ArgRef<AbstractDataNode> input;
         //private ArgRef<AbstractDataNode> pattern;
-	
+
         @Override
 	    protected Signature getSignature() {
             return new Signature(params("input"));
 				 }
-		
+
         @Override
 	    public Object function(Stack stack) {
 		AbstractDataNode hinput = this.input.getValue(stack);
@@ -296,26 +296,36 @@
 
 		DSHandle handle  = new RootArrayDataNode(Types.STRING.arrayType());
 		StringBuffer out = new StringBuffer();
-		Process p;
+		Process proc;
 		int i = 0;
-		
+
 		try {
-		    p = Runtime.getRuntime().exec(input);
-		    p.waitFor();
-		    BufferedReader reader = new BufferedReader( new InputStreamReader(p.getInputStream()));
+		    proc = Runtime.getRuntime().exec(input);
+		    proc.waitFor();
+            int exitcode = proc.exitValue();
+            // If the shell returned a non-zero exit code, attempt to print stderr
+            if ( exitcode != 0 ) {
+                BufferedReader reader = new BufferedReader( new InputStreamReader(proc.getErrorStream()) );
+                String line = "";
+                StringBuffer stderr = new StringBuffer();
+                while ( (line = reader.readLine()) != null ) {
+                    stderr.append(line);
+                }
+                logger.warn("swift:system returned exitcode :" + exitcode);
+                logger.warn("swift:system stderr:\n " + stderr );
+            }
+		    BufferedReader reader = new BufferedReader( new InputStreamReader(proc.getInputStream()) );
 		    String line = "";
-		while ( (line = reader.readLine()) != null ) {
-		    DSHandle el;
-		    el = handle.getField(i++);
-		    el.setValue(line);		    
-		    //out.append(line + "\n");
-		}
-		
+            while ( (line = reader.readLine()) != null ) {
+                DSHandle el;
+                el = handle.getField(i++);
+                el.setValue(line);
+            }
 		} catch (Exception e) {
 		    e.printStackTrace();
 		}
 		handle.closeDeep();
-		
+
 		if (PROVENANCE_ENABLED) {
 		    int provid = SwiftFunction.nextProvenanceID();
 		    SwiftFunction.logProvenanceResult(provid, handle, "system");
@@ -324,8 +334,8 @@
 		return handle;
 	    }
 	}
-	
-	public static class StrSplit extends AbstractSingleValuedFunction {
+
+		public static class StrSplit extends AbstractSingleValuedFunction {
         private ArgRef<AbstractDataNode> input;
         private ArgRef<AbstractDataNode> pattern;
 



From yadunandb at ci.uchicago.edu  Sat Apr  5 22:25:27 2014
From: yadunandb at ci.uchicago.edu (yadunandb at ci.uchicago.edu)
Date: Sat,  5 Apr 2014 22:25:27 -0500 (CDT)
Subject: [Swift-commit] r7759 -
	trunk/src/org/griphyn/vdl/karajan/lib/swiftscript
Message-ID: <20140406032527.36DCA9D065@svn.ci.uchicago.edu>

Author: yadunandb
Date: 2014-04-05 22:25:26 -0500 (Sat, 05 Apr 2014)
New Revision: 7759

Modified:
   trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
Log:
Fixing minor bug which strings passed to system builtin did not 
expand wildcards correctly as they would in a shell. So, pipes
are also working.

Tested with the following strings:

system("ls run*/*.log")
system("ls *"); #this doesn't give us a reasonable format
system("find *");
system("ls run???/*.???");
system("ls run???/*.??? | grep log");
system("find run* -type f");


Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-04 17:07:30 UTC (rev 7758)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-06 03:25:26 UTC (rev 7759)
@@ -300,7 +300,7 @@
 		int i = 0;
 
 		try {
-		    proc = Runtime.getRuntime().exec(input);
+		    proc = Runtime.getRuntime().exec(new String[] {"bash", "-c", input});
 		    proc.waitFor();
             int exitcode = proc.exitValue();
             // If the shell returned a non-zero exit code, attempt to print stderr



From wilde at ci.uchicago.edu  Sun Apr  6 16:28:59 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Sun,  6 Apr 2014 16:28:59 -0500 (CDT)
Subject: [Swift-commit] r7760 - SwiftApps
Message-ID: <20140406212859.306179CFE8@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-06 16:28:58 -0500 (Sun, 06 Apr 2014)
New Revision: 7760

Added:
   SwiftApps/SimulatedAnnealing/
Log:
New project: generalized example of simulated annealing in Swift.



From wilde at ci.uchicago.edu  Sun Apr  6 17:47:27 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Sun,  6 Apr 2014 17:47:27 -0500 (CDT)
Subject: [Swift-commit] r7761 - in SwiftApps/SimulatedAnnealing: . params
Message-ID: <20140406224727.099A09CFE8@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-06 17:47:26 -0500 (Sun, 06 Apr 2014)
New Revision: 7761

Added:
   SwiftApps/SimulatedAnnealing/README
   SwiftApps/SimulatedAnnealing/annealing.swift
   SwiftApps/SimulatedAnnealing/colortext.swift
   SwiftApps/SimulatedAnnealing/math.swift
   SwiftApps/SimulatedAnnealing/params/
   SwiftApps/SimulatedAnnealing/params/ARFull
   SwiftApps/SimulatedAnnealing/params/ARtest01
   SwiftApps/SimulatedAnnealing/params/ARtest02
   SwiftApps/SimulatedAnnealing/params/ARtest03
   SwiftApps/SimulatedAnnealing/params/ARtest04
   SwiftApps/SimulatedAnnealing/params/ARtest05
   SwiftApps/SimulatedAnnealing/params/ARtest06
   SwiftApps/SimulatedAnnealing/params/BetaTest
   SwiftApps/SimulatedAnnealing/params/Fast01
   SwiftApps/SimulatedAnnealing/params/KMtest01
   SwiftApps/SimulatedAnnealing/params/MWtest01
   SwiftApps/SimulatedAnnealing/params/MWtest02
   SwiftApps/SimulatedAnnealing/params/MWtest03
   SwiftApps/SimulatedAnnealing/params/MWtest04
   SwiftApps/SimulatedAnnealing/params/MWtest05
   SwiftApps/SimulatedAnnealing/params/MWtest06
   SwiftApps/SimulatedAnnealing/params/MWtest07
   SwiftApps/SimulatedAnnealing/swiftopt.sh
Log:
Initial revision. Started recoding SciCol to generalize as an example annealing function.

Added: SwiftApps/SimulatedAnnealing/README
===================================================================
--- SwiftApps/SimulatedAnnealing/README	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/README	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,338 @@
+
+*** Getting started
+
+
+To run shorter tests on single local hosts (Mac, Linux, Beagle
+Sandbox, Swift lab machines):
+
+  cd $HOME
+  svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SciColSim
+  cd SciColSim
+
+  ./setup.sh  # just do this once. Will install Boost and Swift, and compile the optimizer
+
+  ./swiftopt.sh -s local -p Fast01 &    # Tells you what run dir its in.  Note the "&" !!!
+
+  cd runNNN                             # cd to that run dir eg run007
+  tail -f swift.out                     # to view progress
+  tail -f best_some_opt_swift.txt       # to view optimization results
+
+  ./swiftopt.sh -s pads  -p params/ARtest01    # Etc.
+
+
+swiftopt.sh command line arguments:
+
+  -s site to run on: can be local, pads, beagle 
+
+  -p param set to use, from param/ dir
+
+  FIXME: specify the variants and how to speciy params.
+  
+  FIXME: Params for gensites can also be in the -p file I think.
+
+  FIXME: create a few more tet variations
+
+
+Contents of parameter file:
+
+#--------  params/Fast01
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         109    # stops before this target innovation value
+target_innovation_increment   50     # increment target innovation by this amount 
+
+annealing_repeats             1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              10     # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 8      # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      1      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     2      # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0 
+  
+#-------
+
+You can add new param sets in the param dir (or anywhere): just copy an existing one and change the params.
+
+You can add new sites in the conf/ dir.  FIXME: for now, add gen.SITENAME files. Limited flexibility w/ tc
+
+
+*** QuickStart
+
+mkdir $HOME/SciColSim # scientific collaboration simulation application
+cd SciColSim
+svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SciColSim src
+cd src
+cp /home/wilde/AndreysOptimizer/src/
+
+wget http://sourceforge.net/projects/boost/files/boost/1.47.0/boost_1_47_0.tar.gz/download
+tar zxf boost_1_47_0.tar.gz
+
+make  # should make openmp-optimizer executable
+
+./testopt.py >& testopt.out
+
+# Needs Swift 0.93 in PATH
+
+./test-swift.sh >& test-swift.out
+
+*** OSG
+
+To run on OSG, run the following commands;
+voms-proxy-init -voms Engage -valid 72:00
+./start-osg <number of nodes to request>
+./swiftopt.sh -s grid -p Fast01 -w &
+
+This has only been tested with the Fast01 parameter set as of now. The "-w" 
+option sets file transfers to be handled by using wget. If you are using OSG
+with another site, you can use the -l option of swiftopt to separate short and 
+long runs based on the value of target_innovation. For example, you can use 
+Ranger for short runs, and OSG for long runs. Adjust conf/grid.cf app
+definitions for bash_short and bash_long according to your needs.
+
+*** Overview
+
+The code simulates the exploration of a chemical network by a research
+community.  The five major parameters determine the strategy of
+exploration, "target" is the number of new interactions discovered and
+"loss" is the number of experiments per one new positive finding.
+
+Terminology and application logic:
+
+  optimizer: the C++ based optimizer that performs simulated annealing
+             optimizetion of parameters to the evolve() function.
+
+  evolve(): the actual science code which does the "collaboration
+            simulation" desribed in Overview above.  Implemented as a
+            function within optimizer. Uses the boost graph lib.
+            Calculates "loss" - amount of effort expended in rerunning
+            scientific experiments.
+
+  multi_annealing(): the function within optimizer that does the 
+            simulated annealing optimization logic (multiple times)
+
+  multi_loss: performs a total of N independent repetitions of evolve()
+           on NWorker cores, in parallel. Each evolve() returns one loss
+           value, a scalar double. Returns average loss over all evolve()
+           simulations, and std dev.
+
+*** Files
+
+Code:
+
+testopt.py: top-most script to run the C++ optimizer (with no
+            swift). Based on a simpler Python script provided by
+            Andrey to run the C++ optimizer.
+
+test-orig.sh: original python script test script for optimizer,
+            provided by Andrey with some mods by Mike.
+
+optirun.swift: replaces top-level py script for the outermost loops
+            (temporarily deprecated: has not been maintained to match changes in optimizer)
+
+annealing.swift: the annealing logic of optimizer, moved to
+             swift. Calls optimizer(op=m) to run 1 multi_loss() of N
+             evolve() runs on NWorker cores. Implements a higher-level multi_loss
+             which spreads the N reruns of evolve() between M runs of optimizer(m)
+             each of which does R evolve runs. N = P*R.  Eg, n=2400, P=100, R=24, 
+             would use 100 nodes (invocations) of 24 cores each.
+
+
+tc specifies:
+
+   optimizer.sh (old) / evolve.sh (current)
+
+   sumloss.sh: calculates avg loss and its std dev
+     did this as a separate app because it was hard to do the sum()
+     logic in swift.  How best to do that?  Try recursion? fold()?
+   
+
+Data:
+
+  input:	movie_data.txt
+  
+  output: best_???  max?  itermiediate?   FIX
+
+
+*** How to build
+
+Copy and untar boost graph library to the current dir:
+Needs: boost_1_47_0.tar.gz
+Available for now from: CI: /home/wilde/AndreysOptimizer/src
+
+Should create dir "boost_1_47_0" in the current directory
+(svn checkout dir, which is:
+  https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SciColSim
+)
+
+Then run "make" -> uses Makefile
+
+  on mac, makes:  openmp-optimizer (multi_loss() changed to use OpenMP),
+                  dispatch-optimizer (orig code logic, only slight corrections)
+
+  on linux, makes: openmp-optimizer
+
+
+*** How to Run
+
+* Non-swift tests (py calling C++ optimizer app):
+
+  cp testopt.py t1.py etc; then edit t1.py etc to set the params you want.
+
+An example of a modified parameter set for testing a fast optimization
+loop on communicado/local.xml is:
+
+> paramset="mw"
+71a73,86
+> elif paramset == "mw":  # FAST TEST PARAMETERS by Mike
+>   
+>   startTarget      = 58
+>   endTarget        = 59
+>   incrTarget       = 50
+>   optimizerRepeats = 1
+>   evolveReruns     = 100
+>   annealingSteps   = 5
+>   NWorkers         = "4"
+>   openmp           = "OMP_NUM_THREADS=" + NWorkers
+>   operation        = "n"  # n=normal, m=manual (runs 1 multi_loss call)
+>   seed             = "1234567"
+>   app              = "./openmp-optimizer";
+> 
+
+Sample output is in sample.testopt.py.output
+
+* Swift tests:
+
+   /test-swift.sh >&ts3.out &
+
+(currently, parameters are set in a few different places in annealing.swift)
+
+Sample output (w/ bugs remaining) is in sample.test-swift.output
+
+
+Optimizer output (both from C++ optimizer and Swift) generates text
+with escape sequences to display colored text. Some versions of more
+and less need special options set (eg in env var?) to display the
+colors correctly.
+
+
+*** C++ Command line args
+
+Usage: super_optimizer alpha_i alpha_m beta gamma delta target_innov [n_epochs n_steps n_reruns] [range] [verbose_level]
+         [T_start T_end Annealing_steps Target_rejection Starting_jump]
+         [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta]
+
+*** Swift command line args and internal control vars
+    The command line options are:
+        -nworkers=<int>
+
+	-minrange=<int>
+	    Starting number for the target innovation value(Inclusive)
+
+	-maxrange=<int>
+	    Ending number for the target innovation value(exclusive)
+
+	-rangeinc=<int>
+	    How much to increment the target innovation value by
+
+	-tstart=<float>
+
+	-tend=<float>
+
+	-trejection=<float>
+
+	-evoreruns=<int>
+	   How many times to re-run the evolve.sh app  // MW: # times to rerun evolve() not evolve.sh ???
+
+	-startingjump=<float>
+	    Controls the dx and/or rejection variable
+
+	-alphai=<float>
+            parameter to the annealing process
+
+	-alpham=<float>
+            parameter to the annealing process
+
+	-beta=<float>
+            parameter to the annealing process
+
+	-gamma=<float>
+            parameter to the annealing process
+
+	-delta=<float>
+            parameter to the annealing process
+
+	-annealingcycles=<int>
+	    Number of times to run the annealing process for a given target innovation
+
+	-rerunsperapp=<int>
+	    Number of evolve reruns to do per app call
+
+	-nreps=<int>
+	    How many times to repeat the annealing process for a given target innovation
+
+
+*** C++ app flow logic ===
+
+for target in range(58, 1009 (used 209), 50):  // 20 values    Target Values
+  for i in range(15):                                          Optimization repeats (runs?)
+    #         P0:                 P1:               D:  P2:                   P3:
+    optimizer |0 0 4 50 -1 target | 40000 20 1000 2 | 1 | 2. 0.01 100 0.3 2.3 | 1 1 0 0 0
+
+    multi_annealing( un[NW], T_start, T_end, Target_rejection, Annealing_repeats, 
+                       starting_jump, Results, Counters, params0, Annealing_repeats);
+
+        Res = multi_loss(un, Results, Counters, x); // Initial
+        curr_x   = Res.first;
+        curr_err = Res.second;
+
+        for i = 0 to annealing_cycles - 1 (100 from .py script)
+          for j = 0 to 4 // 5X: 0..4 is fixed constant  - one iteration for each of 5 evolve() parameters
+            setParameters (for a round of parallel Annealing_repeats
+            Res = multi_loss(un, Results, Counters, Annealing_repeats);
+              In parallel: for 1 to AnnealingRepeats (1,000; 10,000 desired)
+                group repeats among NWorkers
+                  evolve()
+            setParameters  again here, conditionally? (check)
+
+TODO: Merge these two outlines into one:
+
+P  20 targets - py  (target loss)
+P    15 repeats - py
+S      1 initial multi_loss: 1000 to 10000 annealing_repeats
+S      100 Annealing_cycles (groups of 10? : cycle=10 ) (fast:50)
+S         5 repeats (fast: 1)
+P           multi_loss: 1000 to 10000 evolve re-runs
+S             evolve() (leaf) => 2 mins to 10 mins (is that time for one or for 1000?)
+
+# a) 20 targets (parallel)
+# b)   15 repeats (parallel) of optimizer (optimizer == multi_annealing):
+#        1 multi_loss to initialize
+# c)     100 Annealing_cycles (serial)
+# d)        5 repeats (1 per param, serial) of multi_loss:
+# e)          1000 to 10000 annealing_repeats (parallel) == multi_loss  
+# f)            evolve()
+
+Plots
+=====
+
+In order to generate plots from the swift stdout/stderr file, do the following
+
+./extract4plots <swift.out>
+gnuplot plotit
+
+gnuplot must be installed and should be in PATH.
+
+Following plots will be generated:
+activeplot.png: number of active jobs over time
+cumulativeplot.png: Cumulative number of jobs completed
+cumulativeplot-openmp.png: Cumulative number of openmp threads completed
+scs.png: SciColSim T value evolution
+scs_loss.png: Scicolsim Loss 
+multiloss.png: Scicolsim loss with stddev
+
+

Added: SwiftApps/SimulatedAnnealing/annealing.swift
===================================================================
--- SwiftApps/SimulatedAnnealing/annealing.swift	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/annealing.swift	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,487 @@
+import "math";
+import "colortext";
+
+type file;
+
+type Res
+{
+    float loss;
+    float sdev;
+    float tavg;
+    float tsdev;
+}
+
+type Checkpoint
+{
+  int i;
+  int j;
+  float dx0, dx1, dx2, dx3, dx4;
+  float rej0, rej1, rej2, rej3, rej4;
+  float target_innov;
+  float loss;
+  float alpha_i;
+  float alpha_m;
+  float beta;
+  float gamma;
+  float delta;
+  float sdev;
+}
+
+global boolean restart = (@arg("restart","false") == "true");
+tracef("Restart flag is %b\n", restart);
+
+global int iterate_adjust = @toint(@arg("iterate_adjust","1")); // FIXME!!! : 1 for 0.93, 0 for 0.94
+
+global boolean FIX_VARIABLES = true;
+global int var_fixed[] = [0,0,0,0,0];
+global int Nworkers = @toint(@arg("nworkers","4"));
+global int rerunsPerApp= @toint(@arg("rerunsperapp", "100"));
+
+( float nx ) newx( float x, float dx )
+{
+  float r = (random());
+
+  if (r > 0.5) {
+    nx = x + (random())*dx; //Java already returns a float between [0-1]
+  }
+  else {
+    nx = x - (random())*dx;
+  }
+  // tracef("newx(%f,%f)=%f\n",x,dx,nx);
+}
+
+app  (/*file outfile,*/  file objout ) orig_objective ( string args[], file objdata )
+{
+  # evolve @objout  args /*stdout=@outfile*/;  // FIXME: objdata file is passed implicitly
+}
+
+/*
+function y = simple_objective(x)
+   y = (4 - 2.1*x(1)^2 + x(1)^4/3)*x(1)^2 + x(1)*x(2) + ...
+       (-4 + 4*x(2)^2)*x(2)^2;
+*/
+
+(float y) objective ( float x[], file objdata )
+{
+   float x0 = x[0];
+   float x1 = x[1];
+   float x0s = x0 * x0;
+   float x1s = x1 * x1;
+   y = (4.0 - 2.1*x0s + pow(x[0],4.0/3.0))*x0s + x0*x1 + (-4.0 + 4.0*x1s)*x1s;
+}
+
+app (file outfile, file logfile ) sumobjective( file loss[] )
+{
+  sumloss @filename(logfile) @filenames(loss) stdout=@filename(outfile);
+}
+
+/* Program structure:
+
+   main
+     optimizer_sweep() - do N complete annealing optimizations
+       multi_annealing()
+         N_objective()
+           objective()
+         sumobjective()
+*/
+
+(file bestfile, file maxfile) multi_annealing (string best_filename,  # app
+                                               float T_start,         # ann
+                                               float T_end,           # ann
+                                               float Target_rejection,# ann
+                                               int obj_runs,          # app
+                                               float starting_jump,   # ann
+                                               float params0[],       # app f(x)
+                                               int annealing_cycles)  # ann
+{
+  int cycle      = 10;     // const
+  int NEVOPARAMS = 5;      // const - 5 params: alpha_i, alpha_m, beta, gamma, delta
+
+  float rejection[][];  // [i][j] where i is cycle and j is evolve-parameter (alpha_i, alpha_m, beta, gamma, delta)
+  float trejection[][];
+
+  float x[][], dx[][], curr_loss[], curr_sdev[];
+
+  Res mlres[][];
+
+  file ckptFile <single_file_mapper; file=@strcat("ckpt.",target_innov)>;
+  Checkpoint ckpt;
+  int restartIndex;
+
+  if (restart) {
+    ckpt = readData(ckptFile);
+    restartIndex = ckpt.i;
+  }
+  else {
+    restartIndex = -1;
+    mlres[0][0] = multi_loss(T_start, T_end, annealing_cycles, Target_rejection,
+                            starting_jump, 0, 0, params0, target_innov, evolve_reruns ); // FIXME: serves for all evolve-params ???
+    tracef( "multi_annealing: AR: initial: %f +- %f\n", mlres[0][0].loss, mlres[0][0].sdev );
+  
+    foreach j in [0:NEVOPARAMS-1] {
+      x[0][j]         = params0[j];
+      dx[0][j]        = starting_jump;
+      rejection[0][j] = 0.0;
+      curr_loss[j]    = mlres[0][0].loss;
+      curr_sdev[j]    = mlres[0][0].sdev;
+    }
+  }
+  
+  iterate iter_i {  // number of annealing cycles
+    int i = iter_i + 1;  // i ranges [1:n] in the swift script so that [0] can be the initial condition
+
+    // set new temperature, rejection threshold, and dx values for this cycle
+    float temperature = T_start*exp( @tofloat( i-1 ) * ( jlog( T_end ) - jlog( T_start ) ) / @tofloat( annealing_cycles ) );
+
+    tracef( @strcat( "multi_annealing: AR: i=%i ....T = ", color( 3, "%f" ),"\n" ), i, temperature );
+
+    if ( i <  restartIndex ) { 
+      tracef( "skipping index %i - less than restart index %i\n", i, restartIndex );
+    }
+    else { if ( i == restartIndex ) { // reset variables from the restart file
+      dx[i] = [ckpt.dx0, ckpt.dx1, ckpt.dx2, ckpt.dx3, ckpt.dx4];
+      rejection[i] = [ckpt.rej0, ckpt.rej1, ckpt.rej2, ckpt.rej3, ckpt.rej4];
+      x[i][0] = ckpt.alpha_i;
+      x[i][1] = ckpt.alpha_m;
+      x[i][2] = ckpt.beta;
+      x[i][3] = ckpt.gamma;
+      x[i][4] = ckpt.delta;
+      curr_loss[((i+1)*NEVOPARAMS)-1] = ckpt.loss;
+      curr_sdev[((i+1)*NEVOPARAMS)-1] = ckpt.sdev;
+    }
+    else { // i > restartIndex: proceed as normal whether restarting or not
+
+      // On each new "major" cycle within the annealing_cycles (other than the first) set new rejection and dx values
+  
+      if (i %% cycle == 1 && i > 1) {
+        tracef( "multi_annealing: new cycle at i=%i\n", i );
+        tracef( color( Pink, "multi_annealing: AR: New cycle at %i: prev dx[0-4]=[%f %f %f %f %f]\n" ),
+                       i, dx[i-1][0], dx[i-1][1], dx[i-1][2], dx[i-1][3], dx[i-1][4] );
+        foreach k in [0:NEVOPARAMS-1] {
+          float newrejection = rejection[i-1][k] / @tofloat( cycle );
+          if (newrejection > 0.0) {
+            dx[i][k] = dx[i-1][k] / ( newrejection / Target_rejection );
+            // FIXME: re-enable: rejection[i][k]=0.0;
+            trejection[i][k]=0.0;
+          }
+          else {
+            dx[i][k] = dx[i-1][k] * 2.0;
+            // FIXME: re-enable: rejection[i][k]=rejection[i-1][k];
+                      trejection[i][k]=newrejection;
+          }
+                  // FIXME: HANGS? : tracef(color(Red,"Recomputed rejection: i=%d k=%d dx[i][k]=%f\n"), i, k, dx[i][k]);
+        }
+        tracef( color( Blue, "multi_annealing: AR: New cycle at %i: dx[0-4]=[%f %f %f %f %f]\n" ),
+                               i, dx[i][0], dx[i][1], dx[i][2], dx[i][3], dx[i][4] );
+      }
+      else { // If not new cycle, set dx[i][*] from previous dx ([i-1]). rejection[i][j] is set later.
+        foreach k in [0:NEVOPARAMS-1] {
+          dx[i][k] = dx[i-1][k];
+        }
+      }
+      iterate j {  // Try a new value for each non-fixed param; then write results and accept or reject
+        int curr = ( i * NEVOPARAMS ) + j;
+        int prev = curr-1;
+  
+        if ( /* (!FIX_VARIABLES) || */ ( var_fixed[j] == 0 ) ) {  // Adjustable vars
+             // fixed=1,1,0,0,0: FIXME: FIX_VARIABLES flag has faulty logic but OK when TRUE
+          float try_x[];
+          foreach k in [0:NEVOPARAMS-1] { // Select the evolve params to try
+            if (k < j) {
+              try_x[k] = x[i][k]; // already set x[i][k]
+            }
+            else {
+              if (k == j) {
+                try_x[k] = newx(x[i-1][j], dx[i-1][j]); // permute x[i-1][j]
+              }
+              else { // k > j
+                try_x[k] = x[i-1][k]; // use x[i-1][k] (from prior cycle)
+              }
+            }
+          }
+          tracef( @strcat( "multi_annealing: AR: ", color( 10,"%f" ), " ", color( 9,"%i" ),"\n" ), try_x[j], j );
+
+          # Call objective function here:
+
+#         mlres[i][j] = multi_loss( T_start, T_end, annealing_cycles, Target_rejection, starting_jump,
+                                    i, j, try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_rerusn
+          mlres[i][j] = multi_loss( try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_reruns
+
+          tracef( "multi_annealing: AR: %f +- %f\n", mlres[i][j].loss, mlres[i][j].sdev );
+          // Beyond this point, x[] and dx[] are being set for this i,j
+          float ALOT = 100000000000.0; // 100,000,000,000. = 10^11
+          if (mlres[i][j].loss < ALOT) {
+            fprintf( best_filename, "N, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n",
+                    mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss,
+                    try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev );
+                    // Note i-1 field: print that way to match with C++ output
+  
+                    // fprintf( "max_dist_swift.txt", color( Red,"multi_annealing: AF: max_dist.txt - tbd\n" ) );
+                    // FIXME: max_dist is global set in evolve()
+          }
+          else { // does this ever occur? if so did we want to still do the ratio computation above???
+            fprintf( best_filename, "A, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n",
+                     mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss,
+                     try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev );
+                     // Note i-1 field: print that way to match with C++ output
+            //tracef( "multi_annealing: Loss %f > ALOT at [i][j] = [%d][%d]\n", mlres[i][j].loss, i ,j );
+          }
+          float ratio = min( 1.0, exp( -( mlres[i][j].loss - curr_loss[prev] ) / temperature ) );
+          float r = (random()); //Java already returns a random float between [0.0-1.0]
+          tracef("multi_annealing: AR: %f vs %f\n", r, ratio);
+          if (r > ratio) { // Reject new parameter
+            x[i][j] = x[i-1][j];
+            if (i %% cycle == 1 && i > 1) {
+              rejection[i][j] = trejection[i][j] + 1.0;    // FIXME: triple-check this!
+            }
+            else {
+              rejection[i][j] = rejection[i-1][j] + 1.0;  // FIXME: AR: Is this correct? incr rejection?
+            }
+            curr_loss[curr] = curr_loss[prev];
+            curr_sdev[curr] = curr_sdev[prev];
+            // FIXME: AR: the following prints seem to replicate values in the .cpp version - please clarify.
+            tracef( "multi_annealing: AR: %i,%i %i Did not accept: %f (%i)\n", i, j, i, try_x[j], j );
+            tracef( "multi_annealing: AR: %f %f %f %f %f\n", try_x[0],try_x[1],try_x[2],try_x[3],try_x[4] );
+          }
+          else { // Accept new parameter
+            tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i\n",i,j );
+            x[i][j] = try_x[j];
+            if (i %% cycle == 1 && i > 1) {
+              rejection[i][j] = trejection[i][j];    // FIXME: triple-check this!
+            }
+            else {
+              rejection[i][j] = rejection[i-1][j];  // FIXME: AR: Is this correct? no incr of rejection?
+            }
+            curr_loss[curr] = mlres[i][j].loss;
+            curr_sdev[curr] = mlres[i][j].sdev;
+            tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i try_x[j]=%f\n", i, j, try_x[j] );
+            float rj[];
+            foreach k in [0:NEVOPARAMS-1] {
+              if (k <= j) {
+                rj[k] = rejection[i][k]; // Was either set from previous j or just set for this j
+              }
+              else {
+                rj[k] = rejection[i-1][k]; // Not yet set, use previous
+              }
+            }
+            tracef(@strcat("multi_annealing: AR: [%i][%i] ", color(8,"Rejection counts: "),
+                           color(1,"%f"), " ", color(7,"%f"), " ", color(5,"%f"), " ", color(9,"%f"), " ", color(6,"%f"), "\n\n"),
+                           i, j, rj[0], rj[1], rj[2], rj[3], rj[4]);
+            tracef(@strcat("multi_annealing: AR: %i ", color(8,"***** Did accept! "),
+                           color(1,"%f"), " ", color(7,"%f"), " ", color(5,"%f"), " ", color(9,"%f"), " ", color(6,"%f"), "\n\n"),
+                           i, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4]);
+            }
+          }
+          else {// Fixed Vars
+            x[i][j] = x[i-1][j];
+            rejection[i][j] = rejection[i-1][j];
+            curr_loss[curr] = curr_loss[prev];
+            curr_sdev[curr] = curr_sdev[prev];
+            // dx[i][j] not set for fixed vars
+          }
+        } until( j == (NEVOPARAMS-iterate_adjust) );
+      }} // of if/else 
+    } until( iter_i == (annealing_cycles-iterate_adjust) );
+}
+
+/*
+(Res r) multi_loss(float t_start, float t_end, int annealing_steps, float t_rejection, float starting_jump,
+                   int ci, int cj, float x[], float target_innov, int evolve_reruns)
+*/
+
+(Res r) N_objective(float x[], int obj_runs)
+{
+  tracef("%q\n", x);
+  file rfile[];
+  //file ofile[]; // FIXME: to obtain timings and other stats
+
+  tracef( "N_objective: entered: obj_runs=%i x=%q\n", obj_runs, x );
+
+  int appCalls = @toint(@tofloat(obj_runs)/@tofloat(runsPerApp)); // FIXME: handle fractional issues and rounding etc.
+                                                                         // For now must divide evenly
+  tracef("N_objective appCalls=%i\n", appCalls);
+
+  foreach i in [1:appCalls] { // repeats of the objective() - same as n_reruns
+    //file outfile; // FIXME: map and save in future
+    string args[] = [ // FIXME: move this to a setargs() function
+            //    alpha_i           alpha_m        beta           gamma          delta
+                 @strcat(x[0]),  @strcat(x[1]), @strcat(x[2]), @strcat(x[3]), @strcat(x[4]),
+            //     n_epochs n_steps   n_runs                  range
+                  "40000",   "20",  @strcat(rerunsPerApp),     "2",
+            //    verbose_level
+                    "1",
+            //    operation-code:(m,a)  Nworkers           seed
+                  "m",                  @strcat(Nworkers), @arg("seed", "0" ) ];
+
+    file graph <"movie_graph.txt">;
+
+    (/*outfile,*/ rfile[i]) = objective(args, graph);
+    tracef("multi_loss: i=%i calling objective, args=%q\n", i, args);
+    // tracef("multi_objective: after objective: i=%i %k %k\n", i, outfile, rfile[i]);
+  }
+
+  string anneal_cycle;
+  if(ci==0) {
+    anneal_cycle="init";
+  }
+  else {
+    anneal_cycle=@strcat(ci-1);
+  }
+    
+  file sumfile<single_file_mapper; file=@strcat("sumloss/",anneal_cycle,".",cj,".sumloss")>;
+  file logfile<single_file_mapper; file=@strcat("sumloss/",anneal_cycle,".",cj,".sumlog")>;
+
+  (sumfile, logfile) = sumloss(rfile);
+  r = readData(sumfile);
+
+  tracef("multi_loss: returning: ci=%i cj=%i r.loss=%f r.sdev=%f\n",ci,cj,r.loss,r.sdev);
+  // file statfile = sumstats(ofile);  FIXME: to obtain timings and other stats
+  // s = readStat(statsfile);          FIXME: to obtain timings and other stats
+}
+
+optimizer_serial_sweep() // Implements logic of python driver script
+{
+
+  int minrange = @toint(@arg("minrange", "58"));
+  int maxrange = @toint(@arg("maxrange", "59"));
+  int rangeinc = @toint(@arg("rangeinc", "50"));
+
+  // FIXME: add provision for random priming and random param values when x[i] == -100 (see optimizer.cpp main())
+
+  int nreps = @toint(@arg("nreps", "1"));
+
+//file bestfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".best_opt_some")>;
+//file maxfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".max_dist")>;
+
+#  foreach target_innov in [minrange:maxrange:rangeinc] {
+  iterate i {
+    int target_innov = minrange + (i*rangeinc);
+    foreach rep in [1:nreps] {
+
+      string best_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" );
+      file outfile;  // <single_file_mapper; file=@strcat("output/T",target_innov,".R",rep,".out")>;
+      file lossfile; // <single_file_mapper; file=@strcat("output/T",target_innov,".R",rep,".loss_data")>;
+
+      (outfile,lossfile) = multi_annealing(best_filename,
+		                           @tofloat(@arg("tstart", "2.0")),
+                                           @tofloat(@arg("tend", "0.01")),
+                                           @tofloat(@arg("trejection", "0.3")),
+                                           @toint(@arg("evoreruns", "100")),
+                                           @tofloat(@arg("startingjump", "2.3")),
+                                           [ @tofloat(@arg("alphai", "0.0")),
+                                             @tofloat(@arg("alpham", "0.0")),
+                                             @tofloat(@arg("beta", "4.0")),
+                                             @tofloat(@arg("gamma", "50.0")),
+                                             @tofloat(@arg("delta", "-1.0"))
+                                           ],
+                                           @tofloat(target_innov),
+                                           @toint(@arg("annealingcycles", "50")) );
+    }
+  } until(target_innov >= (maxrange-rangeinc));
+}
+
+optimizer_sweep() // Implements logic of python driver script
+{
+
+  int minrange = @toint(@arg("minrange", "58"));
+  int maxrange = @toint(@arg("maxrange", "59"));
+  int rangeinc = @toint(@arg("rangeinc", "50"));
+
+  // FIXME: add provision for random priming and random param values when x[i] == -100 (see optimizer.cpp main())
+
+  int nreps = @toint(@arg("nreps", "1"));
+
+//file bestfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".best_opt_some")>;
+//file maxfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".max_dist")>;
+
+  foreach target_innov in [minrange:maxrange:rangeinc] {
+    foreach rep in [1:nreps] {
+
+      string best_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" );
+      file outfile;  // <single_file_mapper; file=@strcat("output/T",target_innov,".R",rep,".out")>;
+      file lossfile; // <single_file_mapper; file=@strcat("output/T",target_innov,".R",rep,".loss_data")>;
+
+      (outfile,lossfile) = multi_annealing(best_filename,
+		                           @tofloat(@arg("tstart", "2.0")),
+                                           @tofloat(@arg("tend", "0.01")),
+                                           @tofloat(@arg("trejection", "0.3")),
+                                           @toint(@arg("evoreruns", "100")),
+                                           @tofloat(@arg("startingjump", "2.3")),
+                                           [ @tofloat(@arg("alphai", "0.0")),
+                                             @tofloat(@arg("alpham", "0.0")),
+                                             @tofloat(@arg("beta", "4.0")),
+                                             @tofloat(@arg("gamma", "50.0")),
+                                             @tofloat(@arg("delta", "-1.0"))
+                                           ],
+                                           @tofloat(target_innov),
+                                           @toint(@arg("annealingcycles", "50")) );
+    }
+  }
+}
+
+main()
+{
+  optimizer_sweep();
+}
+
+main();
+
+/*
+
+  Program structure:
+
+  main
+  optimizer_sweep()
+  multi_annealing()
+  multi_loss()
+  evolve()
+  sumloss()
+
+  Example parameter sets:
+
+  for target in range(58,59,50):
+  for i in range(1):
+  args="./toptimizer 0 0 4 50 -1 "+target+" 40000 20 75    2 1 2. 0.01 2 0.3 2.3 1 1 1 0 0 m // > out.T"+str(target)+".i"+str(i)
+  os.system(args);
+
+  string fastargs1[] = [
+  "0", "0", "4", "50", "-1", @strcat(target),
+  "40000", "20", "1000", "2",
+  "1",
+  "2.", "0.01", "100", "0.3", "2.3",
+  "1", "1", "0", "0", "0"];
+  string fastargs2[] = [
+  "0", "0", "4", "50", "-1", @strcat(target),
+  "40000", "20", "1000", "2",
+  "1",
+  "2.", "0.01",  "5", "0.3", "2.3",
+  "1", "1", "0", "0", "0", "m"];
+  string fastargs3[] = [
+  "0", "0", "4", "50", "-1", @strcat(target),
+  "40000", "20", @strcat(repeats), "2",
+  "1",
+  "2.", "0.01",  "2", "0.3", "2.3",
+  "1", "1", "0", "0", "0", "m"];
+*/
+
+(string args[]) setargs()
+{
+    // string longargs[] = @strcat("0 0 4 50 -1 ",target," 40000 20 1000 2 1 2. 0.01 100 0.3 2.3 1 1 0 0 0 m");
+
+    //  [alpha_i alpha_m beta gamma delta target_innov
+    //  [n_epochs n_steps n_reruns] [range]
+    //  [verbose_level]
+    //  [T_start T_end Annealing_steps Target_rejection Starting_jump]
+    //  [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta] [operation-code:(m,a) Nworkers]
+}
+
+////////////////// HOLD JUNK
+
+//                    tracef(@strcat("multi_annealing: AR: %i ", color(8,"Rejection counts: "),
+//                                   color( /* 2 */ 1," %f"), "\n\n"),
+//                                   i, rejection[i][j] ); // , rejection[i][1], rejection[i][2], rejection[i][3], rejection[i][4]);
+// FIXME: determine correct rejection[] values to avoid hanging:
+//                    tracef(@strcat("multi_annealing: AR: %i ", color(8,"Rejection counts: "),
+//                                   color( /* 2 */ 1," %f"), color(7," %f"), color(5," %f"), color(9," %f"), color(6," %f"), "\n\n"),
+//                                   rejection[i][0], rejection[i][1], rejection[i][2], rejection[i][3], rejection[i][4]);
+// END FIXME

Added: SwiftApps/SimulatedAnnealing/colortext.swift
===================================================================
--- SwiftApps/SimulatedAnnealing/colortext.swift	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/colortext.swift	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,90 @@
+global string e33; // = @sprintf("\\e") or  @arg("e33");
+
+if( @arg("e33","null") == "null" ) {
+  e33 = @sprintf("\\e");
+}
+else {
+  e33 = @arg("e33");
+}
+
+global string endEscape = "[0m";
+
+// Color codes used in Andrey's cpp optimizer:
+
+global int Black = 0;
+global int Blue = 1;
+global int BlackOnBlue = 2;
+global int Pink = 3;
+global int GoldOnBlue = 4;
+global int BlueOnGray = 5;
+global int Red = 6;
+global int Gold = 7;
+global int BlueOnGold = 8;
+global int Gray = 9;
+global int BlackOnGray = 10;
+
+# Black = 0;	   
+# Blue = 1;	   
+# BlackOnBlue = 2; 
+# Pink = 3;	   
+# GoldOnBlue = 4;  
+# BlueOnGray = 5;  
+# Red = 6;	   
+# Gold = 7;	   
+# BlueOnGold = 8;  
+# Gray = 9;	   
+# BlackOnGray = 10;
+
+global string colorCode[] = [
+ "[1;29m",      # Black = 0;	   
+ "[1;34m",	# Blue = 1;	   
+ "[1;44m",	# BlackOnBlue = 2; 
+ "[1;35m",	# Pink = 3;	   
+ "[1;33;44m",	# GoldOnBlue = 4;  
+ "[1;47;34m",	# BlueOnGray = 5;  
+ "[1;1;31m",	# Red = 6;	   
+ "[1;1;33m",	# Gold = 7;	   
+ "[1;1;43;34m",	# BlueOnGold = 8;  
+ "[1;1;37m",	# Gray = 9;	   
+ "[1;30;47m",	# BlackOnGray = 10;
+];
+
+(string s) color(int c, string ins)
+{
+  s = @strcat(e33,colorCode[c],ins,e33,endEscape);
+}
+
+(string s) ncolor(int c, string ins) // Can use this version if \\e handling is available in current Swift
+{
+  s = @sprintf(@strcat("\\e",colorCode[c],ins,"\\e",endEscape));  // sprintf applies \\e escape processing
+}
+
+(string s) OLDcolor(int c, string ins)
+{
+    switch(c){
+        case 0:
+            s = @strcat(e33,"[1;29m",ins,e33,"[0m");
+        case 1:
+            s = @strcat(e33,"[1;34m",ins,e33,"[0m");
+        case 2:
+            s = @strcat(e33,"[1;44m",ins,e33,"[0m");
+        case 3:
+            s = @strcat(e33,"[1;35m",ins,e33,"[0m");
+        case 4:
+            s = @strcat(e33,"[1;33;44m",ins,e33,"[0m");
+        case 5:
+            s = @strcat(e33,"[1;47;34m",ins,e33,"[0m");
+        case 6:
+            s = @strcat(e33,"[1;1;31m",ins,e33,"[0m");
+        case 7:
+            s = @strcat(e33,"[1;1;33m",ins,e33,"[0m");
+        case 8:
+            s = @strcat(e33,"[1;1;43;34m",ins,e33,"[0m");
+        case 9:
+            s = @strcat(e33,"[1;1;37m",ins,e33,"[0m");
+        case 10:
+            s = @strcat(e33,"[1;30;47m",ins,e33,"[0m");
+        default:
+            s = ins;
+    }
+}

Added: SwiftApps/SimulatedAnnealing/math.swift
===================================================================
--- SwiftApps/SimulatedAnnealing/math.swift	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/math.swift	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,76 @@
+
+(float result) sin(float x)
+{
+   result = @java("java.lang.Math", "sin", x);
+}
+
+(float result) exp(float x)
+{
+   result = @java("java.lang.Math", "exp", x);
+}
+
+(float result) jlog(float x)
+{
+   result = @java("java.lang.Math", "log", x);
+}
+
+(float result) log10(float x)
+{
+   result = @java("java.lang.Math", "log10", x);
+}
+
+(float result) ceil(float x)
+{
+   result = @java("java.lang.Math", "ceil", x);
+}
+
+(float result) floor (float x)
+{
+   result = @java("java.lang.Math", "floor", x);
+}
+
+(float result) min (float a, float b)
+{
+   //result = @java("java.lang.Math", "min", a, b);
+   if ( a < b ) {
+       result = a;
+   }
+   else {
+       result = b;
+   }
+   tracef("math/min: result=%f\n",result);
+}
+
+(float result) pow (float x, float y)
+{
+   result = @java("java.lang.Math", "pow", x, y);
+}
+
+(float result) random ()
+{
+  result = @java("java.lang.Math","random");
+}
+
+// Functions below are exprimental and do not work
+// FIXME: fix 0.93 problems casting ints as doubles internally.
+//  This breaks the @java interface for Swift ints.
+
+(string result) itos (int i)
+{
+  result = @java("java.lang.Integer","toString",i);
+}
+
+(string result) itos2 (int i)
+{
+  result = @java("java.lang.String","valueOf",i);
+}
+
+(string result) ctos (int i)
+{
+  result = @java("java.lang.Character","valueOf",i);
+}
+
+(string result) format (string f, int c)
+{
+  result = @java("java.lang.String", "format", f, c );
+}

Added: SwiftApps/SimulatedAnnealing/params/ARFull
===================================================================
--- SwiftApps/SimulatedAnnealing/params/ARFull	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/ARFull	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,18 @@
+
+min_target_innovation        58     # starting target innovation value to try
+max_target_innovation        1009    # stops before this target innovation value
+target_innovation_increment  50     # increment target innovation by this amount
+
+annealing_repeats            1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles             100    # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                10000   # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                     24      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation    240     # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                      0.0     # 5 evolve() parameters: must be decimal values !!!
+alpha_m                      0.0
+beta                         4.0
+gamma                        50.0
+delta                        -1.0
+

Added: SwiftApps/SimulatedAnnealing/params/ARtest01
===================================================================
--- SwiftApps/SimulatedAnnealing/params/ARtest01	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/ARtest01	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,18 @@
+
+min_target_innovation       = 58     # starting target innovation value to try
+max_target_innovation       = 209    # stops before this target innovation value
+target_innovation_increment = 50     # increment target innovation by this amount 
+
+annealing_repeats           = 15     # times to repeate the entire optimization process for each target_innovation
+annealing_cycles            = 100    # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns               = 1000   # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                    = 2      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation   = 100    # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                     = 0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                     = 0.0    # alpha_i and alpha_m are fixed for now
+beta                        = 4.0    # rest are varied by the optimization process
+gamma                       = 50.0
+delta                       = -1.0 
+ 

Added: SwiftApps/SimulatedAnnealing/params/ARtest02
===================================================================
--- SwiftApps/SimulatedAnnealing/params/ARtest02	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/ARtest02	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,18 @@
+
+min_target_innovation        58     # starting target innovation value to try
+max_target_innovation        59    # stops before this target innovation value
+target_innovation_increment  50     # increment target innovation by this amount 
+
+annealing_repeats            1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles             100    # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                10000   # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                     24      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation    48     # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                      0.0     # 5 evolve() parameters: must be decimal values !!!
+alpha_m                      0.0     # alpha_i and alpha_m are fixed for now
+beta                         4.0     # rest are varied by the optimization process
+gamma                        50.0
+delta                        -1.0 
+ 

Added: SwiftApps/SimulatedAnnealing/params/ARtest03
===================================================================
--- SwiftApps/SimulatedAnnealing/params/ARtest03	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/ARtest03	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,18 @@
+
+min_target_innovation        50    # starting target innovation value to try
+max_target_innovation        51    # stops before this target innovation value
+target_innovation_increment  135    # increment target innovation by this amount 
+
+annealing_repeats            1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles             25     # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                96     # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                     24     # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation    24     # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                      0.0     # 5 evolve() parameters: must be decimal values !!!
+alpha_m                      0.0     # alpha_i and alpha_m are fixed for now
+beta                         0.0     # rest are varied by the optimization process
+gamma                        0.0
+delta                        0.0 
+ 

Added: SwiftApps/SimulatedAnnealing/params/ARtest04
===================================================================
--- SwiftApps/SimulatedAnnealing/params/ARtest04	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/ARtest04	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,18 @@
+
+min_target_innovation        270     # starting target innovation value to try
+max_target_innovation        1357    # stops before this target innovation value
+target_innovation_increment  540     # increment target innovation by this amount 
+
+annealing_repeats            1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles             100      # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                10080    # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                     24     # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation    24     # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                      0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                      0.0    # alpha_i and alpha_m are fixed for now
+beta                         0.0    # rest are varied by the optimization process
+gamma                        0.0
+delta                        0.0 
+ 

Added: SwiftApps/SimulatedAnnealing/params/ARtest05
===================================================================
--- SwiftApps/SimulatedAnnealing/params/ARtest05	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/ARtest05	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,17 @@
+
+min_target_innovation        67     # starting target innovation value to try: 67 134 201 268 335 402 469 536 603 670
+max_target_innovation        672    # stops before this target innovation value
+target_innovation_increment  67     # increment target innovation by this amount 
+
+annealing_repeats            1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles             100      # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                5040   # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                     24     # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation    240    # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                      0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                      0.0    # alpha_i and alpha_m are fixed for now
+beta                         0.0    # rest are varied by the optimization process
+gamma                        0.0
+delta                        0.0 

Added: SwiftApps/SimulatedAnnealing/params/ARtest06
===================================================================
--- SwiftApps/SimulatedAnnealing/params/ARtest06	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/ARtest06	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,19 @@
+
+min_target_innovation        67     # starting target innovation value to try: 67 134 201 268 335 402 469 536 603 670
+max_target_innovation        672    # stops before this target innovation value
+target_innovation_increment  67     # increment target innovation by this amount 
+
+annealing_repeats            1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles             100      # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                5040   # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                     24     # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation    120    # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                      0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                      0.0    # alpha_i and alpha_m are fixed for now
+beta                         0.0    # rest are varied by the optimization process
+gamma                        0.0
+delta                        0.0 
+
+starting_jump                0.3

Added: SwiftApps/SimulatedAnnealing/params/BetaTest
===================================================================
--- SwiftApps/SimulatedAnnealing/params/BetaTest	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/BetaTest	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,20 @@
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         59     # stops before this target innovation value
+target_innovation_increment   50     # increment target innovation by this amount
+
+annealing_repeats             1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              100     # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 60      # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      1      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     2      # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0
+
+
+JOB_THROTTLE                      .26

Added: SwiftApps/SimulatedAnnealing/params/Fast01
===================================================================
--- SwiftApps/SimulatedAnnealing/params/Fast01	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/Fast01	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,20 @@
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         59    # stops before this target innovation value
+target_innovation_increment   50     # increment target innovation by this amount
+
+annealing_repeats             1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              10     # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 100      # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      24      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     48      # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0
+
+
+JOB_THROTTLE                      .05

Added: SwiftApps/SimulatedAnnealing/params/KMtest01
===================================================================
--- SwiftApps/SimulatedAnnealing/params/KMtest01	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/KMtest01	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,20 @@
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         70    # stops before this target innovation value
+target_innovation_increment   1     # increment target innovation by this amount
+
+annealing_repeats             1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              100     # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 1000      # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      20      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     200      # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0
+
+
+JOB_THROTTLE                  9.99

Added: SwiftApps/SimulatedAnnealing/params/MWtest01
===================================================================
--- SwiftApps/SimulatedAnnealing/params/MWtest01	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/MWtest01	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,17 @@
+
+min_target_innovation       = 58
+max_target_innovation       = 1009
+target_innovation_increment = 50
+
+annealing_repeats           = 1
+annealing_cycles            = 100
+evolve_reruns               = 1000
+
+alpha_i                     = 0.0  # 5 parameters: must be decimal values !!!
+alpha_m                     = 0.0
+beta                        = 4.0
+gamma                       = 50.0
+delta                       = -1.0 
+ 
+nworkers                    = 2
+reruns_per_opt_invocation   = 100    # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!

Added: SwiftApps/SimulatedAnnealing/params/MWtest02
===================================================================
--- SwiftApps/SimulatedAnnealing/params/MWtest02	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/MWtest02	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,21 @@
+
+#   Test with one target_innovation and few evolve_reruns to verify numeric behavior
+
+min_target_innovation         67     # starting target innovation value to try
+max_target_innovation         68     # stops before this target innovation value
+target_innovation_increment   135    # increment target innovation by this amount 
+
+annealing_repeats             1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              20     # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 4     # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      1      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     4     # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          0.0    # rest are varied by the optimization process
+gamma                         0.0
+delta                         0.0 
+
+starting_jump                 0.3

Added: SwiftApps/SimulatedAnnealing/params/MWtest03
===================================================================
--- SwiftApps/SimulatedAnnealing/params/MWtest03	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/MWtest03	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,19 @@
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         109    # stops before this target innovation value
+target_innovation_increment   50     # increment target innovation by this amount
+
+annealing_repeats             3      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              100    # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 16      # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      8       # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     16      # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0
+
+JOB_THROTTLE                  .10

Added: SwiftApps/SimulatedAnnealing/params/MWtest04
===================================================================
--- SwiftApps/SimulatedAnnealing/params/MWtest04	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/MWtest04	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,19 @@
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         1009   # stops before this target innovation value
+target_innovation_increment   50     # increment target innovation by this amount
+
+annealing_repeats             1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              100    # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 160    # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      8      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     80     # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0
+
+JOB_THROTTLE                  3.99

Added: SwiftApps/SimulatedAnnealing/params/MWtest05
===================================================================
--- SwiftApps/SimulatedAnnealing/params/MWtest05	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/MWtest05	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,19 @@
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         109    # stops before this target innovation value
+target_innovation_increment   50     # increment target innovation by this amount
+
+annealing_repeats             3      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              100    # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 48      # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      24      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     48      # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0
+
+JOB_THROTTLE                  .10

Added: SwiftApps/SimulatedAnnealing/params/MWtest06
===================================================================
--- SwiftApps/SimulatedAnnealing/params/MWtest06	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/MWtest06	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,18 @@
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         1009   # stops before this target innovation value
+target_innovation_increment   50     # increment target innovation by this amount
+
+annealing_repeats             2      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              100    # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 960     # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      24      # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     240     # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0
+

Added: SwiftApps/SimulatedAnnealing/params/MWtest07
===================================================================
--- SwiftApps/SimulatedAnnealing/params/MWtest07	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/params/MWtest07	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,18 @@
+
+min_target_innovation         58     # starting target innovation value to try
+max_target_innovation         1009   # stops before this target innovation value
+target_innovation_increment   50     # increment target innovation by this amount
+
+annealing_repeats             1      # times to repeate the entire optimization process for each target_innovation
+annealing_cycles              100    # times to perform the simulated annealing loop for all non-fixed parameters
+
+evolve_reruns                 400     # times to perform evolve_to_target_and_save() (objective function - is batched)
+nworkers                      8       # number of parallel threads (cores) to use per invocation of C++ optimizer
+reruns_per_opt_invocation     40      # reruns_per_opt_invocation mod nworkers must == 0 !!! <=== NOTE WELL !!!
+
+alpha_i                       0.0    # 5 evolve() parameters: must be decimal values !!!
+alpha_m                       0.0    # alpha_i and alpha_m are fixed for now
+beta                          4.0    # rest are varied by the optimization process
+gamma                         50.0
+delta                         -1.0
+

Added: SwiftApps/SimulatedAnnealing/swiftopt.sh
===================================================================
--- SwiftApps/SimulatedAnnealing/swiftopt.sh	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/swiftopt.sh	2014-04-06 22:47:26 UTC (rev 7761)
@@ -0,0 +1,191 @@
+#!/bin/bash
+
+# Usage: swiftopt.sh [-s sitename] [-p paramfile] [-w] [-n # for dryrun ]
+#
+# NOTE: this command expects symlink "swift" in the cur dir to point
+# to relese installed by setup.sh If you want to run with a different
+# swift release, replace symlink "swift" with a link to your swift
+# release dir.
+
+usage="$0 [-s sitename] [-p paramfile] [-n] [-w] [-l] # -n for dryrun: just print params and time estimates"
+
+# Function to run Swift
+runswift() {
+   SWIFT_HEAP_MAX=$ram SWIFT_LIB=.. swift >> swift.out 2>&1 -tc.file tc.data -sites.file $1 -config cf annealing.swift -e33="$escapecode" \
+                                            \
+   -minrange=$min_target_innovation         \
+   -maxrange=$max_target_innovation         \
+   -rangeinc=$target_innovation_increment   \
+                                            \
+   -nreps=$annealing_repeats                \
+   -annealingcycles=$annealing_cycles       \
+   -evoreruns=$evolve_reruns                \
+                                            \
+   -alphai=$alpha_i                         \
+   -alpham=$alpha_m                         \
+   -beta=$beta                              \
+   -gamma=$gamma                            \
+   -delta=$delta                            \
+                                            \
+   -startingjump=$starting_jump             \
+                                            \
+   -nworkers=$nworkers                      \
+   -rerunsperapp=$reruns_per_opt_invocation \
+   -wget=$wget				    \
+   -longruns=$longruns                      \
+   -restart=$restart
+}
+
+
+# Default settings
+execsite=local
+paramfile=Fast01
+ram=6000M
+dryrun=
+escapecode=$(printf '\033')
+variable_params=5     # Currently CONSTANT, will want to have this vary
+wget=false
+longruns=false
+
+# Process command line arguments
+while [ $# -gt 0 ]; do
+  case $1 in
+    -l) longruns=true; shift 1;; # FIXME
+    -n) dryrun=true; shift 1;;
+    -p) paramfile=$2; shift 2;;
+    -s) execsite=$2; shift 2;;
+    -w) wget=true; shift 1;;     # FIXME
+    -r) resdir=$2; shift 2;;
+    *) echo $usage 1>&2
+       exit 1;;
+  esac
+done
+
+# Create next unique run id and run directory
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+
+#Exit if rundir already exits. Something is funky
+if [ -d $rundir ];
+then
+    echo "$rundir already exists! exiting." >&2
+    exit 2
+else
+    mkdir $rundir
+fi
+
+if [ _$resdir != _ ]; then
+  if [ -d resdir ]; then
+    restart=true
+    cp $resdir/ckpt.* $rundir
+  else
+    echo $0: no restart directory $resdir found.
+    exit 1
+  fi
+else
+  restart=false
+fi
+
+# Get optimization parameters
+if [ ! -f params/$paramfile ];
+then
+    echo "Could not find parameter file $paramfile in params!"
+    exit 1
+fi
+cp params/$paramfile $rundir/paramfile
+sed -e '/^[[:space:]]*\(#.*\)*$/d' -e 's/#.*//' -e 's/  */=/' -e 's/^/export /' <params/$paramfile >$rundir/params.annealing
+source $rundir/params.annealing
+#swift=../swift/bin/swift # relative to runNNN/ dirs
+echo Optimization $rundir: site=$execsite paramfile=$paramfile
+
+# Report an error if configuration files are missing
+if [ ! -f "conf/$execsite.xml" ] && [ ! -f "conf/$execsite.conf" ]; then
+   echo Unable to find requested configuration file for site $execsite
+   exit 1
+fi
+
+# Use start-coaster-service if site is a .conf file
+if [ -f "conf/$execsite.conf" ]; then
+   USE_SCS=1
+fi
+
+# Check for missing .cf files
+if [ -f "conf/$execsite.xml" ] && [ ! -f "conf/$execsite.cf" ]; then
+   echo Missing configuration file $execsite.cf
+fi
+
+cp movie_graph.txt $rundir
+cp annealing.swift $rundir
+cp colortext.swift $rundir
+cp math.swift $rundir
+
+# Echo parameters
+echo Annealing parameters:
+echo
+cat $rundir/params.annealing
+echo
+
+# Echo runtime estimates
+total_jobs=`python -c "from math import ceil; print int(ceil(($max_target_innovation.00 - $min_target_innovation.00)/$target_innovation_increment.00) * $annealing_repeats * $variable_params * $annealing_cycles * $evolve_reruns/$reruns_per_opt_invocation)"`
+echo Total jobs = $total_jobs
+
+cd $rundir
+
+# Do the run
+export WORK=$PWD/swiftwork
+mkdir -p $PWD/swiftwork/workers
+
+# Use start-coaster-service if the site uses a .conf file
+if [ "$USE_SCS" == "1" ]; then
+   cp ../conf/$execsite.conf coaster-service.conf
+   cp ../conf/$execsite.cf cf
+   sed -i -e "s at _RUNDIR_@$rundir@" coaster-service.conf
+   start-coaster-service
+fi
+
+# Run gensites
+if [ ! "$USE_SCS" == 1 ]; then
+   cp ../conf/$execsite.cf cf
+   # SWIFT_HOME=../swift/bin ../swift/bin/gensites -p ../conf/$execsite.cf ../conf/$execsite.xml > $execsite.xml
+   gensites -p ../conf/$execsite.cf ../conf/$execsite.xml > $execsite.xml
+fi
+
+echo "Run dir=$rundir" >> ABOUT
+echo "Work dir=$WORK" >> ABOUT
+echo "Total jobs=$total_jobs" >> ABOUT
+echo "Run Command: SWIFT_HEAP_MAX=$ram SWIFT_LIB=.. swift >> swift.out 2>&1 -tc.file tc.data -sites.file $execsite.xml -config cf annealing.swift -minrange=$min_target_innovation -maxrange=$max_target_innovation-rangeinc=$target_innovation_increment -nreps=$annealing_repeats -annealingcycles=$annealing_cycles -evoreruns=$evolve_reruns -alphai=$alpha_i -alpham=$alpha_m -beta=$beta -gamma=$gamma -delta=$delta -nworkers=$nworkers -rerunsperapp=$reruns_per_opt_invocation -e33="$escapecode"" >> ABOUT
+
+if [ _$dryrun != _ ]; then
+  exit 0
+fi
+
+if [ "$USE_SCS" == "1" ]; then
+   runswift "sites.xml"
+   stop-coaster-service
+else
+   runswift "$execsite.xml"
+fi
+
+# Run the convertbest.sh script on the generated txt files from within the rundir
+../bin/convertbest.sh *.txt
+
+exit
+
+# @arg("nworkers","4")
+# @arg("rerunsperapp", "100")
+# @arg("seed", "0" )          FIXME
+# @arg("minrange", "58")
+# @arg("maxrange", "59")
+# @arg("rangeinc", "50")
+# @arg("nreps", "1")
+# @arg("tstart", "2.0")       FIXME
+# @arg("tend", "0.01")        FIXME
+# @arg("trejection", "0.3")   FIXME
+# @arg("evoreruns", "100")
+# @arg("startingjump", "2.3")
+# @arg("alphai", "0.0")
+# @arg("alpham", "0.0")
+# @arg("beta", "4.0")
+# @arg("gamma", "50.0")
+# @arg("delta", "-1.0")
+# @arg("annealingcycles", "50")
+


Property changes on: SwiftApps/SimulatedAnnealing/swiftopt.sh
___________________________________________________________________
Added: svn:executable
   + *



From wilde at ci.uchicago.edu  Mon Apr  7 11:54:03 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Mon,  7 Apr 2014 11:54:03 -0500 (CDT)
Subject: [Swift-commit] r7762 - SwiftApps/SimulatedAnnealing
Message-ID: <20140407165403.758DA9D62F@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-07 11:54:02 -0500 (Mon, 07 Apr 2014)
New Revision: 7762

Modified:
   SwiftApps/SimulatedAnnealing/annealing.swift
Log:
Converted to use generic objective function. Converted variable names to reflect more general terminology (eg loss -> result). Used simple avg() function to average N runs of objectve function.

Modified: SwiftApps/SimulatedAnnealing/annealing.swift
===================================================================
--- SwiftApps/SimulatedAnnealing/annealing.swift	2014-04-06 22:47:26 UTC (rev 7761)
+++ SwiftApps/SimulatedAnnealing/annealing.swift	2014-04-07 16:54:02 UTC (rev 7762)
@@ -1,30 +1,29 @@
 import "math";
-import "colortext";
 
 type file;
 
 type Res
 {
-    float loss;
+    float result;
+/*
     float sdev;
     float tavg;
     float tsdev;
+*/
 }
 
 type Checkpoint
 {
   int i;
   int j;
-  float dx0, dx1, dx2, dx3, dx4;
+  float dx0,  dx1,  dx2,  dx3,  dx4;
   float rej0, rej1, rej2, rej3, rej4;
-  float target_innov;
-  float loss;
+  float result;
   float alpha_i;
   float alpha_m;
   float beta;
   float gamma;
   float delta;
-  float sdev;
 }
 
 global boolean restart = (@arg("restart","false") == "true");
@@ -50,31 +49,36 @@
   // tracef("newx(%f,%f)=%f\n",x,dx,nx);
 }
 
-app  (/*file outfile,*/  file objout ) orig_objective ( string args[], file objdata )
-{
-  # evolve @objout  args /*stdout=@outfile*/;  // FIXME: objdata file is passed implicitly
-}
-
 /*
 function y = simple_objective(x)
-   y = (4 - 2.1*x(1)^2 + x(1)^4/3)*x(1)^2 + x(1)*x(2) + ...
-       (-4 + 4*x(2)^2)*x(2)^2;
+   y = (4 - 2.1*x(1)^2 + x(1)^4/3)*x(1)^2 + x(1)*x(2) + (-4 + 4*x(2)^2)*x(2)^2;
 */
 
-(float y) objective ( float x[], file objdata )
+(float y) objective ( float x[] )
 {
    float x0 = x[0];
    float x1 = x[1];
    float x0s = x0 * x0;
    float x1s = x1 * x1;
-   y = (4.0 - 2.1*x0s + pow(x[0],4.0/3.0))*x0s + x0*x1 + (-4.0 + 4.0*x1s)*x1s;
+   float r = random() * .1;
+   tracef("r=%f\n",r);
+   y = ( (4.0 - 2.1*x0s + pow(x[0],4.0/3.0))*x0s + x0*x1 + (-4.0 + 4.0*x1s)*x1s ) + r;
 }
 
-app (file outfile, file logfile ) sumobjective( file loss[] )
+(float s) sum (float a[])
 {
-  sumloss @filename(logfile) @filenames(loss) stdout=@filename(outfile);
+  string t[] = system("echo " + strjoin(a,"+") + "|bc");
+  s = toFloat(t[0]);
 }
 
+(float a) avg (float f[])
+{
+  float n = toFloat( length(f) );
+  float t1 = sum(f);
+  float t2 = t1 / n;
+  a = t2;
+}
+
 /* Program structure:
 
    main
@@ -100,11 +104,11 @@
   float rejection[][];  // [i][j] where i is cycle and j is evolve-parameter (alpha_i, alpha_m, beta, gamma, delta)
   float trejection[][];
 
-  float x[][], dx[][], curr_loss[], curr_sdev[];
+  float x[][], dx[][], curr_result[];
 
-  Res mlres[][];
+  Res objres[][];
 
-  file ckptFile <single_file_mapper; file=@strcat("ckpt.",target_innov)>;
+  file ckptFile <single_file_mapper; file=@strcat("ckpt")>;
   Checkpoint ckpt;
   int restartIndex;
 
@@ -114,16 +118,19 @@
   }
   else {
     restartIndex = -1;
+/*
     mlres[0][0] = multi_loss(T_start, T_end, annealing_cycles, Target_rejection,
                             starting_jump, 0, 0, params0, target_innov, evolve_reruns ); // FIXME: serves for all evolve-params ???
-    tracef( "multi_annealing: AR: initial: %f +- %f\n", mlres[0][0].loss, mlres[0][0].sdev );
+*/
+    objres[0][0] = N_objective(params0, obj_runs ); // FIXME: serves for all evolve-params ???
+
+    tracef( "multi_annealing: initial: %f\n", objres[0][0].result );
   
     foreach j in [0:NEVOPARAMS-1] {
       x[0][j]         = params0[j];
       dx[0][j]        = starting_jump;
       rejection[0][j] = 0.0;
-      curr_loss[j]    = mlres[0][0].loss;
-      curr_sdev[j]    = mlres[0][0].sdev;
+      curr_result[j]    = objres[0][0].result;
     }
   }
   
@@ -133,7 +140,7 @@
     // set new temperature, rejection threshold, and dx values for this cycle
     float temperature = T_start*exp( @tofloat( i-1 ) * ( jlog( T_end ) - jlog( T_start ) ) / @tofloat( annealing_cycles ) );
 
-    tracef( @strcat( "multi_annealing: AR: i=%i ....T = ", color( 3, "%f" ),"\n" ), i, temperature );
+    tracef("multi_annealing: i=%i ....T =%f\n", i, temperature );
 
     if ( i <  restartIndex ) { 
       tracef( "skipping index %i - less than restart index %i\n", i, restartIndex );
@@ -146,8 +153,7 @@
       x[i][2] = ckpt.beta;
       x[i][3] = ckpt.gamma;
       x[i][4] = ckpt.delta;
-      curr_loss[((i+1)*NEVOPARAMS)-1] = ckpt.loss;
-      curr_sdev[((i+1)*NEVOPARAMS)-1] = ckpt.sdev;
+      curr_result[((i+1)*NEVOPARAMS)-1] = ckpt.result;
     }
     else { // i > restartIndex: proceed as normal whether restarting or not
 
@@ -155,7 +161,7 @@
   
       if (i %% cycle == 1 && i > 1) {
         tracef( "multi_annealing: new cycle at i=%i\n", i );
-        tracef( color( Pink, "multi_annealing: AR: New cycle at %i: prev dx[0-4]=[%f %f %f %f %f]\n" ),
+        tracef( "multi_annealing: New cycle at %i: prev dx[0-4]=[%f %f %f %f %f]\n",
                        i, dx[i-1][0], dx[i-1][1], dx[i-1][2], dx[i-1][3], dx[i-1][4] );
         foreach k in [0:NEVOPARAMS-1] {
           float newrejection = rejection[i-1][k] / @tofloat( cycle );
@@ -169,9 +175,9 @@
             // FIXME: re-enable: rejection[i][k]=rejection[i-1][k];
                       trejection[i][k]=newrejection;
           }
-                  // FIXME: HANGS? : tracef(color(Red,"Recomputed rejection: i=%d k=%d dx[i][k]=%f\n"), i, k, dx[i][k]);
+                  // FIXME: HANGS? : tracef("Recomputed rejection: i=%d k=%d dx[i][k]=%f\n", i, k, dx[i][k]);
         }
-        tracef( color( Blue, "multi_annealing: AR: New cycle at %i: dx[0-4]=[%f %f %f %f %f]\n" ),
+        tracef( "multi_annealing: New cycle at %i: dx[0-4]=[%f %f %f %f %f]\n",
                                i, dx[i][0], dx[i][1], dx[i][2], dx[i][3], dx[i][4] );
       }
       else { // If not new cycle, set dx[i][*] from previous dx ([i-1]). rejection[i][j] is set later.
@@ -199,34 +205,24 @@
               }
             }
           }
-          tracef( @strcat( "multi_annealing: AR: ", color( 10,"%f" ), " ", color( 9,"%i" ),"\n" ), try_x[j], j );
+          tracef( "multi_annealing: %f %i\n", try_x[j], j );
 
           # Call objective function here:
 
 #         mlres[i][j] = multi_loss( T_start, T_end, annealing_cycles, Target_rejection, starting_jump,
-                                    i, j, try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_rerusn
-          mlres[i][j] = multi_loss( try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_reruns
+#                                    i, j, try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_rerusn
+          objres[i][j] = N_objective( try_x, obj_runs );
 
-          tracef( "multi_annealing: AR: %f +- %f\n", mlres[i][j].loss, mlres[i][j].sdev );
+          tracef( "multi_annealing: %f\n", objres[i][j].result );
           // Beyond this point, x[] and dx[] are being set for this i,j
-          float ALOT = 100000000000.0; // 100,000,000,000. = 10^11
-          if (mlres[i][j].loss < ALOT) {
-            fprintf( best_filename, "N, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n",
-                    mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss,
-                    try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev );
+          fprintf( best_filename, "%i, %i, %f, %f, | %f [ %f, %f, %f, %f, %f ]\n",
+                    i-1, j, dx[i][j], rejection[i][j], objres[i][j].result,
+                    try_x[0], try_x[1], try_x[2], try_x[3], try_x[4] );
                     // Note i-1 field: print that way to match with C++ output
   
                     // fprintf( "max_dist_swift.txt", color( Red,"multi_annealing: AF: max_dist.txt - tbd\n" ) );
                     // FIXME: max_dist is global set in evolve()
-          }
-          else { // does this ever occur? if so did we want to still do the ratio computation above???
-            fprintf( best_filename, "A, %fs, %i, %i, %f, %f, |, %i, %f, [, %f, %f, %f, %f, %f, ], %f\n",
-                     mlres[i][j].tavg, i-1, j, dx[i][j], rejection[i][j], @toint(target_innov), mlres[i][j].loss,
-                     try_x[0], try_x[1], try_x[2], try_x[3], try_x[4], mlres[i][j].sdev );
-                     // Note i-1 field: print that way to match with C++ output
-            //tracef( "multi_annealing: Loss %f > ALOT at [i][j] = [%d][%d]\n", mlres[i][j].loss, i ,j );
-          }
-          float ratio = min( 1.0, exp( -( mlres[i][j].loss - curr_loss[prev] ) / temperature ) );
+          float ratio = min( 1.0, exp( -( objres[i][j].result - curr_result[prev] ) / temperature ) );
           float r = (random()); //Java already returns a random float between [0.0-1.0]
           tracef("multi_annealing: AR: %f vs %f\n", r, ratio);
           if (r > ratio) { // Reject new parameter
@@ -237,11 +233,10 @@
             else {
               rejection[i][j] = rejection[i-1][j] + 1.0;  // FIXME: AR: Is this correct? incr rejection?
             }
-            curr_loss[curr] = curr_loss[prev];
-            curr_sdev[curr] = curr_sdev[prev];
+            curr_result[curr] = curr_result[prev];
             // FIXME: AR: the following prints seem to replicate values in the .cpp version - please clarify.
-            tracef( "multi_annealing: AR: %i,%i %i Did not accept: %f (%i)\n", i, j, i, try_x[j], j );
-            tracef( "multi_annealing: AR: %f %f %f %f %f\n", try_x[0],try_x[1],try_x[2],try_x[3],try_x[4] );
+            tracef( "multi_annealing: %i,%i %i Did not accept: %f (%i)\n", i, j, i, try_x[j], j );
+            tracef( "multi_annealing: %f %f %f %f %f\n", try_x[0],try_x[1],try_x[2],try_x[3],try_x[4] );
           }
           else { // Accept new parameter
             tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i\n",i,j );
@@ -252,8 +247,7 @@
             else {
               rejection[i][j] = rejection[i-1][j];  // FIXME: AR: Is this correct? no incr of rejection?
             }
-            curr_loss[curr] = mlres[i][j].loss;
-            curr_sdev[curr] = mlres[i][j].sdev;
+            curr_result[curr] = objres[i][j].result;
             tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i try_x[j]=%f\n", i, j, try_x[j] );
             float rj[];
             foreach k in [0:NEVOPARAMS-1] {
@@ -264,19 +258,16 @@
                 rj[k] = rejection[i-1][k]; // Not yet set, use previous
               }
             }
-            tracef(@strcat("multi_annealing: AR: [%i][%i] ", color(8,"Rejection counts: "),
-                           color(1,"%f"), " ", color(7,"%f"), " ", color(5,"%f"), " ", color(9,"%f"), " ", color(6,"%f"), "\n\n"),
+            tracef("multi_annealing: [%i][%i] Rejection counts: %f %f %f %f %f\n\n",
                            i, j, rj[0], rj[1], rj[2], rj[3], rj[4]);
-            tracef(@strcat("multi_annealing: AR: %i ", color(8,"***** Did accept! "),
-                           color(1,"%f"), " ", color(7,"%f"), " ", color(5,"%f"), " ", color(9,"%f"), " ", color(6,"%f"), "\n\n"),
+            tracef("multi_annealing: %i ***** Did accept! %f %f %f %f %f\n\n",
                            i, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4]);
             }
           }
           else {// Fixed Vars
             x[i][j] = x[i-1][j];
             rejection[i][j] = rejection[i-1][j];
-            curr_loss[curr] = curr_loss[prev];
-            curr_sdev[curr] = curr_sdev[prev];
+            curr_result[curr] = curr_result[prev];
             // dx[i][j] not set for fixed vars
           }
         } until( j == (NEVOPARAMS-iterate_adjust) );
@@ -284,59 +275,19 @@
     } until( iter_i == (annealing_cycles-iterate_adjust) );
 }
 
-/*
-(Res r) multi_loss(float t_start, float t_end, int annealing_steps, float t_rejection, float starting_jump,
-                   int ci, int cj, float x[], float target_innov, int evolve_reruns)
-*/
-
 (Res r) N_objective(float x[], int obj_runs)
 {
   tracef("%q\n", x);
-  file rfile[];
-  //file ofile[]; // FIXME: to obtain timings and other stats
+  float result[];
 
   tracef( "N_objective: entered: obj_runs=%i x=%q\n", obj_runs, x );
 
-  int appCalls = @toint(@tofloat(obj_runs)/@tofloat(runsPerApp)); // FIXME: handle fractional issues and rounding etc.
-                                                                         // For now must divide evenly
-  tracef("N_objective appCalls=%i\n", appCalls);
-
-  foreach i in [1:appCalls] { // repeats of the objective() - same as n_reruns
-    //file outfile; // FIXME: map and save in future
-    string args[] = [ // FIXME: move this to a setargs() function
-            //    alpha_i           alpha_m        beta           gamma          delta
-                 @strcat(x[0]),  @strcat(x[1]), @strcat(x[2]), @strcat(x[3]), @strcat(x[4]),
-            //     n_epochs n_steps   n_runs                  range
-                  "40000",   "20",  @strcat(rerunsPerApp),     "2",
-            //    verbose_level
-                    "1",
-            //    operation-code:(m,a)  Nworkers           seed
-                  "m",                  @strcat(Nworkers), @arg("seed", "0" ) ];
-
-    file graph <"movie_graph.txt">;
-
-    (/*outfile,*/ rfile[i]) = objective(args, graph);
-    tracef("multi_loss: i=%i calling objective, args=%q\n", i, args);
-    // tracef("multi_objective: after objective: i=%i %k %k\n", i, outfile, rfile[i]);
+  foreach i in [1:obj_runs] { // repeats of the objective() - same as n_reruns
+    result[i] = objective(x);
   }
 
-  string anneal_cycle;
-  if(ci==0) {
-    anneal_cycle="init";
-  }
-  else {
-    anneal_cycle=@strcat(ci-1);
-  }
-    
-  file sumfile<single_file_mapper; file=@strcat("sumloss/",anneal_cycle,".",cj,".sumloss")>;
-  file logfile<single_file_mapper; file=@strcat("sumloss/",anneal_cycle,".",cj,".sumlog")>;
-
-  (sumfile, logfile) = sumloss(rfile);
-  r = readData(sumfile);
-
-  tracef("multi_loss: returning: ci=%i cj=%i r.loss=%f r.sdev=%f\n",ci,cj,r.loss,r.sdev);
-  // file statfile = sumstats(ofile);  FIXME: to obtain timings and other stats
-  // s = readStat(statsfile);          FIXME: to obtain timings and other stats
+  r.result = avg(result);
+  tracef("multi_loss: returning: r.result=%f\n", r.result);
 }
 
 optimizer_serial_sweep() // Implements logic of python driver script
@@ -353,14 +304,14 @@
 //file bestfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".best_opt_some")>;
 //file maxfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".max_dist")>;
 
-#  foreach target_innov in [minrange:maxrange:rangeinc] {
-  iterate i {
-    int target_innov = minrange + (i*rangeinc);
+# foreach target_innov in [minrange:maxrange:rangeinc] {
+# iterate i {
+#   int target_innov = minrange + (i*rangeinc);
     foreach rep in [1:nreps] {
 
-      string best_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" );
-      file outfile;  // <single_file_mapper; file=@strcat("output/T",target_innov,".R",rep,".out")>;
-      file lossfile; // <single_file_mapper; file=@strcat("output/T",target_innov,".R",rep,".loss_data")>;
+      string best_filename = @strcat( "best.R", @strcat(rep), ".txt" );
+      file outfile;  // <single_file_mapper; file=@strcat("output/R",rep,".out")>;
+      file lossfile; // <single_file_mapper; file=@strcat("output/R",rep,".loss_data")>;
 
       (outfile,lossfile) = multi_annealing(best_filename,
 		                           @tofloat(@arg("tstart", "2.0")),
@@ -374,10 +325,9 @@
                                              @tofloat(@arg("gamma", "50.0")),
                                              @tofloat(@arg("delta", "-1.0"))
                                            ],
-                                           @tofloat(target_innov),
                                            @toint(@arg("annealingcycles", "50")) );
     }
-  } until(target_innov >= (maxrange-rangeinc));
+ # } until(target_innov >= (maxrange-rangeinc));
 }
 
 optimizer_sweep() // Implements logic of python driver script
@@ -394,12 +344,12 @@
 //file bestfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".best_opt_some")>;
 //file maxfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".max_dist")>;
 
-  foreach target_innov in [minrange:maxrange:rangeinc] {
+# foreach target_innov in [minrange:maxrange:rangeinc] {
     foreach rep in [1:nreps] {
 
-      string best_filename = @strcat( "best.T", @strcat(target_innov), ".R", @strcat(rep), ".txt" );
-      file outfile;  // <single_file_mapper; file=@strcat("output/T",target_innov,".R",rep,".out")>;
-      file lossfile; // <single_file_mapper; file=@strcat("output/T",target_innov,".R",rep,".loss_data")>;
+      string best_filename = @strcat( "best.R", @strcat(rep), ".txt" );
+      file outfile;  // <single_file_mapper; file=@strcat("output/R",rep,".out")>;
+      file lossfile; // <single_file_mapper; file=@strcat("output/R",rep,".loss_data")>;
 
       (outfile,lossfile) = multi_annealing(best_filename,
 		                           @tofloat(@arg("tstart", "2.0")),
@@ -413,10 +363,9 @@
                                              @tofloat(@arg("gamma", "50.0")),
                                              @tofloat(@arg("delta", "-1.0"))
                                            ],
-                                           @tofloat(target_innov),
                                            @toint(@arg("annealingcycles", "50")) );
     }
-  }
+# }
 }
 
 main()
@@ -425,63 +374,3 @@
 }
 
 main();
-
-/*
-
-  Program structure:
-
-  main
-  optimizer_sweep()
-  multi_annealing()
-  multi_loss()
-  evolve()
-  sumloss()
-
-  Example parameter sets:
-
-  for target in range(58,59,50):
-  for i in range(1):
-  args="./toptimizer 0 0 4 50 -1 "+target+" 40000 20 75    2 1 2. 0.01 2 0.3 2.3 1 1 1 0 0 m // > out.T"+str(target)+".i"+str(i)
-  os.system(args);
-
-  string fastargs1[] = [
-  "0", "0", "4", "50", "-1", @strcat(target),
-  "40000", "20", "1000", "2",
-  "1",
-  "2.", "0.01", "100", "0.3", "2.3",
-  "1", "1", "0", "0", "0"];
-  string fastargs2[] = [
-  "0", "0", "4", "50", "-1", @strcat(target),
-  "40000", "20", "1000", "2",
-  "1",
-  "2.", "0.01",  "5", "0.3", "2.3",
-  "1", "1", "0", "0", "0", "m"];
-  string fastargs3[] = [
-  "0", "0", "4", "50", "-1", @strcat(target),
-  "40000", "20", @strcat(repeats), "2",
-  "1",
-  "2.", "0.01",  "2", "0.3", "2.3",
-  "1", "1", "0", "0", "0", "m"];
-*/
-
-(string args[]) setargs()
-{
-    // string longargs[] = @strcat("0 0 4 50 -1 ",target," 40000 20 1000 2 1 2. 0.01 100 0.3 2.3 1 1 0 0 0 m");
-
-    //  [alpha_i alpha_m beta gamma delta target_innov
-    //  [n_epochs n_steps n_reruns] [range]
-    //  [verbose_level]
-    //  [T_start T_end Annealing_steps Target_rejection Starting_jump]
-    //  [FREEZE_alpha_i FREEZE_alpha_m FREEZE_beta FREEZE_gamma FREEZE_delta] [operation-code:(m,a) Nworkers]
-}
-
-////////////////// HOLD JUNK
-
-//                    tracef(@strcat("multi_annealing: AR: %i ", color(8,"Rejection counts: "),
-//                                   color( /* 2 */ 1," %f"), "\n\n"),
-//                                   i, rejection[i][j] ); // , rejection[i][1], rejection[i][2], rejection[i][3], rejection[i][4]);
-// FIXME: determine correct rejection[] values to avoid hanging:
-//                    tracef(@strcat("multi_annealing: AR: %i ", color(8,"Rejection counts: "),
-//                                   color( /* 2 */ 1," %f"), color(7," %f"), color(5," %f"), color(9," %f"), color(6," %f"), "\n\n"),
-//                                   rejection[i][0], rejection[i][1], rejection[i][2], rejection[i][3], rejection[i][4]);
-// END FIXME



From ketan at ci.uchicago.edu  Fri Apr 11 14:12:06 2014
From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu)
Date: Fri, 11 Apr 2014 14:12:06 -0500 (CDT)
Subject: [Swift-commit] r7763 - SwiftApps/swift-galaxy/swift
Message-ID: <20140411191206.5C9809CD1D@svn.ci.uchicago.edu>

Author: ketan
Date: 2014-04-11 14:12:06 -0500 (Fri, 11 Apr 2014)
New Revision: 7763

Modified:
   SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
   SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
Log:
small fix

Modified: SwiftApps/swift-galaxy/swift/swiftforeachlist.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachlist.sh	2014-04-07 16:54:02 UTC (rev 7762)
+++ SwiftApps/swift-galaxy/swift/swiftforeachlist.sh	2014-04-11 19:12:06 UTC (rev 7763)
@@ -18,10 +18,10 @@
 shift
 outloc=$1
 shift
+logfile=$1
+shift
 outlistfile=$1
 shift
-logfile=$1
-shift
 stringargs=$1
 shift 
 

Modified: SwiftApps/swift-galaxy/swift/swiftforeachrange.sh
===================================================================
--- SwiftApps/swift-galaxy/swift/swiftforeachrange.sh	2014-04-07 16:54:02 UTC (rev 7762)
+++ SwiftApps/swift-galaxy/swift/swiftforeachrange.sh	2014-04-11 19:12:06 UTC (rev 7763)
@@ -23,10 +23,10 @@
 shift
 outloc=$1
 shift
+logfile=$1
+shift
 outlistfile=$1
 shift
-logfile=$1
-shift
 
 stringargs=$1
 shift



From wilde at ci.uchicago.edu  Fri Apr 11 16:43:06 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Fri, 11 Apr 2014 16:43:06 -0500 (CDT)
Subject: [Swift-commit] r7764 - SwiftApps/SimulatedAnnealing
Message-ID: <20140411214306.A3C829CF85@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-11 16:43:06 -0500 (Fri, 11 Apr 2014)
New Revision: 7764

Added:
   SwiftApps/SimulatedAnnealing/sa.swift
Log:
New simpler kinder gentler annealer.

Added: SwiftApps/SimulatedAnnealing/sa.swift
===================================================================
--- SwiftApps/SimulatedAnnealing/sa.swift	                        (rev 0)
+++ SwiftApps/SimulatedAnnealing/sa.swift	2014-04-11 21:43:06 UTC (rev 7764)
@@ -0,0 +1,388 @@
+import "math";
+
+type file;
+
+type Res
+{
+    float result;
+/*
+    float sdev;
+    float tavg;
+    float tsdev;
+*/
+}
+
+type Checkpoint
+{
+  int i;
+  int j;
+  float dx0,  dx1,  dx2,  dx3,  dx4;
+  float rej0, rej1, rej2, rej3, rej4;
+  float result;
+  float alpha_i;
+  float alpha_m;
+  float beta;
+  float gamma;
+  float delta;
+}
+
+global boolean restart = (@arg("restart","false") == "true");
+tracef("Restart flag is %b\n", restart);
+
+global int iterate_adjust = @toint(@arg("iterate_adjust","1")); // FIXME!!! : 1 for 0.93, 0 for 0.94
+
+global boolean FIX_VARIABLES = true;
+global int var_fixed[] = [0,0,0,0,0];
+global int Nworkers = @toint(@arg("nworkers","4"));
+global int rerunsPerApp= @toint(@arg("rerunsperapp", "100"));
+
+( float nx ) newx( float x, float dx )
+{
+  float r = (random());
+
+  if (r > 0.5) {
+    nx = x + (random())*dx; //Java already returns a float between [0-1]
+  }
+  else {
+    nx = x - (random())*dx;
+  }
+  // tracef("newx(%f,%f)=%f\n",x,dx,nx);
+}
+
+/*
+function y = simple_objective(x)
+   y = (4 - 2.1*x(1)^2 + x(1)^4/3)*x(1)^2 + x(1)*x(2) + (-4 + 4*x(2)^2)*x(2)^2;
+*/
+
+(float y) objective ( float x[] )
+{
+   float x0 = x[0];
+   float x1 = x[1];
+   float x0s = x0 * x0;
+   float x1s = x1 * x1;
+   float r = random() * .1;
+   tracef("r=%f\n",r);
+   y = ( (4.0 - 2.1*x0s + pow(x[0],4.0/3.0))*x0s + x0*x1 + (-4.0 + 4.0*x1s)*x1s ) + r;
+}
+
+(float s) sum (float a[])
+{
+  string t[] = system("echo " + strjoin(a,"+") + "|bc");
+  s = toFloat(t[0]);
+}
+
+(float a) avg (float f[])
+{
+  float n = toFloat( length(f) );
+  float t1 = sum(f);
+  float t2 = t1 / n;
+  a = t2;
+}
+
+/* Program structure:
+
+   main
+     optimizer_sweep() - do N complete annealing optimizations
+       multi_annealing()
+         N_objective()
+           objective()
+         sumobjective()
+*/
+
+(file bestfile, file maxfile) multi_annealing (string best_filename,  # app
+                                               float T_start,         # ann
+                                               float T_end,           # ann
+                                               float Target_rejection,# ann
+                                               int obj_runs,          # app
+                                               float starting_jump,   # ann
+                                               float params0[],       # app f(x)
+                                               int annealing_cycles)  # ann
+{
+  int cycle      = 10;     // const
+  int NEVOPARAMS = 5;      // const - 5 params: alpha_i, alpha_m, beta, gamma, delta
+
+  float rejection[][];  // [i][j] where i is cycle and j is evolve-parameter (alpha_i, alpha_m, beta, gamma, delta)
+  float trejection[][];
+
+  float x[][], dx[][], curr_result[];
+
+  Res objres[][];
+
+  file ckptFile <single_file_mapper; file=@strcat("ckpt")>;
+  Checkpoint ckpt;
+  int restartIndex;
+
+  if (restart) {
+    ckpt = readData(ckptFile);
+    restartIndex = ckpt.i;
+  }
+  else {
+    restartIndex = -1;
+/*
+    mlres[0][0] = multi_loss(T_start, T_end, annealing_cycles, Target_rejection,
+                            starting_jump, 0, 0, params0, target_innov, evolve_reruns ); // FIXME: serves for all evolve-params ???
+*/
+    objres[0][0] = N_objective(params0, obj_runs ); // FIXME: serves for all evolve-params ???
+
+    tracef( "multi_annealing: initial: %f\n", objres[0][0].result );
+  
+    foreach j in [0:NEVOPARAMS-1] {
+      x[0][j]         = params0[j];
+      dx[0][j]        = starting_jump;
+      rejection[0][j] = 0.0;
+      curr_result[j]    = objres[0][0].result;
+    }
+  }
+  
+  iterate iter_i {  // number of annealing cycles
+    int i = iter_i + 1;  // i ranges [1:n] in the swift script so that [0] can be the initial condition
+
+    // set new temperature, rejection threshold, and dx values for this cycle
+    float temperature = T_start*exp( @tofloat( i-1 ) * ( jlog( T_end ) - jlog( T_start ) ) / @tofloat( annealing_cycles ) );
+
+    tracef("multi_annealing: i=%i ....T =%f\n", i, temperature );
+
+    if ( i <  restartIndex ) { 
+      tracef( "skipping index %i - less than restart index %i\n", i, restartIndex );
+    }
+    else { if ( i == restartIndex ) { // reset variables from the restart file
+      dx[i] = [ckpt.dx0, ckpt.dx1, ckpt.dx2, ckpt.dx3, ckpt.dx4];
+      rejection[i] = [ckpt.rej0, ckpt.rej1, ckpt.rej2, ckpt.rej3, ckpt.rej4];
+      x[i][0] = ckpt.alpha_i;
+      x[i][1] = ckpt.alpha_m;
+      x[i][2] = ckpt.beta;
+      x[i][3] = ckpt.gamma;
+      x[i][4] = ckpt.delta;
+      curr_result[((i+1)*NEVOPARAMS)-1] = ckpt.result;
+    }
+    else { // i > restartIndex: proceed as normal whether restarting or not
+
+      // On each new "major" cycle within the annealing_cycles (other than the first) set new rejection and dx values
+  
+      if (i %% cycle == 1 && i > 1) {
+        tracef( "multi_annealing: new cycle at i=%i\n", i );
+        tracef( "multi_annealing: New cycle at %i: prev dx[0-4]=[%f %f %f %f %f]\n",
+                       i, dx[i-1][0], dx[i-1][1], dx[i-1][2], dx[i-1][3], dx[i-1][4] );
+        foreach k in [0:NEVOPARAMS-1] {
+          float newrejection = rejection[i-1][k] / @tofloat( cycle );
+          if (newrejection > 0.0) {
+            dx[i][k] = dx[i-1][k] / ( newrejection / Target_rejection );
+            // FIXME: re-enable: rejection[i][k]=0.0;
+            trejection[i][k]=0.0;
+          }
+          else {
+            dx[i][k] = dx[i-1][k] * 2.0;
+            // FIXME: re-enable: rejection[i][k]=rejection[i-1][k];
+                      trejection[i][k]=newrejection;
+          }
+                  // FIXME: HANGS? : tracef("Recomputed rejection: i=%d k=%d dx[i][k]=%f\n", i, k, dx[i][k]);
+        }
+        tracef( "multi_annealing: New cycle at %i: dx[0-4]=[%f %f %f %f %f]\n",
+                               i, dx[i][0], dx[i][1], dx[i][2], dx[i][3], dx[i][4] );
+      }
+      else { // If not new cycle, set dx[i][*] from previous dx ([i-1]). rejection[i][j] is set later.
+        foreach k in [0:NEVOPARAMS-1] {
+          dx[i][k] = dx[i-1][k];
+        }
+      }
+      iterate j {  // Try a new value for each non-fixed param; then write results and accept or reject
+        int curr = ( i * NEVOPARAMS ) + j;
+        int prev = curr-1;
+  
+        if ( /* (!FIX_VARIABLES) || */ ( var_fixed[j] == 0 ) ) {  // Adjustable vars
+             // fixed=1,1,0,0,0: FIXME: FIX_VARIABLES flag has faulty logic but OK when TRUE
+          float try_x[];
+          foreach k in [0:NEVOPARAMS-1] { // Select the evolve params to try
+            if (k < j) {
+              try_x[k] = x[i][k]; // already set x[i][k]
+            }
+            else {
+              if (k == j) {
+                try_x[k] = newx(x[i-1][j], dx[i-1][j]); // permute x[i-1][j]
+              }
+              else { // k > j
+                try_x[k] = x[i-1][k]; // use x[i-1][k] (from prior cycle)
+              }
+            }
+          }
+          tracef( "multi_annealing: %f %i\n", try_x[j], j );
+
+          # Call objective function here:
+
+#         mlres[i][j] = multi_loss( T_start, T_end, annealing_cycles, Target_rejection, starting_jump,
+#                                    i, j, try_x, target_innov, evolve_reruns ); // do the N evolve()'s, N=evolve_rerusn
+          objres[i][j] = N_objective( try_x, obj_runs );
+
+          tracef( "multi_annealing: %f\n", objres[i][j].result );
+          // Beyond this point, x[] and dx[] are being set for this i,j
+          fprintf( best_filename, "%i, %i, %f, %f, | %f [ %f, %f, %f, %f, %f ]\n",
+                    i-1, j, dx[i][j], rejection[i][j], objres[i][j].result,
+                    try_x[0], try_x[1], try_x[2], try_x[3], try_x[4] );
+                    // Note i-1 field: print that way to match with C++ output
+  
+                    // fprintf( "max_dist_swift.txt", color( Red,"multi_annealing: AF: max_dist.txt - tbd\n" ) );
+                    // FIXME: max_dist is global set in evolve()
+          float ratio = min( 1.0, exp( -( objres[i][j].result - curr_result[prev] ) / temperature ) );
+          float r = (random()); //Java already returns a random float between [0.0-1.0]
+          tracef("multi_annealing: AR: %f vs %f\n", r, ratio);
+          if (r > ratio) { // Reject new parameter
+            x[i][j] = x[i-1][j];
+            if (i %% cycle == 1 && i > 1) {
+              rejection[i][j] = trejection[i][j] + 1.0;    // FIXME: triple-check this!
+            }
+            else {
+              rejection[i][j] = rejection[i-1][j] + 1.0;  // FIXME: AR: Is this correct? incr rejection?
+            }
+            curr_result[curr] = curr_result[prev];
+            // FIXME: AR: the following prints seem to replicate values in the .cpp version - please clarify.
+            tracef( "multi_annealing: %i,%i %i Did not accept: %f (%i)\n", i, j, i, try_x[j], j );
+            tracef( "multi_annealing: %f %f %f %f %f\n", try_x[0],try_x[1],try_x[2],try_x[3],try_x[4] );
+          }
+          else { // Accept new parameter
+            tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i\n",i,j );
+            x[i][j] = try_x[j];
+            if (i %% cycle == 1 && i > 1) {
+              rejection[i][j] = trejection[i][j];    // FIXME: triple-check this!
+            }
+            else {
+              rejection[i][j] = rejection[i-1][j];  // FIXME: AR: Is this correct? no incr of rejection?
+            }
+            curr_result[curr] = objres[i][j].result;
+            tracef( "multi_annealing: Accepting try_x[j], i=%i j=%i try_x[j]=%f\n", i, j, try_x[j] );
+            float rj[];
+            foreach k in [0:NEVOPARAMS-1] {
+              if (k <= j) {
+                rj[k] = rejection[i][k]; // Was either set from previous j or just set for this j
+              }
+              else {
+                rj[k] = rejection[i-1][k]; // Not yet set, use previous
+              }
+            }
+            tracef("multi_annealing: [%i][%i] Rejection counts: %f %f %f %f %f\n\n",
+                           i, j, rj[0], rj[1], rj[2], rj[3], rj[4]);
+            tracef("multi_annealing: %i ***** Did accept! %f %f %f %f %f\n\n",
+                           i, try_x[0], try_x[1], try_x[2], try_x[3], try_x[4]);
+            }
+          }
+          else {// Fixed Vars
+            x[i][j] = x[i-1][j];
+            rejection[i][j] = rejection[i-1][j];
+            curr_result[curr] = curr_result[prev];
+            // dx[i][j] not set for fixed vars
+          }
+        } until( j == (NEVOPARAMS-iterate_adjust) );
+      }} // of if/else 
+    } until( iter_i == (annealing_cycles-iterate_adjust) );
+}
+
+(Res r) N_objective(float x[], int obj_runs)
+{
+  tracef("%q\n", x);
+  float result[];
+
+  tracef( "N_objective: entered: obj_runs=%i x=%q\n", obj_runs, x );
+
+  foreach i in [1:obj_runs] { // repeats of the objective() - same as n_reruns
+    result[i] = objective(x);
+  }
+
+  r.result = avg(result);
+  tracef("multi_loss: returning: r.result=%f\n", r.result);
+}
+
+optimizer_serial_sweep() // Implements logic of python driver script
+{
+
+  int minrange = @toint(@arg("minrange", "58"));
+  int maxrange = @toint(@arg("maxrange", "59"));
+  int rangeinc = @toint(@arg("rangeinc", "50"));
+
+  // FIXME: add provision for random priming and random param values when x[i] == -100 (see optimizer.cpp main())
+
+  int nreps = @toint(@arg("nreps", "1"));
+
+//file bestfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".best_opt_some")>;
+//file maxfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".max_dist")>;
+
+# foreach target_innov in [minrange:maxrange:rangeinc] {
+# iterate i {
+#   int target_innov = minrange + (i*rangeinc);
+    foreach rep in [1:nreps] {
+
+      string best_filename = @strcat( "best.R", @strcat(rep), ".txt" );
+      file outfile;  // <single_file_mapper; file=@strcat("output/R",rep,".out")>;
+      file lossfile; // <single_file_mapper; file=@strcat("output/R",rep,".loss_data")>;
+
+      (outfile,lossfile) = multi_annealing(best_filename,
+		                           @tofloat(@arg("tstart", "2.0")),
+                                           @tofloat(@arg("tend", "0.01")),
+                                           @tofloat(@arg("trejection", "0.3")),
+                                           @toint(@arg("evoreruns", "100")),
+                                           @tofloat(@arg("startingjump", "2.3")),
+                                           [ @tofloat(@arg("alphai", "0.0")),
+                                             @tofloat(@arg("alpham", "0.0")),
+                                             @tofloat(@arg("beta", "4.0")),
+                                             @tofloat(@arg("gamma", "50.0")),
+                                             @tofloat(@arg("delta", "-1.0"))
+                                           ],
+                                           @toint(@arg("annealingcycles", "50")) );
+    }
+ # } until(target_innov >= (maxrange-rangeinc));
+}
+
+optimizer_sweep() // Implements logic of python driver script
+{
+
+  int minrange = @toint(@arg("minrange", "58"));
+  int maxrange = @toint(@arg("maxrange", "59"));
+  int rangeinc = @toint(@arg("rangeinc", "50"));
+
+  // FIXME: add provision for random priming and random param values when x[i] == -100 (see optimizer.cpp main())
+
+  int nreps = @toint(@arg("nreps", "1"));
+
+//file bestfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".best_opt_some")>;
+//file maxfile <single_file_mapper; file=@strcat("output/T",target,".R",rep,".max_dist")>;
+
+# foreach target_innov in [minrange:maxrange:rangeinc] {
+    foreach rep in [1:nreps] {
+
+      string best_filename = @strcat( "best.R", @strcat(rep), ".txt" );
+      file outfile;  // <single_file_mapper; file=@strcat("output/R",rep,".out")>;
+      file lossfile; // <single_file_mapper; file=@strcat("output/R",rep,".loss_data")>;
+
+      (outfile,lossfile) = multi_annealing(best_filename,
+		                           @tofloat(@arg("tstart", "2.0")),
+                                           @tofloat(@arg("tend", "0.01")),
+                                           @tofloat(@arg("trejection", "0.3")),
+                                           @toint(@arg("evoreruns", "100")),
+                                           @tofloat(@arg("startingjump", "2.3")),
+                                           [ @tofloat(@arg("alphai", "0.0")),
+                                             @tofloat(@arg("alpham", "0.0")),
+                                             @tofloat(@arg("beta", "4.0")),
+                                             @tofloat(@arg("gamma", "50.0")),
+                                             @tofloat(@arg("delta", "-1.0"))
+                                           ],
+                                           @toint(@arg("annealingcycles", "50")) );
+    }
+# }
+}
+
+main()
+{
+  # optimizer_sweep();
+
+  float a[][];
+  a[0] = [1.1,1.2,1.3,1.4];
+  float y[];
+  y[0] = objective(a[0]);
+
+  iterate c {
+    float na[] = [ newx(a[0],0.5), newx(a[1],0.5), newx(a[2],0.5), newx(a[3],0.5) ];
+    float ny = objective(na);
+    trace(c,ny);
+  } until (c==10);
+
+}
+
+main();



From wilde at ci.uchicago.edu  Fri Apr 11 16:48:01 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Fri, 11 Apr 2014 16:48:01 -0500 (CDT)
Subject: [Swift-commit] r7765 - SwiftApps/SimulatedAnnealing
Message-ID: <20140411214801.B96CC9CF85@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-11 16:48:01 -0500 (Fri, 11 Apr 2014)
New Revision: 7765

Modified:
   SwiftApps/SimulatedAnnealing/sa.swift
Log:
sa edit

Modified: SwiftApps/SimulatedAnnealing/sa.swift
===================================================================
--- SwiftApps/SimulatedAnnealing/sa.swift	2014-04-11 21:43:06 UTC (rev 7764)
+++ SwiftApps/SimulatedAnnealing/sa.swift	2014-04-11 21:48:01 UTC (rev 7765)
@@ -378,9 +378,10 @@
   y[0] = objective(a[0]);
 
   iterate c {
-    float na[] = [ newx(a[0],0.5), newx(a[1],0.5), newx(a[2],0.5), newx(a[3],0.5) ];
-    float ny = objective(na);
-    trace(c,ny);
+    int cy = c + 1;
+    float na[] = [ newx(a[0][0],0.5), newx(a[0][1],0.5), newx(a[0][2],0.5), newx(a[0][3],0.5) ];
+    y[cy] = objective(na);
+    trace(c,y[c]);
   } until (c==10);
 
 }



From wilde at ci.uchicago.edu  Fri Apr 11 16:49:14 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Fri, 11 Apr 2014 16:49:14 -0500 (CDT)
Subject: [Swift-commit] r7766 - SwiftApps/SimulatedAnnealing
Message-ID: <20140411214914.150839CF85@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-11 16:49:13 -0500 (Fri, 11 Apr 2014)
New Revision: 7766

Modified:
   SwiftApps/SimulatedAnnealing/sa.swift
Log:
sa edit

Modified: SwiftApps/SimulatedAnnealing/sa.swift
===================================================================
--- SwiftApps/SimulatedAnnealing/sa.swift	2014-04-11 21:48:01 UTC (rev 7765)
+++ SwiftApps/SimulatedAnnealing/sa.swift	2014-04-11 21:49:13 UTC (rev 7766)
@@ -379,8 +379,8 @@
 
   iterate c {
     int cy = c + 1;
-    float na[] = [ newx(a[0][0],0.5), newx(a[0][1],0.5), newx(a[0][2],0.5), newx(a[0][3],0.5) ];
-    y[cy] = objective(na);
+    a[cy] = [ newx(a[0][0],0.5), newx(a[0][1],0.5), newx(a[0][2],0.5), newx(a[0][3],0.5) ];
+    y[cy] = objective(a[cy]);
     trace(c,y[c]);
   } until (c==10);
 



From hategan at ci.uchicago.edu  Sat Apr 12 16:33:33 2014
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Sat, 12 Apr 2014 16:33:33 -0500 (CDT)
Subject: [Swift-commit] r7767 - trunk/src/org/griphyn/vdl/engine
Message-ID: <20140412213333.CF2C99D65C@svn.ci.uchicago.edu>

Author: hategan
Date: 2014-04-12 16:33:33 -0500 (Sat, 12 Apr 2014)
New Revision: 7767

Modified:
   trunk/src/org/griphyn/vdl/engine/Karajan.java
Log:
escape open curly bracket in strings (bug 1237)

Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/Karajan.java	2014-04-11 21:49:13 UTC (rev 7766)
+++ trunk/src/org/griphyn/vdl/engine/Karajan.java	2014-04-12 21:33:33 UTC (rev 7767)
@@ -1987,7 +1987,9 @@
 	            case '"':
 	                sb.append("\\\"");
 	                break;
-	            
+	            case '{':
+	            	sb.append("\\{");
+	            	break;
 	            default:
 	                sb.append(c);
 	        }



From swift at ci.uchicago.edu  Sat Apr 12 16:40:04 2014
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Sat, 12 Apr 2014 16:40:04 -0500 (CDT)
Subject: [Swift-commit] cog r3891
Message-ID: <20140412214004.DDF4D8D000A6@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3891 | hategan | 2014-04-12 16:39:49 -0500 (Sat, 12 Apr 2014) | 1 line

don't enable stream compression by default (bug 1233)
------------------------------------------------------------------------
Index: modules/provider-coaster/src/org/globus/cog/coaster/channels/GSSChannel.java
===================================================================
--- modules/provider-coaster/src/org/globus/cog/coaster/channels/GSSChannel.java	(revision 3890)
+++ modules/provider-coaster/src/org/globus/cog/coaster/channels/GSSChannel.java	(working copy)
@@ -37,6 +37,12 @@
 public class GSSChannel extends AbstractTCPChannel {
 	private static final Logger logger = Logger.getLogger(GSSChannel.class);
 
+	private static final boolean streamCompression;
+	
+	static {
+	    streamCompression = "true".equals(System.getProperty("gss.channel.compression.enabled"));
+	}
+
 	private GssSocket socket;
 	private String peerId;
 	private boolean shuttingDown;
@@ -161,12 +167,22 @@
 
 	@Override
     protected void setInputStream(InputStream inputStream) {
-        super.setInputStream(new InflaterInputStream(inputStream));
+	    if (streamCompression) {
+	        super.setInputStream(new InflaterInputStream(inputStream));
+	    }
+	    else {
+	        super.setInputStream(inputStream);
+	    }
     }
 
     @Override
     protected void setOutputStream(OutputStream outputStream) {
-        super.setOutputStream(new DeflaterOutputStream(outputStream, true));
+        if (streamCompression) {
+            super.setOutputStream(new DeflaterOutputStream(outputStream, true));
+        }
+        else {
+            super.setOutputStream(outputStream);
+        }
     }
 
     protected void register() {


From hategan at ci.uchicago.edu  Tue Apr 15 04:17:21 2014
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Tue, 15 Apr 2014 04:17:21 -0500 (CDT)
Subject: [Swift-commit] r7768 - in trunk: resources
	src/org/griphyn/vdl/engine
Message-ID: <20140415091721.E1DB49D537@svn.ci.uchicago.edu>

Author: hategan
Date: 2014-04-15 04:17:17 -0500 (Tue, 15 Apr 2014)
New Revision: 7768

Modified:
   trunk/resources/swiftscript.g
   trunk/src/org/griphyn/vdl/engine/Karajan.java
Log:
fixed wrong procedure call prediction in parser and deal with the issue in the compiler (bug 1239)

Modified: trunk/resources/swiftscript.g
===================================================================
--- trunk/resources/swiftscript.g	2014-04-12 21:33:33 UTC (rev 7767)
+++ trunk/resources/swiftscript.g	2014-04-15 09:17:17 UTC (rev 7768)
@@ -653,36 +653,19 @@
     :
     id=identifier
     ASSIGN
-    (
-      (predictProcedurecallAssign) => code=procedurecallCode
-        { StringTemplate o = template("returnParam");
-          o.setAttribute("name",id);
-          code.setAttribute("outputs",o);
-        }
-    |
-      code=variableAssign
-      {
-          code.setAttribute("lhs",id);
-      }
-    )
-    ;
+    code=variableAssign {
+		code.setAttribute("lhs",id);
+    }
+;
 
 appendStat returns [ StringTemplate code = null ]
 	{ StringTemplate id=null; }
 :
     id=identifier
     APPEND
-    (
-    	(predictProcedurecallAssign) => code=procedurecallCode {
-    		StringTemplate o = template("returnParam");
-			o.setAttribute("name",id);
-			code.setAttribute("outputs",o);
-        }
-    	|
-      	code=arrayAppend {
-			code.setAttribute("array", id);
-		}
-    )
+	code=arrayAppend {
+		code.setAttribute("array", id);
+	}
 ;
 
 arrayAppend returns [ StringTemplate code = null ]
@@ -705,9 +688,6 @@
         }
     ;
 
-predictProcedurecallAssign
-    : ID LPAREN ;
-
 procedurecallCode returns [StringTemplate code=template("call")]
 {StringTemplate f=null;}
     :

Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/Karajan.java	2014-04-12 21:33:33 UTC (rev 7767)
+++ trunk/src/org/griphyn/vdl/engine/Karajan.java	2014-04-15 09:17:17 UTC (rev 7768)
@@ -597,47 +597,67 @@
 
 	public void assign(Assign assign, VariableScope scope) throws CompilationException {
 		try {
-			StringTemplate assignST = template("assign");
-			StringTemplate varST = expressionToKarajan(assign.getAbstractExpressionArray(0), scope, true);
-			String lValueType = datatype(varST);
-			
-			StringTemplate valueST = expressionToKarajan(assign.getAbstractExpressionArray(1), scope, false, lValueType);
-			
-			String rValueType = datatype(valueST);
-			
-			if (isAnyType(lValueType)) {
-			    if (isAnyType(rValueType)) {
-			        // any <- any
-			    }
-			    else {
-			        // any <- someType, so infer lValueType as rValueType
-			        setDatatype(varST, rValueType);
-			    }
-			}
-			else {
-			    if (isAnyType(rValueType)) {
-			        // someType <- any
-			        // only expressions that are allowed to return 'any' are procedures
-			        // for example readData(ret, file). These are special procedures that
-			        // need to look at the return type at run-time.
-			    }
-			    else if (!lValueType.equals(rValueType)){
-			        throw new CompilationException("You cannot assign value of type " + rValueType +
-                        " to a variable of type " + lValueType);
-			    }
-			}
-				
-			assignST.setAttribute("var", varST);
-			assignST.setAttribute("value", valueST);
-			assignST.setAttribute("line", getLine(assign));
-			String rootvar = abstractExpressionToRootVariable(assign.getAbstractExpressionArray(0));
-			scope.addWriter(rootvar, getRootVariableWriteType(assign.getAbstractExpressionArray(0)), assign, assignST);
-			scope.appendStatement(assignST);
+		    XmlObject value = assign.getAbstractExpressionArray(1);
+            if (isCall(value)) {
+                Call call = (Call) value;
+                ActualParameter out = call.addNewOutput();
+                XmlObject src = assign.getAbstractExpressionArray(0);
+                out.getDomNode().appendChild(src.getDomNode());
+                call(call, scope, false);
+            }
+            else {
+    			StringTemplate assignST = template("assign");
+    			StringTemplate varST = expressionToKarajan(assign.getAbstractExpressionArray(0), scope, true);
+    			String lValueType = datatype(varST);
+    			
+    			
+    			StringTemplate valueST = expressionToKarajan(assign.getAbstractExpressionArray(1), scope, false, lValueType);
+    			
+    			String rValueType = datatype(valueST);
+    			
+    			if (isAnyType(lValueType)) {
+    			    if (isAnyType(rValueType)) {
+    			        // any <- any
+    			    }
+    			    else {
+    			        // any <- someType, so infer lValueType as rValueType
+    			        setDatatype(varST, rValueType);
+    			    }
+    			}
+    			else {
+    			    if (isAnyType(rValueType)) {
+    			        // someType <- any
+    			        // only expressions that are allowed to return 'any' are procedures
+    			        // for example readData(ret, file). These are special procedures that
+    			        // need to look at the return type at run-time.
+    			    }
+    			    else if (!lValueType.equals(rValueType)){
+    			        throw new CompilationException("You cannot assign value of type " + rValueType +
+                            " to a variable of type " + lValueType);
+    			    }
+    			}
+    			
+    			assignST.setAttribute("var", varST);
+    			assignST.setAttribute("value", valueST);
+    			assignST.setAttribute("line", getLine(assign));
+    			String rootvar = abstractExpressionToRootVariable(assign.getAbstractExpressionArray(0));
+    			scope.addWriter(rootvar, getRootVariableWriteType(assign.getAbstractExpressionArray(0)), assign, assignST);
+    			scope.appendStatement(assignST);
+            }
 		} catch(CompilationException re) {
 			throw new CompilationException("Compile error in assignment at "+assign.getSrc()+": "+re.getMessage(),re);
 		}
 	}
 	
+    private boolean isCall(XmlObject value) {
+        Node expressionDOM = value.getDomNode();
+        String namespaceURI = expressionDOM.getNamespaceURI();
+        String localName = expressionDOM.getLocalName();
+        QName expressionQName = new QName(namespaceURI, localName);
+        return expressionQName.equals(CALL_EXPR);
+    }
+
+	
     private boolean isAnyType(String type) {
         return ProcedureSignature.ANY.equals(type);
     }



From davidk at ci.uchicago.edu  Wed Apr 16 12:51:49 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 16 Apr 2014 12:51:49 -0500 (CDT)
Subject: [Swift-commit] r7769 - trunk/docs/userguide
Message-ID: <20140416175149.D86419CFED@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-16 12:51:49 -0500 (Wed, 16 Apr 2014)
New Revision: 7769

Modified:
   trunk/docs/userguide/configuration
Log:
Add documentation for ticker properties


Modified: trunk/docs/userguide/configuration
===================================================================
--- trunk/docs/userguide/configuration	2014-04-15 09:17:17 UTC (rev 7768)
+++ trunk/docs/userguide/configuration	2014-04-16 17:51:49 UTC (rev 7769)
@@ -479,12 +479,25 @@
  concurrent transfers can cause the network to be overloaded
  preventing various other signaling traffic from flowing properly.
 
+|ticker.date.format
+|String
+|
+|Describes how to format the ticker date output. The format of this string
+ is documented in the Java SimpleDateFormat class, at 
+ http://docs.oracle.com/javase/6/docs/api/java/text/SimpleDateFormat.html
+
 |ticker.disable
 |true, false
 |false
 |When set to true, suppresses the output progress ticker that Swift
  sends to the console every few seconds during a run
 
+|ticker.prefix
+|String
+|Progress:
+|String to prepend to ticker output
+
+
 |use.wrapper.staging
 |true, false
 |false
@@ -543,4 +556,3 @@
 -----
 workdir=$RUNDIRECTORY
 -----
-



From davidk at ci.uchicago.edu  Wed Apr 16 16:07:52 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 16 Apr 2014 16:07:52 -0500 (CDT)
Subject: [Swift-commit] r7770 - in SwiftApps/tryswift: . css
Message-ID: <20140416210752.097B39CFED@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-16 16:07:51 -0500 (Wed, 16 Apr 2014)
New Revision: 7770

Modified:
   SwiftApps/tryswift/css/tryswift.css
   SwiftApps/tryswift/getfiles.php
   SwiftApps/tryswift/index.php
   SwiftApps/tryswift/tryswift.php
Log:
Various updates to tryswift - remove twitter bootstrap, layout changes


Modified: SwiftApps/tryswift/css/tryswift.css
===================================================================
--- SwiftApps/tryswift/css/tryswift.css	2014-04-16 17:51:49 UTC (rev 7769)
+++ SwiftApps/tryswift/css/tryswift.css	2014-04-16 21:07:51 UTC (rev 7770)
@@ -1,79 +1,44 @@
-#header {
-   text-align: center;
+#body,html {
+   height: 100%;
 }
 
-#infoFrame {
-   width: 100%;
-   background: white;
-   overflow-y: scroll;
+a { 
+   text-decoration: none
 }
 
-#editor {
-   width: 100%;
-   height: 50%;
-}
-
 #scriptNav {
-   position: relative;
-   left: 25%;
-   right: 25%;
    width: 100%;
-}
-
-#scriptDropdown {
-   position: relative;
-   top: 5px;
-}
-
-.b0b0 {
-   background: #b0b0b0;
    border: 1px solid black;
+   float: left;
+   line-height: 2em;
 }
 
-#editorContainer {
+#editor {
+   width: 100%;
+   height: 30%;
    border: 1px solid black;
-   border-bottom: 0px;
 }
 
-#executeButton {
-   position: relative; 
-   left: 47px;
-   width: 100px;
-   font-size: 1.2em;
-}
-
-#resetButton {
-   position: relative;
-   left: 44px;
-   width: 100px;
-   font-size: 1.2em;
-}
-
 #editorButtons {
-   background: #fbf1d3;
+   width: 100%;
+   border: 1px solid black;
 }
 
-#pre {
-   background: #fff;
-   margin: 0;
-   padding: 0;
+#swiftOutputDiv {
+   width: 100%;
+   border: 1px solid black;
 }
 
-#fileOutput {
-   width: 100%;
-   overflow-x: hidden;
-   overflow-y: scroll;
-   background: #fff;
+#example {
+   float: left;
 }
 
-#swiftOutput {
-   overflow-y: scroll;
-   overflow-x: hidden;
-   border: 1px solid black;
-   background: #fff;
+#explanation {
+   float: left;
+   padding-left: 20px;
+   top:50%
 }
 
-#.ace_editor {
-  line-height: normal;
-} 
-
+#outputs {
+   visibility: hidden;
+}

Modified: SwiftApps/tryswift/getfiles.php
===================================================================
--- SwiftApps/tryswift/getfiles.php	2014-04-16 17:51:49 UTC (rev 7769)
+++ SwiftApps/tryswift/getfiles.php	2014-04-16 21:07:51 UTC (rev 7770)
@@ -1,5 +1,4 @@
 <?PHP
-
 $dir = $_POST['dir'];
 $web_directory = "/home/tryswift/tryswift";
 chdir("$web_directory/$dir");
@@ -31,7 +30,8 @@
 	) {
            $size = filesize($file);
            #$type = shell_exec("file -b $file");
-           print "<div class=\"span2\"><a href=\"$dir/$file\" target=\"_blank\">$file</a> [$size bytes]</div>\n";
+           # print "<div class=\"span2\"><a href=\"$dir/$file\" target=\"_blank\">$file</a> [$size bytes]</div>\n";
+           print "<option value=\"$dir/$file\">$file</option>\n";
          }
 }
 ?>

Modified: SwiftApps/tryswift/index.php
===================================================================
--- SwiftApps/tryswift/index.php	2014-04-16 17:51:49 UTC (rev 7769)
+++ SwiftApps/tryswift/index.php	2014-04-16 21:07:51 UTC (rev 7770)
@@ -2,58 +2,48 @@
 <html lang="en">
    <head>
       <title>Try Swift</title>
-      <link href="css/tryswift.css" rel="stylesheet" media="screen">
-      <link href="css/bootstrap.min.css" rel="stylesheet" media="screen">
+      <link rel="stylesheet" type="text/css" href="css/tryswift.css">
    </head>
 
    <body>
-      <div class="container-fluid">
+      <div id="scriptNav">
+         <div id="example">
+            Example: 
+            <select id="scriptDropdown">
+            </select>
+         </div>
+         <div id="explain">
+         </div>
+      </div> <!-- scriptNav -->
+     
+      <!-- Swift code editor -->   
+      <div id="editor">
+         <script src="js/ace/ace.js" type="text/javascript" charset="utf-8"></script>
+         <script>
+            var editor = ace.edit("editor");
+            editor.setFontSize('13px');
+            editor.setTheme("ace/theme/solarized_light");
+            editor.getSession().setMode("ace/mode/text");
+            editor.setShowPrintMargin(false);
+            editor.setHighlightActiveLine(false);
+            editor.setDisplayIndentGuides(false);
+         </script>
+      </div> <!-- editor -->
 
-         <!-- Script navigation -->
-         <div class="row-fluid">
-            <div id="infoContainer" class="span4 b0b0">
-               <div id="scriptNav">
-                  <select id="scriptDropdown">
-                  </select>
-               </div> <!-- scriptNav -->
-               <iframe id="infoFrame"></iframe>
-            </div> <!-- infoContainer -->
-         
-            <!-- Code editor -->
-            <div id="editorContainer" class="span8">
-               <div id="editor">
-                  <script src="js/ace/ace.js" type="text/javascript" charset="utf-8"></script>
-                  <script>
-                     var editor = ace.edit("editor");
-                     editor.setFontSize('13px');
-                     editor.setTheme("ace/theme/solarized_light");
-                     editor.getSession().setMode("ace/mode/text");
-                     editor.setShowPrintMargin(false);
-                     editor.setHighlightActiveLine(false);
-                     editor.setDisplayIndentGuides(false);
-                  </script>
-               </div> <!-- editor -->
-
-               <div id="editorButtons">
-                  <button id="executeButton">Execute</button>
-                  <button id="resetButton">Reset</button>
-               </div> <!-- editorButtons -->
-            </div> <!-- editorContainer -->
-         </div> <!-- row -->
+     <!-- Execute and reset buttons -->
+     <div id="editorButtons">
+        <button id="executeButton">Execute</button>
+        <button id="resetButton">Reset</button>
+        <select id="outputs">
+           <option>File outputs</option>
+        </select>
+     </div> <!-- editorButtons -->
+     
+     <!-- Textual swift output and ticker -->       
+     <div id="swiftOutputDiv">
+        <pre id="swiftOutput"></pre> 
+     </div> <!-- swiftOutputDiv -->
             
-         <div class="row-fluid"> 
-            <div class="span12">
-               <pre id="swiftOutput">Swift output</pre> 
-            </div> <!-- span12 -->
-         </div> <!-- row -->
-            
-         <div class="row-fluid">
-            <div class="span12">
-               <div id="fileOutput"> Output files</div>
-            </div> <!-- span12 -->
-         </div> <!-- row-fluid -->
-      </div> <!-- container -->
-
       <script>
          var scriptCounter=0;
          var scripts=[
@@ -78,29 +68,44 @@
       <script src="http://code.jquery.com/jquery-1.10.1.min.js"></script>
       <script src="js/logtail.js"></script>
       <script type="text/javascript">
-      
+
+      // Popup a new browser window in center of screen
+      function popupwindow(url, name, w, h) {
+         var left = (screen.width/2)-(w/2);
+         var top = (screen.height/2)-(h/2);
+         return window.open(url, name, 'width='+w+', height='+h+', top='+top+', left='+left);
+      } 
+    
+      // Fill editor with script selected from dropdown 
       function updateScript(index) {
          $.get(scripts[index][0], null, function (data) {
             editor.setValue(data);
             editor.gotoLine(1);
          }, "text");
-         $('#infoFrame').attr('src', scripts[index][2]);
+         $('#explain').html("<button id=\"explainButton\" value=\"" + scripts[index][2] + "\">Explain</button>");
       }         
-    
+
       function resize() {
-         $('#editor').height( $(document).height() / 2 );
-         $('#infoContainer').height( $('#editorContainer').height() );
-         $('#infoFrame').height( $('#infoContainer').height() - $('#scriptDropdown').height()-5 );
-         $('#swiftOutput').height( $(document).height() * .20 );
-         $('#fileOutput').height( $(document).height() * .19 );
-         $('#fileOutput').width( $(document).width() );
+         $('#editor').height( $(document).height() * .55 );
+         $('#swiftOutputDiv').height( $(document).height() * .35 );
       }
 
+      function hideOutputs() {
+         $('#outputs').css('visibility', 'hidden');
+      }
+
+      function showOutputs() {
+         $('#outputs').css('visibility', 'visible');
+      } 
+
       $(document).ready(function() {
          resize();
          updateScript(0);
          $(window).resize(function() { resize(); });
-         $('#resetButton').click(function() { updateScript(); })
+         $('#resetButton').click(function() { 
+            updateScript($('#scriptDropdown').find('option:selected').index()); 
+            hideOutputs();
+         })
 
          $.each(scripts, function(key, value) {   
             $('#scriptDropdown')
@@ -111,21 +116,33 @@
 
          $('#scriptDropdown').change(function() {
             updateScript($(this).find('option:selected').index());
+            hideOutputs();
          });
  
+         $(document).on("change", "#outputs", function(){
+            popupwindow($('#outputs').val(), '', 800, 600); 
+            // $('#swiftOutput').load($("#outputs").val());
+         });
+
+         $(document).on("click", "#explainButton", function(){
+            popupwindow($(this).attr("value"), $(this).text(), 800, 600);
+            return false;
+         });
+
          $('#executeButton').click(function() {
            sourceCode=editor.getSession().getValue();
            if(!sourceCode) {
               alert("Source text is empty!");
               return false;
            }
+           $('#outputs').html('<option>File outputs</option>');
 
            $.post("tryswift.php", {source: sourceCode}).done(function(data) { 
-              $('#fileOutput').html("");
               var urlArray = data.split("\n");
               tailf(urlArray[0], "#swiftOutput");
               $.post("getfiles.php", {dir: urlArray[1]}).done(function(filedata) {
-                 $('#fileOutput').html(filedata);
+                 $('#outputs').append(filedata);
+                 showOutputs();
               });
            });
         })

Modified: SwiftApps/tryswift/tryswift.php
===================================================================
--- SwiftApps/tryswift/tryswift.php	2014-04-16 17:51:49 UTC (rev 7769)
+++ SwiftApps/tryswift/tryswift.php	2014-04-16 21:07:51 UTC (rev 7770)
@@ -27,7 +27,7 @@
    }
 
    # Run Swift
-   system("echo Executing.. > $dirname/swift.out");
+   system("echo Swift run starting at $( date +%T ) > $dirname/swift.out");
    system("$swift_cmd >> $dirname/swift.out 2>&1 &");
 
    print "runs/$unique/swift.out\n";



From davidk at ci.uchicago.edu  Wed Apr 16 16:28:53 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 16 Apr 2014 16:28:53 -0500 (CDT)
Subject: [Swift-commit] r7771 - SwiftApps/tryswift/bin
Message-ID: <20140416212853.EB28B9CFED@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-16 16:28:53 -0500 (Wed, 16 Apr 2014)
New Revision: 7771

Added:
   SwiftApps/tryswift/bin/countfiles.sh
Log:
countfiles


Added: SwiftApps/tryswift/bin/countfiles.sh
===================================================================
--- SwiftApps/tryswift/bin/countfiles.sh	                        (rev 0)
+++ SwiftApps/tryswift/bin/countfiles.sh	2014-04-16 21:28:53 UTC (rev 7771)
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+find * -type f \
+   -not -ipath "tc.data" \
+   -not -ipath "sites.xml" \
+   -not -ipath "script.swift" \
+   -not -ipath "swift.out" \
+   -not -ipath "swift.pid" \
+   -not -ipath "cf" \
+   -not -ipath "*.d" \
+   -not -ipath "*.kml" \
+   -not -ipath "*.swiftx" \
+   -not -ipath "*.log" \
+| wc -l


Property changes on: SwiftApps/tryswift/bin/countfiles.sh
___________________________________________________________________
Added: svn:executable
   + *



From davidk at ci.uchicago.edu  Wed Apr 16 17:16:26 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 16 Apr 2014 17:16:26 -0500 (CDT)
Subject: [Swift-commit] r7772 - in SwiftApps/tryswift: . config
Message-ID: <20140416221626.B31059CFE3@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-16 17:16:26 -0500 (Wed, 16 Apr 2014)
New Revision: 7772

Modified:
   SwiftApps/tryswift/config/cf
   SwiftApps/tryswift/tryswift.php
Log:
Output formatting - remove prefix, adjust timestamp, change ticker names


Modified: SwiftApps/tryswift/config/cf
===================================================================
--- SwiftApps/tryswift/config/cf	2014-04-16 21:28:53 UTC (rev 7771)
+++ SwiftApps/tryswift/config/cf	2014-04-16 22:16:26 UTC (rev 7772)
@@ -1,3 +1,5 @@
 foreach.max.threads=10
 execution.retries=0
 wrapperlog.always.transfer=false
+ticker.date.format=hh:mm:ss
+ticker.prefix=

Modified: SwiftApps/tryswift/tryswift.php
===================================================================
--- SwiftApps/tryswift/tryswift.php	2014-04-16 21:28:53 UTC (rev 7771)
+++ SwiftApps/tryswift/tryswift.php	2014-04-16 22:16:26 UTC (rev 7772)
@@ -28,7 +28,7 @@
 
    # Run Swift
    system("echo Swift run starting at $( date +%T ) > $dirname/swift.out");
-   system("$swift_cmd >> $dirname/swift.out 2>&1 &");
+   system("$swift_cmd 2>&1 | sed -u -e 's/^[ \t]*//' -e s/'Selecting site:'/Ready:/g -e s/'Finished successfully:'/Done:/g >> $dirname/swift.out &");
 
    print "runs/$unique/swift.out\n";
    print "runs/$unique\n";



From davidk at ci.uchicago.edu  Wed Apr 16 17:27:36 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 16 Apr 2014 17:27:36 -0500 (CDT)
Subject: [Swift-commit] r7773 - SwiftApps/tryswift
Message-ID: <20140416222736.579119CFE3@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-16 17:27:36 -0500 (Wed, 16 Apr 2014)
New Revision: 7773

Modified:
   SwiftApps/tryswift/tryswift.php
Log:
Adjust timestamp formatting to match swift output


Modified: SwiftApps/tryswift/tryswift.php
===================================================================
--- SwiftApps/tryswift/tryswift.php	2014-04-16 22:16:26 UTC (rev 7772)
+++ SwiftApps/tryswift/tryswift.php	2014-04-16 22:27:36 UTC (rev 7773)
@@ -27,7 +27,7 @@
    }
 
    # Run Swift
-   system("echo Swift run starting at $( date +%T ) > $dirname/swift.out");
+   system("echo Swift run starting at $( date +%I:%M:%S ) > $dirname/swift.out");
    system("$swift_cmd 2>&1 | sed -u -e 's/^[ \t]*//' -e s/'Selecting site:'/Ready:/g -e s/'Finished successfully:'/Done:/g >> $dirname/swift.out &");
 
    print "runs/$unique/swift.out\n";



From davidk at ci.uchicago.edu  Thu Apr 17 11:44:15 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 17 Apr 2014 11:44:15 -0500 (CDT)
Subject: [Swift-commit] r7774 - in SwiftApps/tryswift: css js
Message-ID: <20140417164415.A01089CFE3@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-17 11:44:13 -0500 (Thu, 17 Apr 2014)
New Revision: 7774

Modified:
   SwiftApps/tryswift/css/tryswift.css
   SwiftApps/tryswift/js/logtail.js
Log:
Fixes for overflow and scrolling animation


Modified: SwiftApps/tryswift/css/tryswift.css
===================================================================
--- SwiftApps/tryswift/css/tryswift.css	2014-04-16 22:27:36 UTC (rev 7773)
+++ SwiftApps/tryswift/css/tryswift.css	2014-04-17 16:44:13 UTC (rev 7774)
@@ -27,8 +27,14 @@
 #swiftOutputDiv {
    width: 100%;
    border: 1px solid black;
+   overflow-y: scroll;
+   overflow-x: hidden;
 }
 
+#swiftOutput {
+   width: 100%;
+}
+
 #example {
    float: left;
 }

Modified: SwiftApps/tryswift/js/logtail.js
===================================================================
--- SwiftApps/tryswift/js/logtail.js	2014-04-16 22:27:36 UTC (rev 7773)
+++ SwiftApps/tryswift/js/logtail.js	2014-04-17 16:44:13 UTC (rev 7774)
@@ -114,7 +114,7 @@
 }
 
 function scroll(where) {
-   $("#swiftOutput").animate({ scrollTop: $('#swiftOutput')[0].scrollHeight}, 'slow');
+   $("#swiftOutputDiv").animate({ scrollTop: $('#swiftOutputDiv')[0].scrollHeight}, 'slow');
 }
 
 function show_log() {



From davidk at ci.uchicago.edu  Thu Apr 17 15:33:29 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 17 Apr 2014 15:33:29 -0500 (CDT)
Subject: [Swift-commit] r7775 - trunk
Message-ID: <20140417203329.AD8329CFE3@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-17 15:33:29 -0500 (Thu, 17 Apr 2014)
New Revision: 7775

Modified:
   trunk/build.xml
Log:
Add gensites templates to the build


Modified: trunk/build.xml
===================================================================
--- trunk/build.xml	2014-04-17 16:44:13 UTC (rev 7774)
+++ trunk/build.xml	2014-04-17 20:33:29 UTC (rev 7775)
@@ -226,6 +226,9 @@
 		<copy todir="${dist.dir}/etc/templates">
 			<fileset dir="${cog.dir}/modules/${module.name}/etc/templates"/>
 		</copy>
+                <copy todir="${dist.dir}/etc/sites">
+                        <fileset dir="${cog.dir}/modules/${module.name}/etc/sites"/>
+                </copy>
 		<chmod perm="+x" file="${dist.dir}/bin/swift"/>
 		<chmod perm="+x" file="${dist.dir}/bin/swift-osg-ress-site-catalog"/>
 		<chmod perm="+x" file="${dist.dir}/libexec/cdm_broadcast.sh"/>



From hategan at ci.uchicago.edu  Thu Apr 17 15:56:54 2014
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Thu, 17 Apr 2014 15:56:54 -0500 (CDT)
Subject: [Swift-commit] r7776 - trunk/src/org/griphyn/vdl/engine
Message-ID: <20140417205654.CB50C9D537@svn.ci.uchicago.edu>

Author: hategan
Date: 2014-04-17 15:56:54 -0500 (Thu, 17 Apr 2014)
New Revision: 7776

Modified:
   trunk/src/org/griphyn/vdl/engine/Karajan.java
Log:
fixed function invocation (bug 1235)

Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/Karajan.java	2014-04-17 20:33:29 UTC (rev 7775)
+++ trunk/src/org/griphyn/vdl/engine/Karajan.java	2014-04-17 20:56:54 UTC (rev 7776)
@@ -598,7 +598,7 @@
 	public void assign(Assign assign, VariableScope scope) throws CompilationException {
 		try {
 		    XmlObject value = assign.getAbstractExpressionArray(1);
-            if (isCall(value)) {
+            if (isProcedureCall(value)) {
                 Call call = (Call) value;
                 ActualParameter out = call.addNewOutput();
                 XmlObject src = assign.getAbstractExpressionArray(0);
@@ -613,30 +613,8 @@
     			
     			StringTemplate valueST = expressionToKarajan(assign.getAbstractExpressionArray(1), scope, false, lValueType);
     			
-    			String rValueType = datatype(valueST);
+    			checkOrInferReturnedType(varST, valueST);
     			
-    			if (isAnyType(lValueType)) {
-    			    if (isAnyType(rValueType)) {
-    			        // any <- any
-    			    }
-    			    else {
-    			        // any <- someType, so infer lValueType as rValueType
-    			        setDatatype(varST, rValueType);
-    			    }
-    			}
-    			else {
-    			    if (isAnyType(rValueType)) {
-    			        // someType <- any
-    			        // only expressions that are allowed to return 'any' are procedures
-    			        // for example readData(ret, file). These are special procedures that
-    			        // need to look at the return type at run-time.
-    			    }
-    			    else if (!lValueType.equals(rValueType)){
-    			        throw new CompilationException("You cannot assign value of type " + rValueType +
-                            " to a variable of type " + lValueType);
-    			    }
-    			}
-    			
     			assignST.setAttribute("var", varST);
     			assignST.setAttribute("value", valueST);
     			assignST.setAttribute("line", getLine(assign));
@@ -648,16 +626,45 @@
 			throw new CompilationException("Compile error in assignment at "+assign.getSrc()+": "+re.getMessage(),re);
 		}
 	}
-	
-    private boolean isCall(XmlObject value) {
-        Node expressionDOM = value.getDomNode();
-        String namespaceURI = expressionDOM.getNamespaceURI();
-        String localName = expressionDOM.getLocalName();
-        QName expressionQName = new QName(namespaceURI, localName);
-        return expressionQName.equals(CALL_EXPR);
+
+    private void checkOrInferReturnedType(StringTemplate varST, StringTemplate valueST) throws CompilationException {
+        String lValueType = datatype(varST);
+        String rValueType = datatype(valueST);
+        if (isAnyType(lValueType)) {
+            if (isAnyType(rValueType)) {
+                // any <- any
+            }
+            else {
+                // any <- someType, so infer lValueType as rValueType
+                setDatatype(varST, rValueType);
+            }
+        }
+        else {
+            if (isAnyType(rValueType)) {
+                // someType <- any
+                // only expressions that are allowed to return 'any' are procedures
+                // for example readData(ret, file). These are special procedures that
+                // need to look at the return type at run-time.
+            }
+            else if (!lValueType.equals(rValueType)){
+                throw new CompilationException("You cannot assign value of type " + rValueType +
+                    " to a variable of type " + lValueType);
+            }
+        }
     }
 
-	
+
+    private boolean isProcedureCall(XmlObject value) {
+        if (value instanceof Call) {
+            Call call = (Call) value;
+            return proceduresMap.get(call.getProc().getLocalPart()) != null;
+        }
+        else {
+            return false;
+        }
+    }
+
+
     private boolean isAnyType(String type) {
         return ProcedureSignature.ANY.equals(type);
     }



From hategan at ci.uchicago.edu  Thu Apr 17 20:29:19 2014
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Thu, 17 Apr 2014 20:29:19 -0500 (CDT)
Subject: [Swift-commit] r7777 - in
	branches/release-0.94/src/org/griphyn/vdl: karajan mapping
Message-ID: <20140418012919.4CDDC9D537@svn.ci.uchicago.edu>

Author: hategan
Date: 2014-04-17 20:29:19 -0500 (Thu, 17 Apr 2014)
New Revision: 7777

Added:
   branches/release-0.94/src/org/griphyn/vdl/mapping/FutureArrayOpen.java
Modified:
   branches/release-0.94/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java
   branches/release-0.94/src/org/griphyn/vdl/mapping/ArrayDataNode.java
Log:
fix for busy loop when waiting for a full array for which some elements are available

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java	2014-04-17 20:56:54 UTC (rev 7776)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java	2014-04-18 01:29:19 UTC (rev 7777)
@@ -121,15 +121,19 @@
     public DSHandle getHandle() {
         return node;
     }
+    
+    public void addModificationAction(FutureListener target, VariableStack stack) {
+        addModificationAction(target, stack, true);
+    }
 
-    public void addModificationAction(FutureListener target, VariableStack stack) {
+    public void addModificationAction(FutureListener target, VariableStack stack, boolean partialUpdates) {
         synchronized(node) {
             if (listeners == null) {
                 listeners = new LinkedList<ListenerStackPair>();
             }
             listeners.add(new ListenerStackPair(target, stack));
             WaitingThreadsMonitor.addThread(stack, node);
-            if (!node.isClosed() && keys.isEmpty()) {
+            if (!node.isClosed() && (keys.isEmpty() || !partialUpdates)) {
                 return;
             }
         }

Modified: branches/release-0.94/src/org/griphyn/vdl/mapping/ArrayDataNode.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/mapping/ArrayDataNode.java	2014-04-17 20:56:54 UTC (rev 7776)
+++ branches/release-0.94/src/org/griphyn/vdl/mapping/ArrayDataNode.java	2014-04-18 01:29:19 UTC (rev 7777)
@@ -38,7 +38,7 @@
 	public void getFringePaths(List<Path> list, Path parentPath) throws HandleOpenException {
 		checkMappingException();
 		if (!isClosed()) {
-		    throw new FutureNotYetAvailable(getFutureWrapper());
+		    throw new FutureArrayOpen((ArrayIndexFutureList) getFutureWrapper());
 		}
 		Map<Comparable<?>, DSHandle> handles = getHandles();
 		synchronized (this) {
@@ -55,7 +55,7 @@
 		}
 	}
 	
-	/** Recursively closes arrays through a tree of arrays and complex
+    /** Recursively closes arrays through a tree of arrays and complex
         types. */
     public void closeDeep() {
         assert(this.getType().isArray());
@@ -121,6 +121,22 @@
         return (FutureList) getFutureWrapper();
     }
     
+    @Override
+    public synchronized void waitFor() {
+        if (!isClosed()) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Waiting for " + this);
+            }
+            throw new FutureArrayOpen((ArrayIndexFutureList) getFutureWrapper());
+        }
+        else {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Do not need to wait for " + this);
+            }
+            checkDataException();
+        }
+    }
+
     protected void getFields(List<DSHandle> fields, Path path) throws InvalidPathException {
         if (path.isEmpty()) {
             fields.add(this);

Added: branches/release-0.94/src/org/griphyn/vdl/mapping/FutureArrayOpen.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/mapping/FutureArrayOpen.java	                        (rev 0)
+++ branches/release-0.94/src/org/griphyn/vdl/mapping/FutureArrayOpen.java	2014-04-18 01:29:19 UTC (rev 7777)
@@ -0,0 +1,61 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Apr 17, 2014
+ */
+package org.griphyn.vdl.mapping;
+
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.workflow.futures.Future;
+import org.globus.cog.karajan.workflow.futures.FutureEvaluationException;
+import org.globus.cog.karajan.workflow.futures.FutureListener;
+import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
+import org.griphyn.vdl.karajan.ArrayIndexFutureList;
+
+public class FutureArrayOpen extends FutureNotYetAvailable {
+    public FutureArrayOpen(ArrayIndexFutureList f) {
+        super(new Wrapper(f));
+    }
+
+    @Override
+    public Future getFuture() {
+        return super.getFuture();
+    }
+    
+    private static class Wrapper implements Future {
+        private final ArrayIndexFutureList f;
+
+        public Wrapper(ArrayIndexFutureList f) {
+            this.f = f;
+        }
+
+        @Override
+        public void close() {
+            f.close();
+        }
+
+        @Override
+        public boolean isClosed() {
+            return f.isClosed();
+        }
+
+        @Override
+        public Object getValue() {
+            return f.getValue();
+        }
+
+        @Override
+        public void fail(FutureEvaluationException e) {
+            f.fail(e);
+        }
+
+        @Override
+        public void addModificationAction(FutureListener target, VariableStack stack) {
+            f.addModificationAction(target, stack, false);
+        }
+    }
+}



From swift at ci.uchicago.edu  Thu Apr 17 23:15:05 2014
From: swift at ci.uchicago.edu (swift at ci.uchicago.edu)
Date: Thu, 17 Apr 2014 23:15:05 -0500 (CDT)
Subject: [Swift-commit] cog r3893
Message-ID: <20140418041505.B44088D0010F@bridled.ci.uchicago.edu>

------------------------------------------------------------------------
r3893 | hategan | 2014-04-17 23:13:13 -0500 (Thu, 17 Apr 2014) | 1 line

allow sub-classes to get the method used for invoking a function in a collection
------------------------------------------------------------------------
Index: modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/FunctionsCollection.java
===================================================================
--- modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/FunctionsCollection.java	(revision 3892)
+++ modules/karajan/src/org/globus/cog/karajan/workflow/nodes/functions/FunctionsCollection.java	(working copy)
@@ -11,11 +11,8 @@
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 import org.apache.log4j.Logger;
 import org.globus.cog.karajan.arguments.Arg;
@@ -33,10 +30,10 @@
 
 	public static final Arg[] ARGS_2VALUES = new Arg[] { PA_VALUE1, PA_VALUE2 };
 
-	private static final Class[] sig = new Class[] { VariableStack.class };
+	private static final Class<?>[] sig = new Class[] { VariableStack.class };
 	private Method method;
 	
-	private static Set inlineText = new HashSet();
+	private static Set<String> inlineText = new HashSet<String>();
 
 	protected static void setAcceptsInlineText(String fname, boolean text) {
 		if (!text) {
@@ -94,6 +91,10 @@
 			throw new ExecutionException(e);
 		}
 	}
+	
+	protected final Method getMethod() {
+	    return method;
+	}
 
 	public long currenttime(VariableStack stack) throws ExecutionException {
 		return System.currentTimeMillis();
@@ -114,7 +115,7 @@
 		return method.getName();
 	}
 
-	private static Set quotedArgs = new HashSet();
+	private static Set<String> quotedArgs = new HashSet<String>();
 
 	protected static void setQuotedArgs(String fn) {
 		quotedArgs.add(fn);


From hategan at ci.uchicago.edu  Thu Apr 17 23:17:18 2014
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Thu, 17 Apr 2014 23:17:18 -0500 (CDT)
Subject: [Swift-commit] r7778 - in
	branches/release-0.94/src/org/griphyn/vdl/karajan/lib: . swiftscript
Message-ID: <20140418041718.5C8389CFE3@svn.ci.uchicago.edu>

Author: hategan
Date: 2014-04-17 23:17:18 -0500 (Thu, 17 Apr 2014)
New Revision: 7778

Added:
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/SwiftFunctionsCollection.java
Modified:
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/Tracer.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java
   branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java
Log:
fixed lazy errors not working with assignments and built-in functions/procedures

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -36,6 +36,8 @@
 import org.griphyn.vdl.karajan.WaitingThreadsMonitor;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.InvalidPathException;
 import org.griphyn.vdl.mapping.Mapper;
 import org.griphyn.vdl.mapping.Path;
@@ -126,7 +128,13 @@
     /** make dest look like source - if its a simple value, copy that
 	    and if its an array then recursively copy */
 	public static void deepCopy(DSHandle dest, DSHandle source, VariableStack stack, int level) throws ExecutionException {
-	    ((AbstractDataNode) source).waitFor();
+	    try {
+	        ((AbstractDataNode) source).waitFor();
+	    }
+	    catch (DependentException e) {
+	        dest.setValue(new DataDependentException(dest, e));
+	        return;
+	    }
 		if (source.getType().isPrimitive()) {
 			dest.setValue(source.getValue());
 		}

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/Tracer.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/Tracer.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/Tracer.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -24,6 +24,7 @@
 import org.griphyn.vdl.karajan.FutureWrapper;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.Mapper;
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.type.Types;
@@ -261,20 +262,25 @@
         if (o instanceof AbstractDataNode) {
             AbstractDataNode h = (AbstractDataNode) o;
             if (h.isClosed()) {
-                if (h.getType().isPrimitive()) {
-                    if (Types.STRING.equals(h.getType())) {
-                        return "\"" + h.getValue() + '"';
+                try {
+                    if (h.getType().isPrimitive()) {
+                        if (Types.STRING.equals(h.getType())) {
+                            return "\"" + h.getValue() + '"';
+                        }
+                        else {
+                            return h.getValue();
+                        }
                     }
+                    else if (h.getType().isComposite()){
+                        return getVarName(h);
+                    }
                     else {
-                        return h.getValue();
+                        return fileName(h);
                     }
                 }
-                else if (h.getType().isComposite()){
-                    return getVarName(h);
+                catch (DependentException e) {
+                    return "<exception>";
                 }
-                else {
-                    return fileName(h);
-                }
             }
             else {
                 return "?" + getVarName(h);

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/VDLFunction.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/VDLFunction.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -39,20 +39,19 @@
 import org.globus.cog.karajan.util.TypeUtil;
 import org.globus.cog.karajan.workflow.ExecutionException;
 import org.globus.cog.karajan.workflow.KarajanRuntimeException;
-import org.globus.cog.karajan.workflow.futures.Future;
 import org.globus.cog.karajan.workflow.nodes.SequentialWithArguments;
 import org.globus.cog.karajan.workflow.nodes.restartLog.RestartLog;
 import org.globus.swift.catalog.TCEntry;
 import org.globus.swift.catalog.transformation.File;
 import org.globus.swift.catalog.types.TCType;
 import org.griphyn.vdl.karajan.AssertFailedException;
-import org.griphyn.vdl.karajan.FutureWrapper;
 import org.griphyn.vdl.karajan.Loader;
 import org.griphyn.vdl.karajan.TCCache;
 import org.griphyn.vdl.karajan.functions.ConfigProperty;
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
 import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.GeneralizedFileFormat;
 import org.griphyn.vdl.mapping.HandleOpenException;
@@ -61,6 +60,7 @@
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.mapping.PathComparator;
 import org.griphyn.vdl.mapping.PhysicalFormat;
+import org.griphyn.vdl.mapping.RootDataNode;
 import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 import org.griphyn.vdl.util.FQN;
@@ -95,11 +95,16 @@
             throw e;
         }
 		catch (DependentException e) {
-			// This would not be the primal fault so in non-lazy errors mode it
-			// should not matter
-			throw new ExecutionException(stack, e);
+		    RootDataNode r = new RootDataNode(getReturnType());
+		    r.setValue(new DataDependentException(r, e));
+			ret(stack, r);
+			super.post(stack);
 		}
 	}
+	
+	protected Type getReturnType() {
+	    return Types.ANY;
+	}
 
 	protected void ret(VariableStack stack, final Object value) throws ExecutionException {
 		if (value != null) {

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -29,6 +29,7 @@
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
 import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 
 
@@ -37,7 +38,12 @@
 		setArguments(ExtractInt.class, new Arg[] { PA_VAR });
 	}
 
-	public Object function(VariableStack stack) throws ExecutionException {
+	@Override
+    protected Type getReturnType() {
+        return Types.INT;
+    }
+
+    public Object function(VariableStack stack) throws ExecutionException {
 		AbstractDataNode handle = null;
 		try {
 			handle = (AbstractDataNode) PA_VAR.getValue(stack);

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -23,12 +23,18 @@
 import org.griphyn.vdl.karajan.lib.VDLFunction;
 import org.griphyn.vdl.mapping.DSHandle;
 import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 
 public class FileName extends VDLFunction {
 	static {
 		setArguments(FileName.class, new Arg[] { PA_VAR });
 	}
+	
+	@Override
+    protected Type getReturnType() {
+        return Types.STRING;
+    }
 
 	public Object function(VariableStack stack) throws ExecutionException {
 		String s = argList(filename(stack), true);

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -27,14 +27,19 @@
 import org.griphyn.vdl.mapping.DSHandle;
 import org.griphyn.vdl.mapping.InvalidPathException;
 import org.griphyn.vdl.mapping.Path;
-import org.griphyn.vdl.mapping.RootDataNode;
 import org.griphyn.vdl.mapping.RootArrayDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 
 public class FileNames extends VDLFunction {
 	static {
 		setArguments(FileNames.class, new Arg[] { PA_VAR });
 	}
+	
+	@Override
+    protected Type getReturnType() {
+        return Types.STRING.arrayType();
+    }
 
 	public Object function(VariableStack stack) throws ExecutionException {
 		String[] f = filename(stack);

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -28,6 +28,7 @@
 import org.griphyn.vdl.karajan.lib.VDLFunction;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.type.Types;
 
 /**
@@ -52,20 +53,25 @@
     @Override
     protected Object function(VariableStack stack) 
     throws ExecutionException {
-        AbstractDataNode[] args = waitForAllVargs(stack);
-        
-        check(args);
-        
-        String filename = (String) args[0].getValue();
-        String spec = (String) args[1].getValue(); 
-        DSHandle[] vars = Sprintf.copyArray(args, 2, args.length-2);
-        
-        StringBuilder output = new StringBuilder();
-        Sprintf.format(spec, vars, output);
-        String msg = output.toString();
- 
-        logger.debug("file: " + filename + " msg: " + msg);        
-        write(filename, msg);
+        try {
+            AbstractDataNode[] args = waitForAllVargs(stack);
+            
+            check(args);
+            
+            String filename = (String) args[0].getValue();
+            String spec = (String) args[1].getValue(); 
+            DSHandle[] vars = Sprintf.copyArray(args, 2, args.length-2);
+            
+            StringBuilder output = new StringBuilder();
+            Sprintf.format(spec, vars, output);
+            String msg = output.toString();
+     
+            logger.debug("file: " + filename + " msg: " + msg);        
+            write(filename, msg);
+        }
+        catch (DependentException e) {
+            logger.debug("<exception>");
+        }
         return null;
     }
     

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -27,7 +27,6 @@
 import org.globus.cog.karajan.stack.VariableStack;
 import org.globus.cog.karajan.util.TypeUtil;
 import org.globus.cog.karajan.workflow.ExecutionException;
-import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection;
 import org.griphyn.vdl.karajan.lib.SwiftArg;
 import org.griphyn.vdl.karajan.lib.VDLFunction;
 import org.griphyn.vdl.mapping.AbsFile;
@@ -40,7 +39,7 @@
 import org.griphyn.vdl.type.Types;
 import org.griphyn.vdl.util.VDL2Config;
 
-public class Misc extends FunctionsCollection {
+public class Misc extends SwiftFunctionsCollection {
 
 	private static final Logger logger = Logger.getLogger(Misc.class);
 
@@ -52,24 +51,60 @@
 
 	static {
         setArguments("swiftscript_dirname", new Arg[] { PA_FILE });
+        setReturnType("swiftscript_dirname", Types.STRING);
+        
 	    setArguments("swiftscript_exists", new Arg[] { Arg.VARGS });
+	    setReturnType("swiftscript_exists", Types.BOOLEAN);
+	    
         setArguments("swiftscript_existsfile", new Arg[] { PA_FILE });
+        setReturnType("swiftscript_existsfile", Types.BOOLEAN);
+        
         setArguments("swiftscript_format", new Arg[] { PA_INPUT, PA_TRANSFORM });
+        setReturnType("swiftscript_format", Types.STRING);
+        
         setArguments("swiftscript_length", new Arg[] { PA_ARRAY });
+        setReturnType("swiftscript_length", Types.INT);
+        
         setArguments("swiftscript_pad", new Arg[] { PA_INPUT, PA_TRANSFORM });
+        setReturnType("swiftscript_pad", Types.STRING);
+        
         setArguments("swiftscript_regexp", new Arg[] { PA_INPUT, PA_PATTERN, PA_TRANSFORM });
+        setReturnType("swiftscript_regext", Types.STRING);
+        
         setArguments("swiftscript_strcat",  new Arg[] { Arg.VARGS });
+        setReturnType("swiftscript_strcat", Types.STRING);
+        
         setArguments("swiftscript_strcut", new Arg[] { PA_INPUT, PA_PATTERN });
+        setReturnType("swiftscript_strcut", Types.STRING);
+        
         setArguments("swiftscript_strsplit", new Arg[] { PA_INPUT, PA_PATTERN });
+        setReturnType("swiftscript_strsplit", Types.STRING.arrayType());
+        
         setArguments("swiftscript_strjoin", new Arg[] { PA_ARRAY, PA_INPUT });
+        setReturnType("swiftscript_strjoin", Types.STRING);
+        
         setArguments("swiftscript_strstr", new Arg[] { PA_INPUT, PA_PATTERN });
+        setReturnType("swiftscript_strstr", Types.STRING);
+        
 		setArguments("swiftscript_trace", new Arg[] { Arg.VARGS });
+		
         setArguments("swiftscript_to_int", new Arg[] { PA_INPUT });
+        setReturnType("swiftscript_to_int", Types.INT);
+        
 		setArguments("swiftscript_toint", new Arg[] { PA_INPUT });
+		setReturnType("swiftscript_toint", Types.INT);
+		
         setArguments("swiftscript_to_float", new Arg[] { PA_INPUT });
+        setReturnType("swiftscript_to_float", Types.FLOAT);
+        
 		setArguments("swiftscript_tofloat", new Arg[] { PA_INPUT });
+		setReturnType("swiftscript_tofloat", Types.FLOAT);
+		
         setArguments("swiftscript_to_string", new Arg[] { PA_INPUT });
+        setReturnType("swiftscript_to_string", Types.STRING);
+        
         setArguments("swiftscript_tostring", new Arg[] { PA_INPUT });
+        setReturnType("swiftscript_tostring", Types.STRING);
 	}
 
 	private static final Logger traceLogger =

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -37,6 +37,8 @@
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.InvalidPathException;
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.mapping.PhysicalFormat;
@@ -68,7 +70,13 @@
 		if (tracer.isEnabled()) {
 		    tracer.trace(stack, "readData(" + Tracer.unwrapHandle(src) + ")");
 		}
-		src.waitFor();
+		try {
+		    src.waitFor();
+		}
+		catch (DependentException e) {
+		    dest.setValue(new DataDependentException(dest, e));
+		    return null;
+		}
 		if (src.getType().equals(Types.STRING)) {
 			readData(dest, (String) src.getValue());
 		}

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -34,6 +34,8 @@
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.mapping.PhysicalFormat;
 import org.griphyn.vdl.type.Types;
@@ -63,7 +65,13 @@
 		if (tracer.isEnabled()) {
             tracer.trace(stack, "readData2(" + Tracer.unwrapHandle(src) + ")");
         }
-        src.waitFor();
+        try {
+            src.waitFor();
+        }
+        catch (DependentException e) {
+            dest.setValue(new DataDependentException(dest, e));
+            return null;
+        }
 		if (src.getType().equals(Types.STRING)) {
 			readData(dest, (String) src.getValue());
 			dest.closeDeep();

Added: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/SwiftFunctionsCollection.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/SwiftFunctionsCollection.java	                        (rev 0)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/SwiftFunctionsCollection.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -0,0 +1,70 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Apr 17, 2014
+ */
+package org.griphyn.vdl.karajan.lib.swiftscript;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
+import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
+import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
+import org.griphyn.vdl.type.Types;
+
+public class SwiftFunctionsCollection extends FunctionsCollection {
+    private static Map<String, Type> returnTypes;
+    
+    static {
+        returnTypes = new HashMap<String, Type>();
+    }
+    
+    protected static void setReturnType(String name, Type rt) {
+        returnTypes.put(name, rt);
+    }
+    
+    protected Type getReturnType() {
+        Type t = returnTypes.get(getMethod().getName());
+        if (t == null) {
+            return Types.ANY;
+        }
+        else {
+            return t;
+        }
+    }
+    
+    public Object function(VariableStack stack) throws ExecutionException {
+        try {
+            return getMethod().invoke(this, stack);
+        }
+        catch (InvocationTargetException e) {
+            Throwable ex = e.getTargetException();
+            if (ex instanceof ExecutionException) {
+                throw (ExecutionException) ex;
+            }
+            else if (ex instanceof FutureNotYetAvailable) {
+                throw (FutureNotYetAvailable) ex;
+            }
+            else if (ex instanceof DependentException) {
+                RootDataNode rdn = new RootDataNode(getReturnType());
+                rdn.setValue(new DataDependentException(rdn, (DependentException) ex));
+                return rdn;
+            }
+            throw new ExecutionException(e.getTargetException());
+        }
+        catch (Exception e) {
+            throw new ExecutionException(e);
+        }
+    }
+}

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -23,6 +23,7 @@
 import org.globus.cog.karajan.workflow.ExecutionException;
 import org.griphyn.vdl.karajan.lib.VDLFunction;
 import org.griphyn.vdl.mapping.AbstractDataNode;
+import org.griphyn.vdl.mapping.DependentException;
 
 /**
     Formatted trace output. <br>
@@ -46,11 +47,16 @@
     @Override
     protected Object function(VariableStack stack) 
     throws ExecutionException {
-        AbstractDataNode[] args = waitForAllVargs(stack);
-
-        String msg = Sprintf.format(args);
-        logger.info(msg);
-        System.out.print(msg);
+        try {
+            AbstractDataNode[] args = waitForAllVargs(stack);
+    
+            String msg = Sprintf.format(args);
+            logger.info(msg);
+            System.out.print(msg);
+        }
+        catch (DependentException e) {
+            logger.info("tracef(): <exception>");
+        }
         return null;
     }
 }

Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java
===================================================================
--- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java	2014-04-18 01:29:19 UTC (rev 7777)
+++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java	2014-04-18 04:17:18 UTC (rev 7778)
@@ -33,6 +33,8 @@
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.HandleOpenException;
 import org.griphyn.vdl.mapping.InvalidPathException;
 import org.griphyn.vdl.mapping.Path;
@@ -59,7 +61,13 @@
 		// src can be any of several forms of value
 		AbstractDataNode src = (AbstractDataNode) SRC.getValue(stack);
 
-		src.waitFor();
+		try {
+            src.waitFor();
+        }
+        catch (DependentException e) {
+            dest.setValue(new DataDependentException(dest, e));
+            return null;
+        }
 
 		if (dest.getType().equals(Types.STRING)) {
 			writeData((String)dest.getValue(), src);



From davidk at ci.uchicago.edu  Fri Apr 18 12:01:33 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri, 18 Apr 2014 12:01:33 -0500 (CDT)
Subject: [Swift-commit] r7779 - tags
Message-ID: <20140418170133.1B3B79CFED@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-18 12:01:32 -0500 (Fri, 18 Apr 2014)
New Revision: 7779

Added:
   tags/swift-0.94.2-RC4/
Log:
0.94.2-RC4 tag




From davidk at ci.uchicago.edu  Fri Apr 18 16:25:41 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri, 18 Apr 2014 16:25:41 -0500 (CDT)
Subject: [Swift-commit] r7780 - branches/release-0.94/docs/quickstart
Message-ID: <20140418212541.ADA6A9D537@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-18 16:25:41 -0500 (Fri, 18 Apr 2014)
New Revision: 7780

Modified:
   branches/release-0.94/docs/quickstart/quickstart.txt
Log:
Fix path name to example scripts


Modified: branches/release-0.94/docs/quickstart/quickstart.txt
===================================================================
--- branches/release-0.94/docs/quickstart/quickstart.txt	2014-04-18 17:01:32 UTC (rev 7779)
+++ branches/release-0.94/docs/quickstart/quickstart.txt	2014-04-18 21:25:41 UTC (rev 7780)
@@ -89,7 +89,7 @@
 takes the Swift workflow file name as an argument:
 
 -----
-cd examples/tutorial
+cd swift-0.94.1/examples/swift/tutorial
 swift hello.swift
 -----
 



From davidk at ci.uchicago.edu  Fri Apr 18 16:32:31 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri, 18 Apr 2014 16:32:31 -0500 (CDT)
Subject: [Swift-commit] r7781 - SwiftTutorials/swift-cloud-tutorial/doc
Message-ID: <20140418213231.3B9339D537@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-18 16:32:30 -0500 (Fri, 18 Apr 2014)
New Revision: 7781

Modified:
   SwiftTutorials/swift-cloud-tutorial/doc/README
Log:
Fix file name mismatch


Modified: SwiftTutorials/swift-cloud-tutorial/doc/README
===================================================================
--- SwiftTutorials/swift-cloud-tutorial/doc/README	2014-04-18 21:25:41 UTC (rev 7780)
+++ SwiftTutorials/swift-cloud-tutorial/doc/README	2014-04-18 21:32:30 UTC (rev 7781)
@@ -46,7 +46,7 @@
 -----
 $ cd $HOME
 $ wget http://swiftlang.org/tutorials/cloud/swift-cloud-tutorial.tar.gz
-$ tar xvfz swift-cloud-tutorial.tgz
+$ tar xvfz swift-cloud-tutorial.tar.gz
 $ cd swift-cloud-tutorial
 -----
 



From hategan at ci.uchicago.edu  Sat Apr 19 14:30:29 2014
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Sat, 19 Apr 2014 14:30:29 -0500 (CDT)
Subject: [Swift-commit] r7782 - in
	branches/release-0.95/src/org/griphyn/vdl: karajan/lib
	karajan/lib/swiftscript mapping
Message-ID: <20140419193029.E85059D537@svn.ci.uchicago.edu>

Author: hategan
Date: 2014-04-19 14:30:28 -0500 (Sat, 19 Apr 2014)
New Revision: 7782

Added:
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/AbstractSingleValuedSwiftFunction.java
Modified:
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java
   branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java
   branches/release-0.95/src/org/griphyn/vdl/mapping/RootDataNode.java
Log:
fixed (to some extent) the lazy errors not propagating through built-in functions issue (0.95 version)

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -45,6 +45,8 @@
 import org.griphyn.vdl.karajan.WaitingThreadsMonitor;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.Mapper;
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.type.Type;
@@ -188,7 +190,13 @@
 	
 	protected void deepCopy(DSHandle dest, DSHandle source, Stack stack) {
 	    // don't create a state if only a non-composite is copied
-	    ((AbstractDataNode) source).waitFor(this);
+	    try {
+	        ((AbstractDataNode) source).waitFor(this);
+	    }
+	    catch (DependentException e) {
+	        dest.setValue(new DataDependentException(dest, e));
+	        return;
+	    }
         if (source.getType().isPrimitive()) {
             dest.setValue(source.getValue());
         }
@@ -208,7 +216,13 @@
     /** make dest look like source - if its a simple value, copy that
 	    and if its an array then recursively copy */
 	public void deepCopy(DSHandle dest, DSHandle source, State state, int level) {
-	    ((AbstractDataNode) source).waitFor(this);
+	    try {
+            ((AbstractDataNode) source).waitFor(this);
+        }
+        catch (DependentException e) {
+            dest.setValue(new DataDependentException(dest, e));
+            return;
+        }
         if (source.getType().isPrimitive()) {
             dest.setValue(source.getValue());
         }

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -52,6 +52,7 @@
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
 import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.GeneralizedFileFormat;
 import org.griphyn.vdl.mapping.HandleOpenException;
@@ -60,6 +61,7 @@
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.mapping.PathComparator;
 import org.griphyn.vdl.mapping.PhysicalFormat;
+import org.griphyn.vdl.mapping.RootDataNode;
 import org.griphyn.vdl.mapping.RootHandle;
 import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
@@ -103,8 +105,8 @@
 
     @Override
     public void runBody(LWThread thr) {
+        Stack stack = thr.getStack();
 		try {
-		    Stack stack = thr.getStack();
 		    ret(stack, function(stack));
 		}
 		catch (AssertFailedException e) { 
@@ -112,11 +114,15 @@
             throw e;
         }
         catch (DependentException e) {
-            // This would not be the primal fault so in non-lazy errors mode it
-            // should not matter
-            throw new ExecutionException("Wrapping a dependent exception in VDLFunction.post() - errors in data dependencies",e);
+            RootDataNode rdn = new RootDataNode(getReturnType());
+            rdn.setValue(new DataDependentException(rdn, e));
+            ret(stack, rdn);
         }
     }
+    
+    protected Type getReturnType() {
+        return Types.ANY;
+    }
 	
 	/*
 	 * This will likely break if the engine changes in fundamental ways. It also

Added: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/AbstractSingleValuedSwiftFunction.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/AbstractSingleValuedSwiftFunction.java	                        (rev 0)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/AbstractSingleValuedSwiftFunction.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -0,0 +1,40 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Apr 18, 2014
+ */
+package org.griphyn.vdl.karajan.lib.swiftscript;
+
+import k.rt.Stack;
+import k.thr.LWThread;
+
+import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
+import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
+import org.griphyn.vdl.type.Types;
+
+public abstract class AbstractSingleValuedSwiftFunction extends AbstractSingleValuedFunction {
+
+    @Override
+    public void runBody(LWThread thr) {
+        Stack stack = thr.getStack();
+        try {
+            ret(stack, function(stack));
+        }
+        catch (DependentException e) {
+            RootDataNode rdn = new RootDataNode(getReturnType());
+            rdn.setValue(new DataDependentException(rdn, e));
+            ret(stack, rdn);
+        }
+    }
+    
+    protected Type getReturnType() {
+        return Types.ANY;
+    }
+}

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -28,6 +28,7 @@
 import org.griphyn.vdl.karajan.lib.SwiftFunction;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.type.Types;
 
 /**
@@ -59,17 +60,22 @@
         }
         
         String message;
-        if (hmessage != null) {
-            hmessage.waitFor(this);
-            message = (String) hmessage.getValue();
+        try {
+            if (hmessage != null) {
+                hmessage.waitFor(this);
+                message = (String) hmessage.getValue();
+            }
+            else {
+                message = "Assertion failed";
+            }
+            AbstractDataNode hvalue = fargs.get(0);
+            hvalue.waitFor(this);
+                     
+            checkAssert(hvalue, message);
         }
-        else {
-            message = "Assertion failed";
+        catch (DependentException e) {
+            // cannot make assertion so ignore
         }
-        AbstractDataNode hvalue = fargs.get(0);
-        hvalue.waitFor(this);
-                 
-        checkAssert(hvalue, message);
         
         return null;
     }

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -31,6 +31,7 @@
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
 import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 
 
@@ -41,6 +42,11 @@
     protected Signature getSignature() {
         return new Signature(params("var"));
     }
+    
+    @Override
+    protected Type getReturnType() {
+        return Types.FLOAT;
+    }
 
     @Override
 	public Object function(Stack stack) {

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractInt.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -31,6 +31,7 @@
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
 import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 
 
@@ -43,6 +44,11 @@
     }
 
     @Override
+    protected Type getReturnType() {
+        return Types.INT;
+    }
+
+    @Override
 	public Object function(Stack stack) {
 		AbstractDataNode handle = this.var.getValue(stack);
 		try {

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileName.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -25,6 +25,7 @@
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
 import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 
 public class FileName extends SwiftFunction {
@@ -34,6 +35,11 @@
     protected Signature getSignature() {
         return new Signature(params("var"));
     }
+    
+    @Override
+    protected Type getReturnType() {
+        return Types.STRING;
+    }
 
     @Override
 	public Object function(Stack stack) {

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FileNames.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -31,6 +31,7 @@
 import org.griphyn.vdl.mapping.InvalidPathException;
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.mapping.RootArrayDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 
 public class FileNames extends SwiftFunction {
@@ -40,6 +41,11 @@
     protected Signature getSignature() {
         return new Signature(params("var"));
     }
+    
+    @Override
+    protected Type getReturnType() {
+        return Types.STRING.arrayType();
+    }
 
     @Override
 	public Object function(Stack stack) {

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -36,6 +36,7 @@
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
 import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 
 
@@ -48,6 +49,11 @@
     protected Signature getSignature() {
         return new Signature(params("..."));
     }
+	
+	@Override
+    protected Type getReturnType() {
+        return Types.STRING;
+    }
 
     @Override
     protected void addLocals(Scope scope) {

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -31,6 +31,7 @@
 import org.globus.cog.karajan.analyzer.Signature;
 import org.griphyn.vdl.karajan.lib.SwiftFunction;
 import org.griphyn.vdl.mapping.AbstractDataNode;
+import org.griphyn.vdl.mapping.DependentException;
 
 /**
     Formatted file output. <br>
@@ -57,24 +58,29 @@
     public Object function(Stack stack) {
         AbstractDataNode hfilename = this.filename.getValue(stack);
         AbstractDataNode hspec = this.spec.getValue(stack);
-        hfilename.waitFor(this);
-        hspec.waitFor(this);
-        Channel<AbstractDataNode> args = c_vargs.get(stack);
-        waitForAll(this, args);
-        String filename = (String) hfilename.getValue();
-        String spec = (String) hspec.getValue(); 
-        
-        StringBuilder output = new StringBuilder();
         try {
-            Sprintf.format(spec, args, output);
+            hfilename.waitFor(this);
+            hspec.waitFor(this);
+            Channel<AbstractDataNode> args = c_vargs.get(stack);
+            waitForAll(this, args);
+            String filename = (String) hfilename.getValue();
+            String spec = (String) hspec.getValue(); 
+            
+            StringBuilder output = new StringBuilder();
+            try {
+                Sprintf.format(spec, args, output);
+            }
+            catch (RuntimeException e) {
+                throw new ExecutionException(this, e.getMessage());
+            }
+            String msg = output.toString();
+     
+            logger.debug("file: " + filename + " msg: " + msg);        
+            write(filename, msg);
         }
-        catch (RuntimeException e) {
-            throw new ExecutionException(this, e.getMessage());
+        catch (DependentException e) {
+            logger.debug("<exception>");
         }
-        String msg = output.toString();
- 
-        logger.debug("file: " + filename + " msg: " + msg);        
-        write(filename, msg);
         return null;
     }
     

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -32,14 +32,15 @@
 import org.globus.cog.karajan.analyzer.ChannelRef;
 import org.globus.cog.karajan.analyzer.Signature;
 import org.globus.cog.karajan.compiled.nodes.InternalFunction;
-import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction;
 import org.globus.cog.karajan.util.TypeUtil;
 import org.griphyn.vdl.karajan.lib.SwiftFunction;
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.RootArrayDataNode;
 import org.griphyn.vdl.mapping.RootDataNode;
+import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
 import org.griphyn.vdl.util.VDL2Config;
 
@@ -73,22 +74,26 @@
         @Override
         protected void runBody(LWThread thr) {
         	Channel<AbstractDataNode> vargs = c_vargs.get(thr.getStack());
-            SwiftFunction.waitForAll(this, vargs);
-
-            StringBuilder buf = new StringBuilder();
-            buf.append("SwiftScript trace: ");
-            boolean first = true;
-            for (AbstractDataNode n : vargs) {
-                if (!first) {
-                    buf.append(", ");
+        	try {
+        	    SwiftFunction.waitForAll(this, vargs);
+        	    StringBuilder buf = new StringBuilder();
+                buf.append("SwiftScript trace: ");
+                boolean first = true;
+                for (AbstractDataNode n : vargs) {
+                    if (!first) {
+                        buf.append(", ");
+                    }
+                    else {
+                        first = false;
+                    }
+                    //buf.append(v == null ? args[i] : v);
+                    prettyPrint(buf, n);
                 }
-                else {
-                	first = false;
-                }
-                //buf.append(v == null ? args[i] : v);
-                prettyPrint(buf, n);
-            }
-            traceLogger.warn(buf);
+                traceLogger.warn(buf);
+        	}
+        	catch (DependentException e) {
+        	    traceLogger.warn("<exception>");
+        	}
         }
 	}
 	
@@ -128,7 +133,7 @@
         }
     }
 	
-	public static class StrCat extends AbstractSingleValuedFunction {
+	public static class StrCat extends AbstractSingleValuedSwiftFunction {
         private ChannelRef<AbstractDataNode> c_vargs;
 
         @Override
@@ -137,6 +142,13 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+
+
+
+        @Override
         public Object function(Stack stack) {
             Channel<AbstractDataNode> vargs = c_vargs.get(stack);
             Channel<Object> args = SwiftFunction.unwrapAll(this, vargs);
@@ -161,7 +173,7 @@
         }
 	}
 	
-	public static class Exists extends AbstractSingleValuedFunction {
+	public static class Exists extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> file;
 
         @Override
@@ -170,6 +182,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.BOOLEAN;
+        }
+        
+        @Override
         public Object function(Stack stack) {
         	AbstractDataNode dn = file.getValue(stack);
             String filename = SwiftFunction.unwrap(this, dn);
@@ -190,7 +207,7 @@
         }
     }
 	
-	public static class StrCut extends AbstractSingleValuedFunction {
+	public static class StrCut extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> input;
         private ArgRef<AbstractDataNode> pattern;
 
@@ -200,6 +217,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+        
+        @Override
         public Object function(Stack stack) {
         	AbstractDataNode hinput = this.input.getValue(stack);
         	String input = SwiftFunction.unwrap(this, hinput);
@@ -238,7 +260,7 @@
         }
 	}
 	
-	public static class StrStr extends AbstractSingleValuedFunction {
+	public static class StrStr extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> input;
         private ArgRef<AbstractDataNode> pattern;
 
@@ -248,6 +270,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode hinput = this.input.getValue(stack);
             String input = SwiftFunction.unwrap(this, hinput);
@@ -271,7 +298,7 @@
         }
     }
 	
-	public static class StrSplit extends AbstractSingleValuedFunction {
+	public static class StrSplit extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> input;
         private ArgRef<AbstractDataNode> pattern;
 
@@ -281,6 +308,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING.arrayType();
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode hinput = this.input.getValue(stack);
             String input = SwiftFunction.unwrap(this, hinput);
@@ -318,7 +350,7 @@
 	 * @return DSHandle representing the resulting string
 	 * @throws ExecutionException
 	 */
-	public static class StrJoin extends AbstractSingleValuedFunction {
+	public static class StrJoin extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> array;
         private ArgRef<AbstractDataNode> delim;
 
@@ -328,6 +360,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode harray = this.array.getValue(stack);
             Map<Comparable<?>, DSHandle> arrayValues = SwiftFunction.waitForArray(this, harray);
@@ -359,7 +396,7 @@
         }
     }
     
-	public static class Regexp extends AbstractSingleValuedFunction {
+	public static class Regexp extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> input;
         private ArgRef<AbstractDataNode> pattern;
         private ArgRef<AbstractDataNode> transform;
@@ -370,6 +407,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode hinput = this.input.getValue(stack);
             String input = SwiftFunction.unwrap(this, hinput);
@@ -410,7 +452,7 @@
         }
     }
 	
-	public static class ToInt extends AbstractSingleValuedFunction {
+	public static class ToInt extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> str;
 
         @Override
@@ -419,6 +461,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.INT;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode hstr = str.getValue(stack);
             String str = SwiftFunction.unwrap(this, hstr);
@@ -435,7 +482,7 @@
         }
     }
 	
-	public static class ToFloat extends AbstractSingleValuedFunction {
+	public static class ToFloat extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> str;
 
         @Override
@@ -444,6 +491,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.FLOAT;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode hstr = str.getValue(stack);
             Object obj = SwiftFunction.unwrap(this, hstr);
@@ -474,7 +526,7 @@
 	/*
 	 * Takes in a float and formats to desired precision and returns a string
 	 */
-	public static class Format extends AbstractSingleValuedFunction {
+	public static class Format extends AbstractSingleValuedSwiftFunction {
 	    private ChannelRef<AbstractDataNode> c_vargs;
 
         @Override
@@ -483,6 +535,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             Channel<AbstractDataNode> vargs = c_vargs.get(stack);
             Channel<Object> args = SwiftFunction.unwrapAll(this, vargs);
@@ -507,7 +564,7 @@
 	/*
 	 * Takes in an int and pads zeros to the left and returns a string
 	 */
-	public static class Pad extends AbstractSingleValuedFunction {
+	public static class Pad extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> size;
         private ArgRef<AbstractDataNode> value;
 
@@ -517,6 +574,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode hsize = this.size.getValue(stack);
             Integer size = SwiftFunction.unwrap(this, hsize);
@@ -536,7 +598,7 @@
         }
     }
 	
-	public static class ToString extends AbstractSingleValuedFunction {
+	public static class ToString extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> value;
 
         @Override
@@ -545,6 +607,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode hvalue = this.value.getValue(stack);
             hvalue.waitFor(this);
@@ -570,7 +637,7 @@
      * Good for debugging because array needs to be closed
      *   before the length is determined
      */
-	public static class Dirname extends AbstractSingleValuedFunction {
+	public static class Dirname extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> file;
 
         @Override
@@ -579,6 +646,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.STRING;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode dn = file.getValue(stack);
             String name = SwiftFunction.filename(dn)[0];
@@ -596,7 +668,7 @@
         }
     }
 		
-	public static class Length extends AbstractSingleValuedFunction {
+	public static class Length extends AbstractSingleValuedSwiftFunction {
         private ArgRef<AbstractDataNode> array;
 
         @Override
@@ -605,6 +677,11 @@
         }
         
         @Override
+        protected Type getReturnType() {
+            return Types.INT;
+        }
+        
+        @Override
         public Object function(Stack stack) {
             AbstractDataNode harray = this.array.getValue(stack);
             harray.waitFor(this);

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -38,6 +38,8 @@
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.PhysicalFormat;
 import org.griphyn.vdl.type.Type;
 import org.griphyn.vdl.type.Types;
@@ -59,7 +61,13 @@
 	public Object function(Stack stack) {
 		AbstractDataNode dest = this.dest.getValue(stack);
 		AbstractDataNode src = this.src.getValue(stack);
-		src.waitFor(this);
+		try {
+		    src.waitFor(this);
+		}
+		catch (DependentException e) {
+		    dest.setValue(new DataDependentException(dest, e));
+		    return null;
+		}
 		if (src.getType().equals(Types.STRING)) {
 			readData(dest, (String) src.getValue());
 		}

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadStructured.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -35,6 +35,8 @@
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.mapping.PhysicalFormat;
 import org.griphyn.vdl.type.Types;
@@ -57,7 +59,13 @@
     public Object function(Stack stack) {
         AbstractDataNode dest = this.dest.getValue(stack);
         AbstractDataNode src = this.src.getValue(stack);
-        src.waitFor(this);
+        try {
+            src.waitFor(this);
+        }
+        catch (DependentException e) {
+            dest.setValue(new DataDependentException(dest, e));
+            return null;
+        }
 		if (src.getType().equals(Types.STRING)) {
 			readData(dest, (String) src.getValue());
 			dest.closeDeep();

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -26,6 +26,7 @@
 import org.globus.cog.karajan.analyzer.Signature;
 import org.griphyn.vdl.karajan.lib.SwiftFunction;
 import org.griphyn.vdl.mapping.AbstractDataNode;
+import org.griphyn.vdl.mapping.DependentException;
 
 /**
     Formatted trace output. <br>
@@ -52,12 +53,18 @@
     @Override
     public Object function(Stack stack) {
         AbstractDataNode hspec = this.spec.getValue(stack);
-        hspec.waitFor(this);
-        Channel<AbstractDataNode> args = c_vargs.get(stack);
-        waitForAll(this, args);
-        String spec = (String) hspec.getValue();
-     
-        String msg = Sprintf.format(spec, args);
+        String msg;
+        try {
+            hspec.waitFor(this);
+            Channel<AbstractDataNode> args = c_vargs.get(stack);
+            waitForAll(this, args);
+            String spec = (String) hspec.getValue();
+         
+            msg = Sprintf.format(spec, args);
+        }
+        catch (DependentException e) {
+            msg = "<exception>";
+        }
         logger.info(msg);
         System.out.print(msg);
         return null;

Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -34,6 +34,8 @@
 import org.griphyn.vdl.mapping.AbsFile;
 import org.griphyn.vdl.mapping.AbstractDataNode;
 import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.DataDependentException;
+import org.griphyn.vdl.mapping.DependentException;
 import org.griphyn.vdl.mapping.InvalidPathException;
 import org.griphyn.vdl.mapping.Path;
 import org.griphyn.vdl.mapping.PhysicalFormat;
@@ -62,7 +64,13 @@
 		// src can be any of several forms of value
 		AbstractDataNode src = this.src.getValue(stack);
 
-		src.waitFor(this);
+		try {
+            src.waitFor(this);
+        }
+        catch (DependentException e) {
+            dest.setValue(new DataDependentException(dest, e));
+            return null;
+        }
 
 		if (dest.getType().equals(Types.STRING)) {
 			writeData((String)dest.getValue(), src);

Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/RootDataNode.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/mapping/RootDataNode.java	2014-04-18 21:32:30 UTC (rev 7781)
+++ branches/release-0.95/src/org/griphyn/vdl/mapping/RootDataNode.java	2014-04-19 19:30:28 UTC (rev 7782)
@@ -54,6 +54,10 @@
         this("?", type, value);
     }
 	
+	public RootDataNode(Type type) {
+        this("?", type, null);
+    }
+	
 	public RootDataNode(String name, Type type) {
 	    this(name, type, null);
 	}
@@ -64,7 +68,7 @@
 	    setValue(value);
 	}
 	
-	    public int getLine() {
+	public int getLine() {
         return line;
     }
 



From wozniak at ci.uchicago.edu  Mon Apr 21 15:11:24 2014
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Mon, 21 Apr 2014 15:11:24 -0500 (CDT)
Subject: [Swift-commit] r7783 - in SwiftTutorials/swift-cloud-tutorial: . doc
Message-ID: <20140421201124.83DF0178884@svn.ci.uchicago.edu>

Author: wozniak
Date: 2014-04-21 15:11:24 -0500 (Mon, 21 Apr 2014)
New Revision: 7783

Modified:
   SwiftTutorials/swift-cloud-tutorial/doc/TODO
   SwiftTutorials/swift-cloud-tutorial/setup.sh
Log:
Support ZSH


Modified: SwiftTutorials/swift-cloud-tutorial/doc/TODO
===================================================================
--- SwiftTutorials/swift-cloud-tutorial/doc/TODO	2014-04-19 19:30:28 UTC (rev 7782)
+++ SwiftTutorials/swift-cloud-tutorial/doc/TODO	2014-04-21 20:11:24 UTC (rev 7783)
@@ -77,6 +77,7 @@
  
 3. support for zsh, csh
 [pending]
+ZSH is done. -Justin
  
 4. What results are created and where did the outputs go ?
 and pointers to what the log files mean

Modified: SwiftTutorials/swift-cloud-tutorial/setup.sh
===================================================================
--- SwiftTutorials/swift-cloud-tutorial/setup.sh	2014-04-19 19:30:28 UTC (rev 7782)
+++ SwiftTutorials/swift-cloud-tutorial/setup.sh	2014-04-21 20:11:24 UTC (rev 7783)
@@ -1,16 +1,28 @@
 # ensure that this script is being sourced
-if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" = "${0}" ] ; then
-  echo ERROR: script ${BASH_SOURCE[0]} must be executed as: source ${BASH_SOURCE[0]}
-  exit 1
+
+if [[ $0 =~ .*bash ]] ; then 
+# Bash: 
+  if [ ${BASH_VERSINFO[0]} -gt 2 -a "${BASH_SOURCE[0]}" = "${0}" ] ; then
+    echo ERROR: script ${BASH_SOURCE[0]} must be executed as: source ${BASH_SOURCE[0]}
+    exit 1
+  fi
+  # Lookup tutorial directory based on PATH entry for this script
+  TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 fi
+if [[ $ZSH_VERSION != "" ]] ; then 
+  # ZSH: 
+  if [[  ! $- =~ .*i.* ]] ; then 
+    echo ERROR: script $0 must be executed as: source $0
+    return 1
+  fi
+  # Lookup tutorial directory based on PATH entry for this script
+  TUTDIR="$( cd "$( dirname "$0" )" && /bin/pwd )"
+fi
 
 echo Swift version is $(swift -version)
 rm -f swift.log
 
-# Setting scripts folder to the PATH env var.
-TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-if [ _$(which cleanup 2>/dev/null) != _$TUTDIR/bin/cleanup ]; then
+if [ _$(env which cleanup 2>/dev/null) != _$TUTDIR/bin/cleanup ]; then
   echo Adding $TUTDIR/bin:$TUTDIR/app: to front of PATH
   PATH=$TUTDIR/bin:$TUTDIR/app:$PATH
 else
@@ -56,4 +68,3 @@
 cd ..
 
 return
-



From wozniak at ci.uchicago.edu  Mon Apr 21 15:36:22 2014
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Mon, 21 Apr 2014 15:36:22 -0500 (CDT)
Subject: [Swift-commit] r7784 - SwiftTutorials/swift-cloud-tutorial/doc
Message-ID: <20140421203622.74E62178884@svn.ci.uchicago.edu>

Author: wozniak
Date: 2014-04-21 15:36:22 -0500 (Mon, 21 Apr 2014)
New Revision: 7784

Modified:
   SwiftTutorials/swift-cloud-tutorial/doc/README
Log:
Add note about supported shells


Modified: SwiftTutorials/swift-cloud-tutorial/doc/README
===================================================================
--- SwiftTutorials/swift-cloud-tutorial/doc/README	2014-04-21 20:11:24 UTC (rev 7783)
+++ SwiftTutorials/swift-cloud-tutorial/doc/README	2014-04-21 20:36:22 UTC (rev 7784)
@@ -29,6 +29,8 @@
 compute nodes, and see how more complex workflows can be expressed
 with Swift scripts.
 
+NOTE: This tutorial assumes you are using Bash or ZSH. 
+
 Swift installation
 ------------------
 1. Download the file from http://swiftlang.org/packages/swift-0.94.1-RC2.tar.gz



From wozniak at ci.uchicago.edu  Mon Apr 21 15:53:43 2014
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Mon, 21 Apr 2014 15:53:43 -0500 (CDT)
Subject: [Swift-commit] r7785 - SwiftTutorials/swift-cloud-tutorial/doc
Message-ID: <20140421205343.3C9F3178884@svn.ci.uchicago.edu>

Author: wozniak
Date: 2014-04-21 15:53:43 -0500 (Mon, 21 Apr 2014)
New Revision: 7785

Modified:
   SwiftTutorials/swift-cloud-tutorial/doc/README
Log:
Highlight key sections


Modified: SwiftTutorials/swift-cloud-tutorial/doc/README
===================================================================
--- SwiftTutorials/swift-cloud-tutorial/doc/README	2014-04-21 20:36:22 UTC (rev 7784)
+++ SwiftTutorials/swift-cloud-tutorial/doc/README	2014-04-21 20:53:43 UTC (rev 7785)
@@ -199,8 +199,8 @@
 -----
 
 
-Basic of the Swift language with local execution
-------------------------------------------------
+Unit 1: Basics of the Swift language with local execution
+---------------------------------------------------------
 
 A Summary of Swift in a nutshell
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -489,8 +489,8 @@
 on the site's compute nodes.
 
 
-Running applications on compute nodes with Swift
-------------------------------------------------
+Unit 2: Running applications on compute nodes with Swift
+--------------------------------------------------------
 
 Part 4: Running a parallel ensemble on compute nodes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~



From davidk at ci.uchicago.edu  Wed Apr 23 15:36:29 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 23 Apr 2014 15:36:29 -0500 (CDT)
Subject: [Swift-commit] r7786 - branches/release-0.95/docs/userguide
Message-ID: <20140423203629.13A7A9D0A1@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-23 15:36:28 -0500 (Wed, 23 Apr 2014)
New Revision: 7786

Modified:
   branches/release-0.95/docs/userguide/configuration
Log:
Add undocumented property 'use.provider.staging' and some minor formatting


Modified: branches/release-0.95/docs/userguide/configuration
===================================================================
--- branches/release-0.95/docs/userguide/configuration	2014-04-21 20:53:43 UTC (rev 7785)
+++ branches/release-0.95/docs/userguide/configuration	2014-04-23 20:36:28 UTC (rev 7786)
@@ -173,64 +173,68 @@
 |================================
 |Property|Description|Example
 
-|filesystem|
-Defines how files should be accessed
+|filesystem
+|Defines how files should be accessed
 |site.westmere.filesystem=local
 
-|jobGranularity|
-Specifies the granularity of a job, in nodes
+|jobGranularity
+|Specifies the granularity of a job, in nodes
 |site.westmere.jobGranularity=2
 
-|jobManager|
-Specifies how jobs will be launched. The supported job managers are 
+|jobManager
+|Specifies how jobs will be launched. The supported job managers are 
 "cobalt", "slurm", "condor", "pbs", "lsf", "local", and "sge". 
 |site.westmere.jobManager=slurm
 
-|jobProject|
-Set the project name for the job scheduler
+|jobProject
+|Set the project name for the job scheduler
 |site.westmere.project=myproject
 
-|jobQueue|
-Set the name of the scheduler queue to use.
+|jobQueue
+|Set the name of the scheduler queue to use.
 |site.westmere.jobQueue=westmere
 
-|jobWalltime|
-The maximum number amount of time allocated in a scheduler job, in hh:mm:ss
+|jobWalltime
+|The maximum number amount of time allocated in a scheduler job, in hh:mm:ss
 format.
 |site.westmere.jobWalltime=01:00:00
 
-|maxJobs|
-Maximum number of scheduler jobs to submit
+|maxJobs
+|Maximum number of scheduler jobs to submit
 |site.westmere.maxJobs=20
 
-|maxNodesPerJob|
-The maximum number of nodes to request per scheduler job.
+|maxNodesPerJob
+|The maximum number of nodes to request per scheduler job.
 |site.westmere.maxNodesPerJob=2
 
-|taskDir|
-Tasks will be run from this directory. In the absence of a taskDir definition, 
+|pe
+|The parallel environment to use for SGE schedulers
+|site.sunhpc.pe=mpi
+
+|taskDir
+|Tasks will be run from this directory. In the absence of a taskDir definition, 
 Swift will run the task from workdir.
 |site.westmere.taskDir=/scratch/local/$USER/work
 
-|tasksPerWorker|
-The number of tasks that each worker can run simultaneously.
+|tasksPerWorker
+|The number of tasks that each worker can run simultaneously.
 |site.westmere.tasksPernode=12
 
-|taskThrottle|
-The maximum number of active tasks across all workers.
+|taskThrottle
+|The maximum number of active tasks across all workers.
 |site.westmere.taskThrottle=100
 
-|taskWalltime|
-The maximum amount of time a task may run, in hh:mm:ss.
+|taskWalltime
+|The maximum amount of time a task may run, in hh:mm:ss.
 |site.westmere.taskWalltime=01:00:00
 
-|site    |
-Name of site or sites to run on. This is the same as running with
+|site    
+|Name of site or sites to run on. This is the same as running with
 swift -site <sitename>
 |site=westmere
 
-|workdir |
-The workdirectory element specifies where on the site files can be stored.
+|workdir
+|The workdirectory element specifies where on the site files can be stored.
 This directory must be available on all worker nodes that will be used for 
 execution. A shared cluster filesystem is appropriate for this. Note that 
 you need to specify absolute pathname for this field.
@@ -457,6 +461,11 @@
 |false
 |Determines if the Swift wrapper should do file staging.
 
+|use.provider.staging
+|true, false
+|false
+|If true, files will be staged by Swift over the network.
+
 |wrapper.invocation.mode
 |absolute, relative
 |absolute



From davidk at ci.uchicago.edu  Wed Apr 23 15:49:27 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Wed, 23 Apr 2014 15:49:27 -0500 (CDT)
Subject: [Swift-commit] r7787 - trunk/docs/userguide
Message-ID: <20140423204927.84D8B9D584@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-23 15:49:27 -0500 (Wed, 23 Apr 2014)
New Revision: 7787

Modified:
   trunk/docs/userguide/configuration
Log:
Formatting


Modified: trunk/docs/userguide/configuration
===================================================================
--- trunk/docs/userguide/configuration	2014-04-23 20:36:28 UTC (rev 7786)
+++ trunk/docs/userguide/configuration	2014-04-23 20:49:27 UTC (rev 7787)
@@ -172,52 +172,52 @@
 |================================
 |Property|Description|Example
 
-|condor|
-Pass parameters directly through to the submit script generated for the condor
+|condor
+|Pass parameters directly through to the submit script generated for the condor
 scheduler. For example, the setting "site.osgconnect.condor.+projectname=Swift"
 will generate the line "+projectname = Swift".
 |site.osgconnect.condor.+projectname=Swift
 
-|filesystem|
-Defines how files should be accessed
+|filesystem
+|Defines how files should be accessed
 |site.westmere.filesystem=local
 
-|jobGranularity|
-Specifies the granularity of a job, in nodes
+|jobGranularity
+|Specifies the granularity of a job, in nodes
 |site.westmere.jobGranularity=2
 
-|jobManager|
-Specifies how jobs will be launched. The supported job managers are 
+|jobManager
+|Specifies how jobs will be launched. The supported job managers are 
 "cobalt", "slurm", "condor", "pbs", "lsf", "local", and "sge". 
 |site.westmere.jobManager=slurm
 
-|jobProject|
-Set the project name for the job scheduler
+|jobProject
+|Set the project name for the job scheduler
 |site.westmere.project=myproject
 
-|jobQueue|
-Set the name of the scheduler queue to use.
+|jobQueue
+|Set the name of the scheduler queue to use.
 |site.westmere.jobQueue=westmere
 
-|jobWalltime|
-The maximum number amount of time allocated in a scheduler job, in hh:mm:ss
+|jobWalltime
+|The maximum number amount of time allocated in a scheduler job, in hh:mm:ss
 format.
 |site.westmere.jobWalltime=01:00:00
 
-|maxJobs|
-Maximum number of scheduler jobs to submit
+|maxJobs
+|Maximum number of scheduler jobs to submit
 |site.westmere.maxJobs=20
 
-|maxNodesPerJob|
-The maximum number of nodes to request per scheduler job.
+|maxNodesPerJob
+|The maximum number of nodes to request per scheduler job.
 |site.westmere.maxNodesPerJob=2
 
 |pe
 |The parallel environment to use for SGE schedulers
 |site.sunhpc.pe=mpi
 
-|providerAttributes|
-Allows user to pass attributes through directly to scheduler submit script. Currently
+|providerAttributes
+|Allows user to pass attributes through directly to scheduler submit script. Currently
 only implemented for sites that use PBS.
 |site.beagle.providerAttributes=pbs.aprun;pbs.mpp;depth=24
 
@@ -227,36 +227,36 @@
 the line "#SBATCH --mail-user=username".
 |site.midway.slurm.mail-user=username
 
-|taskDir|
-Tasks will be run from this directory. In the absence of a taskDir definition, 
+|taskDir
+|Tasks will be run from this directory. In the absence of a taskDir definition, 
 Swift will run the task from workdir.
 |site.westmere.taskDir=/scratch/local/$USER/work
 
-|tasksPerWorker|
-The number of tasks that each worker can run simultaneously.
+|tasksPerWorker
+|The number of tasks that each worker can run simultaneously.
 |site.westmere.tasksPernode=12
 
-|taskThrottle|
-The maximum number of active tasks across all workers.
+|taskThrottle
+|The maximum number of active tasks across all workers.
 |site.westmere.taskThrottle=100
 
-|taskWalltime|
-The maximum amount of time a task may run, in hh:mm:ss.
+|taskWalltime
+|The maximum amount of time a task may run, in hh:mm:ss.
 |site.westmere.taskWalltime=01:00:00
 
-|site    |
-Name of site or sites to run on. This is the same as running with
+|site
+|Name of site or sites to run on. This is the same as running with
 swift -site <sitename>
 |site=westmere
 
-|userHomeOverride|
-Sets the Swift user home. This must be a shared filesystem. This defaults
+|userHomeOverride
+|Sets the Swift user home. This must be a shared filesystem. This defaults
 to $HOME. For clusters where $HOME is not accessible to the worker nodes,
 you may override the value to point to a shared directory that you own.
 |site.beagle.userHomeOverride=/lustre/beagle/username
 
-|workdir |
-The workdirectory element specifies where on the site files can be stored.
+|workdir
+|The workdirectory element specifies where on the site files can be stored.
 This directory must be available on all worker nodes that will be used for 
 execution. A shared cluster filesystem is appropriate for this. Note that 
 you need to specify absolute pathname for this field.
@@ -497,12 +497,16 @@
 |Progress:
 |String to prepend to ticker output
 
-
 |use.wrapper.staging
 |true, false
 |false
 |Determines if the Swift wrapper should do file staging.
 
+|use.provider.staging
+|true, false
+|false
+|If true, files will be staged by Swift over the network.
+
 |wrapper.invocation.mode
 |absolute, relative
 |absolute



From wilde at ci.uchicago.edu  Thu Apr 24 09:34:03 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 09:34:03 -0500 (CDT)
Subject: [Swift-commit] r7788 - demo
Message-ID: <20140424143403.94DB49D0A1@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 09:34:02 -0500 (Thu, 24 Apr 2014)
New Revision: 7788

Added:
   demo/xsede.2014.0425/
Log:
create new demo project



From davidk at ci.uchicago.edu  Thu Apr 24 10:16:42 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 10:16:42 -0500 (CDT)
Subject: [Swift-commit] r7789 -
	branches/release-0.95/src/org/griphyn/vdl/karajan/monitor/processors/swift
Message-ID: <20140424151642.BC9959D102@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-24 10:16:42 -0500 (Thu, 24 Apr 2014)
New Revision: 7789

Modified:
   branches/release-0.95/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java
Log:
Ignore heap size messages in summaryprocessor (bug 1263)


Modified: branches/release-0.95/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java
===================================================================
--- branches/release-0.95/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java	2014-04-24 14:34:02 UTC (rev 7788)
+++ branches/release-0.95/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java	2014-04-24 15:16:42 UTC (rev 7789)
@@ -40,6 +40,9 @@
 
 	public void processMessage(SystemState state, Object message, Object details) {
 		String msg = String.valueOf(message);
+		if(msg.contains("CrtHeap")) {
+		   return;
+		}
 		SummaryItem s = (SummaryItem) state.getItemByID(SummaryItem.ID, StatefulItemClass.WORKFLOW);
 		String[] pairs = msg.split("  ");
 		for (ApplicationState key : SummaryItem.STATES) {



From davidk at ci.uchicago.edu  Thu Apr 24 10:32:21 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 10:32:21 -0500 (CDT)
Subject: [Swift-commit] r7790 -
	trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift
Message-ID: <20140424153221.A72019D102@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-24 10:32:21 -0500 (Thu, 24 Apr 2014)
New Revision: 7790

Modified:
   trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java
Log:
Fix for 1263


Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java	2014-04-24 15:16:42 UTC (rev 7789)
+++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java	2014-04-24 15:32:21 UTC (rev 7790)
@@ -40,6 +40,9 @@
 
 	public void processMessage(SystemState state, Object message, Object details) {
 		String msg = String.valueOf(message);
+		if(msg.contains("CrtHeap")) {
+		   return;
+		}
 		SummaryItem s = (SummaryItem) state.getItemByID(SummaryItem.ID, StatefulItemClass.WORKFLOW);
 		String[] pairs = msg.split("  ");
 		for (ApplicationState key : SummaryItem.STATES) {



From wilde at ci.uchicago.edu  Thu Apr 24 11:32:20 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 11:32:20 -0500 (CDT)
Subject: [Swift-commit] r7791 - SwiftApps/SimulatedAnnealing
Message-ID: <20140424163220.E8D9D9D102@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 11:32:20 -0500 (Thu, 24 Apr 2014)
New Revision: 7791

Modified:
   SwiftApps/SimulatedAnnealing/sa.swift
Log:
Commit changes to sa example from Fri Apr 11 session with Jonathan Ozik

Modified: SwiftApps/SimulatedAnnealing/sa.swift
===================================================================
--- SwiftApps/SimulatedAnnealing/sa.swift	2014-04-24 15:32:21 UTC (rev 7790)
+++ SwiftApps/SimulatedAnnealing/sa.swift	2014-04-24 16:32:20 UTC (rev 7791)
@@ -377,13 +377,37 @@
   float y[];
   y[0] = objective(a[0]);
 
-  iterate c {
-    int cy = c + 1;
-    a[cy] = [ newx(a[0][0],0.5), newx(a[0][1],0.5), newx(a[0][2],0.5), newx(a[0][3],0.5) ];
-    y[cy] = objective(a[cy]);
+  iterate i {
+    int c = i + 1;
+    a[c] = [ newx(a[i][0],0.5), newx(a[i][1],0.5), newx(a[i][2],0.5), newx(a[i][3],0.5) ];
+    float ny = objective(a[c]);
+    if ( accept(ny,c,y[c-1]) ) {
+      y[c] = ny;
+    }
+    else {
+      y[c] = y[c-1];
+    }
     trace(c,y[c]);
-  } until (c==10);
+  } until (i==10);
 
 }
 
+(boolean a) accept (float cur, int cy, float prev)
+{
+  float T_start = 1.9;
+  float T_end   = 1.0;
+  int cycles = 10;
+
+  float t =  T_start*exp( @tofloat( cy-1 ) * ( jlog(T_end) - jlog(T_start) ) / @tofloat(cycles) );
+
+  float ratio = min( 1.0, exp( - (cur-prev) / t ) );
+  float r = (random()); // a random float between [0.0-1.0]
+  if (r > ratio) { // Reject new parameter
+    a = false;
+  }
+  else {
+    a = true;
+  }
+}
+
 main();



From ketan at ci.uchicago.edu  Thu Apr 24 12:08:59 2014
From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 12:08:59 -0500 (CDT)
Subject: [Swift-commit] r7792 - SwiftApps/swift-galaxy/swift
Message-ID: <20140424170859.BB1259D0A1@svn.ci.uchicago.edu>

Author: ketan
Date: 2014-04-24 12:08:59 -0500 (Thu, 24 Apr 2014)
New Revision: 7792

Added:
   SwiftApps/swift-galaxy/swift/tryswift.xml
Log:
adding tryswift

Added: SwiftApps/swift-galaxy/swift/tryswift.xml
===================================================================
--- SwiftApps/swift-galaxy/swift/tryswift.xml	                        (rev 0)
+++ SwiftApps/swift-galaxy/swift/tryswift.xml	2014-04-24 17:08:59 UTC (rev 7792)
@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<tool name="TrySwift" id="tryswift" tool_type="data_source">
+    <description>A tool to try Swift Online</description>
+    <command interpreter="python">data_source.py $output $__app__.config.output_size_limit</command>
+    <inputs action="http://swiftvm4.ci.uchicago.edu" check_values="false" method="get"> 
+        <display>go to Try Swift server at $GALAXY_URL</display>
+    </inputs>
+    <request_param_translation>
+        <request_param galaxy_name="data_type" remote_name="data_type" missing="auto" >
+            <value_translation>
+                <value galaxy_value="auto" remote_value="txt" /> <!-- intermine currently always provides 'txt', make this auto detect -->
+            </value_translation>
+        </request_param>
+    </request_param_translation>
+    <uihints minwidth="800"/>
+    <outputs>
+        <data name="output" format="txt" />
+    </outputs>
+    <options sanitize="False" refresh="True"/>
+</tool>



From wilde at ci.uchicago.edu  Thu Apr 24 12:50:23 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 12:50:23 -0500 (CDT)
Subject: [Swift-commit] r7793 - in demo/xsede.2014.0425: . src src/c-ray
Message-ID: <20140424175023.B3C999D102@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 12:50:23 -0500 (Thu, 24 Apr 2014)
New Revision: 7793

Added:
   demo/xsede.2014.0425/src/
   demo/xsede.2014.0425/src/c-ray/
   demo/xsede.2014.0425/src/c-ray/Makefile
   demo/xsede.2014.0425/src/c-ray/README
   demo/xsede.2014.0425/src/c-ray/RUN
   demo/xsede.2014.0425/src/c-ray/c-ray.c
   demo/xsede.2014.0425/src/c-ray/scene.cr
   demo/xsede.2014.0425/src/c-ray/sphfract.cr
Log:
Add stripped-down c-ray source dir.

Added: demo/xsede.2014.0425/src/c-ray/Makefile
===================================================================
--- demo/xsede.2014.0425/src/c-ray/Makefile	                        (rev 0)
+++ demo/xsede.2014.0425/src/c-ray/Makefile	2014-04-24 17:50:23 UTC (rev 7793)
@@ -0,0 +1,23 @@
+#obj = c-ray-mt.o
+bin = c-ray
+
+CC = gcc
+CFLAGS = -O3 -ffast-math
+
+$(bin): $(obj)
+	$(CC) -o $@ $(bin).c -lm -lpthread
+
+#$(bin): $(obj)
+#	$(CC) -o $@ $(obj) -lm -lpthread
+
+.PHONY: clean
+clean:
+	rm -f $(obj) $(bin)
+
+.PHONY: install
+install:
+	cp $(bin) /usr/local/bin/$(bin)
+
+.PHONY: uninstall
+uninstall:
+	rm -f /usr/local/bin/$(bin)

Added: demo/xsede.2014.0425/src/c-ray/README
===================================================================
--- demo/xsede.2014.0425/src/c-ray/README	                        (rev 0)
+++ demo/xsede.2014.0425/src/c-ray/README	2014-04-24 17:50:23 UTC (rev 7793)
@@ -0,0 +1,395 @@
+
+                     C-Ray Simple Raytracing Tests
+
+           http://www.futuretech.blinkenlights.nl/c-ray.html
+
+              By: John Tsiombikas <nuclear at siggraph.org>
+
+       Test suite compiled by: Ian Mapleson <mapesdhs at yahoo.com>
+
+                       Last Change: 10/Apr/2008
+
+
+1. Introduction
+2. The C-Ray Tests (how to compile)
+3. Running the Tests
+4. Submitting Results
+5. Background
+6. Appendix A: Invalid Tests
+
+**********************************************************************
+
+1. Introduction
+
+C-Ray is a simple raytracer written by John Tsiombikas; in his
+own words, it is:
+
+  "...an extremely small program I did one day to figure out how
+  would the simplest raytracer program look like in the least
+  ammount of code lines."
+
+The relevant discussion on Nekochan is at:
+
+  http://forums.nekochan.net/viewtopic.php?f=3&t=15719
+
+The default data set is very small, so C-ray really only tests the pure
+floating-point (fp) speed of a CPU core (or multiple CPUs/cores using the
+threaded version), ie. RAM speed is not a significant factor. John said:
+
+  This thing only measures 'floating point CPU performance' and
+  nothing more, and it's good that nothing else affects the results.
+  A real rendering program/scene would be still CPU-limited meaning
+  that by far the major part of the time spent would be CPU time in
+  the fpu, but it would have more overhead for disk I/O, shader
+  parsing, more strain for the memory bandwidth, and various other
+  things. So it's a good approximation being a renderer itself, but
+  it's definitely not representative."
+
+Nevertheless, the results are certainly interesting:
+
+  http://www.futuretech.blinkenlights.nl/c-ray.html
+
+If you wish to submit your own results, follow the instructions given
+below. Send the data to me, not to John, and I will add them to the
+relevant tables; please include all requested details.
+
+Comments and feedback welcome!
+
+Ian.
+
+mapesdhs at yahoo.com
+sgidepot at blueyonder.co.uk
+http://www.futuretech.blinkenlights.nl/sgidepot/
+
+**********************************************************************
+
+2. The C-Ray Tests
+
+Two programs are included in this archive for testing:
+
+c-ray-f:
+
+  This is for single-CPU systems, or for testing just a single core
+  of a multi-core CPU.
+
+c-ray-mt:
+
+  This is the multithreaded version for testing multi-CPU systems, and/or
+  systems with more than one CPU core. Note that on some systems c-ray-mt
+  with just one thread may be faster than c-ray-f. Use whichever version
+  gives you the best results in each case. Use the -t option to specify
+  the number of threads; without the -t option, only 1 thread is used.
+
+Compile the source files for your target platform with gcc or whatever
+compiler you have. Just enter 'make', though feel free to add any arch-
+specific optimizations for your compiler in CFLAGS first. By default,
+the Makefile is designed for use with GCC. If you are using an SGI and
+want to use MIPS Pro to compile the programs, then enter:
+
+  /bin/cp Makefile.mips Makefile
+
+and then enter 'make'.
+
+Note that the c-ray binaries as supplied were compiled for an SGI
+Octane2 R12K/R14 system (users of other SGI models should recompile
+if possible), while the example x86 binary in the x86 directory was
+compiled by John for a 3GHz P4.
+
+If you don't want to use make, then typical compile lines for each
+program on SGIs using MIPS Pro are as follows (in this case for an
+Octane system - use a different IP number for other SGI systems):
+
+  cc -O3 -mips4 -TARG:platform=ip30 -Ofast=ip30 c-ray-f.c -o c-ray-f -lm
+
+while for the threaded version the pthread library must be included:
+
+  cc -O3 -mips4 -TARG:platform=ip30 -Ofast=ip30 c-ray-mt.c -o c-ray-mt -lm -lpthread
+
+See the 'cc' man page for full details of available optimisation options.
+
+The file 'sgi.txt' has further example compile lines for SGI O2 and
+Octane machines, some with extra example optimisation options. Try
+them out, see which one works best on your system. Those using GCC
+should consult the gcc man page for full details of available options.
+
+NOTE: results for tests done with pre-run profiling/virtualisation
+compiler optimisations will NOT be accepted! (see Appendix A for details).
+
+Before running the tests, naturally you should shut down any other
+applications, processes, etc. which might interfere with the test.
+For example, on SGI systems, I shut down the mediad and sgi_apache
+daemons:
+
+  /etc/init.d/mediad stop
+  /etc/init.d/sgi_apache stop
+
+Better still, turn off timed, nsd, and rlogin remotely to run the tests.
+It should be possible to do the same thing on a Linux/BSD system.
+
+On a Windows machine, shut down all unnecessary processes, close any
+antivirus/firewall applications/processes, and it may be worthwhile
+forcing any pending idle tasks to complete before running the tests,
+ie. select Run from the Start menu and enter:
+
+  Rundll32.exe advapi32.dll,ProcessIdleTasks
+
+Assuming you are now ready to use the binary programs for the tests...
+
+**********************************************************************
+
+3. Running the Tests
+
+There are two data files used for the tests:
+
+'scene' is a simple environment, with just three reflective spheres.
+Examine scene.jpg to see the final image.
+
+'sphract' is a much more complex scenario, with dozens of spheres in
+a fractal pattern (see gen_fract.txt for details of how the scene
+description was created). Examine sphract.jpg to see the final image.
+
+There are four tests; the first is the shortest, the data from which
+are used for the main table on the results page. The tests are:
+
+
+  Test   Data File    Image Resolution     Oversampling
+
+   1.    scene        Default 800x600      NONE
+   2.    sphract      Default 800x600      NONE
+   3.    sphract      1024 x 768           8X
+   4.    scene        7500 x 3500          NONE
+
+If you are using a single-CPU system which only has one core, or wish
+to test just one CPU/core of a multi-CPU/core system, then run the
+tests with c-ray-f, or with c-ray-mt using just 1 thread. For systems
+with multiple CPUs/cores, please submit results for just a single
+CPU/core aswell as the fastest results for using all CPUs/cores (this
+allows one to see how well parallel systems scale).
+
+On UNIX systems, the programs receive the scene description data from
+the standard input and send the results (the image created) to the
+standard output. To run the first test on a UNIX system, enter:
+
+  cat scene | ./c-ray-f > foo.ppm
+
+On a Windows system, enter the following in a Command window:
+
+  c-ray-f -i scene -o foo.ppm
+
+The output will resemble the following (in this case run on a R14000
+550MHz SGI Octane2):
+
+  Rendering took: 1 seconds (1888 milliseconds)
+
+The result to submit is the number of milliseconds. Run each test
+several times if possible to observe a typical result. It's up to
+you whether you submit a typical result or the fastest overall result.
+
+If you have a multi-CPU/core system, now run the test multithreaded with
+c-ray-mt, using the -t option to specify the number of threads, eg.
+
+     UNIX:   cat scene | ./c-ray-mt -t 32 > foo.ppm
+  Windows:   c-ray-mt -t 32 -i scene -o foo.ppm
+
+For multi-CPU/core systems, the optimum number of threads varies
+greatly from one system to another, though a good estimate is 16
+times the number of cores. Try different numbers, eg. 32, 64, 128,
+or some inbetween number such as 40, 60, etc. But also try smaller
+number too, eg. just 8 threads for a quad-core system.
+
+The maximum number of threads c-ray-mt can use is the vertical
+resolution of the output image. As the number of threads increases,
+eventually the speedup obtained by the parallel processing will be
+outweighed by the overhead cost of managing the threads. Experiment
+to find what works best for each test.
+
+Thus, for the other tests, the commands to enter on a UNIX system
+would be as follows, using 32 threads just as an example here:
+
+  cat sphract | ./c-ray-f > foo.ppm
+  cat sphract | ./c-ray-mt -t 32 > foo.ppm
+  cat sphract | ./c-ray-f -s 1024x768 -r 8 > foo.ppm
+  cat sphract | ./c-ray-mt -t 32 -s 1024x768 -r 8 > foo.ppm
+  cat scene | ./c-ray-f -s 7500x3500 > foo.ppm
+  cat scene | ./c-ray-mt -t 32 -s 7500x3500 > foo.ppm
+
+while on a Windows system these would be:
+
+  c-ray-f -i sphract -o foo.ppm
+  c-ray-mt -t 32 -i sphract -o foo.ppm
+  c-ray-f -s 1024x768 -r 8 -i sphract -o foo.ppm
+  c-ray-mt -t 32 -s 1024x768 -r 8 -i sphract -o foo.ppm
+  c-ray-f -s 7500x3500 -i scene -o foo.ppm
+  c-ray-mt -t 32 -s 7500x3500 -i scene -o foo.ppm
+
+If you don't want to bother experimenting, then the script RUN.full
+will execute all the tests, with the c-ray-mt tests done using 32
+threads, but remember 32 threads might not be optimal for your system.
+
+**********************************************************************
+
+4. Submitting Results
+
+Do not send any results to John; instead, send all results to me at
+both of my email addresses (include "C-Ray" in the subject line):
+
+  mapesdhs at yahoo.com
+  sgidepot at blueyonder.co.uk
+
+Apart from the run-times reported by each test, please remember
+to state which tests are multithreaded and how many threads were
+used. Better still, just copy the on-screen text for each test.
+
+With respect to system information, state the type of system (name
+and/or model number), CPU details (name/model, type/speed/cache,
+no. of CPUs/cores), OS name/version (eg. for Linux, what name,
+kernel version/build), what compiler was used (name/version) and
+what extra options if any were employed. The online results page
+also shows the host name where available, but this is optional.
+
+Thus, for example, a system description might look like this:
+
+  SUN Fire X2100, Solaris 10, host name 'kobe'
+  Sun Studio 10 Compiler
+  AMD Opteron 175 2.2 GHz 1MB L2
+
+while for an SGI it might be:
+
+  SGI Octane2, IRIX 6.5.26m
+  MIPS Pro 7.4.3m (7.3 EOE)
+  Dual-R12000 400MHz (2MB L2)
+
+or a Windows system (in this case my own setup):
+
+  WinXP-32bit PC (SP2), Asrock AM2NF3 motherboard, 4GB DDR2/800 RAM
+  Athlon64 X2 6000+ 3.15GHz (overclocked)
+  Supplied x86 binary used (1 core only)
+
+You can also mention the system's RAM, disk, or anything else, but
+they're not essential.
+
+Note for SGI users: you can find out what eoe/dev compiler versions
+your system has installed by entering:
+
+  versions -b | grep compiler_
+
+eg. on my Octane2 this gives:
+
+  I  compiler_dev   05/14/2004  Base Compiler Development Environment, 7.3
+  I  compiler_eoe   07/13/2005  IRIX Standard Execution Environment (Base Compiler Headers and Libraries, 7.4.3m)
+
+**********************************************************************
+
+5. Background
+
+When I asked John about why he created C-Ray, he said:
+
+  This is just an extremely small program I did one day to figure
+  out how would the simplest raytracer program look like in the
+  least amount of code lines :)  It's not useful for anything apart
+  from benchmarking.
+
+  As part of my BSc dissertation project I did a really big and
+  feature-full ray tracer, which could be useful, supporting:
+  programmable shading, network rendering, monte carlo rendering
+  algorithms, etc. But it's big and buggy, slow, and incomplete
+  because I was rushing to finish it the last minute before the
+  deadline (as always) :)
+
+  So I scrapped the damn thing after that, and I'm starting from
+  scratch with a new design if I finish it and it proves to be
+  sucessful I'll let you know :)"
+
+
+I also asked John why the best c-ray-mt results seem to be obtained
+with a number of threads that is much larger than the number of
+CPUs/cores in a system, to which he replied:
+
+  Ian wrote:
+  > I also suspect more threads means the balanced load between
+  > the two CPUs is less affected by the possible differences in
+  > complexity between threads.
+
+  Bingo, each thread takes a bunch of scanlines, if the relative
+  complexity of the rendering calculations between the bunches(sic)
+  is not equal, then one thread may spend much more time calculating
+  than another thread. Of course that doesn't necessarily mean that
+  one "CPU" ends up calculating much more than the other since the
+  threads are not "bound" to any CPU, each CPU takes one of the
+  available ready-to-run threads each timeslice. Anyway having more
+  threads evens it out. I would guess about 4 times as many threads
+  as the CPUs [multiplied by the no. of cores per CPU] would be enough.
+
+
+I also asked John if there was any element of overhead processing
+to handle the results of the multiple threads. He said:
+
+  There's no such overhead. Each thread gets a pointer to the
+  appropriate location of the framebuffer, and stores every pixel
+  as it is calculated directly. Also any processing afterwards
+  (output of the PPM image) is done *after* timing stops.
+
+
+See the Nekochan thread for more discussion about C-Ray, including
+further comments by John.
+
+**********************************************************************
+
+6. Appendix A: Invalid Tests
+
+Results for tests done with pre-run profiling/virtualisation compiler
+optimisations will NOT be accepted! What does this mean? Read on...
+
+Someone emailed me to say they had been able to halve the test run
+times by using the following compilation/execution sequence:
+
+  XC="gcc -O3 -ffast-math -fomit-frame-pointer c-ray-f.c
+  -finline-limit=10000 -ftree-vectorize -fwhole-program
+  -fbranch-probabilities -ffunction-sections -o c-ray-f -lm" && $XC
+  -fprofile-generate && ./run > /dev/null 2>&1 && $XC -fprofile-use
+  && ./run && ./run && ./run && ./run
+
+with ./run containing:
+
+  #!/bin/sh
+  cat scene | ./c-ray-f > foo2.ppm
+
+The explanation was as follows:
+
+  What this does is to create an executable with profiling and
+  virtualization instructions, execute it in order to create "real life"
+  information and then recompile it using that information to better
+  optimize it without the need of "guessing" (which is what usually a
+  plain -O3 does). Doing this speeds up the execution a lot. Another
+  important thing was to increase the inline limit from 600 (or whatever
+  the default value is) to something big (like 10000) so more functions
+  will be inlined instead of called. The usage of "-fwhole-program" tells
+  to gcc that the .c file is the one and only source code file for the
+  program, so it will use all functions as static and will make all
+  "inline-able" functions, inline. This is another great speedup :-).
+
+My problem with this is that, by definition, the test has to be run
+multiple times in order to provide execution profiling data used to
+optimise the test for the final run. Thus, the real total run time spent
+to obtain the final result was not that much less or perhaps longer than
+just running the test a single time without this sort of execution
+profiling.
+
+I asked John about these optimisations; he said:
+
+  Hahaha :)
+  Yes I bet the biggest advantage was the profiling run and use of that
+  profiling information, and maybe also the whole program optimizations.
+  Both of which can't be done in real-life programs :)
+
+  The profiling information helps the compiler issue branch prediction
+  hints to the processor, and also do prefetches. The first one makes a
+  lot of difference in modern x86 CPUs with their huge execution
+  pipelines. If the branch prediction fails, you end up flushing the
+  pipeline and backtracking tens of instructions. Explicit prefetching,
+  would also make a big difference if the data set was bigger. I don't
+  think it helps here.
+
+This method of optimisation is certainly interesting, but I don't think
+it's appropriate for the C-Ray tests.
+

Added: demo/xsede.2014.0425/src/c-ray/RUN
===================================================================
--- demo/xsede.2014.0425/src/c-ray/RUN	                        (rev 0)
+++ demo/xsede.2014.0425/src/c-ray/RUN	2014-04-24 17:50:23 UTC (rev 7793)
@@ -0,0 +1,2 @@
+#!/bin/sh
+cat scene.cr | ./c-ray > scene.ppm


Property changes on: demo/xsede.2014.0425/src/c-ray/RUN
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/src/c-ray/c-ray.c
===================================================================
--- demo/xsede.2014.0425/src/c-ray/c-ray.c	                        (rev 0)
+++ demo/xsede.2014.0425/src/c-ray/c-ray.c	2014-04-24 17:50:23 UTC (rev 7793)
@@ -0,0 +1,671 @@
+/* c-ray-mt - a simple multithreaded raytracing filter.
+ * Copyright (C) 2006 John Tsiombikas <nuclear at siggraph.org>
+ *
+ * You are free to use, modify and redistribute this program under the
+ * terms of the GNU General Public License v2 or (at your option) later.
+ * see "http://www.gnu.org/licenses/gpl.txt" for details.
+ * ---------------------------------------------------------------------
+ * Usage:
+ *   compile:  just type make
+ *              (add any arch-specific optimizations for your compiler in CFLAGS first)
+ *       run:  cat scene | ./c-ray-mt [-t num-threads] >foo.ppm
+ *              (on broken systems such as windows try: c-ray-mt -i scene -o foo.ppm)
+ *     enjoy:  display foo.ppm
+ *              (with imagemagick, or use your favorite image viewer)
+ * ---------------------------------------------------------------------
+ * Scene file format:
+ *   # sphere (many)
+ *   s  x y z  rad   r g b   shininess   reflectivity
+ *   # light (many)
+ *   l  x y z
+ *   # camera (one)
+ *   c  x y z  fov   tx ty tz
+ * ---------------------------------------------------------------------
+ */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <ctype.h>
+#include <errno.h>
+#include <pthread.h>
+
+#define VER_MAJOR	1
+#define VER_MINOR	1
+#define VER_STR		"c-ray-mt v%d.%d\n"
+
+#if !defined(unix) && !defined(__unix__)
+#ifdef __MACH__
+#define unix		1
+#define __unix__	1
+#endif	/* __MACH__ */
+#endif	/* unix */
+
+/* find the appropriate way to define explicitly sized types */
+/* for C99 or GNU libc (also mach's libc) we can use stdint.h */
+#if (__STDC_VERSION__ >= 199900) || defined(__GLIBC__) || defined(__MACH__)
+#include <stdint.h>
+#elif defined(unix) || defined(__unix__)	/* some UNIX systems have them in sys/types.h */
+#include <sys/types.h>
+#elif defined(__WIN32__) || defined(WIN32)	/* the nameless one */
+typedef unsigned __int8 uint8_t;
+typedef unsigned __int32 uint32_t;
+#endif	/* sized type detection */
+
+struct vec3 {
+	double x, y, z;
+};
+
+struct ray {
+	struct vec3 orig, dir;
+};
+
+struct material {
+	struct vec3 col;	/* color */
+	double spow;		/* specular power */
+	double refl;		/* reflection intensity */
+};
+
+struct sphere {
+	struct vec3 pos;
+	double rad;
+	struct material mat;
+	struct sphere *next;
+};
+
+struct spoint {
+	struct vec3 pos, normal, vref;	/* position, normal and view reflection */
+	double dist;		/* parametric distance of intersection along the ray */
+};
+
+struct camera {
+	struct vec3 pos, targ;
+	double fov;
+};
+
+struct thread_data {
+	pthread_t tid;
+	int sl_start, sl_count;
+
+	uint32_t *pixels;
+};
+
+void render_scanline(int xsz, int ysz, int sl, uint32_t *fb, int samples);
+struct vec3 trace(struct ray ray, int depth);
+struct vec3 shade(struct sphere *obj, struct spoint *sp, int depth);
+struct vec3 reflect(struct vec3 v, struct vec3 n);
+struct vec3 cross_product(struct vec3 v1, struct vec3 v2);
+struct ray get_primary_ray(int x, int y, int sample);
+struct vec3 get_sample_pos(int x, int y, int sample);
+struct vec3 jitter(int x, int y, int s);
+int ray_sphere(const struct sphere *sph, struct ray ray, struct spoint *sp);
+void load_scene(FILE *fp);
+unsigned long get_msec(void);
+
+void *thread_func(void *tdata);
+
+#define MAX_LIGHTS		16				/* maximum number of lights */
+#define RAY_MAG			1000.0			/* trace rays of this magnitude */
+#define MAX_RAY_DEPTH	5				/* raytrace recursion limit */
+#define FOV				0.78539816		/* field of view in rads (pi/4) */
+#define HALF_FOV		(FOV * 0.5)
+#define ERR_MARGIN		1e-6			/* an arbitrary error margin to avoid surface acne */
+
+/* bit-shift ammount for packing each color into a 32bit uint */
+#ifdef LITTLE_ENDIAN
+#define RSHIFT	16
+#define BSHIFT	0
+#else	/* big endian */
+#define RSHIFT	0
+#define BSHIFT	16
+#endif	/* endianess */
+#define GSHIFT	8	/* this is the same in both byte orders */
+
+/* some helpful macros... */
+#define SQ(x)		((x) * (x))
+#define MAX(a, b)	((a) > (b) ? (a) : (b))
+#define MIN(a, b)	((a) < (b) ? (a) : (b))
+#define DOT(a, b)	((a).x * (b).x + (a).y * (b).y + (a).z * (b).z)
+#define NORMALIZE(a)  do {\
+	double len = sqrt(DOT(a, a));\
+	(a).x /= len; (a).y /= len; (a).z /= len;\
+} while(0);
+
+/* global state */
+int xres = 800;
+int yres = 600;
+int rays_per_pixel = 1;
+double aspect = 1.333333;
+struct sphere *obj_list;
+struct vec3 lights[MAX_LIGHTS];
+int lnum = 0;
+struct camera cam;
+
+int thread_num = 1;
+struct thread_data *threads;
+
+int start = 0;
+pthread_mutex_t start_mutex = PTHREAD_MUTEX_INITIALIZER;
+pthread_cond_t start_cond = PTHREAD_COND_INITIALIZER;
+
+#define NRAN	1024
+#define MASK	(NRAN - 1)
+struct vec3 urand[NRAN];
+int irand[NRAN];
+
+const char *usage = {
+	"Usage: c-ray-mt [options]\n"
+	"  Reads a scene file from stdin, writes the image to stdout, and stats to stderr.\n\n"
+	"Options:\n"
+	"  -t <num>   how many threads to use (default: 1)\n"
+	"  -s WxH     where W is the width and H the height of the image\n"
+	"  -r <rays>  shoot <rays> rays per pixel (antialiasing)\n"
+	"  -i <file>  read from <file> instead of stdin\n"
+	"  -o <file>  write to <file> instead of stdout\n"
+	"  -h         this help screen\n\n"
+};
+
+
+
+int main(int argc, char **argv) {
+	int i;
+	unsigned long rend_time, start_time;
+	uint32_t *pixels;
+	double sl, sl_per_thread;
+	FILE *infile = stdin, *outfile = stdout;
+
+	for(i=1; i<argc; i++) {
+		if(argv[i][0] == '-' && argv[i][2] == 0) {
+			char *sep;
+			switch(argv[i][1]) {
+			case 't':
+				if(!isdigit(argv[++i][0])) {
+					fprintf(stderr, "-t mus be followed by the number of worker threads to spawn\n");
+					return EXIT_FAILURE;
+				}
+				thread_num = atoi(argv[i]);
+				if(!thread_num) {
+					fprintf(stderr, "invalid number of threads specified: %d\n", thread_num);
+					return EXIT_FAILURE;
+				}
+				break;
+					
+			case 's':
+				if(!isdigit(argv[++i][0]) || !(sep = strchr(argv[i], 'x')) || !isdigit(*(sep + 1))) {
+					fputs("-s must be followed by something like \"640x480\"\n", stderr);
+					return EXIT_FAILURE;
+				}
+				xres = atoi(argv[i]);
+				yres = atoi(sep + 1);
+				aspect = (double)xres / (double)yres;
+				break;
+
+			case 'i':
+				if(!(infile = fopen(argv[++i], "rb"))) {
+					fprintf(stderr, "failed to open input file %s: %s\n", argv[i], strerror(errno));
+					return EXIT_FAILURE;
+				}
+				break;
+
+			case 'o':
+				if(!(outfile = fopen(argv[++i], "wb"))) {
+					fprintf(stderr, "failed to open output file %s: %s\n", argv[i], strerror(errno));
+					return EXIT_FAILURE;
+				}
+				break;
+
+			case 'r':
+				if(!isdigit(argv[++i][0])) {
+					fputs("-r must be followed by a number (rays per pixel)\n", stderr);
+					return EXIT_FAILURE;
+				}
+				rays_per_pixel = atoi(argv[i]);
+				break;
+
+			case 'h':
+				fputs(usage, stdout);
+				return 0;
+				
+			default:
+				fprintf(stderr, "unrecognized argument: %s\n", argv[i]);
+				fputs(usage, stderr);
+				return EXIT_FAILURE;
+			}
+		} else {
+			fprintf(stderr, "unrecognized argument: %s\n", argv[i]);
+			fputs(usage, stderr);
+			return EXIT_FAILURE;
+		}
+	}
+
+	if(!(pixels = malloc(xres * yres * sizeof *pixels))) {
+		perror("pixel buffer allocation failed");
+		return EXIT_FAILURE;
+	}
+	load_scene(infile);
+
+	/* initialize the random number tables for the jitter */
+	for(i=0; i<NRAN; i++) urand[i].x = (double)rand() / RAND_MAX - 0.5;
+	for(i=0; i<NRAN; i++) urand[i].y = (double)rand() / RAND_MAX - 0.5;
+	for(i=0; i<NRAN; i++) irand[i] = (int)(NRAN * ((double)rand() / RAND_MAX));
+
+	if(thread_num > yres) {
+		fprintf(stderr, "more threads than scanlines specified, reducing number of threads to %d\n", yres);
+		thread_num = yres;
+	}
+
+	if(!(threads = malloc(thread_num * sizeof *threads))) {
+		perror("failed to allocate thread table");
+		return EXIT_FAILURE;
+	}
+
+	sl = 0.0;
+	sl_per_thread = (double)yres / (double)thread_num;
+	for(i=0; i<thread_num; i++) {
+		threads[i].sl_start = (int)sl;
+		sl += sl_per_thread;
+		threads[i].sl_count = (int)sl - threads[i].sl_start;
+		threads[i].pixels = pixels;
+		
+		if(pthread_create(&threads[i].tid, 0, thread_func, &threads[i]) != 0) {
+			perror("failed to spawn thread");
+			return EXIT_FAILURE;
+		}
+	}
+	threads[thread_num - 1].sl_count = yres - threads[thread_num - 1].sl_start;
+
+	fprintf(stderr, VER_STR, VER_MAJOR, VER_MINOR);
+	
+	pthread_mutex_lock(&start_mutex);
+	start_time = get_msec();
+	start = 1;
+	pthread_cond_broadcast(&start_cond);
+	pthread_mutex_unlock(&start_mutex);
+
+	for(i=0; i<thread_num; i++) {
+		pthread_join(threads[i].tid, 0);
+	}
+	rend_time = get_msec() - start_time;
+	
+	/* output statistics to stderr */
+	fprintf(stderr, "Rendering took: %lu seconds (%lu milliseconds)\n", rend_time / 1000, rend_time);
+
+	/* output the image */
+	fprintf(outfile, "P6\n%d %d\n255\n", xres, yres);
+	for(i=0; i<xres * yres; i++) {
+		fputc((pixels[i] >> RSHIFT) & 0xff, outfile);
+		fputc((pixels[i] >> GSHIFT) & 0xff, outfile);
+		fputc((pixels[i] >> BSHIFT) & 0xff, outfile);
+	}
+	fflush(outfile);
+
+	if(infile != stdin) fclose(infile);
+	if(outfile != stdout) fclose(outfile);
+	return 0;
+}
+
+/* render a frame of xsz/ysz dimensions into the provided framebuffer */
+void render_scanline(int xsz, int ysz, int sl, uint32_t *fb, int samples) {
+	int i, s;
+	double rcp_samples = 1.0 / (double)samples;
+
+	for(i=0; i<xsz; i++) {
+		double r, g, b;
+		r = g = b = 0.0;
+			
+		for(s=0; s<samples; s++) {
+			struct vec3 col = trace(get_primary_ray(i, sl, s), 0);
+			r += col.x;
+			g += col.y;
+			b += col.z;
+		}
+
+		r = r * rcp_samples;
+		g = g * rcp_samples;
+		b = b * rcp_samples;
+			
+		fb[sl * xsz + i] = ((uint32_t)(MIN(r, 1.0) * 255.0) & 0xff) << RSHIFT |
+							((uint32_t)(MIN(g, 1.0) * 255.0) & 0xff) << GSHIFT |
+							((uint32_t)(MIN(b, 1.0) * 255.0) & 0xff) << BSHIFT;
+	}
+}
+
+/* trace a ray throught the scene recursively (the recursion happens through
+ * shade() to calculate reflection rays if necessary).
+ */
+struct vec3 trace(struct ray ray, int depth) {
+	struct vec3 col;
+	struct spoint sp, nearest_sp;
+	struct sphere *nearest_obj = 0;
+	struct sphere *iter = obj_list->next;
+
+	/* if we reached the recursion limit, bail out */
+	if(depth >= MAX_RAY_DEPTH) {
+		col.x = col.y = col.z = 0.0;
+		return col;
+	}
+	
+	/* find the nearest intersection ... */
+	while(iter) {
+		if(ray_sphere(iter, ray, &sp)) {
+			if(!nearest_obj || sp.dist < nearest_sp.dist) {
+				nearest_obj = iter;
+				nearest_sp = sp;
+			}
+		}
+		iter = iter->next;
+	}
+
+	/* and perform shading calculations as needed by calling shade() */
+	if(nearest_obj) {
+		col = shade(nearest_obj, &nearest_sp, depth);
+	} else {
+		col.x = col.y = col.z = 0.0;
+	}
+
+	return col;
+}
+
+/* Calculates direct illumination with the phong reflectance model.
+ * Also handles reflections by calling trace again, if necessary.
+ */
+struct vec3 shade(struct sphere *obj, struct spoint *sp, int depth) {
+	int i;
+	struct vec3 col = {0, 0, 0};
+
+	/* for all lights ... */
+	for(i=0; i<lnum; i++) {
+		double ispec, idiff;
+		struct vec3 ldir;
+		struct ray shadow_ray;
+		struct sphere *iter = obj_list->next;
+		int in_shadow = 0;
+
+		ldir.x = lights[i].x - sp->pos.x;
+		ldir.y = lights[i].y - sp->pos.y;
+		ldir.z = lights[i].z - sp->pos.z;
+
+		shadow_ray.orig = sp->pos;
+		shadow_ray.dir = ldir;
+
+		/* shoot shadow rays to determine if we have a line of sight with the light */
+		while(iter) {
+			if(ray_sphere(iter, shadow_ray, 0)) {
+				in_shadow = 1;
+				break;
+			}
+			iter = iter->next;
+		}
+
+		/* and if we're not in shadow, calculate direct illumination with the phong model. */
+		if(!in_shadow) {
+			NORMALIZE(ldir);
+
+			idiff = MAX(DOT(sp->normal, ldir), 0.0);
+			ispec = obj->mat.spow > 0.0 ? pow(MAX(DOT(sp->vref, ldir), 0.0), obj->mat.spow) : 0.0;
+
+			col.x += idiff * obj->mat.col.x + ispec;
+			col.y += idiff * obj->mat.col.y + ispec;
+			col.z += idiff * obj->mat.col.z + ispec;
+		}
+	}
+
+	/* Also, if the object is reflective, spawn a reflection ray, and call trace()
+	 * to calculate the light arriving from the mirror direction.
+	 */
+	if(obj->mat.refl > 0.0) {
+		struct ray ray;
+		struct vec3 rcol;
+
+		ray.orig = sp->pos;
+		ray.dir = sp->vref;
+		ray.dir.x *= RAY_MAG;
+		ray.dir.y *= RAY_MAG;
+		ray.dir.z *= RAY_MAG;
+
+		rcol = trace(ray, depth + 1);
+		col.x += rcol.x * obj->mat.refl;
+		col.y += rcol.y * obj->mat.refl;
+		col.z += rcol.z * obj->mat.refl;
+	}
+
+	return col;
+}
+
+/* calculate reflection vector */
+struct vec3 reflect(struct vec3 v, struct vec3 n) {
+	struct vec3 res;
+	double dot = v.x * n.x + v.y * n.y + v.z * n.z;
+	res.x = -(2.0 * dot * n.x - v.x);
+	res.y = -(2.0 * dot * n.y - v.y);
+	res.z = -(2.0 * dot * n.z - v.z);
+	return res;
+}
+
+struct vec3 cross_product(struct vec3 v1, struct vec3 v2) {
+	struct vec3 res;
+	res.x = v1.y * v2.z - v1.z * v2.y;
+	res.y = v1.z * v2.x - v1.x * v2.z;
+	res.z = v1.x * v2.y - v1.y * v2.x;
+	return res;
+}
+
+/* determine the primary ray corresponding to the specified pixel (x, y) */
+struct ray get_primary_ray(int x, int y, int sample) {
+	struct ray ray;
+	float m[3][3];
+	struct vec3 i, j = {0, 1, 0}, k, dir, orig, foo;
+
+	k.x = cam.targ.x - cam.pos.x;
+	k.y = cam.targ.y - cam.pos.y;
+	k.z = cam.targ.z - cam.pos.z;
+	NORMALIZE(k);
+
+	i = cross_product(j, k);
+	j = cross_product(k, i);
+	m[0][0] = i.x; m[0][1] = j.x; m[0][2] = k.x;
+	m[1][0] = i.y; m[1][1] = j.y; m[1][2] = k.y;
+	m[2][0] = i.z; m[2][1] = j.z; m[2][2] = k.z;
+	
+	ray.orig.x = ray.orig.y = ray.orig.z = 0.0;
+	ray.dir = get_sample_pos(x, y, sample);
+	ray.dir.z = 1.0 / HALF_FOV;
+	ray.dir.x *= RAY_MAG;
+	ray.dir.y *= RAY_MAG;
+	ray.dir.z *= RAY_MAG;
+	
+	dir.x = ray.dir.x + ray.orig.x;
+	dir.y = ray.dir.y + ray.orig.y;
+	dir.z = ray.dir.z + ray.orig.z;
+	foo.x = dir.x * m[0][0] + dir.y * m[0][1] + dir.z * m[0][2];
+	foo.y = dir.x * m[1][0] + dir.y * m[1][1] + dir.z * m[1][2];
+	foo.z = dir.x * m[2][0] + dir.y * m[2][1] + dir.z * m[2][2];
+
+	orig.x = ray.orig.x * m[0][0] + ray.orig.y * m[0][1] + ray.orig.z * m[0][2] + cam.pos.x;
+	orig.y = ray.orig.x * m[1][0] + ray.orig.y * m[1][1] + ray.orig.z * m[1][2] + cam.pos.y;
+	orig.z = ray.orig.x * m[2][0] + ray.orig.y * m[2][1] + ray.orig.z * m[2][2] + cam.pos.z;
+
+	ray.orig = orig;
+	ray.dir.x = foo.x + orig.x;
+	ray.dir.y = foo.y + orig.y;
+	ray.dir.z = foo.z + orig.z;
+	
+	return ray;
+}
+
+
+struct vec3 get_sample_pos(int x, int y, int sample) {
+	struct vec3 pt;
+	static double sf = 0.0;
+
+	if(sf == 0.0) {
+		sf = 1.5 / (double)xres;
+	}
+
+	pt.x = ((double)x / (double)xres) - 0.5;
+	pt.y = -(((double)y / (double)yres) - 0.65) / aspect;
+
+	if(sample) {
+		struct vec3 jt = jitter(x, y, sample);
+		pt.x += jt.x * sf;
+		pt.y += jt.y * sf / aspect;
+	}
+	return pt;
+}
+
+/* jitter function taken from Graphics Gems I. */
+struct vec3 jitter(int x, int y, int s) {
+	struct vec3 pt;
+	pt.x = urand[(x + (y << 2) + irand[(x + s) & MASK]) & MASK].x;
+	pt.y = urand[(y + (x << 2) + irand[(y + s) & MASK]) & MASK].y;
+	return pt;
+}
+
+/* Calculate ray-sphere intersection, and return {1, 0} to signify hit or no hit.
+ * Also the surface point parameters like position, normal, etc are returned through
+ * the sp pointer if it is not NULL.
+ */
+int ray_sphere(const struct sphere *sph, struct ray ray, struct spoint *sp) {
+	double a, b, c, d, sqrt_d, t1, t2;
+	
+	a = SQ(ray.dir.x) + SQ(ray.dir.y) + SQ(ray.dir.z);
+	b = 2.0 * ray.dir.x * (ray.orig.x - sph->pos.x) +
+				2.0 * ray.dir.y * (ray.orig.y - sph->pos.y) +
+				2.0 * ray.dir.z * (ray.orig.z - sph->pos.z);
+	c = SQ(sph->pos.x) + SQ(sph->pos.y) + SQ(sph->pos.z) +
+				SQ(ray.orig.x) + SQ(ray.orig.y) + SQ(ray.orig.z) +
+				2.0 * (-sph->pos.x * ray.orig.x - sph->pos.y * ray.orig.y - sph->pos.z * ray.orig.z) - SQ(sph->rad);
+	
+	if((d = SQ(b) - 4.0 * a * c) < 0.0) return 0;
+
+	sqrt_d = sqrt(d);
+	t1 = (-b + sqrt_d) / (2.0 * a);
+	t2 = (-b - sqrt_d) / (2.0 * a);
+
+	if((t1 < ERR_MARGIN && t2 < ERR_MARGIN) || (t1 > 1.0 && t2 > 1.0)) return 0;
+
+	if(sp) {
+		if(t1 < ERR_MARGIN) t1 = t2;
+		if(t2 < ERR_MARGIN) t2 = t1;
+		sp->dist = t1 < t2 ? t1 : t2;
+		
+		sp->pos.x = ray.orig.x + ray.dir.x * sp->dist;
+		sp->pos.y = ray.orig.y + ray.dir.y * sp->dist;
+		sp->pos.z = ray.orig.z + ray.dir.z * sp->dist;
+		
+		sp->normal.x = (sp->pos.x - sph->pos.x) / sph->rad;
+		sp->normal.y = (sp->pos.y - sph->pos.y) / sph->rad;
+		sp->normal.z = (sp->pos.z - sph->pos.z) / sph->rad;
+
+		sp->vref = reflect(ray.dir, sp->normal);
+		NORMALIZE(sp->vref);
+	}
+	return 1;
+}
+
+/* Load the scene from an extremely simple scene description file */
+#define DELIM	" \t\n"
+void load_scene(FILE *fp) {
+	char line[256], *ptr, type;
+
+	obj_list = malloc(sizeof(struct sphere));
+	obj_list->next = 0;
+	
+	while((ptr = fgets(line, 256, fp))) {
+		int i;
+		struct vec3 pos, col;
+		double rad, spow, refl;
+		
+		while(*ptr == ' ' || *ptr == '\t') ptr++;
+		if(*ptr == '#' || *ptr == '\n') continue;
+
+		if(!(ptr = strtok(line, DELIM))) continue;
+		type = *ptr;
+		
+		for(i=0; i<3; i++) {
+			if(!(ptr = strtok(0, DELIM))) break;
+			*((double*)&pos.x + i) = atof(ptr);
+		}
+
+		if(type == 'l') {
+			lights[lnum++] = pos;
+			continue;
+		}
+
+		if(!(ptr = strtok(0, DELIM))) continue;
+		rad = atof(ptr);
+
+		for(i=0; i<3; i++) {
+			if(!(ptr = strtok(0, DELIM))) break;
+			*((double*)&col.x + i) = atof(ptr);
+		}
+
+		if(type == 'c') {
+			cam.pos = pos;
+			cam.targ = col;
+			cam.fov = rad;
+			continue;
+		}
+
+		if(!(ptr = strtok(0, DELIM))) continue;
+		spow = atof(ptr);
+
+		if(!(ptr = strtok(0, DELIM))) continue;
+		refl = atof(ptr);
+
+		if(type == 's') {
+			struct sphere *sph = malloc(sizeof *sph);
+			sph->next = obj_list->next;
+			obj_list->next = sph;
+
+			sph->pos = pos;
+			sph->rad = rad;
+			sph->mat.col = col;
+			sph->mat.spow = spow;
+			sph->mat.refl = refl;
+		} else {
+			fprintf(stderr, "unknown type: %c\n", type);
+		}
+	}
+}
+
+
+/* provide a millisecond-resolution timer for each system */
+#if defined(unix) || defined(__unix__)
+#include <time.h>
+#include <sys/time.h>
+unsigned long get_msec(void) {
+	static struct timeval timeval, first_timeval;
+	
+	gettimeofday(&timeval, 0);
+	if(first_timeval.tv_sec == 0) {
+		first_timeval = timeval;
+		return 0;
+	}
+	return (timeval.tv_sec - first_timeval.tv_sec) * 1000 + (timeval.tv_usec - first_timeval.tv_usec) / 1000;
+}
+#elif defined(__WIN32__) || defined(WIN32)
+#include <windows.h>
+unsigned long get_msec(void) {
+	return GetTickCount();
+}
+#else
+#error "I don't know how to measure time on your platform"
+#endif
+
+void *thread_func(void *tdata) {
+	int i;
+	struct thread_data *td = (struct thread_data*)tdata;
+
+	pthread_mutex_lock(&start_mutex);
+	while(!start) {
+		pthread_cond_wait(&start_cond, &start_mutex);
+	}
+	pthread_mutex_unlock(&start_mutex);
+
+	for(i=0; i<td->sl_count; i++) {
+		render_scanline(xres, yres, i + td->sl_start, td->pixels, rays_per_pixel);
+	}
+
+	return 0;
+}

Added: demo/xsede.2014.0425/src/c-ray/scene.cr
===================================================================
--- demo/xsede.2014.0425/src/c-ray/scene.cr	                        (rev 0)
+++ demo/xsede.2014.0425/src/c-ray/scene.cr	2014-04-24 17:50:23 UTC (rev 7793)
@@ -0,0 +1,18 @@
+# spheres
+#	position		radius	color			shininess	reflectivity
+s	-1.5 -0.3 -1	0.7		1.0 0.2 0.05		50.0	0.3
+s	1.5 -0.4 0		0.6		0.1 0.85 1.0		50.0	0.4
+
+# walls
+s	0 -1000 2		999		0.1 0.2 0.6			80.0	0.8
+
+# bouncing ball
+s	0 0 2			1		1.0 0.5 0.1			60.0	0.7
+
+# lights...
+l	-50 100 -50
+l	40 40 150
+
+# camera (there can be only one!)
+#	position	FOV		target
+c	0 6 -17		45		0 -1 0

Added: demo/xsede.2014.0425/src/c-ray/sphfract.cr
===================================================================
--- demo/xsede.2014.0425/src/c-ray/sphfract.cr	                        (rev 0)
+++ demo/xsede.2014.0425/src/c-ray/sphfract.cr	2014-04-24 17:50:23 UTC (rev 7793)
@@ -0,0 +1,192 @@
+s	0 0 0 	1.0	0.25 0.25 0.25  50.0	0.65
+s	1.4 0 0 	0.4	0.25 0.25 0.25  50.0	0.65
+s	1.96 0 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	2.184 0 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.96 0.224 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.96 -0.224 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.96 0 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.96 0 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0.56 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	1.624 0.56 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.176 0.56 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0.784 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0.56 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0.56 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 -0.56 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	1.624 -0.56 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.176 -0.56 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 -0.784 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 -0.56 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 -0.56 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0 0.56 	0.16	0.25 0.25 0.25  50.0	0.65
+s	1.624 0 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.176 0 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0.224 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 -0.224 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0 0.784 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0 -0.56 	0.16	0.25 0.25 0.25  50.0	0.65
+s	1.624 0 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.176 0 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0.224 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 -0.224 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	1.4 0 -0.784 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0 0 	0.4	0.25 0.25 0.25  50.0	0.65
+s	-1.96 0 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-2.184 0 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.96 0.224 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.96 -0.224 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.96 0 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.96 0 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0.56 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-1.176 0.56 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.624 0.56 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0.784 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0.56 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0.56 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 -0.56 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-1.176 -0.56 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.624 -0.56 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 -0.784 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 -0.56 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 -0.56 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0 0.56 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-1.176 0 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.624 0 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0.224 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 -0.224 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0 0.784 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0 -0.56 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-1.176 0 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.624 0 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0.224 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 -0.224 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-1.4 0 -0.784 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.4 0 	0.4	0.25 0.25 0.25  50.0	0.65
+s	0.56 1.4 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.784 1.4 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 1.624 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 1.176 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 1.4 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 1.4 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 1.4 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-0.784 1.4 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 1.624 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 1.176 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 1.4 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 1.4 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.96 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 1.96 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 1.96 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 2.184 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.96 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.96 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.4 0.56 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 1.4 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 1.4 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.624 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.176 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.4 0.784 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.4 -0.56 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 1.4 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 1.4 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.624 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.176 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 1.4 -0.784 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.4 0 	0.4	0.25 0.25 0.25  50.0	0.65
+s	0.56 -1.4 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.784 -1.4 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 -1.176 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 -1.624 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 -1.4 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 -1.4 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 -1.4 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-0.784 -1.4 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 -1.176 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 -1.624 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 -1.4 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 -1.4 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.96 0 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 -1.96 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 -1.96 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -2.184 0 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.96 0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.96 -0.224 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.4 0.56 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 -1.4 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 -1.4 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.176 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.624 0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.4 0.784 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.4 -0.56 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 -1.4 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 -1.4 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.176 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.624 -0.56 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -1.4 -0.784 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0 1.4 	0.4	0.25 0.25 0.25  50.0	0.65
+s	0.56 0 1.4 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.784 0 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 0.224 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 -0.224 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 0 1.624 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 0 1.176 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 0 1.4 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-0.784 0 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 0.224 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 -0.224 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 0 1.624 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 0 1.176 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.56 1.4 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 0.56 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 0.56 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.784 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.56 1.624 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.56 1.176 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.56 1.4 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 -0.56 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 -0.56 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.784 1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.56 1.624 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.56 1.176 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0 1.96 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 0 1.96 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 0 1.96 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.224 1.96 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.224 1.96 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0 2.184 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0 -1.4 	0.4	0.25 0.25 0.25  50.0	0.65
+s	0.56 0 -1.4 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.784 0 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 0.224 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 -0.224 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 0 -1.176 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0.56 0 -1.624 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 0 -1.4 	0.16	0.25 0.25 0.25  50.0	0.65
+s	-0.784 0 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 0.224 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 -0.224 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 0 -1.176 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.56 0 -1.624 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.56 -1.4 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 0.56 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 0.56 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.784 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.56 -1.176 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.56 -1.624 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.56 -1.4 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 -0.56 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 -0.56 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.784 -1.4 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.56 -1.176 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.56 -1.624 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0 -1.96 	0.16	0.25 0.25 0.25  50.0	0.65
+s	0.224 0 -1.96 	0.064	0.25 0.25 0.25  50.0	0.65
+s	-0.224 0 -1.96 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0.224 -1.96 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 -0.224 -1.96 	0.064	0.25 0.25 0.25  50.0	0.65
+s	0 0 -2.184 	0.064	0.25 0.25 0.25  50.0	0.65
+s  0 -10002.25 0  10000  0.2 0.35 0.5  80.0  0.4
+s  0  10100.00 0  10000  0.5 0.2 0.1  40.0  0.0
+l	-50 68 -50
+l	40 40 150
+c	-7 6 -12 45  0 -0.65 0



From wilde at ci.uchicago.edu  Thu Apr 24 12:59:24 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 12:59:24 -0500 (CDT)
Subject: [Swift-commit] r7794 - in demo/xsede.2014.0425/src: . md
Message-ID: <20140424175924.0AE1F9D102@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 12:59:23 -0500 (Thu, 24 Apr 2014)
New Revision: 7794

Added:
   demo/xsede.2014.0425/src/md/
   demo/xsede.2014.0425/src/md/Makefile
   demo/xsede.2014.0425/src/md/clean.sh
   demo/xsede.2014.0425/src/md/md.c
   demo/xsede.2014.0425/src/md/test.sh
Log:
Add md app

Added: demo/xsede.2014.0425/src/md/Makefile
===================================================================
--- demo/xsede.2014.0425/src/md/Makefile	                        (rev 0)
+++ demo/xsede.2014.0425/src/md/Makefile	2014-04-24 17:59:23 UTC (rev 7794)
@@ -0,0 +1,12 @@
+bin = md
+
+CC = gcc
+#CFLAGS = -O3 -ffast-math
+
+$(bin): $(obj)
+	$(CC) -o $@ $(bin).c
+
+.PHONY: clean
+clean:
+	rm -f $(obj) $(bin)
+

Added: demo/xsede.2014.0425/src/md/clean.sh
===================================================================
--- demo/xsede.2014.0425/src/md/clean.sh	                        (rev 0)
+++ demo/xsede.2014.0425/src/md/clean.sh	2014-04-24 17:59:23 UTC (rev 7794)
@@ -0,0 +1,2 @@
+#! /bin/sh
+rm -f md??.trj md.dat


Property changes on: demo/xsede.2014.0425/src/md/clean.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/src/md/md.c
===================================================================
--- demo/xsede.2014.0425/src/md/md.c	                        (rev 0)
+++ demo/xsede.2014.0425/src/md/md.c	2014-04-24 17:59:23 UTC (rev 7794)
@@ -0,0 +1,765 @@
+# include <stdlib.h>
+# include <stdio.h>
+# include <time.h>
+# include <math.h>
+
+int main ( int argc, char *argv[] );
+void compute ( int np, int nd, double pos[], double vel[], 
+  double mass, double f[], double *pot, double *kin );
+double cpu_time ( void );
+double dist ( int nd, double r1[], double r2[], double dr[] );
+void initialize ( int np, int nd, double box[], int *seed, double pos[], 
+  double vel[], double acc[] );
+double r8_uniform_01 ( int *seed );
+void timestamp ( void );
+void update ( int np, int nd, double pos[], double vel[], double f[], 
+  double acc[], double mass, double dt );
+void snap ( int np, int nd, double pos[], double vel[], double f[], 
+  double acc[], double mass, double dt );
+
+double scale_factor = 2.5, scale_offset = -2.0;
+char default_printinfo[100] = "0.05 1.0 0.2 0.05 50.0 0.1";
+char *printinfo = default_printinfo;
+
+/******************************************************************************/
+
+int main ( int argc, char *argv[] )
+
+/******************************************************************************/
+/*
+  Purpose:
+
+    MAIN is the main program for MD.
+
+  Discussion:
+
+    MD implements a simple molecular dynamics simulation.
+
+    The velocity Verlet time integration scheme is used. 
+
+    The particles interact with a central pair potential.
+
+  Usage:
+
+    md nd np step_num print_step_num dt mass printinfo scale_factor scale_offset
+    where
+    * nd is the spatial dimension (2 or 3);
+    * np is the number of particles (500, for instance);
+    * step_num is the number of time steps (500, for instance);
+    * print_step_num is the number of snapshot prints (10 for instance);
+    * dt is size of timestep;
+    * mass is particle mass;
+    * printinfo is a string to append to each particle coord
+    * scale_offset and scale_factor are used to scale particle positions for logging/rendering (FIXME)
+
+  Licensing:
+
+    This code is distributed under the GNU LGPL license. 
+
+  Modified:
+
+    05 November 2010
+
+  Author:
+
+    Original FORTRAN90 version by Bill Magro.
+    C version by John Burkardt.
+
+  Parameters:
+
+    None
+*/
+{
+  double *acc;
+  double *box;
+  double ctime;
+  double ctime1;
+  double ctime2;
+  double dt = 0.0001;
+  double e0;
+  double *force;
+  int i;
+  int id;
+  double kinetic;
+  double mass = 1.0 * .0001;
+  int nd;
+  int np;
+  double *pos;
+  double potential;
+  int seed = 123456789;
+  int step;
+  int step_num;
+  int step_print;
+  int step_print_index;
+  int step_print_num=10;
+  double *vel;
+
+  timestamp ( );
+  printf ( "\n" );
+  printf ( "MD\n" );
+  printf ( "  C version\n" );
+  printf ( "  A molecular dynamics program.\n" );
+/*
+  Get the spatial dimension.
+*/
+  if ( 1 < argc )
+  {
+    nd = atoi ( argv[1] );
+  }
+  else
+  {
+    printf ( "\n" );
+    printf ( "  Enter ND, the spatial dimension (2 or 3).\n" );
+    scanf ( "%d", &nd );
+  }
+//
+//  Get the number of points.
+//
+  if ( 2 < argc )
+  {
+    np = atoi ( argv[2] );
+  }
+  else
+  {
+    printf ( "\n" );
+    printf ( "  Enter NP, the number of points (500, for instance).\n" );
+    scanf ( "%d", &np );
+  }
+//
+//  Get the number of time steps.
+//
+  if ( 3 < argc )
+  {
+    step_num = atoi ( argv[3] );
+  }
+  else
+  {
+    printf ( "\n" );
+    printf ( "  Enter ND, the number of time steps (500 or 1000, for instance).\n" );
+    scanf ( "%d", &step_num );
+  }
+  /*
+	Get any additional args (command-line only)
+	md nd np step_num [ step__print_num dt mass printinfo scale_factor scale_offset ]
+  */
+  if ( 4 < argc )
+  {
+    step_print_num = atoi ( argv[4] );
+  }
+  if ( 5 < argc )
+  {
+    dt = atof ( argv[5] );
+  }
+  if ( 6 < argc )
+  {
+    mass = atof ( argv[6] );
+  }
+  if ( 7 < argc )
+  {
+    printinfo = ( argv[7] );
+  }
+  if ( 8 < argc )
+  {
+    scale_factor = atof ( argv[8] );
+  }
+  if ( 9 < argc )
+  {
+    scale_offset = atof ( argv[9] );
+  }
+
+/*
+  Report.
+*/
+  printf ( "\n" );
+  printf ( "  ND, the spatial dimension, is %d\n", nd );
+  printf ( "  NP, the number of particles in the simulation, is %d\n", np );
+  printf ( "  STEP_NUM, the number of time steps, is %d\n", step_num );
+  printf ( "  STEP_PRINT_NUM, the number of snapshots to print, is %d\n", step_print_num );
+  printf ( "  DT, the size of each time step, is %f\n", dt );
+  printf ( "  MASS, the particle mass, is %f\n", mass );
+  printf ( "  SCALE_FACTOR, the particle position scaling factor, is %f\n", scale_factor );
+  printf ( "  SCALE_OFFSET, the particle position scaling offset, is %f\n", scale_offset );
+/*
+  Allocate memory.
+*/
+  acc = ( double * ) malloc ( nd * np * sizeof ( double ) );
+  box = ( double * ) malloc ( nd * sizeof ( double ) );
+  force = ( double * ) malloc ( nd * np * sizeof ( double ) );
+  pos = ( double * ) malloc ( nd * np * sizeof ( double ) );
+  vel = ( double * ) malloc ( nd * np * sizeof ( double ) );
+/*
+  Set the dimensions of the box.
+*/
+  for ( i = 0; i < nd; i++ )
+  {
+    box[i] = 10.0;
+  }
+
+  printf ( "\n" );
+  printf ( "  Initializing positions, velocities, and accelerations.\n" );
+/*
+  Set initial positions, velocities, and accelerations.
+*/
+  initialize ( np, nd, box, &seed, pos, vel, acc );
+/*
+  Compute the forces and energies.
+*/
+  printf ( "\n" );
+  printf ( "  Computing initial forces and energies.\n" );
+
+  compute ( np, nd, pos, vel, mass, force, &potential, &kinetic );
+
+  e0 = potential + kinetic;
+/*
+  This is the main time stepping loop:
+    Compute forces and energies,
+    Update positions, velocities, accelerations.
+*/
+  printf ( "\n" );
+  printf ( "  At each step, we report the potential and kinetic energies.\n" );
+  printf ( "  The sum of these energies should be a constant.\n" );
+  printf ( "  As an accuracy check, we also print the relative error\n" );
+  printf ( "  in the total energy.\n" );
+  printf ( "\n" );
+  printf ( "      Step      Potential       Kinetic        (P+K-E0)/E0\n" );
+  printf ( "                Energy P        Energy K       Relative Energy Error\n" );
+  printf ( "\n" );
+
+  FILE *ofile = fopen("md.dat","w");
+  fprintf (ofile, "      Step      Potential       Kinetic        RelativeErr\n" );
+
+  step_print = 0;
+  step_print_index = 0;
+  
+  step = 0;
+  printf ( "  %8d  %14f  %14f  %14e\n",
+    step, potential, kinetic, ( potential + kinetic - e0 ) / e0 );
+  fprintf ( ofile, "  %8d  %14f  %14f  %14e\n",
+    step, potential, kinetic, ( potential + kinetic - e0 ) / e0 );
+  step_print_index = step_print_index + 1;
+  step_print = ( step_print_index * step_num ) / step_print_num;
+
+  ctime1 = cpu_time ( );
+
+  for ( step = 1; step <= step_num; step++ )
+  {
+    compute ( np, nd, pos, vel, mass, force, &potential, &kinetic );
+
+    if ( step == step_print )
+    {
+      printf ( "  %8d  %14f  %14f  %14e\n", step, potential, kinetic,
+       ( potential + kinetic - e0 ) / e0 );
+      fprintf ( ofile, "  %8d  %14f  %14f  %14e\n", step, potential, kinetic,
+       ( potential + kinetic - e0 ) / e0 );
+      step_print_index = step_print_index + 1;
+      step_print = ( step_print_index * step_num ) / step_print_num;
+    snap ( np, nd, pos, vel, force, acc, mass, dt );
+    }
+    update ( np, nd, pos, vel, force, acc, mass, dt );
+  }
+  ctime2 = cpu_time ( );
+  ctime = ctime2 - ctime1;
+
+  printf ( "\n" );
+  printf ( "  Elapsed cpu time for main computation:\n" );
+  printf ( "  %f seconds.\n", ctime );
+
+  free ( acc );
+  free ( box );
+  free ( force );
+  free ( pos );
+  free ( vel );
+/*
+  Terminate.
+*/
+  printf ( "\n" );
+  printf ( "MD\n" );
+  printf ( "  Normal end of execution.\n" );
+
+  printf ( "\n" );
+  timestamp ( );
+
+  fclose(ofile);
+  return 0;
+}
+/******************************************************************************/
+
+void compute ( int np, int nd, double pos[], double vel[], 
+  double mass, double f[], double *pot, double *kin )
+
+/******************************************************************************/
+/*
+  Purpose:
+
+    COMPUTE computes the forces and energies.
+
+  Discussion:
+
+    The computation of forces and energies is fully parallel.
+
+    The potential function V(X) is a harmonic well which smoothly
+    saturates to a maximum value at PI/2:
+
+      v(x) = ( sin ( min ( x, PI2 ) ) )^2
+
+    The derivative of the potential is:
+
+      dv(x) = 2.0 * sin ( min ( x, PI2 ) ) * cos ( min ( x, PI2 ) )
+            = sin ( 2.0 * min ( x, PI2 ) )
+
+  Licensing:
+
+    This code is distributed under the GNU LGPL license. 
+
+  Modified:
+
+    21 November 2007
+
+  Author:
+
+    Original FORTRAN90 version by Bill Magro.
+    C version by John Burkardt.
+
+  Parameters:
+
+    Input, int NP, the number of particles.
+
+    Input, int ND, the number of spatial dimensions.
+
+    Input, double POS[ND*NP], the position of each particle.
+
+    Input, double VEL[ND*NP], the velocity of each particle.
+
+    Input, double MASS, the mass of each particle.
+
+    Output, double F[ND*NP], the forces.
+
+    Output, double *POT, the total potential energy.
+
+    Output, double *KIN, the total kinetic energy.
+*/
+{
+  double d;
+  double d2;
+  int i;
+  int j;
+  int k;
+  double ke;
+  double pe;
+  double PI2 = 3.141592653589793 / 2.0;
+  double rij[3];
+
+  pe = 0.0;
+  ke = 0.0;
+
+  for ( k = 0; k < np; k++ )
+  {
+/*
+  Compute the potential energy and forces.
+*/
+    for ( i = 0; i < nd; i++ )
+    {
+      f[i+k*nd] = 0.0;
+    }
+
+    for ( j = 0; j < np; j++ )
+    {
+      if ( k != j )
+      {
+        d = dist ( nd, pos+k*nd, pos+j*nd, rij );
+/*
+  Attribute half of the potential energy to particle J.
+*/
+        if ( d < PI2 )
+        {
+          d2 = d;
+        }
+        else
+        {
+          d2 = PI2;
+        }
+
+        pe = pe + 0.5 * pow ( sin ( d2 ), 2 );
+
+        for ( i = 0; i < nd; i++ )
+        {
+          f[i+k*nd] = f[i+k*nd] - rij[i] * sin ( 2.0 * d2 ) / d;
+        }
+      }
+    }
+/*
+  Compute the kinetic energy.
+*/
+    for ( i = 0; i < nd; i++ )
+    {
+      ke = ke + vel[i+k*nd] * vel[i+k*nd];
+    }
+  }
+
+  ke = ke * 0.5 * mass;
+  
+  *pot = pe;
+  *kin = ke;
+
+  return;
+}
+/*******************************************************************************/
+
+double cpu_time ( void )
+
+/*******************************************************************************/
+/*
+  Purpose:
+ 
+    CPU_TIME reports the total CPU time for a program.
+
+  Licensing:
+
+    This code is distributed under the GNU LGPL license. 
+
+  Modified:
+
+    27 September 2005
+
+  Author:
+
+    John Burkardt
+
+  Parameters:
+
+    Output, double CPU_TIME, the current total elapsed CPU time in second.
+*/
+{
+  double value;
+
+  value = ( double ) clock ( ) / ( double ) CLOCKS_PER_SEC;
+
+  return value;
+}
+/******************************************************************************/
+
+double dist ( int nd, double r1[], double r2[], double dr[] )
+
+/******************************************************************************/
+/*
+  Purpose:
+
+    DIST computes the displacement (and its norm) between two particles.
+
+  Licensing:
+
+    This code is distributed under the GNU LGPL license. 
+
+  Modified:
+
+    21 November 2007
+
+  Author:
+
+    Original FORTRAN90 version by Bill Magro.
+    C version by John Burkardt.
+
+  Parameters:
+
+    Input, int ND, the number of spatial dimensions.
+
+    Input, double R1[ND], R2[ND], the positions of the particles.
+
+    Output, double DR[ND], the displacement vector.
+
+    Output, double D, the Euclidean norm of the displacement.
+*/
+{
+  double d;
+  int i;
+
+  d = 0.0;
+  for ( i = 0; i < nd; i++ )
+  {
+    dr[i] = r1[i] - r2[i];
+    d = d + dr[i] * dr[i];
+  }
+  d = sqrt ( d );
+
+  return d;
+}
+/******************************************************************************/
+
+void initialize ( int np, int nd, double box[], int *seed, double pos[], 
+  double vel[], double acc[] )
+
+/******************************************************************************/
+/*
+  Purpose:
+
+    INITIALIZE initializes the positions, velocities, and accelerations.
+
+  Licensing:
+
+    This code is distributed under the GNU LGPL license. 
+
+  Modified:
+
+    20 July 2008
+
+  Author:
+
+    Original FORTRAN90 version by Bill Magro.
+    C version by John Burkardt.
+
+  Parameters:
+
+    Input, int NP, the number of particles.
+
+    Input, int ND, the number of spatial dimensions.
+
+    Input, double BOX[ND], specifies the maximum position
+    of particles in each dimension.
+
+    Input, int *SEED, a seed for the random number generator.
+
+    Output, double POS[ND*NP], the position of each particle.
+
+    Output, double VEL[ND*NP], the velocity of each particle.
+
+    Output, double ACC[ND*NP], the acceleration of each particle.
+*/
+{
+  int i;
+  int j;
+/*
+  Give the particles random positions within the box.
+*/
+  for ( j = 0; j < np; j++ )
+  {
+    for ( i = 0; i < nd; i++ )
+    {
+      pos[i+j*nd] = box[i] * r8_uniform_01 ( seed );
+      vel[i+j*nd] = 0.0;
+      acc[i+j*nd] = 0.0;
+    }
+  }
+  return;
+}
+/******************************************************************************/
+
+double r8_uniform_01 ( int *seed )
+
+/******************************************************************************/
+/*
+  Purpose:
+
+    R8_UNIFORM_01 is a unit pseudorandom R8.
+
+  Discussion:
+
+    This routine implements the recursion
+
+      seed = 16807 * seed mod ( 2^31 - 1 )
+      unif = seed / ( 2^31 - 1 )
+
+    The integer arithmetic never requires more than 32 bits,
+    including a sign bit.
+
+  Licensing:
+
+    This code is distributed under the GNU LGPL license. 
+
+  Modified:
+
+    11 August 2004
+
+  Author:
+
+    John Burkardt
+
+  Reference:
+
+    Paul Bratley, Bennett Fox, Linus Schrage,
+    A Guide to Simulation,
+    Springer Verlag, pages 201-202, 1983.
+
+    Bennett Fox,
+    Algorithm 647:
+    Implementation and Relative Efficiency of Quasirandom
+    Sequence Generators,
+    ACM Transactions on Mathematical Software,
+    Volume 12, Number 4, pages 362-376, 1986.
+
+  Parameters:
+
+    Input/output, int *SEED, a seed for the random number generator.
+
+    Output, double R8_UNIFORM_01, a new pseudorandom variate, strictly between
+    0 and 1.
+*/
+{
+  int k;
+  double r;
+
+  k = *seed / 127773;
+
+  *seed = 16807 * ( *seed - k * 127773 ) - k * 2836;
+
+  if ( *seed < 0 )
+  {
+    *seed = *seed + 2147483647;
+  }
+
+  r = ( double ) ( *seed ) * 4.656612875E-10;
+
+  return r;
+}
+/******************************************************************************/
+
+void timestamp ( void )
+
+/******************************************************************************/
+/*
+  Purpose:
+
+    TIMESTAMP prints the current YMDHMS date as a time stamp.
+
+  Example:
+
+    31 May 2001 09:45:54 AM
+
+  Licensing:
+
+    This code is distributed under the GNU LGPL license. 
+
+  Modified:
+
+    24 September 2003
+
+  Author:
+
+    John Burkardt
+
+  Parameters:
+
+    None
+*/
+{
+# define TIME_SIZE 40
+
+  static char time_buffer[TIME_SIZE];
+  const struct tm *tm;
+  size_t len;
+  time_t now;
+
+  now = time ( NULL );
+  tm = localtime ( &now );
+
+  len = strftime ( time_buffer, TIME_SIZE, "%d %B %Y %I:%M:%S %p", tm );
+
+  printf ( "%s\n", time_buffer );
+
+  return;
+# undef TIME_SIZE
+}
+/******************************************************************************/
+
+void update ( int np, int nd, double pos[], double vel[], double f[], 
+  double acc[], double mass, double dt )
+
+/******************************************************************************/
+/*
+  Purpose:
+
+    UPDATE updates positions, velocities and accelerations.
+
+  Discussion:
+
+    The time integration is fully parallel.
+
+    A velocity Verlet algorithm is used for the updating.
+
+    x(t+dt) = x(t) + v(t) * dt + 0.5 * a(t) * dt * dt
+    v(t+dt) = v(t) + 0.5 * ( a(t) + a(t+dt) ) * dt
+    a(t+dt) = f(t) / m
+
+  Licensing:
+
+    This code is distributed under the GNU LGPL license. 
+
+  Modified:
+
+    21 November 2007
+
+  Author:
+
+    Original FORTRAN90 version by Bill Magro.
+    C version by John Burkardt.
+
+  Parameters:
+
+    Input, int NP, the number of particles.
+
+    Input, int ND, the number of spatial dimensions.
+
+    Input/output, double POS[ND*NP], the position of each particle.
+
+    Input/output, double VEL[ND*NP], the velocity of each particle.
+
+    Input, double F[ND*NP], the force on each particle.
+
+    Input/output, double ACC[ND*NP], the acceleration of each particle.
+
+    Input, double MASS, the mass of each particle.
+
+    Input, double DT, the time step.
+*/
+{
+  int i;
+  int j;
+  double rmass;
+
+  rmass = 1.0 / mass;
+
+  for ( j = 0; j < np; j++ )
+  {
+    for ( i = 0; i < nd; i++ )
+    {
+      pos[i+j*nd] = pos[i+j*nd] + vel[i+j*nd] * dt + 0.5 * acc[i+j*nd] * dt * dt;
+      vel[i+j*nd] = vel[i+j*nd] + 0.5 * dt * ( f[i+j*nd] * rmass + acc[i+j*nd] );
+      acc[i+j*nd] = f[i+j*nd] * rmass;
+    }
+	/*
+s	-0.5 -0.3 -1	0.05	1.0 0.2 0.05		50.0	0.1
+s	0.5 -0.4 0		0.05	0.1 0.85 1.0		50.0	0.1
+	*/
+    char *s = "0.05 1.0 0.2 0.05 50.0 0.1";
+    /* printf("s %f %f %f %s\n", pos[0+j*nd]/20.0, pos[1+j*nd]/20.0, pos[2+j*nd]/20.0, s); */
+  }
+
+  return;
+}
+
+
+double scale (double x)
+{
+  return( (x / scale_factor) - scale_offset);
+  /* return( (x / 2.5) - 2.0); */
+}
+
+static int snapid = 0;
+
+void snap ( int np, int nd, double pos[], double vel[], double f[], 
+  double acc[], double mass, double dt )
+{
+  int j;
+  
+  char snapfile[100];
+  sprintf(snapfile, "md%02d.trj", snapid);
+
+  FILE *sf = fopen(snapfile, "w");
+
+  for ( j = 0; j < np; j++ )
+  {
+    fprintf(sf, "s %f %f %f %s\n", scale(pos[0+j*nd]), scale(pos[1+j*nd]), scale(pos[2+j*nd]), printinfo);
+  }
+
+  fclose(sf);
+  snapid++;
+}

Added: demo/xsede.2014.0425/src/md/test.sh
===================================================================
--- demo/xsede.2014.0425/src/md/test.sh	                        (rev 0)
+++ demo/xsede.2014.0425/src/md/test.sh	2014-04-24 17:59:23 UTC (rev 7794)
@@ -0,0 +1,3 @@
+#! /bin/sh
+
+./md 3 50 10000


Property changes on: demo/xsede.2014.0425/src/md/test.sh
___________________________________________________________________
Added: svn:executable
   + *



From wilde at ci.uchicago.edu  Thu Apr 24 13:04:53 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 13:04:53 -0500 (CDT)
Subject: [Swift-commit] r7795 - in demo/xsede.2014.0425/src: . shdemo
Message-ID: <20140424180453.2721F9D102@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 13:04:53 -0500 (Thu, 24 Apr 2014)
New Revision: 7795

Added:
   demo/xsede.2014.0425/src/shdemo/
   demo/xsede.2014.0425/src/shdemo/d01.sh
   demo/xsede.2014.0425/src/shdemo/d02.sh
   demo/xsede.2014.0425/src/shdemo/d03.sh
Log:
shell demo tools to test md and cr

Added: demo/xsede.2014.0425/src/shdemo/d01.sh
===================================================================
--- demo/xsede.2014.0425/src/shdemo/d01.sh	                        (rev 0)
+++ demo/xsede.2014.0425/src/shdemo/d01.sh	2014-04-24 18:04:53 UTC (rev 7795)
@@ -0,0 +1,9 @@
+#! /bin/sh
+
+cr=../c-ray-1.1
+md=../mdlite
+
+for ((i=0;i<50;i++)); do
+  $md/md 3 50 30000 | tee md$i.trj | grep ^s:$i | cat - $cr/ts2 | $cr/cr >fr$i.ppm ; convert fr$i.ppm fr$i.png 
+done
+convert fr*png t.gif; open -a Safari t.gif


Property changes on: demo/xsede.2014.0425/src/shdemo/d01.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/src/shdemo/d02.sh
===================================================================
--- demo/xsede.2014.0425/src/shdemo/d02.sh	                        (rev 0)
+++ demo/xsede.2014.0425/src/shdemo/d02.sh	2014-04-24 18:04:53 UTC (rev 7795)
@@ -0,0 +1,13 @@
+#! /bin/sh
+
+cr=../c-ray-1.1
+md=../mdlite
+
+$md/md 3 50 30000
+
+for t in md??.trj; do
+  cat $t $cr/ts2 | $cr/cr >t.ppm
+  convert t.ppm $t.png 
+done
+
+convert -delay 20 md??.trj.png t.gif; open -a Safari t.gif


Property changes on: demo/xsede.2014.0425/src/shdemo/d02.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/src/shdemo/d03.sh
===================================================================
--- demo/xsede.2014.0425/src/shdemo/d03.sh	                        (rev 0)
+++ demo/xsede.2014.0425/src/shdemo/d03.sh	2014-04-24 18:04:53 UTC (rev 7795)
@@ -0,0 +1,14 @@
+#! /bin/sh
+
+cr=../c-ray-1.1
+md=../mdlite
+
+# $md/md 3 50 30000
+  $md/md 3 50 30000 50  .0001  .005  "0.03 1.0 0.2 0.05 50.0 0.1" 2.5 2.0
+
+for t in md??.trj; do
+  cat $t $cr/ts2 | $cr/cr >t.ppm
+  convert t.ppm $t.png 
+done
+
+convert -delay 20 md??.trj.png t.gif; open -a Safari t.gif


Property changes on: demo/xsede.2014.0425/src/shdemo/d03.sh
___________________________________________________________________
Added: svn:executable
   + *



From wilde at ci.uchicago.edu  Thu Apr 24 13:08:56 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 13:08:56 -0500 (CDT)
Subject: [Swift-commit] r7796 - demo/xsede.2014.0425/src/md
Message-ID: <20140424180856.115709D102@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 13:08:55 -0500 (Thu, 24 Apr 2014)
New Revision: 7796

Modified:
   demo/xsede.2014.0425/src/md/Makefile
   demo/xsede.2014.0425/src/md/test.sh
Log:
edits for midway: needs -lm; runs slower so shorten test.

Modified: demo/xsede.2014.0425/src/md/Makefile
===================================================================
--- demo/xsede.2014.0425/src/md/Makefile	2014-04-24 18:04:53 UTC (rev 7795)
+++ demo/xsede.2014.0425/src/md/Makefile	2014-04-24 18:08:55 UTC (rev 7796)
@@ -4,7 +4,7 @@
 #CFLAGS = -O3 -ffast-math
 
 $(bin): $(obj)
-	$(CC) -o $@ $(bin).c
+	$(CC) -o $@ $(bin).c -lm
 
 .PHONY: clean
 clean:

Modified: demo/xsede.2014.0425/src/md/test.sh
===================================================================
--- demo/xsede.2014.0425/src/md/test.sh	2014-04-24 18:04:53 UTC (rev 7795)
+++ demo/xsede.2014.0425/src/md/test.sh	2014-04-24 18:08:55 UTC (rev 7796)
@@ -1,3 +1,3 @@
 #! /bin/sh
 
-./md 3 50 10000
+./md 3 50 1000



From wilde at ci.uchicago.edu  Thu Apr 24 13:34:47 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 13:34:47 -0500 (CDT)
Subject: [Swift-commit] r7797 - in demo/xsede.2014.0425: . app bin doc
	doc/figs doc/images doc/images/icons part01 part02 part03
	part04 part05 part06 src/c-ray
Message-ID: <20140424183448.017E39D102@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 13:34:47 -0500 (Thu, 24 Apr 2014)
New Revision: 7797

Added:
   demo/xsede.2014.0425/README
   demo/xsede.2014.0425/app/
   demo/xsede.2014.0425/app/simulate
   demo/xsede.2014.0425/app/stats
   demo/xsede.2014.0425/bin/
   demo/xsede.2014.0425/bin/cleanup
   demo/xsede.2014.0425/bin/hosts
   demo/xsede.2014.0425/bin/package_tutorial.sh
   demo/xsede.2014.0425/bin/plot.sh
   demo/xsede.2014.0425/doc/
   demo/xsede.2014.0425/doc/README
   demo/xsede.2014.0425/doc/TODO
   demo/xsede.2014.0425/doc/activeplot.png
   demo/xsede.2014.0425/doc/asciidoc.css
   demo/xsede.2014.0425/doc/build_docs.sh
   demo/xsede.2014.0425/doc/cumulativeplot.png
   demo/xsede.2014.0425/doc/figs/
   demo/xsede.2014.0425/doc/figs/modis.dia
   demo/xsede.2014.0425/doc/figs/modis.png
   demo/xsede.2014.0425/doc/images/
   demo/xsede.2014.0425/doc/images/icons/
   demo/xsede.2014.0425/doc/images/icons/caution.png
   demo/xsede.2014.0425/doc/images/icons/example.png
   demo/xsede.2014.0425/doc/images/icons/home.png
   demo/xsede.2014.0425/doc/images/icons/important.png
   demo/xsede.2014.0425/doc/images/icons/next.png
   demo/xsede.2014.0425/doc/images/icons/note.png
   demo/xsede.2014.0425/doc/images/icons/prev.png
   demo/xsede.2014.0425/doc/images/icons/tip.png
   demo/xsede.2014.0425/doc/images/icons/up.png
   demo/xsede.2014.0425/doc/images/icons/warning.png
   demo/xsede.2014.0425/doc/part01.png
   demo/xsede.2014.0425/doc/part02.png
   demo/xsede.2014.0425/doc/part03.png
   demo/xsede.2014.0425/doc/part04.png
   demo/xsede.2014.0425/doc/part05.png
   demo/xsede.2014.0425/doc/part06.png
   demo/xsede.2014.0425/doc/push.sh
   demo/xsede.2014.0425/part01/
   demo/xsede.2014.0425/part01/p1.swift
   demo/xsede.2014.0425/part01/swift.properties
   demo/xsede.2014.0425/part02/
   demo/xsede.2014.0425/part02/p2.swift
   demo/xsede.2014.0425/part02/swift.properties
   demo/xsede.2014.0425/part03/
   demo/xsede.2014.0425/part03/p3.swift
   demo/xsede.2014.0425/part03/swift.properties
   demo/xsede.2014.0425/part04/
   demo/xsede.2014.0425/part04/p4.swift
   demo/xsede.2014.0425/part04/swift.properties
   demo/xsede.2014.0425/part05/
   demo/xsede.2014.0425/part05/p5.swift
   demo/xsede.2014.0425/part05/swift.properties
   demo/xsede.2014.0425/part06/
   demo/xsede.2014.0425/part06/p6.swift
   demo/xsede.2014.0425/part06/swift.properties
   demo/xsede.2014.0425/setup.sh
   demo/xsede.2014.0425/src/c-ray/test.sh
   demo/xsede.2014.0425/tutorial.pptx
Removed:
   demo/xsede.2014.0425/src/c-ray/RUN
Log:
Copy midway tutorial as base demo code.

Added: demo/xsede.2014.0425/README
===================================================================
--- demo/xsede.2014.0425/README	                        (rev 0)
+++ demo/xsede.2014.0425/README	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1 @@
+link doc/README
\ No newline at end of file


Property changes on: demo/xsede.2014.0425/README
___________________________________________________________________
Added: svn:special
   + *

Added: demo/xsede.2014.0425/app/simulate
===================================================================
--- demo/xsede.2014.0425/app/simulate	                        (rev 0)
+++ demo/xsede.2014.0425/app/simulate	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,135 @@
+#! /bin/bash
+
+printparams()
+{
+  printf "\nSimulation parameters:\n\n"
+  echo bias=$bias
+  echo biasfile=$biasfile
+  echo initseed=$initseed
+  echo log=$log
+  echo paramfile=$paramfile
+  echo range=$range
+  echo scale=$scale
+  echo seedfile=$seedfile
+  echo timesteps=$timesteps
+  echo output width=$width
+}
+
+log() {
+  printf "\nCalled as: $0: $cmdargs\n\n"
+  printf "Start time: "; /bin/date
+  printf "Running as user: "; /usr/bin/id
+  printf "Running on node: "; /bin/hostname
+  printf "Node IP address: "; /bin/hostname -I
+  printparams
+  printf "\nEnvironment:\n\n"
+  printenv | sort
+}
+
+addsims() {
+  while read f1 ; do
+    read -u 3 f2 
+    if [ _$f1 = _ ]; then f1=$lastf1; fi
+    if [ _$f2 = _ ]; then f2=$lastf2; fi
+    printf "%${width}d\n" $(($f1+$f2)) 
+    lastf1=$f1
+    lastf2=$f2
+  done <$1 3<$2
+}
+
+# set defaults
+
+bias=0
+biasfile=none
+initseed=none
+log=yes
+paramfile=none
+range=100
+scale=1
+seedfile=none
+timesteps=0
+nvalues=1
+width=8
+cmdargs="$*"
+
+usage()
+{
+  echo $( basename $0 ): usage:
+  cat <<END
+    -b|--bias       offset bias: add this integer to all results
+    -B|--biasfile   file of integer biases to add to results
+    -l|--log        generate a log in stderr if not null
+    -n|--nvalues    print this many values per simulation            
+    -r|--range      range (limit) of generated results
+    -s|--seed       use this integer [0..32767] as a seed
+    -S|--seedfile   use this file (containing integer seeds [0..32767]) one per line
+    -t|--timesteps  number of simulated "timesteps" in seconds (determines runtime)
+    -x|--scale      scale the results by this integer
+    -h|-?|?|--help  print this help
+END
+}
+
+# FIXME: NOT YET IMPLEMENTED:
+#    -p|--paramfile  take these parameters (in form param=value) from this file 
+#    -p|--paramfile) paramfile=$2 ;;
+
+while [ $# -gt 0 ]; do
+  case $1 in
+    -b|--bias)      bias=$2      ;;
+    -B|--biasfile)  biasfile=$2  ;;
+    -l|--log)       log=$2       ;;
+    -n|--nvalues)   nvalues=$2   ;;       
+    -s|--seed)      initseed=$2  ;;
+    -S|--seedfile)  seedfile=$2  ;;
+    -t|--timesteps) timesteps=$2 ;;
+    -r|--range)     range=$2     ;;   
+    -w|--width)     width=$2     ;;
+    -x|--scale)     scale=$2     ;;
+    -h|-?|--help|*) usage; exit  ;;
+  esac
+  shift 2
+done
+    
+# process initial seed
+
+if [ $initseed != none ]; then
+  RANDOM=$initseed
+fi
+
+# process file of seeds
+
+if [ $seedfile != none ]; then
+  seed=0
+  while read $seedfile s; do
+    seed=$(($seed+$s))
+  done <$seedfile
+  RANDOM=$seed
+fi
+
+# run for some number of "timesteps"
+
+sleep $timesteps
+
+# emit N (nvalues) "simulation results" scaled and biased by argument values
+
+simout=$(mktemp simout.XXXX)
+for ((i=0;i<nvalues;i++)); do
+  # value=$(( (($RANDOM)*(2**16))+$RANDOM ))
+  value=$(( (($RANDOM)*(2**48)) + (($RANDOM)*(2**32)) + (($RANDOM)*(2**16)) + $RANDOM ))
+  printf "%${width}d\n" $(( ($value%range)*scale+bias))
+done  >$simout
+
+# process file of biases
+
+if [ $biasfile != none ]; then
+  addsims $simout $biasfile
+else
+  cat $simout
+fi
+rm $simout
+
+# log environmental data
+
+if [ $log != off ]; then
+  log 1>&2
+fi


Property changes on: demo/xsede.2014.0425/app/simulate
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/app/stats
===================================================================
--- demo/xsede.2014.0425/app/stats	                        (rev 0)
+++ demo/xsede.2014.0425/app/stats	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,19 @@
+#! /bin/sh
+
+log() {
+  printf "\nCalled as: $0: $cmdargs\n\n"
+  printf "Start time: "; /bin/date
+  printf "Running as user: "; /usr/bin/id
+  printf "Running on node: "; /bin/hostname
+  printf "Node IP address: "; /bin/hostname -I
+  printf "\nEnvironment:\n\n"
+  printenv | sort
+}
+
+awk '
+
+{ sum += $1}
+
+END { printf("%d\n",sum/NR) }
+' $*
+log 1>&2


Property changes on: demo/xsede.2014.0425/app/stats
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/bin/cleanup
===================================================================
--- demo/xsede.2014.0425/bin/cleanup	                        (rev 0)
+++ demo/xsede.2014.0425/bin/cleanup	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,2 @@
+rm -rf *.rlog *.kml *.swiftx *.out output outdir logs hi.* _concurrent .swift/tmp run0*
+


Property changes on: demo/xsede.2014.0425/bin/cleanup
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/bin/hosts
===================================================================
--- demo/xsede.2014.0425/bin/hosts	                        (rev 0)
+++ demo/xsede.2014.0425/bin/hosts	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1 @@
+for h in $(cat hosts.txt); do ssh $h hostname -f; done


Property changes on: demo/xsede.2014.0425/bin/hosts
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/bin/package_tutorial.sh
===================================================================
--- demo/xsede.2014.0425/bin/package_tutorial.sh	                        (rev 0)
+++ demo/xsede.2014.0425/bin/package_tutorial.sh	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,15 @@
+#! /bin/sh
+ 
+BIN=$(cd $(dirname $0); pwd)
+BASE=$(cd $BIN/..; pwd)
+LOC=$(cd $BASE/..; pwd)
+
+echo executing $0 from BIN=$BIN
+echo making tutorial package of BASE=$BASE
+echo placing tutorial package in LOC=$LOC
+
+(
+  cd $LOC
+  tar zcf swift-cray-tutorial.tgz  --exclude-vcs swift-cray-tutorial
+  scp swift-cray-tutorial.tgz p01532 at raven.cray.com:
+)


Property changes on: demo/xsede.2014.0425/bin/package_tutorial.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/bin/plot.sh
===================================================================
--- demo/xsede.2014.0425/bin/plot.sh	                        (rev 0)
+++ demo/xsede.2014.0425/bin/plot.sh	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+#usage: ./plotswiftlogs.ketan <swift-logfile.log>
+
+SWIFTLOGFILE=$1
+
+#TMPDIR=`mktemp -d plotlog.XXX`
+
+grep -i ProgressTicker $SWIFTLOGFILE > swiftoutfile.out
+
+SWIFTOUTFILE=swiftoutfile.out
+
+#extract start time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | head -n 1 | cut -f1-2 -d ' '`
+START_TIME=`date +%s -d "$TMPDATE"`
+
+#extract end time
+TMPDATE=`grep -i progress $SWIFTOUTFILE 2>/dev/null | tail -n 1 | cut -f1-2 -d ' '`
+END_TIME=`date +%s -d "$TMPDATE"`
+
+#duration
+DIFFTIME=$((END_TIME - START_TIME))
+
+#extract active runs in a file
+(grep -o -i "Active:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' >active.txt)
+
+#extract successful completions in a file
+(grep -o -i "Successfully:[0-9]*" $SWIFTOUTFILE 2>/dev/null | awk -F: '{print $2}' > cumulative.txt)
+
+#prepare tics
+activelines=`wc -l active.txt | awk '{print $1}'`
+cumulines=`wc -l cumulative.txt | awk '{print $1}'`
+
+if [ $activelines -ne 0 ]
+then
+  activelinespertic=`echo "scale=5 ; $DIFFTIME / $activelines" | bc`
+fi
+
+seq 0 $activelinespertic $DIFFTIME > activetics.txt
+
+if [ $cumulines -ne 0 ]
+then
+    cumulinespertic=`echo "scale=5 ; $DIFFTIME / $cumulines" | bc`
+fi
+
+seq 0 $cumulinespertic $DIFFTIME > cumultics.txt
+
+#final plot data
+paste activetics.txt active.txt > plot_active.txt
+paste cumultics.txt cumulative.txt > plot_cumulative.txt
+
+cat << EOF1 > plotit.gp
+set terminal png enhanced
+set nokey
+set output "cumulativeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of completed apps"
+set title "Cumulative Apps Completed"
+plot "plot_cumulative.txt" using 1:2 with lines
+set output "activeplot.png"
+set xlabel "Time in sec"
+set ylabel "number of active apps"
+set title "Active Apps"
+plot "plot_active.txt" using 1:2 with line
+EOF1
+
+gnuplot plotit.gp 2>/dev/null
+rm swiftoutfile.out plotit.gp active.txt cumulative.txt activetics.txt plot_active.txt plot_cumulative.txt cumultics.txt


Property changes on: demo/xsede.2014.0425/bin/plot.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/doc/README
===================================================================
--- demo/xsede.2014.0425/doc/README	                        (rev 0)
+++ demo/xsede.2014.0425/doc/README	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,717 @@
+Swift Tutorial for RCC Midway Cluster
+=====================================
+
+//// 
+
+This is the asciidoc input file.
+Its content is viewable as a plain-text README file.
+
+////
+
+This tutorial is viewable at:
+http://swiftlang.org/tutorials/midway/tutorial.html
+
+////
+
+Tutorial Outline:
+
+Introductory example, running apps locally on login node:
+
+  p1 - Run an application under Swift
+  p2 - Parallel loops with foreach
+  p3 - Merging/reducing the results of a parallel foreach loop
+
+Compute-node exercises, running apps via qsub and aprun:
+
+  p4 - Running apps on RCC Midway cluster
+  p5 - Running on multiple pools of compute nodes
+  p6 - Running a more complex workflow pattern
+
+////
+
+Introduction: Why Parallel Scripting?
+------------------------------------
+
+Swift is a simple scripting language for executing many instances of
+ordinary application programs on distributed parallel resources.
+Swift scripts run many copies of ordinary programs concurrently, using
+statements like this:
+-----
+foreach protein in proteinList {
+  runBLAST(protein);
+}
+-----
+Swift acts like a structured "shell" language. It runs programs
+concurrently as soon as their inputs are available, reducing the need
+for complex parallel programming.  Swift expresses your workflow in a
+portable fashion: The same script runs on multicore computers,
+clusters, clouds, grids, and supercomputers.
+
+In this tutorial, you'll be able to first try a few Swift examples
+(parts 1-3) on a Midway login host, to get a sense of the
+language. Then in parts 4-6 you'll run similar workflows on Midway
+compute nodes, and see how more complex workflows can be expressed
+with Swift scripts.
+
+Swift tutorial setup
+--------------------
+If you are using a temporary/guest account to access Midway, follow
+the instructions at http://docs.rcc.uchicago.edu/tutorials/intro-to-rcc-workshop.html
+for more information on using a yubikey to log in.
+
+Once you are logged into the Midway login host, run the following commands 
+to set up your environment.
+
+-----
+$ cd $HOME
+$ wget http://swiftlang.org/tutorials/midway/swift-midway-tutorial.tar.gz
+$ tar xvfz swift-midway-tutorial.tar.gz
+$ cd swift-midway-tutorial
+$ source setup.sh 
+-----
+
+Verify your environment
+~~~~~~~~~~~~~~~~~~~~~~~
+
+To verify that Swift loaded, do:
+
+-----
+$ swift -version  # verify that you have Swift 0.95 RC1
+-----
+
+NOTE: If you re-login or open new ssh sessions, you must re-run `source setup.sh` in each ssh shell/window.
+
+To check out the tutorial scripts from SVN
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you later want to get the most recent version of this tutorial from
+the Swift Subversion repository, do:
+
+-----
+$ svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftTutorials/swift-midway-tutorial
+-----
+
+This will create a directory called "swift-midway-tutorial" which contains all of the
+files used in this tutorial.
+
+
+Simple "science applications" for the workflow tutorial
+-------------------------------------------------------
+
+This tutorial is based on two intentionally trivial example programs,
+`simulation` and `stats`, (implemented as bash shell scripts)
+that serve as easy-to-understand proxies for real science
+applications. These "programs" behave as follows.
+
+simulate.sh
+~~~~~~~~~~~
+
+The simulation script serves as a trivial proxy for any more
+complex scientific simulation application. It generates and prints a
+set of one or more random integers in the range [0-2^62) as controlled
+by its command line arguments, which are:
+
+-----
+$ simulate -help
+simulate: usage:
+    -b|--bias       offset bias: add this integer to all results
+    -B|--biasfile   file of integer biases to add to results
+    -l|--log        generate a log in stderr if not null
+    -n|--nvalues    print this many values per simulation            
+    -r|--range      range (limit) of generated results
+    -s|--seed       use this integer [0..32767] as a seed
+    -S|--seedfile   use this file (containing integer seeds [0..32767]) one per line
+    -t|--timesteps  number of simulated "timesteps" in seconds (determines runtime)
+    -x|--scale      scale the results by this integer
+    -h|-?|?|--help  print this help
+-----
+
+All of these arguments are optional, with default values indicated above as `[n]`.
+
+////
+.simulation arguments
+[width="80%",cols="^2,10",options="header"]
+
+|=======================
+|Argument|Short|Description
+|1    |runtime: sets run time of simulation in seconds
+|2    |range: limits generated values to the range [0,range-1]
+|3    |biasfile: add the integer contained in this file to each value generated
+|4    |scale: multiplies each generated value by this integer
+|5    |count: number of values to generate in the simulation
+|=======================
+////
+
+With no arguments, simulate prints 1 number in the range of
+1-100. Otherwise it generates n numbers of the form (R*scale)+bias
+where R is a random integer. By default it logs information about its
+execution environment to stderr.  Here's some examples of its usage:
+
+-----
+$ simulate 2>log
+      51
+$ head -5 log
+
+Called as: /home/davidkelly999/swift-midway-tutorial/app/simulate: 
+Start time: Mon Dec  2 13:47:41 CST 2013
+Running as user: uid=88848(davidkelly999) gid=88848(davidkelly999) groups=88848(davidkelly999),10008(rcc),10030(pi-gavoth),10031(sp-swift),10036(swift),10058(pi-joshuaelliott),10084(pi-wilde),10118(cron-account),10124(cmts),10138(cmtsworkshop)
+
+$ simulate -n 4 -r 1000000 2>log
+  239454
+  386702
+   13849
+  873526
+
+$ simulate -n 3 -r 1000000 -x 100 2>log
+ 6643700
+62182300
+ 5230600
+
+$ simulate -n 2 -r 1000 -x 1000 2>log
+  565000
+  636000
+
+$ time simulate -n 2 -r 1000 -x 1000 -t 3 2>log
+  336000
+  320000
+real    0m3.012s
+user    0m0.005s
+sys     0m0.006s
+-----
+
+stats
+~~~~~
+
+The stats script serves as a trivial model of an "analysis"
+program. It reads N files each containing M integers and simply prints
+the average of all those numbers to stdout. Similarly to simulate
+it logs environmental information to the stderr.
+
+-----
+$ ls f*
+f1  f2  f3  f4
+
+$ cat f*
+25
+60
+40
+75
+
+$ stats f* 2>log
+50
+-----
+
+
+Basic of the Swift language with local execution
+------------------------------------------------
+
+A Summary of Swift in a nutshell
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Swift scripts are text files ending in `.swift` The `swift` command
+runs on any host, and executes these scripts. `swift` is a Java
+application, which you can install almost anywhere.  On Linux, just
+unpack the distribution `tar` file and add its `bin/` directory to
+your `PATH`.
+
+* Swift scripts run ordinary applications, just like shell scripts
+do. Swift makes it easy to run these applications on parallel and
+remote computers (from laptops to supercomputers). If you can `ssh` to
+the system, Swift can likely run applications there.
+
+* The details of where to run applications and how to get files back
+and forth are described in configuration files separate from your
+program. Swift speaks ssh, PBS, Condor, SLURM, LSF, SGE, Cobalt, and
+Globus to run applications, and scp, http, ftp, and GridFTP to move
+data.
+
+* The Swift language has 5 main data types: `boolean`, `int`,
+`string`, `float`, and `file`. Collections of these are dynamic,
+sparse arrays of arbitrary dimension and structures of scalars and/or
+arrays defined by the `type` declaration.
+
+* Swift file variables are "mapped" to external files. Swift sends
+files to and from remote systems for you automatically.
+
+* Swift variables are "single assignment": once you set them you can't
+change them (in a given block of code).  This makes Swift a natural,
+"parallel data flow" language. This programming model keeps your
+workflow scripts simple and easy to write and understand.
+
+* Swift lets you define functions to "wrap" application programs, and
+to cleanly structure more complex scripts. Swift `app` functions take
+files and parameters as inputs and return files as outputs.
+
+* A compact set of built-in functions for string and file
+manipulation, type conversions, high level IO, etc. is provided.
+Swift's equivalent of `printf()` is `tracef()`, with limited and
+slightly different format codes.
+
+* Swift's `foreach {}` statement is the main parallel workhorse of the
+language, and executes all iterations of the loop concurrently. The
+actual number of parallel tasks executed is based on available
+resources and settable "throttles".
+
+* In fact, Swift conceptually executes *all* the statements,
+expressions and function calls in your program in parallel, based on
+data flow. These are similarly throttled based on available resources
+and settings.
+
+* Swift also has `if` and `switch` statements for conditional
+execution. These are seldom needed in simple workflows but they enable
+very dynamic workflow patterns to be specified.
+
+We'll see many of these points in action in the examples below. Lets
+get started!
+
+Part 1: Run a single application under Swift
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The first swift script, p1.swift, runs simulate to generate a
+single random number. It writes the number to a file.
+
+image::part01.png["p1 workflow",align="center"]
+
+.p1.swift
+-----
+sys::[cat ../part01/p1.swift]
+-----
+
+
+To run this script, run the following command:
+-----
+$ cd part01
+$ swift p1.swift
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1413-oa6fdib2
+Progress:  time: Tue, 27 Aug 2013 14:13:33 -0500
+Final status: Tue, 27 Aug 2013 14:13:33 -0500  Finished successfully:1
+$ cat sim.out 
+      84
+$ swift p1.swift
+$ cat sim.out 
+      36
+-----
+
+To cleanup the directory and remove all outputs (including the log
+files and directories that Swift generates), run the cleanup script
+which is located in the tutorial PATH:
+
+-----
+$ cleanup
+------
+
+NOTE: You'll also find a Swift configuration file in each `partNN`
+directory of this tutorial. This file specifies the environment-specific
+details of how to run. These files will be explained in more
+detail in parts 4-6, and can be ignored for now.
+
+Part 2: Running an ensemble of many apps in parallel with a "foreach" loop
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The `p2.swift` script introduces the `foreach` parallel iteration
+construct to run many concurrent simulations.
+
+image::part02.png[align="center"]
+
+.p2.swift
+-----
+sys::[cat ../part02/p2.swift]
+-----
+
+The script also shows an
+example of naming the output files of an ensemble run. In this case, the output files will be named
+`output/sim_N.out`.
+
+To run the script and view the output:
+-----
+$ cd ../part02
+$ swift p2.swift
+$ ls output
+sim_0.out  sim_1.out  sim_2.out  sim_3.out  sim_4.out  sim_5.out  sim_6.out  sim_7.out  sim_8.out  sim_9.out
+$ more output/*
+::::::::::::::
+output/sim_0.out
+::::::::::::::
+      44
+::::::::::::::
+output/sim_1.out
+::::::::::::::
+      55
+...
+::::::::::::::
+output/sim_9.out
+::::::::::::::
+      82
+-----
+
+Part 3: Analyzing results of a parallel ensemble
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+After all the parallel simulations in an ensemble run have completed,
+its typically necessary to gather and analyze their results with some
+kind of post-processing analysis program or script.  p3.swift
+introduces such a postprocessing step. In this case, the files created
+by all of the parallel runs of `simulation` will be averaged by by
+the trivial "analysis application" `stats`:
+
+image::part03.png[align="center"]
+
+.p3.swift
+----
+sys::[cat ../part03/p3.swift]
+----
+
+To run:
+----
+$ cd part03
+$ swift p3.swift
+----
+
+Note that in `p3.swift` we expose more of the capabilities of the
+`simulate` application to the `simulation()` app function:
+
+-----
+app (file o) simulation (int sim_steps, int sim_range, int sim_values)
+{
+  simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@filename(o);
+}
+-----
+
+`p3.swift` also shows how to fetch application-specific values from
+the `swift` command line in a Swift script using `arg()` which
+accepts a keyword-style argument and its default value:
+
+-----
+int nsim   = toInt(arg("nsim","10"));
+int steps  = toInt(arg("steps","1"));
+int range  = toInt(arg("range","100"));
+int values = toInt(arg("values","5"));
+-----
+
+Now we can specify that more runs should be performed and that each should run 
+for more timesteps, and produce more that one value each, within a specified range, 
+using command line arguments placed after the Swift script name in the form `-parameterName=value`:
+
+-----
+$ swift p3.swift -nsim=3 -steps=10 -values=4 -range=1000000
+
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1439-s3vvo809
+Progress:  time: Tue, 27 Aug 2013 14:39:42 -0500
+Progress:  time: Tue, 27 Aug 2013 14:39:53 -0500  Active:2  Stage out:1
+Final status: Tue, 27 Aug 2013 14:39:53 -0500  Finished successfully:4
+
+$ ls output/
+average.out  sim_0.out  sim_1.out  sim_2.out
+$ more output/*
+::::::::::::::
+output/average.out
+::::::::::::::
+651368
+::::::::::::::
+output/sim_0.out
+::::::::::::::
+  735700
+  886206
+  997391
+  982970
+::::::::::::::
+output/sim_1.out
+::::::::::::::
+  260071
+  264195
+  869198
+  933537
+::::::::::::::
+output/sim_2.out
+::::::::::::::
+  201806
+  213540
+  527576
+  944233
+-----
+
+Now try running (`-nsim=`) 100 simulations of (`-steps=`) 1 second each:
+
+-----
+$ swift p3.swift -nsim=100 -steps=1 
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1444-rq809ts6
+Progress:  time: Tue, 27 Aug 2013 14:44:55 -0500
+Progress:  time: Tue, 27 Aug 2013 14:44:56 -0500  Selecting site:79  Active:20  Stage out:1
+Progress:  time: Tue, 27 Aug 2013 14:44:58 -0500  Selecting site:58  Active:20  Stage out:1  Finished successfully:21
+Progress:  time: Tue, 27 Aug 2013 14:44:59 -0500  Selecting site:37  Active:20  Stage out:1  Finished successfully:42
+Progress:  time: Tue, 27 Aug 2013 14:45:00 -0500  Selecting site:16  Active:20  Stage out:1  Finished successfully:63
+Progress:  time: Tue, 27 Aug 2013 14:45:02 -0500  Active:15  Stage out:1  Finished successfully:84
+Progress:  time: Tue, 27 Aug 2013 14:45:03 -0500  Finished successfully:101
+Final status: Tue, 27 Aug 2013 14:45:03 -0500  Finished successfully:101
+-----
+
+We can see from Swift's "progress" status that the tutorial's default
+parameters for local execution allow Swift to run up to 20
+application invocations concurrently on the login node. We'll look at
+this in more detail in the next sections where we execute applications
+on the site's compute nodes.
+
+
+Running applications on Midway compute nodes with Swift
+-------------------------------------------------------
+
+Part 4: Running a parallel ensemble on Midway compute nodes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+`p4.swift` will run our mock "simulation" applications on Midway 
+compute nodes. The script is similar to as `p3.swift`, but specifies 
+that each simulation app invocation should additionally return the 
+log file which the application writes to `stderr`.
+
+Now when you run `swift p4.swift` you'll see that two types output
+files will placed in the `output/` directory: `sim_N.out` and
+`sim_N.log`.  The log files provide data on the runtime environment of
+each app invocation. For example:
+
+-----
+$ cat output/sim_0.log
+Called as: /home/davidkelly999/swift-midway-tutorial/app/simulate: --timesteps 1 --range 100 --nvalues 5
+
+Start time: Mon Dec  2 12:17:06 CST 2013
+Running as user: uid=88848(davidkelly999) gid=88848(davidkelly999) groups=88848(davidkelly999),10008(rcc),10030(pi-gavoth),10031(sp-swift),10036(swift),10058(pi-joshuaelliott),10084(pi-wilde),10118(cron-account),10124(cmts),10138(cmtsworkshop)
+Running on node: midway002
+Node IP address: 10.50.181.2 172.25.181.2 
+
+Simulation parameters:
+
+bias=0
+biasfile=none
+initseed=none
+log=yes
+paramfile=none
+range=100
+scale=1
+seedfile=none
+timesteps=1
+output width=8
+
+Environment:
+
+ANTLR_ROOT=/software/antlr-2.7-el6-x86_64
+ANT_HOME=/software/ant-1.8.4-all
+ANT_HOME_modshare=/software/ant-1.8.4-all:3
+...
+-----
+
+Swift's `swift.properties` configuration file allows many parameters to
+specify how jobs should be run on a given cluster.
+
+Consider for example that Midway has several Slurm partitions. The sandyb
+partition has 16 cores, and the westmere partition has 12 cores. Depending
+on the application and which partitions are busy, you may want to modify
+where you run. 
+
+Here is an example of the swift.properties in the part04 directory: 
+
+-----
+site=westmere
+site.westmere.slurm.reservation=workshop
+site.westmere.slurm.exclusive=false
+-----
+
+The first line, site=westmere, allows you to define which partition to run on. Swift includes
+templates for each partition that provides some reasonable default values. Other valid partitions 
+are amd, bigmem, gpu, local, and sandyb.
+
+The second and third line override some of the default values, by specifying a reservation that will
+be used for this session, and enabling node sharing.
+
+Try changing the queue to run on the sandyb queue. The new swift.properties should look like this:
+-----
+site=sandyb
+site.sandyb.slurm.reservation=workshop
+site.sandyb.slurm.exclusive=false
+----
+
+Performing larger Swift runs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To test with larger runs, there are two changes that are required. The first is a 
+change to the command line arguments. The example below will run 1000 simulations
+with each simulation taking 5 seconds.
+
+-----
+$ swift p6.swift -steps=5 -nsim=1000
+-----
+
+Plotting run activity
+~~~~~~~~~~~~~~~~~~~~~
+
+The tutorial `bin` directory in your `PATH` provides a script
+`plot.sh` to plot the progress of a Swift script.  It generates two
+image files: `activeplot.png`, which shows the number of active jobs
+over time, and `cumulativeplot.png`, which shows the total number of
+app calls completed as the Swift script progresses.
+
+After each Swift run, a new run directory is created called runNNN.
+Each run directory will have a log file with a similar name called
+runNNN.log. Once you have identified the log file name, run the 
+command `plot.sh` <logfile>` to generate the plots for that
+specific run. For example:
+
+-----
+$ ls
+output  p3.swift  run000  swift.properties
+
+$ cd run000/
+
+$ ls
+apps  cf  p3-20131202-2004-0kh4ha6e.d  run000.log  sites.xml
+
+$ plot.sh run000.log 
+-----
+
+This yields plots like:
+
+image::activeplot.png[width=700,align=center]
+image::cumulativeplot.png[width=700,align=center]
+
+Part 5: Controlling the compute-node pools where applications run
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This section is under development.
+
+Part 6: Specifying more complex workflow patterns
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+p6.swift expands the workflow pattern of p4.swift to add additional
+stages to the workflow. Here, we generate a dynamic seed value that
+will be used by all of the simulations, and for each simulation, we
+run an pre-processing application to generate a unique "bias
+file". This pattern is shown below, followed by the Swift script.
+
+image::part06.png[align="center"]
+
+.p6.swift
+----
+sys::[cat ../part06/p6.swift]
+----
+
+Note that the workflow is based on data flow dependencies: each simulation depends on the seed value, calculated in this statement:
+-----
+seedfile = genseed(1);
+-----
+and on the bias file, computed and then consumed in these two dependent statements:
+-----
+  biasfile = genbias(1000, 20);
+  (simout,simlog) = simulation(steps, range, biasfile, 1000000, values, seedfile);
+-----
+
+To run:
+----
+$ cd ../part06
+$ swift p6.swift
+----
+
+The default parameters result in the following execution log:
+
+-----
+$ swift p6.swift
+Swift 0.94.1 RC2 swift-r6895 cog-r3765
+
+RunID: 20130827-1917-jvs4gqm5
+Progress:  time: Tue, 27 Aug 2013 19:17:56 -0500
+
+*** Script parameters: nsim=10 range=100 num values=10
+
+Progress:  time: Tue, 27 Aug 2013 19:17:57 -0500  Stage in:1  Submitted:10
+Generated seed=382537
+Progress:  time: Tue, 27 Aug 2013 19:17:59 -0500  Active:9  Stage out:1  Finished successfully:11
+Final status: Tue, 27 Aug 2013 19:18:00 -0500  Finished successfully:22
+-----
+which produces the following output:
+-----
+$ ls -lrt output
+total 264
+-rw-r--r-- 1 p01532 61532     9 Aug 27 19:17 seed.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_9.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_8.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_7.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_6.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_5.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_4.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_3.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_2.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_1.dat
+-rw-r--r-- 1 p01532 61532   180 Aug 27 19:17 bias_0.dat
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:17 sim_9.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_9.log
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_8.log
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:17 sim_7.out
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:17 sim_6.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_6.log
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:17 sim_5.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_5.log
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:17 sim_4.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_4.log
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:17 sim_1.log
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:18 sim_8.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_7.log
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:18 sim_3.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_3.log
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:18 sim_2.out
+-rw-r--r-- 1 p01532 61532 14898 Aug 27 19:18 sim_2.log
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:18 sim_1.out
+-rw-r--r-- 1 p01532 61532    90 Aug 27 19:18 sim_0.out
+-rw-r--r-- 1 p01532 61532 14897 Aug 27 19:18 sim_0.log
+-rw-r--r-- 1 p01532 61532     9 Aug 27 19:18 average.out
+-rw-r--r-- 1 p01532 61532 14675 Aug 27 19:18 average.log
+-----
+
+Each sim_N.out file is the sum of its bias file plus newly "simulated" random output scaled by 1,000,000:
+
+-----
+$ cat output/bias_0.dat
+     302
+     489
+      81
+     582
+     664
+     290
+     839
+     258
+     506
+     310
+     293
+     508
+      88
+     261
+     453
+     187
+      26
+     198
+     402
+     555
+
+$ cat output/sim_0.out
+64000302
+38000489
+32000081
+12000582
+46000664
+36000290
+35000839
+22000258
+49000506
+75000310
+-----
+
+We produce 20 values in each bias file. Simulations of less than that
+number of values ignore the unneeded number, while simualtions of more
+than 20 will use the last bias number for all remoaining values past
+20.  As an exercise, adjust the code to produce the same number of
+bias values as is needed for each simulation.  As a further exercise,
+modify the script to generate a unique seed value for each simulation,
+which is a common practice in ensemble computations.
+

Added: demo/xsede.2014.0425/doc/TODO
===================================================================
--- demo/xsede.2014.0425/doc/TODO	                        (rev 0)
+++ demo/xsede.2014.0425/doc/TODO	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,137 @@
+PLAN
+
+basics
+cloud
+uc3
+multisystems
+(modis)
+
+get hosts.txt for cloud dynamically
+
+Show variety of UC3 activity and what ran where
+
+get hostname and app_resources into provider staging (hostname into all _swiftwrap)
+
+Show gridftp and go connect to process the modis dataset
+
+Show SwiftR for R users, SwiftPy for Enthought PY?
+
+fetching apps vs sending apps vs parrot/cvmfs/oasis etc ????
+
+Demo script-aps, bin apps, building apps, caching apps. etc.
+
+Do a Swift BLAST workflow  -- with viz !!!  (spice it up with help from Dina)
+
+OSG page from Mats: "Is your job HTC-ready"
+
+Use new Trunk config in tutorial?
+
+
+Add viz and status display; add node probing to tutorial.
+
+Debug failure with pinfiles
+
+
+What "exercises to suggest?"
+
+- run stats app locally
+- installed vs transferred apps
+- transferred programs vs transferred scripts
+- summarize things about the env from the .log files, ala stats.
+
+Y-style dag: 2 sims and average
+
+Show how to find temp files at runtime...
+
+Show how to find hung processes at runtime...  (or get other trace data...)
+
+make your owb versions of the programs; verify that you can substitute them
+
+Add a layer to isolate shell version v installed version
+
+Show how to handle a paramfile
+
+Test if OSGCOnnect works and account string (PorjectName) works OK for both!
+
+SHow how to handle long lists of files (eg writeData and other swift.properties)
+
+Expand arith to arb prec using bc
+
+
+ISSUES
+
+error in pin files? debug and turn back on
+PATH issues for various providers
+
+bin dir?
+
+setup.csh and .csh testing
+
+IMPROVEMENTS
+
+1. Cleanup.sh mentioned in the tutorial
+[fixed]
+ 
+2. cd ../part02
+[fixed]
+ 
+3. support for zsh, csh
+[pending]
+ 
+4. What results are created and where did the outputs go ?
+and pointers to what the log files mean
+[pending]
+ 
+5. Part4,
+- cd ../part04 is missing. Cleanup the README
+[fixed]
+ 
+6. Part5
+- cd ../part05 is missing. Cleanup
+[fixed]
+ 
+7. Part7
+-Change from running on Tukey analysis cluster compute nodes
+[fixed]
+ 
+8. Part6
+command line args for nsim and steps, tell user the different behavior
+that can be expected with different arg options
+[pending]
+ 
+9. Draw workflow for the modis demos
+[pending]
+ 
+10. Link to more technical swift-lang references
+[pending]
+ 
+11. Links to detailed language constructs, when explaining
+apps, link to the section on apps in the userguide.
+[pending]
+ 
+ 
+ 
+---
+
+From Yadu:
+
+I've committed all the changes that I made on Friday and you can get them 
+from -> https://svn.ci.uchicago.edu/svn/vdl2/SwiftTutorials/CIC_2013-08-09
+
+I'm trying to make this tutorial easily work on several sites so that, we
+do not have to tweak things for every single tutorial. Once this is done,
+I can also add it to the test-battery as well. I think this has value 
+considering that we spend atleast 1-2 days for every demo.
+
+Now, the major pending item is the Modis demo, specifically the issue of 
+bringing data to any site and scripts. Slight tweaks to allow for sending
+the scripts as args to apps would also be needed.
+
+So the major changes are: 
+1. Source the setup.sh script with the target site name
+eg. source setups.sh <cloud|uc3|midway|beagle>
+2. No separate cloud folder
+3. [pending] fix modis data and scripts
+4. [pending] support for zsh and other shells? 
+
+---

Added: demo/xsede.2014.0425/doc/activeplot.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/activeplot.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/asciidoc.css
===================================================================
--- demo/xsede.2014.0425/doc/asciidoc.css	                        (rev 0)
+++ demo/xsede.2014.0425/doc/asciidoc.css	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,26 @@
+a:link { color:navy; }
+a:visited { color:navy; }
+
+.monospaced, code, pre {
+  font-family: "Courier New", Courier, monospace;
+  font-size: medium; /* inherit; */
+  color: black;
+  padding: 0;
+  margin: 0;
+}
+
+/*
+  background: #f8f8f8;
+  border: 1px solid #dddddd;
+  border-left: 5px solid #f0f0f0;
+  margin-right: 10%;
+*/
+
+div.listingblock > div.content {
+  padding: 0.5em;
+  background: none;
+  border: none;
+  border-left: none;
+  margin-right: none;
+}
+

Added: demo/xsede.2014.0425/doc/build_docs.sh
===================================================================
--- demo/xsede.2014.0425/doc/build_docs.sh	                        (rev 0)
+++ demo/xsede.2014.0425/doc/build_docs.sh	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,3 @@
+#!/bin/bash -e
+
+asciidoc -a icons -a toc -a toplevels=2 -a stylesheet=$PWD/asciidoc.css -a max-width=800px -o tutorial.html README


Property changes on: demo/xsede.2014.0425/doc/build_docs.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/doc/cumulativeplot.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/cumulativeplot.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/figs/modis.dia
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/figs/modis.dia
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/figs/modis.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/figs/modis.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/caution.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/caution.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/example.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/example.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/home.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/home.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/important.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/important.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/next.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/next.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/note.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/note.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/prev.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/prev.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/tip.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/tip.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/up.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/up.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/images/icons/warning.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/images/icons/warning.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/part01.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/part01.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/part02.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/part02.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/part03.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/part03.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/part04.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/part04.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/part05.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/part05.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/part06.png
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/doc/part06.png
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: demo/xsede.2014.0425/doc/push.sh
===================================================================
--- demo/xsede.2014.0425/doc/push.sh	                        (rev 0)
+++ demo/xsede.2014.0425/doc/push.sh	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+# scp -r swift-cray-tutorial.html images *png login.ci.uchicago.edu:/ci/www/projects/swift/tutorials/cray
+
+tar zcf - --exclude-vcs *html *png images | ssh ci "cd /ci/www/projects/swift/tutorials/midway; tar zxf -"


Property changes on: demo/xsede.2014.0425/doc/push.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: demo/xsede.2014.0425/part01/p1.swift
===================================================================
--- demo/xsede.2014.0425/part01/p1.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part01/p1.swift	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,9 @@
+type file;
+
+app (file o) simulation ()
+{
+  simulate stdout=filename(o);
+}
+
+file f <"sim.out">;
+f = simulation();

Added: demo/xsede.2014.0425/part01/swift.properties
===================================================================
--- demo/xsede.2014.0425/part01/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part01/swift.properties	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1 @@
+site=local

Added: demo/xsede.2014.0425/part02/p2.swift
===================================================================
--- demo/xsede.2014.0425/part02/p2.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part02/p2.swift	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,11 @@
+type file;
+
+app (file o) simulation ()
+{
+  simulate stdout=filename(o);
+}
+
+foreach i in [0:9] {
+  file f <single_file_mapper; file=strcat("output/sim_",i,".out")>;
+  f = simulation();
+}

Added: demo/xsede.2014.0425/part02/swift.properties
===================================================================
--- demo/xsede.2014.0425/part02/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part02/swift.properties	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1 @@
+site=local

Added: demo/xsede.2014.0425/part03/p3.swift
===================================================================
--- demo/xsede.2014.0425/part03/p3.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part03/p3.swift	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,27 @@
+type file;
+
+app (file o) simulation (int sim_steps, int sim_range, int sim_values)
+{
+  simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=filename(o);
+}
+
+app (file o) analyze (file s[])
+{
+  stats filenames(s) stdout=filename(o);
+}
+
+int nsim   = toInt(arg("nsim","10"));
+int steps  = toInt(arg("steps","1"));
+int range  = toInt(arg("range","100"));
+int values = toInt(arg("values","5"));
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+  file simout <single_file_mapper; file=strcat("output/sim_",i,".out")>;
+  simout = simulation(steps,range,values);
+  sims[i] = simout;
+}
+
+file stats<"output/average.out">;
+stats = analyze(sims);

Added: demo/xsede.2014.0425/part03/swift.properties
===================================================================
--- demo/xsede.2014.0425/part03/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part03/swift.properties	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1 @@
+site=local

Added: demo/xsede.2014.0425/part04/p4.swift
===================================================================
--- demo/xsede.2014.0425/part04/p4.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part04/p4.swift	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,29 @@
+type file;
+
+app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values)
+{
+  simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log;
+}
+
+app (file out, file log) analyze (file s[])
+{
+  stats filenames(s) stdout=@out stderr=@log;
+}
+
+int nsim   = toInt(arg("nsim",   "10"));
+int steps  = toInt(arg("steps",  "1"));
+int range  = toInt(arg("range",  "100"));
+int values = toInt(arg("values", "5"));
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+  file simout <single_file_mapper; file=strcat("output/sim_",i,".out")>;
+  file simlog <single_file_mapper; file=strcat("output/sim_",i,".log")>;
+  (simout,simlog) = simulation(steps,range,values);
+  sims[i] = simout;
+}
+
+file stats_out<"output/average.out">;
+file stats_log<"output/average.log">;
+(stats_out, stats_log) = analyze(sims);

Added: demo/xsede.2014.0425/part04/swift.properties
===================================================================
--- demo/xsede.2014.0425/part04/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part04/swift.properties	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,2 @@
+site=westmere
+site.westmere.slurm.exclusive=false

Added: demo/xsede.2014.0425/part05/p5.swift
===================================================================
--- demo/xsede.2014.0425/part05/p5.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part05/p5.swift	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,29 @@
+type file;
+
+app (file out, file log) simulation (int sim_steps, int sim_range, int sim_values)
+{
+  simulate "--timesteps" sim_steps "--range" sim_range "--nvalues" sim_values stdout=@out stderr=@log;
+}
+
+app (file out, file log) analyze (file s[])
+{
+  stats filenames(s) stdout=@out stderr=@log;
+}
+
+int nsim   = toInt(arg("nsim",   "10"));
+int steps  = toInt(arg("steps",  "1"));
+int range  = toInt(arg("range",  "100"));
+int values = toInt(arg("values", "5"));
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+  file simout <single_file_mapper; file=strcat("output/sim_",i,".out")>;
+  file simlog <single_file_mapper; file=strcat("output/sim_",i,".log")>;
+  (simout,simlog) = simulation(steps,range,values);
+  sims[i] = simout;
+}
+
+file stats_out<"output/average.out">;
+file stats_log<"output/average.log">;
+(stats_out, stats_log) = analyze(sims);

Added: demo/xsede.2014.0425/part05/swift.properties
===================================================================
--- demo/xsede.2014.0425/part05/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part05/swift.properties	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,2 @@
+site=westmere
+site.westmere.slurm.exclusive=false

Added: demo/xsede.2014.0425/part06/p6.swift
===================================================================
--- demo/xsede.2014.0425/part06/p6.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part06/p6.swift	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,53 @@
+type file;
+
+# app() functions for application programs to be called:
+
+app (file out) genseed (int nseeds)
+{
+  simulate "-r" 2000000 "-n" nseeds stdout=@out;
+}
+
+app (file out) genbias (int bias_range, int nvalues)
+{
+  simulate "-r" bias_range "-n" nvalues stdout=@out;
+}
+
+app (file out, file log) simulation (int timesteps, int sim_range, file bias_file, 
+                                     int scale, int sim_count, file seed_file)
+{
+  simulate "-t" timesteps "-r" sim_range "-B" @bias_file "-x" scale
+           "-n" sim_count "-S" @seed_file stdout=@out stderr=@log;
+}
+
+app (file out, file log) analyze (file s[])
+{
+  stats filenames(s) stdout=@out stderr=@log;
+}
+
+# Command line arguments
+
+int   nsim = toInt(arg("nsim",   "10"));  # number of simulation programs to run
+int  steps = toInt(arg("steps",  "1"));   # number of timesteps (seconds) per simulation
+int  range = toInt(arg("range",  "100")); # range of the generated random numbers
+int values = toInt(arg("values", "10"));  # number of values generated per simulation
+
+# Main script and data
+file seedfile <"output/seed.dat">;        # Dynamically generated bias for simulation ensemble
+
+tracef("\n*** Script parameters: nsim=%i range=%i num values=%i\n\n", nsim, range, values);
+seedfile = genseed(1);
+
+file sims[];                      # Array of files to hold each simulation output
+
+foreach i in [0:nsim-1] {
+  file biasfile <single_file_mapper; file=strcat("output/bias_",i,".dat")>;
+  file simout   <single_file_mapper; file=strcat("output/sim_",i,".out")>;
+  file simlog   <single_file_mapper; file=strcat("output/sim_",i,".log")>;
+  biasfile = genbias(1000, 20);
+  (simout,simlog) = simulation(steps, range, biasfile, 1000000, values, seedfile);
+  sims[i] = simout;
+}
+
+file stats_out<"output/average.out">;
+file stats_log<"output/average.log">;
+(stats_out,stats_log) = analyze(sims);

Added: demo/xsede.2014.0425/part06/swift.properties
===================================================================
--- demo/xsede.2014.0425/part06/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part06/swift.properties	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,2 @@
+site=westmere
+site.westmere.slurm.exclusive=false

Added: demo/xsede.2014.0425/setup.sh
===================================================================
--- demo/xsede.2014.0425/setup.sh	                        (rev 0)
+++ demo/xsede.2014.0425/setup.sh	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,8 @@
+# Load modules
+module unload swift
+module load swift/0.95-RC1
+echo Swift version is $(swift -version)
+
+# Add applications to $PATH
+TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PATH=$TUTDIR/bin:$TUTDIR/app:$PATH

Deleted: demo/xsede.2014.0425/src/c-ray/RUN
===================================================================
--- demo/xsede.2014.0425/src/c-ray/RUN	2014-04-24 18:08:55 UTC (rev 7796)
+++ demo/xsede.2014.0425/src/c-ray/RUN	2014-04-24 18:34:47 UTC (rev 7797)
@@ -1,2 +0,0 @@
-#!/bin/sh
-cat scene.cr | ./c-ray > scene.ppm

Copied: demo/xsede.2014.0425/src/c-ray/test.sh (from rev 7795, demo/xsede.2014.0425/src/c-ray/RUN)
===================================================================
--- demo/xsede.2014.0425/src/c-ray/test.sh	                        (rev 0)
+++ demo/xsede.2014.0425/src/c-ray/test.sh	2014-04-24 18:34:47 UTC (rev 7797)
@@ -0,0 +1,2 @@
+#!/bin/sh
+cat scene.cr | ./c-ray > scene.ppm

Added: demo/xsede.2014.0425/tutorial.pptx
===================================================================
(Binary files differ)


Property changes on: demo/xsede.2014.0425/tutorial.pptx
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream



From davidk at ci.uchicago.edu  Thu Apr 24 15:37:07 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 15:37:07 -0500 (CDT)
Subject: [Swift-commit] r7798 - branches/release-0.95/bin
Message-ID: <20140424203707.35BE09D102@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-24 15:37:07 -0500 (Thu, 24 Apr 2014)
New Revision: 7798

Modified:
   branches/release-0.95/bin/swiftrun
Log:
Verify swift properties


Modified: branches/release-0.95/bin/swiftrun
===================================================================
--- branches/release-0.95/bin/swiftrun	2014-04-24 18:34:47 UTC (rev 7797)
+++ branches/release-0.95/bin/swiftrun	2014-04-24 20:37:07 UTC (rev 7798)
@@ -26,42 +26,84 @@
 my @new_swift_args;		# New command line arguments
 my @original_swift_args;	# Original arguments
 
-# List of xml shortcuts
+# List of allowable site properties and their xml namespace
 my %site_aliases = (
-                   'allocationstepsize'        => 'globus.allocationStepSize',
-                   'condor'                    => 'globus.condor',
-                   'delaybase'                 => 'karajan.delayBase',
-                   'filesystem'                => 'filesystem',
-                   'highoverallocation'        => 'globus.highOverallocation',
-                   'initialscore'              => 'karajan.initialScore',
-                   'internalhostname'          => 'globus.internalhostname',
-                   'jobgranularity'            => 'globus.nodeGranularity',
-                   'jobmanager'                => 'jobmanager',
-		   'jobproject'                => 'globus.project',
-                   'jobqueue'                  => 'globus.queue',
-                   'jobtype'                   => 'globus.jobType',
-		   'jobwalltime'               => 'globus.maxtime',
-                   'jobwalltimereserve'        => 'globus.reserve',
-                   'lowoverallocation'         => 'globus.lowOverallocation',
-		   'maxjobs'                   => 'globus.slots',
-                   'maxnodesperjob'            => 'globus.maxNodes',
-                   'maxsubmitrate'             => 'karajan.maxSubmitRate',
-                   'overallocationdecayfactor' => 'globus.overallocationDecayFactor',
-		   'pe'                        => 'globus.pe',
-                   'providerattributes'        => 'globus.providerAttributes',
-                   'remotemonitorenabled'      => 'globus.remoteMonitorEnabled',
-                   'slurm'                     => 'globus.slurm',
-                   'taskdir'                   => 'globus.scratch',
-                   'tasksperworker'            => 'globus.jobsPerNode',
-                   'taskthrottle'              => 'karajan.jobThrottle',
-                   'taskwalltime'              => 'globus.maxwalltime',
-		   'userhomeoverride'	       => 'globus.userHomeOverride',
-                   'workdir'                   => 'workdir',
-		   'workerloglevel'            => 'globus.workerlogginglevel',
-                   'workerlogdirectory'        => 'globus.workerloggingdirectory',
-		   'workermanager'             => 'globus.workerManager',
+   'allocationstepsize'        => 'globus.allocationStepSize',
+   'condor'                    => 'globus.condor',
+   'delaybase'                 => 'karajan.delayBase',
+   'filesystem'                => 'filesystem',
+   'highoverallocation'        => 'globus.highOverallocation',
+   'initialscore'              => 'karajan.initialScore',
+   'internalhostname'          => 'globus.internalhostname',
+   'jobgranularity'            => 'globus.nodeGranularity',
+   'jobmanager'                => 'jobmanager',
+   'jobproject'                => 'globus.project',
+   'jobqueue'                  => 'globus.queue',
+   'jobtype'                   => 'globus.jobType',
+   'jobwalltime'               => 'globus.maxtime',
+   'jobwalltimereserve'        => 'globus.reserve',
+   'lowoverallocation'         => 'globus.lowOverallocation',
+   'maxjobs'                   => 'globus.slots',
+   'maxnodesperjob'            => 'globus.maxNodes',
+   'maxsubmitrate'             => 'karajan.maxSubmitRate',
+   'overallocationdecayfactor' => 'globus.overallocationDecayFactor',
+   'pe'                        => 'globus.pe',
+   'providerattributes'        => 'globus.providerAttributes',
+   'remotemonitorenabled'      => 'globus.remoteMonitorEnabled',
+   'slurm'                     => 'globus.slurm',
+   'taskdir'                   => 'globus.scratch',
+   'tasksperworker'            => 'globus.jobsPerNode',
+   'taskthrottle'              => 'karajan.jobThrottle',
+   'taskwalltime'              => 'globus.maxwalltime',
+   'userhomeoverride'	       => 'globus.userHomeOverride',
+   'workdir'                   => 'workdir',
+   'workerloglevel'            => 'globus.workerlogginglevel',
+   'workerlogdirectory'        => 'globus.workerloggingdirectory',
+   'workermanager'             => 'globus.workerManager',
 );
 
+# List of allowable swift properties and aliases
+my %property_aliases = (
+   'caching.algorithm'               => 'caching.algorithm',
+   'clustering.enabled'              => 'clustering.enabled',
+   'clustering.min.time'             => 'clustering.min.time',
+   'clustering.queue.delay'          => 'clustering.queue.delay',
+   'config.rundirs'                  => 'config.rundirs',
+   'execution.retries'               => 'execution.retries',
+   'foreach.max.threads'             => 'foreach.max.threads',
+   'kickstart.enabled'               => 'kickstart.enabled',
+   'kickstart.always.transfer'       => 'kickstart.always.transfer',
+   'lazy.errors'                     => 'lazy.errors',
+   'pgraph'                          => 'pgraph',                 
+   'pgraph.graph.options'            => 'pgraph.graph.options',
+   'pgraph.node.options'             => 'pgraph.node.options',
+   'provenance.log'                  => 'provenance.log',
+   'provider.staging.pin.swiftfiles' => 'provider.staging.pin.swiftfiles',
+   'replication.enabled'             => 'replication.enabled',
+   'replication.min.queue.time'      => 'replication.min.queue.time',
+   'replication.limit'               => 'replication.limit',
+   'sitedir.keep'                    => 'sitedir.keep',
+   'sites.file'                      => 'sites.file',
+   'status.mode'                     => 'status.mode',
+   'swift.home'                      => 'swift.home',
+   'tc.file'                         => 'tc.file',
+   'tcp.port.range'                  => 'tcp.port.range',
+   'throttle.file.operations'        => 'throttle.file.operations',
+   'throttle.host.submit'            => 'throttle.host.submit',
+   'throttle.score.job.factor'       => 'throttle.score.job.factor',
+   'throttle.submit'                 => 'throttle.submit',
+   'throttle.transfers'              => 'throttle.transfers',
+   'ticker.date.format'              => 'ticker.date.format',
+   'ticker.disable'                  => 'ticker.disable',
+   'ticker.prefix'                   => 'ticker.prefix',
+   'tracing.enabled'                 => 'tracing.enabled',
+   'use.wrapper.staging'             => 'use.wrapper.staging',
+   'use.provider.staging'            => 'use.provider.staging',
+   'wrapper.invocation.mode'         => 'wrapper.invocation.mode',
+   'wrapper.parameter.mode'          => 'wrapper.parameter.mode',
+   'wrapperlog.always.transfer'      => 'wrapperlog.always.transfer',
+);
+
 # Verify a property file exists and add it to the search path
 sub add_property_file {
    my $property_file = $_[0];
@@ -230,9 +272,13 @@
    open( PROPERTY_FILE, ">$run_directory/$property_filename" ) || &swiftquit("Unable to open $run_directory/$property_filename");
    while( my( $key, $value ) = each( %properties ) ) {
       # Ignore properties that start with app. or site.
-      next if $key =~ /^\s*app./;
-      next if $key =~ /^\s*site/;
-      print PROPERTY_FILE "$key=$value\n";
+      next if $key =~ /^\s*app\./;
+      next if $key =~ /^\s*site$/;
+      next if $key =~ /^\s*site\./;
+      if( !defined( $property_aliases{ $key } )) {
+         swiftquit("Error: Unknown property $key");
+      }
+      print PROPERTY_FILE "$property_aliases{$key}=$value\n";
    }
    push( @new_swift_args, "-config $run_directory/$property_filename" );
 }



From davidk at ci.uchicago.edu  Thu Apr 24 15:38:11 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 15:38:11 -0500 (CDT)
Subject: [Swift-commit] r7799 - trunk/bin
Message-ID: <20140424203811.E395F9D102@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-24 15:38:11 -0500 (Thu, 24 Apr 2014)
New Revision: 7799

Modified:
   trunk/bin/swiftrun
Log:
Verify swift properties


Modified: trunk/bin/swiftrun
===================================================================
--- trunk/bin/swiftrun	2014-04-24 20:37:07 UTC (rev 7798)
+++ trunk/bin/swiftrun	2014-04-24 20:38:11 UTC (rev 7799)
@@ -26,42 +26,84 @@
 my @new_swift_args;		# New command line arguments
 my @original_swift_args;	# Original arguments
 
-# List of xml shortcuts
+# List of allowable site properties and their xml namespace
 my %site_aliases = (
-                   'allocationstepsize'        => 'globus.allocationStepSize',
-                   'condor'                    => 'globus.condor',
-                   'delaybase'                 => 'karajan.delayBase',
-                   'filesystem'                => 'filesystem',
-                   'highoverallocation'        => 'globus.highOverallocation',
-                   'initialscore'              => 'karajan.initialScore',
-                   'internalhostname'          => 'globus.internalhostname',
-                   'jobgranularity'            => 'globus.nodeGranularity',
-                   'jobmanager'                => 'jobmanager',
-		   'jobproject'                => 'globus.project',
-                   'jobqueue'                  => 'globus.queue',
-                   'jobtype'                   => 'globus.jobType',
-		   'jobwalltime'               => 'globus.maxtime',
-                   'jobwalltimereserve'        => 'globus.reserve',
-                   'lowoverallocation'         => 'globus.lowOverallocation',
-		   'maxjobs'                   => 'globus.slots',
-                   'maxnodesperjob'            => 'globus.maxNodes',
-                   'maxsubmitrate'             => 'karajan.maxSubmitRate',
-                   'overallocationdecayfactor' => 'globus.overallocationDecayFactor',
-		   'pe'                        => 'globus.pe',
-                   'providerattributes'        => 'globus.providerAttributes',
-                   'remotemonitorenabled'      => 'globus.remoteMonitorEnabled',
-                   'slurm'                     => 'globus.slurm',
-                   'taskdir'                   => 'globus.scratch',
-                   'tasksperworker'            => 'globus.jobsPerNode',
-                   'taskthrottle'              => 'karajan.jobThrottle',
-                   'taskwalltime'              => 'globus.maxwalltime',
-		   'userhomeoverride'	       => 'globus.userHomeOverride',
-                   'workdir'                   => 'workdir',
-		   'workerloglevel'            => 'globus.workerlogginglevel',
-                   'workerlogdirectory'        => 'globus.workerloggingdirectory',
-		   'workermanager'             => 'globus.workerManager',
+   'allocationstepsize'        => 'globus.allocationStepSize',
+   'condor'                    => 'globus.condor',
+   'delaybase'                 => 'karajan.delayBase',
+   'filesystem'                => 'filesystem',
+   'highoverallocation'        => 'globus.highOverallocation',
+   'initialscore'              => 'karajan.initialScore',
+   'internalhostname'          => 'globus.internalhostname',
+   'jobgranularity'            => 'globus.nodeGranularity',
+   'jobmanager'                => 'jobmanager',
+   'jobproject'                => 'globus.project',
+   'jobqueue'                  => 'globus.queue',
+   'jobtype'                   => 'globus.jobType',
+   'jobwalltime'               => 'globus.maxtime',
+   'jobwalltimereserve'        => 'globus.reserve',
+   'lowoverallocation'         => 'globus.lowOverallocation',
+   'maxjobs'                   => 'globus.slots',
+   'maxnodesperjob'            => 'globus.maxNodes',
+   'maxsubmitrate'             => 'karajan.maxSubmitRate',
+   'overallocationdecayfactor' => 'globus.overallocationDecayFactor',
+   'pe'                        => 'globus.pe',
+   'providerattributes'        => 'globus.providerAttributes',
+   'remotemonitorenabled'      => 'globus.remoteMonitorEnabled',
+   'slurm'                     => 'globus.slurm',
+   'taskdir'                   => 'globus.scratch',
+   'tasksperworker'            => 'globus.jobsPerNode',
+   'taskthrottle'              => 'karajan.jobThrottle',
+   'taskwalltime'              => 'globus.maxwalltime',
+   'userhomeoverride'	       => 'globus.userHomeOverride',
+   'workdir'                   => 'workdir',
+   'workerloglevel'            => 'globus.workerlogginglevel',
+   'workerlogdirectory'        => 'globus.workerloggingdirectory',
+   'workermanager'             => 'globus.workerManager',
 );
 
+# List of allowable swift properties and aliases
+my %property_aliases = (
+   'caching.algorithm'               => 'caching.algorithm',
+   'clustering.enabled'              => 'clustering.enabled',
+   'clustering.min.time'             => 'clustering.min.time',
+   'clustering.queue.delay'          => 'clustering.queue.delay',
+   'config.rundirs'                  => 'config.rundirs',
+   'execution.retries'               => 'execution.retries',
+   'foreach.max.threads'             => 'foreach.max.threads',
+   'kickstart.enabled'               => 'kickstart.enabled',
+   'kickstart.always.transfer'       => 'kickstart.always.transfer',
+   'lazy.errors'                     => 'lazy.errors',
+   'pgraph'                          => 'pgraph',                 
+   'pgraph.graph.options'            => 'pgraph.graph.options',
+   'pgraph.node.options'             => 'pgraph.node.options',
+   'provenance.log'                  => 'provenance.log',
+   'provider.staging.pin.swiftfiles' => 'provider.staging.pin.swiftfiles',
+   'replication.enabled'             => 'replication.enabled',
+   'replication.min.queue.time'      => 'replication.min.queue.time',
+   'replication.limit'               => 'replication.limit',
+   'sitedir.keep'                    => 'sitedir.keep',
+   'sites.file'                      => 'sites.file',
+   'status.mode'                     => 'status.mode',
+   'swift.home'                      => 'swift.home',
+   'tc.file'                         => 'tc.file',
+   'tcp.port.range'                  => 'tcp.port.range',
+   'throttle.file.operations'        => 'throttle.file.operations',
+   'throttle.host.submit'            => 'throttle.host.submit',
+   'throttle.score.job.factor'       => 'throttle.score.job.factor',
+   'throttle.submit'                 => 'throttle.submit',
+   'throttle.transfers'              => 'throttle.transfers',
+   'ticker.date.format'              => 'ticker.date.format',
+   'ticker.disable'                  => 'ticker.disable',
+   'ticker.prefix'                   => 'ticker.prefix',
+   'tracing.enabled'                 => 'tracing.enabled',
+   'use.wrapper.staging'             => 'use.wrapper.staging',
+   'use.provider.staging'            => 'use.provider.staging',
+   'wrapper.invocation.mode'         => 'wrapper.invocation.mode',
+   'wrapper.parameter.mode'          => 'wrapper.parameter.mode',
+   'wrapperlog.always.transfer'      => 'wrapperlog.always.transfer',
+);
+
 # Verify a property file exists and add it to the search path
 sub add_property_file {
    my $property_file = $_[0];
@@ -230,9 +272,13 @@
    open( PROPERTY_FILE, ">$run_directory/$property_filename" ) || &swiftquit("Unable to open $run_directory/$property_filename");
    while( my( $key, $value ) = each( %properties ) ) {
       # Ignore properties that start with app. or site.
-      next if $key =~ /^\s*app./;
-      next if $key =~ /^\s*site/;
-      print PROPERTY_FILE "$key=$value\n";
+      next if $key =~ /^\s*app\./;
+      next if $key =~ /^\s*site$/;
+      next if $key =~ /^\s*site\./;
+      if( !defined( $property_aliases{ $key } )) {
+         swiftquit("Error: Unknown property $key");
+      }
+      print PROPERTY_FILE "$property_aliases{$key}=$value\n";
    }
    push( @new_swift_args, "-config $run_directory/$property_filename" );
 }



From wilde at ci.uchicago.edu  Thu Apr 24 16:13:51 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 16:13:51 -0500 (CDT)
Subject: [Swift-commit] r7800 - in demo/xsede.2014.0425: . app part01m
	part02m part03m part04 src/c-ray src/md
Message-ID: <20140424211351.151209D102@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 16:13:50 -0500 (Thu, 24 Apr 2014)
New Revision: 7800

Added:
   demo/xsede.2014.0425/part01m/
   demo/xsede.2014.0425/part01m/p1.swift
   demo/xsede.2014.0425/part01m/swift.properties
   demo/xsede.2014.0425/part02m/
   demo/xsede.2014.0425/part02m/p2.swift
   demo/xsede.2014.0425/part02m/swift.properties
   demo/xsede.2014.0425/part03m/
   demo/xsede.2014.0425/part03m/p3.swift
   demo/xsede.2014.0425/part03m/swift.properties
   demo/xsede.2014.0425/setup.midway.sh
Removed:
   demo/xsede.2014.0425/setup.sh
Modified:
   demo/xsede.2014.0425/app/stats
   demo/xsede.2014.0425/part04/swift.properties
   demo/xsede.2014.0425/src/c-ray/Makefile
   demo/xsede.2014.0425/src/md/Makefile
Log:
Provide md-based example for parts 1-3.

Modified: demo/xsede.2014.0425/app/stats
===================================================================
--- demo/xsede.2014.0425/app/stats	2014-04-24 20:38:11 UTC (rev 7799)
+++ demo/xsede.2014.0425/app/stats	2014-04-24 21:13:50 UTC (rev 7800)
@@ -12,8 +12,8 @@
 
 awk '
 
-{ sum += $1}
+{ sum += $4}
 
-END { printf("%d\n",sum/NR) }
+END { printf("%f\n",sum/NR) }
 ' $*
 log 1>&2

Added: demo/xsede.2014.0425/part01m/p1.swift
===================================================================
--- demo/xsede.2014.0425/part01m/p1.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part01m/p1.swift	2014-04-24 21:13:50 UTC (rev 7800)
@@ -0,0 +1,9 @@
+type file;
+
+app (file o) simulation ()
+{
+  md 3 50 1000 stdout=filename(o);
+}
+
+file f <"sim.out">;
+f = simulation();

Added: demo/xsede.2014.0425/part01m/swift.properties
===================================================================
--- demo/xsede.2014.0425/part01m/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part01m/swift.properties	2014-04-24 21:13:50 UTC (rev 7800)
@@ -0,0 +1,3 @@
+site=local
+ticker.date.format=HH:mm:ss
+ticker.prefix= 

Added: demo/xsede.2014.0425/part02m/p2.swift
===================================================================
--- demo/xsede.2014.0425/part02m/p2.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part02m/p2.swift	2014-04-24 21:13:50 UTC (rev 7800)
@@ -0,0 +1,11 @@
+type file;
+
+app (file o) simulation ()
+{
+  sh "-c" "md 3 50 1000 >/dev/null; cat md.dat" stdout=filename(o);
+}
+
+foreach i in [0:9] {
+  file f <single_file_mapper; file=strcat("output/sim_",i,".out")>;
+  f = simulation();
+}

Added: demo/xsede.2014.0425/part02m/swift.properties
===================================================================
--- demo/xsede.2014.0425/part02m/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part02m/swift.properties	2014-04-24 21:13:50 UTC (rev 7800)
@@ -0,0 +1,3 @@
+site=local
+ticker.date.format=HH:mm:ss
+ticker.prefix= 

Added: demo/xsede.2014.0425/part03m/p3.swift
===================================================================
--- demo/xsede.2014.0425/part03m/p3.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part03m/p3.swift	2014-04-24 21:13:50 UTC (rev 7800)
@@ -0,0 +1,44 @@
+type file;
+
+/*
+    md nd np step_num print_step_num dt mass printinfo scale_factor scale_offset
+    where
+    * nd is the spatial dimension (2 or 3);
+    * np is the number of particles (500, for instance);
+    * step_num is the number of time steps (500, for instance);
+    * print_step_num is the number of snapshot prints (10 for instance);
+    * dt is size of timestep;
+    * mass is particle mass;
+    * printinfo is a string to append to each particle coord
+    * scale_offset and scale_factor are used to scale particle positions for logging/rendering (FIXME)
+
+    e.g.:
+
+    md 3 50 30000 50  .0001  .005  "0.03 1.0 0.2 0.05 50.0 0.1" 2.5 2.0
+*/
+
+app (file o) simulation (string npart, string steps, string mass)
+{
+  sh "-c" strjoin(["md","3",npart,steps,">/dev/null; cat md.dat"]," ") stdout=filename(o);
+}
+
+app (file o) analyze (file s[])
+{
+  mdstats filenames(s) stdout=filename(o);
+}
+
+int    nsim   = toInt(arg("nsim","10"));
+string npart  = arg("npart","50");
+string steps  = arg("steps","1000");
+string mass   = arg("mass",".0001");
+
+file sims[];
+
+foreach i in [0:nsim-1] {
+  file simout <single_file_mapper; file=strcat("output/sim_",i,".out")>;
+  simout = simulation(npart,steps,mass);
+  sims[i] = simout;
+}
+
+file stats<"output/average.out">;
+stats = analyze(sims);

Added: demo/xsede.2014.0425/part03m/swift.properties
===================================================================
--- demo/xsede.2014.0425/part03m/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part03m/swift.properties	2014-04-24 21:13:50 UTC (rev 7800)
@@ -0,0 +1,3 @@
+site=local
+ticker.date.format=HH:mm:ss
+ticker.prefix= 

Modified: demo/xsede.2014.0425/part04/swift.properties
===================================================================
--- demo/xsede.2014.0425/part04/swift.properties	2014-04-24 20:38:11 UTC (rev 7799)
+++ demo/xsede.2014.0425/part04/swift.properties	2014-04-24 21:13:50 UTC (rev 7800)
@@ -1,2 +1,2 @@
-site=westmere
+site=midway-westmere
 site.westmere.slurm.exclusive=false

Copied: demo/xsede.2014.0425/setup.midway.sh (from rev 7797, demo/xsede.2014.0425/setup.sh)
===================================================================
--- demo/xsede.2014.0425/setup.midway.sh	                        (rev 0)
+++ demo/xsede.2014.0425/setup.midway.sh	2014-04-24 21:13:50 UTC (rev 7800)
@@ -0,0 +1,8 @@
+# Load modules
+module unload swift
+module load swift/0.95-RC1
+echo Swift version is $(swift -version)
+
+# Add applications to $PATH
+TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PATH=$TUTDIR/bin:$TUTDIR/app:$PATH

Deleted: demo/xsede.2014.0425/setup.sh
===================================================================
--- demo/xsede.2014.0425/setup.sh	2014-04-24 20:38:11 UTC (rev 7799)
+++ demo/xsede.2014.0425/setup.sh	2014-04-24 21:13:50 UTC (rev 7800)
@@ -1,8 +0,0 @@
-# Load modules
-module unload swift
-module load swift/0.95-RC1
-echo Swift version is $(swift -version)
-
-# Add applications to $PATH
-TUTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-PATH=$TUTDIR/bin:$TUTDIR/app:$PATH

Modified: demo/xsede.2014.0425/src/c-ray/Makefile
===================================================================
--- demo/xsede.2014.0425/src/c-ray/Makefile	2014-04-24 20:38:11 UTC (rev 7799)
+++ demo/xsede.2014.0425/src/c-ray/Makefile	2014-04-24 21:13:50 UTC (rev 7800)
@@ -16,8 +16,8 @@
 
 .PHONY: install
 install:
-	cp $(bin) /usr/local/bin/$(bin)
+	cp $(bin) ../../bin/$(bin)
 
 .PHONY: uninstall
 uninstall:
-	rm -f /usr/local/bin/$(bin)
+	rm -f ../../bin/$(bin)

Modified: demo/xsede.2014.0425/src/md/Makefile
===================================================================
--- demo/xsede.2014.0425/src/md/Makefile	2014-04-24 20:38:11 UTC (rev 7799)
+++ demo/xsede.2014.0425/src/md/Makefile	2014-04-24 21:13:50 UTC (rev 7800)
@@ -10,3 +10,10 @@
 clean:
 	rm -f $(obj) $(bin)
 
+.PHONY: install
+install:
+	cp $(bin) ../../bin/$(bin)
+
+.PHONY: uninstall
+uninstall:
+	rm -f ../../bin/$(bin)



From wilde at ci.uchicago.edu  Thu Apr 24 16:14:57 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 16:14:57 -0500 (CDT)
Subject: [Swift-commit] r7801 - demo/xsede.2014.0425/app
Message-ID: <20140424211457.CD98D9D102@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 16:14:57 -0500 (Thu, 24 Apr 2014)
New Revision: 7801

Added:
   demo/xsede.2014.0425/app/mdstats
Modified:
   demo/xsede.2014.0425/app/stats
Log:
Revert stats back to original. Add mdstats.

Added: demo/xsede.2014.0425/app/mdstats
===================================================================
--- demo/xsede.2014.0425/app/mdstats	                        (rev 0)
+++ demo/xsede.2014.0425/app/mdstats	2014-04-24 21:14:57 UTC (rev 7801)
@@ -0,0 +1,23 @@
+#! /bin/sh
+
+log() {
+  printf "\nCalled as: $0: $cmdargs\n\n"
+  printf "Start time: "; /bin/date
+  printf "Running as user: "; /usr/bin/id
+  printf "Running on node: "; /bin/hostname
+  printf "Node IP address: "; /bin/hostname -I
+  printf "\nEnvironment:\n\n"
+  printenv | sort
+}
+
+awk '
+
+{
+  sum2 += $2
+  sum3 += $3
+  sum4 += $4
+}
+
+END { printf("%f %f %f\n", sum2/NR, sum3/NR, sum4/NR) }
+' $*
+log 1>&2


Property changes on: demo/xsede.2014.0425/app/mdstats
___________________________________________________________________
Added: svn:executable
   + *

Modified: demo/xsede.2014.0425/app/stats
===================================================================
--- demo/xsede.2014.0425/app/stats	2014-04-24 21:13:50 UTC (rev 7800)
+++ demo/xsede.2014.0425/app/stats	2014-04-24 21:14:57 UTC (rev 7801)
@@ -12,7 +12,7 @@
 
 awk '
 
-{ sum += $4}
+{ sum += $1}
 
 END { printf("%f\n",sum/NR) }
 ' $*



From wilde at ci.uchicago.edu  Thu Apr 24 22:05:59 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Thu, 24 Apr 2014 22:05:59 -0500 (CDT)
Subject: [Swift-commit] r7802 - demo/xsede.2014.0425/app
Message-ID: <20140425030559.479799D0A1@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-24 22:05:59 -0500 (Thu, 24 Apr 2014)
New Revision: 7802

Added:
   demo/xsede.2014.0425/app/mdviz
Log:
app to simulate and render in one invocation

Added: demo/xsede.2014.0425/app/mdviz
===================================================================
--- demo/xsede.2014.0425/app/mdviz	                        (rev 0)
+++ demo/xsede.2014.0425/app/mdviz	2014-04-25 03:05:59 UTC (rev 7802)
@@ -0,0 +1,39 @@
+#! /bin/sh
+
+mdout=$1
+vzout=$2
+shift 2
+
+md $* >& md.out     # Run the MD command
+
+cat <<EOF >c-ray-cmds
+# spheres
+#	position		radius	color			shininess	reflectivity
+#s	-0.5 -0.3 -1	0.05	1.0 0.2 0.05		50.0	0.1
+#s	0.5 -0.4 0		0.05	0.1 0.85 1.0		50.0	0.1
+
+
+# walls
+s	0 -1000 2		99		0.2 0.2 0.2			0.0	0.0
+
+# bouncing ball
+#s	0 0 2			1		1.0 0.5 0.1			60.0	0.7
+
+# lights...
+l	-50 100 -50
+#l	40 40 150
+
+# camera (there can be only one!)
+#	position	FOV		target
+c	0 6 -17		45		0 -1 0
+EOF
+
+for t in md??.trj; do
+  cat $t c-ray-cmds | c-ray >$t.ppm
+  convert $t.ppm $t.png 
+done
+
+convert -delay 20 md??.trj.png t.gif
+
+mv md.dat $mdout
+mv t.gif  $vzout


Property changes on: demo/xsede.2014.0425/app/mdviz
___________________________________________________________________
Added: svn:executable
   + *



From davidk at ci.uchicago.edu  Fri Apr 25 16:11:25 2014
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri, 25 Apr 2014 16:11:25 -0500 (CDT)
Subject: [Swift-commit] r7803 - trunk/docs/userguide
Message-ID: <20140425211125.8D0199D584@svn.ci.uchicago.edu>

Author: davidk
Date: 2014-04-25 16:11:25 -0500 (Fri, 25 Apr 2014)
New Revision: 7803

Added:
   trunk/docs/userguide/HTTP-monitor.jpg
   trunk/docs/userguide/Swing-monitor.jpg
   trunk/docs/userguide/TUI-monitor.jpg
Removed:
   trunk/docs/userguide/swift_monitor.png
Modified:
   trunk/docs/userguide/configuration
   trunk/docs/userguide/debugging
Log:
Userguide updates for the various monitors


Added: trunk/docs/userguide/HTTP-monitor.jpg
===================================================================
(Binary files differ)


Property changes on: trunk/docs/userguide/HTTP-monitor.jpg
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: trunk/docs/userguide/Swing-monitor.jpg
===================================================================
(Binary files differ)


Property changes on: trunk/docs/userguide/Swing-monitor.jpg
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Added: trunk/docs/userguide/TUI-monitor.jpg
===================================================================
(Binary files differ)


Property changes on: trunk/docs/userguide/TUI-monitor.jpg
___________________________________________________________________
Added: svn:mime-type
   + application/octet-stream

Modified: trunk/docs/userguide/configuration
===================================================================
--- trunk/docs/userguide/configuration	2014-04-25 03:05:59 UTC (rev 7802)
+++ trunk/docs/userguide/configuration	2014-04-25 21:11:25 UTC (rev 7803)
@@ -397,6 +397,13 @@
  information enabling this will increase the size of log files,
  sometimes significantly.
 
+|provider.staging.pin.swiftfiles
+|true, false
+|false
+|When provider staging is enabled and provider.staging.pin.swiftfiles
+|is set, cache some small files needed by Swift to avoid the cost
+|of staging more than once.
+
 |sitedir.keep
 |true, false
 |false
@@ -497,6 +504,15 @@
 |Progress:
 |String to prepend to ticker output
 
+|tracing.enabled
+|true, false
+|true
+|Enables tracing of procedure invocations, assignments,
+ iteration constructs, as well as certain dataflow events
+ such as data intialization and waiting. This is done at 
+ a slight decrease in performance. Traces will be available
+ in the log file.
+
 |use.wrapper.staging
 |true, false
 |false

Modified: trunk/docs/userguide/debugging
===================================================================
--- trunk/docs/userguide/debugging	2014-04-25 03:05:59 UTC (rev 7802)
+++ trunk/docs/userguide/debugging	2014-04-25 21:11:25 UTC (rev 7803)
@@ -71,29 +71,52 @@
 Monitoring Swift
 ~~~~~~~~~~~~~~~
 
-A Swift run can be monitored for progress and resource usage. To monitor the resource usage, use the +-monitor+ option with the Swift commandline. For example:
+Swift runs can be monitored for progress and resource usage. There are three basic monitors available: Swing, TUI, and http.
 
-----
-swift -tc.file tc -sites.file sites.xml -config cf modis04.swift -monitor
-----
+HTTP Monitor
+^^^^^^^^^^^^
+The HTTP monitor will allow for the monitoring of Swift via a web browser. To start the HTTP monitor, run Swift with the +-ui http:<port>+
+command line option. For example:
 
-This will produce a gui/X window consisting of the following quantities:
+-----
+swift -ui http:8000 modis.swift
+-----
+image:HTTP-monitor.jpg[]
 
-* Allocated memory
-* Heap Size
-* Total Threads
-* Total Workers
+This will create a server running on port 8000 on the machine where Swift is running. Point your web browser to http://<ip_address>:8000 to view
+progress.
 
-.Figure 4. Resource Monitor
-image:swift_monitor.png[]
 
-Figure 4 shows a snapshot of a Swift resource monitor.
+Swing Monitor
+^^^^^^^^^^^^^
 
-The progress of a Swift run can be monitored using the +-tui+ option. For example:
+The Swing monitor displays information via a Java gui/X window. To start the Swing monitor, run Swift with the
++-ui Swing+ command line option. For example:
 
+-----
+swift -ui Swing modis.swift
+-----
+image:Swing-monitor.jpg[]
+
+This will produce a gui/X window consisting of the following tabs:
+
+* Summary
+* Graphs
+* Applications
+* Tasks
+* Gantt Chart
+
+TUI Monitor
+^^^^^^^^^^^
+The TUI (textual user interface) monitor is one option for monitoring Swift on the console using a curses-like
+library.
+
+The progress of a Swift run can be monitored using the +-ui TUI+ option. For example:
+
 ----
-swift -tc.file tc -sites.file sites.xml -config cf modis04.swift -tui
+swift -ui TUI modis.swift
 ----
+image:TUI-monitor.jpg[]
 
 This will produce a textual user interface with multiple tabs, each showing the following features of the current Swift run:
 
@@ -102,7 +125,7 @@
 * A jobs tab 
 * A transfer tab 
 * A scheduler tab 
-* A Tast statistics tab 
+* A Task statistics tab 
 * A customized tab called 'Ben's View'
 
 Navigation between these tabs can be done using the function keys f2 through f8. 

Deleted: trunk/docs/userguide/swift_monitor.png
===================================================================
(Binary files differ)



From wilde at ci.uchicago.edu  Sat Apr 26 13:18:38 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Sat, 26 Apr 2014 13:18:38 -0500 (CDT)
Subject: [Swift-commit] r7804 - in demo/xsede.2014.0425: app src/md
Message-ID: <20140426181838.AB4D39D04B@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-26 13:18:37 -0500 (Sat, 26 Apr 2014)
New Revision: 7804

Modified:
   demo/xsede.2014.0425/app/mdviz
   demo/xsede.2014.0425/src/md/Makefile
   demo/xsede.2014.0425/src/md/md.c
Log:
Add viz wrapper script. Minor fixes to makefile and md.c

Modified: demo/xsede.2014.0425/app/mdviz
===================================================================
--- demo/xsede.2014.0425/app/mdviz	2014-04-25 21:11:25 UTC (rev 7803)
+++ demo/xsede.2014.0425/app/mdviz	2014-04-26 18:18:37 UTC (rev 7804)
@@ -4,26 +4,16 @@
 vzout=$2
 shift 2
 
-md $* >& md.out     # Run the MD command
+md "$@" >& md.out     # Run the MD command
 
 cat <<EOF >c-ray-cmds
-# spheres
-#	position		radius	color			shininess	reflectivity
-#s	-0.5 -0.3 -1	0.05	1.0 0.2 0.05		50.0	0.1
-#s	0.5 -0.4 0		0.05	0.1 0.85 1.0		50.0	0.1
-
-
 # walls
 s	0 -1000 2		99		0.2 0.2 0.2			0.0	0.0
 
-# bouncing ball
-#s	0 0 2			1		1.0 0.5 0.1			60.0	0.7
-
-# lights...
+# lights
 l	-50 100 -50
-#l	40 40 150
 
-# camera (there can be only one!)
+# camera
 #	position	FOV		target
 c	0 6 -17		45		0 -1 0
 EOF

Modified: demo/xsede.2014.0425/src/md/Makefile
===================================================================
--- demo/xsede.2014.0425/src/md/Makefile	2014-04-25 21:11:25 UTC (rev 7803)
+++ demo/xsede.2014.0425/src/md/Makefile	2014-04-26 18:18:37 UTC (rev 7804)
@@ -1,9 +1,10 @@
 bin = md
+src = md.c
 
 CC = gcc
 #CFLAGS = -O3 -ffast-math
 
-$(bin): $(obj)
+$(bin): $(src)
 	$(CC) -o $@ $(bin).c -lm
 
 .PHONY: clean

Modified: demo/xsede.2014.0425/src/md/md.c
===================================================================
--- demo/xsede.2014.0425/src/md/md.c	2014-04-25 21:11:25 UTC (rev 7803)
+++ demo/xsede.2014.0425/src/md/md.c	2014-04-26 18:18:37 UTC (rev 7804)
@@ -171,12 +171,14 @@
   Report.
 */
   printf ( "\n" );
+  printf ( "  MD: Argument count: %d\n", argc );
   printf ( "  ND, the spatial dimension, is %d\n", nd );
   printf ( "  NP, the number of particles in the simulation, is %d\n", np );
   printf ( "  STEP_NUM, the number of time steps, is %d\n", step_num );
   printf ( "  STEP_PRINT_NUM, the number of snapshots to print, is %d\n", step_print_num );
   printf ( "  DT, the size of each time step, is %f\n", dt );
   printf ( "  MASS, the particle mass, is %f\n", mass );
+  printf ( "  PRINTINFO, the pass-through info to c-ray, is %s\n", printinfo );
   printf ( "  SCALE_FACTOR, the particle position scaling factor, is %f\n", scale_factor );
   printf ( "  SCALE_OFFSET, the particle position scaling offset, is %f\n", scale_offset );
 /*



From wilde at ci.uchicago.edu  Sat Apr 26 16:11:05 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Sat, 26 Apr 2014 16:11:05 -0500 (CDT)
Subject: [Swift-commit] r7805 - in demo/xsede.2014.0425: . part04m src/md
Message-ID: <20140426211105.062899D04B@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-26 16:11:04 -0500 (Sat, 26 Apr 2014)
New Revision: 7805

Added:
   demo/xsede.2014.0425/part04m/
   demo/xsede.2014.0425/part04m/p4.swift
   demo/xsede.2014.0425/part04m/swift.properties
Modified:
   demo/xsede.2014.0425/src/md/md.c
Log:
Added random state var to md command line. Added part04m to generate N simulations with one viz output per sim.

Added: demo/xsede.2014.0425/part04m/p4.swift
===================================================================
--- demo/xsede.2014.0425/part04m/p4.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part04m/p4.swift	2014-04-26 21:11:04 UTC (rev 7805)
@@ -0,0 +1,43 @@
+type file;
+
+(int result) randomInt ()
+{
+  float range = 9999999.0;
+  float rand = java("java.lang.Math","random");
+  string s[] = strsplit(toString(range*rand),"\\.");
+  result = toInt(s[0]);
+}
+
+app (file o, file v) simulation (string npart, string steps, string mass)
+{
+# mdviz  @o @v 3 npart steps 50 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt();
+  mdviz  @o @v 3 npart steps 10 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt();
+}
+
+app (file o) analyze (file s[])
+{
+  mdstats filenames(s) stdout=filename(o);
+}
+
+app (file o) convert (file s[])
+{
+  convert filenames(s) filename(o);
+}
+
+int    nsim   = toInt(arg("nsim","10"));
+string npart  = arg("npart","50");
+string steps  = arg("steps","1000");
+string mass   = arg("mass",".005");
+
+file sims[] <simple_mapper; prefix="output/sim_", suffix=".out">;
+file gifs[] <simple_mapper; prefix="output/sim_", suffix=".gif">;
+
+foreach i in [0:nsim-1] {
+  (sims[i],gifs[i]) = simulation(npart,steps,mass);
+}
+
+file stats_out<"output/average.out">;
+stats_out = analyze(sims);
+
+file viz_all<"output/all.gif">;
+viz_all = convert(gifs);

Added: demo/xsede.2014.0425/part04m/swift.properties
===================================================================
--- demo/xsede.2014.0425/part04m/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part04m/swift.properties	2014-04-26 21:11:04 UTC (rev 7805)
@@ -0,0 +1,2 @@
+site=midway-westmere
+site.midway-westmere.slurm.exclusive=false

Modified: demo/xsede.2014.0425/src/md/md.c
===================================================================
--- demo/xsede.2014.0425/src/md/md.c	2014-04-26 18:18:37 UTC (rev 7804)
+++ demo/xsede.2014.0425/src/md/md.c	2014-04-26 21:11:04 UTC (rev 7805)
@@ -166,6 +166,10 @@
   {
     scale_offset = atof ( argv[9] );
   }
+  if ( 10 < argc )
+  {
+    seed = atof ( argv[10] );
+  }
 
 /*
   Report.
@@ -181,6 +185,7 @@
   printf ( "  PRINTINFO, the pass-through info to c-ray, is %s\n", printinfo );
   printf ( "  SCALE_FACTOR, the particle position scaling factor, is %f\n", scale_factor );
   printf ( "  SCALE_OFFSET, the particle position scaling offset, is %f\n", scale_offset );
+  printf ( "  SEED, the simulation randomization seed, is %d\n", seed );
 /*
   Allocate memory.
 */



From wilde at ci.uchicago.edu  Sun Apr 27 16:16:52 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Sun, 27 Apr 2014 16:16:52 -0500 (CDT)
Subject: [Swift-commit] r7806 - in demo/xsede.2014.0425: . app part04m src/md
Message-ID: <20140427211652.E55039D659@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-27 16:16:51 -0500 (Sun, 27 Apr 2014)
New Revision: 7806

Added:
   demo/xsede.2014.0425/app/mdmaxk
Modified:
   demo/xsede.2014.0425/part04m/p4.swift
   demo/xsede.2014.0425/setup.midway.sh
   demo/xsede.2014.0425/src/md/md.c
Log:
Improvements: added app to find most energetic sim. Make md put trajectory in tarfile. IMprive setup for midway demo/test.

Added: demo/xsede.2014.0425/app/mdmaxk
===================================================================
--- demo/xsede.2014.0425/app/mdmaxk	                        (rev 0)
+++ demo/xsede.2014.0425/app/mdmaxk	2014-04-27 21:16:51 UTC (rev 7806)
@@ -0,0 +1,37 @@
+#! /bin/sh
+
+log() {
+  printf "\nCalled as: $0: $cmdargs\n\n"
+  printf "Start time: "; /bin/date
+  printf "Running as user: "; /usr/bin/id
+  printf "Running on node: "; /bin/hostname
+  printf "Node IP address: "; /bin/hostname -I
+  printf "\nEnvironment:\n\n"
+  printenv | sort
+}
+
+nfiles=$1
+shift
+
+gawk '
+
+BEGIN { maxKe = 0.0; LASTFILE="";}
+
+{
+  if ( $1 == "Step" ) next;
+  if ( (LASTFILE != "") && (FILENAME != LASTFILE)) {
+     printf("%s %f\n", LASTFILE, maxKe )
+     maxKe = 0.0;
+  }
+  Ke = $3 + 0.0;
+  if ( Ke > maxKe ) maxKe = Ke;
+  LASTFILE=FILENAME
+}
+
+END { printf("max Ke for %s is %f\n", LASTFILE, maxKe ) }
+
+
+
+' $* | sort -n -k2 | tail -$nfiles
+
+# log 1>&2


Property changes on: demo/xsede.2014.0425/app/mdmaxk
___________________________________________________________________
Added: svn:executable
   + *

Modified: demo/xsede.2014.0425/part04m/p4.swift
===================================================================
--- demo/xsede.2014.0425/part04m/p4.swift	2014-04-26 21:11:04 UTC (rev 7805)
+++ demo/xsede.2014.0425/part04m/p4.swift	2014-04-27 21:16:51 UTC (rev 7806)
@@ -8,10 +8,10 @@
   result = toInt(s[0]);
 }
 
-app (file o, file v) simulation (string npart, string steps, string mass)
+app (file out, file traj) simulation (string npart, string steps, string mass)
 {
-# mdviz  @o @v 3 npart steps 50 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt();
-  mdviz  @o @v 3 npart steps 10 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt();
+# mdviz  @out @traj 3 npart steps 50 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt();
+  md     3 npart steps 10 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt() @out @traj;
 }
 
 app (file o) analyze (file s[])
@@ -29,15 +29,15 @@
 string steps  = arg("steps","1000");
 string mass   = arg("mass",".005");
 
-file sims[] <simple_mapper; prefix="output/sim_", suffix=".out">;
-file gifs[] <simple_mapper; prefix="output/sim_", suffix=".gif">;
+file sim[] <simple_mapper; prefix="output/sim_", suffix=".out">;
+file trj[] <simple_mapper; prefix="output/sim_", suffix=".trj.tgz">;
 
 foreach i in [0:nsim-1] {
-  (sims[i],gifs[i]) = simulation(npart,steps,mass);
+  (sim[i],trj[i]) = simulation(npart,steps,mass);
 }
 
 file stats_out<"output/average.out">;
-stats_out = analyze(sims);
+stats_out = analyze(sim);
 
-file viz_all<"output/all.gif">;
-viz_all = convert(gifs);
+#file viz_all<"output/all.gif">;
+#viz_all = convert(gifs);

Modified: demo/xsede.2014.0425/setup.midway.sh
===================================================================
--- demo/xsede.2014.0425/setup.midway.sh	2014-04-26 21:11:04 UTC (rev 7805)
+++ demo/xsede.2014.0425/setup.midway.sh	2014-04-27 21:16:51 UTC (rev 7806)
@@ -1,6 +1,11 @@
 # Load modules
-module unload swift
-module load swift/0.95-RC1
+#module unload swift
+#module load swift/0.95-RC1
+
+module load ant
+
+PATH=/home/wilde/swift/src/trunk/cog/modules/swift/dist/swift-svn/bin:$PATH
+
 echo Swift version is $(swift -version)
 
 # Add applications to $PATH

Modified: demo/xsede.2014.0425/src/md/md.c
===================================================================
--- demo/xsede.2014.0425/src/md/md.c	2014-04-26 21:11:04 UTC (rev 7805)
+++ demo/xsede.2014.0425/src/md/md.c	2014-04-27 21:16:51 UTC (rev 7806)
@@ -18,9 +18,11 @@
   double acc[], double mass, double dt );
 
 double scale_factor = 2.5, scale_offset = -2.0;
-char default_printinfo[100] = "0.05 1.0 0.2 0.05 50.0 0.1";
-char *printinfo = default_printinfo;
+char *printinfo = "0.05 1.0 0.2 0.05 50.0 0.1";
+char *outfile = "md.dat";
+char *trjfile = "md.trj.tgz";
 
+
 /******************************************************************************/
 
 int main ( int argc, char *argv[] )
@@ -41,7 +43,7 @@
 
   Usage:
 
-    md nd np step_num print_step_num dt mass printinfo scale_factor scale_offset
+    md nd np step_num print_step_num dt mass printinfo scale_factor scale_offset seed outFile trajectoryFile
     where
     * nd is the spatial dimension (2 or 3);
     * np is the number of particles (500, for instance);
@@ -51,6 +53,8 @@
     * mass is particle mass;
     * printinfo is a string to append to each particle coord
     * scale_offset and scale_factor are used to scale particle positions for logging/rendering (FIXME)
+    * seed sets the initial configuration
+    
 
   Licensing:
 
@@ -140,7 +144,7 @@
   }
   /*
 	Get any additional args (command-line only)
-	md nd np step_num [ step__print_num dt mass printinfo scale_factor scale_offset ]
+	md nd np step_num [ step__print_num dt mass printinfo scale_factor scale_offset randomseed outfile trjfile ]
   */
   if ( 4 < argc )
   {
@@ -170,6 +174,14 @@
   {
     seed = atof ( argv[10] );
   }
+  if ( 11 < argc )
+  {
+    outfile = argv[11];
+  }
+  if ( 12 < argc )
+  {
+    trjfile = argv[12];
+  }
 
 /*
   Report.
@@ -232,7 +244,7 @@
   printf ( "                Energy P        Energy K       Relative Energy Error\n" );
   printf ( "\n" );
 
-  FILE *ofile = fopen("md.dat","w");
+  FILE *ofile = fopen(outfile,"w");
   fprintf (ofile, "      Step      Potential       Kinetic        RelativeErr\n" );
 
   step_print = 0;
@@ -276,6 +288,9 @@
   free ( force );
   free ( pos );
   free ( vel );
+  char tarcmd[2000];
+  sprintf(tarcmd,"tar zcf %s md??.trj",trjfile);
+  system(tarcmd);
 /*
   Terminate.
 */



From wilde at ci.uchicago.edu  Sun Apr 27 16:22:48 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Sun, 27 Apr 2014 16:22:48 -0500 (CDT)
Subject: [Swift-commit] r7807 - in demo/xsede.2014.0425: . part06m
Message-ID: <20140427212248.A35D39D659@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-27 16:22:48 -0500 (Sun, 27 Apr 2014)
New Revision: 7807

Added:
   demo/xsede.2014.0425/part06m/
   demo/xsede.2014.0425/part06m/p4.swift
   demo/xsede.2014.0425/part06m/swift.properties
Log:
cloning part6 for demo with viz.

Added: demo/xsede.2014.0425/part06m/p4.swift
===================================================================
--- demo/xsede.2014.0425/part06m/p4.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part06m/p4.swift	2014-04-27 21:22:48 UTC (rev 7807)
@@ -0,0 +1,43 @@
+type file;
+
+(int result) randomInt ()
+{
+  float range = 9999999.0;
+  float rand = java("java.lang.Math","random");
+  string s[] = strsplit(toString(range*rand),"\\.");
+  result = toInt(s[0]);
+}
+
+app (file out, file traj) simulation (string npart, string steps, string mass)
+{
+# mdviz  @out @traj 3 npart steps 50 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt();
+  md     3 npart steps 10 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt() @out @traj;
+}
+
+app (file o) analyze (file s[])
+{
+  mdstats filenames(s) stdout=filename(o);
+}
+
+app (file o) convert (file s[])
+{
+  convert filenames(s) filename(o);
+}
+
+int    nsim   = toInt(arg("nsim","10"));
+string npart  = arg("npart","50");
+string steps  = arg("steps","1000");
+string mass   = arg("mass",".005");
+
+file sim[] <simple_mapper; prefix="output/sim_", suffix=".out">;
+file trj[] <simple_mapper; prefix="output/sim_", suffix=".trj.tgz">;
+
+foreach i in [0:nsim-1] {
+  (sim[i],trj[i]) = simulation(npart,steps,mass);
+}
+
+file stats_out<"output/average.out">;
+stats_out = analyze(sim);
+
+#file viz_all<"output/all.gif">;
+#viz_all = convert(gifs);

Added: demo/xsede.2014.0425/part06m/swift.properties
===================================================================
--- demo/xsede.2014.0425/part06m/swift.properties	                        (rev 0)
+++ demo/xsede.2014.0425/part06m/swift.properties	2014-04-27 21:22:48 UTC (rev 7807)
@@ -0,0 +1,2 @@
+site=midway-westmere
+site.midway-westmere.slurm.exclusive=false



From ketan at ci.uchicago.edu  Mon Apr 28 15:27:08 2014
From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu)
Date: Mon, 28 Apr 2014 15:27:08 -0500 (CDT)
Subject: [Swift-commit] r7808 - SwiftApps/swift-galaxy
Message-ID: <20140428202708.1303D9D587@svn.ci.uchicago.edu>

Author: ketan
Date: 2014-04-28 15:27:07 -0500 (Mon, 28 Apr 2014)
New Revision: 7808

Modified:
   SwiftApps/swift-galaxy/README
Log:
update galaxy swift tutorial

Modified: SwiftApps/swift-galaxy/README
===================================================================
--- SwiftApps/swift-galaxy/README	2014-04-27 21:22:48 UTC (rev 7807)
+++ SwiftApps/swift-galaxy/README	2014-04-28 20:27:07 UTC (rev 7808)
@@ -39,6 +39,9 @@
 
 Installation
 -------------
+This installation recipe assumes that you have Galaxy preinstalled. If not,
+installation instructions for Galaxy can be found
+https://wiki.galaxyproject.org/Admin/GetGalaxy[here].
 
 Step 0. Checkout the swift-galaxy SVN directory as follows:
 
@@ -46,18 +49,18 @@
 svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/swift-galaxy
 ----
 
-Step 1. Create a symbolic link of the swift directory in the current directory from galaxy's top
-level directory. For example, if Galaxy top level directory is galaxy-dist and
-swift directory is /home/ketan/SwiftApps/swift-galaxy/swift:
+Step 1. Create a symbolic link to the +swift-galaxy/swift+ directory from
+galaxy's +tools+ directory. For example, if Galaxy top level directory is
++galaxy-dist+ and swift directory is $HOME/swift-galaxy/swift:
 
 ----
-cd galaxy-dist
-ln -s /home/ketan/SwiftApps/swift-galaxy/swift swift
+cd galaxy-dist/tools
+ln -s $HOME/swift-galaxy/swift swift
 ----
 
 The advantage of symbolic links is that as users update the tool directory via
-`svn up`, they need not worry about reinstallation. Everything gets propagated
-to Galaxy.
+`svn up`, they need not worry about reinstallation. Updates in installation
+gets automatically propagated to Galaxy.
 
 Step 2. Add the tool definitions of the tools contained in this Swift to
 Galaxy's tool_conf.xml. This can be done by copying the text in
@@ -66,7 +69,7 @@
 
 ----
 <section name="swift" id="swift">
- <tool file="swift/genswift.xml" />
+ <tool file="swift/tryswift.xml" />
  <tool file="swift/genericswift.xml" />
  <tool file="swift/swiftforeach.xml" />
 </section>
@@ -159,7 +162,7 @@
 cp -r swift-galaxy/swift /location/of/galaxy-server/tools/
 ----
 
-Step 3. Edit the tool_conf.xml file present at the top-level of galaxy directory tree to add the following lines between the <toolbox> and </toolbox> xml tags:
+Step 3. Edit the +tool_conf.xml+ file present at the top-level of galaxy directory tree to add the following lines between the +<toolbox>+ and +</toolbox>+ xml tags:
 
 ----
 <section name="swift-galaxy" id="swift-galaxy">



From wilde at ci.uchicago.edu  Wed Apr 30 10:01:59 2014
From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu)
Date: Wed, 30 Apr 2014 10:01:59 -0500 (CDT)
Subject: [Swift-commit] r7809 - in demo/xsede.2014.0425: app part06m
Message-ID: <20140430150159.4F6389D04B@svn.ci.uchicago.edu>

Author: wilde
Date: 2014-04-30 10:01:57 -0500 (Wed, 30 Apr 2014)
New Revision: 7809

Added:
   demo/xsede.2014.0425/part06m/p6m.swift
Removed:
   demo/xsede.2014.0425/part06m/p4.swift
Modified:
   demo/xsede.2014.0425/app/mdmaxk
Log:
Snapshot of devel changes for MD-based tutorial/demo

Modified: demo/xsede.2014.0425/app/mdmaxk
===================================================================
--- demo/xsede.2014.0425/app/mdmaxk	2014-04-28 20:27:07 UTC (rev 7808)
+++ demo/xsede.2014.0425/app/mdmaxk	2014-04-30 15:01:57 UTC (rev 7809)
@@ -28,10 +28,11 @@
   LASTFILE=FILENAME
 }
 
-END { printf("max Ke for %s is %f\n", LASTFILE, maxKe ) }
+#END { printf("max Ke for %s is %f\n", LASTFILE, maxKe ) }
+ END { printf("%s %f\n", LASTFILE, maxKe ) }
 
 
 
-' $* | sort -n -k2 | tail -$nfiles
+' $* | sort -n -k2 | tail -$nfiles | awk '{print $1}'
 
 # log 1>&2

Deleted: demo/xsede.2014.0425/part06m/p4.swift
===================================================================
--- demo/xsede.2014.0425/part06m/p4.swift	2014-04-28 20:27:07 UTC (rev 7808)
+++ demo/xsede.2014.0425/part06m/p4.swift	2014-04-30 15:01:57 UTC (rev 7809)
@@ -1,43 +0,0 @@
-type file;
-
-(int result) randomInt ()
-{
-  float range = 9999999.0;
-  float rand = java("java.lang.Math","random");
-  string s[] = strsplit(toString(range*rand),"\\.");
-  result = toInt(s[0]);
-}
-
-app (file out, file traj) simulation (string npart, string steps, string mass)
-{
-# mdviz  @out @traj 3 npart steps 50 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt();
-  md     3 npart steps 10 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt() @out @traj;
-}
-
-app (file o) analyze (file s[])
-{
-  mdstats filenames(s) stdout=filename(o);
-}
-
-app (file o) convert (file s[])
-{
-  convert filenames(s) filename(o);
-}
-
-int    nsim   = toInt(arg("nsim","10"));
-string npart  = arg("npart","50");
-string steps  = arg("steps","1000");
-string mass   = arg("mass",".005");
-
-file sim[] <simple_mapper; prefix="output/sim_", suffix=".out">;
-file trj[] <simple_mapper; prefix="output/sim_", suffix=".trj.tgz">;
-
-foreach i in [0:nsim-1] {
-  (sim[i],trj[i]) = simulation(npart,steps,mass);
-}
-
-file stats_out<"output/average.out">;
-stats_out = analyze(sim);
-
-#file viz_all<"output/all.gif">;
-#viz_all = convert(gifs);

Copied: demo/xsede.2014.0425/part06m/p6m.swift (from rev 7807, demo/xsede.2014.0425/part06m/p4.swift)
===================================================================
--- demo/xsede.2014.0425/part06m/p6m.swift	                        (rev 0)
+++ demo/xsede.2014.0425/part06m/p6m.swift	2014-04-30 15:01:57 UTC (rev 7809)
@@ -0,0 +1,54 @@
+type file;
+
+(int result) randomInt ()
+{
+  float range = 9999999.0;
+  float rand = java("java.lang.Math","random");
+  string s[] = strsplit(toString(range*rand),"\\.");
+  result = toInt(s[0]);
+}
+
+app (file out, file traj) simulation (string npart, string steps, string mass, int randval)
+{
+# mdviz  @out @traj 3 npart steps 50 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randomInt();
+  md     3 npart steps 10 ".0001" mass "0.1 1.0 0.2 0.05 50.0 0.1" 2.5 2.0 randval @out @traj;
+}
+
+app (file out) genconfig ()
+{
+  simulate "-n" 1 "-r" 1000000 stdout=filename(out);
+}
+
+app (file o) analyze (file s[])
+{
+  mdstats filenames(s) stdout=filename(o);
+}
+
+app (file o) maxkinetic (file s[])
+{
+  mdmaxk "3" filenames(s) stdout=filename(o);
+}
+
+app (file o) convert (file s[])
+{
+  convert filenames(s) filename(o);
+}
+
+int    nsim  = toInt(arg("nsim","10"));
+string npart = arg("npart","50");
+string steps = arg("steps","1000");
+string mass  = arg("mass",".005");
+
+file sim[] <simple_mapper; prefix="output/sim_", suffix=".out">;
+file trj[] <simple_mapper; prefix="output/sim_", suffix=".trj.tgz">;
+
+foreach i in [0:nsim-1] {
+  int startstate = readData(genconfig());
+  (sim[i],trj[i]) = simulation(npart,steps,mass,startstate);
+}
+
+file stats_out<"output/average.out">;
+stats_out = analyze(sim);
+
+#file viz_all<"output/all.gif">;
+#viz_all = convert(gifs);