From hategan at ci.uchicago.edu Wed Apr 3 14:46:27 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 3 Apr 2013 14:46:27 -0500 (CDT) Subject: [Swift-commit] r6411 - trunk/libexec Message-ID: <20130403194627.C18129CD0A@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-03 14:46:27 -0500 (Wed, 03 Apr 2013) New Revision: 6411 Modified: trunk/libexec/vdl-int.k Log: fixed error file processing in status.mode=files Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2013-03-28 16:10:02 UTC (rev 6410) +++ trunk/libexec/vdl-int.k 2013-04-03 19:46:27 UTC (rev 6411) @@ -46,15 +46,15 @@ log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") ) sequential( - try ( - msg = checkErrorFile(rhost, wfdir, jobid, jobdir) + msg := try ( + checkErrorFile(rhost, wfdir, jobid, jobdir) sequential ( log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing") throw("No status file was found. Check the shared filesystem on {rhost}") ) ) + throw(msg) ) - throw(checkErrorFile(rhost, wfdir, jobid, jobdir)) ) ) From hategan at ci.uchicago.edu Wed Apr 3 14:48:57 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 3 Apr 2013 14:48:57 -0500 (CDT) Subject: [Swift-commit] r6412 - branches/release-0.94/libexec Message-ID: <20130403194857.AD51A9CD0A@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-03 14:48:57 -0500 (Wed, 03 Apr 2013) New Revision: 6412 Modified: branches/release-0.94/libexec/vdl-int.k Log: fix for bug 971 from trunk Modified: branches/release-0.94/libexec/vdl-int.k =================================================================== --- branches/release-0.94/libexec/vdl-int.k 2013-04-03 19:46:27 UTC (rev 6411) +++ branches/release-0.94/libexec/vdl-int.k 2013-04-03 19:48:57 UTC (rev 6412) @@ -46,15 +46,15 @@ log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") ) sequential( - try ( - msg = checkErrorFile(rhost, wfdir, jobid, jobdir) + msg := try ( + checkErrorFile(rhost, wfdir, jobid, jobdir) sequential ( log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing") throw("No status file was found. Check the shared filesystem on {rhost}") ) ) + throw(msg) ) - throw(checkErrorFile(rhost, wfdir, jobid, jobdir)) ) ) From hategan at ci.uchicago.edu Wed Apr 3 21:44:38 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 3 Apr 2013 21:44:38 -0500 (CDT) Subject: [Swift-commit] r6413 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20130404024438.6BB8A9CD0A@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-03 21:44:38 -0500 (Wed, 03 Apr 2013) New Revision: 6413 Modified: trunk/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: replace parent dir references with "__parent__" and references to the root dir with "__root__" when figuring out remote paths; the last one is an older issue of the potential clash between "/dir/file" and "dir/file" if the strategy is only to remove the leading slash Modified: trunk/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java 2013-04-03 19:48:57 UTC (rev 6412) +++ trunk/src/org/griphyn/vdl/karajan/lib/CacheUnlockFiles.java 2013-04-04 02:44:38 UTC (rev 6413) @@ -53,7 +53,7 @@ Iterator i = pairs.iterator(); while (i.hasNext()) { String file = (String) i.next(); - File f = new File(new AbsFile(file).getPath(), dir, host, 0); + File f = new File(PathUtils.remotePathName(new AbsFile(file).getPath()), dir, host, 0); CacheReturn cr = cache.unlockEntry(f, TypeUtil.toBoolean(FORCE.getValue(stack))); rem.addAll(cr.remove); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2013-04-03 19:48:57 UTC (rev 6412) +++ trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2013-04-04 02:44:38 UTC (rev 6413) @@ -47,16 +47,7 @@ AbsFile af = new AbsFile(path); if ("file".equals(af.getProtocol())) { String dir = af.getDir(); - // there could be a clash here since - // "/a/b/c.txt" would be remotely the same - // as "a/b/c.txt". Perhaps absolute paths - // should have a unique prefix. - if (dir.startsWith("/") && dir.length() != 1) { - ret.append(dir.substring(1)); - } - else if (dir.length() != 0) { - ret.append(dir); - } + ret.append(PathUtils.remotePathName(dir)); } else { // also prepend host name to the path Modified: trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2013-04-03 19:48:57 UTC (rev 6412) +++ trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2013-04-04 02:44:38 UTC (rev 6413) @@ -49,14 +49,152 @@ public String vdl_reldirname(VariableStack stack) throws ExecutionException { String path = TypeUtil.toString(PATH.getValue(stack)); String dir = new AbsFile(path).getDir(); - if (dir.startsWith("/")) { - return dir.substring(1); + return remotePathName(dir); + } + + private static final char EOL = '\0'; + /** + * Replace leading slash if present with "__root__" and replace + * parent dir references with "__parent__" + */ + public static String remotePathName(String dir) { + if (dir.length() == 0) { + return dir; } + StringBuilder sb = null; + + // parse it by hand to avoid creating too much string object garbage + boolean modified = false; + boolean nondot = false; + int dotcount = 0; + int start = 0; + for (int i = 0; i <= dir.length(); i++) { + boolean skip = false; + char c; + if (i == dir.length()) { + c = EOL; + } + else { + c = dir.charAt(i); + } + switch (c) { + case EOL: + case '/': + if (i == 0) { + sb = new StringBuilder(); + sb.append("__root__/"); + skip = true; + modified = true; + } + else if (nondot) { + // do nothing + } + else { + // only dots. If zero or one, remove completely ("//", "/./") + switch (dotcount) { + case 0: + case 1: + case 2: + modified = true; + skip = true; + if (sb == null) { + sb = new StringBuilder(); + append(sb, dir, 0, i - dotcount); + } + if (dotcount == 2) { + sb.append("__parent__"); + if (c != EOL) { + sb.append('/'); + } + } + default: + // pass along + } + } + nondot = false; + dotcount = 0; + break; + case '.': + if (nondot) { + // a path element containing a dot among other things + // so leave it alone + } + else { + dotcount++; + skip = true; + } + break; + default: + nondot = true; + if (dotcount > 0) { + if (modified) { + for (int j = 0; j < dotcount; j++) { + sb.append('.'); + } + } + dotcount = 0; + } + } + if (modified) { + if (sb == null) { + sb = new StringBuilder(); + append(sb, dir, 0, i - dotcount); + } + if (!skip && c != EOL) { + sb.append(c); + } + } + } + if (modified) { + return sb.toString(); + } else { return dir; } } + public static void testMakeRelative(String str, String expected, boolean samestr) { + String result = remotePathName(str); + if (!result.equals(expected)) { + throw new RuntimeException("input: '" + str + "', expected: '" + expected + "', result: '" + result + "'"); + } + if (samestr && (str != result)) { + throw new RuntimeException("Expected same string for '" + str + "'"); + } + System.out.println("OK '" + str + "' -> '" + result + "'"); + } + + public static void main(String[] args) { + testMakeRelative("onething", "onething", true); + testMakeRelative("two/things", "two/things", true); + testMakeRelative("/absolute/path", "__root__/absolute/path", false); + testMakeRelative("../in/the/beginning", "__parent__/in/the/beginning", false); + testMakeRelative("in/the/../middle", "in/the/__parent__/middle", false); + // nonsensical, but meh + testMakeRelative("/../in/the/beginning/absolute", "__root__/__parent__/in/the/beginning/absolute", false); + testMakeRelative("/in/the/../middle/absolute", "__root__/in/the/__parent__/middle/absolute", false); + testMakeRelative("../in/../many/../places", "__parent__/in/__parent__/many/__parent__/places", false); + testMakeRelative("/../in/../many/../places/../absolute", "__root__/__parent__/in/__parent__/many/__parent__/places/__parent__/absolute", false); + testMakeRelative("a/single/./dot", "a/single/dot", false); + testMakeRelative("double//slash", "double/slash", false); + testMakeRelative("./single/dot/at/start", "single/dot/at/start", false); + testMakeRelative("/./single/dot/at/start/absolute", "__root__/single/dot/at/start/absolute", false); + testMakeRelative("multiple/single/././././dots", "multiple/single/dots", false); + // technically this isn't valid, but that's not our problem + testMakeRelative("three/.../dots", "three/.../dots", true); + testMakeRelative("two/..valid/dots", "two/..valid/dots", true); + testMakeRelative("more/val..id/dots", "more/val..id/dots", true); + testMakeRelative("/everything/./in/../../one//single../../path", "__root__/everything/in/__parent__/__parent__/one/single../__parent__/path", false); + testMakeRelative("..", "__parent__", false); + testMakeRelative("ends/in/..", "ends/in/__parent__", false); + } + + private static void append(StringBuilder sb, String str, int begin, int end) { + for (int i = begin; i < end; i++) { + sb.append(str.charAt(i)); + } + } + static { setArguments("vdl_basename", new Arg[] { PATH }); } @@ -125,10 +263,10 @@ } public Object[] vdl_pathnames(VariableStack stack) throws ExecutionException { - List l = new ArrayList(); - Iterator i = TypeUtil.toIterator(FILES.getValue(stack)); + List l = new ArrayList(); + Iterator i = TypeUtil.toIterator(FILES.getValue(stack)); while (i.hasNext()) { - l.add(new AbsFile((String) i.next()).getPath()); + l.add(remotePathName(new AbsFile((String) i.next()).getPath())); } return l.toArray(new String[0]); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2013-04-03 19:48:57 UTC (rev 6412) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2013-04-04 02:44:38 UTC (rev 6413) @@ -39,14 +39,12 @@ import org.globus.cog.karajan.util.TypeUtil; import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.KarajanRuntimeException; -import org.globus.cog.karajan.workflow.futures.Future; import org.globus.cog.karajan.workflow.nodes.SequentialWithArguments; import org.globus.cog.karajan.workflow.nodes.restartLog.RestartLog; import org.globus.swift.catalog.TCEntry; import org.globus.swift.catalog.transformation.File; import org.globus.swift.catalog.types.TCType; import org.griphyn.vdl.karajan.AssertFailedException; -import org.griphyn.vdl.karajan.FutureWrapper; import org.griphyn.vdl.karajan.Loader; import org.griphyn.vdl.karajan.TCCache; import org.griphyn.vdl.karajan.functions.ConfigProperty; @@ -337,12 +335,7 @@ */ public static String relativize(String name) { name = pathOnly(name); - if (name != null && name.length() > 0 && name.charAt(0) == '/') { - return name.substring(1); - } - else { - return name; - } + return PathUtils.remotePathName(name); } protected static Map getLogData(VariableStack stack) throws ExecutionException { From davidk at ci.uchicago.edu Fri Apr 5 10:58:06 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Fri, 5 Apr 2013 10:58:06 -0500 (CDT) Subject: [Swift-commit] r6414 - www/support Message-ID: <20130405155806.5B2609CD0A@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-05 10:58:06 -0500 (Fri, 05 Apr 2013) New Revision: 6414 Modified: www/support/index.php Log: Add a link to swift-support Modified: www/support/index.php =================================================================== --- www/support/index.php 2013-04-04 02:44:38 UTC (rev 6413) +++ www/support/index.php 2013-04-05 15:58:06 UTC (rev 6414) @@ -34,8 +34,8 @@

Mailing Lists

- - + + @@ -56,7 +56,7 @@ + + + + + + +
ListList Info
(subscribe here)
ListList Info
(subscribe here)
Archives Purpose
- swift-developer + swift-devel list page @@ -66,9 +66,21 @@ - Swift developers mailing list. + The Swift developers mailing list.
+ swift-support + + Contact the Swift support team directly with any questions + +

 

 

From wilde at ci.uchicago.edu Mon Apr 8 19:04:05 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Mon, 8 Apr 2013 19:04:05 -0500 (CDT) Subject: [Swift-commit] r6415 - SwiftTutorials Message-ID: <20130409000405.489449CCFD@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-08 19:04:05 -0500 (Mon, 08 Apr 2013) New Revision: 6415 Added: SwiftTutorials/UofC_2013-04-09/ Log: Initial dir. From ketan at ci.uchicago.edu Mon Apr 8 23:16:40 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 8 Apr 2013 23:16:40 -0500 (CDT) Subject: [Swift-commit] r6416 - in trunk: libexec src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20130409041640.AA21D9CCFD@svn.ci.uchicago.edu> Author: ketan Date: 2013-04-08 23:16:40 -0500 (Mon, 08 Apr 2013) New Revision: 6416 Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java Modified: trunk/libexec/vdl-lib.xml trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java Log: adding extractFloat function to read floating point numbers from file a la extractInt Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2013-04-09 00:04:05 UTC (rev 6415) +++ trunk/libexec/vdl-lib.xml 2013-04-09 04:16:40 UTC (rev 6416) @@ -3,6 +3,7 @@ + Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2013-04-09 00:04:05 UTC (rev 6415) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2013-04-09 04:16:40 UTC (rev 6416) @@ -185,6 +185,7 @@ add(functionsMap, "arg", returns(STRING), args(STRING, optional(STRING))); add(functionsMap, "extractInt", returns(INT), args(ANY)); + add(functionsMap, "extractFloat", returns(FLOAT), args(ANY)); add(functionsMap, "filename", returns(STRING), args(ANY)); add(functionsMap, "filenames", returns(STRING_ARRAY), args(ANY)); add(functionsMap, "dirname", returns(STRING), args(ANY)); Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ExtractFloat.java 2013-04-09 04:16:40 UTC (rev 6416) @@ -0,0 +1,61 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.griphyn.vdl.karajan.lib.swiftscript; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.mapping.AbstractDataNode; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.type.Types; + + +public class ExtractFloat extends VDLFunction { + static { + setArguments(ExtractFloat.class, new Arg[] { PA_VAR }); + } + + public Object function(VariableStack stack) throws ExecutionException { + AbstractDataNode handle = null; + try { + handle = (AbstractDataNode) PA_VAR.getValue(stack); + handle.waitFor(); + + String fn = argList(filename(handle), true); + Reader freader = new FileReader(fn); + BufferedReader breader = new BufferedReader(freader); + String str = breader.readLine(); + freader.close(); + DSHandle result = new RootDataNode(Types.FLOAT, Float.parseFloat(str)); + int provid = VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, result, "extractfloat"); + VDLFunction.logProvenanceParameter(provid, handle, "filename"); + return result; + } + catch (IOException ioe) { + throw new ExecutionException("Reading floatingpoint content of file", ioe); + } + } +} From ketan at ci.uchicago.edu Mon Apr 8 23:32:52 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 8 Apr 2013 23:32:52 -0500 (CDT) Subject: [Swift-commit] r6417 - trunk/docs/userguide Message-ID: <20130409043252.DB31F9CCFD@svn.ci.uchicago.edu> Author: ketan Date: 2013-04-08 23:32:52 -0500 (Mon, 08 Apr 2013) New Revision: 6417 Modified: trunk/docs/userguide/app_procedures Log: adding extractfloat bits to userguide Modified: trunk/docs/userguide/app_procedures =================================================================== --- trunk/docs/userguide/app_procedures 2013-04-09 04:16:40 UTC (rev 6416) +++ trunk/docs/userguide/app_procedures 2013-04-09 04:32:52 UTC (rev 6417) @@ -297,6 +297,10 @@ @extractInt(file) will read the specified file, parse an integer from the file contents and return that integer. + at extractFloat +~~~~~~~~~~~~~ +Similar to @extractInt, @extractFloat(file) will read the specified file, parse a float from +the file contents and return that float. @filename ~~~~~~~~~ @@ -464,7 +468,7 @@ Where the contents of the "emps.txt" file are: ---- -name id address +name id loc Thomas 2222 Chicago Gina 3333 Boston Anne 4444 Houston @@ -520,7 +524,6 @@ trace will log its parameters. By default these will appear on both stdout and in the run log file. Some formatting occurs to produce the log message. The particular output format should not be relied upon. -(since Swift 0.4) tracef ~~~~~~ From wilde at ci.uchicago.edu Tue Apr 9 01:58:55 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 9 Apr 2013 01:58:55 -0500 (CDT) Subject: [Swift-commit] r6419 - SwiftTutorials/UofC_2013-04-09/data Message-ID: <20130409065855.32D039CCFD@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 01:58:54 -0500 (Tue, 09 Apr 2013) New Revision: 6419 Removed: SwiftTutorials/UofC_2013-04-09/data/sat/ Log: Remove large test data. From wilde at ci.uchicago.edu Tue Apr 9 09:46:48 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 9 Apr 2013 09:46:48 -0500 (CDT) Subject: [Swift-commit] r6420 - in SwiftTutorials/UofC_2013-04-09: bin part04 part05 Message-ID: <20130409144648.57DEF9CCFD@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 09:46:48 -0500 (Tue, 09 Apr 2013) New Revision: 6420 Added: SwiftTutorials/UofC_2013-04-09/bin/highest.sh SwiftTutorials/UofC_2013-04-09/bin/lowest.sh Modified: SwiftTutorials/UofC_2013-04-09/bin/cleanup SwiftTutorials/UofC_2013-04-09/bin/random2.sh SwiftTutorials/UofC_2013-04-09/bin/random3.sh SwiftTutorials/UofC_2013-04-09/bin/random4.sh SwiftTutorials/UofC_2013-04-09/part04/README SwiftTutorials/UofC_2013-04-09/part05/README SwiftTutorials/UofC_2013-04-09/part05/p5.swift Log: Added new parts for arg passing and cluster execution. Modified: SwiftTutorials/UofC_2013-04-09/bin/cleanup =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/cleanup 2013-04-09 06:58:54 UTC (rev 6419) +++ SwiftTutorials/UofC_2013-04-09/bin/cleanup 2013-04-09 14:46:48 UTC (rev 6420) @@ -2,14 +2,14 @@ echo Removing: echo -ls -ltd *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log applist config sites.xml pathlist 2>/dev/null +ls -ltd *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log 2>/dev/null echo echo Enter y to proceed: read ok if [ _$ok = _y -o _$ok = _Y -o _$ok = _yes ]; then echo set -x - rm -rf *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log applist config sites.xml pathlist + rm -rf *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log else echo echo No files removed. Added: SwiftTutorials/UofC_2013-04-09/bin/highest.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/highest.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/highest.sh 2013-04-09 14:46:48 UTC (rev 6420) @@ -0,0 +1,2 @@ +#! /bin/sh +sort -nr | head -$1 Property changes on: SwiftTutorials/UofC_2013-04-09/bin/highest.sh ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/lowest.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/lowest.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/lowest.sh 2013-04-09 14:46:48 UTC (rev 6420) @@ -0,0 +1,2 @@ +#! /bin/sh +sort -n | head -$1 Property changes on: SwiftTutorials/UofC_2013-04-09/bin/lowest.sh ___________________________________________________________________ Added: svn:executable + * Modified: SwiftTutorials/UofC_2013-04-09/bin/random2.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/random2.sh 2013-04-09 06:58:54 UTC (rev 6419) +++ SwiftTutorials/UofC_2013-04-09/bin/random2.sh 2013-04-09 14:46:48 UTC (rev 6420) @@ -1,5 +1,7 @@ #! /bin/sh -range=$1 +runtime=${1:-0} -echo $(($RANDOM%range)) +sleep $runtime + +echo $(($RANDOM)) Modified: SwiftTutorials/UofC_2013-04-09/bin/random3.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/random3.sh 2013-04-09 06:58:54 UTC (rev 6419) +++ SwiftTutorials/UofC_2013-04-09/bin/random3.sh 2013-04-09 14:46:48 UTC (rev 6420) @@ -1,9 +1,10 @@ #! /bin/sh -range=$1 -n=$2 +runtime=${1:-0} +range=$2 -for ((i=0;i was a shorthand for but the latter is more flexible. @@ -23,22 +23,20 @@ Try: -- changing the output names +- Adjust the script to take the number of simulations to do from the command line -- @arg("argname") returns arguments after the .swift script on the swift command line: + @arg("argname") returns arguments after the .swift script on the swift command line: swift -tc.file tc p4.swift -myarg1=value1 ... + @arg("argname","default") returns "default" if the argname is not given on the command line + @toInt() converts strings to integers: @toInt(@arg("myArg1")) - Adjust the script to take the number of simulations to do from the command line - trace(expr1,...) traces expressions on stdout. Try inserting a few traces. But remember: Swift is *very* concurrent! - - - Modified: SwiftTutorials/UofC_2013-04-09/part05/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part05/README 2013-04-09 06:58:54 UTC (rev 6419) +++ SwiftTutorials/UofC_2013-04-09/part05/README 2013-04-09 14:46:48 UTC (rev 6420) @@ -12,7 +12,7 @@ Run the program: -$ swift -tc.file tc p4.swift +$ swift -tc.file tc p5.swift Look at the output: Modified: SwiftTutorials/UofC_2013-04-09/part05/p5.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part05/p5.swift 2013-04-09 06:58:54 UTC (rev 6419) +++ SwiftTutorials/UofC_2013-04-09/part05/p5.swift 2013-04-09 14:46:48 UTC (rev 6420) @@ -12,7 +12,9 @@ file sims[]; -foreach i in [0:9] { +int nsim = @toInt(@arg("nsim","10")); + +foreach i in [0:nsim-1] { file simout ; simout = mysim(); sims[i] = simout; From wilde at ci.uchicago.edu Tue Apr 9 09:48:47 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 9 Apr 2013 09:48:47 -0500 (CDT) Subject: [Swift-commit] r6421 - in SwiftTutorials/UofC_2013-04-09: . part06 part07 Message-ID: <20130409144847.5065A9CCFD@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 09:48:47 -0500 (Tue, 09 Apr 2013) New Revision: 6421 Added: SwiftTutorials/UofC_2013-04-09/part06/ SwiftTutorials/UofC_2013-04-09/part06/README SwiftTutorials/UofC_2013-04-09/part06/p6.swift SwiftTutorials/UofC_2013-04-09/part06/tc SwiftTutorials/UofC_2013-04-09/part07/ SwiftTutorials/UofC_2013-04-09/part07/README SwiftTutorials/UofC_2013-04-09/part07/p7.swift SwiftTutorials/UofC_2013-04-09/part07/sites.xml SwiftTutorials/UofC_2013-04-09/part07/tc Log: Added new parts for arg passing and cluster execution. Added: SwiftTutorials/UofC_2013-04-09/part06/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part06/README (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part06/README 2013-04-09 14:48:47 UTC (rev 6421) @@ -0,0 +1,28 @@ + +Swift script - add arguments to the "simulation" program to control number of steps, + range of output, and number of output lines, + + +Test new versions of the "simulation" program: + + random.sh # original + random2.sh [runtime] # runtime in seconds; defaults to 1 sec + random3.sh [runtime] [range] # requests random numbers in {0:range-1} + random4.sh [runtime] [range] [count] # requests "count" random numbers in {0:range} + +p6.swift changes to random2 for the simulation program and adds an argument. + +Run the program: + +$ swift -tc.file tc p6.swift -steps 3 # each simulation takes 3 seconds + +Next we need to use the parallel cluster for our simulations! + +Try: + +- change to random3.sh and add the extra argument "range" + +- change to random4.sh and add yet another argument "count" + + + Added: SwiftTutorials/UofC_2013-04-09/part06/p6.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part06/p6.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part06/p6.swift 2013-04-09 14:48:47 UTC (rev 6421) @@ -0,0 +1,25 @@ +type file; + +app (file o) mysim2 (int timesteps) +{ + random2 timesteps stdout=@filename(o); +} + +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} + +file sims[]; +int nsim = @toInt(@arg("nsim","10")); + +int steps = @toInt(@arg("steps","1")); + +foreach i in [0:nsim-1] { + file simout ; + simout = mysim2(steps); + sims[i] = simout; +} + +file stats<"output/average.out">; +stats = analyze(sims); Added: SwiftTutorials/UofC_2013-04-09/part06/tc =================================================================== --- SwiftTutorials/UofC_2013-04-09/part06/tc (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part06/tc 2013-04-09 14:48:47 UTC (rev 6421) @@ -0,0 +1,2 @@ +localhost random2 random2.sh +localhost average avg.sh Added: SwiftTutorials/UofC_2013-04-09/part07/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part07/README (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part07/README 2013-04-09 14:48:47 UTC (rev 6421) @@ -0,0 +1,81 @@ + +Swift script - execution on the parallel cluster + + +Til now we've been running our "simulations" on the local login host. + +Now we'll run them on the parallel cluster. + +The script will be the same as previous, but we change the tool catalog to +indicate that our "simulation app" (random4.sh) should be run on the +"westmere" partition of the Midway cluster using the "slurm" scheduler: + +We changed file "tc" from: + +localhost random4 random4.sh +localhost average avg.sh + +to: + +westmere random4 random4.sh +localhost average avg.sh + +And we added the "sites.xml" file which specifies two "pools" of resources, +the "local host" pool (which we've been using till now) and the "westmere" pool. + +To run, we now add the sites.xml file to the swift command line: + +$ swift -tc.file tc -sites.file sites.xml p7.swift -steps=10 -range=1000 -count=6 -nsim=100 + +Note that we're now asking for 100 simulations to be run, each of which +is to run for 10 steps (ie 10 seconds). + +The sites file looks like this: + + + + + + + /scratch/midway/{env.USER}/swiftwork + + + + + + + westmere + 12 + + + + 1 + 1 + 1 + 12 + .11 + + + 00:15:00 + 1800 + + + + /scratch/midway/{env.USER}/swiftwork + + + false + 100 + 100 + 10000 + + + + +Try: + +- running more app() calls at once: change jobsPerNode and jobThrottle + +- requesting more than 1 slurm job at once: change slots + +- requesting nodes in units greater than 1: change maxnodes and nodeGranularity Added: SwiftTutorials/UofC_2013-04-09/part07/p7.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part07/p7.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part07/p7.swift 2013-04-09 14:48:47 UTC (rev 6421) @@ -0,0 +1,27 @@ +type file; + +app (file o) mysim4 (int timesteps, int range, int count) +{ + random4 timesteps range count stdout=@filename(o); +} + +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} + +file sims[]; +int nsim = @toInt(@arg("nsim", "10")); + +int steps = @toInt(@arg("steps", "1")); +int range = @toInt(@arg("range", "100")); +int count = @toInt(@arg("count", "10")); + +foreach i in [0:nsim-1] { + file simout ; + simout = mysim4(steps,range,count); + sims[i] = simout; +} + +file stats<"output/average.out">; +stats = analyze(sims); Added: SwiftTutorials/UofC_2013-04-09/part07/sites.xml =================================================================== --- SwiftTutorials/UofC_2013-04-09/part07/sites.xml (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part07/sites.xml 2013-04-09 14:48:47 UTC (rev 6421) @@ -0,0 +1,39 @@ + + + + + + /scratch/midway/{env.USER}/swiftwork + + + + + + + westmere + 12 + + + + 1 + 1 + 1 + 12 + .11 + + + 00:15:00 + 1800 + + + + /scratch/midway/{env.USER}/swiftwork + + + false + 100 + 100 + 10000 + + + Added: SwiftTutorials/UofC_2013-04-09/part07/tc =================================================================== --- SwiftTutorials/UofC_2013-04-09/part07/tc (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part07/tc 2013-04-09 14:48:47 UTC (rev 6421) @@ -0,0 +1,2 @@ +westmere random4 random4.sh +localhost average avg.sh From wilde at ci.uchicago.edu Tue Apr 9 09:50:39 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 9 Apr 2013 09:50:39 -0500 (CDT) Subject: [Swift-commit] r6422 - SwiftTutorials/UofC_2013-04-09 Message-ID: <20130409145039.B71079CCFD@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 09:50:39 -0500 (Tue, 09 Apr 2013) New Revision: 6422 Removed: SwiftTutorials/UofC_2013-04-09/scripts/ Log: Remove old demo scripts From wilde at ci.uchicago.edu Tue Apr 9 10:30:37 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 9 Apr 2013 10:30:37 -0500 (CDT) Subject: [Swift-commit] r6423 - in SwiftTutorials/UofC_2013-04-09: . bin part01 part02 part03 part04 part05 part06 part07 Message-ID: <20130409153037.23E619CCFD@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 10:30:37 -0500 (Tue, 09 Apr 2013) New Revision: 6423 Modified: SwiftTutorials/UofC_2013-04-09/README SwiftTutorials/UofC_2013-04-09/bin/setup.sh SwiftTutorials/UofC_2013-04-09/part01/README SwiftTutorials/UofC_2013-04-09/part02/README SwiftTutorials/UofC_2013-04-09/part03/README SwiftTutorials/UofC_2013-04-09/part04/README SwiftTutorials/UofC_2013-04-09/part05/README SwiftTutorials/UofC_2013-04-09/part06/README SwiftTutorials/UofC_2013-04-09/part07/README Log: Update READMEs and setup script. Modified: SwiftTutorials/UofC_2013-04-09/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/README 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/README 2013-04-09 15:30:37 UTC (rev 6423) @@ -12,70 +12,43 @@ bin: script tools for the tutorial scripts: Swift scripts for the tutorial -An output directory will be created by the Swift scripts - - Use bin/setup.sh to add tutorial programs to your PATH Run: source bin/setup.sh -EXERCISE 1 - Run a simple command under Swift +verify: -This script simply runs the getinfo command found in bin/getinfo.sh +$ random.sh # should be found in your PATH now -You can run this on the login node by typing: -bin/getinfo.sh -Inspect getinfo.sh to see what it does. +The tutorial is arranged in parts. To begin: -The Swift syntax in scripts/script01.swift directs the output of -getinfo.sh to file output/info.txt +$ cd part01 +$ cat README -Run the Swift script by typing: +When finished: -runswift scripts/script01.swift +$ cd ../part02 # etc -Check that the output is created in the output directory +In each part, you can type "cleanup" after running a few swift scripts +to remove the old logs that build up, and (usually) the output files. -EXERCISE 2 - Run parallel jobs -The Swift syntax in scripts/script02.swift runs the getinfo -script 25 times in parallel. Each job produces an output -file with a different name. +The topics of the tutorial parts are: -runswift scripts/script02.swift +part01: First Swift script - run a "simulation" program, random.sh -Check that the output is created successfully +part02: Specifying the output filenames -EXERCISE 3 - Run parallel jobs with pipelines +part03: Run many instances of a "simulation" program in a foreach() loop -The Swift syntax in scripts/script02.swift runs the getinfo -script 25 times in parallel. Each job produces an output -file with a different name. Each output file is then read by the -filterinfo task, which produces output in _concurrent +part04: run a program in a foreach() loop with explicitly named output files. -Inspect bin/filterinfo.sh to see what it does to the output of getinfo.sh +part05: Add a program that "analyzes" the output the parallel simulations -runswift scripts/script03.swift +part06: Add arguments to the "simulation" program -EXERCISE 4 - Name files in pipelines - -In Exercise 3, the final outputs were named by Swift. -Exercise 4 is similar to 3 but allows you to name the final -outputs. Edit script04.swift to name the files in an appropriate -way. - -EXERCISE 5 - Reduction - -Exercise 5 allows you to collect all of the filtered outputs -and summarize them into one file. This Swift script allows -suminfo.sh to read all the files and collect a statistic. - -Inspect bin/suminfo.sh to see what it will do to the files it reads. - -CLEAN UP - -Simply run: bin/cleanup and enter y to confirm +part07: Run the "simulations" on the parallel cluster Modified: SwiftTutorials/UofC_2013-04-09/bin/setup.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/setup.sh 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/bin/setup.sh 2013-04-09 15:30:37 UTC (rev 6423) @@ -1,13 +1,9 @@ module load swift -echo 0=$0 - DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -demobin=$(cd $(dirname $0); pwd) +echo Adding DIR=$DIR to PATH: -echo demobin=$demobin DIR=$DIR +PATH=$DIR:$PATH -return - -PATH=$PWD/bin:$PATH +echo $PATH Modified: SwiftTutorials/UofC_2013-04-09/part01/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part01/README 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/part01/README 2013-04-09 15:30:37 UTC (rev 6423) @@ -1,6 +1,7 @@ -First Swift script - run a "simulation" program, random.sh +TOPIC: First Swift script - run a "simulation" program, random.sh + Verify that random.sh is in your path, and that it works: $ which random.sh Modified: SwiftTutorials/UofC_2013-04-09/part02/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part02/README 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/part02/README 2013-04-09 15:30:37 UTC (rev 6423) @@ -1,5 +1,5 @@ -Swift script - run a "simulation" program, random.sh, specifying the output filename +TOPIC: specifying the output filenames Study the Swift program: Modified: SwiftTutorials/UofC_2013-04-09/part03/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part03/README 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/part03/README 2013-04-09 15:30:37 UTC (rev 6423) @@ -1,5 +1,5 @@ -Swift script - run many instances of a "simulation" program in a foreach() loop +TOPIC: run many instances of a "simulation" program in a foreach() loop Run the program: Modified: SwiftTutorials/UofC_2013-04-09/part04/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part04/README 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/part04/README 2013-04-09 15:30:37 UTC (rev 6423) @@ -1,7 +1,7 @@ -Swift script - run many instances of a "simulation" program in a foreach() loop, - with explicitly named output files. +TOPIC: run a program in a foreach() loop with explicitly named output files. + Study the program: - @strcat() is a Swift builtin function (the @name() form is used for many but not all builtins) Modified: SwiftTutorials/UofC_2013-04-09/part05/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part05/README 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/part05/README 2013-04-09 15:30:37 UTC (rev 6423) @@ -1,15 +1,42 @@ -Swift script - run many instances of a "simulation" program in a foreach() loop, - with explicitly named output files. +TOPIC: Add a program that "analyzes" the output the parallel simulations + + Study the program: -- @strcat() is a Swift builtin function (the @name() form is used for many bu not all builtins) +We added a new app to "analyze" all the simulation outputs: -- @strcat() coerces numeric args to strings +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} -- <"filename"> was a shorthand for but the latter is more flexible. +Nots that it expects an array of files as its input, and that +it uses @filenames() to place the names of all the files on +the command line to the "average" program. +Average does: +-------------------- +#! /bin/sh + +awk ' + +{ sum += $1} + +END { print sum/NR } +' $* +------------------- + + +At the end of the script we insert: + +file stats<"output/average.out">; +stats = analyze(sims); + +...which runs the analysis *after* all the simulations complete. + + Run the program: $ swift -tc.file tc p5.swift @@ -18,10 +45,7 @@ $ cd output $ ls -l -$ cat * +$ cat sim*.out +$ cat average.out -Try: - -- changing the output names - Modified: SwiftTutorials/UofC_2013-04-09/part06/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part06/README 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/part06/README 2013-04-09 15:30:37 UTC (rev 6423) @@ -1,6 +1,6 @@ -Swift script - add arguments to the "simulation" program to control number of steps, - range of output, and number of output lines, +TOPIC: add arguments to the "simulation" program +(to control number of steps, range of output, and number of output lines) Test new versions of the "simulation" program: Modified: SwiftTutorials/UofC_2013-04-09/part07/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part07/README 2013-04-09 14:50:39 UTC (rev 6422) +++ SwiftTutorials/UofC_2013-04-09/part07/README 2013-04-09 15:30:37 UTC (rev 6423) @@ -1,5 +1,5 @@ -Swift script - execution on the parallel cluster +TOPIC: Run the "simulations" on the parallel cluster Til now we've been running our "simulations" on the local login host. From wilde at ci.uchicago.edu Tue Apr 9 10:37:02 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 9 Apr 2013 10:37:02 -0500 (CDT) Subject: [Swift-commit] r6424 - SwiftTutorials/UofC_2013-04-09/bin Message-ID: <20130409153702.454B59CCFD@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 10:37:02 -0500 (Tue, 09 Apr 2013) New Revision: 6424 Removed: SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh SwiftTutorials/UofC_2013-04-09/bin/genapplist SwiftTutorials/UofC_2013-04-09/bin/getinfo.sh SwiftTutorials/UofC_2013-04-09/bin/runswift SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh Log: Remove old tools from bin. Deleted: SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh 2013-04-09 15:30:37 UTC (rev 6423) +++ SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh 2013-04-09 15:37:02 UTC (rev 6424) @@ -1,2 +0,0 @@ -#! /bin/sh -grep getinfo.sh $1 | wc -l >$2 Deleted: SwiftTutorials/UofC_2013-04-09/bin/genapplist =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/genapplist 2013-04-09 15:30:37 UTC (rev 6423) +++ SwiftTutorials/UofC_2013-04-09/bin/genapplist 2013-04-09 15:37:02 UTC (rev 6424) @@ -1,7 +0,0 @@ -cat <pathlist -rule .*_concurrent/.* DIRECT $PWD/tmp -rule .* DIRECT / -END - -cat <config -status.mode=provider -use.provider.staging=false -use.wrapper.staging=false -wrapperlog.always.transfer=true -execution.retries=0 -lazy.errors=false -provider.staging.pin.swiftfiles=false -sitedir.keep=true -file.gc.enabled=false -tcp.port.range=50000,51000 -END - -reservationID=${reservationID:-swift} - -cat <sites.xml - - - - - - - 0.03 - 10000 - - $PWD/swiftwork - - - - - - - - 12 - 12 - 1 - 1 - 1 - westmere - - slurm.resource_list=advres=$reservationID - - 5.99 - 10000 - - 100 - 100 - 18000 - 02:00:00 - - - $PWD/swiftwork - - - - - - - - 1 - 12 - 1 - 1 - 1 - single - westmere - - slurm.resource_list=advres=$reservationID - - 0.64 - 10000 - - 100 - 100 - 9000 - 01:00:00 - - - $PWD/swiftwork - - - - - -END - -genapplist >applist - -set -x -#swift -config config -tc.file applist -sites.file sites.xml -cdm.file pathlist $* -swift -config config -tc.file applist -sites.file sites.xml $* - - - - Deleted: SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh 2013-04-09 15:30:37 UTC (rev 6423) +++ SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh 2013-04-09 15:37:02 UTC (rev 6424) @@ -1,2 +0,0 @@ -#! /bin/sh -awk '{ sum += $1 } END { print "Total processes observed: " sum }' $* From wilde at ci.uchicago.edu Tue Apr 9 11:47:45 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 9 Apr 2013 11:47:45 -0500 (CDT) Subject: [Swift-commit] r6425 - in SwiftTutorials/UofC_2013-04-09: . bin part06 Message-ID: <20130409164745.5C0189D103@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 11:47:44 -0500 (Tue, 09 Apr 2013) New Revision: 6425 Added: SwiftTutorials/UofC_2013-04-09/setup.sh Removed: SwiftTutorials/UofC_2013-04-09/bin/setup.sh Modified: SwiftTutorials/UofC_2013-04-09/README SwiftTutorials/UofC_2013-04-09/part06/README Log: Moved setup to base dir; updated main README; fixed README type in part06. Modified: SwiftTutorials/UofC_2013-04-09/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/README 2013-04-09 15:37:02 UTC (rev 6424) +++ SwiftTutorials/UofC_2013-04-09/README 2013-04-09 16:47:44 UTC (rev 6425) @@ -12,14 +12,12 @@ bin: script tools for the tutorial scripts: Swift scripts for the tutorial -Use bin/setup.sh to add tutorial programs to your PATH +TO START, do: -Run: +$ source setup.sh # to add tutorial programs to your PATH -source bin/setup.sh +...and then verify: -verify: - $ random.sh # should be found in your PATH now Deleted: SwiftTutorials/UofC_2013-04-09/bin/setup.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/setup.sh 2013-04-09 15:37:02 UTC (rev 6424) +++ SwiftTutorials/UofC_2013-04-09/bin/setup.sh 2013-04-09 16:47:44 UTC (rev 6425) @@ -1,9 +0,0 @@ -module load swift - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -echo Adding DIR=$DIR to PATH: - -PATH=$DIR:$PATH - -echo $PATH Modified: SwiftTutorials/UofC_2013-04-09/part06/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part06/README 2013-04-09 15:37:02 UTC (rev 6424) +++ SwiftTutorials/UofC_2013-04-09/part06/README 2013-04-09 16:47:44 UTC (rev 6425) @@ -14,7 +14,7 @@ Run the program: -$ swift -tc.file tc p6.swift -steps 3 # each simulation takes 3 seconds +$ swift -tc.file tc p6.swift -steps=3 # each simulation takes 3 seconds Next we need to use the parallel cluster for our simulations! Copied: SwiftTutorials/UofC_2013-04-09/setup.sh (from rev 6423, SwiftTutorials/UofC_2013-04-09/bin/setup.sh) =================================================================== --- SwiftTutorials/UofC_2013-04-09/setup.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/setup.sh 2013-04-09 16:47:44 UTC (rev 6425) @@ -0,0 +1,33 @@ +module load swift + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +echo Adding $DIR/bin to PATH: + +PATH=$DIR/bin:$PATH + +echo $PATH + +mkdir -p $HOME/.swift + +if [ -f $HOME/.swift/swift.properties ]; then + echo Adding properties to end of $HOME/.swift/swift.properties +else + echo creating $HOME/.swift/swift.properties +fi + +cat >>$HOME/.swift/swift.properties < Author: wilde Date: 2013-04-09 11:53:03 -0500 (Tue, 09 Apr 2013) New Revision: 6426 Modified: SwiftTutorials/UofC_2013-04-09/SwiftRCCTutorial.2013.0409.pptx Log: Update slides. Modified: SwiftTutorials/UofC_2013-04-09/SwiftRCCTutorial.2013.0409.pptx =================================================================== (Binary files differ) From wilde at ci.uchicago.edu Tue Apr 9 11:54:32 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 9 Apr 2013 11:54:32 -0500 (CDT) Subject: [Swift-commit] r6427 - SwiftTutorials/UofC_2013-04-09 Message-ID: <20130409165432.018699D015@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 11:54:31 -0500 (Tue, 09 Apr 2013) New Revision: 6427 Modified: SwiftTutorials/UofC_2013-04-09/setup.sh Log: Add SLURM reservation variable. Modified: SwiftTutorials/UofC_2013-04-09/setup.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/setup.sh 2013-04-09 16:53:03 UTC (rev 6426) +++ SwiftTutorials/UofC_2013-04-09/setup.sh 2013-04-09 16:54:31 UTC (rev 6427) @@ -1,5 +1,8 @@ + module load swift +export SBATCH_RESERVATION=swift + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" echo Adding $DIR/bin to PATH: From wozniak at ci.uchicago.edu Tue Apr 9 13:10:42 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Tue, 9 Apr 2013 13:10:42 -0500 (CDT) Subject: [Swift-commit] r6428 - SwiftTutorials/UofC_2013-04-09 Message-ID: <20130409181042.231B39D015@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-09 13:10:41 -0500 (Tue, 09 Apr 2013) New Revision: 6428 Added: SwiftTutorials/UofC_2013-04-09/setup.csh Log: CSH version of setup.sh Added: SwiftTutorials/UofC_2013-04-09/setup.csh =================================================================== --- SwiftTutorials/UofC_2013-04-09/setup.csh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/setup.csh 2013-04-09 18:10:41 UTC (rev 6428) @@ -0,0 +1,35 @@ + +module load swift + +setenv SBATCH_RESERVATION swift + +set DIR=`dirname ${0}` +cd $DIR +set DIR=`pwd` + +echo Adding $DIR/bin to PATH: +setenv PATH $DIR/bin:$PATH + +mkdir -p $HOME/.swift + +if ( -f $HOME/.swift/swift.properties ) then + echo Adding properties to end of $HOME/.swift/swift.properties +else + echo creating $HOME/.swift/swift.properties +endif + +cat >>$HOME/.swift/swift.properties < Author: ketan Date: 2013-04-12 10:26:59 -0500 (Fri, 12 Apr 2013) New Revision: 6431 Modified: trunk/tests/sites/intrepid/sites.template.xml Log: update to intrepid sites file Modified: trunk/tests/sites/intrepid/sites.template.xml =================================================================== --- trunk/tests/sites/intrepid/sites.template.xml 2013-04-11 20:29:12 UTC (rev 6430) +++ trunk/tests/sites/intrepid/sites.template.xml 2013-04-12 15:26:59 UTC (rev 6431) @@ -18,14 +18,14 @@ _QUEUE_ zeptoos true - 21 + 5.11 10000 1 DEBUG 1 900 - 64 - 64 + 512 + 512 _WORK_ From ketan at ci.uchicago.edu Fri Apr 12 10:28:44 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Fri, 12 Apr 2013 10:28:44 -0500 (CDT) Subject: [Swift-commit] r6432 - trunk/docs/siteguide Message-ID: <20130412152844.93B0C9CCC2@svn.ci.uchicago.edu> Author: ketan Date: 2013-04-12 10:28:44 -0500 (Fri, 12 Apr 2013) New Revision: 6432 Modified: trunk/docs/siteguide/intrepid Log: updates to Intrepid sites guide Modified: trunk/docs/siteguide/intrepid =================================================================== --- trunk/docs/siteguide/intrepid 2013-04-12 15:26:59 UTC (rev 6431) +++ trunk/docs/siteguide/intrepid 2013-04-12 15:28:44 UTC (rev 6432) @@ -68,6 +68,13 @@ What You Need To Know Before Running Swift ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Note that on Intrepid, the compute nodes can not create or write to a /home filesystem. Consequently, in order for Swift to interface correctly from login node to the compute nodes, Swift must write all internal and intermediate files to /intrepid-fs0, which is writable by the compute nodes. In order to accomplish this, export the environment variable SWIFT_USERHOME as follows: + +---- +export SWIFT_USERHOME=/intrepid-fs0/users/`whoami`/scratch +---- + Before you can create a Swift configuration file, there are some things you will need to know. From swift at ci.uchicago.edu Fri Apr 12 23:05:04 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 12 Apr 2013 23:05:04 -0500 (CDT) Subject: [Swift-commit] cog r3649 Message-ID: <20130413040505.D28758D001E9@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3649 | hategan | 2013-04-12 23:02:53 -0500 (Fri, 12 Apr 2013) | 1 line links are not supported in windows and they are not necessary. The build scripts automatically look in both lib and the main lib dir for a jar file ------------------------------------------------------------------------ Index: modules/util/lib/log4j-1.2.16.jar =================================================================== --- modules/util/lib/log4j-1.2.16.jar (revision 3648) +++ modules/util/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file From hategan at ci.uchicago.edu Fri Apr 12 23:08:56 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 12 Apr 2013 23:08:56 -0500 (CDT) Subject: [Swift-commit] r6433 - branches/release-0.94/lib Message-ID: <20130413040856.848E49CCFD@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-12 23:08:56 -0500 (Fri, 12 Apr 2013) New Revision: 6433 Removed: branches/release-0.94/lib/log4j-1.2.16.jar Log: removed link to log4j jar: does not work on windows and the build scripts use the common one automatically anyway Deleted: branches/release-0.94/lib/log4j-1.2.16.jar =================================================================== --- branches/release-0.94/lib/log4j-1.2.16.jar 2013-04-12 15:28:44 UTC (rev 6432) +++ branches/release-0.94/lib/log4j-1.2.16.jar 2013-04-13 04:08:56 UTC (rev 6433) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file From swift at ci.uchicago.edu Fri Apr 12 23:10:06 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 12 Apr 2013 23:10:06 -0500 (CDT) Subject: [Swift-commit] cog r3650 Message-ID: <20130413041006.B46CD8D001E9@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3650 | hategan | 2013-04-12 23:07:35 -0500 (Fri, 12 Apr 2013) | 1 line removed links to log4j; see previous commit for why ------------------------------------------------------------------------ Index: modules/provider-mpichg2/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-mpichg2/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-mpichg2/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-localscheduler/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-localscheduler/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-localscheduler/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-local/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-local/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-local/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-gt2/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-gt2/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-gt2/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-ssh/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-ssh/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-ssh/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-coaster/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-coaster/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-coaster/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/abstraction-common/lib/log4j-1.2.16.jar =================================================================== --- modules/abstraction-common/lib/log4j-1.2.16.jar (revision 3649) +++ modules/abstraction-common/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-dcache/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-dcache/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-dcache/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/setup/lib/log4j-1.2.16.jar =================================================================== --- modules/setup/lib/log4j-1.2.16.jar (revision 3649) +++ modules/setup/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-webdav/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-webdav/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-webdav/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-slocal/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-slocal/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-slocal/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/certrequest/lib/log4j-1.2.16.jar =================================================================== --- modules/certrequest/lib/log4j-1.2.16.jar (revision 3649) +++ modules/certrequest/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/karajan/lib/log4j-1.2.16.jar =================================================================== --- modules/karajan/lib/log4j-1.2.16.jar (revision 3649) +++ modules/karajan/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/grapheditor/lib/log4j-1.2.16.jar =================================================================== --- modules/grapheditor/lib/log4j-1.2.16.jar (revision 3649) +++ modules/grapheditor/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/provider-condor/lib/log4j-1.2.16.jar =================================================================== --- modules/provider-condor/lib/log4j-1.2.16.jar (revision 3649) +++ modules/provider-condor/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file Index: modules/examples/lib/log4j-1.2.16.jar =================================================================== --- modules/examples/lib/log4j-1.2.16.jar (revision 3649) +++ modules/examples/lib/log4j-1.2.16.jar (working copy) @@ -1 +0,0 @@ -link ../../../lib/log4j-1.2.16.jar \ No newline at end of file From hategan at ci.uchicago.edu Fri Apr 12 23:46:24 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 12 Apr 2013 23:46:24 -0500 (CDT) Subject: [Swift-commit] r6434 - branches/release-0.94/libexec Message-ID: <20130413044624.483119D6CE@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-12 23:46:24 -0500 (Fri, 12 Apr 2013) New Revision: 6434 Modified: branches/release-0.94/libexec/_swiftwrap.vbs Log: Fixed windows wrapper (it wasn't aware of the cdmfile argument) Modified: branches/release-0.94/libexec/_swiftwrap.vbs =================================================================== --- branches/release-0.94/libexec/_swiftwrap.vbs 2013-04-13 04:08:56 UTC (rev 6433) +++ branches/release-0.94/libexec/_swiftwrap.vbs 2013-04-13 04:46:24 UTC (rev 6434) @@ -187,6 +187,10 @@ expectArg("k") KICKSTART=getOptArg() +expectArg("cdmfile") +'ignored, but read if specified +CDMFILE=getOptArg() + expectArg("status") STATUSMODE=getArgVal(MandatoryArg, "status") From hategan at ci.uchicago.edu Sat Apr 13 20:12:14 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 20:12:14 -0500 (CDT) Subject: [Swift-commit] r6435 - branches/faster/resources Message-ID: <20130414011214.4B5999CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 20:12:13 -0500 (Sat, 13 Apr 2013) New Revision: 6435 Modified: branches/faster/resources/swift-sites-1.0.xsd Log: updated sites schema Modified: branches/faster/resources/swift-sites-1.0.xsd =================================================================== --- branches/faster/resources/swift-sites-1.0.xsd 2013-04-13 04:46:24 UTC (rev 6434) +++ branches/faster/resources/swift-sites-1.0.xsd 2013-04-14 01:12:13 UTC (rev 6435) @@ -21,135 +21,190 @@ + + + + + + + + + + - root element aggregating all pool information there is. + root element aggregating all sites information there is. - Describes a singl e pool. + Describes a single site. - - - Administrative profile defaults associated with a pool. - - - - - - - - - - - - - Each pool may report to multiple LRCs. - - - - - - - - Each pool may have multiple gridftp servers. - - - - - - Stores the bandwidth informaion related to each gridftp server. - - - - - - - - - - - - - - - - The URL (actually, it may be more a URI, but hey, so what) is the access URL to the gridftp server. Each pool may have multiple gridftp servers, or run multiple versions of Globus on different ports. - - - - - This element is the storage mount point prefix. Of course, thi s may get turned over into other things, augmented by user and system requirements etc. I believe that default works quite well for default Globus setups. - - - - - - - - - - Each pool supports various (usually two) jobmanagers. - - - - - The universe name is actually the primary key for the jobmanager identification. - - - - - The contact string is the secondary key for any job manager. - - - - - Any pool may have multiple versions of Globus installed, and these versions may have multiple jobmanagers listening on different ports. - - - - - - - - - - - - - - - - - - - Each pool may have one and only one work directory mount point. - - - - - - - - - - + + + + + + - - - This is the path to grid launch. As attributes are optional, each pool can have at most one grid launch path. More does not make sense, unless there are different launchers for different remote schedulers, and a pool su pports more than two (fork + one other) remote schedulers. - - + + + + + Describes a single site. + + + + + + + + + + + + + - + + + + Administrative profile defaults associated with a pool. + + + + + + + + + + + + + Each pool may have one and only one work directory mount point. + + + + + + + + + + + + + Describes the mechanism used to submit jobs to the pool + + + + + + + + + + + Describes the mechanism used to interact with the filesystem on the given pool + + + + + + + + + + + Each pool may have multiple gridftp servers. + + + + + + Stores the bandwidth informaion related to each gridftp server. + + + + + + + + + + + + + + + + The URL (actually, it may be more a URI, but hey, so what) is the access URL to the gridftp server. Each pool may have multiple gridftp servers, or run multiple versions of Globus on different ports. + + + + + This element is the storage mount point prefix. Of course, this may get turned over into other things, augmented by user and system requirements etc. I believe that default works quite well for default Globus setups. + + + + + + + + + + Each pool supports various (usually two) jobmanagers. + + + + + The universe name is actually the primary key for the jobmanager identification. + + + + + The contact string is the secondary key for any job manager. + + + + + Any pool may have multiple versions of Globus installed, and these versions may have multiple jobmanagers listening on different ports. + + + + + + + + + + + + + + + + + + + Specifies the details of an application installed on a site + + + + + + + + + \ No newline at end of file From hategan at ci.uchicago.edu Sat Apr 13 20:13:18 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 20:13:18 -0500 (CDT) Subject: [Swift-commit] r6436 - branches/faster/src/org/griphyn/vdl/karajan/lib Message-ID: <20130414011318.6C9FF9CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 20:13:18 -0500 (Sat, 13 Apr 2013) New Revision: 6436 Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java Log: optimize data copying a bit Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2013-04-14 01:12:13 UTC (rev 6435) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2013-04-14 01:13:18 UTC (rev 6436) @@ -185,25 +185,32 @@ } protected void deepCopy(DSHandle dest, DSHandle source, Stack stack) { - State state = this.state.getValue(stack); - if (state == null) { - state = new State(); - this.state.setValue(stack, state); + // don't create a state if only a non-composite is copied + ((AbstractDataNode) source).waitFor(this); + if (source.getType().isPrimitive()) { + dest.setValue(source.getValue()); } - - deepCopy(dest, source, state, 0); - - this.state.setValue(stack, null); + else { + State state = this.state.getValue(stack); + if (state == null) { + state = new State(); + this.state.setValue(stack, state); + } + + deepCopy(dest, source, state, 0); + + this.state.setValue(stack, null); + } } /** make dest look like source - if its a simple value, copy that and if its an array then recursively copy */ public void deepCopy(DSHandle dest, DSHandle source, State state, int level) { ((AbstractDataNode) source).waitFor(this); - if (source.getType().isPrimitive()) { - dest.setValue(source.getValue()); - } - else if (source.getType().isArray()) { + if (source.getType().isPrimitive()) { + dest.setValue(source.getValue()); + } + else if (source.getType().isArray()) { copyArray(dest, source, state, level); } else if (source.getType().isComposite()) { From hategan at ci.uchicago.edu Sat Apr 13 20:14:09 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 20:14:09 -0500 (CDT) Subject: [Swift-commit] r6437 - branches/faster/src/org/griphyn/vdl/engine Message-ID: <20130414011409.E0D3C9CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 20:14:09 -0500 (Sat, 13 Apr 2013) New Revision: 6437 Modified: branches/faster/src/org/griphyn/vdl/engine/Karajan.java Log: updates to keyword parameter passing Modified: branches/faster/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- branches/faster/src/org/griphyn/vdl/engine/Karajan.java 2013-04-14 01:13:18 UTC (rev 6436) +++ branches/faster/src/org/griphyn/vdl/engine/Karajan.java 2013-04-14 01:14:09 UTC (rev 6437) @@ -776,35 +776,63 @@ callST.setAttribute("inputs", argST); } } else if (keywordArgsInput) { - /* if ALL arguments are specified by name=value */ - int noOfMandArgs = 0; - for (int i = 0; i < call.sizeOfInputArray(); i++) { - ActualParameter input = call.getInputArray(i); - StringTemplate argST = actualParameter(input, scope); + /* if ALL arguments are specified by name=value */ + /* Re-order all (which re-orders positionals), then pass optionals by keyword */ + ActualParameter[] actuals = new ActualParameter[proc.sizeOfInputArray()]; + for (int i = 0; i < call.sizeOfInputArray(); i++) { + ActualParameter actual = call.getInputArray(i); + boolean found = false; + for (int j = 0; j < proc.sizeOfInputArray(); j++) { + FormalArgumentSignature formal = proc.getInputArray(j); + if (actual.getBind().equals(formal.getName())) { + actuals[j] = actual; + found = true; + break; + } + } + if (!found) { + throw new CompilationException("Formal argument " + actual.getBind() + " doesn't exist"); + } + } + + int noOfMandArgs = 0; + for (ActualParameter actual : actuals) { + if (actual == null) { + // an optional formal parameter with no actual parameter + continue; + } + FormalArgumentSignature formal = inArgs.get(actual.getBind()); + + StringTemplate argST; + if (formal.isOptional()) { + argST = actualParameter(actual, formal.getName(), scope); + } + else { + argST = actualParameter(actual, scope); + } callST.setAttribute("inputs", argST); - if (!inArgs.containsKey(input.getBind())) - throw new CompilationException("Formal argument " + input.getBind() + " doesn't exist"); - FormalArgumentSignature formalArg = inArgs.get(input.getBind()); - String formalType = formalArg.getType(); + String formalType = formal.getType(); String actualType = datatype(argST); - if (!formalArg.isAnyType() && !actualType.equals(formalType)) - throw new CompilationException("Wrong type for parameter number " + i + - ", expected " + formalType + ", got " + actualType); + if (!formal.isAnyType() && !actualType.equals(formalType)) + throw new CompilationException("Wrong type for '" + formal.getName() + "' parameter;" + + " expected " + formalType + ", got " + actualType); - if (!formalArg.isOptional()) + if (!formal.isOptional()) { noOfMandArgs++; + } } - if (!proc.getAnyNumOfInputArgs() && noOfMandArgs < proc.sizeOfInputArray() - noOfOptInArgs) + if (!proc.getAnyNumOfInputArgs() && noOfMandArgs < proc.sizeOfInputArray() - noOfOptInArgs) { throw new CompilationException("Mandatory argument missing"); + } } else { /* Positional arguments */ /* Checking types of mandatory arguments */ for (int i = 0; i < proc.sizeOfInputArray() - noOfOptInArgs; i++) { ActualParameter input = call.getInputArray(i); + FormalArgumentSignature formalArg = proceduresMap.get(procName).getInputArray(i); StringTemplate argST = actualParameter(input, scope); callST.setAttribute("inputs", argST); - - FormalArgumentSignature formalArg = proceduresMap.get(procName).getInputArray(i); + String formalType = formalArg.getType(); String actualType = datatype(argST); if (!formalArg.isAnyType() && !actualType.equals(formalType)) @@ -814,13 +842,15 @@ /* Checking types of optional arguments */ for (int i = proc.sizeOfInputArray() - noOfOptInArgs; i < call.sizeOfInputArray(); i++) { ActualParameter input = call.getInputArray(i); - StringTemplate argST = actualParameter(input, scope); - callST.setAttribute("inputs", argST); String formalName = input.getBind(); if (!inArgs.containsKey(formalName)) throw new CompilationException("Formal argument " + formalName + " doesn't exist"); FormalArgumentSignature formalArg = inArgs.get(formalName); + + StringTemplate argST = actualParameter(input, formalArg.getName(), scope); + callST.setAttribute("inputs", argST); + String formalType = formalArg.getType(); String actualType = datatype(argST); if (!formalArg.isAnyType() && !actualType.equals(formalType)) @@ -847,21 +877,35 @@ } if (keywordArgsOutput) { /* if ALL arguments are specified by name=value */ - for (int i = 0; i < call.sizeOfOutputArray(); i++) { - ActualParameter output = call.getOutputArray(i); - StringTemplate argST = actualParameter(output, scope); + /* Re-order to match the formal output args */ + ActualParameter[] actuals = new ActualParameter[proc.sizeOfOutputArray()]; + for (int i = 0; i < call.sizeOfOutputArray(); i++) { + ActualParameter actual = call.getOutputArray(i); + boolean found = false; + for (int j = 0; j < proc.sizeOfOutputArray(); j++) { + FormalArgumentSignature formal = proc.getOutputArray(j); + if (actual.getBind().equals(formal.getName())) { + actuals[j] = actual; + found = true; + break; + } + } + if (!found) { + throw new CompilationException("Formal argument " + actual.getBind() + " doesn't exist"); + } + } + for (ActualParameter actual : actuals) { + StringTemplate argST = actualParameter(actual, null, scope); callST.setAttribute("outputs", argST); - if (!outArgs.containsKey(output.getBind())) - throw new CompilationException("Formal argument " + output.getBind() + " doesn't exist"); - FormalArgumentSignature formalArg = outArgs.get(output.getBind()); + FormalArgumentSignature formalArg = outArgs.get(actual.getBind()); String formalType = formalArg.getType(); String actualType = datatype(argST); if (!formalArg.isAnyType() && !actualType.equals(formalType)) - throw new CompilationException("Wrong type for output parameter number " + i + - ", expected " + formalType + ", got " + actualType); + throw new CompilationException("Wrong type for output parameter '" + actual.getBind() + + "', expected " + formalType + ", got " + actualType); - addWriterToScope(scope, call.getOutputArray(i).getAbstractExpression(), call, callST); + addWriterToScope(scope, actual.getAbstractExpression(), call, callST); } } else { /* Positional arguments */ for (int i = 0; i < call.sizeOfOutputArray(); i++) { @@ -1091,13 +1135,23 @@ } public StringTemplate actualParameter(ActualParameter arg, VariableScope scope) throws CompilationException { - return actualParameter(arg, scope, false); + return actualParameter(arg, null, scope, false); + } + + public StringTemplate actualParameter(ActualParameter arg, String bind, VariableScope scope) throws CompilationException { + return actualParameter(arg, bind, scope, false); } + + public StringTemplate actualParameter(ActualParameter arg, VariableScope scope, boolean lvalue) throws CompilationException { + return actualParameter(arg, null, scope, lvalue); + } - public StringTemplate actualParameter(ActualParameter arg, VariableScope scope, boolean lvalue) throws CompilationException { + public StringTemplate actualParameter(ActualParameter arg, String bind, VariableScope scope, boolean lvalue) throws CompilationException { StringTemplate argST = template("call_arg"); StringTemplate expST = expressionToKarajan(arg.getAbstractExpression(), scope, lvalue); - argST.setAttribute("bind", arg.getBind()); + if (bind != null) { + argST.setAttribute("bind", arg.getBind()); + } argST.setAttribute("expr", expST); argST.setAttribute("datatype", datatype(expST)); return argST; From hategan at ci.uchicago.edu Sat Apr 13 20:14:39 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 20:14:39 -0500 (CDT) Subject: [Swift-commit] r6438 - branches/faster/src/org/griphyn/vdl/karajan/lib Message-ID: <20130414011439.754E59CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 20:14:39 -0500 (Sat, 13 Apr 2013) New Revision: 6438 Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java Log: properly wait for app arguments that are not mapped types Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java 2013-04-14 01:14:09 UTC (rev 6437) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/UnwrapClosedList.java 2013-04-14 01:14:39 UTC (rev 6438) @@ -24,17 +24,19 @@ import java.util.List; import java.util.Map; +import k.rt.ExecutionException; import k.rt.Stack; import org.apache.log4j.Logger; import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; +import org.griphyn.vdl.mapping.AbstractDataNode; import org.griphyn.vdl.mapping.DSHandle; public class UnwrapClosedList extends SwiftFunction { public static final Logger logger = Logger.getLogger(UnwrapClosedList.class); - private ArgRef> list; + private ArgRef> list; @Override protected Signature getSignature() { @@ -43,20 +45,34 @@ @Override public Object function(Stack stack) { - List l = this.list.getValue(stack); + List l = this.list.getValue(stack); List r = new ArrayList(l.size()); - for (DSHandle h : l) { + for (AbstractDataNode h : l) { if (h.getType().isArray()) { + h.waitFor(this); Map m = h.getArrayValue(); for (DSHandle h2 : m.values()) { - r.add(h2.getValue()); + if (h2.getType().isPrimitive()) { + ((AbstractDataNode) h2).waitFor(this); + r.add(h2.getValue()); + } + else { + throw new ExecutionException(this, "Cannot pass an array of non-primitives as an application parameter"); + } } } - else { + else if (h.getType().isPrimitive()) { + // stagein only waits for the root of the parameter + // and the fringes as returned by getFringePaths(), but + // the latter only returns mapped types + h.waitFor(this); r.add(h.getValue()); } + else { + throw new ExecutionException(this, "Cannot pass a structure as an application parameter"); + } } return r; From hategan at ci.uchicago.edu Sat Apr 13 20:15:12 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 20:15:12 -0500 (CDT) Subject: [Swift-commit] r6439 - branches/faster/src/org/griphyn/vdl/mapping Message-ID: <20130414011512.EE76C9CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 20:15:12 -0500 (Sat, 13 Apr 2013) New Revision: 6439 Modified: branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java Log: fix sync issues with external type Modified: branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2013-04-14 01:14:39 UTC (rev 6438) +++ branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2013-04-14 01:15:12 UTC (rev 6439) @@ -233,6 +233,17 @@ return prefix; } + public String getFullName() { + String name = getDisplayableName(); + Path p = getPathFromRoot(); + if (p.isEmpty()) { + return name; + } + else { + return name + "." + p; + } + } + public String getDeclarationLine() { String line = getRoot().getParam(MappingParam.SWIFT_LINE); if (line == null || line.length() == 0) { @@ -398,7 +409,7 @@ } } - public Object getValue() { + public synchronized Object getValue() { checkNoValue(); checkDataException(); if (field.getType().isArray()) { @@ -420,16 +431,20 @@ } public void setValue(Object value) { - if (this.closed) { - throw new IllegalArgumentException(this.getDisplayableName() - + " is closed with a value of " + this.value); + synchronized(this) { + if (this.closed) { + throw new IllegalArgumentException(this.getFullName() + + " is closed with a value of " + this.value); + } + if (this.value != null) { + throw new IllegalArgumentException(this.getFullName() + + " is already assigned with a value of " + this.value); + } + + this.value = value; + this.closed = true; } - if (this.value != null) { - throw new IllegalArgumentException(this.getDisplayableName() - + " is already assigned with a value of " + this.value); - } - this.value = value; - closeShallow(); + postCloseActions(); } public Collection getFringePaths() throws HandleOpenException { @@ -466,7 +481,7 @@ } } } - + public void closeShallow() { synchronized(this) { if (this.closed) { @@ -474,6 +489,10 @@ } this.closed = true; } + postCloseActions(); + } + + private void postCloseActions() { // closed notifyListeners(); if (logger.isDebugEnabled()) { @@ -687,10 +706,10 @@ } AbstractDataNode parent = getParentNode(); if (parent != null && parent.getType().isArray()) { - throw new IndexOutOfBoundsException("Invalid index [" + field.getId() + "] for " + parent.getDisplayableName()); + throw new IndexOutOfBoundsException("Invalid index [" + field.getId() + "] for " + parent.getFullName()); } else { - throw new RuntimeException(getDisplayableName() + " has no value"); + throw new RuntimeException(getFullName() + " has no value"); } } } From hategan at ci.uchicago.edu Sat Apr 13 20:16:17 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 20:16:17 -0500 (CDT) Subject: [Swift-commit] r6440 - branches/faster/src/org/griphyn/vdl/mapping Message-ID: <20130414011617.A8AD49CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 20:16:17 -0500 (Sat, 13 Apr 2013) New Revision: 6440 Modified: branches/faster/src/org/griphyn/vdl/mapping/ExternalDataNode.java Log: ... Modified: branches/faster/src/org/griphyn/vdl/mapping/ExternalDataNode.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/ExternalDataNode.java 2013-04-14 01:15:12 UTC (rev 6439) +++ branches/faster/src/org/griphyn/vdl/mapping/ExternalDataNode.java 2013-04-14 01:16:17 UTC (rev 6440) @@ -124,4 +124,14 @@ } return (String) params.get(p); } + + @Override + public void closeDeep() { + /* + * Need to override this and set a value since + * this is skipped by the normal stageout mechanism which + * does that + */ + this.setValue(FILE_VALUE); + } } From hategan at ci.uchicago.edu Sat Apr 13 20:16:45 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 20:16:45 -0500 (CDT) Subject: [Swift-commit] r6441 - branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20130414011645.0E2C39CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 20:16:44 -0500 (Sat, 13 Apr 2013) New Revision: 6441 Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: fix a bunch of functions Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2013-04-14 01:16:17 UTC (rev 6440) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2013-04-14 01:16:44 UTC (rev 6441) @@ -32,7 +32,7 @@ import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.compiled.nodes.InternalFunction; -import org.globus.cog.karajan.compiled.nodes.functions.AbstractFunction; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; import org.globus.cog.karajan.util.TypeUtil; import org.griphyn.vdl.karajan.lib.SwiftFunction; import org.griphyn.vdl.mapping.AbsFile; @@ -128,7 +128,7 @@ } } - public static class StrCat extends AbstractFunction { + public static class StrCat extends AbstractSingleValuedFunction { private ChannelRef c_vargs; @Override @@ -161,7 +161,7 @@ } } - public static class Exists extends AbstractFunction { + public static class Exists extends AbstractSingleValuedFunction { private ArgRef file; @Override @@ -190,7 +190,7 @@ } } - public static class StrCut extends AbstractFunction { + public static class StrCut extends AbstractSingleValuedFunction { private ArgRef input; private ArgRef pattern; @@ -238,7 +238,7 @@ } } - public static class StrStr extends AbstractFunction { + public static class StrStr extends AbstractSingleValuedFunction { private ArgRef input; private ArgRef pattern; @@ -271,7 +271,7 @@ } } - public static class StrSplit extends AbstractFunction { + public static class StrSplit extends AbstractSingleValuedFunction { private ArgRef input; private ArgRef pattern; @@ -318,7 +318,7 @@ * @return DSHandle representing the resulting string * @throws ExecutionException */ - public static class StrJoin extends AbstractFunction { + public static class StrJoin extends AbstractSingleValuedFunction { private ArgRef array; private ArgRef delim; @@ -359,7 +359,7 @@ } } - public static class Regexp extends AbstractFunction { + public static class Regexp extends AbstractSingleValuedFunction { private ArgRef input; private ArgRef pattern; private ArgRef transform; @@ -412,7 +412,7 @@ } } - public static class ToInt extends AbstractFunction { + public static class ToInt extends AbstractSingleValuedFunction { private ArgRef str; @Override @@ -437,7 +437,7 @@ } } - public static class ToFloat extends AbstractFunction { + public static class ToFloat extends AbstractSingleValuedFunction { private ArgRef str; @Override @@ -476,7 +476,7 @@ /* * Takes in a float and formats to desired precision and returns a string */ - public static class Format extends AbstractFunction { + public static class Format extends AbstractSingleValuedFunction { private ArgRef format; private ArgRef value; @@ -508,7 +508,7 @@ /* * Takes in an int and pads zeros to the left and returns a string */ - public static class Pad extends AbstractFunction { + public static class Pad extends AbstractSingleValuedFunction { private ArgRef size; private ArgRef value; @@ -537,7 +537,7 @@ } } - public static class ToString extends AbstractFunction { + public static class ToString extends AbstractSingleValuedFunction { private ArgRef value; @Override @@ -548,6 +548,7 @@ @Override public Object function(Stack stack) { AbstractDataNode hvalue = this.value.getValue(stack); + hvalue.waitFor(this); StringBuilder sb = new StringBuilder(); prettyPrint(sb, hvalue); @@ -570,7 +571,7 @@ * Good for debugging because array needs to be closed * before the length is determined */ - public static class Dirname extends AbstractFunction { + public static class Dirname extends AbstractSingleValuedFunction { private ArgRef file; @Override @@ -596,7 +597,7 @@ } } - public static class Length extends AbstractFunction { + public static class Length extends AbstractSingleValuedFunction { private ArgRef array; @Override From hategan at ci.uchicago.edu Sat Apr 13 23:03:18 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 23:03:18 -0500 (CDT) Subject: [Swift-commit] r6442 - branches/faster/tests/functions Message-ID: <20130414040318.109709CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 23:03:17 -0500 (Sat, 13 Apr 2013) New Revision: 6442 Modified: branches/faster/tests/functions/105-assert-loop.check.sh Log: don't check for iterations completed when an assertion is inside a loop Modified: branches/faster/tests/functions/105-assert-loop.check.sh =================================================================== --- branches/faster/tests/functions/105-assert-loop.check.sh 2013-04-14 01:16:44 UTC (rev 6441) +++ branches/faster/tests/functions/105-assert-loop.check.sh 2013-04-14 04:03:17 UTC (rev 6442) @@ -4,7 +4,15 @@ grep I_IS_4 105-assert-loop.stdout || exit 1 -LINES=$( grep -c "i:" 105-assert-loop.stdout ) -[[ ${LINES} == 5 ]] || exit 1 +# +# Assert fails the entire run whenever the assertion +# fails, and not when the loop in which the assert statement +# exists completes. It is therefore incorrect to assume anything +# about the number of parallel iterations that are +# completed when assert aborts everything. +# +#LINES=$( grep -c "i:" 105-assert-loop.stdout ) +#[[ ${LINES} == 5 ]] || exit 1 + exit 0 From hategan at ci.uchicago.edu Sat Apr 13 23:03:56 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 23:03:56 -0500 (CDT) Subject: [Swift-commit] r6443 - branches/faster/resources Message-ID: <20130414040356.372B99CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 23:03:56 -0500 (Sat, 13 Apr 2013) New Revision: 6443 Modified: branches/faster/resources/Karajan.stg Log: invert previously incorrect condition check in iterate Modified: branches/faster/resources/Karajan.stg =================================================================== --- branches/faster/resources/Karajan.stg 2013-04-14 04:03:17 UTC (rev 6442) +++ branches/faster/resources/Karajan.stg 2013-04-14 04:03:56 UTC (rev 6443) @@ -198,7 +198,7 @@ iterate(declarations, statements, cond, var, refs, cleanups, trace, line) ::= << swift:while($var$$if(trace)$, _traceline="$line$"$endif$$if(refs)$, refs="$refs;separator=" "$"$endif$) { - if (swift:getFieldValue($cond$)) { + if (!swift:getFieldValue($cond$)) { $sub_comp(declarations=declarations, statements=statements, cleanups=cleanups)$ next(swiftop:inc($var$)) } From hategan at ci.uchicago.edu Sat Apr 13 23:04:27 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sat, 13 Apr 2013 23:04:27 -0500 (CDT) Subject: [Swift-commit] r6444 - branches/faster/src/org/griphyn/vdl/karajan/lib Message-ID: <20130414040427.AEADD9CCC2@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-13 23:04:27 -0500 (Sat, 13 Apr 2013) New Revision: 6444 Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java Log: set value on target mapped data when doing a remapping Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2013-04-14 04:03:56 UTC (rev 6443) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2013-04-14 04:04:27 UTC (rev 6444) @@ -289,7 +289,7 @@ logger.debug("Remapping " + dest + " to " + source); } dmapper.remap(dpath, source.getMapper(), source.getPathFromRoot()); - dest.closeShallow(); + dest.setValue(AbstractDataNode.FILE_VALUE); } else { StateEntry se = getStateEntry(state, level); @@ -303,7 +303,7 @@ throw new ExecutionException("Failed to copy " + source + " to " + dest, fc.getException()); } } - dest.closeShallow(); + dest.setValue(AbstractDataNode.FILE_VALUE); } else { fc = new FileCopier(source.getMapper().map(source.getPathFromRoot()), From hategan at ci.uchicago.edu Sun Apr 14 14:29:32 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 14 Apr 2013 14:29:32 -0500 (CDT) Subject: [Swift-commit] r6445 - branches/faster/resources Message-ID: <20130414192932.3F127178884@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-14 14:29:31 -0500 (Sun, 14 Apr 2013) New Revision: 6445 Modified: branches/faster/resources/Karajan.stg Log: fixed restart log file name Modified: branches/faster/resources/Karajan.stg =================================================================== --- branches/faster/resources/Karajan.stg 2013-04-14 04:04:27 UTC (rev 6444) +++ branches/faster/resources/Karajan.stg 2013-04-14 19:29:31 UTC (rev 6445) @@ -23,7 +23,7 @@ $procedures;separator="\n"$ $if(statements)$ -restartLog() { +restartLog(name = k:concat(contextAttribute("SWIFT:SCRIPT_NAME"), "-", contextAttribute("SWIFT:RUN_ID"))) { swift:mains( swift:startProgressTicker() swift:mainp( From hategan at ci.uchicago.edu Sun Apr 14 14:29:57 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 14 Apr 2013 14:29:57 -0500 (CDT) Subject: [Swift-commit] r6446 - branches/faster/libexec Message-ID: <20130414192957.43CFE178884@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-14 14:29:57 -0500 (Sun, 14 Apr 2013) New Revision: 6446 Modified: branches/faster/libexec/swift-int.k Log: fixed d dir name Modified: branches/faster/libexec/swift-int.k =================================================================== --- branches/faster/libexec/swift-int.k 2013-04-14 19:29:31 UTC (rev 6445) +++ branches/faster/libexec/swift-int.k 2013-04-14 19:29:57 UTC (rev 6446) @@ -102,7 +102,7 @@ } initDDir := function() { - ddir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" + ddir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d" once(ddir) { if(!file:exists(ddir)) { From hategan at ci.uchicago.edu Sun Apr 14 14:31:00 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 14 Apr 2013 14:31:00 -0500 (CDT) Subject: [Swift-commit] r6447 - branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20130414193100.DD0E3178884@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-14 14:31:00 -0500 (Sun, 14 Apr 2013) New Revision: 6447 Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java Log: fixed @arg with default value Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java 2013-04-14 19:29:57 UTC (rev 6446) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/swiftscript/FnArg.java 2013-04-14 19:31:00 UTC (rev 6447) @@ -46,7 +46,7 @@ @Override protected Signature getSignature() { - return new Signature(params("name", optional("value", null))); + return new Signature(params("name", "value")); } @Override From wilde at ci.uchicago.edu Tue Apr 16 11:58:12 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 16 Apr 2013 11:58:12 -0500 (CDT) Subject: [Swift-commit] r6448 - / demo Message-ID: <20130416165812.C4E949CCC9@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-16 11:58:12 -0500 (Tue, 16 Apr 2013) New Revision: 6448 Added: demo/ demo/cray/ Log: tart new dir for demos; create first one for Cray. From ketan at ci.uchicago.edu Tue Apr 16 14:07:46 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 16 Apr 2013 14:07:46 -0500 (CDT) Subject: [Swift-commit] r6449 - branches/release-0.94/docs/siteguide Message-ID: <20130416190746.F35D09CCC9@svn.ci.uchicago.edu> Author: ketan Date: 2013-04-16 14:07:46 -0500 (Tue, 16 Apr 2013) New Revision: 6449 Modified: branches/release-0.94/docs/siteguide/mcs Log: adding softenv info on mcs Modified: branches/release-0.94/docs/siteguide/mcs =================================================================== --- branches/release-0.94/docs/siteguide/mcs 2013-04-16 16:58:12 UTC (rev 6448) +++ branches/release-0.94/docs/siteguide/mcs 2013-04-16 19:07:46 UTC (rev 6449) @@ -2,8 +2,12 @@ ------------------------------------- This sections describes how to use the general use compute servers for -the MCS division of Argonne National Laboratory. +the MCS division of Argonne National Laboratory. Swift is available as a soft package on mcs machines. Add +swift to your .soft and run resoft command. +---- +$ resoft +---- + Create a coaster-service.conf ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To begin, copy the text below and paste it into your Swift distribution's etc From wilde at ci.uchicago.edu Tue Apr 16 15:54:50 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 16 Apr 2013 15:54:50 -0500 (CDT) Subject: [Swift-commit] r6451 - in demo/cray: . bin data part01 part02 part03 part04 part05 part06 part07 part08 part08/data Message-ID: <20130416205450.4E9CC9CCC9@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-16 15:54:50 -0500 (Tue, 16 Apr 2013) New Revision: 6451 Added: demo/cray/README demo/cray/SwiftRCCTutorial.2013.0409.pptx demo/cray/bin/ demo/cray/bin/avg.sh demo/cray/bin/cleanup demo/cray/bin/getlanduse.pl demo/cray/bin/highest.sh demo/cray/bin/lowest.sh demo/cray/bin/makeinput demo/cray/bin/myapp demo/cray/bin/random.sh demo/cray/bin/random2.sh demo/cray/bin/random3.sh demo/cray/bin/random4.sh demo/cray/bin/random5.sh demo/cray/data/ demo/cray/data/data.txt demo/cray/part01/ demo/cray/part01/README demo/cray/part01/p1.swift demo/cray/part01/tc demo/cray/part02/ demo/cray/part02/README demo/cray/part02/p2.swift demo/cray/part02/p2b.swift demo/cray/part02/tc demo/cray/part03/ demo/cray/part03/README demo/cray/part03/p3.swift demo/cray/part03/tc demo/cray/part04/ demo/cray/part04/README demo/cray/part04/p4.swift demo/cray/part04/tc demo/cray/part05/ demo/cray/part05/README demo/cray/part05/p5.swift demo/cray/part05/tc demo/cray/part06/ demo/cray/part06/README demo/cray/part06/p6.swift demo/cray/part06/tc demo/cray/part07/ demo/cray/part07/README demo/cray/part07/p7.swift demo/cray/part07/sites.xml demo/cray/part07/tc demo/cray/part08/ demo/cray/part08/README demo/cray/part08/cf demo/cray/part08/cray.xml demo/cray/part08/data/ demo/cray/part08/data/bias.dat demo/cray/part08/dm demo/cray/part08/p8.swift demo/cray/part08/pbs.xml demo/cray/part08/tc demo/cray/setup.csh demo/cray/setup.sh Log: Initial Cray demo version, based on latest RCC demo/tutorial. Added: demo/cray/README =================================================================== --- demo/cray/README (rev 0) +++ demo/cray/README 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,52 @@ + +INTRODUCTION TO SWIFT + +SETUP + +Obtain the tutorial package + +Change directory into the created directory called "tutorial" + +This directory contains: + +bin: script tools for the tutorial +scripts: Swift scripts for the tutorial + +TO START, do: + +$ source setup.sh # to add tutorial programs to your PATH + +...and then verify: + +$ random.sh # should be found in your PATH now + + + +The tutorial is arranged in parts. To begin: + +$ cd part01 +$ cat README + +When finished: + +$ cd ../part02 # etc + +In each part, you can type "cleanup" after running a few swift scripts +to remove the old logs that build up, and (usually) the output files. + + +The topics of the tutorial parts are: + +part01: First Swift script - run a "simulation" program, random.sh + +part02: Specifying the output filenames + +part03: Run many instances of a "simulation" program in a foreach() loop + +part04: run a program in a foreach() loop with explicitly named output files. + +part05: Add a program that "analyzes" the output the parallel simulations + +part06: Add arguments to the "simulation" program + +part07: Run the "simulations" on the parallel cluster Added: demo/cray/SwiftRCCTutorial.2013.0409.pptx =================================================================== (Binary files differ) Property changes on: demo/cray/SwiftRCCTutorial.2013.0409.pptx ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: demo/cray/bin/avg.sh =================================================================== --- demo/cray/bin/avg.sh (rev 0) +++ demo/cray/bin/avg.sh 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,9 @@ +#! /bin/sh + +awk ' + +{ sum += $1} + +END { print sum/NR } +' $* + Property changes on: demo/cray/bin/avg.sh ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/cleanup =================================================================== --- demo/cray/bin/cleanup (rev 0) +++ demo/cray/bin/cleanup 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,17 @@ +#! /bin/sh + +echo Removing: +echo +ls -ltd *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log 2>/dev/null +echo +echo Enter y to proceed: +read ok +if [ _$ok = _y -o _$ok = _Y -o _$ok = _yes ]; then + echo + set -x + rm -rf *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log +else + echo + echo No files removed. +fi + Property changes on: demo/cray/bin/cleanup ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/getlanduse.pl =================================================================== --- demo/cray/bin/getlanduse.pl (rev 0) +++ demo/cray/bin/getlanduse.pl 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,33 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +if($#ARGV != 1) { + die "usage $0: imageFile runTime" +} + +my $image_filename = shift; +my $runtime = shift; + +open(IMAGEFILE, "$image_filename") || die "$0: Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d\n", $sortedcount[$count], $count); +} + +sleep($runtime); Property changes on: demo/cray/bin/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/highest.sh =================================================================== --- demo/cray/bin/highest.sh (rev 0) +++ demo/cray/bin/highest.sh 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,2 @@ +#! /bin/sh +sort -nr | head -$1 Property changes on: demo/cray/bin/highest.sh ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/lowest.sh =================================================================== --- demo/cray/bin/lowest.sh (rev 0) +++ demo/cray/bin/lowest.sh 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,2 @@ +#! /bin/sh +sort -n | head -$1 Property changes on: demo/cray/bin/lowest.sh ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/makeinput =================================================================== --- demo/cray/bin/makeinput (rev 0) +++ demo/cray/bin/makeinput 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,22 @@ +#! /bin/sh + +nfiles=${1:-10} + +rm -rf input +mkdir input +cp ../data/sat/t? input/ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(($RANDOM%10)) + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) Property changes on: demo/cray/bin/makeinput ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/myapp =================================================================== --- demo/cray/bin/myapp (rev 0) +++ demo/cray/bin/myapp 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,6 @@ +#! /bin/sh + +hostname -f +date +sleep 3 +ps -u $USER -H -f Property changes on: demo/cray/bin/myapp ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/random.sh =================================================================== --- demo/cray/bin/random.sh (rev 0) +++ demo/cray/bin/random.sh 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,2 @@ +#! /bin/sh +echo $RANDOM Property changes on: demo/cray/bin/random.sh ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/random2.sh =================================================================== --- demo/cray/bin/random2.sh (rev 0) +++ demo/cray/bin/random2.sh 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,7 @@ +#! /bin/sh + +runtime=${1:-0} + +sleep $runtime + +echo $(($RANDOM)) Property changes on: demo/cray/bin/random2.sh ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/random3.sh =================================================================== --- demo/cray/bin/random3.sh (rev 0) +++ demo/cray/bin/random3.sh 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,10 @@ +#! /bin/sh + +runtime=${1:-0} +range=$2 + +sleep $runtime + +echo $(($RANDOM%range)) + + Property changes on: demo/cray/bin/random3.sh ___________________________________________________________________ Added: svn:executable + * Added: demo/cray/bin/random4.sh =================================================================== --- demo/cray/bin/random4.sh (rev 0) +++ demo/cray/bin/random4.sh 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,12 @@ +#! /bin/sh + +runtime=${1:-0} +range=$2 +n=$3 + +sleep $runtime + +for ((i=0;i; +f = mysim(); Added: demo/cray/part02/p2b.swift =================================================================== --- demo/cray/part02/p2b.swift (rev 0) +++ demo/cray/part02/p2b.swift 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,9 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +file f<"output/sim.out">; +f = mysim(); Added: demo/cray/part02/tc =================================================================== --- demo/cray/part02/tc (rev 0) +++ demo/cray/part02/tc 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1 @@ +localhost random random.sh Added: demo/cray/part03/README =================================================================== --- demo/cray/part03/README (rev 0) +++ demo/cray/part03/README 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,20 @@ + +TOPIC: run many instances of a "simulation" program in a foreach() loop + + +Run the program: + +$ swift -tc.file tc p3.swift + +Look at the output: + +$ cd _concurrent +$ ls -l +$ cat * + +Look at the order in which the files were written: + +$ ls -lt + +We'll come back to this... + Added: demo/cray/part03/p3.swift =================================================================== --- demo/cray/part03/p3.swift (rev 0) +++ demo/cray/part03/p3.swift 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,11 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +foreach i in [0:9] { + file f = mysim(); +} + Added: demo/cray/part03/tc =================================================================== --- demo/cray/part03/tc (rev 0) +++ demo/cray/part03/tc 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1 @@ +localhost random random.sh Added: demo/cray/part04/README =================================================================== --- demo/cray/part04/README (rev 0) +++ demo/cray/part04/README 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,42 @@ + +TOPIC: run a program in a foreach() loop with explicitly named output files. + + +Study the program: + +- @strcat() is a Swift builtin function (the @name() form is used for many but not all builtins) + +- Note that @strcat() coerces numeric args to strings + +- <"filename"> was a shorthand for but the latter is more flexible. + +Run the program: + +$ swift -tc.file tc p4.swift + +Look at the output: + +$ cd output +$ ls -l +$ cat * + + +Try: + +- Adjust the script to take the number of simulations to do from the command line + + @arg("argname") returns arguments after the .swift script on the swift command line: + + swift -tc.file tc p4.swift -myarg1=value1 ... + + @arg("argname","default") returns "default" if the argname is not given on the command line + + @toInt() converts strings to integers: + + @toInt(@arg("myArg1")) + + +- trace(expr1,...) traces expressions on stdout. + + Try inserting a few traces. But remember: Swift is *very* concurrent! + Added: demo/cray/part04/p4.swift =================================================================== --- demo/cray/part04/p4.swift (rev 0) +++ demo/cray/part04/p4.swift 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,12 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +foreach i in [0:9] { + file f ; + f = mysim(); +} + Added: demo/cray/part04/tc =================================================================== --- demo/cray/part04/tc (rev 0) +++ demo/cray/part04/tc 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1 @@ +localhost random random.sh Added: demo/cray/part05/README =================================================================== --- demo/cray/part05/README (rev 0) +++ demo/cray/part05/README 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,51 @@ + + +TOPIC: Add a program that "analyzes" the output the parallel simulations + + +Study the program: + +We added a new app to "analyze" all the simulation outputs: + +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} + +Nots that it expects an array of files as its input, and that +it uses @filenames() to place the names of all the files on +the command line to the "average" program. + +Average does: +-------------------- +#! /bin/sh + +awk ' + +{ sum += $1} + +END { print sum/NR } +' $* +------------------- + + +At the end of the script we insert: + +file stats<"output/average.out">; +stats = analyze(sims); + +...which runs the analysis *after* all the simulations complete. + + +Run the program: + +$ swift -tc.file tc p5.swift + +Look at the output: + +$ cd output +$ ls -l +$ cat sim*.out +$ cat average.out + + Added: demo/cray/part05/p5.swift =================================================================== --- demo/cray/part05/p5.swift (rev 0) +++ demo/cray/part05/p5.swift 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,24 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} + +file sims[]; + +int nsim = @toInt(@arg("nsim","10")); + +foreach i in [0:nsim-1] { + file simout ; + simout = mysim(); + sims[i] = simout; +} + +file stats<"output/average.out">; +stats = analyze(sims); Added: demo/cray/part05/tc =================================================================== --- demo/cray/part05/tc (rev 0) +++ demo/cray/part05/tc 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,2 @@ +localhost random random.sh +localhost average avg.sh Added: demo/cray/part06/README =================================================================== --- demo/cray/part06/README (rev 0) +++ demo/cray/part06/README 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,28 @@ + +TOPIC: add arguments to the "simulation" program +(to control number of steps, range of output, and number of output lines) + + +Test new versions of the "simulation" program: + + random.sh # original + random2.sh [runtime] # runtime in seconds; defaults to 1 sec + random3.sh [runtime] [range] # requests random numbers in {0:range-1} + random4.sh [runtime] [range] [count] # requests "count" random numbers in {0:range} + +p6.swift changes to random2 for the simulation program and adds an argument. + +Run the program: + +$ swift -tc.file tc p6.swift -steps=3 # each simulation takes 3 seconds + +Next we need to use the parallel cluster for our simulations! + +Try: + +- change to random3.sh and add the extra argument "range" + +- change to random4.sh and add yet another argument "count" + + + Added: demo/cray/part06/p6.swift =================================================================== --- demo/cray/part06/p6.swift (rev 0) +++ demo/cray/part06/p6.swift 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,25 @@ +type file; + +app (file o) mysim2 (int timesteps) +{ + random2 timesteps stdout=@filename(o); +} + +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} + +file sims[]; +int nsim = @toInt(@arg("nsim","10")); + +int steps = @toInt(@arg("steps","1")); + +foreach i in [0:nsim-1] { + file simout ; + simout = mysim2(steps); + sims[i] = simout; +} + +file stats<"output/average.out">; +stats = analyze(sims); Added: demo/cray/part06/tc =================================================================== --- demo/cray/part06/tc (rev 0) +++ demo/cray/part06/tc 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,2 @@ +localhost random2 random2.sh +localhost average avg.sh Added: demo/cray/part07/README =================================================================== --- demo/cray/part07/README (rev 0) +++ demo/cray/part07/README 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,81 @@ + +TOPIC: Run the "simulations" on the parallel cluster + + +Til now we've been running our "simulations" on the local login host. + +Now we'll run them on the parallel cluster. + +The script will be the same as previous, but we change the tool catalog to +indicate that our "simulation app" (random4.sh) should be run on the +"westmere" partition of the Midway cluster using the "slurm" scheduler: + +We changed file "tc" from: + +localhost random4 random4.sh +localhost average avg.sh + +to: + +westmere random4 random4.sh +localhost average avg.sh + +And we added the "sites.xml" file which specifies two "pools" of resources, +the "local host" pool (which we've been using till now) and the "westmere" pool. + +To run, we now add the sites.xml file to the swift command line: + +$ swift -tc.file tc -sites.file sites.xml p7.swift -steps=10 -range=1000 -count=6 -nsim=100 + +Note that we're now asking for 100 simulations to be run, each of which +is to run for 10 steps (ie 10 seconds). + +The sites file looks like this: + + + + + + + /scratch/midway/{env.USER}/swiftwork + + + + + + + westmere + 12 + + + + 1 + 1 + 1 + 12 + .11 + + + 00:15:00 + 1800 + + + + /scratch/midway/{env.USER}/swiftwork + + + false + 100 + 100 + 10000 + + + + +Try: + +- running more app() calls at once: change jobsPerNode and jobThrottle + +- requesting more than 1 slurm job at once: change slots + +- requesting nodes in units greater than 1: change maxnodes and nodeGranularity Added: demo/cray/part07/p7.swift =================================================================== --- demo/cray/part07/p7.swift (rev 0) +++ demo/cray/part07/p7.swift 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,27 @@ +type file; + +app (file o) mysim4 (int timesteps, int range, int count) +{ + random4 timesteps range count stdout=@filename(o); +} + +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} + +file sims[]; +int nsim = @toInt(@arg("nsim", "10")); + +int steps = @toInt(@arg("steps", "1")); +int range = @toInt(@arg("range", "100")); +int count = @toInt(@arg("count", "10")); + +foreach i in [0:nsim-1] { + file simout ; + simout = mysim4(steps,range,count); + sims[i] = simout; +} + +file stats<"output/average.out">; +stats = analyze(sims); Added: demo/cray/part07/sites.xml =================================================================== --- demo/cray/part07/sites.xml (rev 0) +++ demo/cray/part07/sites.xml 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,39 @@ + + + + + + /scratch/midway/{env.USER}/swiftwork + + + + + + + westmere + 12 + + + + 1 + 1 + 1 + 12 + .11 + + + 00:15:00 + 1800 + + + + /scratch/midway/{env.USER}/swiftwork + + + false + 100 + 100 + 10000 + + + Added: demo/cray/part07/tc =================================================================== --- demo/cray/part07/tc (rev 0) +++ demo/cray/part07/tc 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,2 @@ +westmere random4 random4.sh +localhost average avg.sh Added: demo/cray/part08/README =================================================================== --- demo/cray/part08/README (rev 0) +++ demo/cray/part08/README 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,81 @@ + +TOPIC: Run the "simulations" on the parallel cluster + + +Til now we've been running our "simulations" on the local login host. + +Now we'll run them on the parallel cluster. + +The script will be the same as previous, but we change the tool catalog to +indicate that our "simulation app" (random4.sh) should be run on the +"westmere" partition of the Midway cluster using the "slurm" scheduler: + +We changed file "tc" from: + +localhost random4 random4.sh +localhost average avg.sh + +to: + +westmere random4 random4.sh +localhost average avg.sh + +And we added the "sites.xml" file which specifies two "pools" of resources, +the "local host" pool (which we've been using till now) and the "westmere" pool. + +To run, we now add the sites.xml file to the swift command line: + +$ swift -tc.file tc -sites.file sites.xml p7.swift -steps=10 -range=1000 -count=6 -nsim=100 + +Note that we're now asking for 100 simulations to be run, each of which +is to run for 10 steps (ie 10 seconds). + +The sites file looks like this: + + + + + + + /scratch/midway/{env.USER}/swiftwork + + + + + + + westmere + 12 + + + + 1 + 1 + 1 + 12 + .11 + + + 00:15:00 + 1800 + + + + /scratch/midway/{env.USER}/swiftwork + + + false + 100 + 100 + 10000 + + + + +Try: + +- running more app() calls at once: change jobsPerNode and jobThrottle + +- requesting more than 1 slurm job at once: change slots + +- requesting nodes in units greater than 1: change maxnodes and nodeGranularity Added: demo/cray/part08/cf =================================================================== --- demo/cray/part08/cf (rev 0) +++ demo/cray/part08/cf 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,9 @@ +user.home=/lustre/beagle/wilde +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.wrapper.staging=false +use.provider.staging=false +provider.staging.pin.swiftfiles=false Added: demo/cray/part08/cray.xml =================================================================== --- demo/cray/part08/cray.xml (rev 0) +++ demo/cray/part08/cray.xml 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,31 @@ + + + + + + /scratch/midway/{env.USER}/swiftwork + + + + + + + pbs.aprun;pbs.mpp;depth=24 + + batch + + 24 + 1800 + + 1 + 1 + 1 + + 12 + 10000 + + + /lustre/beagle/wilde/swift/lab/swiftwork + + + Added: demo/cray/part08/data/bias.dat =================================================================== --- demo/cray/part08/data/bias.dat (rev 0) +++ demo/cray/part08/data/bias.dat 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1 @@ +1000000 123 Added: demo/cray/part08/dm =================================================================== --- demo/cray/part08/dm (rev 0) +++ demo/cray/part08/dm 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1 @@ +rule .*/data.* DIRECT /lustre/beagle/wilde/swift/demo/cray/svn/part08 Added: demo/cray/part08/p8.swift =================================================================== --- demo/cray/part08/p8.swift (rev 0) +++ demo/cray/part08/p8.swift 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,29 @@ +type file; + +app (file o) mysim5 (int timesteps, int range, file bias, int count) +{ + random5 timesteps range @filename(bias) count stdout=@filename(o); +} + +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} + +file sims[]; +int nsim = @toInt(@arg("nsim", "10")); + +int steps = @toInt(@arg("steps", "1")); +int range = @toInt(@arg("range", "100")); +int count = @toInt(@arg("count", "10")); + +file bias<"data/bias.dat">; + +foreach i in [0:nsim-1] { + file simout ; + simout = mysim5(steps,range,bias,count); + sims[i] = simout; +} + +file stats<"output/average.out">; +stats = analyze(sims); Added: demo/cray/part08/pbs.xml =================================================================== --- demo/cray/part08/pbs.xml (rev 0) +++ demo/cray/part08/pbs.xml 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,23 @@ + + + + + + pbs.aprun;pbs.mpp;depth=24 + + batch + + 24 + 1800 + + 1 + 1 + 1 + + 12 + 10000 + + + /lustre/beagle/wilde/swift/lab/swiftwork + + Added: demo/cray/part08/tc =================================================================== --- demo/cray/part08/tc (rev 0) +++ demo/cray/part08/tc 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,2 @@ +cray random5 /lustre/beagle/wilde/swift/demo/cray/svn/bin/random5.sh +cray average /lustre/beagle/wilde/swift/demo/cray/svn/bin/avg.sh Added: demo/cray/setup.csh =================================================================== --- demo/cray/setup.csh (rev 0) +++ demo/cray/setup.csh 2013-04-16 20:54:50 UTC (rev 6451) @@ -0,0 +1,35 @@ + +module load swift + +setenv SBATCH_RESERVATION swift + +set DIR=`dirname ${0}` +cd $DIR +set DIR=`pwd` + +echo Adding $DIR/bin to PATH: +setenv PATH $DIR/bin:$PATH + +mkdir -p $HOME/.swift + +if ( -f $HOME/.swift/swift.properties ) then + echo Adding properties to end of $HOME/.swift/swift.properties +else + echo creating $HOME/.swift/swift.properties +endif + +cat >>$HOME/.swift/swift.properties <>$HOME/.swift/swift.properties < Author: wilde Date: 2013-04-16 17:57:46 -0500 (Tue, 16 Apr 2013) New Revision: 6452 Added: demo/cray/part08/raven.xml Modified: demo/cray/part08/README demo/cray/part08/dm demo/cray/part08/tc Log: Updates for Raven. Fixed cdm file 'dm' for relative paths to data/ dir. Modified: demo/cray/part08/README =================================================================== --- demo/cray/part08/README 2013-04-16 20:54:50 UTC (rev 6451) +++ demo/cray/part08/README 2013-04-16 22:57:46 UTC (rev 6452) @@ -1,4 +1,10 @@ +swift -config cf -tc.file tc -sites.file cray.xml -cdm.file dm p8.swift -steps=1 -range=1000 -count=6 -nsim=1 + + + +OLD: FIXME: + TOPIC: Run the "simulations" on the parallel cluster Modified: demo/cray/part08/dm =================================================================== --- demo/cray/part08/dm 2013-04-16 20:54:50 UTC (rev 6451) +++ demo/cray/part08/dm 2013-04-16 22:57:46 UTC (rev 6452) @@ -1 +1,4 @@ -rule .*/data.* DIRECT /lustre/beagle/wilde/swift/demo/cray/svn/part08 +#rule .*/data.* DIRECT /lustre/beagle/wilde/swift/demo/cray/svn/part08 + +rule .*data/.* DIRECT /home/users/p01532/swift/demo/svn/part08 + Added: demo/cray/part08/raven.xml =================================================================== --- demo/cray/part08/raven.xml (rev 0) +++ demo/cray/part08/raven.xml 2013-04-16 22:57:46 UTC (rev 6452) @@ -0,0 +1,33 @@ + + + + + + /home/users/{env.USER}/swiftwork + + + + + + + pbs.aprun;pbs.mpp;depth=32 + + + small + + 32 + 1800 + + 6 + 4 + 4 + + 6.00 + 10000 + + + /home/users/{env.USER}/swiftwork + + + + Modified: demo/cray/part08/tc =================================================================== --- demo/cray/part08/tc 2013-04-16 20:54:50 UTC (rev 6451) +++ demo/cray/part08/tc 2013-04-16 22:57:46 UTC (rev 6452) @@ -1,2 +1,6 @@ -cray random5 /lustre/beagle/wilde/swift/demo/cray/svn/bin/random5.sh -cray average /lustre/beagle/wilde/swift/demo/cray/svn/bin/avg.sh +#cray random5 /lustre/beagle/wilde/swift/demo/cray/svn/bin/random5.sh +#cray average /lustre/beagle/wilde/swift/demo/cray/svn/bin/avg.sh + +cray random5 /home/users/p01532/swift/demo/svn/bin/random5.sh +cray average /home/users/p01532/swift/demo/svn/bin/avg.sh + From wilde at ci.uchicago.edu Tue Apr 16 18:32:41 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 16 Apr 2013 18:32:41 -0500 (CDT) Subject: [Swift-commit] r6453 - demo/cray/part08 Message-ID: <20130416233241.C26AC9CD1F@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-16 18:32:41 -0500 (Tue, 16 Apr 2013) New Revision: 6453 Added: demo/cray/part08/cf.ps demo/cray/part08/ravenps.xml Modified: demo/cray/part08/README Log: Additional config for provider staging on Raven. Modified: demo/cray/part08/README =================================================================== --- demo/cray/part08/README 2013-04-16 22:57:46 UTC (rev 6452) +++ demo/cray/part08/README 2013-04-16 23:32:41 UTC (rev 6453) @@ -1,8 +1,14 @@ -swift -config cf -tc.file tc -sites.file cray.xml -cdm.file dm p8.swift -steps=1 -range=1000 -count=6 -nsim=1 +Regular staging with direct data management: + swift -config cf -tc.file tc -sites.file cray.xml -cdm.file dm p8.swift -steps=1 -range=1000 -count=6 -nsim=100 +Provider staging: + swift -config cf.ps -tc.file tc -sites.file ravenps.xml p8.swift -steps=5 -range=1000 -count=6 -nsim=100 + + + OLD: FIXME: TOPIC: Run the "simulations" on the parallel cluster Added: demo/cray/part08/cf.ps =================================================================== --- demo/cray/part08/cf.ps (rev 0) +++ demo/cray/part08/cf.ps 2013-04-16 23:32:41 UTC (rev 6453) @@ -0,0 +1,9 @@ +#user.home=/lustre/beagle/wilde +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.wrapper.staging=false +use.provider.staging=true +provider.staging.pin.swiftfiles=true Added: demo/cray/part08/ravenps.xml =================================================================== --- demo/cray/part08/ravenps.xml (rev 0) +++ demo/cray/part08/ravenps.xml 2013-04-16 23:32:41 UTC (rev 6453) @@ -0,0 +1,33 @@ + + + + + + /home/users/{env.USER}/swiftwork + + + + + + + pbs.aprun;pbs.mpp;depth=32 + + + small + + 32 + 1800 + + 6 + 4 + 4 + + 6.00 + 10000 + + + /dev/shm/{env.USER}/swiftwork + + + + From wilde at ci.uchicago.edu Wed Apr 17 10:00:32 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Wed, 17 Apr 2013 10:00:32 -0500 (CDT) Subject: [Swift-commit] r6454 - demo/cray/part08 Message-ID: <20130417150032.B8EC09CD1F@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-17 10:00:32 -0500 (Wed, 17 Apr 2013) New Revision: 6454 Modified: demo/cray/part08/README Log: Test of commit from Raven. Modified: demo/cray/part08/README =================================================================== --- demo/cray/part08/README 2013-04-16 23:32:41 UTC (rev 6453) +++ demo/cray/part08/README 2013-04-17 15:00:32 UTC (rev 6454) @@ -1,4 +1,5 @@ + Regular staging with direct data management: swift -config cf -tc.file tc -sites.file cray.xml -cdm.file dm p8.swift -steps=1 -range=1000 -count=6 -nsim=100 From davidk at ci.uchicago.edu Wed Apr 17 12:02:37 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 17 Apr 2013 12:02:37 -0500 (CDT) Subject: [Swift-commit] r6455 - branches/release-0.94/tests/language-behaviour/cleanup Message-ID: <20130417170237.A96E99CD1F@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-17 12:02:37 -0500 (Wed, 17 Apr 2013) New Revision: 6455 Modified: branches/release-0.94/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh branches/release-0.94/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh Log: Fix for some versions of grep which may cause these tests to fail Modified: branches/release-0.94/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh =================================================================== --- branches/release-0.94/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh 2013-04-17 15:00:32 UTC (rev 6454) +++ branches/release-0.94/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh 2013-04-17 17:02:37 UTC (rev 6455) @@ -12,12 +12,12 @@ exit 1 fi -if ! grep "Cleaning file [file://*$PREFIX.t1.tmp*]" $PREFIX.stdout; then +if ! grep "^Cleaning file file://.*$PREFIX.t1.tmp" $PREFIX.stdout; then echo "Test mapper did not claim to clean t1" exit 1 fi -if ! grep "Not cleaning [file://*$PREFIX.t2.out*]" $PREFIX.stdout; then +if ! grep "^Not cleaning file://.*$PREFIX.t2.out" $PREFIX.stdout; then echo "Test mapper did not identify t2 as persistent" exit 1 fi Modified: branches/release-0.94/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh =================================================================== --- branches/release-0.94/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh 2013-04-17 15:00:32 UTC (rev 6454) +++ branches/release-0.94/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh 2013-04-17 17:02:37 UTC (rev 6455) @@ -17,12 +17,12 @@ exit 1 fi -if ! grep "Cleaning file [file://*$PREFIX.1.tmp*]" $PREFIX.stdout; then +if ! grep "Cleaning file file://.*$PREFIX.1.tmp" $PREFIX.stdout; then echo "Test mapper didn't clean what it should have" exit 1 fi -if ! grep "Not cleaning [file://*$PREFIX.2.out*]" $PREFIX.stdout; then +if ! grep "Not cleaning file://.*$PREFIX.2.out" $PREFIX.stdout; then echo "Test mapper did not identify 2 as persistent" exit 1 fi From davidk at ci.uchicago.edu Wed Apr 17 12:04:57 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 17 Apr 2013 12:04:57 -0500 (CDT) Subject: [Swift-commit] r6456 - trunk/tests/language-behaviour/cleanup Message-ID: <20130417170457.12AA39CD1F@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-17 12:04:56 -0500 (Wed, 17 Apr 2013) New Revision: 6456 Modified: trunk/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh trunk/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh Log: Fix for some versions of grep which may cause these tests to fail Modified: trunk/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh =================================================================== --- trunk/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh 2013-04-17 17:02:37 UTC (rev 6455) +++ trunk/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh 2013-04-17 17:04:56 UTC (rev 6456) @@ -12,12 +12,12 @@ exit 1 fi -if ! grep "Cleaning file [file://*$PREFIX.t1.tmp*]" $PREFIX.stdout; then +if ! grep "^Cleaning file file://.*$PREFIX.t1.tmp" $PREFIX.stdout; then echo "Test mapper did not claim to clean t1" exit 1 fi -if ! grep "Not cleaning [file://*$PREFIX.t2.out*]" $PREFIX.stdout; then +if ! grep "^Not cleaning file://.*$PREFIX.t2.out" $PREFIX.stdout; then echo "Test mapper did not identify t2 as persistent" exit 1 fi Modified: trunk/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh =================================================================== --- trunk/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh 2013-04-17 17:02:37 UTC (rev 6455) +++ trunk/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh 2013-04-17 17:04:56 UTC (rev 6456) @@ -17,12 +17,12 @@ exit 1 fi -if ! grep "Cleaning file [file://*$PREFIX.1.tmp*]" $PREFIX.stdout; then +if ! grep "Cleaning file file://.*$PREFIX.1.tmp" $PREFIX.stdout; then echo "Test mapper didn't clean what it should have" exit 1 fi -if ! grep "Not cleaning [file://*$PREFIX.2.out*]" $PREFIX.stdout; then +if ! grep "Not cleaning file://.*$PREFIX.2.out" $PREFIX.stdout; then echo "Test mapper did not identify 2 as persistent" exit 1 fi From davidk at ci.uchicago.edu Wed Apr 17 12:06:59 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Wed, 17 Apr 2013 12:06:59 -0500 (CDT) Subject: [Swift-commit] r6457 - branches/faster/tests/language-behaviour/cleanup Message-ID: <20130417170659.BB0E09CD1F@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-17 12:06:59 -0500 (Wed, 17 Apr 2013) New Revision: 6457 Modified: branches/faster/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh branches/faster/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh Log: Fix for some versions of grep which may cause these tests to fail Modified: branches/faster/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh =================================================================== --- branches/faster/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh 2013-04-17 17:04:56 UTC (rev 6456) +++ branches/faster/tests/language-behaviour/cleanup/001-proc-scope-cleanup.check.sh 2013-04-17 17:06:59 UTC (rev 6457) @@ -12,12 +12,12 @@ exit 1 fi -if ! grep "Cleaning file [file://*$PREFIX.t1.tmp*]" $PREFIX.stdout; then +if ! grep "^Cleaning file file://.*$PREFIX.t1.tmp" $PREFIX.stdout; then echo "Test mapper did not claim to clean t1" exit 1 fi -if ! grep "Not cleaning [file://*$PREFIX.t2.out*]" $PREFIX.stdout; then +if ! grep "^Not cleaning file://.*$PREFIX.t2.out" $PREFIX.stdout; then echo "Test mapper did not identify t2 as persistent" exit 1 fi Modified: branches/faster/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh =================================================================== --- branches/faster/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh 2013-04-17 17:04:56 UTC (rev 6456) +++ branches/faster/tests/language-behaviour/cleanup/011-proc-scope-alias-cleanup.check.sh 2013-04-17 17:06:59 UTC (rev 6457) @@ -17,12 +17,12 @@ exit 1 fi -if ! grep "Cleaning file [file://*$PREFIX.1.tmp*]" $PREFIX.stdout; then +if ! grep "Cleaning file file://.*$PREFIX.1.tmp" $PREFIX.stdout; then echo "Test mapper didn't clean what it should have" exit 1 fi -if ! grep "Not cleaning [file://*$PREFIX.2.out*]" $PREFIX.stdout; then +if ! grep "Not cleaning file://.*$PREFIX.2.out" $PREFIX.stdout; then echo "Test mapper did not identify 2 as persistent" exit 1 fi From wilde at ci.uchicago.edu Wed Apr 17 14:00:52 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:00:52 -0500 (CDT) Subject: [Swift-commit] r6458 - demo/cray Message-ID: <20130417190052.95EB99CC97@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-17 14:00:52 -0500 (Wed, 17 Apr 2013) New Revision: 6458 Added: demo/cray/swift-1.0/ demo/cray/swift-2.0/ Log: Break demo into Swift/K and Swift/T sections) From wozniak at ci.uchicago.edu Wed Apr 17 14:00:59 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:00:59 -0500 (CDT) Subject: [Swift-commit] r6459 - demo/cray Message-ID: <20130417190059.C535C9CC97@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-17 14:00:59 -0500 (Wed, 17 Apr 2013) New Revision: 6459 Added: demo/cray/swift-t/ Log: Adding swift-t From wozniak at ci.uchicago.edu Wed Apr 17 14:01:34 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:01:34 -0500 (CDT) Subject: [Swift-commit] r6460 - demo/cray/swift-t Message-ID: <20130417190134.EF2969CC97@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-17 14:01:34 -0500 (Wed, 17 Apr 2013) New Revision: 6460 Added: demo/cray/swift-t/part08/ Log: Copy part08 for Swift/T From wilde at ci.uchicago.edu Wed Apr 17 14:04:18 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:04:18 -0500 (CDT) Subject: [Swift-commit] r6461 - in demo/cray: . swift-1.0 Message-ID: <20130417190418.956599CC97@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-17 14:04:18 -0500 (Wed, 17 Apr 2013) New Revision: 6461 Added: demo/cray/swift-1.0/part01/ demo/cray/swift-1.0/part02/ demo/cray/swift-1.0/part03/ demo/cray/swift-1.0/part04/ demo/cray/swift-1.0/part05/ demo/cray/swift-1.0/part06/ demo/cray/swift-1.0/part07/ demo/cray/swift-1.0/part08/ Removed: demo/cray/part01/ demo/cray/part02/ demo/cray/part03/ demo/cray/part04/ demo/cray/part05/ demo/cray/part06/ demo/cray/part07/ demo/cray/part08/ Log: Move current demo parts to swift-1.0. Can copy from there for swift-2.0 as desired. From wozniak at ci.uchicago.edu Wed Apr 17 14:07:58 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:07:58 -0500 (CDT) Subject: [Swift-commit] r6462 - in demo/cray/swift-2.0: . part08 Message-ID: <20130417190758.96C749CC97@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-17 14:07:58 -0500 (Wed, 17 Apr 2013) New Revision: 6462 Added: demo/cray/swift-2.0/part08/ Removed: demo/cray/swift-2.0/part08/cf demo/cray/swift-2.0/part08/cf.ps demo/cray/swift-2.0/part08/cray.xml demo/cray/swift-2.0/part08/pbs.xml demo/cray/swift-2.0/part08/raven.xml demo/cray/swift-2.0/part08/ravenps.xml demo/cray/swift-2.0/part08/tc Log: Copy part08, drop swift-1.0 cfg files Deleted: demo/cray/swift-2.0/part08/cf =================================================================== --- demo/cray/swift-1.0/part08/cf 2013-04-17 19:04:18 UTC (rev 6461) +++ demo/cray/swift-2.0/part08/cf 2013-04-17 19:07:58 UTC (rev 6462) @@ -1,9 +0,0 @@ -user.home=/lustre/beagle/wilde -wrapperlog.always.transfer=true -sitedir.keep=true -execution.retries=0 -lazy.errors=false -status.mode=provider -use.wrapper.staging=false -use.provider.staging=false -provider.staging.pin.swiftfiles=false Deleted: demo/cray/swift-2.0/part08/cf.ps =================================================================== --- demo/cray/swift-1.0/part08/cf.ps 2013-04-17 19:04:18 UTC (rev 6461) +++ demo/cray/swift-2.0/part08/cf.ps 2013-04-17 19:07:58 UTC (rev 6462) @@ -1,9 +0,0 @@ -#user.home=/lustre/beagle/wilde -wrapperlog.always.transfer=true -sitedir.keep=true -execution.retries=0 -lazy.errors=false -status.mode=provider -use.wrapper.staging=false -use.provider.staging=true -provider.staging.pin.swiftfiles=true Deleted: demo/cray/swift-2.0/part08/cray.xml =================================================================== --- demo/cray/swift-1.0/part08/cray.xml 2013-04-17 19:04:18 UTC (rev 6461) +++ demo/cray/swift-2.0/part08/cray.xml 2013-04-17 19:07:58 UTC (rev 6462) @@ -1,31 +0,0 @@ - - - - - - /scratch/midway/{env.USER}/swiftwork - - - - - - - pbs.aprun;pbs.mpp;depth=24 - - batch - - 24 - 1800 - - 1 - 1 - 1 - - 12 - 10000 - - - /lustre/beagle/wilde/swift/lab/swiftwork - - - Deleted: demo/cray/swift-2.0/part08/pbs.xml =================================================================== --- demo/cray/swift-1.0/part08/pbs.xml 2013-04-17 19:04:18 UTC (rev 6461) +++ demo/cray/swift-2.0/part08/pbs.xml 2013-04-17 19:07:58 UTC (rev 6462) @@ -1,23 +0,0 @@ - - - - - - pbs.aprun;pbs.mpp;depth=24 - - batch - - 24 - 1800 - - 1 - 1 - 1 - - 12 - 10000 - - - /lustre/beagle/wilde/swift/lab/swiftwork - - Deleted: demo/cray/swift-2.0/part08/raven.xml =================================================================== --- demo/cray/swift-1.0/part08/raven.xml 2013-04-17 19:04:18 UTC (rev 6461) +++ demo/cray/swift-2.0/part08/raven.xml 2013-04-17 19:07:58 UTC (rev 6462) @@ -1,33 +0,0 @@ - - - - - - /home/users/{env.USER}/swiftwork - - - - - - - pbs.aprun;pbs.mpp;depth=32 - - - small - - 32 - 1800 - - 6 - 4 - 4 - - 6.00 - 10000 - - - /home/users/{env.USER}/swiftwork - - - - Deleted: demo/cray/swift-2.0/part08/ravenps.xml =================================================================== --- demo/cray/swift-1.0/part08/ravenps.xml 2013-04-17 19:04:18 UTC (rev 6461) +++ demo/cray/swift-2.0/part08/ravenps.xml 2013-04-17 19:07:58 UTC (rev 6462) @@ -1,33 +0,0 @@ - - - - - - /home/users/{env.USER}/swiftwork - - - - - - - pbs.aprun;pbs.mpp;depth=32 - - - small - - 32 - 1800 - - 6 - 4 - 4 - - 6.00 - 10000 - - - /dev/shm/{env.USER}/swiftwork - - - - Deleted: demo/cray/swift-2.0/part08/tc =================================================================== --- demo/cray/swift-1.0/part08/tc 2013-04-17 19:04:18 UTC (rev 6461) +++ demo/cray/swift-2.0/part08/tc 2013-04-17 19:07:58 UTC (rev 6462) @@ -1,6 +0,0 @@ -#cray random5 /lustre/beagle/wilde/swift/demo/cray/svn/bin/random5.sh -#cray average /lustre/beagle/wilde/swift/demo/cray/svn/bin/avg.sh - -cray random5 /home/users/p01532/swift/demo/svn/bin/random5.sh -cray average /home/users/p01532/swift/demo/svn/bin/avg.sh - From wilde at ci.uchicago.edu Wed Apr 17 14:28:45 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:28:45 -0500 (CDT) Subject: [Swift-commit] r6463 - in demo/cray: . swift-1.0 Message-ID: <20130417192845.50CE89CD1F@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-17 14:28:45 -0500 (Wed, 17 Apr 2013) New Revision: 6463 Added: demo/cray/swift-1.0/README demo/cray/swift-1.0/setup.csh demo/cray/swift-1.0/setup.sh Removed: demo/cray/README demo/cray/setup.csh demo/cray/setup.sh Log: a bit more re-working for the T/K split. Added a common intro README. Deleted: demo/cray/README =================================================================== --- demo/cray/README 2013-04-17 19:07:58 UTC (rev 6462) +++ demo/cray/README 2013-04-17 19:28:45 UTC (rev 6463) @@ -1,52 +0,0 @@ - -INTRODUCTION TO SWIFT - -SETUP - -Obtain the tutorial package - -Change directory into the created directory called "tutorial" - -This directory contains: - -bin: script tools for the tutorial -scripts: Swift scripts for the tutorial - -TO START, do: - -$ source setup.sh # to add tutorial programs to your PATH - -...and then verify: - -$ random.sh # should be found in your PATH now - - - -The tutorial is arranged in parts. To begin: - -$ cd part01 -$ cat README - -When finished: - -$ cd ../part02 # etc - -In each part, you can type "cleanup" after running a few swift scripts -to remove the old logs that build up, and (usually) the output files. - - -The topics of the tutorial parts are: - -part01: First Swift script - run a "simulation" program, random.sh - -part02: Specifying the output filenames - -part03: Run many instances of a "simulation" program in a foreach() loop - -part04: run a program in a foreach() loop with explicitly named output files. - -part05: Add a program that "analyzes" the output the parallel simulations - -part06: Add arguments to the "simulation" program - -part07: Run the "simulations" on the parallel cluster Deleted: demo/cray/setup.csh =================================================================== --- demo/cray/setup.csh 2013-04-17 19:07:58 UTC (rev 6462) +++ demo/cray/setup.csh 2013-04-17 19:28:45 UTC (rev 6463) @@ -1,35 +0,0 @@ - -module load swift - -setenv SBATCH_RESERVATION swift - -set DIR=`dirname ${0}` -cd $DIR -set DIR=`pwd` - -echo Adding $DIR/bin to PATH: -setenv PATH $DIR/bin:$PATH - -mkdir -p $HOME/.swift - -if ( -f $HOME/.swift/swift.properties ) then - echo Adding properties to end of $HOME/.swift/swift.properties -else - echo creating $HOME/.swift/swift.properties -endif - -cat >>$HOME/.swift/swift.properties <>$HOME/.swift/swift.properties <>$HOME/.swift/swift.properties <>$HOME/.swift/swift.properties < Author: wozniak Date: 2013-04-17 14:28:47 -0500 (Wed, 17 Apr 2013) New Revision: 6464 Modified: demo/cray/bin/random5.sh Log: This script requires bash (sh is not bash on Ubuntu) Modified: demo/cray/bin/random5.sh =================================================================== --- demo/cray/bin/random5.sh 2013-04-17 19:28:45 UTC (rev 6463) +++ demo/cray/bin/random5.sh 2013-04-17 19:28:47 UTC (rev 6464) @@ -1,4 +1,4 @@ -#! /bin/sh +#! /bin/bash runtime=${1:-0} range=$2 From wilde at ci.uchicago.edu Wed Apr 17 14:35:11 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:35:11 -0500 (CDT) Subject: [Swift-commit] r6465 - demo/cray/swift-1.0 Message-ID: <20130417193511.73CA79CD1F@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-17 14:35:11 -0500 (Wed, 17 Apr 2013) New Revision: 6465 Added: demo/cray/swift-1.0/part09/ Log: create part09 for the provider staging demo. From wilde at ci.uchicago.edu Wed Apr 17 14:38:05 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:38:05 -0500 (CDT) Subject: [Swift-commit] r6466 - demo/cray/swift-1.0/part08 Message-ID: <20130417193805.4746B9CD1F@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-17 14:38:05 -0500 (Wed, 17 Apr 2013) New Revision: 6466 Added: demo/cray/swift-1.0/part08/cf.ps demo/cray/swift-1.0/part08/ravenps.xml Removed: demo/cray/swift-1.0/part08/cf.ps demo/cray/swift-1.0/part08/ravenps.xml Log: Decided to leave provider staging in the same demo part so user can compare the configs Deleted: demo/cray/swift-1.0/part08/cf.ps =================================================================== --- demo/cray/swift-1.0/part08/cf.ps 2013-04-17 19:35:11 UTC (rev 6465) +++ demo/cray/swift-1.0/part08/cf.ps 2013-04-17 19:38:05 UTC (rev 6466) @@ -1,9 +0,0 @@ -#user.home=/lustre/beagle/wilde -wrapperlog.always.transfer=true -sitedir.keep=true -execution.retries=0 -lazy.errors=false -status.mode=provider -use.wrapper.staging=false -use.provider.staging=true -provider.staging.pin.swiftfiles=true Added: demo/cray/swift-1.0/part08/cf.ps =================================================================== --- demo/cray/swift-1.0/part08/cf.ps (rev 0) +++ demo/cray/swift-1.0/part08/cf.ps 2013-04-17 19:38:05 UTC (rev 6466) @@ -0,0 +1,9 @@ +#user.home=/lustre/beagle/wilde +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.wrapper.staging=false +use.provider.staging=true +provider.staging.pin.swiftfiles=true Deleted: demo/cray/swift-1.0/part08/ravenps.xml =================================================================== --- demo/cray/swift-1.0/part08/ravenps.xml 2013-04-17 19:35:11 UTC (rev 6465) +++ demo/cray/swift-1.0/part08/ravenps.xml 2013-04-17 19:38:05 UTC (rev 6466) @@ -1,33 +0,0 @@ - - - - - - /home/users/{env.USER}/swiftwork - - - - - - - pbs.aprun;pbs.mpp;depth=32 - - - small - - 32 - 1800 - - 6 - 4 - 4 - - 6.00 - 10000 - - - /dev/shm/{env.USER}/swiftwork - - - - Added: demo/cray/swift-1.0/part08/ravenps.xml =================================================================== --- demo/cray/swift-1.0/part08/ravenps.xml (rev 0) +++ demo/cray/swift-1.0/part08/ravenps.xml 2013-04-17 19:38:05 UTC (rev 6466) @@ -0,0 +1,33 @@ + + + + + + /home/users/{env.USER}/swiftwork + + + + + + + pbs.aprun;pbs.mpp;depth=32 + + + small + + 32 + 1800 + + 6 + 4 + 4 + + 6.00 + 10000 + + + /dev/shm/{env.USER}/swiftwork + + + + From wozniak at ci.uchicago.edu Wed Apr 17 14:45:29 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:45:29 -0500 (CDT) Subject: [Swift-commit] r6467 - demo/cray/swift-2.0/part08 Message-ID: <20130417194529.A68A19CD1F@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-17 14:45:29 -0500 (Wed, 17 Apr 2013) New Revision: 6467 Modified: demo/cray/swift-2.0/part08/p8.swift Log: Convert to Swift/T Modified: demo/cray/swift-2.0/part08/p8.swift =================================================================== --- demo/cray/swift-2.0/part08/p8.swift 2013-04-17 19:38:05 UTC (rev 6466) +++ demo/cray/swift-2.0/part08/p8.swift 2013-04-17 19:45:29 UTC (rev 6467) @@ -1,29 +1,38 @@ -type file; -app (file o) mysim5 (int timesteps, int range, file bias, int count) +#include +#include +#include +#include +#include + +app (file o) mysim5 (int timesteps, int rnge, file bias, int count) { - random5 timesteps range @filename(bias) count stdout=@filename(o); + "random5.sh" timesteps rnge bias count @stdout=o; } app (file o) analyze (file s[]) { - average @filenames(s) stdout=@filename(o); + "avg.sh" s @stdout=o; } -file sims[]; -int nsim = @toInt(@arg("nsim", "10")); -int steps = @toInt(@arg("steps", "1")); -int range = @toInt(@arg("range", "100")); -int count = @toInt(@arg("count", "10")); +main +{ + file sims[]; + int nsim = toint(argv("nsim", "10")); -file bias<"data/bias.dat">; + int steps = toint(argv("steps", "1")); + int rnge = toint(argv("range", "100")); + int count = toint(argv("count", "10")); -foreach i in [0:nsim-1] { - file simout ; - simout = mysim5(steps,range,bias,count); - sims[i] = simout; + file bias = input_file("data/bias.dat"); + + foreach i in [0:nsim-1] { + file simout ; + simout = mysim5(steps,rnge,bias,count); + sims[i] = simout; + } + + file stats<"output/average.out">; + stats = analyze(sims); } - -file stats<"output/average.out">; -stats = analyze(sims); From wozniak at ci.uchicago.edu Wed Apr 17 14:45:37 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:45:37 -0500 (CDT) Subject: [Swift-commit] r6468 - demo/cray/swift-2.0/part08 Message-ID: <20130417194537.1C4DD178884@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-17 14:45:36 -0500 (Wed, 17 Apr 2013) New Revision: 6468 Removed: demo/cray/swift-2.0/part08/dm Log: Drop unused file Deleted: demo/cray/swift-2.0/part08/dm =================================================================== --- demo/cray/swift-2.0/part08/dm 2013-04-17 19:45:29 UTC (rev 6467) +++ demo/cray/swift-2.0/part08/dm 2013-04-17 19:45:36 UTC (rev 6468) @@ -1,4 +0,0 @@ -#rule .*/data.* DIRECT /lustre/beagle/wilde/swift/demo/cray/svn/part08 - -rule .*data/.* DIRECT /home/users/p01532/swift/demo/svn/part08 - From wozniak at ci.uchicago.edu Wed Apr 17 14:51:35 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:51:35 -0500 (CDT) Subject: [Swift-commit] r6469 - demo/cray/bin Message-ID: <20130417195135.A19179CD1F@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-17 14:51:35 -0500 (Wed, 17 Apr 2013) New Revision: 6469 Added: demo/cray/bin/run-swift-2-local.sh Log: Run script for swift-2.0 Added: demo/cray/bin/run-swift-2-local.sh =================================================================== --- demo/cray/bin/run-swift-2-local.sh (rev 0) +++ demo/cray/bin/run-swift-2-local.sh 2013-04-17 19:51:35 UTC (rev 6469) @@ -0,0 +1,15 @@ +#!/bin/bash + +SCRIPT_SWIFT=$1 +shift +ARGS=${*} + +SCRIPT_TCL=${SCRIPT_SWIFT%.swift}.tcl + +stc ${SCRIPT_SWIFT} ${SCRIPT_TCL} || exit 1 + +export TURBINE_LOG=0 +turbine ${SCRIPT_TCL} ${ARGS} + + + Property changes on: demo/cray/bin/run-swift-2-local.sh ___________________________________________________________________ Added: svn:executable + * From wozniak at ci.uchicago.edu Wed Apr 17 14:52:32 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Wed, 17 Apr 2013 14:52:32 -0500 (CDT) Subject: [Swift-commit] r6470 - demo/cray/swift-2.0/part08 Message-ID: <20130417195232.6A4599CD1F@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-17 14:52:32 -0500 (Wed, 17 Apr 2013) New Revision: 6470 Modified: demo/cray/swift-2.0/part08/README demo/cray/swift-2.0/part08/p8.swift Log: Update README; add success message Modified: demo/cray/swift-2.0/part08/README =================================================================== --- demo/cray/swift-2.0/part08/README 2013-04-17 19:51:35 UTC (rev 6469) +++ demo/cray/swift-2.0/part08/README 2013-04-17 19:52:32 UTC (rev 6470) @@ -1,94 +1,4 @@ +To run: -Regular staging with direct data management: - - swift -config cf -tc.file tc -sites.file cray.xml -cdm.file dm p8.swift -steps=1 -range=1000 -count=6 -nsim=100 - -Provider staging: - - swift -config cf.ps -tc.file tc -sites.file ravenps.xml p8.swift -steps=5 -range=1000 -count=6 -nsim=100 - - - -OLD: FIXME: - -TOPIC: Run the "simulations" on the parallel cluster - - -Til now we've been running our "simulations" on the local login host. - -Now we'll run them on the parallel cluster. - -The script will be the same as previous, but we change the tool catalog to -indicate that our "simulation app" (random4.sh) should be run on the -"westmere" partition of the Midway cluster using the "slurm" scheduler: - -We changed file "tc" from: - -localhost random4 random4.sh -localhost average avg.sh - -to: - -westmere random4 random4.sh -localhost average avg.sh - -And we added the "sites.xml" file which specifies two "pools" of resources, -the "local host" pool (which we've been using till now) and the "westmere" pool. - -To run, we now add the sites.xml file to the swift command line: - -$ swift -tc.file tc -sites.file sites.xml p7.swift -steps=10 -range=1000 -count=6 -nsim=100 - -Note that we're now asking for 100 simulations to be run, each of which -is to run for 10 steps (ie 10 seconds). - -The sites file looks like this: - - - - - - - /scratch/midway/{env.USER}/swiftwork - - - - - - - westmere - 12 - - - - 1 - 1 - 1 - 12 - .11 - - - 00:15:00 - 1800 - - - - /scratch/midway/{env.USER}/swiftwork - - - false - 100 - 100 - 10000 - - - - -Try: - -- running more app() calls at once: change jobsPerNode and jobThrottle - -- requesting more than 1 slurm job at once: change slots - -- requesting nodes in units greater than 1: change maxnodes and nodeGranularity +run-swift-2-local.sh p8.swift --steps=0 ... Modified: demo/cray/swift-2.0/part08/p8.swift =================================================================== --- demo/cray/swift-2.0/part08/p8.swift 2013-04-17 19:51:35 UTC (rev 6469) +++ demo/cray/swift-2.0/part08/p8.swift 2013-04-17 19:52:32 UTC (rev 6470) @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -35,4 +36,7 @@ file stats<"output/average.out">; stats = analyze(sims); + wait (stats) { + printf("SUCCESS!"); + } } From wilde at ci.uchicago.edu Wed Apr 17 19:13:29 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Wed, 17 Apr 2013 19:13:29 -0500 (CDT) Subject: [Swift-commit] r6471 - in demo/cray/swift-1.0: . part00 part08 Message-ID: <20130418001329.86B069CD1F@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-17 19:13:29 -0500 (Wed, 17 Apr 2013) New Revision: 6471 Added: demo/cray/swift-1.0/part00/ demo/cray/swift-1.0/part00/README demo/cray/swift-1.0/part00/p0.swift demo/cray/swift-1.0/part00/p0.swift.original demo/cray/swift-1.0/part08/apps demo/cray/swift-1.0/part08/dmrules Removed: demo/cray/swift-1.0/part08/dm demo/cray/swift-1.0/part08/tc Modified: demo/cray/swift-1.0/part08/README demo/cray/swift-1.0/part08/cf demo/cray/swift-1.0/part08/raven.xml demo/cray/swift-1.0/part08/ravenps.xml demo/cray/swift-1.0/setup.sh Log: Add part00 for a basic intro to Swift language elements. Added: demo/cray/swift-1.0/part00/README =================================================================== --- demo/cray/swift-1.0/part00/README (rev 0) +++ demo/cray/swift-1.0/part00/README 2013-04-18 00:13:29 UTC (rev 6471) @@ -0,0 +1,21 @@ + +TOPIC: Basics of the Swift language + +A walk through comments, trace() statements (Swift's only form of direct +printing, variables, arrays, and loops. + + +/*Remove the comments marked <--, one at a time, + to try the following examples + using "swift p0.swift" +*/ + + +// (1) Declare, assign, and trace variables: + +/* <-- (1) + +p0.swift.original is provided in case you need to get back to the original +script. + + Added: demo/cray/swift-1.0/part00/p0.swift =================================================================== --- demo/cray/swift-1.0/part00/p0.swift (rev 0) +++ demo/cray/swift-1.0/part00/p0.swift 2013-04-18 00:13:29 UTC (rev 6471) @@ -0,0 +1,76 @@ + +tracef("Hello, World!\n"); + +# this is a one-line comment + +// as it this + +/* This is a + multi-line c + +*/ + +/*Remove the comments marked <--, one at a time, + to try the following examples + using "swift p0.swift" +*/ + + +// (1) Declare, assign, and trace variables: + +/* <-- (1) +int n; +string str; + +n=99; +str = "\n\n hi there! \n\n"; + +tracef("\n n=%i str=%s\n\n", n, str); + + +// (2) Declare and trace a vector + +/* <-- (2) +int[] ivec = [12,43,56,87,123]; +tracef("ivec: %q\n",ivec); +trace("ivec",ivec); + + +// (3) foreach: do each loop body in parallel + +/* <-- (3) +foreach i, value in ivec { + tracef("i=%i value=%i\n", i, value); +} + + +// (4) Use an array constructor to control a foreach loop + +/* <-- (4) +foreach j, value in [0:9] { + tracef("j=%i value=%i\n", j, value); +} + + +// (5) Data flow evaluation: the statements below execute in reverse order + +/* <-- (5) +int a, b, c; + +c = 1234; +a = b; +b = c; + +trace("dataflow", a, b, c); + + +// (6) The iterate statement executes sequentially + +/* <-- (6) +iterate iter { + trace("iter",iter); +} until(iter==10); + +*/ + +// TO ADD: functions, if, select... Associative arrays... More? Added: demo/cray/swift-1.0/part00/p0.swift.original =================================================================== --- demo/cray/swift-1.0/part00/p0.swift.original (rev 0) +++ demo/cray/swift-1.0/part00/p0.swift.original 2013-04-18 00:13:29 UTC (rev 6471) @@ -0,0 +1,74 @@ + +tracef("Hello, World!\n"); + +# this is a one-line comment + +// as it this + +/* This is a + multi-line c + +*/ + +/*Remove the comments marked <--, one at a time, + to try the following examples + using "swift p0.swift" +*/ + + +// (1) Declare, assign, and trace variables: + +/* <-- (1) +int n; +string str; + +n=99; +str = "\n\n hi there! \n\n"; + +tracef("\n n=%i str=%s\n\n", n, str); + + +// (2) Declare and trace a vector + +/* <-- (2) +int[] ivec = [12,43,56,87,123]; +tracef("ivec: %q\n",ivec); +trace("ivec",ivec); + + +// (3) foreach: do each loop body in parallel + +/* <-- (3) +foreach i, value in ivec { + tracef("i=%i value=%i\n", i, value); +} + + +// (4) Use an array constructor to control a foreach loop + +/* <-- (4) +foreach j, value in [0:9] { + tracef("j=%i value=%i\n", j, value); +} + + +// (5) Data flow evaluation: the statements below execute in reverse order + +/* <-- (5) +int a, b, c; + +c = 1234; +a = b; +b = c; + +trace("dataflow", a, b, c); + + +// (6) The iterate statement executes sequentially + +/* <-- (6) +iterate iter { + trace("iter",iter); +} until(iter==10); + +*/ Modified: demo/cray/swift-1.0/part08/README =================================================================== --- demo/cray/swift-1.0/part08/README 2013-04-17 19:52:32 UTC (rev 6470) +++ demo/cray/swift-1.0/part08/README 2013-04-18 00:13:29 UTC (rev 6471) @@ -1,15 +1,27 @@ +Regular staging with deafult data management: + swift -config cf -tc.file apps -sites.file cray.xml p8.swift -steps=1 -range=1000 -count=6 -nsim=100 + Regular staging with direct data management: - swift -config cf -tc.file tc -sites.file cray.xml -cdm.file dm p8.swift -steps=1 -range=1000 -count=6 -nsim=100 + swift -config cf -tc.file apps -sites.file cray.xml -cdm.file dmrules p8.swift -steps=1 -range=1000 -count=6 -nsim=100 Provider staging: - swift -config cf.ps -tc.file tc -sites.file ravenps.xml p8.swift -steps=5 -range=1000 -count=6 -nsim=100 + swift -config cf.ps -tc.file apps -sites.file ravenps.xml p8.swift -steps=5 -range=1000 -count=6 -nsim=100 +Adjust the job sizes: + 6 + 4 + 4 + + +The app list + + OLD: FIXME: TOPIC: Run the "simulations" on the parallel cluster Copied: demo/cray/swift-1.0/part08/apps (from rev 6464, demo/cray/swift-1.0/part08/tc) =================================================================== --- demo/cray/swift-1.0/part08/apps (rev 0) +++ demo/cray/swift-1.0/part08/apps 2013-04-18 00:13:29 UTC (rev 6471) @@ -0,0 +1,8 @@ +#cray random5 /lustre/beagle/wilde/swift/demo/cray/svn/bin/random5.sh +#cray average /lustre/beagle/wilde/swift/demo/cray/svn/bin/avg.sh + +#cray random5 /home/users/p01532/swift/demo/svn/bin/random5.sh +#cray average /home/users/p01532/swift/demo/svn/bin/avg.sh + +cray random5 random5.sh +cray average avg.sh Modified: demo/cray/swift-1.0/part08/cf =================================================================== --- demo/cray/swift-1.0/part08/cf 2013-04-17 19:52:32 UTC (rev 6470) +++ demo/cray/swift-1.0/part08/cf 2013-04-18 00:13:29 UTC (rev 6471) @@ -1,4 +1,11 @@ -user.home=/lustre/beagle/wilde +#sites.file=${swift.home}/etc/sites.xml +#tc.file=${swift.home}/etc/tc.data + +sites.file=./raven.xml +tc.file=./apps + +#user.home=/lustre/beagle/wilde + wrapperlog.always.transfer=true sitedir.keep=true execution.retries=0 Deleted: demo/cray/swift-1.0/part08/dm =================================================================== --- demo/cray/swift-1.0/part08/dm 2013-04-17 19:52:32 UTC (rev 6470) +++ demo/cray/swift-1.0/part08/dm 2013-04-18 00:13:29 UTC (rev 6471) @@ -1,4 +0,0 @@ -#rule .*/data.* DIRECT /lustre/beagle/wilde/swift/demo/cray/svn/part08 - -rule .*data/.* DIRECT /home/users/p01532/swift/demo/svn/part08 - Copied: demo/cray/swift-1.0/part08/dmrules (from rev 6464, demo/cray/swift-1.0/part08/dm) =================================================================== --- demo/cray/swift-1.0/part08/dmrules (rev 0) +++ demo/cray/swift-1.0/part08/dmrules 2013-04-18 00:13:29 UTC (rev 6471) @@ -0,0 +1,4 @@ +#rule .*/data.* DIRECT /lustre/beagle/wilde/swift/demo/cray/svn/part08 + +rule .*data/.* DIRECT /home/users/p01532/swift/demo/svn/part08 + Modified: demo/cray/swift-1.0/part08/raven.xml =================================================================== --- demo/cray/swift-1.0/part08/raven.xml 2013-04-17 19:52:32 UTC (rev 6470) +++ demo/cray/swift-1.0/part08/raven.xml 2013-04-18 00:13:29 UTC (rev 6471) @@ -3,7 +3,7 @@ - /home/users/{env.USER}/swiftwork + {env.PWD}/swiftwork @@ -18,15 +18,17 @@ 32 1800 - 6 - 4 - 4 + 1 + 1 + 1 6.00 10000 + {env.PATHPREFIX} + - /home/users/{env.USER}/swiftwork + {env.PWD}/swiftwork Modified: demo/cray/swift-1.0/part08/ravenps.xml =================================================================== --- demo/cray/swift-1.0/part08/ravenps.xml 2013-04-17 19:52:32 UTC (rev 6470) +++ demo/cray/swift-1.0/part08/ravenps.xml 2013-04-18 00:13:29 UTC (rev 6471) @@ -3,7 +3,7 @@ - /home/users/{env.USER}/swiftwork + {env.PWD}/swiftwork @@ -18,13 +18,15 @@ 32 1800 - 6 - 4 - 4 + 1 + 1 + 1 6.00 10000 + {env.PATHPREFIX} + /dev/shm/{env.USER}/swiftwork Deleted: demo/cray/swift-1.0/part08/tc =================================================================== --- demo/cray/swift-1.0/part08/tc 2013-04-17 19:52:32 UTC (rev 6470) +++ demo/cray/swift-1.0/part08/tc 2013-04-18 00:13:29 UTC (rev 6471) @@ -1,6 +0,0 @@ -#cray random5 /lustre/beagle/wilde/swift/demo/cray/svn/bin/random5.sh -#cray average /lustre/beagle/wilde/swift/demo/cray/svn/bin/avg.sh - -cray random5 /home/users/p01532/swift/demo/svn/bin/random5.sh -cray average /home/users/p01532/swift/demo/svn/bin/avg.sh - Modified: demo/cray/swift-1.0/setup.sh =================================================================== --- demo/cray/swift-1.0/setup.sh 2013-04-17 19:52:32 UTC (rev 6470) +++ demo/cray/swift-1.0/setup.sh 2013-04-18 00:13:29 UTC (rev 6471) @@ -1,18 +1,20 @@ -module load swift +# module load java # to get Oracle/Sun JDK +# module load swift -export SBATCH_RESERVATION=swift -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +# export SBATCH_RESERVATION=swift -echo Adding $DIR/bin to PATH: +export PATHPREFIX="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../bin" && pwd )" +echo Set PATHPREFIX=$PATHPREFIX -PATH=$DIR/bin:$PATH +PATH=/home/users/p01532/swift/rev/0.94/bin:$PATHPREFIX:$PATH -echo $PATH +return mkdir -p $HOME/.swift + if [ -f $HOME/.swift/swift.properties ]; then echo Adding properties to end of $HOME/.swift/swift.properties else From davidk at ci.uchicago.edu Thu Apr 18 01:35:29 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Thu, 18 Apr 2013 01:35:29 -0500 (CDT) Subject: [Swift-commit] r6472 - in trunk/docs: . designs Message-ID: <20130418063529.A565A9CD08@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-18 01:35:29 -0500 (Thu, 18 Apr 2013) New Revision: 6472 Added: trunk/docs/designs/ trunk/docs/designs/images trunk/docs/designs/swiftrun.txt Log: A place to store ideas for future improvements Start of a swiftrun document Added: trunk/docs/designs/images =================================================================== --- trunk/docs/designs/images (rev 0) +++ trunk/docs/designs/images 2013-04-18 06:35:29 UTC (rev 6472) @@ -0,0 +1 @@ +link ../tutorial/images \ No newline at end of file Property changes on: trunk/docs/designs/images ___________________________________________________________________ Added: svn:special + * Added: trunk/docs/designs/swiftrun.txt =================================================================== --- trunk/docs/designs/swiftrun.txt (rev 0) +++ trunk/docs/designs/swiftrun.txt 2013-04-18 06:35:29 UTC (rev 6472) @@ -0,0 +1,197 @@ +Swiftrun +======== + +:toc: +:icons: +:numbered: + +New method for running Swift +---------------------------- +This document describes a new method of running which aims to make +Swift easier to use. + +Currently, Swift configuration settings are split up into multiple files. +Sites.xml defines the site definition, tc.data defines applications, +swift.properties defines other various settings, and cdm.data defines CDM +rules. + +Below is a fairly typical Swift command: +----- +swift -sites.file sites.xml -tc.file tc.data -config cf -cdm.file fs.data hello.swift +----- + +We should keep this syntax for backward compatibility, but try to simplify +this and also add some flexibility. There are a few ways to go about this. + +.Goals +1. Define all settings in a single, swift.properties-like file. +2. Make the settings in swift.properties simple to understand, and use a + consistent format for all settings (eliminate the need to + know the formats of tc.data and sites.xml). +3. Make it easier to use predefined site templates +4. Increase flexibility when running on multiple sites. + +Swift Properties +---------------- +This new configuration file will be very similar to the current format of +swift.properties. Settings that are currently defined in swift.properties +will keep the same names and formatting. + +----- +execution.retries=0 +sitedir.keep=true +----- + +Where will Swift look for swift.properties? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Settings for swift.properties get set in the following order. + +1. Swift default values +2. The etc/swift.properties file in the swift installation directory +3. The swift.properties file located in the $SWIFT_CONFIG directory +4. $HOME/.swift/swift.properties +5. One or both of $PWD/swift.properties and $PWD/.swift/swift.properties (to + be determined) + +Settings are accumulative. Every swift.properties files that is found is read +and values are set, with values from the most recent file taking precedence. + +Apps +---- +Applications currently defined in tc.data will instead be defined in the +new swift.properties file in the following format: + +----- +app.beagle.wc = /usr/bin/wc +app.midway.bash = /bin/bash +----- + +Special cases +~~~~~~~~~~~~~ +For most app definitions you need only the path to the application. You should +also be able to handle special cases in a similar way. For example, to define +that an app is WIN32, or to set an app specific walltime: +----- +app.beagle.wc.win32 = true +app.beagle.wc.walltime = 00:05:00 +----- + +App wildcards +~~~~~~~~~~~~~ +The existing tc.data format should be modified to allow wildcards. A * in your +tc.data would allow you to run any application defined in your swift script on +the site. It would search your PATH for the executable location. + +----- +beagle * * null null null +----- + +In the new tc format, this same expression would be: +----- +app.beagle.* = * +----- + +CDM +--- +CDM configuration files will also be defined in swift.properties. + +----- +pathrule.type=path,path # type=DIRECT,LOCAL, etc +---- + +Sites +----- +To specify which site a swift script should run on, use the -site command line argument. + +----- +$ swift -site midway hello.swift +----- + +To run on multiple sites, separate site names by a comma. + +----- +swift -site midway,beagle hello.swift +----- + +NOTE: Perhaps we should also think about adding the ability to define "site + groups". For example, the group "xsede" could define the sites for + Trestles, Stampede, and Kraken. You could then also mix and match + multiple groups. + +How does Swift know how to run on these sites? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Swift uses XML to shape a specific run. These XML files define things that +Swift needs to know about a particular site, for example, which scheduler +should be used, what is the location of the work directory, and what degree +of parallelism should be used. + +Swift will include several XML templates that allow a user to run on a +variety of commonly used sites. These site templates could be found in +multiple locations. + +. In your current working directory, Swift will search for a matching + template in $PWD/.swift/sites. + +. Next, Swift will look in $HOME/.swift/sites. + +. Directories defined in $SWIFT_TEMPLATE_PATH. The format of this environment + variable is similar to $PATH, /dir/location1:/dir/location2:/dir/location3. + +. Finally, the Swift installation directory etc/sites. + +How can I change the the values in these templates dynamically? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Values defined in your Swift configuration file can override the default values +stored in XML templates. Here is one example. + +Assume that you are looking at a site templates that looks like this: +----- + + + 24 + 100 + 100 + pbs.aprun;pbs.mpp;depth=24 + _WALLTIME_ + 50 + 1 + 12.00 + 10000 + + /lustre/beagle/{env.USER}/swiftwork + +----- + +By default in this template, jobsPerNode is set to 24. However, for our +application we would like to set this to 12. To do this, we add this +line to the Swift configuration file we are using: +----- +site.beagle.jobsPerNode=12 +----- + +Run directories and logs +------------------------ +Every time Swift runs, it should create a directory called called +$PWD/.swift/runNNN, where NNN is a numeric value that gets incremented +with every run. + +These run directories contain: + +* kml and swiftx files +* Swift log files +* Submit scripts +* tc.data +* sites.xml +* swift.properties +* cdm.data + +What changes need to be made to make this work? +----------------------------------------------- +The vast majority of this can done with changes to the swift shell script and +a new perl script. + +The swift shell script will check the command line parameters it receives. If +the newer style of running is specified, where no configuration files are +specified, swift and swiftrun will generate sites.xml, tc.data, +swift.properties, cdm.data as described in this document. They will be created +in $PWD/.swift/runNNN and referenced. From wilde at ci.uchicago.edu Thu Apr 18 11:13:50 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 18 Apr 2013 11:13:50 -0500 (CDT) Subject: [Swift-commit] r6473 - in demo/cray/swift-1.0: . part08 Message-ID: <20130418161350.71CEC9CC92@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-18 11:13:50 -0500 (Thu, 18 Apr 2013) New Revision: 6473 Modified: demo/cray/swift-1.0/part08/cf demo/cray/swift-1.0/part08/cf.ps demo/cray/swift-1.0/setup.sh Log: Update setup process. Modified: demo/cray/swift-1.0/part08/cf =================================================================== --- demo/cray/swift-1.0/part08/cf 2013-04-18 06:35:29 UTC (rev 6472) +++ demo/cray/swift-1.0/part08/cf 2013-04-18 16:13:50 UTC (rev 6473) @@ -1,11 +1,7 @@ -#sites.file=${swift.home}/etc/sites.xml -#tc.file=${swift.home}/etc/tc.data sites.file=./raven.xml tc.file=./apps -#user.home=/lustre/beagle/wilde - wrapperlog.always.transfer=true sitedir.keep=true execution.retries=0 Modified: demo/cray/swift-1.0/part08/cf.ps =================================================================== --- demo/cray/swift-1.0/part08/cf.ps 2013-04-18 06:35:29 UTC (rev 6472) +++ demo/cray/swift-1.0/part08/cf.ps 2013-04-18 16:13:50 UTC (rev 6473) @@ -1,4 +1,7 @@ -#user.home=/lustre/beagle/wilde + +sites.file=./raven.xml +tc.file=./apps + wrapperlog.always.transfer=true sitedir.keep=true execution.retries=0 Modified: demo/cray/swift-1.0/setup.sh =================================================================== --- demo/cray/swift-1.0/setup.sh 2013-04-18 06:35:29 UTC (rev 6472) +++ demo/cray/swift-1.0/setup.sh 2013-04-18 16:13:50 UTC (rev 6473) @@ -1,38 +1,41 @@ -# module load java # to get Oracle/Sun JDK +module load java # to get Oracle/Sun JDK # module load swift - -# export SBATCH_RESERVATION=swift - export PATHPREFIX="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../bin" && pwd )" -echo Set PATHPREFIX=$PATHPREFIX +echo Setting PATHPREFIX=$PATHPREFIX PATH=/home/users/p01532/swift/rev/0.94/bin:$PATHPREFIX:$PATH +echo Setting PATH=$PATH -return - mkdir -p $HOME/.swift - if [ -f $HOME/.swift/swift.properties ]; then echo Adding properties to end of $HOME/.swift/swift.properties else echo creating $HOME/.swift/swift.properties fi +mkdir -p $HOME/.swift cat >>$HOME/.swift/swift.properties < Author: wilde Date: 2013-04-18 11:16:51 -0500 (Thu, 18 Apr 2013) New Revision: 6474 Added: demo/cray/swift-1.0/part08-cray-loop/ Removed: demo/cray/swift-1.0/part08/ Log: Start giving the partNN dirs functional name suffixes. From wilde at ci.uchicago.edu Thu Apr 18 12:14:58 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 18 Apr 2013 12:14:58 -0500 (CDT) Subject: [Swift-commit] r6475 - in demo/cray: bin swift-1.0/part08-cray-loop Message-ID: <20130418171458.D3F83178884@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-18 12:14:58 -0500 (Thu, 18 Apr 2013) New Revision: 6475 Modified: demo/cray/bin/cleanup demo/cray/swift-1.0/part08-cray-loop/README Log: Adjusting part08. Modified: demo/cray/bin/cleanup =================================================================== --- demo/cray/bin/cleanup 2013-04-18 16:16:51 UTC (rev 6474) +++ demo/cray/bin/cleanup 2013-04-18 17:14:58 UTC (rev 6475) @@ -1,9 +1,15 @@ #! /bin/sh -echo Removing: -echo -ls -ltd *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log 2>/dev/null -echo +if [ _$(ls -ltd *log *.d swiftwork _concurrent output *.kml *.swiftx 2>/dev/null) = _ ]; then + echo Nothing to clean up. + exit +else + echo Removing: + echo + ls -ltd *log *.d swiftwork _concurrent output *.kml *.swiftx 2>/dev/null + echo +fi + echo Enter y to proceed: read ok if [ _$ok = _y -o _$ok = _Y -o _$ok = _yes ]; then Modified: demo/cray/swift-1.0/part08-cray-loop/README =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/README 2013-04-18 16:16:51 UTC (rev 6474) +++ demo/cray/swift-1.0/part08-cray-loop/README 2013-04-18 17:14:58 UTC (rev 6475) @@ -1,8 +1,41 @@ -Regular staging with deafult data management: +The script we'll run here is p8.swift, which executes N mock "simulations" in +parallel, and runs all the results through an "analysis" program. This script +is explained in the demo parts01-06 where it is built up gradually. - swift -config cf -tc.file apps -sites.file cray.xml p8.swift -steps=1 -range=1000 -count=6 -nsim=100 +The simulation app reads from data/bias.dat and writes output files to +the data/output/ directory. +You can use the "cleanup" script to remove logs and the output of prior runs, +so that its easier to tell what each test run created: + + $ cleanup # will prompt for 'y' before deleting files + +First try running just one simulation (followed by an analysis) + + $ cleanup + $ swift -sites.file raven.xml p8.swift -nsim=1 + +To avoid specifying the sites file every time, copy raven.xml to sites.xml: + + $ cp raven.xml sites.xml # use the default sites file name + $ swift p8.swift -nsim=1 # so you dont need to specify -sites.file + +Then you can experiment with different script parameters to set the number of +simulations you want or to adjust the behavior of each simulation. The +parameters (from the p8.swift source code) are: + +int nsim = @toInt(@arg("nsim", "10")); # number of simulation programs to run +int steps = @toInt(@arg("steps", "1")); # number of "steps" each simulation (==seconds of runtime) +int range = @toInt(@arg("range", "100")); # range of the generated random numbers +int count = @toInt(@arg("count", "10")); # number of random numbers generated per simulation + +The default values of these parameters are the second parameter to the @arg() function. +Try, for example: + + $ swift p8.swift -nsim=10 -steps=5 -range=1000 -count=5 + + Regular staging with direct data management: swift -config cf -tc.file apps -sites.file cray.xml -cdm.file dmrules p8.swift -steps=1 -range=1000 -count=6 -nsim=100 From wilde at ci.uchicago.edu Thu Apr 18 12:18:08 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 18 Apr 2013 12:18:08 -0500 (CDT) Subject: [Swift-commit] r6476 - in demo/cray: bin swift-1.0/part08-cray-loop Message-ID: <20130418171808.10583178884@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-18 12:18:07 -0500 (Thu, 18 Apr 2013) New Revision: 6476 Modified: demo/cray/bin/cleanup demo/cray/swift-1.0/part08-cray-loop/apps demo/cray/swift-1.0/part08-cray-loop/p8.swift Log: Adjusting part08. Modified: demo/cray/bin/cleanup =================================================================== --- demo/cray/bin/cleanup 2013-04-18 17:14:58 UTC (rev 6475) +++ demo/cray/bin/cleanup 2013-04-18 17:18:07 UTC (rev 6476) @@ -1,6 +1,6 @@ #! /bin/sh -if [ _$(ls -ltd *log *.d swiftwork _concurrent output *.kml *.swiftx 2>/dev/null) = _ ]; then +if [ _$(ls -ltd *log *.d swiftwork _concurrent data/output *.kml *.swiftx 2>/dev/null) = _ ]; then echo Nothing to clean up. exit else Modified: demo/cray/swift-1.0/part08-cray-loop/apps =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/apps 2013-04-18 17:14:58 UTC (rev 6475) +++ demo/cray/swift-1.0/part08-cray-loop/apps 2013-04-18 17:18:07 UTC (rev 6476) @@ -1,8 +1,3 @@ -#cray random5 /lustre/beagle/wilde/swift/demo/cray/svn/bin/random5.sh -#cray average /lustre/beagle/wilde/swift/demo/cray/svn/bin/avg.sh -#cray random5 /home/users/p01532/swift/demo/svn/bin/random5.sh -#cray average /home/users/p01532/swift/demo/svn/bin/avg.sh - cray random5 random5.sh cray average avg.sh Modified: demo/cray/swift-1.0/part08-cray-loop/p8.swift =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/p8.swift 2013-04-18 17:14:58 UTC (rev 6475) +++ demo/cray/swift-1.0/part08-cray-loop/p8.swift 2013-04-18 17:18:07 UTC (rev 6476) @@ -10,20 +10,24 @@ average @filenames(s) stdout=@filename(o); } -file sims[]; -int nsim = @toInt(@arg("nsim", "10")); +file sims[]; # Array of files to hold each simulation output -int steps = @toInt(@arg("steps", "1")); -int range = @toInt(@arg("range", "100")); -int count = @toInt(@arg("count", "10")); +int nsim = @toInt(@arg("nsim", "10")); # number of simulation programs to run -file bias<"data/bias.dat">; +int steps = @toInt(@arg("steps", "1")); # number of "steps" each simulation + # (==seconds of runtime) +int range = @toInt(@arg("range", "100")); # range of the generated random numbers +int count = @toInt(@arg("count", "10")); # number of random numbers generated per simulation + +file bias<"data/bias.dat">; # Input data file to "bias" the numbers: + # 1 line: scale offset ( N = n*scale + offset) + foreach i in [0:nsim-1] { - file simout ; + file simout ; simout = mysim5(steps,range,bias,count); sims[i] = simout; } -file stats<"output/average.out">; +file stats<"output/average.out">; # Final output file: average of all "simulations" stats = analyze(sims); From wilde at ci.uchicago.edu Thu Apr 18 12:46:12 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 18 Apr 2013 12:46:12 -0500 (CDT) Subject: [Swift-commit] r6477 - demo/cray/swift-1.0 Message-ID: <20130418174612.375B89CC92@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-18 12:46:12 -0500 (Thu, 18 Apr 2013) New Revision: 6477 Modified: demo/cray/swift-1.0/setup.sh Log: Add output spacing in setup.sh Modified: demo/cray/swift-1.0/setup.sh =================================================================== --- demo/cray/swift-1.0/setup.sh 2013-04-18 17:18:07 UTC (rev 6476) +++ demo/cray/swift-1.0/setup.sh 2013-04-18 17:46:12 UTC (rev 6477) @@ -3,11 +3,13 @@ # module load swift export PATHPREFIX="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../bin" && pwd )" +echo echo Setting PATHPREFIX=$PATHPREFIX PATH=/home/users/p01532/swift/rev/0.94/bin:$PATHPREFIX:$PATH +echo echo Setting PATH=$PATH - +echo mkdir -p $HOME/.swift if [ -f $HOME/.swift/swift.properties ]; then @@ -15,6 +17,7 @@ else echo creating $HOME/.swift/swift.properties fi +echo mkdir -p $HOME/.swift cat >>$HOME/.swift/swift.properties < Author: davidk Date: 2013-04-18 12:46:54 -0500 (Thu, 18 Apr 2013) New Revision: 6478 Modified: trunk/docs/designs/swiftrun.txt Log: Updtes to swiftrun doc Modified: trunk/docs/designs/swiftrun.txt =================================================================== --- trunk/docs/designs/swiftrun.txt 2013-04-18 17:46:12 UTC (rev 6477) +++ trunk/docs/designs/swiftrun.txt 2013-04-18 17:46:54 UTC (rev 6478) @@ -52,6 +52,8 @@ 4. $HOME/.swift/swift.properties 5. One or both of $PWD/swift.properties and $PWD/.swift/swift.properties (to be determined) +6. Command line options. Swift will accept parameters on the command line, in + the format of -Dexecution.retries=5. Settings are accumulative. Every swift.properties files that is found is read and values are set, with values from the most recent file taking precedence. @@ -169,6 +171,17 @@ site.beagle.jobsPerNode=12 ----- +Defining sites in swift.properties +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +We may also want to talk about the ability to define sites directly in +swift.properties using the existing XML format. The program that parses the +properties file can attempt to find pool entries there. + +NOTE: This came up in discussions, but I think we should try to avoid this +one. We should be able to shape a sites file using only x.y.z=foo properties. +In cases where a user wants to use XML directly, I think they should +use the -sites.file option. + Run directories and logs ------------------------ Every time Swift runs, it should create a directory called called From wilde at ci.uchicago.edu Thu Apr 18 13:05:18 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 18 Apr 2013 13:05:18 -0500 (CDT) Subject: [Swift-commit] r6479 - in demo/cray/swift-1.0: . part08-cray-loop Message-ID: <20130418180518.254AC9CC92@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-18 13:05:17 -0500 (Thu, 18 Apr 2013) New Revision: 6479 Modified: demo/cray/swift-1.0/part08-cray-loop/README demo/cray/swift-1.0/setup.sh Log: set provider staging as default; provide instructions for switching to CDM DIRECT. Modified: demo/cray/swift-1.0/part08-cray-loop/README =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/README 2013-04-18 17:46:54 UTC (rev 6478) +++ demo/cray/swift-1.0/part08-cray-loop/README 2013-04-18 18:05:17 UTC (rev 6479) @@ -36,15 +36,11 @@ $ swift p8.swift -nsim=10 -steps=5 -range=1000 -count=5 -Regular staging with direct data management: +!!! STOP HERE - remaining needs adjustments - swift -config cf -tc.file apps -sites.file cray.xml -cdm.file dmrules p8.swift -steps=1 -range=1000 -count=6 -nsim=100 -Provider staging: +Next, you can experiment with different size runs: - swift -config cf.ps -tc.file apps -sites.file ravenps.xml p8.swift -steps=5 -range=1000 -count=6 -nsim=100 - - Adjust the job sizes: 6 @@ -52,6 +48,25 @@ 4 + + + +In these runs, Swift was moving data to and from the data/ directory by +staging to the compute node local (RAM) filesystem. For some workflows its +better to bypass staging and access all input and output files directly. + +We do this by specifying a "data management rules" file on the command line +and turning off staging to the local filesystem: + + $ sed -i -e '/SET PROVIDER STAGING/s/=true/=false/p' $HOME/.swift/swift.properties + +Now run the script with "direct" data acces + + swift -sites.file ravenps.xml -cdm.file dmrules p8.swift -steps=1 -range=1000 -count=6 -nsim=10 + + + + The app list Modified: demo/cray/swift-1.0/setup.sh =================================================================== --- demo/cray/swift-1.0/setup.sh 2013-04-18 17:46:54 UTC (rev 6478) +++ demo/cray/swift-1.0/setup.sh 2013-04-18 18:05:17 UTC (rev 6479) @@ -39,6 +39,6 @@ provider.staging.pin.swiftfiles=true # Edit this line to change data management mode: -use.provider.staging=true +use.provider.staging=true # SET PROVIDER STAGING END From wilde at ci.uchicago.edu Thu Apr 18 13:12:35 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 18 Apr 2013 13:12:35 -0500 (CDT) Subject: [Swift-commit] r6480 - demo/cray/bin Message-ID: <20130418181235.1FC259CC92@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-18 13:12:35 -0500 (Thu, 18 Apr 2013) New Revision: 6480 Modified: demo/cray/bin/cleanup Log: fix cleanup Modified: demo/cray/bin/cleanup =================================================================== --- demo/cray/bin/cleanup 2013-04-18 18:05:17 UTC (rev 6479) +++ demo/cray/bin/cleanup 2013-04-18 18:12:35 UTC (rev 6480) @@ -1,6 +1,6 @@ #! /bin/sh -if [ _$(ls -ltd *log *.d swiftwork _concurrent data/output *.kml *.swiftx 2>/dev/null) = _ ]; then +if [ _"$(ls -ltd *log *.d swiftwork _concurrent data/output *.kml *.swiftx 2>/dev/null)" = _ ]; then echo Nothing to clean up. exit else From wilde at ci.uchicago.edu Thu Apr 18 13:24:07 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Thu, 18 Apr 2013 13:24:07 -0500 (CDT) Subject: [Swift-commit] r6481 - demo/cray/swift-1.0/part08-cray-loop Message-ID: <20130418182407.B5838178884@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-18 13:24:07 -0500 (Thu, 18 Apr 2013) New Revision: 6481 Added: demo/cray/swift-1.0/part08-cray-loop/beagle.xml demo/cray/swift-1.0/part08-cray-loop/raven-direct.xml demo/cray/swift-1.0/part08-cray-loop/raven.xml Removed: demo/cray/swift-1.0/part08-cray-loop/cray.xml demo/cray/swift-1.0/part08-cray-loop/pbs.xml demo/cray/swift-1.0/part08-cray-loop/raven.xml demo/cray/swift-1.0/part08-cray-loop/ravenps.xml Modified: demo/cray/swift-1.0/part08-cray-loop/README Log: Adjust README and sites file names. Modified: demo/cray/swift-1.0/part08-cray-loop/README =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/README 2013-04-18 18:12:35 UTC (rev 6480) +++ demo/cray/swift-1.0/part08-cray-loop/README 2013-04-18 18:24:07 UTC (rev 6481) @@ -36,21 +36,29 @@ $ swift p8.swift -nsim=10 -steps=5 -range=1000 -count=5 + !!! STOP HERE - remaining needs adjustments + +PART 2: + Next, you can experiment with different size runs: -Adjust the job sizes: +Adjust the job sizes in sites.xml: 6 4 4 +TODO: Explain the above!!! +PART 3: +Next, you can experiment with different data management strategy + In these runs, Swift was moving data to and from the data/ directory by staging to the compute node local (RAM) filesystem. For some workflows its better to bypass staging and access all input and output files directly. @@ -59,17 +67,13 @@ and turning off staging to the local filesystem: $ sed -i -e '/SET PROVIDER STAGING/s/=true/=false/p' $HOME/.swift/swift.properties + $ cp raven-direct.xml sites.xml Now run the script with "direct" data acces - swift -sites.file ravenps.xml -cdm.file dmrules p8.swift -steps=1 -range=1000 -count=6 -nsim=10 + swift -cdm.file dmrules p8.swift -steps=1 -range=1000 -count=6 -nsim=10 - - -The app list - - OLD: FIXME: TOPIC: Run the "simulations" on the parallel cluster Copied: demo/cray/swift-1.0/part08-cray-loop/beagle.xml (from rev 6480, demo/cray/swift-1.0/part08-cray-loop/cray.xml) =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/beagle.xml (rev 0) +++ demo/cray/swift-1.0/part08-cray-loop/beagle.xml 2013-04-18 18:24:07 UTC (rev 6481) @@ -0,0 +1,31 @@ + + + + + + /scratch/midway/{env.USER}/swiftwork + + + + + + + pbs.aprun;pbs.mpp;depth=24 + + batch + + 24 + 1800 + + 1 + 1 + 1 + + 12 + 10000 + + + /lustre/beagle/wilde/swift/lab/swiftwork + + + Deleted: demo/cray/swift-1.0/part08-cray-loop/cray.xml =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/cray.xml 2013-04-18 18:12:35 UTC (rev 6480) +++ demo/cray/swift-1.0/part08-cray-loop/cray.xml 2013-04-18 18:24:07 UTC (rev 6481) @@ -1,31 +0,0 @@ - - - - - - /scratch/midway/{env.USER}/swiftwork - - - - - - - pbs.aprun;pbs.mpp;depth=24 - - batch - - 24 - 1800 - - 1 - 1 - 1 - - 12 - 10000 - - - /lustre/beagle/wilde/swift/lab/swiftwork - - - Deleted: demo/cray/swift-1.0/part08-cray-loop/pbs.xml =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/pbs.xml 2013-04-18 18:12:35 UTC (rev 6480) +++ demo/cray/swift-1.0/part08-cray-loop/pbs.xml 2013-04-18 18:24:07 UTC (rev 6481) @@ -1,23 +0,0 @@ - - - - - - pbs.aprun;pbs.mpp;depth=24 - - batch - - 24 - 1800 - - 1 - 1 - 1 - - 12 - 10000 - - - /lustre/beagle/wilde/swift/lab/swiftwork - - Copied: demo/cray/swift-1.0/part08-cray-loop/raven-direct.xml (from rev 6480, demo/cray/swift-1.0/part08-cray-loop/raven.xml) =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/raven-direct.xml (rev 0) +++ demo/cray/swift-1.0/part08-cray-loop/raven-direct.xml 2013-04-18 18:24:07 UTC (rev 6481) @@ -0,0 +1,35 @@ + + + + + + {env.PWD}/swiftwork + + + + + + + pbs.aprun;pbs.mpp;depth=32 + + + small + + 32 + 1800 + + 1 + 1 + 1 + + 6.00 + 10000 + + {env.PATHPREFIX} + + + {env.PWD}/swiftwork + + + + Deleted: demo/cray/swift-1.0/part08-cray-loop/raven.xml =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/raven.xml 2013-04-18 18:12:35 UTC (rev 6480) +++ demo/cray/swift-1.0/part08-cray-loop/raven.xml 2013-04-18 18:24:07 UTC (rev 6481) @@ -1,35 +0,0 @@ - - - - - - {env.PWD}/swiftwork - - - - - - - pbs.aprun;pbs.mpp;depth=32 - - - small - - 32 - 1800 - - 1 - 1 - 1 - - 6.00 - 10000 - - {env.PATHPREFIX} - - - {env.PWD}/swiftwork - - - - Copied: demo/cray/swift-1.0/part08-cray-loop/raven.xml (from rev 6480, demo/cray/swift-1.0/part08-cray-loop/ravenps.xml) =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/raven.xml (rev 0) +++ demo/cray/swift-1.0/part08-cray-loop/raven.xml 2013-04-18 18:24:07 UTC (rev 6481) @@ -0,0 +1,35 @@ + + + + + + {env.PWD}/swiftwork + + + + + + + pbs.aprun;pbs.mpp;depth=32 + + + small + + 32 + 1800 + + 1 + 1 + 1 + + 6.00 + 10000 + + {env.PATHPREFIX} + + + /dev/shm/{env.USER}/swiftwork + + + + Deleted: demo/cray/swift-1.0/part08-cray-loop/ravenps.xml =================================================================== --- demo/cray/swift-1.0/part08-cray-loop/ravenps.xml 2013-04-18 18:12:35 UTC (rev 6480) +++ demo/cray/swift-1.0/part08-cray-loop/ravenps.xml 2013-04-18 18:24:07 UTC (rev 6481) @@ -1,35 +0,0 @@ - - - - - - {env.PWD}/swiftwork - - - - - - - pbs.aprun;pbs.mpp;depth=32 - - - small - - 32 - 1800 - - 1 - 1 - 1 - - 6.00 - 10000 - - {env.PATHPREFIX} - - - /dev/shm/{env.USER}/swiftwork - - - - From wozniak at ci.uchicago.edu Thu Apr 18 16:07:36 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Thu, 18 Apr 2013 16:07:36 -0500 (CDT) Subject: [Swift-commit] r6482 - demo/cray/swift-2.0/part08 Message-ID: <20130418210736.07910178884@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-18 16:07:35 -0500 (Thu, 18 Apr 2013) New Revision: 6482 Added: demo/cray/swift-2.0/part08/README.txt Removed: demo/cray/swift-2.0/part08/README Modified: demo/cray/swift-2.0/part08/ demo/cray/swift-2.0/part08/p8.swift Log: Latest everything Property changes on: demo/cray/swift-2.0/part08 ___________________________________________________________________ Added: svn:ignore + output p8.tcl Deleted: demo/cray/swift-2.0/part08/README =================================================================== --- demo/cray/swift-2.0/part08/README 2013-04-18 18:24:07 UTC (rev 6481) +++ demo/cray/swift-2.0/part08/README 2013-04-18 21:07:35 UTC (rev 6482) @@ -1,4 +0,0 @@ - -To run: - -run-swift-2-local.sh p8.swift --steps=0 ... Added: demo/cray/swift-2.0/part08/README.txt =================================================================== --- demo/cray/swift-2.0/part08/README.txt (rev 0) +++ demo/cray/swift-2.0/part08/README.txt 2013-04-18 21:07:35 UTC (rev 6482) @@ -0,0 +1,43 @@ + +SUMMARY: + +This test runs the p8.swift script, running some simple scripts as +apps and producing output in the user directory. + +USAGE: + +Make a work directory called, say, WORK + +$ cd WORK + +# Copy in the Swift script: + +$ cp -r /home/users/p01577/Public/demo/part08/p8.swift . + +# Copy in the input data: + +$ cp -r /home/users/p01577/Public/demo/part08/data . + +$ PATH=$PATH:/home/users/p01577/Public/demo/bin + +$ run-swift-2-qsub.sh p8.swift --work=$PWD + +# Inspect the output: + +$ ls output + +$ ls *.OU + +FURTHER: + +run-swift-2-qsub.sh compiles and runs the Swift script. It sets +everything up, including the locations of Swift programs. + +run-swift-2-qsub.sh is user code. You can edit this +to change compiler options, run time settings, etc. + +p8.swift is also user code. + +run-swift-2-qsub.sh calls turbine-aprun-run.zsh which is the provided +interface to Swift. This script is documented at: +http://www.mcs.anl.gov/exm/local/guides/turbine-sites.html#_raven Modified: demo/cray/swift-2.0/part08/p8.swift =================================================================== --- demo/cray/swift-2.0/part08/p8.swift 2013-04-18 18:24:07 UTC (rev 6481) +++ demo/cray/swift-2.0/part08/p8.swift 2013-04-18 21:07:35 UTC (rev 6482) @@ -26,15 +26,17 @@ int rnge = toint(argv("range", "100")); int count = toint(argv("count", "10")); - file bias = input_file("data/bias.dat"); + string work = argv("work"); + file bias = input_file(work+"/data/bias.dat"); + foreach i in [0:nsim-1] { - file simout ; + file simout ; simout = mysim5(steps,rnge,bias,count); sims[i] = simout; } - file stats<"output/average.out">; + file stats; stats = analyze(sims); wait (stats) { printf("SUCCESS!"); From wozniak at ci.uchicago.edu Thu Apr 18 16:09:18 2013 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Thu, 18 Apr 2013 16:09:18 -0500 (CDT) Subject: [Swift-commit] r6483 - demo/cray/bin Message-ID: <20130418210918.91D47178884@svn.ci.uchicago.edu> Author: wozniak Date: 2013-04-18 16:09:18 -0500 (Thu, 18 Apr 2013) New Revision: 6483 Added: demo/cray/bin/run-swift-2-qsub.sh Log: Run script for Raven: points to Swift/T installation in Wozniak's Public directory Added: demo/cray/bin/run-swift-2-qsub.sh =================================================================== --- demo/cray/bin/run-swift-2-qsub.sh (rev 0) +++ demo/cray/bin/run-swift-2-qsub.sh 2013-04-18 21:09:18 UTC (rev 6483) @@ -0,0 +1,32 @@ +#!/bin/bash + +PATH+=:/home/users/p01577/Public/turbine/bin +PATH+=:/home/users/p01577/Public/turbine/scripts/submit/cray +PATH+=:/home/users/p01577/Public/stc/bin + +echo Turbine tools: +which turbine || exit 1 +which turbine-aprun-run.zsh || exit 1 +echo STC: +which stc || exit 1 +echo + +SCRIPT_SWIFT=$1 +shift +ARGS=${*} + +if [[ ${SCRIPT_SWIFT} == "" ]] +then + echo "No script!" + exit 1 +fi + +SCRIPT_TCL=${SCRIPT_SWIFT%.swift}.tcl + +stc ${SCRIPT_SWIFT} ${SCRIPT_TCL} || exit 1 + +export TURBINE_LOG=0 +export TURBINE_OUTPUT=${PWD} +export NODES=3 +export PPN=3 +turbine-aprun-run.zsh ${SCRIPT_TCL} ${ARGS} Property changes on: demo/cray/bin/run-swift-2-qsub.sh ___________________________________________________________________ Added: svn:executable + * From swift at ci.uchicago.edu Wed Apr 24 15:10:07 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 24 Apr 2013 15:10:07 -0500 (CDT) Subject: [Swift-commit] cog r3655 Message-ID: <20130424201007.5D8F58D00281@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3655 | davidkelly999 | 2013-04-24 15:05:16 -0500 (Wed, 24 Apr 2013) | 2 lines Verify that only valid RSL attributes get set ------------------------------------------------------------------------ Index: modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java =================================================================== --- modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java (revision 3654) +++ modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java (working copy) @@ -11,6 +11,8 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.HashSet; +import java.util.Arrays; import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.AbstractDelegatedTaskHandler; @@ -376,6 +378,18 @@ private void setOtherAttributes(JobSpecification spec, RslNode rsl) throws IllegalSpecException { + + HashSet RSLAttributes = new HashSet( + Arrays.asList("directory", "executable", "arguments", "stdin", + "stdout", "stderr", "count", "environment", + "maxTime", "maxWallTime", "maxCpuTime", "jobType", + "gramMyJob", "queue", "project", "hostCount", + "dryRun", "minMemory", "maxMemory", "save_state", + "two_phase", "restart", "stdout_position", + "stderr_position", "remote_io_url") + ); + + for (String key : spec.getAttributeNames()) { try { String value = String.valueOf(spec.getAttribute(key)); @@ -385,7 +399,9 @@ if (key.equalsIgnoreCase("maxwalltime")) { value = WallTime.normalize(value, jobManager); } - rsl.add(new NameOpValue(key, NameOpValue.EQ, value)); + if(RSLAttributes.contains(key)) { + rsl.add(new NameOpValue(key, NameOpValue.EQ, value)); + } } catch (Exception e) { throw new IllegalSpecException( From swift at ci.uchicago.edu Wed Apr 24 15:10:10 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 24 Apr 2013 15:10:10 -0500 (CDT) Subject: [Swift-commit] cog r3656 Message-ID: <20130424201010.92F348D00281@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3656 | davidkelly999 | 2013-04-24 15:08:34 -0500 (Wed, 24 Apr 2013) | 2 lines Fix to verify rsl attributes ------------------------------------------------------------------------ Index: modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java =================================================================== --- modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java (revision 3655) +++ modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java (working copy) @@ -11,6 +11,8 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.HashSet; +import java.util.Arrays; import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.AbstractDelegatedTaskHandler; @@ -376,6 +378,18 @@ private void setOtherAttributes(JobSpecification spec, RslNode rsl) throws IllegalSpecException { + + HashSet RSLAttributes = new HashSet( + Arrays.asList("directory", "executable", "arguments", "stdin", + "stdout", "stderr", "count", "environment", + "maxTime", "maxWallTime", "maxCpuTime", "jobType", + "gramMyJob", "queue", "project", "hostCount", + "dryRun", "minMemory", "maxMemory", "save_state", + "two_phase", "restart", "stdout_position", + "stderr_position", "remote_io_url") + ); + + for (String key : spec.getAttributeNames()) { try { String value = String.valueOf(spec.getAttribute(key)); @@ -385,7 +399,9 @@ if (key.equalsIgnoreCase("maxwalltime")) { value = WallTime.normalize(value, jobManager); } - rsl.add(new NameOpValue(key, NameOpValue.EQ, value)); + if(RSLAttributes.contains(key)) { + rsl.add(new NameOpValue(key, NameOpValue.EQ, value)); + } } catch (Exception e) { throw new IllegalSpecException( From swift at ci.uchicago.edu Thu Apr 25 15:10:06 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 25 Apr 2013 15:10:06 -0500 (CDT) Subject: [Swift-commit] cog r3657 Message-ID: <20130425201006.B84818D00281@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3657 | davidkelly999 | 2013-04-25 15:07:47 -0500 (Thu, 25 Apr 2013) | 2 lines Make properties case insensitive ------------------------------------------------------------------------ Index: modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java =================================================================== --- modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java (revision 3656) +++ modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java (working copy) @@ -382,9 +382,9 @@ HashSet RSLAttributes = new HashSet( Arrays.asList("directory", "executable", "arguments", "stdin", "stdout", "stderr", "count", "environment", - "maxTime", "maxWallTime", "maxCpuTime", "jobType", - "gramMyJob", "queue", "project", "hostCount", - "dryRun", "minMemory", "maxMemory", "save_state", + "maxtime", "maxwalltime", "maxcputime", "jobtype", + "grammyjob", "queue", "project", "hostcount", + "dryrun", "minmemory", "maxmemory", "save_state", "two_phase", "restart", "stdout_position", "stderr_position", "remote_io_url") ); @@ -399,7 +399,7 @@ if (key.equalsIgnoreCase("maxwalltime")) { value = WallTime.normalize(value, jobManager); } - if(RSLAttributes.contains(key)) { + if(RSLAttributes.contains(key.toLowerCase())) { rsl.add(new NameOpValue(key, NameOpValue.EQ, value)); } } From swift at ci.uchicago.edu Thu Apr 25 15:10:09 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 25 Apr 2013 15:10:09 -0500 (CDT) Subject: [Swift-commit] cog r3658 Message-ID: <20130425201009.4D2998D00281@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3658 | davidkelly999 | 2013-04-25 15:09:07 -0500 (Thu, 25 Apr 2013) | 2 lines Make properties case insensitive ------------------------------------------------------------------------ Index: modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java =================================================================== --- modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java (revision 3657) +++ modules/provider-gt2/src/org/globus/cog/abstraction/impl/execution/gt2/JobSubmissionTaskHandler.java (working copy) @@ -382,9 +382,9 @@ HashSet RSLAttributes = new HashSet( Arrays.asList("directory", "executable", "arguments", "stdin", "stdout", "stderr", "count", "environment", - "maxTime", "maxWallTime", "maxCpuTime", "jobType", - "gramMyJob", "queue", "project", "hostCount", - "dryRun", "minMemory", "maxMemory", "save_state", + "maxtime", "maxwalltime", "maxcputime", "jobtype", + "grammyjob", "queue", "project", "hostcount", + "dryrun", "minmemory", "maxmemory", "save_state", "two_phase", "restart", "stdout_position", "stderr_position", "remote_io_url") ); @@ -399,7 +399,7 @@ if (key.equalsIgnoreCase("maxwalltime")) { value = WallTime.normalize(value, jobManager); } - if(RSLAttributes.contains(key)) { + if(RSLAttributes.contains(key.toLowerCase())) { rsl.add(new NameOpValue(key, NameOpValue.EQ, value)); } } From davidk at ci.uchicago.edu Mon Apr 29 15:26:18 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 29 Apr 2013 15:26:18 -0500 (CDT) Subject: [Swift-commit] r6484 - in SwiftTutorials/OSG_2013-03-11/MODIS: . test.fusion Message-ID: <20130429202618.74A9B9CCF5@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-29 15:26:18 -0500 (Mon, 29 Apr 2013) New Revision: 6484 Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/ SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/cf SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/fusion.xml SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/getlanduse.pl SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/run SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/swift.log SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/tc Modified: SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh Log: Modis tests for fusion Modified: SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh 2013-04-18 21:09:18 UTC (rev 6483) +++ SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh 2013-04-29 20:26:18 UTC (rev 6484) @@ -24,6 +24,8 @@ (cd test.uc3; clean; makeinput 100) +(cd test.fusion; clean; makeinput 100) + (cd test.multiple; clean; makeinput 3000) for d in test.*; do Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/cf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/cf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/cf 2013-04-29 20:26:18 UTC (rev 6484) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/fusion.xml =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/fusion.xml (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/fusion.xml 2013-04-29 20:26:18 UTC (rev 6484) @@ -0,0 +1,17 @@ + + + + + 3600 + 8 + shared + 1 + 1 + 1 + 1.0 + 100 + 100 + 100 + /home/{env.USER}/swiftwork + + Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/getlanduse.pl =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/getlanduse.pl (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/getlanduse.pl 2013-04-29 20:26:18 UTC (rev 6484) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/run =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/run (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/run 2013-04-29 20:26:18 UTC (rev 6484) @@ -0,0 +1,5 @@ +#! /bin/sh + +set -x + +swift -config cf -sites.file fusion.xml -tc.file tc ../modis.swift -modisdir=input/ Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/run ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/swift.log =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/swift.log (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/swift.log 2013-04-29 20:26:18 UTC (rev 6484) @@ -0,0 +1 @@ +2013-04-29 15:24:16,354-0500 DEBUG Loader Swift started Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/tc =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/tc (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.fusion/tc 2013-04-29 20:26:18 UTC (rev 6484) @@ -0,0 +1 @@ +fusion perl /usr/bin/perl null null null From davidk at ci.uchicago.edu Mon Apr 29 17:00:54 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 29 Apr 2013 17:00:54 -0500 (CDT) Subject: [Swift-commit] r6485 - in SwiftTutorials/OSG_2013-03-11/MODIS: . test.midway test.raven Message-ID: <20130429220054.43D359CC97@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-29 17:00:54 -0500 (Mon, 29 Apr 2013) New Revision: 6485 Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/ SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/cf SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/getlanduse.pl SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/input/ SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/raven.xml SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/run SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/tc Modified: SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh SwiftTutorials/OSG_2013-03-11/MODIS/test.midway/run Log: Modis test for Raven Modified: SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh 2013-04-29 20:26:18 UTC (rev 6484) +++ SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh 2013-04-29 22:00:54 UTC (rev 6485) @@ -26,6 +26,8 @@ (cd test.fusion; clean; makeinput 100) +(cd test.raven; clean; makeinput 100) + (cd test.multiple; clean; makeinput 3000) for d in test.*; do Modified: SwiftTutorials/OSG_2013-03-11/MODIS/test.midway/run =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.midway/run 2013-04-29 20:26:18 UTC (rev 6484) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.midway/run 2013-04-29 22:00:54 UTC (rev 6485) @@ -1,6 +1,6 @@ #! /bin/sh -queue=${1:-sandyb} +queue=${1:-westmere} echo -e '\nQueues with idle nodes are:\n' Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/cf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/cf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/cf 2013-04-29 22:00:54 UTC (rev 6485) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/getlanduse.pl =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/getlanduse.pl (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/getlanduse.pl 2013-04-29 22:00:54 UTC (rev 6485) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/raven.xml =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/raven.xml (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/raven.xml 2013-04-29 22:00:54 UTC (rev 6485) @@ -0,0 +1,19 @@ + + + + CI-SES000031 + + KEEP + 24 + pbs.aprun;pbs.mpp;depth=24 + 13800 + 03:00:00 + 50 + 4 + 4 + 60.99 + 10000 + + /home/users/p01537/DSSAT/swiftwork + + Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/run =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/run (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/run 2013-04-29 22:00:54 UTC (rev 6485) @@ -0,0 +1,5 @@ +#! /bin/sh + +set -x + +swift -config cf -sites.file raven.xml -tc.file tc ../modis.swift -modisdir=input/ Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/run ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/tc =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/tc (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.raven/tc 2013-04-29 22:00:54 UTC (rev 6485) @@ -0,0 +1 @@ +raven perl /usr/bin/perl null null null From davidk at ci.uchicago.edu Mon Apr 29 17:31:09 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 29 Apr 2013 17:31:09 -0500 (CDT) Subject: [Swift-commit] r6486 - in SwiftTutorials/OSG_2013-03-11/MODIS: . test.stampede Message-ID: <20130429223109.427969CCF5@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-29 17:31:09 -0500 (Mon, 29 Apr 2013) New Revision: 6486 Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/ SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/cf SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/getlanduse.pl SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/run SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/stampede.xml SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/tc Modified: SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh Log: Modis test for stampede Modified: SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh 2013-04-29 22:00:54 UTC (rev 6485) +++ SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh 2013-04-29 22:31:09 UTC (rev 6486) @@ -16,20 +16,15 @@ echo -e "Cleaning test.* dirs and creating input/ dirs...\\n" +(cd test.beagle; clean; makeinput 100) +(cd test.fusion; clean; makeinput 100) (cd test.local; clean; makeinput 10) - (cd test.midway; clean; makeinput 100) - -(cd test.beagle; clean; makeinput 100) - +(cd test.multiple; clean; makeinput 3000) +(cd test.raven; clean; makeinput 100) +(cd test.stampede; clean; makeinput 100) (cd test.uc3; clean; makeinput 100) -(cd test.fusion; clean; makeinput 100) - -(cd test.raven; clean; makeinput 100) - -(cd test.multiple; clean; makeinput 3000) - for d in test.*; do (cd $d; echo -e "$d: $(find input -type f -print | wc -l) files, $(grep sleep getlanduse.pl | sed -e 's/[^0-9]//g') second task duration" ) done Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/cf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/cf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/cf 2013-04-29 22:31:09 UTC (rev 6486) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/getlanduse.pl =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/getlanduse.pl (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/getlanduse.pl 2013-04-29 22:31:09 UTC (rev 6486) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/run =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/run (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/run 2013-04-29 22:31:09 UTC (rev 6486) @@ -0,0 +1,7 @@ +#! /bin/sh + +set -x +export GLOBUS_HOSTNAME=$( hostname -f ) +#myproxy-logon -l dkelly -s myproxy.teragrid.org + +swift -config cf -sites.file stampede.xml -tc.file tc ../modis.swift -modisdir=input/ Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/run ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/stampede.xml =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/stampede.xml (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/stampede.xml 2013-04-29 22:31:09 UTC (rev 6486) @@ -0,0 +1,19 @@ + + + + + 16 + 16 + 3600 + 00:05:00 + 100 + 100 + normal + 1 + 1 + TG-EAR130015 + .1499 + 10000 + /scratch/01503/davidkel + + Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/tc =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/tc (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.stampede/tc 2013-04-29 22:31:09 UTC (rev 6486) @@ -0,0 +1 @@ +stampede perl /usr/bin/perl null null null From hategan at ci.uchicago.edu Mon Apr 29 19:26:26 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Mon, 29 Apr 2013 19:26:26 -0500 (CDT) Subject: [Swift-commit] r6487 - branches/faster/src/org/griphyn/vdl/mapping Message-ID: <20130430002626.7A38E9CC97@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-29 19:26:26 -0500 (Mon, 29 Apr 2013) New Revision: 6487 Added: branches/faster/src/org/griphyn/vdl/mapping/MissingDataException.java Modified: branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java Log: better error message when a mapped piece of data is closed but has no value Modified: branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2013-04-29 22:31:09 UTC (rev 6486) +++ branches/faster/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2013-04-30 00:26:26 UTC (rev 6487) @@ -708,9 +708,12 @@ if (parent != null && parent.getType().isArray()) { throw new IndexOutOfBoundsException("Invalid index [" + field.getId() + "] for " + parent.getFullName()); } - else { + else if (getType().isPrimitive()) { throw new RuntimeException(getFullName() + " has no value"); } + else { + throw new MissingDataException(this, getMapper().map(getPathFromRoot())); + } } } Added: branches/faster/src/org/griphyn/vdl/mapping/MissingDataException.java =================================================================== --- branches/faster/src/org/griphyn/vdl/mapping/MissingDataException.java (rev 0) +++ branches/faster/src/org/griphyn/vdl/mapping/MissingDataException.java 2013-04-30 00:26:26 UTC (rev 6487) @@ -0,0 +1,16 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Apr 16, 2013 + */ +package org.griphyn.vdl.mapping; + +public class MissingDataException extends RuntimeException { + public MissingDataException(AbstractDataNode n, PhysicalFormat pf) { + super("Data not found for " + n.getFullName() + ": " + pf); + } +} From davidk at ci.uchicago.edu Mon Apr 29 20:10:26 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Mon, 29 Apr 2013 20:10:26 -0500 (CDT) Subject: [Swift-commit] r6488 - in SwiftTutorials/OSG_2013-03-11/MODIS: . test.mcs Message-ID: <20130430011026.2750C9CC97@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-29 20:10:26 -0500 (Mon, 29 Apr 2013) New Revision: 6488 Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/ SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/cf SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/getlanduse.pl SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/input/ SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/mcs.xml SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/run SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/tc Modified: SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh Log: Modis test for mcs Modified: SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh 2013-04-30 00:26:26 UTC (rev 6487) +++ SwiftTutorials/OSG_2013-03-11/MODIS/setup.sh 2013-04-30 01:10:26 UTC (rev 6488) @@ -19,6 +19,7 @@ (cd test.beagle; clean; makeinput 100) (cd test.fusion; clean; makeinput 100) (cd test.local; clean; makeinput 10) +(cd test.mcs; clean; makeinput 100) (cd test.midway; clean; makeinput 100) (cd test.multiple; clean; makeinput 3000) (cd test.raven; clean; makeinput 100) Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/cf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/cf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/cf 2013-04-30 01:10:26 UTC (rev 6488) @@ -0,0 +1,7 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/getlanduse.pl =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/getlanduse.pl (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/getlanduse.pl 2013-04-30 01:10:26 UTC (rev 6488) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/mcs.xml =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/mcs.xml (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/mcs.xml 2013-04-30 01:10:26 UTC (rev 6488) @@ -0,0 +1,123 @@ + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + + + 8 + 100 + 100 + 3600 + 00:05:00 + 0.0799 + 10000 + /sandbox/{env.USER}/swiftwork + + + Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/run =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/run (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/run 2013-04-30 01:10:26 UTC (rev 6488) @@ -0,0 +1,6 @@ +#! /bin/sh + +set -x +export GLOBUS_HOSTNAME=$( hostname -f ) + +swift -config cf -sites.file mcs.xml -tc.file tc ../modis.swift -modisdir=input/ Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/run ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/tc =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/tc (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.mcs/tc 2013-04-30 01:10:26 UTC (rev 6488) @@ -0,0 +1,12 @@ +crank perl /usr/bin/perl null null null +churn perl /usr/bin/perl null null null +crush perl /usr/bin/perl null null null +grind perl /usr/bin/perl null null null +octagon perl /usr/bin/perl null null null +octopus perl /usr/bin/perl null null null +steamroller perl /usr/bin/perl null null null +stomp perl /usr/bin/perl null null null +thrash perl /usr/bin/perl null null null +thwomp perl /usr/bin/perl null null null +trounce perl /usr/bin/perl null null null +vanquish perl /usr/bin/perl null null null From hategan at ci.uchicago.edu Mon Apr 29 20:46:44 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Mon, 29 Apr 2013 20:46:44 -0500 (CDT) Subject: [Swift-commit] r6489 - branches/faster/tests/functions Message-ID: <20130430014644.279949CCF5@svn.ci.uchicago.edu> Author: hategan Date: 2013-04-29 20:46:44 -0500 (Mon, 29 Apr 2013) New Revision: 6489 Added: branches/faster/tests/functions/450-tracef.setup.sh Log: new policy is that mapped input data must have the files there Added: branches/faster/tests/functions/450-tracef.setup.sh =================================================================== --- branches/faster/tests/functions/450-tracef.setup.sh (rev 0) +++ branches/faster/tests/functions/450-tracef.setup.sh 2013-04-30 01:46:44 UTC (rev 6489) @@ -0,0 +1,5 @@ +#!/bin/bash + +touch test.txt + +exit 0 Property changes on: branches/faster/tests/functions/450-tracef.setup.sh ___________________________________________________________________ Added: svn:executable + * From davidk at ci.uchicago.edu Tue Apr 30 11:28:34 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 30 Apr 2013 11:28:34 -0500 (CDT) Subject: [Swift-commit] r6490 - in SwiftTutorials/OSG_2013-03-11/MODIS: . test.eureka Message-ID: <20130430162834.2382D9CC9B@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-30 11:28:33 -0500 (Tue, 30 Apr 2013) New Revision: 6490 Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/ SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/cf SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/coaster-service.conf SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/getlanduse.pl SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/run SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/tc Log: Modis test for Eureka Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/cf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/cf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/cf 2013-04-30 16:28:33 UTC (rev 6490) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/coaster-service.conf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/coaster-service.conf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/coaster-service.conf 2013-04-30 16:28:33 UTC (rev 6490) @@ -0,0 +1,53 @@ +# Keep all interesting settings in one place +# User should modify this to fit environment + +# Location of SWIFT. If empty, PATH is searched +export SWIFT= + +# Where to copy worker.pl on the remote machine for sites.xml +export WORKER_WORK=$HOME/work + +# How to launch workers: local, ssh, cobalt, or futuregrid +export WORKER_MODE=cobalt + +# Kernel profile for cobalt jobs +export KERNEL=zeptoos + +# SSH5B hosts to start workers on (ssh mode only) +export WORKER_HOSTS="host1 host2 host3" + +# Do all the worker nodes you're using have a shared filesystem? (yes/no) +export SHARED_FILESYSTEM=yes + +# Username to use on worker nodes +export WORKER_USERNAME=$USER + +# Enable SSH tunneling? (yes/no) +export SSH_TUNNELING=no + +# Directory to keep log files, relative to working directory when launching start-coaster-service +export LOG_DIR=logs + +# Manually define ports. If not specified, an available port will be used +export LOCAL_PORT=50321 +export SERVICE_PORT=50322 + +# start-coaster-service tries to automatically detect the IP address of this system. Specify here if you have multiple network interfaces +export IPADDR=172.17.9.151 + +# Location of the swift-vm-boot scripts +export SWIFTVMBOOT_DIR=$HOME/swift-vm-boot + +export JOBSPERNODE=4 +export JOBTHROTTLE=1000 + +# Setting for WORKER_INIT_CMD +export WORKER_ENVIRONMENT="WORKER_INIT_CMD=$PWD/worker-init.pl" + +# Swift information for creating sites.xml +export PROJECT=MTCScienceApps +export WORK=$HOME/work +export QUEUE=default +export MAXTIME=59 +export NODES=2 + Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/getlanduse.pl =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/getlanduse.pl (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/getlanduse.pl 2013-04-30 16:28:33 UTC (rev 6490) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/run =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/run (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/run 2013-04-30 16:28:33 UTC (rev 6490) @@ -0,0 +1,7 @@ +#! /bin/sh + +set -x + +start-coaster-service +swift -config cf -sites.file sites.xml -tc.file tc ../modis.swift -modisdir=input/ +stop-coaster-service Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/run ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/tc =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/tc (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.eureka/tc 2013-04-30 16:28:33 UTC (rev 6490) @@ -0,0 +1 @@ +persistent-coasters perl /usr/bin/perl null null null From davidk at ci.uchicago.edu Tue Apr 30 11:31:51 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 30 Apr 2013 11:31:51 -0500 (CDT) Subject: [Swift-commit] r6491 - in SwiftTutorials/OSG_2013-03-11/MODIS: . test.intrepid Message-ID: <20130430163151.1E4F79CC9B@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-30 11:31:50 -0500 (Tue, 30 Apr 2013) New Revision: 6491 Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/ SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/cf SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/getlanduse.pl SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/intrepid.xml SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/run SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/tc Log: Modis intrepid test Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/cf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/cf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/cf 2013-04-30 16:31:50 UTC (rev 6491) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/getlanduse.pl =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/getlanduse.pl (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/getlanduse.pl 2013-04-30 16:31:50 UTC (rev 6491) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/intrepid.xml =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/intrepid.xml (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/intrepid.xml 2013-04-30 16:31:50 UTC (rev 6491) @@ -0,0 +1,22 @@ + + + + + 172.17.5.141 + zeptoos + true + MTCScienceApps + prod + 0 + 10000 + 1 + DEBUG + /intrepid-fs0/users/davidk/scratch/MODIS/test.intrepid + 1 + 900 + 512 + 512 + /intrepid-fs0/users/davidk/scratch/MODIS/test.intrepid/scratch + + + Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/run =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/run (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/run 2013-04-30 16:31:50 UTC (rev 6491) @@ -0,0 +1,5 @@ +#! /bin/sh + +set -x + +swift -config cf -sites.file intrepid.xml -tc.file tc ../modis.swift -modisdir=input/ Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/run ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/tc =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/tc (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.intrepid/tc 2013-04-30 16:31:50 UTC (rev 6491) @@ -0,0 +1 @@ +intrepid perl /usr/bin/perl null null null From davidk at ci.uchicago.edu Tue Apr 30 11:33:11 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 30 Apr 2013 11:33:11 -0500 (CDT) Subject: [Swift-commit] r6492 - in SwiftTutorials/OSG_2013-03-11/MODIS: . test.surveyor Message-ID: <20130430163311.17E289CC9B@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-30 11:33:10 -0500 (Tue, 30 Apr 2013) New Revision: 6492 Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/ SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/cf SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/coaster-service.conf SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/getlanduse.pl SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/run SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/tc Log: Modis surveyor test Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/cf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/cf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/cf 2013-04-30 16:33:10 UTC (rev 6492) @@ -0,0 +1,7 @@ +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/coaster-service.conf =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/coaster-service.conf (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/coaster-service.conf 2013-04-30 16:33:10 UTC (rev 6492) @@ -0,0 +1,52 @@ +# Keep all interesting settings in one place +# User should modify this to fit environment + +# Location of SWIFT. If empty, PATH is searched +export SWIFT= + +# Where to copy worker.pl on the remote machine for sites.xml +export WORKER_WORK=$HOME/work + +# How to launch workers: local, ssh, cobalt, or futuregrid +export WORKER_MODE=cobalt + +# Kernel profile for cobalt jobs +export KERNEL=zeptoos + +# SSH5B hosts to start workers on (ssh mode only) +export WORKER_HOSTS="host1 host2 host3" + +# Do all the worker nodes you're using have a shared filesystem? (yes/no) +export SHARED_FILESYSTEM=yes + +# Username to use on worker nodes +export WORKER_USERNAME=$USER + +# Enable SSH tunneling? (yes/no) +export SSH_TUNNELING=no + +# Directory to keep log files, relative to working directory when launching start-coaster-service +export LOG_DIR=logs + +# Manually define ports. If not specified, an available port will be used +export LOCAL_PORT=12345 +export SERVICE_PORT=12346 + +# start-coaster-service tries to automatically detect the IP address of this system. Specify here if you have multiple network interfaces +export IPADDR=172.17.3.13 + +# Location of the swift-vm-boot scripts +export SWIFTVMBOOT_DIR=$HOME/swift-vm-boot + +export JOBSPERNODE=4 +export JOBTHROTTLE=1000 + +# Setting for WORKER_INIT_CMD +export WORKER_ENVIRONMENT="WORKER_INIT_CMD=$PWD/worker-init.pl" + +# Swift information for creating sites.xml +export PROJECT=MTCScienceApps +export WORK=$HOME/work +export QUEUE=default +export MAXTIME=59 +export NODES=2 Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/getlanduse.pl =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/getlanduse.pl (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/getlanduse.pl 2013-04-30 16:33:10 UTC (rev 6492) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/run =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/run (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/run 2013-04-30 16:33:10 UTC (rev 6492) @@ -0,0 +1,7 @@ +#! /bin/sh + +# soft add +ibmjava6 +set -x +start-coaster-service +swift -config cf -sites.file sites.xml -tc.file tc ../modis.swift -modisdir=input/ +stop-coaster-service Property changes on: SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/run ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/tc =================================================================== --- SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/tc (rev 0) +++ SwiftTutorials/OSG_2013-03-11/MODIS/test.surveyor/tc 2013-04-30 16:33:10 UTC (rev 6492) @@ -0,0 +1,2 @@ +surveyor perl /usr/bin/perl null null null +persistent-coasters perl /usr/bin/perl null null null From davidk at ci.uchicago.edu Tue Apr 30 11:45:11 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 30 Apr 2013 11:45:11 -0500 (CDT) Subject: [Swift-commit] r6493 - in www: downloads inc Message-ID: <20130430164511.A0E2F9CC9B@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-30 11:45:11 -0500 (Tue, 30 Apr 2013) New Revision: 6493 Modified: www/downloads/index.php www/inc/downloads_sidebar.php Log: Update download links Modified: www/downloads/index.php =================================================================== --- www/downloads/index.php 2013-04-30 16:33:10 UTC (rev 6492) +++ www/downloads/index.php 2013-04-30 16:45:11 UTC (rev 6493) @@ -37,7 +37,7 @@

Latest Release

-

Swift 0.94 - 2013/03/01

+

Swift 0.94 - 2013/04/30

For the majority of users, this is the version you will want to download. This package contains the latest Stable release of Swift. Since Swift is written in Java, this package will run on all supported platforms with Java Runtime Environment 1.6 or greater. Modified: www/inc/downloads_sidebar.php =================================================================== --- www/inc/downloads_sidebar.php 2013-04-30 16:33:10 UTC (rev 6492) +++ www/inc/downloads_sidebar.php 2013-04-30 16:45:11 UTC (rev 6493) @@ -1,7 +1,7 @@

Latest Release

-0.94 current version
2013/03/01 +0.94 current version
2013/04/30

 

From davidk at ci.uchicago.edu Tue Apr 30 11:48:31 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 30 Apr 2013 11:48:31 -0500 (CDT) Subject: [Swift-commit] r6494 - www/main Message-ID: <20130430164831.305149CC9B@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-30 11:48:31 -0500 (Tue, 30 Apr 2013) New Revision: 6494 Modified: www/main/index.php Log: Update download link Modified: www/main/index.php =================================================================== --- www/main/index.php 2013-04-30 16:45:11 UTC (rev 6493) +++ www/main/index.php 2013-04-30 16:48:31 UTC (rev 6494) @@ -62,7 +62,7 @@
-
0.94 current version
2013/03/01 +
0.94 current version
2013/04/30
Read our Quick Start Guide and start using Swift today!
From davidk at ci.uchicago.edu Tue Apr 30 12:35:05 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 30 Apr 2013 12:35:05 -0500 (CDT) Subject: [Swift-commit] r6495 - in branches/release-0.94/docs: . siteguide Message-ID: <20130430173505.6609D9CCF5@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-30 12:35:05 -0500 (Tue, 30 Apr 2013) New Revision: 6495 Added: branches/release-0.94/docs/siteguide/stampede Modified: branches/release-0.94/docs/build_docs.sh branches/release-0.94/docs/siteguide/siteguide.txt Log: Stampede site guide entry Modified: branches/release-0.94/docs/build_docs.sh =================================================================== --- branches/release-0.94/docs/build_docs.sh 2013-04-30 16:48:31 UTC (rev 6494) +++ branches/release-0.94/docs/build_docs.sh 2013-04-30 17:35:05 UTC (rev 6495) @@ -17,7 +17,7 @@ # Output group GROUP="vdl2-svn" # Make PDFs iff MAKE_PDF=1 -MAKE_PDF=1 +MAKE_PDF=0 # See getopts loop below for options usage() Modified: branches/release-0.94/docs/siteguide/siteguide.txt =================================================================== --- branches/release-0.94/docs/siteguide/siteguide.txt 2013-04-30 16:48:31 UTC (rev 6494) +++ branches/release-0.94/docs/siteguide/siteguide.txt 2013-04-30 17:35:05 UTC (rev 6495) @@ -22,4 +22,6 @@ include::mcs[] +include::stampede[] + include::uc3[] Added: branches/release-0.94/docs/siteguide/stampede =================================================================== --- branches/release-0.94/docs/siteguide/stampede (rev 0) +++ branches/release-0.94/docs/siteguide/stampede 2013-04-30 17:35:05 UTC (rev 6495) @@ -0,0 +1,106 @@ +Stampede (x86 cluster) +---------------------- +Stampede is a cluster managed by the Texas Advanced Computing Center (TACC). It +is a part of the XSEDE project. For more information about how to request an +account, a project, how to log in and manage SSH keys, please see the +More information about the system can be found in the +https://portal.xsede.org/web/guest/tacc-stampede[Stampede User Guide]. + +Downloading and building Swift +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The most recent versions of Swift can be found at +http://www.ci.uchicago.edu/swift/downloads/index.php[the Swift downloads page]. Follow the instructions +provided on that site to download and build Swift. + +Overview of How to Run +~~~~~~~~~~~~~~~~~~~~~~ +You will need to do the following steps in order to run. + +1. Connect to a system that has the Globus myproxy-init command. This will be + the system where Swift runs and from where Swift submits jobs to Stampede. +2. Obtain a grid certificate. +3. Run Swift with configuration files that define how to start remote jobs to + Stampede via gram. + +Verify System Requirements and Environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The system where you run Swift needs to have the myproxy-init tool installed. +Ideally it should also have globus-job-run for testing purposes. + +Swift uses two environment variables in order for remote job execution to work. +They are $GLOBUS_HOSTNAME and $GLOBUS_TCP_PORT_RANGE. + +GLOBUS_HOSTNAME should contain the full hostname of the system where you are +running. It may also contain the IP address of the machine. + +GLOBUS_TCP_PORT_RANGE defines a range of ports to which a remote system may +connect. You will likely need this defined if you are behind a firewall with +somewhat restricted access. + +Obtain a Grid Certificate +~~~~~~~~~~~~~~~~~~~~~~~~~ +Once you have verified you have everything you need on the submit host where +you are running, you can obtain an XSEDE grid certificate with following +command: + +----- +$ myproxy-logon -l username -s myproxy.teragrid.org +----- + +Create sites.xml file +~~~~~~~~~~~~~~~~~~~~~ +You may use the following example as a guide to run on Stampede. You will +likely need to make a few modifications, as described below. +----- + + + + + 16 + 16 + 3600 + 00:05:00 + 100 + 100 + normal + 1 + 1 + yourproject + .3199 + 10000 + /scratch/01503/yourusername + + +----- + +You will need to modify the XSEDE project name to the match the name that has +been allocated to you. In most cases you'll want to set the work directory to +your Stampede scratch directory. This is defined, on Stampede, in the +environment variable $SCRATCH. + +Running Swift +~~~~~~~~~~~~~ +You may now run your Swift script exactly as you have before. + +----- +$ swift -sites.file sites.xml -tc.file tc -config cf myscript.swift +----- + +Debugging +~~~~~~~~~ +If you are having problems getting this working correctly, there are a few +places where you may look to help you debug. Since this configuration is +slightly more complicated, there are several log files produced. + +1. The standard swift log, created in your current working directory on the + machine where you are running from. This will be named something along the + lines of myscript-.log. +2. The bootstrap log. These are located on Stampede in your home directory and + have the name coaster-bootstrap-.log. +3. The coaster log. This is located on Stampede in your $HOME/.globus/coasters/coasters.log. +4. The gram log. This is located on Stamped in $HOME/gram-.log. + +For help in getting this configuration working, feel free to contact the Swift +support team at swift-support at ci.uchicago.edu. + + From davidk at ci.uchicago.edu Tue Apr 30 12:36:41 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 30 Apr 2013 12:36:41 -0500 (CDT) Subject: [Swift-commit] r6496 - branches/release-0.94/docs Message-ID: <20130430173641.9399B9CCF5@svn.ci.uchicago.edu> Author: davidk Date: 2013-04-30 12:36:41 -0500 (Tue, 30 Apr 2013) New Revision: 6496 Modified: branches/release-0.94/docs/build_docs.sh Log: Default to build PDFs Modified: branches/release-0.94/docs/build_docs.sh =================================================================== --- branches/release-0.94/docs/build_docs.sh 2013-04-30 17:35:05 UTC (rev 6495) +++ branches/release-0.94/docs/build_docs.sh 2013-04-30 17:36:41 UTC (rev 6496) @@ -17,7 +17,7 @@ # Output group GROUP="vdl2-svn" # Make PDFs iff MAKE_PDF=1 -MAKE_PDF=0 +MAKE_PDF=1 # See getopts loop below for options usage() From wilde at ci.uchicago.edu Tue Apr 9 01:58:16 2013 From: wilde at ci.uchicago.edu (wilde at ci.uchicago.edu) Date: Tue, 09 Apr 2013 06:58:16 -0000 Subject: [Swift-commit] r6418 - in SwiftTutorials/UofC_2013-04-09: . bin data data/sat part01 part02 part03 part04 part05 scripts Message-ID: <20130409065812.547219CCFD@svn.ci.uchicago.edu> Author: wilde Date: 2013-04-09 01:57:55 -0500 (Tue, 09 Apr 2013) New Revision: 6418 Added: SwiftTutorials/UofC_2013-04-09/README SwiftTutorials/UofC_2013-04-09/SwiftRCCTutorial.2013.0409.pptx SwiftTutorials/UofC_2013-04-09/bin/ SwiftTutorials/UofC_2013-04-09/bin/avg.sh SwiftTutorials/UofC_2013-04-09/bin/cleanup SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh SwiftTutorials/UofC_2013-04-09/bin/genapplist SwiftTutorials/UofC_2013-04-09/bin/getinfo.sh SwiftTutorials/UofC_2013-04-09/bin/getlanduse.pl SwiftTutorials/UofC_2013-04-09/bin/makeinput SwiftTutorials/UofC_2013-04-09/bin/myapp SwiftTutorials/UofC_2013-04-09/bin/random.sh SwiftTutorials/UofC_2013-04-09/bin/random2.sh SwiftTutorials/UofC_2013-04-09/bin/random3.sh SwiftTutorials/UofC_2013-04-09/bin/random4.sh SwiftTutorials/UofC_2013-04-09/bin/runswift SwiftTutorials/UofC_2013-04-09/bin/setup.sh SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh SwiftTutorials/UofC_2013-04-09/data/ SwiftTutorials/UofC_2013-04-09/data/data.txt SwiftTutorials/UofC_2013-04-09/data/sat/ SwiftTutorials/UofC_2013-04-09/data/sat/t0 SwiftTutorials/UofC_2013-04-09/data/sat/t1 SwiftTutorials/UofC_2013-04-09/data/sat/t2 SwiftTutorials/UofC_2013-04-09/data/sat/t3 SwiftTutorials/UofC_2013-04-09/data/sat/t4 SwiftTutorials/UofC_2013-04-09/data/sat/t5 SwiftTutorials/UofC_2013-04-09/data/sat/t6 SwiftTutorials/UofC_2013-04-09/data/sat/t7 SwiftTutorials/UofC_2013-04-09/data/sat/t8 SwiftTutorials/UofC_2013-04-09/data/sat/t9 SwiftTutorials/UofC_2013-04-09/part01/ SwiftTutorials/UofC_2013-04-09/part01/README SwiftTutorials/UofC_2013-04-09/part01/p1.swift SwiftTutorials/UofC_2013-04-09/part01/tc SwiftTutorials/UofC_2013-04-09/part02/ SwiftTutorials/UofC_2013-04-09/part02/README SwiftTutorials/UofC_2013-04-09/part02/p2.swift SwiftTutorials/UofC_2013-04-09/part02/p2b.swift SwiftTutorials/UofC_2013-04-09/part02/tc SwiftTutorials/UofC_2013-04-09/part03/ SwiftTutorials/UofC_2013-04-09/part03/README SwiftTutorials/UofC_2013-04-09/part03/p3.swift SwiftTutorials/UofC_2013-04-09/part03/tc SwiftTutorials/UofC_2013-04-09/part04/ SwiftTutorials/UofC_2013-04-09/part04/README SwiftTutorials/UofC_2013-04-09/part04/p4.swift SwiftTutorials/UofC_2013-04-09/part04/tc SwiftTutorials/UofC_2013-04-09/part05/ SwiftTutorials/UofC_2013-04-09/part05/README SwiftTutorials/UofC_2013-04-09/part05/p5.swift SwiftTutorials/UofC_2013-04-09/part05/tc SwiftTutorials/UofC_2013-04-09/scripts/ SwiftTutorials/UofC_2013-04-09/scripts/catsn.swift SwiftTutorials/UofC_2013-04-09/scripts/script01.swift SwiftTutorials/UofC_2013-04-09/scripts/script02.swift SwiftTutorials/UofC_2013-04-09/scripts/script03.swift SwiftTutorials/UofC_2013-04-09/scripts/script04.swift Log: Initial version Added: SwiftTutorials/UofC_2013-04-09/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/README (rev 0) +++ SwiftTutorials/UofC_2013-04-09/README 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,81 @@ + +INTRODUCTION TO SWIFT + +SETUP + +Obtain the tutorial package + +Change directory into the created directory called "tutorial" + +This directory contains: + +bin: script tools for the tutorial +scripts: Swift scripts for the tutorial + +An output directory will be created by the Swift scripts + + +Use bin/setup.sh to add tutorial programs to your PATH + +Run: + +source bin/setup.sh + +EXERCISE 1 - Run a simple command under Swift + +This script simply runs the getinfo command found in bin/getinfo.sh + +You can run this on the login node by typing: + +bin/getinfo.sh + +Inspect getinfo.sh to see what it does. + +The Swift syntax in scripts/script01.swift directs the output of +getinfo.sh to file output/info.txt + +Run the Swift script by typing: + +runswift scripts/script01.swift + +Check that the output is created in the output directory + +EXERCISE 2 - Run parallel jobs + +The Swift syntax in scripts/script02.swift runs the getinfo +script 25 times in parallel. Each job produces an output +file with a different name. + +runswift scripts/script02.swift + +Check that the output is created successfully + +EXERCISE 3 - Run parallel jobs with pipelines + +The Swift syntax in scripts/script02.swift runs the getinfo +script 25 times in parallel. Each job produces an output +file with a different name. Each output file is then read by the +filterinfo task, which produces output in _concurrent + +Inspect bin/filterinfo.sh to see what it does to the output of getinfo.sh + +runswift scripts/script03.swift + +EXERCISE 4 - Name files in pipelines + +In Exercise 3, the final outputs were named by Swift. +Exercise 4 is similar to 3 but allows you to name the final +outputs. Edit script04.swift to name the files in an appropriate +way. + +EXERCISE 5 - Reduction + +Exercise 5 allows you to collect all of the filtered outputs +and summarize them into one file. This Swift script allows +suminfo.sh to read all the files and collect a statistic. + +Inspect bin/suminfo.sh to see what it will do to the files it reads. + +CLEAN UP + +Simply run: bin/cleanup and enter y to confirm Added: SwiftTutorials/UofC_2013-04-09/SwiftRCCTutorial.2013.0409.pptx =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/SwiftRCCTutorial.2013.0409.pptx ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/bin/avg.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/avg.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/avg.sh 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,9 @@ +#! /bin/sh + +awk ' + +{ sum += $1} + +END { print sum/NR } +' $* + Property changes on: SwiftTutorials/UofC_2013-04-09/bin/avg.sh ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/cleanup =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/cleanup (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/cleanup 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,17 @@ +#! /bin/sh + +echo Removing: +echo +ls -ltd *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log applist config sites.xml pathlist 2>/dev/null +echo +echo Enter y to proceed: +read ok +if [ _$ok = _y -o _$ok = _Y -o _$ok = _yes ]; then + echo + set -x + rm -rf *log *.d swiftwork _concurrent output *.kml *.swiftx swift.log applist config sites.xml pathlist +else + echo + echo No files removed. +fi + Property changes on: SwiftTutorials/UofC_2013-04-09/bin/cleanup ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,2 @@ +#! /bin/sh +grep getinfo.sh $1 | wc -l >$2 Property changes on: SwiftTutorials/UofC_2013-04-09/bin/filterinfo.sh ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/genapplist =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/genapplist (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/genapplist 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,7 @@ +cat <) { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d\n", $sortedcount[$count], $count); +} + +sleep($runtime); Property changes on: SwiftTutorials/UofC_2013-04-09/bin/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/makeinput =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/makeinput (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/makeinput 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,22 @@ +#! /bin/sh + +nfiles=${1:-10} + +rm -rf input +mkdir input +cp ../data/sat/t? input/ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(($RANDOM%10)) + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) Property changes on: SwiftTutorials/UofC_2013-04-09/bin/makeinput ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/myapp =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/myapp (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/myapp 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,6 @@ +#! /bin/sh + +hostname -f +date +sleep 3 +ps -u $USER -H -f Property changes on: SwiftTutorials/UofC_2013-04-09/bin/myapp ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/random.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/random.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/random.sh 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,2 @@ +#! /bin/sh +echo $RANDOM Property changes on: SwiftTutorials/UofC_2013-04-09/bin/random.sh ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/random2.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/random2.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/random2.sh 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,5 @@ +#! /bin/sh + +range=$1 + +echo $(($RANDOM%range)) Property changes on: SwiftTutorials/UofC_2013-04-09/bin/random2.sh ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/random3.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/random3.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/random3.sh 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,9 @@ +#! /bin/sh + +range=$1 +n=$2 + +for ((i=0;ipathlist +rule .*_concurrent/.* DIRECT $PWD/tmp +rule .* DIRECT / +END + +cat <config +status.mode=provider +use.provider.staging=false +use.wrapper.staging=false +wrapperlog.always.transfer=true +execution.retries=0 +lazy.errors=false +provider.staging.pin.swiftfiles=false +sitedir.keep=true +file.gc.enabled=false +tcp.port.range=50000,51000 +END + +reservationID=${reservationID:-swift} + +cat <sites.xml + + + + + + + 0.03 + 10000 + + $PWD/swiftwork + + + + + + + + 12 + 12 + 1 + 1 + 1 + westmere + + slurm.resource_list=advres=$reservationID + + 5.99 + 10000 + + 100 + 100 + 18000 + 02:00:00 + + + $PWD/swiftwork + + + + + + + + 1 + 12 + 1 + 1 + 1 + single + westmere + + slurm.resource_list=advres=$reservationID + + 0.64 + 10000 + + 100 + 100 + 9000 + 01:00:00 + + + $PWD/swiftwork + + + + + +END + +genapplist >applist + +set -x +#swift -config config -tc.file applist -sites.file sites.xml -cdm.file pathlist $* +swift -config config -tc.file applist -sites.file sites.xml $* + + + + Property changes on: SwiftTutorials/UofC_2013-04-09/bin/runswift ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/bin/setup.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/setup.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/setup.sh 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,13 @@ +module load swift + +echo 0=$0 + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +demobin=$(cd $(dirname $0); pwd) + +echo demobin=$demobin DIR=$DIR + +return + +PATH=$PWD/bin:$PATH Added: SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh =================================================================== --- SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh (rev 0) +++ SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,2 @@ +#! /bin/sh +awk '{ sum += $1 } END { print "Total processes observed: " sum }' $* Property changes on: SwiftTutorials/UofC_2013-04-09/bin/suminfo.sh ___________________________________________________________________ Added: svn:executable + * Added: SwiftTutorials/UofC_2013-04-09/data/data.txt =================================================================== --- SwiftTutorials/UofC_2013-04-09/data/data.txt (rev 0) +++ SwiftTutorials/UofC_2013-04-09/data/data.txt 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1 @@ +Hi Swift! Added: SwiftTutorials/UofC_2013-04-09/data/sat/t0 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t0 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/data/sat/t1 =================================================================== --- SwiftTutorials/UofC_2013-04-09/data/sat/t1 (rev 0) +++ SwiftTutorials/UofC_2013-04-09/data/sat/t1 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,13792 @@ +    + + + + + + + + +      + + +  + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + +         + + + + + +              + + + + + + + + + + + + + + +      + + +       + + + + + + + + + + + + + + +       + + +  + + +   + + +  + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + +      + + + + + +               + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +       + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +  + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +  + + + + + +        + + + + + + + + +   + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + +  + + +    + + + + + +       + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + + + + + + + + + + + + + + + + + + + +   + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +  + + +   + + + + + +  + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +  + + + + + + + + + + + +   + + + + + + + + + + + +      + + + + + + + + + + + +  + + + + + + + + +    + + + + + + + + + + + +  + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +        + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + + + + + + + +  + + +      + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + + + + +   + + + + + + + + +  + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + + + + +    + + + + + + + + + + + +  + + + + + + + + +    + + + + + + + + + + + +     + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +      + + +   + + + + + + + + + + + + + + +   + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +     + + +   + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + +     + + + + + +    + + + + + + + + +    + + +   + + +   + + + + + + + + + + + + + + +    + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +       + + + + + +    + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + +      + + + + + +    + + + + + + + + +  + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + +         + + +   + + + + + + + + + + + + + + + + + +  + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +               + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + + + + + + + + + + +      + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +  + + + + + +    + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +   + + +       + + + + + + + + + + + +      + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + +       + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + +      + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + +    + + + + + +   + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +          + + + + + + + + + + + + + + +    + + + + + + + + + + + +   + + + + + + + + + + + + + + +  + + + + + + + + +          + + +  + + +    + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + +   + + +        + + +     + + +    + + + + + +   + + + + + + + + +  + + + + + + + + + + + +     + + + + + + + + +  + + +            + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +            + + + + + + + + + + + + + + +      + + + + + +  + + + + + +  + + + + + + + + + + + + + + +   + + +   + + +      + + +  + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +         + + + + + +    + + + + + +  + + + + + +    + + + + + + + + +                 + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + +        + + + + + +   + + +  + + + + + + + + +   + + +      + + +     + + +  + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + +  + + + + + +    + + + + + + + + +    + + + + + + + + + + + +      + + + + + +                   + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +  + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + +  + + + + + + + + + + + +     + + + + + + + + +     + + +    + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + +   + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + +                  + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + +   + + + + + + + + +   + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + +    + + +   + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +                   + + + + + + + + + + + + + + + + + + + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + +  + + + + + +    + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + +           + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + +  + + +                  + + + + + + + + + + + + + + +   + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + + + + + + + + + + + + + + + + +  + + + + + + + + +     + + +    + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + + + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +                 + + + + + + + + + + + + + + +   + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + +  + + + + + + + + + + + + + + + + + +    + + +     + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + +      + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +                   + + + + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + + + + +  + + +  + + + + + +   + + + + + +            + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +    + + + + + +  + + +     + + + + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +               + + + + + +    + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + + + + +     + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + + + + +  + + + + + +   + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + +                + + + + + +               + + + + + + + + + + + +   + + +         + + + + + + + + +   + + + + + + + + + + + +  + + +   + + +  + + + + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + +   + + +  + + +   + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + +                  + + +          + + + + + + + + + + + +     + + +     + + +    + + +  + + +  + + +    + + +   + + + + + + + + + + + + + + +       + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + +  + + + + + +      + + +  + + + + + + + + + + + + + + + + + + + + +     + + +       + + +                        + + +         + + + + + + + + +    + + + + + +    + + + + + +         + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +   + + +   + + + + + +  + + + + + + + + + + + +     + + + + + + + + + + + +  + + + + + +   + + +                    + + +        + + + + + + + + + + + + + + + + + +      + + + + + +       + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + + + + +   + + +  + + + + + + + + + + + + + + + + + +        + + + + + + + + +   + + + + + +    + + + + + + + + +                           + + +                 + + + + + +   + + +  + + + + + + + + + + + +        + + + + + + + + +  + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +  + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + +      + + +                   + + + + + +                 + + + + + + + + + + + +        + + +   + + +   + + +    + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + + + + + + + +         + + +                              + + + + + +       + + +         + + +     + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +   + + + + + +  + + + + + + + + + + + + + + + + + +      + + +                           + + + + + +               + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + +             + + + + + + + + +    + + + + + +                                   + + + + + + + + + + + + + + + + + +             + + + + + +    + + + + + +    + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + +    + + +    + + + + + + + + +                       + + + + + + + + + + + + + + + + + + + + +         + + +     + + +  + + + + + +  + + +             + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + +   + + + + + +     + + + + + +                       + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +      + + + + + + + + +     + + + + + + + + + + + + + + + + + +           + + + + + + + + + + + + + + + + + +                        + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + + + + +  + + + + + +   + + + + + +    + + + + + +   + + + + + + + + + + + + + + +  + + + + + + + + +        + + +   + + +  + + +      + + + + + + + + +                         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + +    + + +  + + +    + + + + + +      + + + + + +      + + +    + + + + + + + + +                             + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +  + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + +     + + +  + + + + + + + + + + + +    + + +             + + + + + +                + + +           + + + + + + + + +         + + + + + + + + + + + + + + + + + + + + +         + + +  + + + + + +  + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + +  + + +   + + + + + + + + + + + +   + + +                + + + + + +                 + + + + + +        + + +             + + + + + + + + + + + + + + + + + + + + +           + + +         + + + + + + + + + + + +   + + + + + +  + + + + + + + + + + + + + + +      + + + + + +    + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +   + + +  + + + + + + + + +    + + +                   + + +                              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +             + + + + + + + + + + + + + + + + + + + + + + + +   + + +  + + +  + + + + + +     + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +       + + + + + +    + + + + + +    + + + + + + + + +  + + +      + + +         + + +  + + + + + +                     + + + + + +    + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + +   + + + + + +     + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + +   + + + + + + + + +    + + + + + +     + + + + + +      + + + + + +  + + +  + + + + + +    + + +     + + + + + + + + +     + + +          + + +  + + +      + + + + + +    + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + + + + + + + + + + +  + + +   + + +   + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + +  + + + + + +    + + + + + + + + +  + + + + + +     + + +        + + + + + +   + + + + + +   + + + + + +  + + +   + + +  + + + + + + + + + + + +  + + + + + + + + +    + + + + + + + + + + + +     + + +         + + +  + + +       + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + +             + + + + + + + + +  + + + + + + + + +  + + + + + +   + + + + + + + + + + + +     + + + + + + + + + + + + + + +   + + +   + + + + + + + + + + + + + + +    + + +           + + +  + + + + + + + + +  + + + + + + + + + + + + + + + + + +                                + + + + + + + + +                   + + + + + + + + + + + +  + + +  + + + + + + + + + + + +       + + + + + +  + + + + + +   + + + + + + + + +  + + + + + + + + + + + + + + +  + + +  + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + +             + + +    + + + + + + + + + + + + + + + + + + + + +  + + + + + +                               + + +           + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + +  + + + + + + + + + + + +  + + + + + +  + + +    + + +    + + + + + + + + + + + + + + +  + + +      + + + + + +         + + +  + + + + + + + + + + + + + + +         + + +     + + + + + +                     + + +    + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + +  + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + +    + + +   + + + + + + + + +  + + +       + + +                  + + + + + + + + + + + +  + + + + + + + + +           + + +   + + + + + +                      + + +       + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +  + + + + + +   + + +   + + + + + + + + +         + + +         + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +                     + + + + + +                       + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +    + + +  + + + + + + + + + + + + + + + + + + + + +    + + +  + + +     + + + + + + + + +  + + +  + + + + + + + + + + + +  + + + + + +       + + +  + + + + + +  + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                     + + +     + + +       + + +          + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +   + + +  + + + + + +      + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +         + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +   + + +  + + + + + +           + + + + + + + + +        + + + + + +                   + + + + + +  + + +  + + + + + +  + + +  + + + + + +   + + +   + + + + + +  + + +      + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + +    + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + +              + + + + + + + + +            + + +        + + +  + + +  + + + + + + + + +  + + + + + + + + + + + +    + + +     + + +     + + + + + + + + + + + +  + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +           + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +             + + +            + + +          + + +        + + + + + + + + + + + + + + + + + +  + + + + + +  + + +    + + + + + + + + +  + + +   + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + +               + + +    + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + +                      + + + + + + + + + + + +            + + + + + +    + + + + + + + + + + + + + + +  + + + + + + + + +   + + +        + + + + + + + + +     + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + +  + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +                                + + + + + + + + + + + + + + +         + + +   + + + + + +    + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +       + + + + + +   + + + + + + + + +  + + + + + +      + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + +    + + +  + + +  + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                                + + + + + + + + + + + +        + + + + + +   + + + + + +  + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +  + + +   + + +        + + +      + + +  + + + + + +    + + +  + + +  + + + + + + + + + + + +       + + +    + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + +      + + +            + + + + + +              + + +     + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +  + + + + + +   + + +   + + +    + + +      + + + + + + + + +    + + + + + +   + + +     + + + + + +   + + + + + + + + + + + +          + + + + + +  + + + + + + + + + + + + + + +   + + +                             + + +      + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +     + + + + + +    + + +   + + +             + + +  + + + + + +    + + +    + + + + + +   + + +          + + + + + + + + +                                 + + + + + +  + + +    + + +    + + +   + + + + + + + + + + + +   + + +   + + + + + + + + + + + +   + + +         + + + + + +     + + + + + +      + + +       + + +         + + + + + + + + +  + + + + + +                           + + + + + +      + + +  + + + + + + + + +       + + + + + + + + + + + +       + + + + + +  + + +    + + +    + + +          + + +  + + + + + +      + + + + + +  + + + + + +                                      + + +   + + +  + + +            + + +      + + +   + + + + + +                  + + + + + + + + +         + + + + + + + + +   + + + + + + + + + + + +           + + +                                 + + + + + + + + + + + +  + + +    + + + + + +   + + +    + + + + + + + + + + + +   + + + + + + + + + + + +  + + +                + + +     + + + + + +      + + +  + + + + + +  + + +        + + +   + + + + + +   + + + + + +          + + + + + + + + +             + + +  + + +  + + + + + +    + + +  + + + + + + + + +     + + + + + +  + + + + + +   + + +                   + + + + + + + + +      + + +              + + +     + + + + + +           + + + + + + + + +          + + + + + + + + + + + +  + + + + + +   + + +    + + + + + +   + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +               + + + + + +         + + +       + + + + + +                + + + + + +  + + + + + +      + + +             + + + + + + + + +   + + +     + + +         + + + + + +    + + +  + + + + + + + + + + + + + + + + + + + + +       + + + + + +             + + + + + +            + + + + + +              + + +   + + +     + + + + + +   + + +            + + + + + +  + + +        + + +  + + + + + + + + +   + + + + + + + + + + + +    + + + + + +    + + +  + + + + + + + + + + + + + + + + + + + + +  + + +     + + +                    + + +  + + +  + + +                              + + + + + + + + +          + + +  + + + + + + + + + + + +  + + + + + +     + + +         + + +     + + + + + + + + + + + +     + + + + + + + + + + + +    + + +   + + +                           + + + + + +                   + + + + + + + + +          + + + + + + + + + + + +     + + + + + +  + + + + + + + + + + + +      + + + + + + + + +   + + +  + + +  + + + + + + + + +      + + + + + +      + + + + + + + + +  + + + + + +  + + +                  + + + + + + + + +            + + +                     + + +    + + + + + +  + + +  + + + + + + + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + +   + + + + + +    + + + + + + + + +  + + +  + + +                       + + + + + + + + + + + + + + + + + +    + + +          + + +                         + + + + + + + + +  + + + + + +  + + +     + + + + + + + + +    + + + + + + + + + + + + + + +  + + +  + + + + + + + + +     + + +   + + +  + + + + + +                 + + +  + + + + + +         + + + + + + + + +                  + + +       + + +                 + + + + + + + + +  + + + + + + + + + + + +  + + + + + +   + + +   + + +   + + + + + +    + + + + + +     + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + + + + +  + + + + + + + + + + + +                     + + +         + + + + + + + + + + + +         + + + + + +           + + +                     + + + + + +        + + + + + +  + + + + + +       + + +       + + + + + +   + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + +  + + + + + +                                     + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + +      + + + + + + + + + + + +          + + +                               + + +   + + +  + + + + + + + + + + + +       + + + + + +    + + + + + + + + +  + + +    + + + + + + + + + + + + + + + + + +            + + +               + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + +                                    + + + + + + + + + + + +    + + + + + + + + + + + + + + +        + + +       + + +   + + + + + + + + +   + + + + + +       + + +        + + + + + +                     + + + + + + + + + + + + + + +   + + + + + +    + + + + + + + + + + + + + + + + + +         + + + + + +                                + + + + + +     + + +  + + +   + + +  + + + + + +       + + + + + + + + + + + +   + + +  + + + + + + + + + + + +    + + +  + + + + + +     + + + + + + + + +                    + + + + + + + + +      + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + +           + + +  + + + + + +    + + + + + +  + + +                           + + + + + +     + + +  + + + + + +  + + + + + + + + + + + + + + +  + + + + + +  + + +           + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + +   + + +     + + +     + + +                  + + +  + + +  + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + +         + + +     + + +  + + + + + + + + + + + + + + + + + +                          + + +     + + + + + + + + + + + + + + + + + + + + +  + + +      + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + +    + + +      + + +    + + +                 + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +         + + +    + + + + + + + + + + + + + + +   + + + + + +                             + + + + + + + + + + + + + + +    + + + + + + + + +   + + + + + + + + +   + + +  + + + + + +   + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +                          + + + + + + + + + + + +  + + +   + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +               + + + + + + + + + + + +                               + + +  + + +  + + +   + + +  + + + + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +  + + +        + + + + + +          + + + + + + + + + + + + + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                + + + + + + + + + + + + + + + + + +                   + + +   + + +   + + + + + + + + +  + + +   + + + + + + + + +  + + + + + +        + + + + + +    + + +   + + + + + +   + + + + + +         + + + + + +           + + + + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +               + + + + + + + + + + + + + + +           + + +               + + +   + + +   + + + + + + + + +             + + + + + + + + +    + + +   + + +                            + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                 + + + + + + + + + + + +         + + +                           + + +        + + + + + +                          + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +              + + +               + + +        + + + + + +   + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +        + + + + + +                            + + + + + + + + + + + + + + + + + + + + + + + +   + + +             + + + + + +            + + +     + + +       + + +       + + + + + + + + + + + + + + + + + +   + + + + + +    + + + + + +  + + +    + + + + + +                   + + + + + +        + + + + + + + + + + + + + + + + + +  + + +                   + + +             + + +        + + + + + + + + + + + + + + + + + + + + +    + + + + + + + + +  + + + + + +          + + + + + + + + + + + +       + + +              + + +        + + + + + + + + + + + +                    + + +  + + +        + + + + + +     + + +         + + + + + + + + +     + + + + + + + + +    + + +      + + +       + + + + + +      + + +             + + + + + +   + + +            + + +        + + + + + +           + + + + + +              + + + + + + + + +  + + +        + + +               + + + + + +           + + +   + + +        + + +      + + + + + + + + +         + + + + + +                  + + +  + + +    + + +  + + + + + + + + +         + + +                         + + +       + + +      + + +          + + +      + + + + + +   + + +             + + +             + + + + + +   + + +  + + + + + + + + + + + + + + + + + +  + + +   + + + + + + + + +  + + +        + + + + + +  + + + + + +                  + + +         + + + + + + + + + + + +             + + +        + + +             + + + + + + + + +           + + + + + + + + +  + + + + + + + + +   + + + + + + + + + + + +    + + + + + +        + + + + + + + + +  + + +  + + + + + + + + +                                         + + +        + + +              + + +               + + + + + + + + +  + + + + + +  + + +  + + + + + + + + + + + + + + +      + + + + + +        + + + + + +   + + + + + +                   + + + + + +    + + + + + + + + +                         + + +            + + + + + +         + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + +       + + +  + + + + + +  + + +                 + + + + + +       + + +     + + +                       + + +             + + + + + + + + + + + +            + + + + + +   + + +   + + + + + + + + +  + + + + + + + + +  + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + + + + + + + + + + +       + + + + + +                 + + +    + + + + + + + + + + + + + + + + + + + + + + + +   + + +                             + + +  + + + + + + + + +             + + +  + + + + + + + + + + + +  + + +       + + +    + + + + + + + + + + + + + + + + + + + + + + + +     + + +   + + + + + + + + + + + +             + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + +          + + + + + +    + + + + + +               + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + +     + + +        + + +   + + +       + + +    + + +   + + + + + + + + + + + + + + + + + + + + +      + + +   + + + + + + + + + + + +               + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +  + + + + + +         + + +    + + + + + +           + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +           + + + + + + + + + + + +     + + +     + + + + + +    + + + + + + + + + + + + + + +       + + + + + + + + + + + + + + + + + +             + + +  + + + + + + + + + + + + + + + + + + + + + + + +     + + +   + + +              + + + + + + + + +   + + + + + + + + +     + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + +     + + + + + + + + + + + + + + + + + + + + +   + + +     + + + + + + + + + + + +     + + +    + + +     + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + +                  + + + + + +  + + + + + + + + + + + + + + + + + + + + +   + + + + + +            + + +  + + + + + +            + + +  + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + +       + + + + + + + + + + + + + + +   + + + + + +  + + + + + +      + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +       + + +         + + +  + + + + + + + + + + + + + + + + + + + + +                   + + + + + + + + + + + + + + + + + +        + + +            + + +                + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +           + + + + + +  + + + + + + + + + + + + + + +   + + + + + + + + + + + +      + + + + + +  + + + + + + + + + + + + + + + + + +     + + +           + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                  + + + + + + + + +  + + + + + + + + + + + + + + + + + +        + + +           + + +              + + + + + + + + + + + + + + + + + + + + + + + +     + + +     + + +   + + + + + +  + + + + + +      + + + + + + + + + + + + + + + + + +       + + +           + + + + + +  + + + + + + + + + + + + + + + + + + + + +                      + + +     + + + + + + + + + + + + + + + + + +         + + +            + + + + + +    + + +  + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + +           + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +    + + + + + + + + +       + + +           + + + + + +               + + +  + + + + + + + + + + + + + + + + + + + + +                     + + + + + +   + + + + + + + + +        + + +   + + + + + + + + + + + + + + + + + +        + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + +  + + + + + +  \ No newline at end of file Added: SwiftTutorials/UofC_2013-04-09/data/sat/t2 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t2 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/data/sat/t3 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t3 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/data/sat/t4 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t4 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/data/sat/t5 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t5 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/data/sat/t6 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t6 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/data/sat/t7 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t7 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/data/sat/t8 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t8 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/data/sat/t9 =================================================================== (Binary files differ) Property changes on: SwiftTutorials/UofC_2013-04-09/data/sat/t9 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: SwiftTutorials/UofC_2013-04-09/part01/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part01/README (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part01/README 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,39 @@ + +First Swift script - run a "simulation" program, random.sh + +Verify that random.sh is in your path, and that it works: + +$ which random.sh +$ random.sh + +Study the Swift program: + +$ cat p1.swift + +Examine the tool catalog file: + +$ cat tc + +Run the program: + +$ swift -tc.file tc p1.swift + +Look at the output: + +$ cat _concurrent/* + +Look at the other files generated: + +$ ls -l + +We'll talk more about these files later + +$ cleanup + +Then test the sript a few more times. + +$ swift -tc.file tc p1.swift; cat _concurrent/* + +$ swift -tc.file tc p1.swift; cat _concurrent/* + +$ swift -tc.file tc p1.swift; cat _concurrent/* Added: SwiftTutorials/UofC_2013-04-09/part01/p1.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part01/p1.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part01/p1.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,8 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +file f = mysim(); Added: SwiftTutorials/UofC_2013-04-09/part01/tc =================================================================== --- SwiftTutorials/UofC_2013-04-09/part01/tc (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part01/tc 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1 @@ +localhost random random.sh Added: SwiftTutorials/UofC_2013-04-09/part02/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part02/README (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part02/README 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,33 @@ + +Swift script - run a "simulation" program, random.sh, specifying the output filename + +Study the Swift program: + +$ cat p2.swift + +Examine the tool catalog file: + +$ cat tc + +Run the program: + +$ swift -tc.file tc p2.swift + +Look at the output: + +$ cat sim.out + +Then test the sript a few more times. + +$ swift -tc.file tc p1.swift; cat sim.out + +$ swift -tc.file tc p1.swift; cat sim.out + +$ swift -tc.file tc p1.swift; cat sim.out + +Things to try: + +- change the output filename + +- add a dir component to the output filename + Added: SwiftTutorials/UofC_2013-04-09/part02/p2.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part02/p2.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part02/p2.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,9 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +file f<"sim.out">; +f = mysim(); Added: SwiftTutorials/UofC_2013-04-09/part02/p2b.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part02/p2b.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part02/p2b.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,9 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +file f<"output/sim.out">; +f = mysim(); Added: SwiftTutorials/UofC_2013-04-09/part02/tc =================================================================== --- SwiftTutorials/UofC_2013-04-09/part02/tc (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part02/tc 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1 @@ +localhost random random.sh Added: SwiftTutorials/UofC_2013-04-09/part03/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part03/README (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part03/README 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,20 @@ + +Swift script - run many instances of a "simulation" program in a foreach() loop + + +Run the program: + +$ swift -tc.file tc p3.swift + +Look at the output: + +$ cd _concurrent +$ ls -l +$ cat * + +Look at the order in which the files were written: + +$ ls -lt + +We'll come back to this... + Added: SwiftTutorials/UofC_2013-04-09/part03/p3.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part03/p3.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part03/p3.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,11 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +foreach i in [0:9] { + file f = mysim(); +} + Added: SwiftTutorials/UofC_2013-04-09/part03/tc =================================================================== --- SwiftTutorials/UofC_2013-04-09/part03/tc (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part03/tc 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1 @@ +localhost random random.sh Added: SwiftTutorials/UofC_2013-04-09/part04/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part04/README (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part04/README 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,44 @@ + +Swift script - run many instances of a "simulation" program in a foreach() loop, + with explicitly named output files. + +Study the program: + +- @strcat() is a Swift builtin function (the @name() form is used for many bu not all builtins) + +- @strcat() coerces numeric args to strings + +- <"filename"> was a shorthand for but the latter is more flexible. + +Run the program: + +$ swift -tc.file tc p4.swift + +Look at the output: + +$ cd output +$ ls -l +$ cat * + + +Try: + +- changing the output names + +- @arg("argname") returns arguments after the .swift script on the swift command line: + + swift -tc.file tc p4.swift -myarg1=value1 ... + + @toInt() converts strings to integers: + + @toInt(@arg("myArg1")) + + Adjust the script to take the number of simulations to do from the command line + +- trace(expr1,...) traces expressions on stdout. + + Try inserting a few traces. But remember: Swift is *very* concurrent! + + + + Added: SwiftTutorials/UofC_2013-04-09/part04/p4.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part04/p4.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part04/p4.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,12 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +foreach i in [0:9] { + file f ; + f = mysim(); +} + Added: SwiftTutorials/UofC_2013-04-09/part04/tc =================================================================== --- SwiftTutorials/UofC_2013-04-09/part04/tc (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part04/tc 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1 @@ +localhost random random.sh Added: SwiftTutorials/UofC_2013-04-09/part05/README =================================================================== --- SwiftTutorials/UofC_2013-04-09/part05/README (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part05/README 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,27 @@ + +Swift script - run many instances of a "simulation" program in a foreach() loop, + with explicitly named output files. + +Study the program: + +- @strcat() is a Swift builtin function (the @name() form is used for many bu not all builtins) + +- @strcat() coerces numeric args to strings + +- <"filename"> was a shorthand for but the latter is more flexible. + +Run the program: + +$ swift -tc.file tc p4.swift + +Look at the output: + +$ cd output +$ ls -l +$ cat * + + +Try: + +- changing the output names + Added: SwiftTutorials/UofC_2013-04-09/part05/p5.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/part05/p5.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part05/p5.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,22 @@ +type file; + +app (file o) mysim () +{ + random stdout=@filename(o); +} + +app (file o) analyze (file s[]) +{ + average @filenames(s) stdout=@filename(o); +} + +file sims[]; + +foreach i in [0:9] { + file simout ; + simout = mysim(); + sims[i] = simout; +} + +file stats<"output/average.out">; +stats = analyze(sims); Added: SwiftTutorials/UofC_2013-04-09/part05/tc =================================================================== --- SwiftTutorials/UofC_2013-04-09/part05/tc (rev 0) +++ SwiftTutorials/UofC_2013-04-09/part05/tc 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,2 @@ +localhost random random.sh +localhost average avg.sh Added: SwiftTutorials/UofC_2013-04-09/scripts/catsn.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/scripts/catsn.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/scripts/catsn.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,12 @@ +type file; + +app (file o) cat (file i) +{ + cat @i stdout=@o; +} + +file out[]; +foreach j in [1:@toint(@arg("n","1"))] { + file data<"data.txt">; + out[j] = cat(data); +} Added: SwiftTutorials/UofC_2013-04-09/scripts/script01.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/scripts/script01.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/scripts/script01.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,9 @@ +type file; + +app (file o) getinfo () +{ + getinfo stdout=@o; +} + +file info <"output/info.txt">; +info = getinfo(); Added: SwiftTutorials/UofC_2013-04-09/scripts/script02.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/scripts/script02.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/scripts/script02.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,13 @@ +type file; + +app (file o) getinfo () +{ + getinfo stdout=@o; +} + +foreach i in [0:24] { + string fname = @strcat("output/info.",i,".txt"); + file info ; + info = getinfo(); +} + Added: SwiftTutorials/UofC_2013-04-09/scripts/script03.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/scripts/script03.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/scripts/script03.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,19 @@ +type file; + +app (file o) getinfo () +{ + getinfo stdout=@o; +} + +app (file o) filterinfo (file i) +{ + filterinfo @i @o; +} + +foreach i in [0:24] { + string fname = @strcat("output/info.",i,".txt"); + file info ; + info = getinfo(); + file filtered = filterinfo(info); +} + Added: SwiftTutorials/UofC_2013-04-09/scripts/script04.swift =================================================================== --- SwiftTutorials/UofC_2013-04-09/scripts/script04.swift (rev 0) +++ SwiftTutorials/UofC_2013-04-09/scripts/script04.swift 2013-04-09 06:57:55 UTC (rev 6418) @@ -0,0 +1,25 @@ +type file; + +app (file o) getinfo () +{ + getinfo stdout=@o; +} + +app (file o) filter (file i) +{ + filterinfo @i @o; +} + +file filtered[]; + +foreach i in [0:24] { + string fname = @strcat("output/info.",i,".txt"); + file info ; + info = getinfo(); + // replace: filtered[i] = filter(info); + // with: + string filfname = ??? + filfname = + filtered[i] = +} + From yadunandb at ci.uchicago.edu Thu Apr 11 15:29:13 2013 From: yadunandb at ci.uchicago.edu (yadunandb at ci.uchicago.edu) Date: Thu, 11 Apr 2013 20:29:13 -0000 Subject: [Swift-commit] r6430 - in branches/release-0.94/tests: groups stress stress/IO stress/IO/i1 stress/IO/i2 stress/apps stress/apps/data stress/apps/modis_beagle stress/apps/modis_local stress/apps/modis_midway stress/apps/modis_multiple stress/apps/modis_uc3 stress/internals Message-ID: <20130411202913.5D2409D79A@svn.ci.uchicago.edu> Author: yadunandb Date: 2013-04-11 15:29:12 -0500 (Thu, 11 Apr 2013) New Revision: 6430 Added: branches/release-0.94/tests/groups/group-stress-heavy.sh branches/release-0.94/tests/groups/group-stress-midway.sh branches/release-0.94/tests/stress/IO/ branches/release-0.94/tests/stress/IO/i1/ branches/release-0.94/tests/stress/IO/i1/cf branches/release-0.94/tests/stress/IO/i1/filegen.sh branches/release-0.94/tests/stress/IO/i1/gendata.pl branches/release-0.94/tests/stress/IO/i1/i1.kml branches/release-0.94/tests/stress/IO/i1/i1.swift branches/release-0.94/tests/stress/IO/i1/i1.swiftx branches/release-0.94/tests/stress/IO/i1/input/ branches/release-0.94/tests/stress/IO/i1/run branches/release-0.94/tests/stress/IO/i1/sites.xml branches/release-0.94/tests/stress/IO/i1/sites.xml.bak branches/release-0.94/tests/stress/IO/i1/sumall.pl branches/release-0.94/tests/stress/IO/i1/tc branches/release-0.94/tests/stress/IO/i2/ branches/release-0.94/tests/stress/IO/i2/cf branches/release-0.94/tests/stress/IO/i2/filegen.sh branches/release-0.94/tests/stress/IO/i2/gendata.pl branches/release-0.94/tests/stress/IO/i2/i1.kml branches/release-0.94/tests/stress/IO/i2/i1.swift branches/release-0.94/tests/stress/IO/i2/i1.swiftx branches/release-0.94/tests/stress/IO/i2/input/ branches/release-0.94/tests/stress/IO/i2/run branches/release-0.94/tests/stress/IO/i2/sites.xml branches/release-0.94/tests/stress/IO/i2/sites.xml.bak branches/release-0.94/tests/stress/IO/i2/sumall.pl branches/release-0.94/tests/stress/IO/i2/tc branches/release-0.94/tests/stress/apps/ branches/release-0.94/tests/stress/apps/data/ branches/release-0.94/tests/stress/apps/data/t0 branches/release-0.94/tests/stress/apps/data/t1 branches/release-0.94/tests/stress/apps/data/t2 branches/release-0.94/tests/stress/apps/data/t3 branches/release-0.94/tests/stress/apps/data/t4 branches/release-0.94/tests/stress/apps/data/t5 branches/release-0.94/tests/stress/apps/data/t6 branches/release-0.94/tests/stress/apps/data/t7 branches/release-0.94/tests/stress/apps/data/t8 branches/release-0.94/tests/stress/apps/data/t9 branches/release-0.94/tests/stress/apps/get.sh branches/release-0.94/tests/stress/apps/modis_beagle/ branches/release-0.94/tests/stress/apps/modis_beagle/clean branches/release-0.94/tests/stress/apps/modis_beagle/getlanduse.pl branches/release-0.94/tests/stress/apps/modis_beagle/input/ branches/release-0.94/tests/stress/apps/modis_beagle/makeinput branches/release-0.94/tests/stress/apps/modis_beagle/modis.args branches/release-0.94/tests/stress/apps/modis_beagle/modis.kml branches/release-0.94/tests/stress/apps/modis_beagle/modis.setup.sh branches/release-0.94/tests/stress/apps/modis_beagle/modis.swift branches/release-0.94/tests/stress/apps/modis_beagle/modis.timeout branches/release-0.94/tests/stress/apps/modis_beagle/run branches/release-0.94/tests/stress/apps/modis_beagle/sites.template.xml branches/release-0.94/tests/stress/apps/modis_beagle/sites.template.xml.bak branches/release-0.94/tests/stress/apps/modis_beagle/swift.properties branches/release-0.94/tests/stress/apps/modis_beagle/tc.template.data branches/release-0.94/tests/stress/apps/modis_beagle/title.txt branches/release-0.94/tests/stress/apps/modis_local/ branches/release-0.94/tests/stress/apps/modis_local/clean branches/release-0.94/tests/stress/apps/modis_local/getlanduse.pl branches/release-0.94/tests/stress/apps/modis_local/makeinput branches/release-0.94/tests/stress/apps/modis_local/modis.args branches/release-0.94/tests/stress/apps/modis_local/modis.setup.sh branches/release-0.94/tests/stress/apps/modis_local/modis.swift branches/release-0.94/tests/stress/apps/modis_local/modis.timeout branches/release-0.94/tests/stress/apps/modis_local/run branches/release-0.94/tests/stress/apps/modis_local/sites.template.xml branches/release-0.94/tests/stress/apps/modis_local/swift.properties branches/release-0.94/tests/stress/apps/modis_local/tc.template.data branches/release-0.94/tests/stress/apps/modis_local/title.txt branches/release-0.94/tests/stress/apps/modis_midway/ branches/release-0.94/tests/stress/apps/modis_midway/clean branches/release-0.94/tests/stress/apps/modis_midway/getlanduse.pl branches/release-0.94/tests/stress/apps/modis_midway/makeinput branches/release-0.94/tests/stress/apps/modis_midway/modis.args branches/release-0.94/tests/stress/apps/modis_midway/modis.setup.sh branches/release-0.94/tests/stress/apps/modis_midway/modis.swift branches/release-0.94/tests/stress/apps/modis_midway/modis.timeout branches/release-0.94/tests/stress/apps/modis_midway/run branches/release-0.94/tests/stress/apps/modis_midway/sandyb.xml branches/release-0.94/tests/stress/apps/modis_midway/sites.template.xml branches/release-0.94/tests/stress/apps/modis_midway/swift.properties branches/release-0.94/tests/stress/apps/modis_midway/tc.template.data branches/release-0.94/tests/stress/apps/modis_midway/title.txt branches/release-0.94/tests/stress/apps/modis_midway/westmere.xml branches/release-0.94/tests/stress/apps/modis_multiple/ branches/release-0.94/tests/stress/apps/modis_multiple/clean branches/release-0.94/tests/stress/apps/modis_multiple/getlanduse.pl branches/release-0.94/tests/stress/apps/modis_multiple/makeinput branches/release-0.94/tests/stress/apps/modis_multiple/modis.args branches/release-0.94/tests/stress/apps/modis_multiple/modis.setup.sh branches/release-0.94/tests/stress/apps/modis_multiple/modis.swift branches/release-0.94/tests/stress/apps/modis_multiple/modis.timeout branches/release-0.94/tests/stress/apps/modis_multiple/run branches/release-0.94/tests/stress/apps/modis_multiple/sites.template.xml branches/release-0.94/tests/stress/apps/modis_multiple/swift.properties branches/release-0.94/tests/stress/apps/modis_multiple/tc.template.data branches/release-0.94/tests/stress/apps/modis_multiple/title.txt branches/release-0.94/tests/stress/apps/modis_uc3/ branches/release-0.94/tests/stress/apps/modis_uc3/clean branches/release-0.94/tests/stress/apps/modis_uc3/getlanduse.pl branches/release-0.94/tests/stress/apps/modis_uc3/input/ branches/release-0.94/tests/stress/apps/modis_uc3/makeinput branches/release-0.94/tests/stress/apps/modis_uc3/modis.args branches/release-0.94/tests/stress/apps/modis_uc3/modis.kml branches/release-0.94/tests/stress/apps/modis_uc3/modis.setup.sh branches/release-0.94/tests/stress/apps/modis_uc3/modis.swift branches/release-0.94/tests/stress/apps/modis_uc3/modis.timeout branches/release-0.94/tests/stress/apps/modis_uc3/run branches/release-0.94/tests/stress/apps/modis_uc3/sites.template.xml branches/release-0.94/tests/stress/apps/modis_uc3/swift.properties branches/release-0.94/tests/stress/apps/modis_uc3/tc.template.data branches/release-0.94/tests/stress/apps/modis_uc3/title.txt branches/release-0.94/tests/stress/internals/ branches/release-0.94/tests/stress/internals/.args.m4 branches/release-0.94/tests/stress/internals/m4_template_gen.sh branches/release-0.94/tests/stress/internals/scripts.sh branches/release-0.94/tests/stress/internals/test.args.m4 branches/release-0.94/tests/stress/internals/title.txt branches/release-0.94/tests/stress/internals/x_append_iter.args branches/release-0.94/tests/stress/internals/x_append_iter.args.m4 branches/release-0.94/tests/stress/internals/x_append_iter.swift branches/release-0.94/tests/stress/internals/x_foreach.args branches/release-0.94/tests/stress/internals/x_foreach.args.m4 branches/release-0.94/tests/stress/internals/x_foreach.swift branches/release-0.94/tests/stress/internals/x_func.args branches/release-0.94/tests/stress/internals/x_func.args.m4 branches/release-0.94/tests/stress/internals/x_func.swift branches/release-0.94/tests/stress/internals/x_func.timeout branches/release-0.94/tests/stress/internals/x_func_float.args branches/release-0.94/tests/stress/internals/x_func_float.args.m4 branches/release-0.94/tests/stress/internals/x_func_float.swift branches/release-0.94/tests/stress/internals/x_func_float.timeout branches/release-0.94/tests/stress/internals/x_iterate.args branches/release-0.94/tests/stress/internals/x_iterate.args.m4 branches/release-0.94/tests/stress/internals/x_iterate.swift branches/release-0.94/tests/stress/internals/x_recursion.args branches/release-0.94/tests/stress/internals/x_recursion.args.m4 branches/release-0.94/tests/stress/internals/x_recursion.swift branches/release-0.94/tests/stress/internals/x_recursion.timeout branches/release-0.94/tests/stress/internals/x_sleep.args branches/release-0.94/tests/stress/internals/x_sleep.args.m4 branches/release-0.94/tests/stress/internals/x_sleep.swift branches/release-0.94/tests/stress/internals/x_sleep.timeout Log: Adding group files for site specific tests (Midway,MODIS) and generic stress tests Adding stress tests. Added: branches/release-0.94/tests/groups/group-stress-heavy.sh =================================================================== --- branches/release-0.94/tests/groups/group-stress-heavy.sh (rev 0) +++ branches/release-0.94/tests/groups/group-stress-heavy.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,8 @@ +# GROUPLIST definition to run all local tests \ + +GROUPLIST=( + $TESTDIR/stress/IO \ + $TESTDIR/stress/internals \ + ) + +checkvars WORK \ No newline at end of file Added: branches/release-0.94/tests/groups/group-stress-midway.sh =================================================================== --- branches/release-0.94/tests/groups/group-stress-midway.sh (rev 0) +++ branches/release-0.94/tests/groups/group-stress-midway.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,11 @@ +# GROUPLIST definition to run all local tests \ + +GROUPLIST=( + $TESTDIR/stress/apps/modis_local \ + $TESTDIR/stress/apps/modis_midway \ + $TESTDIR/stress/apps/modis_beagle \ + $TESTDIR/stress/apps/modis_uc3 \ + $TESTDIR/stress/apps/modis_multiple \ + ) + +checkvars WORK \ No newline at end of file Added: branches/release-0.94/tests/stress/IO/i1/cf =================================================================== --- branches/release-0.94/tests/stress/IO/i1/cf (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/cf 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: branches/release-0.94/tests/stress/IO/i1/filegen.sh =================================================================== --- branches/release-0.94/tests/stress/IO/i1/filegen.sh (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/filegen.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,9 @@ +#!/bin/bash + +FILES=10 +rm input/* -rf +for (( i=1; i <= 20; i=$(($i+1)) )) +do + top=$(($i*5120000)) + ./gendata.pl $top > input/file_$i.inp & +done; \ No newline at end of file Property changes on: branches/release-0.94/tests/stress/IO/i1/filegen.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/IO/i1/gendata.pl =================================================================== --- branches/release-0.94/tests/stress/IO/i1/gendata.pl (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/gendata.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,21 @@ +#!/usr/bin/perl + +use strict; + +my @characters=('0'..'9'); +my $string=""; +my $length = $ARGV[0]; +my $width_count=0; + +foreach (1..$length) +{ + if($width_count == 5) { + $string .= "\n"; + $width_count=0; + next; + } + $string .= $characters[rand @characters]; + $width_count++; +} + +print $string; Property changes on: branches/release-0.94/tests/stress/IO/i1/gendata.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/IO/i1/i1.kml =================================================================== --- branches/release-0.94/tests/stress/IO/i1/i1.kml (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/i1.kml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,180 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + perl + + + + + + + ps + + + input + + + + + output + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Indir + + + + + + + + + + + + + Indir + + + + + + + + + + + + + nFiles + + + swift#string#17000swift#string#17001 + + + + + + + + Indir + + swift#string#17002swift#string#17003 + + + + + + + swift#mapper#17004 + swift#string#17005 + + + + + + swift#mapper#17006 + swift#string#17007 + + + + + + + inputs + + + + + + + + + + + + + outputs + i + + input + sumall_pl + + + + + + + + + + + + + + + + + Added: branches/release-0.94/tests/stress/IO/i1/i1.swift =================================================================== --- branches/release-0.94/tests/stress/IO/i1/i1.swift (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/i1.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,21 @@ +type file; +type perlscript; + +perlscript sumall_pl <"sumall.pl">; + +app (file output) do_sum (file input, perlscript ps) +{ + perl @ps @filename(input) stdout=@filename(output); +} + +# Constants and command line arguments +int nFiles = @toInt(@arg("nfiles", "1000")); +string Indir = @arg("Indir", "./input/"); + +# Input Dataset +file inputs[] ; +file outputs[] ; + +foreach input,i in inputs { + outputs[i] = do_sum(input, sumall_pl); +} Added: branches/release-0.94/tests/stress/IO/i1/i1.swiftx =================================================================== --- branches/release-0.94/tests/stress/IO/i1/i1.swiftx (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/i1.swiftx 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,69 @@ + + + + file + string + + + + perlscript + string + + + + + + + + + + + + + perl + + output + ps + input + + + + + + nFiles + + + nfiles1000 + + + + Indir + + Indir./input/ + + + + Indir + .inp + + + + + Indir + .out + + + + inputs + + + outputs + i + + input + sumall_pl + + + Added: branches/release-0.94/tests/stress/IO/i1/run =================================================================== --- branches/release-0.94/tests/stress/IO/i1/run (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/run 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,7 @@ +#! /bin/sh + +set -x + +export GLOBUS_HOSTNAME="swift.rcc.uchicago.edu" + +swift -config cf -sites.file sites.xml -tc.file tc i1.swift Property changes on: branches/release-0.94/tests/stress/IO/i1/run ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/IO/i1/sites.xml =================================================================== --- branches/release-0.94/tests/stress/IO/i1/sites.xml (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/sites.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,53 @@ + + + + + 10.00 + 10000 + 1 + 3600 + 00:05:00 + 100 + 100 + 1000 + 1 + 1 + "group_friends.{env.USER}" + nonshared + + . + + + + + sandyb + 16 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .64 + 10000 + /tmp/{env.USER} + + + + + westmere + 12 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .48 + 10000 + /tmp/{env.USER} + + + Added: branches/release-0.94/tests/stress/IO/i1/sites.xml.bak =================================================================== --- branches/release-0.94/tests/stress/IO/i1/sites.xml.bak (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/sites.xml.bak 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,71 @@ + + + + + 10.00 + 10000 + 1 + 3600 + 00:05:00 + 100 + 100 + 1000 + 1 + 1 + "group_friends.{env.USER}" + nonshared + + . + + + + + 24 + 100 + 100 + + pbs.aprun;pbs.mpp;depth=24;pbs.resource_list=advres=wilde.1768 + 3600 + 00:05:00 + /lustre/beagle/{env.USER}/swiftwork + 5 + 1 + 1 + 4.80 + 10000 + /tmp/{env.USER}/swiftwork + + + + + sandyb + 16 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .64 + 10000 + /tmp/{env.USER} + + + + + westmere + 12 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .48 + 10000 + /tmp/{env.USER} + + + Added: branches/release-0.94/tests/stress/IO/i1/sumall.pl =================================================================== --- branches/release-0.94/tests/stress/IO/i1/sumall.pl (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/sumall.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,9 @@ +#!/usr/bin/perl + +my $sum = 0; +my $filename = shift; +open(my $fd, "<", $filename) or die "Couldn;t access '$filename' \n"; +while ( my $line = <$fd> ) { + $sum += $line; +} +print "$sum\n"; Property changes on: branches/release-0.94/tests/stress/IO/i1/sumall.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/IO/i1/tc =================================================================== --- branches/release-0.94/tests/stress/IO/i1/tc (rev 0) +++ branches/release-0.94/tests/stress/IO/i1/tc 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,4 @@ +uc3 perl /usr/bin/perl null null null +beagle perl /usr/bin/perl null null null +#sandy perl /usr/bin/perl null null null +westmere perl /usr/bin/perl null null null Added: branches/release-0.94/tests/stress/IO/i2/cf =================================================================== --- branches/release-0.94/tests/stress/IO/i2/cf (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/cf 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: branches/release-0.94/tests/stress/IO/i2/filegen.sh =================================================================== --- branches/release-0.94/tests/stress/IO/i2/filegen.sh (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/filegen.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,31 @@ +#!/bin/bash + +case $STRESS in + "S1") + FILES=100 + SIZE=102400 + ;; + "S2") + FILES=1000 + SIZE=102400 + ;; + "S3") + FILES=1000 + SIZE=1024000 + ;; + "S4") + FILES=10000 + SIZE=1024000 + ;; + *) + SIZE=102400 + FILES=1000 + ;; +esac + +rm input/* -rf +for (( i=1; i <= $FILES ; i=$(($i+1)) )) +do + top=$SIZE + ./gendata.pl $top > input/file_$i.inp +done; \ No newline at end of file Property changes on: branches/release-0.94/tests/stress/IO/i2/filegen.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/IO/i2/gendata.pl =================================================================== --- branches/release-0.94/tests/stress/IO/i2/gendata.pl (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/gendata.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,21 @@ +#!/usr/bin/perl + +use strict; + +my @characters=('0'..'9'); +my $string=""; +my $length = $ARGV[0]; +my $width_count=0; + +foreach (1..$length) +{ + if($width_count == 5) { + $string .= "\n"; + $width_count=0; + next; + } + $string .= $characters[rand @characters]; + $width_count++; +} + +print $string; Property changes on: branches/release-0.94/tests/stress/IO/i2/gendata.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/IO/i2/i1.kml =================================================================== --- branches/release-0.94/tests/stress/IO/i2/i1.kml (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/i1.kml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,180 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + perl + + + + + + + ps + + + input + + + + + output + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Indir + + + + + + + + + + + + + Indir + + + + + + + + + + + + + nFiles + + + swift#string#17000swift#string#17001 + + + + + + + + Indir + + swift#string#17002swift#string#17003 + + + + + + + swift#mapper#17004 + swift#string#17005 + + + + + + swift#mapper#17006 + swift#string#17007 + + + + + + + inputs + + + + + + + + + + + + + outputs + i + + input + sumall_pl + + + + + + + + + + + + + + + + + Added: branches/release-0.94/tests/stress/IO/i2/i1.swift =================================================================== --- branches/release-0.94/tests/stress/IO/i2/i1.swift (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/i1.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,21 @@ +type file; +type perlscript; + +perlscript sumall_pl <"sumall.pl">; + +app (file output) do_sum (file input, perlscript ps) +{ + perl @ps @filename(input) stdout=@filename(output); +} + +# Constants and command line arguments +int nFiles = @toInt(@arg("nfiles", "1000")); +string Indir = @arg("Indir", "./input/"); + +# Input Dataset +file inputs[] ; +file outputs[] ; + +foreach input,i in inputs { + outputs[i] = do_sum(input, sumall_pl); +} Added: branches/release-0.94/tests/stress/IO/i2/i1.swiftx =================================================================== --- branches/release-0.94/tests/stress/IO/i2/i1.swiftx (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/i1.swiftx 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,69 @@ + + + + file + string + + + + perlscript + string + + + + + + + + + + + + + perl + + output + ps + input + + + + + + nFiles + + + nfiles1000 + + + + Indir + + Indir./input/ + + + + Indir + .inp + + + + + Indir + .out + + + + inputs + + + outputs + i + + input + sumall_pl + + + Added: branches/release-0.94/tests/stress/IO/i2/run =================================================================== --- branches/release-0.94/tests/stress/IO/i2/run (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/run 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,9 @@ +#! /bin/sh + +./filegen.sh + +set -x + +export GLOBUS_HOSTNAME="swift.rcc.uchicago.edu" + +swift -config cf -sites.file sites.xml -tc.file tc i1.swift Property changes on: branches/release-0.94/tests/stress/IO/i2/run ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/IO/i2/sites.xml =================================================================== --- branches/release-0.94/tests/stress/IO/i2/sites.xml (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/sites.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,53 @@ + + + + + 10.00 + 10000 + 1 + 3600 + 00:05:00 + 100 + 100 + 1000 + 1 + 1 + "group_friends.{env.USER}" + nonshared + + . + + + + + sandyb + 16 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .64 + 10000 + /tmp/{env.USER} + + + + + westmere + 12 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .48 + 10000 + /tmp/{env.USER} + + + Added: branches/release-0.94/tests/stress/IO/i2/sites.xml.bak =================================================================== --- branches/release-0.94/tests/stress/IO/i2/sites.xml.bak (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/sites.xml.bak 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,71 @@ + + + + + 10.00 + 10000 + 1 + 3600 + 00:05:00 + 100 + 100 + 1000 + 1 + 1 + "group_friends.{env.USER}" + nonshared + + . + + + + + 24 + 100 + 100 + + pbs.aprun;pbs.mpp;depth=24;pbs.resource_list=advres=wilde.1768 + 3600 + 00:05:00 + /lustre/beagle/{env.USER}/swiftwork + 5 + 1 + 1 + 4.80 + 10000 + /tmp/{env.USER}/swiftwork + + + + + sandyb + 16 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .64 + 10000 + /tmp/{env.USER} + + + + + westmere + 12 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .48 + 10000 + /tmp/{env.USER} + + + Added: branches/release-0.94/tests/stress/IO/i2/sumall.pl =================================================================== --- branches/release-0.94/tests/stress/IO/i2/sumall.pl (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/sumall.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,9 @@ +#!/usr/bin/perl + +my $sum = 0; +my $filename = shift; +open(my $fd, "<", $filename) or die "Couldn;t access '$filename' \n"; +while ( my $line = <$fd> ) { + $sum += $line; +} +print "$sum\n"; Property changes on: branches/release-0.94/tests/stress/IO/i2/sumall.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/IO/i2/tc =================================================================== --- branches/release-0.94/tests/stress/IO/i2/tc (rev 0) +++ branches/release-0.94/tests/stress/IO/i2/tc 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,4 @@ +uc3 perl /usr/bin/perl null null null +beagle perl /usr/bin/perl null null null +#sandy perl /usr/bin/perl null null null +westmere perl /usr/bin/perl null null null Added: branches/release-0.94/tests/stress/apps/data/t0 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t0 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/data/t1 =================================================================== --- branches/release-0.94/tests/stress/apps/data/t1 (rev 0) +++ branches/release-0.94/tests/stress/apps/data/t1 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,13792 @@ +    + + + + + + + + +      + + +  + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + +         + + + + + +              + + + + + + + + + + + + + + +      + + +       + + + + + + + + + + + + + + +       + + +  + + +   + + +  + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + +      + + + + + +               + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +       + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +  + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +  + + + + + +        + + + + + + + + +   + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + +  + + +    + + + + + +       + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + + + + + + + + + + + + + + + + + + + +   + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +  + + +   + + + + + +  + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +  + + + + + + + + + + + +   + + + + + + + + + + + +      + + + + + + + + + + + +  + + + + + + + + +    + + + + + + + + + + + +  + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +        + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + + + + + + + +  + + +      + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + + + + +   + + + + + + + + +  + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + + + + +    + + + + + + + + + + + +  + + + + + + + + +    + + + + + + + + + + + +     + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +      + + +   + + + + + + + + + + + + + + +   + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +     + + +   + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + +     + + + + + +    + + + + + + + + +    + + +   + + +   + + + + + + + + + + + + + + +    + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +       + + + + + +    + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + +      + + + + + +    + + + + + + + + +  + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + +         + + +   + + + + + + + + + + + + + + + + + +  + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +               + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + + + + + + + + + + +      + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +  + + + + + +    + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +   + + +       + + + + + + + + + + + +      + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + +       + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + +      + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + +    + + + + + +   + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +          + + + + + + + + + + + + + + +    + + + + + + + + + + + +   + + + + + + + + + + + + + + +  + + + + + + + + +          + + +  + + +    + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + +   + + +        + + +     + + +    + + + + + +   + + + + + + + + +  + + + + + + + + + + + +     + + + + + + + + +  + + +            + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +            + + + + + + + + + + + + + + +      + + + + + +  + + + + + +  + + + + + + + + + + + + + + +   + + +   + + +      + + +  + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +         + + + + + +    + + + + + +  + + + + + +    + + + + + + + + +                 + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + +        + + + + + +   + + +  + + + + + + + + +   + + +      + + +     + + +  + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + +  + + + + + +    + + + + + + + + +    + + + + + + + + + + + +      + + + + + +                   + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +  + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + +  + + + + + + + + + + + +     + + + + + + + + +     + + +    + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + +   + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + +                  + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + +   + + + + + + + + +   + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + +    + + +   + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +                   + + + + + + + + + + + + + + + + + + + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + +  + + + + + +    + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + +           + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + +  + + +                  + + + + + + + + + + + + + + +   + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + + + + + + + + + + + + + + + + +  + + + + + + + + +     + + +    + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + + + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +                 + + + + + + + + + + + + + + +   + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + +  + + + + + + + + + + + + + + + + + +    + + +     + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + +      + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +                   + + + + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + + + + +  + + +  + + + + + +   + + + + + +            + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +    + + + + + +  + + +     + + + + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +               + + + + + +    + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + + + + +     + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + + + + +  + + + + + +   + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + +                + + + + + +               + + + + + + + + + + + +   + + +         + + + + + + + + +   + + + + + + + + + + + +  + + +   + + +  + + + + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + +   + + +  + + +   + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + +                  + + +          + + + + + + + + + + + +     + + +     + + +    + + +  + + +  + + +    + + +   + + + + + + + + + + + + + + +       + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + +  + + + + + +      + + +  + + + + + + + + + + + + + + + + + + + + +     + + +       + + +                        + + +         + + + + + + + + +    + + + + + +    + + + + + +         + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +   + + +   + + + + + +  + + + + + + + + + + + +     + + + + + + + + + + + +  + + + + + +   + + +                    + + +        + + + + + + + + + + + + + + + + + +      + + + + + +       + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + + + + +   + + +  + + + + + + + + + + + + + + + + + +        + + + + + + + + +   + + + + + +    + + + + + + + + +                           + + +                 + + + + + +   + + +  + + + + + + + + + + + +        + + + + + + + + +  + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +  + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + +      + + +                   + + + + + +                 + + + + + + + + + + + +        + + +   + + +   + + +    + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + + + + + + + +         + + +                              + + + + + +       + + +         + + +     + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +   + + + + + +  + + + + + + + + + + + + + + + + + +      + + +                           + + + + + +               + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + +             + + + + + + + + +    + + + + + +                                   + + + + + + + + + + + + + + + + + +             + + + + + +    + + + + + +    + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + +    + + +    + + + + + + + + +                       + + + + + + + + + + + + + + + + + + + + +         + + +     + + +  + + + + + +  + + +             + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + +   + + + + + +     + + + + + +                       + + + + + + + + + + + + + + + + + + + + + + + + + + +          + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +      + + + + + + + + +     + + + + + + + + + + + + + + + + + +           + + + + + + + + + + + + + + + + + +                        + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + + + + +  + + + + + +   + + + + + +    + + + + + +   + + + + + + + + + + + + + + +  + + + + + + + + +        + + +   + + +  + + +      + + + + + + + + +                         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + +    + + +  + + +    + + + + + +      + + + + + +      + + +    + + + + + + + + +                             + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +  + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + +     + + +  + + + + + + + + + + + +    + + +             + + + + + +                + + +           + + + + + + + + +         + + + + + + + + + + + + + + + + + + + + +         + + +  + + + + + +  + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + +  + + +   + + + + + + + + + + + +   + + +                + + + + + +                 + + + + + +        + + +             + + + + + + + + + + + + + + + + + + + + +           + + +         + + + + + + + + + + + +   + + + + + +  + + + + + + + + + + + + + + +      + + + + + +    + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +   + + +  + + + + + + + + +    + + +                   + + +                              + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +             + + + + + + + + + + + + + + + + + + + + + + + +   + + +  + + +  + + + + + +     + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +       + + + + + +    + + + + + +    + + + + + + + + +  + + +      + + +         + + +  + + + + + +                     + + + + + +    + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + +            + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + +   + + + + + +     + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + +   + + + + + + + + +    + + + + + +     + + + + + +      + + + + + +  + + +  + + + + + +    + + +     + + + + + + + + +     + + +          + + +  + + +      + + + + + +    + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + + + + + + + + + + +  + + +   + + +   + + + + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + +  + + + + + +    + + + + + + + + +  + + + + + +     + + +        + + + + + +   + + + + + +   + + + + + +  + + +   + + +  + + + + + + + + + + + +  + + + + + + + + +    + + + + + + + + + + + +     + + +         + + +  + + +       + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + +             + + + + + + + + +  + + + + + + + + +  + + + + + +   + + + + + + + + + + + +     + + + + + + + + + + + + + + +   + + +   + + + + + + + + + + + + + + +    + + +           + + +  + + + + + + + + +  + + + + + + + + + + + + + + + + + +                                + + + + + + + + +                   + + + + + + + + + + + +  + + +  + + + + + + + + + + + +       + + + + + +  + + + + + +   + + + + + + + + +  + + + + + + + + + + + + + + +  + + +  + + + + + + + + + + + + + + +     + + + + + + + + + + + + + + +             + + +    + + + + + + + + + + + + + + + + + + + + +  + + + + + +                               + + +           + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + +  + + + + + + + + + + + +  + + + + + +  + + +    + + +    + + + + + + + + + + + + + + +  + + +      + + + + + +         + + +  + + + + + + + + + + + + + + +         + + +     + + + + + +                     + + +    + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + +  + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + +    + + +   + + + + + + + + +  + + +       + + +                  + + + + + + + + + + + +  + + + + + + + + +           + + +   + + + + + +                      + + +       + + + + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +  + + + + + +   + + +   + + + + + + + + +         + + +         + + +        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +                     + + + + + +                       + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +    + + +  + + + + + + + + + + + + + + + + + + + + +    + + +  + + +     + + + + + + + + +  + + +  + + + + + + + + + + + +  + + + + + +       + + +  + + + + + +  + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                     + + +     + + +       + + +          + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +   + + +  + + + + + +      + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +         + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +   + + +  + + + + + +           + + + + + + + + +        + + + + + +                   + + + + + +  + + +  + + + + + +  + + +  + + + + + +   + + +   + + + + + +  + + +      + + + + + +  + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +         + + +    + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + +              + + + + + + + + +            + + +        + + +  + + +  + + + + + + + + +  + + + + + + + + + + + +    + + +     + + +     + + + + + + + + + + + +  + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +           + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +             + + +            + + +          + + +        + + + + + + + + + + + + + + + + + +  + + + + + +  + + +    + + + + + + + + +  + + +   + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + +               + + +    + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + + + + +                      + + + + + + + + + + + +            + + + + + +    + + + + + + + + + + + + + + +  + + + + + + + + +   + + +        + + + + + + + + +     + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +        + + +  + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +                                + + + + + + + + + + + + + + +         + + +   + + + + + +    + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +       + + + + + +   + + + + + + + + +  + + + + + +      + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + +    + + +  + + +  + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                                + + + + + + + + + + + +        + + + + + +   + + + + + +  + + +  + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +  + + +   + + +        + + +      + + +  + + + + + +    + + +  + + +  + + + + + + + + + + + +       + + +    + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + +              + + + + + +      + + +            + + + + + +              + + +     + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + + + + +  + + + + + +   + + +   + + +    + + +      + + + + + + + + +    + + + + + +   + + +     + + + + + +   + + + + + + + + + + + +          + + + + + +  + + + + + + + + + + + + + + +   + + +                             + + +      + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + +     + + + + + +    + + +   + + +             + + +  + + + + + +    + + +    + + + + + +   + + +          + + + + + + + + +                                 + + + + + +  + + +    + + +    + + +   + + + + + + + + + + + +   + + +   + + + + + + + + + + + +   + + +         + + + + + +     + + + + + +      + + +       + + +         + + + + + + + + +  + + + + + +                           + + + + + +      + + +  + + + + + + + + +       + + + + + + + + + + + +       + + + + + +  + + +    + + +    + + +          + + +  + + + + + +      + + + + + +  + + + + + +                                      + + +   + + +  + + +            + + +      + + +   + + + + + +                  + + + + + + + + +         + + + + + + + + +   + + + + + + + + + + + +           + + +                                 + + + + + + + + + + + +  + + +    + + + + + +   + + +    + + + + + + + + + + + +   + + + + + + + + + + + +  + + +                + + +     + + + + + +      + + +  + + + + + +  + + +        + + +   + + + + + +   + + + + + +          + + + + + + + + +             + + +  + + +  + + + + + +    + + +  + + + + + + + + +     + + + + + +  + + + + + +   + + +                   + + + + + + + + +      + + +              + + +     + + + + + +           + + + + + + + + +          + + + + + + + + + + + +  + + + + + +   + + +    + + + + + +   + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +               + + + + + +         + + +       + + + + + +                + + + + + +  + + + + + +      + + +             + + + + + + + + +   + + +     + + +         + + + + + +    + + +  + + + + + + + + + + + + + + + + + + + + +       + + + + + +             + + + + + +            + + + + + +              + + +   + + +     + + + + + +   + + +            + + + + + +  + + +        + + +  + + + + + + + + +   + + + + + + + + + + + +    + + + + + +    + + +  + + + + + + + + + + + + + + + + + + + + +  + + +     + + +                    + + +  + + +  + + +                              + + + + + + + + +          + + +  + + + + + + + + + + + +  + + + + + +     + + +         + + +     + + + + + + + + + + + +     + + + + + + + + + + + +    + + +   + + +                           + + + + + +                   + + + + + + + + +          + + + + + + + + + + + +     + + + + + +  + + + + + + + + + + + +      + + + + + + + + +   + + +  + + +  + + + + + + + + +      + + + + + +      + + + + + + + + +  + + + + + +  + + +                  + + + + + + + + +            + + +                     + + +    + + + + + +  + + +  + + + + + + + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + +   + + + + + +    + + + + + + + + +  + + +  + + +                       + + + + + + + + + + + + + + + + + +    + + +          + + +                         + + + + + + + + +  + + + + + +  + + +     + + + + + + + + +    + + + + + + + + + + + + + + +  + + +  + + + + + + + + +     + + +   + + +  + + + + + +                 + + +  + + + + + +         + + + + + + + + +                  + + +       + + +                 + + + + + + + + +  + + + + + + + + + + + +  + + + + + +   + + +   + + +   + + + + + +    + + + + + +     + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + + + + +  + + + + + + + + + + + +                     + + +         + + + + + + + + + + + +         + + + + + +           + + +                     + + + + + +        + + + + + +  + + + + + +       + + +       + + + + + +   + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + +  + + + + + +                                     + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + +      + + + + + + + + + + + +          + + +                               + + +   + + +  + + + + + + + + + + + +       + + + + + +    + + + + + + + + +  + + +    + + + + + + + + + + + + + + + + + +            + + +               + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + +                                    + + + + + + + + + + + +    + + + + + + + + + + + + + + +        + + +       + + +   + + + + + + + + +   + + + + + +       + + +        + + + + + +                     + + + + + + + + + + + + + + +   + + + + + +    + + + + + + + + + + + + + + + + + +         + + + + + +                                + + + + + +     + + +  + + +   + + +  + + + + + +       + + + + + + + + + + + +   + + +  + + + + + + + + + + + +    + + +  + + + + + +     + + + + + + + + +                    + + + + + + + + +      + + + + + + + + + + + +   + + + + + + + + + + + + + + + + + + + + + + + +           + + +  + + + + + +    + + + + + +  + + +                           + + + + + +     + + +  + + + + + +  + + + + + + + + + + + + + + +  + + + + + +  + + +           + + + + + + + + + + + + + + + + + + + + +   + + + + + + + + +   + + +     + + +     + + +                  + + +  + + +  + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + +         + + +     + + +  + + + + + + + + + + + + + + + + + +                          + + +     + + + + + + + + + + + + + + + + + + + + +  + + +      + + + + + + + + + + + + + + + + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + +    + + +      + + +    + + +                 + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +         + + +    + + + + + + + + + + + + + + +   + + + + + +                             + + + + + + + + + + + + + + +    + + + + + + + + +   + + + + + + + + +   + + +  + + + + + +   + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +                          + + + + + + + + + + + +  + + +   + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +               + + + + + + + + + + + +                               + + +  + + +  + + +   + + +  + + + + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +  + + +        + + + + + +          + + + + + + + + + + + + + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                + + + + + + + + + + + + + + + + + +                   + + +   + + +   + + + + + + + + +  + + +   + + + + + + + + +  + + + + + +        + + + + + +    + + +   + + + + + +   + + + + + +         + + + + + +           + + + + + + + + +       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +               + + + + + + + + + + + + + + +           + + +               + + +   + + +   + + + + + + + + +             + + + + + + + + +    + + +   + + +                            + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                 + + + + + + + + + + + +         + + +                           + + +        + + + + + +                          + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +              + + +               + + +        + + + + + +   + + + + + + + + + + + + + + + + + +  + + + + + + + + + + + +        + + + + + +                            + + + + + + + + + + + + + + + + + + + + + + + +   + + +             + + + + + +            + + +     + + +       + + +       + + + + + + + + + + + + + + + + + +   + + + + + +    + + + + + +  + + +    + + + + + +                   + + + + + +        + + + + + + + + + + + + + + + + + +  + + +                   + + +             + + +        + + + + + + + + + + + + + + + + + + + + +    + + + + + + + + +  + + + + + +          + + + + + + + + + + + +       + + +              + + +        + + + + + + + + + + + +                    + + +  + + +        + + + + + +     + + +         + + + + + + + + +     + + + + + + + + +    + + +      + + +       + + + + + +      + + +             + + + + + +   + + +            + + +        + + + + + +           + + + + + +              + + + + + + + + +  + + +        + + +               + + + + + +           + + +   + + +        + + +      + + + + + + + + +         + + + + + +                  + + +  + + +    + + +  + + + + + + + + +         + + +                         + + +       + + +      + + +          + + +      + + + + + +   + + +             + + +             + + + + + +   + + +  + + + + + + + + + + + + + + + + + +  + + +   + + + + + + + + +  + + +        + + + + + +  + + + + + +                  + + +         + + + + + + + + + + + +             + + +        + + +             + + + + + + + + +           + + + + + + + + +  + + + + + + + + +   + + + + + + + + + + + +    + + + + + +        + + + + + + + + +  + + +  + + + + + + + + +                                         + + +        + + +              + + +               + + + + + + + + +  + + + + + +  + + +  + + + + + + + + + + + + + + +      + + + + + +        + + + + + +   + + + + + +                   + + + + + +    + + + + + + + + +                         + + +            + + + + + +         + + +         + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + +       + + +  + + + + + +  + + +                 + + + + + +       + + +     + + +                       + + +             + + + + + + + + + + + +            + + + + + +   + + +   + + + + + + + + +  + + + + + + + + +  + + + + + + + + + + + + + + + + + +      + + + + + + + + + + + + + + + + + + + + + + + +       + + + + + +                 + + +    + + + + + + + + + + + + + + + + + + + + + + + +   + + +                             + + +  + + + + + + + + +             + + +  + + + + + + + + + + + +  + + +       + + +    + + + + + + + + + + + + + + + + + + + + + + + +     + + +   + + + + + + + + + + + +             + + + + + + + + + + + + + + + + + + + + + + + +  + + + + + +    + + +          + + + + + +    + + + + + +               + + + + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + +     + + +        + + +   + + +       + + +    + + +   + + + + + + + + + + + + + + + + + + + + +      + + +   + + + + + + + + + + + +               + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +     + + +  + + + + + +         + + +    + + + + + +           + + + + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +           + + + + + + + + + + + +     + + +     + + + + + +    + + + + + + + + + + + + + + +       + + + + + + + + + + + + + + + + + +             + + +  + + + + + + + + + + + + + + + + + + + + + + + +     + + +   + + +              + + + + + + + + +   + + + + + + + + +     + + + + + +          + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + +     + + + + + + + + + + + + + + + + + + + + +   + + +     + + + + + + + + + + + +     + + +    + + +     + + + + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + + + + +                  + + + + + +  + + + + + + + + + + + + + + + + + + + + +   + + + + + +            + + +  + + + + + +            + + +  + + +      + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + +       + + + + + + + + + + + + + + +   + + + + + +  + + + + + +      + + + + + + + + + + + +  + + + + + + + + + + + + + + + + + + + + + + + + + + +  + + +       + + +         + + +  + + + + + + + + + + + + + + + + + + + + +                   + + + + + + + + + + + + + + + + + +        + + +            + + +                + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +           + + + + + +  + + + + + + + + + + + + + + +   + + + + + + + + + + + +      + + + + + +  + + + + + + + + + + + + + + + + + +     + + +           + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +                  + + + + + + + + +  + + + + + + + + + + + + + + + + + +        + + +           + + +              + + + + + + + + + + + + + + + + + + + + + + + +     + + +     + + +   + + + + + +  + + + + + +      + + + + + + + + + + + + + + + + + +       + + +           + + + + + +  + + + + + + + + + + + + + + + + + + + + +                      + + +     + + + + + + + + + + + + + + + + + +         + + +            + + + + + +    + + +  + + + + + +     + + + + + + + + + + + + + + + + + + + + + + + +           + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +    + + +    + + + + + + + + +       + + +           + + + + + +               + + +  + + + + + + + + + + + + + + + + + + + + +                     + + + + + +   + + + + + + + + +        + + +   + + + + + + + + + + + + + + + + + +        + + +    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +   + + + + + +  + + + + + + + + +  + + + + + +  \ No newline at end of file Added: branches/release-0.94/tests/stress/apps/data/t2 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t2 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/data/t3 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t3 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/data/t4 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t4 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/data/t5 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t5 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/data/t6 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t6 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/data/t7 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t7 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/data/t8 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t8 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/data/t9 =================================================================== (Binary files differ) Property changes on: branches/release-0.94/tests/stress/apps/data/t9 ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Added: branches/release-0.94/tests/stress/apps/get.sh =================================================================== --- branches/release-0.94/tests/stress/apps/get.sh (rev 0) +++ branches/release-0.94/tests/stress/apps/get.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,4 @@ +#!/bin/bash + + + Property changes on: branches/release-0.94/tests/stress/apps/get.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_beagle/clean =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/clean (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/clean 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,3 @@ +#! /bin/sh + +rm -rf *~ *log *.d ../*.{kml,swiftx} *.out landuse Property changes on: branches/release-0.94/tests/stress/apps/modis_beagle/clean ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_beagle/getlanduse.pl =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/getlanduse.pl (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/getlanduse.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: branches/release-0.94/tests/stress/apps/modis_beagle/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_beagle/makeinput =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/makeinput (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/makeinput 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,22 @@ +#! /bin/sh + +nfiles=${1:-10} + +rm -rf input +mkdir input +cp ../data/t? input/ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) Property changes on: branches/release-0.94/tests/stress/apps/modis_beagle/makeinput ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_beagle/modis.args =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/modis.args (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/modis.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-modisdir=input/ \ No newline at end of file Added: branches/release-0.94/tests/stress/apps/modis_beagle/modis.kml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/modis.kml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/modis.kml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,200 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + perl + + + + + + + ps + + + input + + + + + output + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MODISdir + + + + + + + + + + + + + + + + + geos + + swift#mapper#17008 + + + + + + + + + + + + nFiles + + + swift#string#17000swift#string#17001 + + + + + + + + MODISdir + + swift#string#17002swift#string#17003 + + + + + + + swift#mapper#17004 + swift#string#17005 + + + + + + swift#mapper#17006 + swift#string#17007 + + + + + + swift#mapper#17008 + + swift#string#17009 + + + + + + + + geos + + + + + + + + + + + + + land + i + + g + getlanduse_pl + + + + + + + + + + + + + + + + + Added: branches/release-0.94/tests/stress/apps/modis_beagle/modis.setup.sh =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/modis.setup.sh (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/modis.setup.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,47 @@ +#!/bin/bash + +case $STRESS in + "S1") + FILES=10 + ;; + "S2") + FILES=1000 + ;; + "S3") + FILES=10000 + ;; + "S4") + FILES=10000 + ;; + *) + FILES=1000 + ;; +esac + +export GLOBUS_HOSTNAME="128.135.112.73" +export GLOBUS_TCP_PORT_RANGE=50000,51000 +#OVERRIDE_GLOBUS_HOSTNAME "128.135.112.73" + +nfiles=${FILES:-10} + +rm -rf input +mkdir input +cp $(dirname $GROUP)/data/t? input/ +cp $GROUP/getlanduse.pl ./ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) + +#makeinput \ No newline at end of file Property changes on: branches/release-0.94/tests/stress/apps/modis_beagle/modis.setup.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_beagle/modis.swift =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/modis.swift (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/modis.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,25 @@ +type imagefile; +type landuse; +type perlscript; + +perlscript getlanduse_pl <"getlanduse.pl">; + +app (landuse output) getLandUse (imagefile input, perlscript ps) +{ + perl @ps @filename(input) stdout=@filename(output); +} + +# Constants and command line arguments +int nFiles = @toInt(@arg("nfiles", "1000")); +string MODISdir = @arg("modisdir", "../data/modis/2002"); + +# Input Dataset +imagefile geos[] ; + +# Compute the land use summary of each MODIS tile +landuse land[] ; + +foreach g,i in geos { + land[i] = getLandUse(g, getlanduse_pl); +} Added: branches/release-0.94/tests/stress/apps/modis_beagle/modis.timeout =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/modis.timeout (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/modis.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +999999 Added: branches/release-0.94/tests/stress/apps/modis_beagle/run =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/run (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/run 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,7 @@ +#! /bin/sh + +set -x +export GLOBUS_HOSTNAME="128.135.112.73" +export GLOBUS_TCP_PORT_RANGE=50000,51000 + +swift -config swift.properties -sites.file sites.template.xml -tc.file tc.template.data modis.swift -modisdir=input/ Property changes on: branches/release-0.94/tests/stress/apps/modis_beagle/run ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_beagle/sites.template.xml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/sites.template.xml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/sites.template.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,35 @@ + + + + + + + 24 + + 100 + + 100 + + pbs.aprun;pbs.mpp;depth=24 + + 3600 + + 00:05:00 + + /lustre/beagle/$USER/swiftwork + + 20 + + 1 + + 1 + + 4.80 + + 10000 + + /tmp/$USER/swiftwork + + + + \ No newline at end of file Added: branches/release-0.94/tests/stress/apps/modis_beagle/sites.template.xml.bak =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/sites.template.xml.bak (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/sites.template.xml.bak 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,19 @@ + + + + 24 + 100 + 100 + + pbs.aprun;pbs.mpp;depth=24;pbs.resource_list=advres=wilde.1768 + 3600 + 00:05:00 + /lustre/beagle/{env.USER}/swiftwork + 20 + 1 + 1 + 4.80 + 10000 + /tmp/{env.USER}/swiftwork + + Added: branches/release-0.94/tests/stress/apps/modis_beagle/swift.properties =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/swift.properties (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/swift.properties 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: branches/release-0.94/tests/stress/apps/modis_beagle/tc.template.data =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/tc.template.data (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/tc.template.data 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +beagle perl /usr/bin/perl null null null Added: branches/release-0.94/tests/stress/apps/modis_beagle/title.txt =================================================================== --- branches/release-0.94/tests/stress/apps/modis_beagle/title.txt (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_beagle/title.txt 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +Modis-Beagle Added: branches/release-0.94/tests/stress/apps/modis_local/clean =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/clean (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/clean 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,3 @@ +#! /bin/sh + +rm -rf *~ *log *.d ../*.{kml,swiftx} *.out landuse Property changes on: branches/release-0.94/tests/stress/apps/modis_local/clean ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_local/getlanduse.pl =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/getlanduse.pl (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/getlanduse.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: branches/release-0.94/tests/stress/apps/modis_local/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_local/makeinput =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/makeinput (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/makeinput 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,22 @@ +#! /bin/sh + +nfiles=${1:-10} + +rm -rf input +mkdir input +cp ../data/t? input/ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) Property changes on: branches/release-0.94/tests/stress/apps/modis_local/makeinput ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_local/modis.args =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/modis.args (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/modis.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-modisdir=input/ \ No newline at end of file Added: branches/release-0.94/tests/stress/apps/modis_local/modis.setup.sh =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/modis.setup.sh (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/modis.setup.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,43 @@ +#!/bin/bash + +case $STRESS in + "S1") + FILES=100 + ;; + "S2") + FILES=1000 + ;; + "S3") + FILES=10000 + ;; + "S4") + FILES=10000 + ;; + *) + FILES=1000 + ;; +esac + +nfiles=${FILES:-10} + +rm -rf input +mkdir input +cp $(dirname $GROUP)/data/t? input/ +cp $GROUP/getlanduse.pl ./ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) + +#makeinput \ No newline at end of file Property changes on: branches/release-0.94/tests/stress/apps/modis_local/modis.setup.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_local/modis.swift =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/modis.swift (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/modis.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,25 @@ +type imagefile; +type landuse; +type perlscript; + +perlscript getlanduse_pl <"getlanduse.pl">; + +app (landuse output) getLandUse (imagefile input, perlscript ps) +{ + perl @ps @filename(input) stdout=@filename(output); +} + +# Constants and command line arguments +int nFiles = @toInt(@arg("nfiles", "1000")); +string MODISdir = @arg("modisdir", "../data/modis/2002"); + +# Input Dataset +imagefile geos[] ; + +# Compute the land use summary of each MODIS tile +landuse land[] ; + +foreach g,i in geos { + land[i] = getLandUse(g, getlanduse_pl); +} Added: branches/release-0.94/tests/stress/apps/modis_local/modis.timeout =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/modis.timeout (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/modis.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +999999 Added: branches/release-0.94/tests/stress/apps/modis_local/run =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/run (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/run 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,5 @@ +#! /bin/sh + +set -x + +swift -config cf -sites.file local.xml -tc.file tc ../modis.swift -modisdir=input/ Property changes on: branches/release-0.94/tests/stress/apps/modis_local/run ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_local/sites.template.xml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/sites.template.xml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/sites.template.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,16 @@ + + + + 4 + 00:05:00 + 100 + 100 + 1 + 1 + 1 + 0.03 + 10000 + + /scratch/midway/{env.USER} + + Added: branches/release-0.94/tests/stress/apps/modis_local/swift.properties =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/swift.properties (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/swift.properties 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=false +provider.staging.pin.swiftfiles=false Added: branches/release-0.94/tests/stress/apps/modis_local/tc.template.data =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/tc.template.data (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/tc.template.data 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +local perl /usr/bin/perl null null null Added: branches/release-0.94/tests/stress/apps/modis_local/title.txt =================================================================== --- branches/release-0.94/tests/stress/apps/modis_local/title.txt (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_local/title.txt 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +Modis Local Added: branches/release-0.94/tests/stress/apps/modis_midway/clean =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/clean (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/clean 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,3 @@ +#! /bin/sh + +rm -rf *~ *log *.d ../*.{kml,swiftx} *.out landuse Property changes on: branches/release-0.94/tests/stress/apps/modis_midway/clean ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_midway/getlanduse.pl =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/getlanduse.pl (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/getlanduse.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: branches/release-0.94/tests/stress/apps/modis_midway/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_midway/makeinput =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/makeinput (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/makeinput 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,22 @@ +#! /bin/sh + +nfiles=${1:-10} + +rm -rf input +mkdir input +cp ../data/t? input/ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) Property changes on: branches/release-0.94/tests/stress/apps/modis_midway/makeinput ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_midway/modis.args =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/modis.args (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/modis.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-modisdir=input/ \ No newline at end of file Added: branches/release-0.94/tests/stress/apps/modis_midway/modis.setup.sh =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/modis.setup.sh (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/modis.setup.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,43 @@ +#!/bin/bash + +case $STRESS in + "S1") + FILES=10 + ;; + "S2") + FILES=1000 + ;; + "S3") + FILES=10000 + ;; + "S4") + FILES=10000 + ;; + *) + FILES=1000 + ;; +esac + +nfiles=${FILES:-10} + +rm -rf input +mkdir input +cp $(dirname $GROUP)/data/t? input/ +cp $GROUP/getlanduse.pl ./ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) + +#makeinput \ No newline at end of file Property changes on: branches/release-0.94/tests/stress/apps/modis_midway/modis.setup.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_midway/modis.swift =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/modis.swift (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/modis.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,25 @@ +type imagefile; +type landuse; +type perlscript; + +perlscript getlanduse_pl <"getlanduse.pl">; + +app (landuse output) getLandUse (imagefile input, perlscript ps) +{ + perl @ps @filename(input) stdout=@filename(output); +} + +# Constants and command line arguments +int nFiles = @toInt(@arg("nfiles", "1000")); +string MODISdir = @arg("modisdir", "../data/modis/2002"); + +# Input Dataset +imagefile geos[] ; + +# Compute the land use summary of each MODIS tile +landuse land[] ; + +foreach g,i in geos { + land[i] = getLandUse(g, getlanduse_pl); +} Added: branches/release-0.94/tests/stress/apps/modis_midway/modis.timeout =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/modis.timeout (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/modis.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +999999 Added: branches/release-0.94/tests/stress/apps/modis_midway/run =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/run (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/run 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,5 @@ +#! /bin/sh + +set -x + +swift -config cf -sites.file local.xml -tc.file tc ../modis.swift -modisdir=input/ Property changes on: branches/release-0.94/tests/stress/apps/modis_midway/run ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_midway/sandyb.xml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/sandyb.xml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/sandyb.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,17 @@ + + + + sandyb + 16 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .64 + 10000 + /tmp/{env.USER} + + Added: branches/release-0.94/tests/stress/apps/modis_midway/sites.template.xml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/sites.template.xml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/sites.template.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,17 @@ + + + + sandyb + 16 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .64 + 10000 + /tmp/{env.USER} + + Added: branches/release-0.94/tests/stress/apps/modis_midway/swift.properties =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/swift.properties (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/swift.properties 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,9 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=true + Added: branches/release-0.94/tests/stress/apps/modis_midway/tc.template.data =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/tc.template.data (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/tc.template.data 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +midway perl /usr/bin/perl null null null Added: branches/release-0.94/tests/stress/apps/modis_midway/title.txt =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/title.txt (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/title.txt 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +Modis Midway Added: branches/release-0.94/tests/stress/apps/modis_midway/westmere.xml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_midway/westmere.xml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_midway/westmere.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,19 @@ + + + + + westmere + 12 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .48 + 10000 + /tmp/{env.USER} + + + Added: branches/release-0.94/tests/stress/apps/modis_multiple/clean =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/clean (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/clean 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,3 @@ +#! /bin/sh + +rm -rf *~ *log *.d ../*.{kml,swiftx} *.out landuse Property changes on: branches/release-0.94/tests/stress/apps/modis_multiple/clean ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_multiple/getlanduse.pl =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/getlanduse.pl (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/getlanduse.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: branches/release-0.94/tests/stress/apps/modis_multiple/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_multiple/makeinput =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/makeinput (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/makeinput 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,22 @@ +#! /bin/sh + +nfiles=${1:-10} + +rm -rf input +mkdir input +cp ../data/t? input/ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) Property changes on: branches/release-0.94/tests/stress/apps/modis_multiple/makeinput ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_multiple/modis.args =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/modis.args (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/modis.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-modisdir=input/ \ No newline at end of file Added: branches/release-0.94/tests/stress/apps/modis_multiple/modis.setup.sh =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/modis.setup.sh (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/modis.setup.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,43 @@ +#!/bin/bash + +case $STRESS in + "S1") + FILES=10 + ;; + "S2") + FILES=1000 + ;; + "S3") + FILES=10000 + ;; + "S4") + FILES=10000 + ;; + *) + FILES=1000 + ;; +esac + +nfiles=${FILES:-10} +#OVERRIDE_GLOBUS_HOSTNAME "swift.rcc.uchicago.edu" +rm -rf input +mkdir input +cp $(dirname $GROUP)/data/t? input/ +cp $GROUP/getlanduse.pl ./ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) + +#makeinput \ No newline at end of file Property changes on: branches/release-0.94/tests/stress/apps/modis_multiple/modis.setup.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_multiple/modis.swift =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/modis.swift (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/modis.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,25 @@ +type imagefile; +type landuse; +type perlscript; + +perlscript getlanduse_pl <"getlanduse.pl">; + +app (landuse output) getLandUse (imagefile input, perlscript ps) +{ + perl @ps @filename(input) stdout=@filename(output); +} + +# Constants and command line arguments +int nFiles = @toInt(@arg("nfiles", "1000")); +string MODISdir = @arg("modisdir", "../data/modis/2002"); + +# Input Dataset +imagefile geos[] ; + +# Compute the land use summary of each MODIS tile +landuse land[] ; + +foreach g,i in geos { + land[i] = getLandUse(g, getlanduse_pl); +} Added: branches/release-0.94/tests/stress/apps/modis_multiple/modis.timeout =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/modis.timeout (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/modis.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +999999 Added: branches/release-0.94/tests/stress/apps/modis_multiple/run =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/run (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/run 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,5 @@ +#! /bin/sh + +set -x + +swift -config cf -sites.file local.xml -tc.file tc ../modis.swift -modisdir=input/ Property changes on: branches/release-0.94/tests/stress/apps/modis_multiple/run ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_multiple/sites.template.xml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/sites.template.xml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/sites.template.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,71 @@ + + + + + 10.00 + 10000 + 1 + 3600 + 00:05:00 + 100 + 100 + 1000 + 1 + 1 + "group_friends.{env.USER}" + nonshared + + . + + + + + 24 + 100 + 100 + + pbs.aprun;pbs.mpp;depth=24;pbs.resource_list=advres=wilde.1768 + 3600 + 00:05:00 + /lustre/beagle/{env.USER}/swiftwork + 5 + 1 + 1 + 4.80 + 10000 + /tmp/{env.USER}/swiftwork + + + + + sandyb + 16 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .64 + 10000 + /tmp/{env.USER} + + + + + westmere + 12 + 00:05:00 + 3600 + 100 + 100 + 4 + 1 + 1 + .48 + 10000 + /tmp/{env.USER} + + + Added: branches/release-0.94/tests/stress/apps/modis_multiple/swift.properties =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/swift.properties (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/swift.properties 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: branches/release-0.94/tests/stress/apps/modis_multiple/tc.template.data =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/tc.template.data (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/tc.template.data 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,4 @@ +uc3 perl /usr/bin/perl null null null +beagle perl /usr/bin/perl null null null +#sandy perl /usr/bin/perl null null null +westmere perl /usr/bin/perl null null null Added: branches/release-0.94/tests/stress/apps/modis_multiple/title.txt =================================================================== --- branches/release-0.94/tests/stress/apps/modis_multiple/title.txt (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_multiple/title.txt 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +Modis Multiple Sites Added: branches/release-0.94/tests/stress/apps/modis_uc3/clean =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/clean (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/clean 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,3 @@ +#! /bin/sh + +rm -rf *~ *log *.d ../*.{kml,swiftx} *.out landuse Property changes on: branches/release-0.94/tests/stress/apps/modis_uc3/clean ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_uc3/getlanduse.pl =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/getlanduse.pl (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/getlanduse.pl 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,29 @@ +#!/usr/bin/perl -w + +# Input to this program should be a raw, greyscale RGB file +# Usage: rgb_histogram.pl myfile.rgb + +my $image_filename = shift; +open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n"; +binmode IMAGEFILE; + +my @pixelcount; +foreach my $count (0..255) { $pixelcount[$count] = 0; } + +# Read values, three bytes at a time +$/ = \3; +foreach() { + $pixelcount[unpack('C', $_)]++; +} +close(IMAGEFILE); + + at sortedcount = sort {$b <=> $a} @pixelcount; + +foreach my $count (0..255) { + if($sortedcount[$count] == 0) { next; } + printf("%d %d %02x\n", $sortedcount[$count], $count, $count); +} + +system("hostname -f 2>&1"); + +sleep(1); Property changes on: branches/release-0.94/tests/stress/apps/modis_uc3/getlanduse.pl ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_uc3/makeinput =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/makeinput (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/makeinput 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,22 @@ +#! /bin/sh + +nfiles=${1:-10} + +rm -rf input +mkdir input +cp ../data/t? input/ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) Property changes on: branches/release-0.94/tests/stress/apps/modis_uc3/makeinput ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_uc3/modis.args =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/modis.args (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/modis.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-modisdir=input/ \ No newline at end of file Added: branches/release-0.94/tests/stress/apps/modis_uc3/modis.kml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/modis.kml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/modis.kml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,200 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + perl + + + + + + + ps + + + input + + + + + output + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MODISdir + + + + + + + + + + + + + + + + + geos + + swift#mapper#17008 + + + + + + + + + + + + nFiles + + + swift#string#17000swift#string#17001 + + + + + + + + MODISdir + + swift#string#17002swift#string#17003 + + + + + + + swift#mapper#17004 + swift#string#17005 + + + + + + swift#mapper#17006 + swift#string#17007 + + + + + + swift#mapper#17008 + + swift#string#17009 + + + + + + + + geos + + + + + + + + + + + + + land + i + + g + getlanduse_pl + + + + + + + + + + + + + + + + + Added: branches/release-0.94/tests/stress/apps/modis_uc3/modis.setup.sh =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/modis.setup.sh (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/modis.setup.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,43 @@ +#!/bin/bash + +case $STRESS in + "S1") + FILES=10 + ;; + "S2") + FILES=1000 + ;; + "S3") + FILES=10000 + ;; + "S4") + FILES=10000 + ;; + *) + FILES=1000 + ;; +esac + +nfiles=${FILES:-10} +#OVERRIDE_GLOBUS_HOSTNAME "swift.rcc.uchicago.edu" +rm -rf input +mkdir input +cp $(dirname $GROUP)/data/t? input/ +cp $GROUP/getlanduse.pl ./ +( cd input + n=0 + for h in $(seq -w 00 99); do + for v in $(seq -w 00 99); do + n=$((n+1)) + if [ $n -gt $nfiles ]; then + break; + else + f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/') + ln $f h${h}v${v}.rgb + fi + done + done + rm t? +) + +#makeinput \ No newline at end of file Property changes on: branches/release-0.94/tests/stress/apps/modis_uc3/modis.setup.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_uc3/modis.swift =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/modis.swift (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/modis.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,25 @@ +type imagefile; +type landuse; +type perlscript; + +perlscript getlanduse_pl <"getlanduse.pl">; + +app (landuse output) getLandUse (imagefile input, perlscript ps) +{ + perl @ps @filename(input) stdout=@filename(output); +} + +# Constants and command line arguments +int nFiles = @toInt(@arg("nfiles", "1000")); +string MODISdir = @arg("modisdir", "../data/modis/2002"); + +# Input Dataset +imagefile geos[] ; + +# Compute the land use summary of each MODIS tile +landuse land[] ; + +foreach g,i in geos { + land[i] = getLandUse(g, getlanduse_pl); +} Added: branches/release-0.94/tests/stress/apps/modis_uc3/modis.timeout =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/modis.timeout (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/modis.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +999999 Added: branches/release-0.94/tests/stress/apps/modis_uc3/run =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/run (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/run 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,5 @@ +#! /bin/sh + +set -x + +swift -config swift.properties -sites.file sites.template.xml -tc.file tc.template.data modis.swift -modisdir=input/ Property changes on: branches/release-0.94/tests/stress/apps/modis_uc3/run ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/apps/modis_uc3/sites.template.xml =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/sites.template.xml (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/sites.template.xml 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,19 @@ + + + + 10.00 + 10000 + 1 + 3600 + 00:05:00 + 100 + 100 + 1000 + 1 + 1 + "group_friends.{env.USER}" + nonshared + . + + + Added: branches/release-0.94/tests/stress/apps/modis_uc3/swift.properties =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/swift.properties (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/swift.properties 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,8 @@ + +wrapperlog.always.transfer=true +sitedir.keep=true +execution.retries=0 +lazy.errors=false +status.mode=provider +use.provider.staging=true +provider.staging.pin.swiftfiles=false Added: branches/release-0.94/tests/stress/apps/modis_uc3/tc.template.data =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/tc.template.data (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/tc.template.data 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +uc3 perl /usr/bin/perl null null null Added: branches/release-0.94/tests/stress/apps/modis_uc3/title.txt =================================================================== --- branches/release-0.94/tests/stress/apps/modis_uc3/title.txt (rev 0) +++ branches/release-0.94/tests/stress/apps/modis_uc3/title.txt 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +Modis UC3 Added: branches/release-0.94/tests/stress/internals/.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,7 @@ +#NIGHTLY 1000 10000 +#WEEKLY 10000 100000 1000000 +define(S3, 100000) +define(S2, 10000) +define(S1, 1000) +-loops=$(esyscmd(`printf $STRESS') ) + Added: branches/release-0.94/tests/stress/internals/m4_template_gen.sh =================================================================== --- branches/release-0.94/tests/stress/internals/m4_template_gen.sh (rev 0) +++ branches/release-0.94/tests/stress/internals/m4_template_gen.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,13 @@ +#!/bin/bash + +for i in $(ls *swift) +do + NAME=${i%.swift}; + ARGS="$NAME.args.m4" + if [ -f $ARGS ]; then + rm $ARGS; + fi; + grep "#NIGHTLY" $i >> $ARGS + grep "#WEEKLY" $i >> $ARGS + cat test.args.m4 >> $ARGS +done \ No newline at end of file Property changes on: branches/release-0.94/tests/stress/internals/m4_template_gen.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/internals/scripts.sh =================================================================== --- branches/release-0.94/tests/stress/internals/scripts.sh (rev 0) +++ branches/release-0.94/tests/stress/internals/scripts.sh 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,12 @@ +#!/bin/bash + +SWIFTSCRIPT=x_func.swift; + +SWIFT_ITERATIONS=`grep "#NIGHTLY" $SWIFTSCRIPT`; +if [ "$?" == "0" ]; then + echo "SWIFT_ITERATIONS : $SWIFT_ITERATIONS"; + SWIFT_ITERATIONS=($SWIFT_ITERATIONS); + echo "${SWIFT_ITERATIONS[1]}"; + echo "${SWIFT_ITERATIONS[2]}"; + +fi; Property changes on: branches/release-0.94/tests/stress/internals/scripts.sh ___________________________________________________________________ Added: svn:executable + * Added: branches/release-0.94/tests/stress/internals/test.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/test.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/test.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,6 @@ +define(S4, 100000)dnl +define(S3, 100000)dnl +define(S2, 10000)dnl +define(S1, 1000)dnl +-loops=esyscmd(`printf $STRESS') dnl + Added: branches/release-0.94/tests/stress/internals/title.txt =================================================================== --- branches/release-0.94/tests/stress/internals/title.txt (rev 0) +++ branches/release-0.94/tests/stress/internals/title.txt 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +Stress-Swift Internals Added: branches/release-0.94/tests/stress/internals/x_append_iter.args =================================================================== --- branches/release-0.94/tests/stress/internals/x_append_iter.args (rev 0) +++ branches/release-0.94/tests/stress/internals/x_append_iter.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-loops=100 Added: branches/release-0.94/tests/stress/internals/x_append_iter.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/x_append_iter.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/x_append_iter.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,6 @@ +define(S4, 10000)dnl +define(S3, 10000)dnl +define(S2, 1000)dnl +define(S1, 100)dnl +-loops=esyscmd(`printf $STRESS') dnl + Added: branches/release-0.94/tests/stress/internals/x_append_iter.swift =================================================================== --- branches/release-0.94/tests/stress/internals/x_append_iter.swift (rev 0) +++ branches/release-0.94/tests/stress/internals/x_append_iter.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,27 @@ +/* + Iterate loops with variable number of loops + to be called with cmdline arg -loops= + + todo : Fails to compile on 0.94 ? check + Compiles on 0.93 (Regression noted) + perhaps this isn't a very scalable test :( + stats: 1K -> 2.458s real + 10K -> Java heap space ? exec failed + 100K -> ? + 1M -> java.langOutOfMemoryError thrown + 10M -> ? +#NIGHTLY 1000 +#WEEKLY 1000 10000 +*/ + +int limit = @toint(@arg("loops")); +string result[]; +result[0] = "banana"; + +iterate current { + result[current+1] = @strcat(result[current], "banana"); + // tracef("result[current] = %s \n", result[current]); +} until ( current > limit ); + +tracef("Result[0] = %s \n", result[0]); +tracef("Result[%i] = %s \n", limit, result[limit]); \ No newline at end of file Added: branches/release-0.94/tests/stress/internals/x_foreach.args =================================================================== --- branches/release-0.94/tests/stress/internals/x_foreach.args (rev 0) +++ branches/release-0.94/tests/stress/internals/x_foreach.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-loops=1000 Added: branches/release-0.94/tests/stress/internals/x_foreach.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/x_foreach.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/x_foreach.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,6 @@ +define(S4, 1000000)dnl +define(S3, 100000)dnl +define(S2, 10000)dnl +define(S1, 1000)dnl +-loops=esyscmd(`printf $STRESS') dnl + Added: branches/release-0.94/tests/stress/internals/x_foreach.swift =================================================================== --- branches/release-0.94/tests/stress/internals/x_foreach.swift (rev 0) +++ branches/release-0.94/tests/stress/internals/x_foreach.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,17 @@ +/* + Foreach loops with variable number of loops + to be called with cmdline arg -loops= + +#NIGHTLY 10000 100000 +#WEEKLY 1000 10000 1000000 +*/ + +int limit = @toint(@arg("loops")); +int result[]; + +foreach num, index in [1:limit:1] { + result[index] = index; +} + +tracef("Result[0] = %i \n", result[0]); +tracef("Result[%i] = %i \n", limit, result[limit]); \ No newline at end of file Added: branches/release-0.94/tests/stress/internals/x_func.args =================================================================== --- branches/release-0.94/tests/stress/internals/x_func.args (rev 0) +++ branches/release-0.94/tests/stress/internals/x_func.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-loops=1000 Added: branches/release-0.94/tests/stress/internals/x_func.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/x_func.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/x_func.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,6 @@ +define(S4, 1000000)dnl +define(S3, 100000)dnl +define(S2, 10000)dnl +define(S1, 1000)dnl +-loops=esyscmd(`printf $STRESS') dnl + Added: branches/release-0.94/tests/stress/internals/x_func.swift =================================================================== --- branches/release-0.94/tests/stress/internals/x_func.swift (rev 0) +++ branches/release-0.94/tests/stress/internals/x_func.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,36 @@ +/* + Test reliability of function calls over large scale + + tested on : 0.94 + using ints + + stats: 1K -> 3.059s + 10K -> 5.047s (result is probably wrong) + 100K -> 31.85s + 1M -> Exception in thread "Hang checker" + java.lang.StackOverflowError + at java.util.HashMap.put(HashMap.java:484) + at java.util.HashSet.add(HashSet.java:217) + +#NIGHTLY 1000 10000 100000 +#WEEKLY 1000 10000 100000 1000000 + */ + +int limit = @toInt(@arg("loops")); +int range[] = [2:limit:1]; +int array[]; + +(int out) sum (int a, int b){ + out = a + b ; +} + +array[0] = 0; +array[1] = 1; + +foreach num in range { + array[num] = sum(array[num-1], array[num-2]); +} + +tracef("Fibonacci[2] = %i \n", array[2]); +tracef("Fibinacci[%i] = %i \n", limit, array[limit]); + Added: branches/release-0.94/tests/stress/internals/x_func.timeout =================================================================== --- branches/release-0.94/tests/stress/internals/x_func.timeout (rev 0) +++ branches/release-0.94/tests/stress/internals/x_func.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +99999 \ No newline at end of file Added: branches/release-0.94/tests/stress/internals/x_func_float.args =================================================================== --- branches/release-0.94/tests/stress/internals/x_func_float.args (rev 0) +++ branches/release-0.94/tests/stress/internals/x_func_float.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-loops=1000 Added: branches/release-0.94/tests/stress/internals/x_func_float.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/x_func_float.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/x_func_float.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,6 @@ +define(S4, 1000000)dnl +define(S3, 100000)dnl +define(S2, 10000)dnl +define(S1, 1000)dnl +-loops=esyscmd(`printf $STRESS') dnl + Added: branches/release-0.94/tests/stress/internals/x_func_float.swift =================================================================== --- branches/release-0.94/tests/stress/internals/x_func_float.swift (rev 0) +++ branches/release-0.94/tests/stress/internals/x_func_float.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,36 @@ +/* + Test reliability of function calls over large scale + + tested on : 0.94 + todo : Redo with floats as numbers either become too big + Floats give infinity for fib[10K] + stats: 10K -> ? + 100K -> + 1M -> Exception in thread "Hang checker" + java.lang.StackOverflowError + at java.util.HashMap.put(HashMap.java:484) + at java.util.HashSet.add(HashSet.java:217) + +#NIGHTLY 1000 10000 100000 +#WEEKLY 1000 10000 100000 1000000 + + */ + +int limit = @toInt(@arg("loops")); +int range[] = [2:limit:1]; +float array[]; + +(float out) sum (float a, float b){ + out = a + b ; +} + +array[0] = 0.0; +array[1] = 1.0; + +foreach num in range { + array[num] = sum(array[num-1], array[num-2]); +} + +tracef("Fibonacci[2] = %f \n", array[2]); +tracef("Fibinacci[%i] = %f \n", limit, array[limit]); + Added: branches/release-0.94/tests/stress/internals/x_func_float.timeout =================================================================== --- branches/release-0.94/tests/stress/internals/x_func_float.timeout (rev 0) +++ branches/release-0.94/tests/stress/internals/x_func_float.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +999999 \ No newline at end of file Added: branches/release-0.94/tests/stress/internals/x_iterate.args =================================================================== --- branches/release-0.94/tests/stress/internals/x_iterate.args (rev 0) +++ branches/release-0.94/tests/stress/internals/x_iterate.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-loops=100000 Added: branches/release-0.94/tests/stress/internals/x_iterate.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/x_iterate.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/x_iterate.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,6 @@ +define(S4, 1000000)dnl +define(S3, 1000000)dnl +define(S2, 100000)dnl +define(S1, 10000)dnl +-loops=esyscmd(`printf $STRESS') dnl + Added: branches/release-0.94/tests/stress/internals/x_iterate.swift =================================================================== --- branches/release-0.94/tests/stress/internals/x_iterate.swift (rev 0) +++ branches/release-0.94/tests/stress/internals/x_iterate.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,23 @@ +/* + Iterate loops with variable number of loops + to be called with cmdline arg -loops= + + todo : Fails to compile on 0.94 + Compiles on 0.93 (Regression noted) + stats: 100k -> 17s (core i5) + 1M -> java.langOutOfMemoryError thrown + +#NIGHTLY 1000 10000 100000 +#WEEKLY 1000 10000 100000 1000000 +*/ + +int limit = @toint(@arg("loops")); +int result[]; + +iterate current { + result[current] = current; + // tracef("result[current] = %i \n", result[current]); +} until ( current > limit ); + +tracef("Result[0] = %i \n", result[0]); +tracef("Result[%i] = %i \n", limit, result[limit]); \ No newline at end of file Added: branches/release-0.94/tests/stress/internals/x_recursion.args =================================================================== --- branches/release-0.94/tests/stress/internals/x_recursion.args (rev 0) +++ branches/release-0.94/tests/stress/internals/x_recursion.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-loops=10000 Added: branches/release-0.94/tests/stress/internals/x_recursion.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/x_recursion.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/x_recursion.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,6 @@ +define(S4, 100000)dnl +define(S3, 100000)dnl +define(S2, 10000)dnl +define(S1, 1000)dnl +-loops=esyscmd(`printf $STRESS') dnl + Added: branches/release-0.94/tests/stress/internals/x_recursion.swift =================================================================== --- branches/release-0.94/tests/stress/internals/x_recursion.swift (rev 0) +++ branches/release-0.94/tests/stress/internals/x_recursion.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,45 @@ +/* + Linear tail recursion. + Expect linear growth only, so usual loop numbers match + as for earlier scripts. + + stats: 1K -> 25s real (i5 8gb) + 10K -> (10% ram 8gb in use by java) +Uncaught exception: java.lang.StackOverflowError in vdl:unitstart @ x_recursion.kml, line: 45 +java.lang.StackOverflowError + at java.lang.String.valueOf(String.java:2959) + at org.globus.cog.karajan.util.ThreadingContext.toString(ThreadingContext.java:87) + at org.globus.cog.karajan.util.ThreadingContext.toString(ThreadingContext.java:87) + ... +Exception is: java.lang.StackOverflowError +Near Karajan line: vdl:unitstart @ x_recursion.kml, line: 45 +Another uncaught exception while handling an uncaught exception. +java.lang.StackOverflowError + at org.globus.cog.karajan.workflow.nodes.FlowNode.failImmediately(FlowNode.java:77) + at org.globus.cog.karajan.workflow.nodes.FlowNode.failed(FlowNode.java:245) + ... +The initial exception was +java.lang.StackOverflowError + at java.lang.String.valueOf(String.java:2959) + at org.globus.cog.karajan.util.ThreadingContext.toString(ThreadingContext.java:87) + at org.globus.cog.karajan.util.ThreadingContext.toString(ThreadingContext.java:87) + + 100K -> ? + +#NIGHTLY 1000 10000 +#WEEKLY 1000 10000 100000 +*/ + +int limit = @toint(@arg("loops")); + +(int out) sum (int n){ + if ( n == 0 ){ + out = 0; + }else{ + out = n + sum( n-1 ); + } +} + +int result = sum(limit); + +tracef("Sum(%i) = %i \n", limit, result); \ No newline at end of file Added: branches/release-0.94/tests/stress/internals/x_recursion.timeout =================================================================== --- branches/release-0.94/tests/stress/internals/x_recursion.timeout (rev 0) +++ branches/release-0.94/tests/stress/internals/x_recursion.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +99999 \ No newline at end of file Added: branches/release-0.94/tests/stress/internals/x_sleep.args =================================================================== --- branches/release-0.94/tests/stress/internals/x_sleep.args (rev 0) +++ branches/release-0.94/tests/stress/internals/x_sleep.args 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +-loops=10000 Added: branches/release-0.94/tests/stress/internals/x_sleep.args.m4 =================================================================== --- branches/release-0.94/tests/stress/internals/x_sleep.args.m4 (rev 0) +++ branches/release-0.94/tests/stress/internals/x_sleep.args.m4 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,6 @@ +define(S4, 1000000)dnl +define(S3, 100000)dnl +define(S2, 10000)dnl +define(S1, 1000)dnl +-loops=esyscmd(`printf $STRESS') dnl + Added: branches/release-0.94/tests/stress/internals/x_sleep.swift =================================================================== --- branches/release-0.94/tests/stress/internals/x_sleep.swift (rev 0) +++ branches/release-0.94/tests/stress/internals/x_sleep.swift 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1,24 @@ +/* + Foreach loops with variable number of loops + to be called with cmdline arg -loops= + todo: measure actual times with expected times. + + This is probably better for a larger system. + Could take forever to execute on i5s for even modest scales. + stats: 1K -> real 41m55.779s + +#NIGHTLY 1000 10000 +#WEEKLY 10000 100000 1000000 +*/ + +int limit = @toint(@arg("loops")); +int range[] = [0:limit:1]; +int result[]; + +app sleep (int seconds){ + sleep seconds; +} + +foreach num in range { + sleep(num%%5); +} \ No newline at end of file Added: branches/release-0.94/tests/stress/internals/x_sleep.timeout =================================================================== --- branches/release-0.94/tests/stress/internals/x_sleep.timeout (rev 0) +++ branches/release-0.94/tests/stress/internals/x_sleep.timeout 2013-04-11 20:29:12 UTC (rev 6430) @@ -0,0 +1 @@ +999999 \ No newline at end of file From yadunandb at ci.uchicago.edu Thu Apr 11 15:00:26 2013 From: yadunandb at ci.uchicago.edu (yadunandb at ci.uchicago.edu) Date: Thu, 11 Apr 2013 20:00:26 -0000 Subject: [Swift-commit] r6429 - branches/release-0.94/tests Message-ID: <20130411200025.C66C99D79A@svn.ci.uchicago.edu> Author: yadunandb Date: 2013-04-11 15:00:25 -0500 (Thu, 11 Apr 2013) New Revision: 6429 Modified: branches/release-0.94/tests/suite.sh Log: Added support for stress levels for stress testing. Added minor change to override environment variables from setup script Modified: branches/release-0.94/tests/suite.sh =================================================================== --- branches/release-0.94/tests/suite.sh 2013-04-09 18:10:41 UTC (rev 6428) +++ branches/release-0.94/tests/suite.sh 2013-04-11 20:00:25 UTC (rev 6429) @@ -16,6 +16,7 @@ printf "\t -t Tree mode (alias: -a,-c,-g,-p,-s) \n" printf "\t -x Do not continue after a failure \n" printf "\t -v Verbose (set -x, HTML comments) \n" + printf "\t -l Stress level < 1/ 2/ 3/ 4> \n" printf "\t -o output Location for cog and output \n" printf "\t GROUP argument \n" } @@ -24,6 +25,7 @@ TEXTREPORT=0 DEFAULT_TIMEOUT=30 # seconds RUN_ANT=1 +STRESS=2 # If true, run "ant clean" CLEAN=1 SKIP_TESTS=0 @@ -81,6 +83,9 @@ -x) ALWAYS_EXITONFAILURE=1 shift;; + -l) + STRESS=$2 + shift 2;; -v) VERBOSE=1 shift;; @@ -110,7 +115,7 @@ GRAY="" fi - +export STRESS="S$STRESS" # Iterations per test (may want to run each test multiple times?) ITERS_LOCAL=1 @@ -398,6 +403,20 @@ fi } +override_globals() { + FILE=$1; + index=0; + while read line + do + if echo $line | grep -q "#OVERRIDE_" ; then + line=($line) + var_lhs=${line[0]#\#OVERRIDE_} + var_rhs=${line[1]}; + eval export $var_lhs=$var_rhs; + fi + done < $FILE +} + start_group() { G=$1 echo @@ -785,16 +804,29 @@ TEST_SHOULD_FAIL=0 OUTPUT=$NAME.setup.stdout + if [ -x "$GROUP/$SETUPSCRIPT" ]; then cp "$GROUP/$SETUPSCRIPT" . script_exec ./$SETUPSCRIPT "S" + globus_var=`grep "#OVERRIDE_" ./$SETUPSCRIPT`; + echo "Globus_var = $globus_var" + if [ $? == 0 ] + then + globus_var=($globus_var); + lhs_var=${globus_var[0]##OVERRIDE_} # Add check here to confirm if GLOBUS_HOSTNAME + rhs_val=${globus_var[1]} + export GLOBUS_HOSTNAME=$rhs_val + fi else stage_files $GROUP $NAME fi ARGS="" if [ -f $GROUP/$ARGSFILE ]; then - ARGS=`cat $GROUP/$ARGSFILE` + cp "$GROUP/$ARGSFILE" . + ARGS=`cat $GROUP/$ARGSFILE` + elif [ -f $ARGSFILE ]; then + ARGS=`cat $ARGSFILE` fi CDM= From tmoore at ci.uchicago.edu Tue Apr 16 14:13:37 2013 From: tmoore at ci.uchicago.edu (tmoore at ci.uchicago.edu) Date: Tue, 16 Apr 2013 19:13:37 -0000 Subject: [Swift-commit] r6450 - SwiftApps/ParVis/HiRAMTools Message-ID: <20130416191337.24A9F9CCC9@svn.ci.uchicago.edu> Author: tmoore Date: 2013-04-16 14:13:37 -0500 (Tue, 16 Apr 2013) New Revision: 6450 Added: SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.6month SwiftApps/ParVis/HiRAMTools/removetrans.sh Modified: SwiftApps/ParVis/HiRAMTools/combine_realization.sh SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh Log: updated for maxWallTime changes Modified: SwiftApps/ParVis/HiRAMTools/combine_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2013-04-16 19:07:46 UTC (rev 6449) +++ SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2013-04-16 19:13:37 UTC (rev 6450) @@ -70,7 +70,7 @@ 32 28000 - 00:20:00 + 03:20:00 100 100 1 Added: SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.6month =================================================================== --- SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.6month (rev 0) +++ SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.6month 2013-04-16 19:13:37 UTC (rev 6450) @@ -0,0 +1,59 @@ +#! /bin/sh + +realid=$1 +realdir=$2 +outdir=$3 +scriptlist=$PWD/$4 +scriptfilter=$5 + +mkdir -p $outdir/$realid/scripts +cd $outdir/$realid/scripts + +tcsh /home/wilde/LZ/HiRAMTools/pfrepp.csh \ + -a $realdir \ + -b $outdir/$realid \ + -s 1950 \ + -i /intrepid-fs0/users/lzamboni/persistent/bin/fms/pfrepp/remap_file/C90_to_180x90/C90_mosaic.nc \ + -x 180 -y 90 \ + -r /intrepid-fs0/users/lzamboni/persistent/bin/fms/pfrepp/remap_file/C90_to_180x90/C90_to_180x90.nc \ + -m 6 -n 50 -q \ + -u + +echo $0: scriptfilter=$scriptfilter + +if [ "_$scriptfilter" != _ ]; then + echo $0: using scriptfilter + scripts=$(/bin/ls -1 pfrepp_script* | sh -c "$scriptfilter" ) +else + scripts=$(/bin/ls -1 pfrepp_script* ) +fi + +echo id path >$scriptlist +for f in $scripts; do + echo $realid $f >>$scriptlist +done + +exit + +# -a /intrepid-fs0/users/lzamboni/persistent/yearly-nco/en1eo12Ic3/run001/run001/ \ + +# datasets needed by LZ: + +Pfrepp-month.time_average +Pfrepp-month.time_series +Pfrepp-daily.time_series (4X???) +Pfrepp-8xdaily.time_series +Pfrepp-8xdaily_instant.time_series +Pfrepp-4xdaily.time_series + +which means: + +pfrepp_script.atmos_month.time_average.1950* +pfrepp_script.atmos_month.time_series.1950* +pfrepp_script.atmos_4xdaily.time_series.1950* +pfrepp_script.atmos_8xdaily.time_series.1950* +pfrepp_script.atmos_8xdaily_instant.time_series.1950* +pfrepp_script.atmos_daily.time_series.1950* + +pfrepp_script.ice_month.time_average.1950* +pfrepp_script.ice_month.time_series.1950* Property changes on: SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.6month ___________________________________________________________________ Added: svn:executable + * Modified: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2013-04-16 19:07:46 UTC (rev 6449) +++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2013-04-16 19:13:37 UTC (rev 6450) @@ -75,7 +75,7 @@ 32 28000 - 00:20:00 + 04:20:00 100 100 1 Added: SwiftApps/ParVis/HiRAMTools/removetrans.sh =================================================================== --- SwiftApps/ParVis/HiRAMTools/removetrans.sh (rev 0) +++ SwiftApps/ParVis/HiRAMTools/removetrans.sh 2013-04-16 19:13:37 UTC (rev 6450) @@ -0,0 +1,9 @@ +#!/bin/sh + +for d in 19* +do + echo $d + cd $d + rm -Rf *.trans + cd .. +done Property changes on: SwiftApps/ParVis/HiRAMTools/removetrans.sh ___________________________________________________________________ Added: svn:executable + *