Swift 0.94 - 2013/03/01

For the majority of users, this is the version you will want to download. This package contains the latest Stable release of Swift. Since Swift is written in Java, this package will run on all supported platforms with Java Runtime Environment 1.6 or greater.

Precompiled binary distribution -[swift-0.94RC4.tar.gz] +[swift-0.94.tar.gz]

Source Code

Getting Started

Use the Quickstart Guide to help you install and configure swift and run a simple 'Hello World' example.

Bug Reports

Download Now

0.94RC4 current version
2013/02/22 +

Download Now

0.94 current version
2013/03/01

Get Started

Read our Quick Start Guide +

Get Started

Read our Quick Start Guide and start using Swift today!

From davidk at ci.uchicago.edu Sat Mar 2 15:30:49 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Sat, 2 Mar 2013 15:30:49 -0600 (CST) Subject: [Swift-commit] r6323 - branches/release-0.94/bin Message-ID: <20130302213049.308959CD06@svn.ci.uchicago.edu> Author: davidk Date: 2013-03-02 15:30:48 -0600 (Sat, 02 Mar 2013) New Revision: 6323 Modified: branches/release-0.94/bin/gensites Log: Fix the order in which gensites searches for templates: full pathname, $HOME/.swift, then etc/sites Modified: branches/release-0.94/bin/gensites =================================================================== --- branches/release-0.94/bin/gensites 2013-03-01 12:48:13 UTC (rev 6322) +++ branches/release-0.94/bin/gensites 2013-03-02 21:30:48 UTC (rev 6323) @@ -123,10 +123,10 @@ if [ ! -f "$TEMPLATE_PATH" ]; then if [ -f "$TEMPLATE" ]; then TEMPLATE_PATH=$TEMPLATE + elif [ -f "$HOME/.swift/sites/$TEMPLATE" ]; then + TEMPLATE_PATH=$HOME/.swift/sites/$TEMPLATE elif [ -f "$SWIFT_HOME/$TEMPLATE" ]; then TEMPLATE_PATH=$SWIFT_HOME/$TEMPLATE - elif [ -f "$HOME/.swift/sites/$TEMPLATE" ]; then - TEMPLATE_PATH=$HOME/.swift/sites/$TEMPLATE fi if [ ! -f "$TEMPLATE_PATH" ]; then crash "Cannot find template for $TEMPLATE" From davidk at ci.uchicago.edu Sat Mar 2 15:31:23 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Sat, 2 Mar 2013 15:31:23 -0600 (CST) Subject: [Swift-commit] r6324 - trunk/bin Message-ID: <20130302213123.A1A5F9CD06@svn.ci.uchicago.edu> Author: davidk Date: 2013-03-02 15:31:23 -0600 (Sat, 02 Mar 2013) New Revision: 6324 Modified: trunk/bin/gensites Log: Update to gensites search paths Modified: trunk/bin/gensites =================================================================== --- trunk/bin/gensites 2013-03-02 21:30:48 UTC (rev 6323) +++ trunk/bin/gensites 2013-03-02 21:31:23 UTC (rev 6324) @@ -123,10 +123,10 @@ if [ ! -f "$TEMPLATE_PATH" ]; then if [ -f "$TEMPLATE" ]; then TEMPLATE_PATH=$TEMPLATE + elif [ -f "$HOME/.swift/sites/$TEMPLATE" ]; then + TEMPLATE_PATH=$HOME/.swift/sites/$TEMPLATE elif [ -f "$SWIFT_HOME/$TEMPLATE" ]; then TEMPLATE_PATH=$SWIFT_HOME/$TEMPLATE - elif [ -f "$HOME/.swift/sites/$TEMPLATE" ]; then - TEMPLATE_PATH=$HOME/.swift/sites/$TEMPLATE fi if [ ! -f "$TEMPLATE_PATH" ]; then crash "Cannot find template for $TEMPLATE" From davidk at ci.uchicago.edu Sat Mar 2 18:21:48 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Sat, 2 Mar 2013 18:21:48 -0600 (CST) Subject: [Swift-commit] r6325 - branches/release-0.94/bin Message-ID: <20130303002148.E4E7B9D5A3@svn.ci.uchicago.edu> Author: davidk Date: 2013-03-02 18:21:48 -0600 (Sat, 02 Mar 2013) New Revision: 6325 Modified: branches/release-0.94/bin/stop-coaster-service Log: Fix for stopping coaster jobs locally Modified: branches/release-0.94/bin/stop-coaster-service =================================================================== --- branches/release-0.94/bin/stop-coaster-service 2013-03-02 21:31:23 UTC (rev 6324) +++ branches/release-0.94/bin/stop-coaster-service 2013-03-03 00:21:48 UTC (rev 6325) @@ -86,7 +86,7 @@ if [ -n "$WORKER_RELAY_HOST" ]; then ssh $WORKER_USERNAME@$WORKER_RELAY_HOST "condor_rm $WORKER_USERNAME" else - condor_rm $CONDOR_USERNAME + condor_rm $WORKER_USERNAME fi fi From hategan at ci.uchicago.edu Sun Mar 3 01:06:00 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 3 Mar 2013 01:06:00 -0600 (CST) Subject: [Swift-commit] r6326 - in branches/release-0.94: resources src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib Message-ID: <20130303070600.A07319CCA1@svn.ci.uchicago.edu> Author: hategan Date: 2013-03-03 01:06:00 -0600 (Sun, 03 Mar 2013) New Revision: 6326 Added: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/RefCount.java branches/release-0.94/src/org/griphyn/vdl/karajan/lib/StaticRefCount.java Modified: branches/release-0.94/resources/Karajan.stg branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java branches/release-0.94/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java branches/release-0.94/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java Log: fixed array writing in iterate bug reported by Yadu (see http://lists.ci.uchicago.edu/pipermail/swift-devel/2013-March/010100.html) Modified: branches/release-0.94/resources/Karajan.stg =================================================================== --- branches/release-0.94/resources/Karajan.stg 2013-03-03 00:21:48 UTC (rev 6325) +++ branches/release-0.94/resources/Karajan.stg 2013-03-03 07:06:00 UTC (rev 6326) @@ -219,9 +219,8 @@ >> -iterate(declarations,statements,cond,var,cleanups,trace,line) ::= << - - +iterate(declarations, statements, cond, var, refs, cleanups, trace, line) ::= << + $sub_comp(declarations=declarations, statements=statements, cleanups=cleanups)$ $cond$ Modified: branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java 2013-03-03 00:21:48 UTC (rev 6325) +++ branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java 2013-03-03 07:06:00 UTC (rev 6326) @@ -908,7 +908,7 @@ } public void iterateStat(Iterate iterate, VariableScope scope) throws CompilationException { - VariableScope loopScope = new VariableScope(this, scope, EnclosureType.LOOP, iterate); + VariableScope loopScope = new VariableScope(this, scope, EnclosureType.ALL, iterate); VariableScope innerScope = new VariableScope(this, loopScope, EnclosureType.LOOP, iterate); loopScope.addVariable(iterate.getVar(), "int", "Iteration variable", iterate); @@ -1531,8 +1531,6 @@ throw new CompilationException("Procedure " + name + " must have exactly one " + "return value to be used in an expression."); } - - Warnings.warn(c, "Procedure " + name + " is deprecated"); StringTemplate call = template("callexpr"); Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java =================================================================== --- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java 2013-03-03 00:21:48 UTC (rev 6325) +++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java 2013-03-03 07:06:00 UTC (rev 6326) @@ -34,8 +34,10 @@ public static final String COUNTER_NAME = "$"; public static final Arg.Positional VAR = new Arg.Positional("var"); + public static final Arg O_REFS = new Arg.Optional("refs", null); private Tracer tracer; + private List srefs; public InfiniteCountingWhile() { setOptimize(false); @@ -45,18 +47,21 @@ protected void initializeStatic() { super.initializeStatic(); tracer = Tracer.getTracer(this); + srefs = StaticRefCount.build((String) O_REFS.getStatic(this)); } public void pre(VariableStack stack) throws ExecutionException { ThreadingContext tc = (ThreadingContext)stack.getVar("#thread"); stack.setVar("#iteratethread", tc); stack.setVar("#thread", tc.split(0)); + stack.setVar("#refs", RefCount.build(srefs, stack)); stack.setVar(COUNTER_NAME, Collections.singletonList(0)); String var = (String) VAR.getStatic(this); if (tracer.isEnabled()) { tracer.trace(tc.toString(), var + " = 0"); } stack.setVar(var, new RootDataNode(Types.INT, 0)); + incRefs(stack); super.pre(stack); } @@ -89,6 +94,7 @@ tracer.trace(ntc.toString(), var + " = " + i); } stack.setVar(var, new RootDataNode(Types.INT, i)); + incRefs(stack); } if (index >= elementCount()) { // starting new iteration @@ -102,6 +108,16 @@ startElement(fn, stack); } + private void incRefs(VariableStack stack) { + @SuppressWarnings("unchecked") + List rcs = (List) stack.currentFrame().getVar("#refs"); + if (rcs != null) { + for (RefCount rc : rcs) { + rc.var.updateWriteRefCount(rc.count); + } + } + } + public void failed(VariableStack stack, ExecutionException e) throws ExecutionException { if (e instanceof While.Break) { Added: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/RefCount.java =================================================================== --- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/RefCount.java (rev 0) +++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/RefCount.java 2013-03-03 07:06:00 UTC (rev 6326) @@ -0,0 +1,38 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 2, 2013 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.ArrayList; +import java.util.List; + +import org.globus.cog.karajan.stack.VariableNotFoundException; +import org.globus.cog.karajan.stack.VariableStack; +import org.griphyn.vdl.mapping.DSHandle; + +public class RefCount { + public final DSHandle var; + public final int count; + + public RefCount(DSHandle var, int count) { + this.var = var; + this.count = count; + } + + public static List build(List srefs, VariableStack stack) throws VariableNotFoundException { + if (srefs == null) { + return null; + } + List l = new ArrayList(srefs.size()); + for (StaticRefCount s : srefs) { + l.add(new RefCount((DSHandle) stack.getVar(s.name), s.count)); + } + return l; + } +} \ No newline at end of file Added: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/StaticRefCount.java =================================================================== --- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/StaticRefCount.java (rev 0) +++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/StaticRefCount.java 2013-03-03 07:06:00 UTC (rev 6326) @@ -0,0 +1,45 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Mar 2, 2013 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.ArrayList; +import java.util.List; +import java.util.StringTokenizer; + +public class StaticRefCount { + public final String name; + public final int count; + + public StaticRefCount(String name, int count) { + this.name = name; + this.count = count; + } + + public static List build(String refs) { + if (refs == null) { + return null; + } + List l = new ArrayList(); + String name = null; + boolean flip = true; + StringTokenizer st = new StringTokenizer(refs); + while (st.hasMoreTokens()) { + if (flip) { + name = st.nextToken(); + } + else { + int count = Integer.parseInt(st.nextToken()); + l.add(new StaticRefCount(name.toLowerCase(), count)); + } + flip = !flip; + } + return l; + } +} \ No newline at end of file Modified: branches/release-0.94/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java =================================================================== --- branches/release-0.94/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2013-03-03 00:21:48 UTC (rev 6325) +++ branches/release-0.94/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2013-03-03 07:06:00 UTC (rev 6326) @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.StringTokenizer; import org.apache.log4j.Logger; import org.globus.cog.karajan.arguments.Arg; @@ -40,7 +39,6 @@ import org.globus.cog.karajan.workflow.futures.ListenerStackPair; import org.globus.cog.karajan.workflow.nodes.AbstractParallelIterator; import org.griphyn.vdl.karajan.Pair; -import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.util.VDL2Config; public class ThrottledParallelFor extends AbstractParallelIterator { @@ -64,77 +62,16 @@ private String kvar, vvar; private List srefs; - private static class StaticRefCount { - public final String name; - public final int count; - - public StaticRefCount(String name, int count) { - this.name = name; - this.count = count; - } - } - - private static class RefCount { - public final DSHandle var; - public final int count; - - public RefCount(DSHandle var, int count) { - this.var = var; - this.count = count; - } - - public void inc() { - - } - - public void dec() { - - } - } - - @Override + @Override protected void initializeStatic() { super.initializeStatic(); forTracer = Tracer.getTracer(this, "FOREACH"); iterationTracer = Tracer.getTracer(this, "ITERATION"); kvar = (String) getProperty("_kvar"); vvar = (String) getProperty("_vvar"); - srefs = buildStaticRefs(); + srefs = StaticRefCount.build((String) O_REFS.getStatic(this)); } - private List buildStaticRefs() { - String refs = (String) O_REFS.getStatic(this); - if (refs == null) { - return null; - } - List l = new ArrayList(); - String name = null; - boolean flip = true; - StringTokenizer st = new StringTokenizer(refs); - while (st.hasMoreTokens()) { - if (flip) { - name = st.nextToken(); - } - else { - int count = Integer.parseInt(st.nextToken()); - l.add(new StaticRefCount(name.toLowerCase(), count)); - } - flip = !flip; - } - return l; - } - - private List buildRefs(VariableStack stack) throws VariableNotFoundException { - if (srefs == null) { - return null; - } - List l = new ArrayList(srefs.size()); - for (StaticRefCount s : srefs) { - l.add(new RefCount((DSHandle) stack.getVar(s.name), s.count)); - } - return l; - } - protected void partialArgumentsEvaluated(VariableStack stack) throws ExecutionException { if (forTracer.isEnabled()) { @@ -313,7 +250,7 @@ maxThreadCount = DEFAULT_MAX_THREADS; } } - stack.setVar(THREAD_COUNT, new ThreadCount(maxThreadCount, selfClose, i, buildRefs(stack))); + stack.setVar(THREAD_COUNT, new ThreadCount(maxThreadCount, selfClose, i, RefCount.build(srefs, stack))); } private ThreadCount getThreadCount(VariableStack stack) From hategan at ci.uchicago.edu Sun Mar 3 14:26:14 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Sun, 3 Mar 2013 14:26:14 -0600 (CST) Subject: [Swift-commit] r6327 - branches/release-0.94/src/org/griphyn/vdl/engine Message-ID: <20130303202614.CC5D19D015@svn.ci.uchicago.edu> Author: hategan Date: 2013-03-03 14:26:13 -0600 (Sun, 03 Mar 2013) New Revision: 6327 Modified: branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java Log: some change I had locally which I assume is in line with pre-second-copy 0.94 Modified: branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java 2013-03-03 07:06:00 UTC (rev 6326) +++ branches/release-0.94/src/org/griphyn/vdl/engine/Karajan.java 2013-03-03 20:26:13 UTC (rev 6327) @@ -694,7 +694,7 @@ ProcedureSignature proc = proceduresMap.get(procName); if (proc.isDeprecated()) { - /* warn(call, "Procedure " + procName + " is deprecated"); */ + Warnings.warn(call, "Procedure " + procName + " is deprecated"); } StringTemplate callST; From swift at ci.uchicago.edu Sun Mar 3 15:25:05 2013 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Sun, 3 Mar 2013 15:25:05 -0600 (CST) Subject: [Swift-commit] cog r3616 Message-ID: <20130303212505.764518D000A2@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3616 | hategan | 2013-03-03 15:24:15 -0600 (Sun, 03 Mar 2013) | 1 line use a sorted map to sort blocks by their time left since a comparator cannot guaranteed to be correct because calls to block.sizeLeft() are dependent on the time when the call is made ------------------------------------------------------------------------ Index: modules/provider-coaster/src/org/globus/cog/abstraction/coaster/service/job/manager/BlockQueueProcessor.java =================================================================== --- modules/provider-coaster/src/org/globus/cog/abstraction/coaster/service/job/manager/BlockQueueProcessor.java (revision 3615) +++ modules/provider-coaster/src/org/globus/cog/abstraction/coaster/service/job/manager/BlockQueueProcessor.java (working copy) @@ -6,14 +6,13 @@ import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.Date; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedMap; import java.util.TreeMap; import org.apache.log4j.Logger; @@ -429,31 +428,18 @@ * */ protected void removeIdleBlocks() { - ArrayList sorted; - + SortedMap sorted = new TreeMap(); + synchronized (blocks) { - sorted = new ArrayList(blocks.values()); - Collections.sort(sorted, new Comparator() { - public int compare(Block b1, Block b2) { - double s1 = b1.sizeLeft(); - double s2 = b2.sizeLeft(); - if (s1 == s2) { - return 0; - } - else if (s1 < s2) { - return -1; - } - else { - return 1; - } - } - }); + for (Block b : blocks.values()) { + sorted.put(b.sizeLeft(), b); + } } double needed = queued.getJSize() + running.getSize(); double sum = 0; - for (Block b : sorted) { + for (Block b : sorted.values()) { if (sum >= needed && !b.isSuspended() && (System.currentTimeMillis() - b.getLastUsed()) > Block.SUSPEND_SHUTDOWN_DELAY) { From ketan at ci.uchicago.edu Sun Mar 3 23:53:21 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Sun, 3 Mar 2013 23:53:21 -0600 (CST) Subject: [Swift-commit] r6328 - trunk/docs/userguide Message-ID: <20130304055321.90FC39CCA1@svn.ci.uchicago.edu> Author: ketan Date: 2013-03-03 23:53:21 -0600 (Sun, 03 Mar 2013) New Revision: 6328 Modified: trunk/docs/userguide/coasters Log: adding subsections to coasters Modified: trunk/docs/userguide/coasters =================================================================== --- trunk/docs/userguide/coasters 2013-03-03 20:26:13 UTC (rev 6327) +++ trunk/docs/userguide/coasters 2013-03-04 05:53:21 UTC (rev 6328) @@ -90,3 +90,13 @@ |remoteMonitorEnabled|If true, show a graphical display of the status of the coaster service |================== + +Submitting to a specific job queue +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to submit to a specific job schedular queue, the +queue+ parameter of coasters sites file should be used. For example, submitting to the +normal+ queue of a PBS schedular, one would use the following sites line: + +---- +normal +---- + From ketan at ci.uchicago.edu Mon Mar 4 21:25:51 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Mon, 4 Mar 2013 21:25:51 -0600 (CST) Subject: [Swift-commit] r6329 - trunk/docs/userguide Message-ID: <20130305032551.E70519CD1D@svn.ci.uchicago.edu> Author: ketan Date: 2013-03-04 21:25:51 -0600 (Mon, 04 Mar 2013) New Revision: 6329 Added: trunk/docs/userguide/swift_monitor.png Modified: trunk/docs/userguide/configuration_properties Log: formatting, tui and monitor Modified: trunk/docs/userguide/configuration_properties =================================================================== --- trunk/docs/userguide/configuration_properties 2013-03-04 05:53:21 UTC (rev 6328) +++ trunk/docs/userguide/configuration_properties 2013-03-05 03:25:51 UTC (rev 6329) @@ -1,17 +1,16 @@ Swift configuration properties ------------------------------ -Various aspects of the behavior of the Swift Engine can be configured -through properties. The Swift Engine recognizes a global, per -installation properties file which can found in etc/swift.properties -in the Swift installation directory and a user properties file which can -be created by each user in ~/.swift/swift.properties. The Swift Engine -will first load the global properties file. It will then try to load the -user properties file. If a user properties file is found, individual -properties explicitly set in that file will override the respective -properties in the global properties file. Furthermore, some of the -properties can be overridden directly using command line arguments to -the *swift* command. +Various aspects of the behavior of Swift can be configured through properties. +Swift recognizes a global, per installation properties file which can found in +etc/swift.properties in the Swift installation directory and a user properties +file which can be created by each user in ~/.swift/swift.properties. +Swift will first load the global properties file. It will then try to load the +user properties file. If a user properties file is found, individual properties +explicitly set in that file will override the respective properties in the +global properties file. Furthermore, some of the properties can be overridden +directly using command line arguments to the *swift* command. + Swift properties are specified in the following format: = @@ -452,3 +451,43 @@ ip.address=192.168.0.1 ---- +Monitoring Swift +~~~~~~~~~~~~~~~~ + +A Swift run can be monitored for progress and resource usage. To monitor the resource usage, use the +-monitor+ option with the Swift commandline. For example: + +---- +swift -tc.file tc -sites.file sites.xml -config cf modis04.swift -monitor +---- + +This will produce a gui/X window consisting of the following quantities: + +* Allocated memory +* Heap Size +* Total Threads +* Total Workers + +.Figure 4. Resource Monitor +image:swift_monitor.png[] + +Figure 4 shows a snapshot of a Swift resource monitor. + + +The progress of a Swift run can be monitored using the +-tui+ option. For example: + +---- +swift -tc.file tc -sites.file sites.xml -config cf modis04.swift -tui +---- + +This will produce a textual user interface with multiple tabs, each showing the following features of the current Swift run: + +* A summary view showing task status +* An apps tab +* A jobs tab +* A transfer tab +* A scheduler tab +* A Tast statistics tab +* A customized tab called 'Ben's View' + +Navigation between these tabs can be done using the function keys f2 through f8. + Added: trunk/docs/userguide/swift_monitor.png =================================================================== (Binary files differ) Property changes on: trunk/docs/userguide/swift_monitor.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream From davidk at ci.uchicago.edu Tue Mar 5 10:29:26 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 5 Mar 2013 10:29:26 -0600 (CST) Subject: [Swift-commit] r6330 - SwiftTutorials/IO Message-ID: <20130305162926.73D5B9CC97@svn.ci.uchicago.edu> Author: davidk Date: 2013-03-05 10:29:26 -0600 (Tue, 05 Mar 2013) New Revision: 6330 Added: SwiftTutorials/IO/TODO Log: TODO Added: SwiftTutorials/IO/TODO =================================================================== --- SwiftTutorials/IO/TODO (rev 0) +++ SwiftTutorials/IO/TODO 2013-03-05 16:29:26 UTC (rev 6330) @@ -0,0 +1,32 @@ +lets desribe all the sources of file management and IO overhead that we know of, and how to eliminate them. +1. strings to prevent staging +2. use of scratch to reduce the effect of small-block IO (iff it applies: it might not apply here if the files are tiny) +3. workdir overhead pushed to local node filesystem +4. reduce/elim overhead of swift files like info and the .d dir + +You can do the same analysis/tests for CDM +The trick there is ensuring that a) CDM files match as expected and (b) that CDM itself doesnt add unacceptable overhead +ANd we have never stress tested the CDM file caching option that we always turn of in the cf file + +First one, just do a simplified test version of what you did for dssat on stampede? +ie a version of that that strips out the "noise" and lists the things that made the improvement +I think those things are the things I mentioned above +SO, lets say for the app, you use a shell script that does 3 dd statements: the stagein dd, the "app" dd (using a small block size like 8KB) and the stageout dd ? +one file in, ine file out, for starters. +Later we can parameterize these tests and demonstrate the overhead of different aspects changing (ie add more in and/or out files) ? + +So this User Guide setion needs to start out very simple, but cover all the cases in a well-structured manner that makes it no more complex than is necessary. +So I would structure this as a set of directories, where each dir has one test case/example and a README saying what it shows and how its used. +You can just number the dirs, or give them longish names like "local_scratch+full_pathnames/" +Each should have a cf and a sites.xml +and save its workdirectory below it, +and run an app that returns a detailed ls/find dummary of its cwd and arg list. +because the added complexity is not just how the jobdir gets set, but also what gets passed to the app on its command line +So, now, I *think* we have laidout out here all the issues: + - where is jobdir + - what is copied where + - what is linked where + - what is passed to command + - what is copied out after command + - what is cleaned up (under what options) + From davidk at ci.uchicago.edu Tue Mar 5 21:03:15 2013 From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu) Date: Tue, 5 Mar 2013 21:03:15 -0600 (CST) Subject: [Swift-commit] r6331 - trunk/libexec Message-ID: <20130306030315.C894A9CCA1@svn.ci.uchicago.edu> Author: davidk Date: 2013-03-05 21:03:15 -0600 (Tue, 05 Mar 2013) New Revision: 6331 Modified: trunk/libexec/_swiftwrap.wrapperstaging Log: A few fixes for local copying Modified: trunk/libexec/_swiftwrap.wrapperstaging =================================================================== --- trunk/libexec/_swiftwrap.wrapperstaging 2013-03-05 16:29:26 UTC (rev 6330) +++ trunk/libexec/_swiftwrap.wrapperstaging 2013-03-06 03:03:15 UTC (rev 6331) @@ -58,6 +58,8 @@ localPath() { # remove protocol:// PATH="${1#*://}" + # Remove hostname + PATH="${PATH#*/}" # remove leading "/" if present PATH="${PATH#/}" log "Transformed $1 to $PATH" @@ -67,7 +69,7 @@ stagein() { SRC=$1 DST=$2 - + case $SRC in *://*) ;; @@ -85,7 +87,7 @@ if [ ! -f $SRC ]; then fail 254 "Cannot stage in $SRC. File not found." fi - cp $SRC $DST 2>&1 >& "$INFO" + dd if=$SRC of=$DST bs=16M 2>&1 >& "$INFO" ;; http://*) log "Fetching $SRC using wget" @@ -453,8 +455,10 @@ eval cdm_action $DIR "INPUT" $L $CDM_POLICY continue fi - - stagein $L "$DIR/`localPath $L`" + + SRC=$( localPath $L ) + DST=$DIR/$( localPath $L ) + stagein $SRC $DST done if [[ $CDM_FILE != "" ]]; then From ketan at ci.uchicago.edu Tue Mar 5 22:25:34 2013 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 5 Mar 2013 22:25:34 -0600 (CST) Subject: [Swift-commit] r6332 - trunk/docs/userguide Message-ID: <20130306042534.E3F9E9CCA1@svn.ci.uchicago.edu> Author: ketan Date: 2013-03-05 22:25:34 -0600 (Tue, 05 Mar 2013) New Revision: 6332 Modified: trunk/docs/userguide/site_catalog Log: scratch tag Modified: trunk/docs/userguide/site_catalog =================================================================== --- trunk/docs/userguide/site_catalog 2013-03-06 03:03:15 UTC (rev 6331) +++ trunk/docs/userguide/site_catalog 2013-03-06 04:25:34 UTC (rev 6332) @@ -11,7 +11,6 @@ The sites file is formatted as XML. It consists of elements, one for each site that Swift will use. - Pool element ~~~~~~~~~~~~ Each pool element must have a handle attribute, giving a symbolic @@ -25,7 +24,6 @@ a remote working directory. Optionally, profile settings can be specified. - File transfer method ~~~~~~~~~~~~~~~~~~~~ Transfer methods are specified with either the element or @@ -158,6 +156,20 @@ be used for execution. A shared cluster scratch filesystem is appropriate for this. +Scratch +~~~~~~~ + +The scratch element takes in a value of a directory on a shared filesystem. For example: + +---- +/work/01739/ketan/lab/swift/myscratch +---- + +The scratch element specifies that the underlying Swift wrapper will copy the +input files into the scratch directory and the job will be run from that +directory. In the absence of scratch tag, Swift will run the job from +workdirectory by creating symlinks to input files in a shared directory. + Profiles ~~~~~~~~ Profile keys can be specified using the element. From hategan at ci.uchicago.edu Wed Mar 6 01:22:48 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 6 Mar 2013 01:22:48 -0600 (CST) Subject: [Swift-commit] r6333 - branches/faster/src/org/griphyn/vdl/karajan/lib Message-ID: <20130306072248.20B979CCA1@svn.ci.uchicago.edu> Author: hategan Date: 2013-03-06 01:22:47 -0600 (Wed, 06 Mar 2013) New Revision: 6333 Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java Log: faster runtime stats Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2013-03-06 04:25:34 UTC (rev 6332) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2013-03-06 07:22:47 UTC (rev 6333) @@ -159,6 +159,33 @@ } } + private static class MutableInt { + private int value; + + public MutableInt() { + value = 0; + } + + public MutableInt(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + + public void setValue(int value) { + this.value = value; + } + + public void inc() { + value++; + } + + public String toString() { + return String.valueOf(value); + } + } public static class StopProgressTicker extends Node { private VarRef context; @@ -265,13 +292,13 @@ printStates("Final status:"); } - private Map getSummary() { + private Map getSummary() { List states; synchronized(this.states) { states = new ArrayList(this.states); } - Map m = new HashMap(); + Map m = new HashMap(); for (ProgressState s : states) { inc(m, s); @@ -279,20 +306,19 @@ return m; } - private void inc(Map m, ProgressState s) { + private void inc(Map m, ProgressState s) { String v = s.crt; - Integer i = m.get(v); + MutableInt i = m.get(v); if (i == null) { - i = 1; + i = new MutableInt(); + m.put(v, i); } - else { - i = i + 1; - } - m.put(v, i); + + i.inc(); } synchronized void printStates(String prefix) { - Map summary = getSummary(); + Map summary = getSummary(); StringBuilder sb = new StringBuilder(); @@ -305,7 +331,7 @@ for (int pos = 0; pos < preferredOutputOrder.length; pos++) { String key = preferredOutputOrder[pos]; - Integer value = summary.get(key); + MutableInt value = summary.get(key); if(value != null) { sb.append(" "); sb.append(key); @@ -315,7 +341,7 @@ summary.remove(key); } - for (Map.Entry s : summary.entrySet()) { + for (Map.Entry s : summary.entrySet()) { sb.append(" "); sb.append(s.getKey()); sb.append(":"); From hategan at ci.uchicago.edu Wed Mar 6 01:23:15 2013 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 6 Mar 2013 01:23:15 -0600 (CST) Subject: [Swift-commit] r6334 - branches/faster/src/org/griphyn/vdl/karajan/lib Message-ID: <20130306072315.A011F9D670@svn.ci.uchicago.edu> Author: hategan Date: 2013-03-06 01:23:14 -0600 (Wed, 06 Mar 2013) New Revision: 6334 Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java Log: avoid creating arrays when not necessary Modified: branches/faster/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java =================================================================== --- branches/faster/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java 2013-03-06 07:22:47 UTC (rev 6333) +++ branches/faster/src/org/griphyn/vdl/karajan/lib/ExpandArguments.java 2013-03-06 07:23:14 UTC (rev 6334) @@ -24,6 +24,7 @@ import java.util.TreeMap; import k.rt.Channel; +import k.rt.ExecutionException; import k.rt.Stack; import org.apache.log4j.Logger; @@ -44,40 +45,60 @@ } public Object function(Stack stack) { - ArrayList l = new ArrayList(); - Channel

Latest Release

Source Code

Latest Release

Getting Started

Bug Reports