From ketan at ci.uchicago.edu Tue Mar 3 12:58:25 2015 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 3 Mar 2015 12:58:25 -0600 (CST) Subject: [Swift-commit] r8382 - SwiftApps/subjobs/mpicatsnsleep Message-ID: <20150303185825.658019D539@svn.ci.uchicago.edu> Author: ketan Date: 2015-03-03 12:58:25 -0600 (Tue, 03 Mar 2015) New Revision: 8382 Added: SwiftApps/subjobs/mpicatsnsleep/swift.conf Modified: SwiftApps/subjobs/mpicatsnsleep/catsnsleepmpi.swift SwiftApps/subjobs/mpicatsnsleep/runcetus.sh Log: adding conf Modified: SwiftApps/subjobs/mpicatsnsleep/catsnsleepmpi.swift =================================================================== --- SwiftApps/subjobs/mpicatsnsleep/catsnsleepmpi.swift 2015-02-27 20:02:44 UTC (rev 8381) +++ SwiftApps/subjobs/mpicatsnsleep/catsnsleepmpi.swift 2015-03-03 18:58:25 UTC (rev 8382) @@ -7,9 +7,9 @@ file out[]; -foreach j in [1:toInt(arg("n","1"))] { +foreach i in [1:toInt(arg("n","1"))] { file data<"data.txt">; - file sout; - file serr; - (out[j],sout,serr) = cat(data); + file sout; + file serr; + (out[i],sout,serr) = cat(data); } Modified: SwiftApps/subjobs/mpicatsnsleep/runcetus.sh =================================================================== --- SwiftApps/subjobs/mpicatsnsleep/runcetus.sh 2015-02-27 20:02:44 UTC (rev 8381) +++ SwiftApps/subjobs/mpicatsnsleep/runcetus.sh 2015-03-03 18:58:25 UTC (rev 8382) @@ -7,7 +7,8 @@ #mpixlc mpicatnap.c -o mpicatnap #New method -/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file cobalt.cetus.xml -config cf -tc.file apps catsnsleepmpi.swift -n=16 -s=1 +#/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file cobalt.cetus.xml -config cf -tc.file apps catsnsleepmpi.swift -n=16 -s=1 +/home/ketan/swift-k/dist/swift-svn/bin/swift catsnsleepmpi.swift -n=16 -s=1 #A testblock run #qsub -A ATPESC2013 -n 32 -t 5 -q low --mode c16 --mode script testblock.sh # --disable_preboot Added: SwiftApps/subjobs/mpicatsnsleep/swift.conf =================================================================== --- SwiftApps/subjobs/mpicatsnsleep/swift.conf (rev 0) +++ SwiftApps/subjobs/mpicatsnsleep/swift.conf 2015-03-03 18:58:25 UTC (rev 8382) @@ -0,0 +1,35 @@ +sites : cluster +site.cluster { + execution { + type: "coaster" + URL: "localhost" + jobManager: "local:cobalt" + options { + maxNodesPerJob: 32 + maxJobs: 1 + tasksPerNode: 2 + # Option ignored: globus:mode = script + #workerLoggingLevel = "DEBUG" + nodeGranularity: 32 + } + } + filesystem { + type: "local" + URL: "localhost" + } + staging : swift + workDirectory: "/home/ketan/swiftwork" + maxParallelTasks: 30 + initialParallelTasks: 29 + app.bgsh { + executable: "/home/ketan/SwiftApps/subjobs/bg.sh" + maxWallTime: "00:04:00" + env.SUBBLOCK_SIZE="16" + } +} + +executionRetries: 0 +keepSiteDir: false +providerStagingPinSwiftFiles: false +alwaysTransferWrapperLog: false + From ketan at ci.uchicago.edu Tue Mar 3 14:17:16 2015 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 3 Mar 2015 14:17:16 -0600 (CST) Subject: [Swift-commit] r8383 - SwiftApps/subjobs/mpicatsnsleep Message-ID: <20150303201716.D96349D539@svn.ci.uchicago.edu> Author: ketan Date: 2015-03-03 14:17:16 -0600 (Tue, 03 Mar 2015) New Revision: 8383 Modified: SwiftApps/subjobs/mpicatsnsleep/runcetus.sh Log: small change Modified: SwiftApps/subjobs/mpicatsnsleep/runcetus.sh =================================================================== --- SwiftApps/subjobs/mpicatsnsleep/runcetus.sh 2015-03-03 18:58:25 UTC (rev 8382) +++ SwiftApps/subjobs/mpicatsnsleep/runcetus.sh 2015-03-03 20:17:16 UTC (rev 8383) @@ -7,8 +7,8 @@ #mpixlc mpicatnap.c -o mpicatnap #New method -#/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file cobalt.cetus.xml -config cf -tc.file apps catsnsleepmpi.swift -n=16 -s=1 -/home/ketan/swift-k/dist/swift-svn/bin/swift catsnsleepmpi.swift -n=16 -s=1 +#/home/ketan/swift-0.95/cog/modules/swift/dist/swift-svn/bin/swift -sites.file cobalt.cetus.xml -config cf -tc.file apps catsnsleepmpi.swift -n=8 -s=1 +/home/ketan/swift-k/dist/swift-svn/bin/swift catsnsleepmpi.swift -n=8 -s=1 #A testblock run #qsub -A ATPESC2013 -n 32 -t 5 -q low --mode c16 --mode script testblock.sh # --disable_preboot From ketan at ci.uchicago.edu Tue Mar 3 14:27:11 2015 From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu) Date: Tue, 3 Mar 2015 14:27:11 -0600 (CST) Subject: [Swift-commit] r8384 - SwiftApps/subjobs Message-ID: <20150303202711.DE7D19D539@svn.ci.uchicago.edu> Author: ketan Date: 2015-03-03 14:27:11 -0600 (Tue, 03 Mar 2015) New Revision: 8384 Modified: SwiftApps/subjobs/bg.sh SwiftApps/subjobs/runbash.sh Log: minor Modified: SwiftApps/subjobs/bg.sh =================================================================== --- SwiftApps/subjobs/bg.sh 2015-03-03 20:17:16 UTC (rev 8383) +++ SwiftApps/subjobs/bg.sh 2015-03-03 20:27:11 UTC (rev 8384) @@ -1,5 +1,7 @@ #!/bin/bash +set -x + mname=$(hostname) # vesta and mira has different path than cetus @@ -10,6 +12,8 @@ export PATH=/soft/cobalt/cetus/bgq_hardware_mapper:$PATH fi +#export SUBBLOCK_SIZE=16 + # Prepare shape based on subblock size # provided by user in sites environment case "$SUBBLOCK_SIZE" in @@ -57,7 +61,7 @@ #without timeout #runjob --strace none --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$((16*$SUBBLOCK_SIZE))" : "$@" - runjob --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np 2 : "$@" + runjob --block "$COBALT_PARTNAME" --corner "$CORNER" --shape "$SHAPE" -p 16 --np "$((16*$SUBBLOCK_SIZE))" : "$@" echo "Runjob finished." else Modified: SwiftApps/subjobs/runbash.sh =================================================================== --- SwiftApps/subjobs/runbash.sh 2015-03-03 20:17:16 UTC (rev 8383) +++ SwiftApps/subjobs/runbash.sh 2015-03-03 20:27:11 UTC (rev 8384) @@ -3,3 +3,5 @@ #To run: qsub -n 512 -t 02:00:00 --mode script subjob.sh +#qsub -e WORKER_LOGGING_LEVEL=NONE --proccount 32 -n 32 -t 40 + From wozniak at ci.uchicago.edu Thu Mar 5 13:23:40 2015 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Thu, 5 Mar 2015 13:23:40 -0600 (CST) Subject: [Swift-commit] r8385 - www/Swift-T Message-ID: <20150305192340.E22CE9CD26@svn.ci.uchicago.edu> Author: wozniak Date: 2015-03-05 13:23:39 -0600 (Thu, 05 Mar 2015) New Revision: 8385 Modified: www/Swift-T/gallery.html Log: Update gallery Modified: www/Swift-T/gallery.html =================================================================== --- www/Swift-T/gallery.html 2015-03-03 20:27:11 UTC (rev 8384) +++ www/Swift-T/gallery.html 2015-03-05 19:23:39 UTC (rev 8385) @@ -773,10 +773,8 @@
import io;
-main
-{
-  printf("Hello world!");
-}
+ +printf("Hello world!");
@@ -792,12 +790,9 @@ "od" "-b" i @stdout=o; } -main -{ - file x = input("mtc1.swift"); - file y<"mtc1.octal">; - y = f(x); -} +file x = input("mtc1.swift"); +file y<"mtc1.octal">; +y = f(x);

This script splits itself into lines, where line i is in file out- i .txt

@@ -812,13 +807,10 @@ "/bin/echo" s @stdout=o; } -main +string lines[] = file_lines(input("mtc2.swift")); +foreach line,i in lines { - string lines[] = file_lines(input("mtc2.swift")); - foreach line,i in lines - { - file y <sprintf("out-%i.txt",i)> = f(line); - } + file y <sprintf("out-%i.txt",i)> = f(line); }

Note that each /bin/echo is eligible to run concurrently. See @@ -827,8 +819,42 @@

-

3. Reductions

+

3. Running many tasks on a cluster

+

It’s easy to launch these kinds of workloads on a cluster.

+

If using a plain host list:

+

File: mtc/hosts.txt

+
+
+
machine1.some.edu
+machine2.some.edu
+machine3.some.edu
+machine4.some.edu
+
+

Just run with:

+
+
+
stc mtc1.swift
+turbine -f hosts.txt -n 4 mtc1.tic
+
+

As shown, turbine accepts an MPI hosts file and number of +processes, just like mpiexec.

+

A shorter, equivalent form of that command sequence is:

+
+
+
swift-t -t:f hosts.txt -n 4 mtc1.swift
+
+

On a PBS system, run with:

+
+
+
swift-t -m pbs -n 4 mtc1.swift
+
+

Many other systems are supported!

+
+
+
+

4. Reductions

+

A simplified version of the MapReduce model is to just compute many things and assemble them together at the end.

This script splits itself into lines, then reassembles the original @@ -845,24 +871,21 @@ "/bin/echo" s @stdout=o; } -main +string lines[] = file_lines(input("mtc2.swift")); +file fragments[]; +foreach line,i in lines { - string lines[] = file_lines(input("mtc2.swift")); - file fragments[]; - foreach line,i in lines - { - file y <sprintf("out-%i.txt",i)> = g(line); - fragments[i] = y; - } - file result <"assembled.txt"> = cat(fragments); -} + file y <sprintf("out-%i.txt",i)> = g(line); + fragments[i] = y; +} +file result <"assembled.txt"> = cat(fragments);

Note that leading whitespace is trimmed by file_lines(), and cat() is part of the Swift/T standard library in module unix.

-

4. Recursion

+

5. Recursion

This script computes the given Fibonacci number:

@@ -887,11 +910,8 @@ } } -main -{ - int n = toint(argv("n")); - trace(fib(n)); -} +int n = toint(argv("n")); +trace(fib(n));

Run it as:

File: fib/run.sh

@@ -938,27 +958,69 @@ } } -main -{ - file result <"sorted.txt"> = merge(0,7); -} +file result <"sorted.txt"> = merge(0,7);

This code runs the sort invocations concurrently, limited only by available processors and data dependencies.

-

5. Python and Numpy

+

6. Python and Numpy

See this section for information about calling Python or Numpy: Swift/T Guide: Python

+
+

7. Static executables

+
+

This section demonstrates a complete, concrete example of the +static executable feature. First, compose a +Swift script.

+

File: static-exec/hello.swift

+
+
+
import io;
+printf("HELLO");
+
+

Then, copy Turbine’s example manifest file and edit. +This manifest has all comments removed for simplicity.

+

File: static-exec/hello.manifest

+
+
+
pkg_name = hello
+pkg_version = 0.1
+
+main_script = hello.tic
+
+

Then, build:

+

File: static-exec/build.sh

+
+
+
#!/bin/bash
+set -eu
+
+# Obtain the Turbine build configuration variables
+TURBINE=/homes/wozniak/sfw/fusion/compute/turbine-static
+source ${TURBINE}/scripts/turbine-build-config.sh
+
+# Generate hello.tic
+stc hello.swift
+# Bundle hello.tic and Turbine into hello_main.c
+mkstatic.tcl hello.manifest -c hello_main.c
+
+# Compile hello_main.c and link as standalone, static executable
+CFLAGS=-std=gnu99
+gcc -c ${CFLAGS} ${TURBINE_INCLUDES} hello_main.c
+mpicc -static  -o hello.x hello_main.o ${TURBINE_LIBS} ${TURBINE_RPATH}
+
+
+

From wozniak at ci.uchicago.edu Thu Mar 5 13:45:44 2015 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Thu, 5 Mar 2015 13:45:44 -0600 (CST) Subject: [Swift-commit] r8386 - in www/Swift-T: . downloads Message-ID: <20150305194544.A178B9CD26@svn.ci.uchicago.edu> Author: wozniak Date: 2015-03-05 13:45:44 -0600 (Thu, 05 Mar 2015) New Revision: 8386 Added: www/Swift-T/downloads/gallery.tar.gz Modified: www/Swift-T/gallery.html Log: Gallery downloads Added: www/Swift-T/downloads/gallery.tar.gz =================================================================== (Binary files differ) Property changes on: www/Swift-T/downloads/gallery.tar.gz ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Modified: www/Swift-T/gallery.html =================================================================== --- www/Swift-T/gallery.html 2015-03-05 19:23:39 UTC (rev 8385) +++ www/Swift-T/gallery.html 2015-03-05 19:45:44 UTC (rev 8386) @@ -761,9 +761,18 @@ General documentation for Swift/T

+
  • +

    +Post to the ExM + user list with questions about these examples. +

    +
  • +
  • +

    +These example scripts may be downloaded here. +

    +
  • -

    Post to the ExM -user list with questions about these examples.

    @@ -779,7 +788,7 @@
    -

    2. Running shell commands

    +

    2. Swift/T for shell users

    This script converts itself to octal in mtc.octal.

    File: mtc/mtc1.swift

    @@ -819,7 +828,7 @@
    -

    3. Running many tasks on a cluster

    +

    3. Swift/T for cluster users

    It’s easy to launch these kinds of workloads on a cluster.

    If using a plain host list:

    @@ -853,7 +862,7 @@
    -

    4. Reductions

    +

    4. Swift/T for MapReduce users

    A simplified version of the MapReduce model is to just compute many things and assemble them together at the end.

    @@ -885,7 +894,7 @@
    -

    5. Recursion

    +

    5. Swift/T for recursive algorithms

    This script computes the given Fibonacci number:

    @@ -965,15 +974,65 @@
    -

    6. Python and Numpy

    +

    6. Swift/T for Python and Numpy users

    +

    Swift/T can run Python as an ordinary external program or via a +bundled interpreter! You can load Python packages, including +Python-wrapped native code- just set PYTHONPATH and import what +you need.

    See this section for information about calling Python or Numpy: Swift/T Guide: Python

    -

    7. Static executables

    +

    7. Swift/T for Tcl users

    +

    Swift/T is a great way to parallelize Tcl applications. You can run +tclsh as an ordinary external program, or use the bundled Tcl +interpreter! (Swift/T always has a Tcl interpreter for basic +operation.) You can load Tcl packages, including Tcl-wrapped native +code- just set SWIFT_PATH and package require what you need.

    +

    File: swift-tcl/tcl.swift

    +
    +
    +
    import io;
    +
    +(int o) add(int i, int j) "turbine" "0.0"
    +[
    +"""
    +set i <<i>>
    +set j <<j>>
    +set o [ expr $i + $j ]
    +puts "tcl: o=$o"
    +set <<o>> $o
    +"""
    +];
    +
    +i = 3;
    +j = 4;
    +o = add(i,j);
    +printf("o should be: %i", i+j);
    +printf("o is: %i", o);
    +
    +

    Run this with:

    +
    +
    +
    swift-t -p gallery/swift-tcl/tcl.swift
    +
    +

    (swift-t -p turns off the C preprocessor and allows the triple-quote +syntax.)

    +

    It outputs:

    +
    +
    +
    o should be: 7
    +tcl: o=7
    +o is: 7
    +
    +
    +
    +
    +

    8. Static executables

    +

    This section demonstrates a complete, concrete example of the static executable feature. First, compose a Swift script.

    @@ -1020,7 +1079,7 @@

    From wozniak at ci.uchicago.edu Thu Mar 5 13:47:27 2015 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Thu, 5 Mar 2015 13:47:27 -0600 (CST) Subject: [Swift-commit] r8387 - www/Swift-T Message-ID: <20150305194727.23C349CD26@svn.ci.uchicago.edu> Author: wozniak Date: 2015-03-05 13:47:27 -0600 (Thu, 05 Mar 2015) New Revision: 8387 Modified: www/Swift-T/gallery.html Log: Fix path Modified: www/Swift-T/gallery.html =================================================================== --- www/Swift-T/gallery.html 2015-03-05 19:45:44 UTC (rev 8386) +++ www/Swift-T/gallery.html 2015-03-05 19:47:27 UTC (rev 8387) @@ -769,7 +769,8 @@
  • -These example scripts may be downloaded here. +These example scripts may be downloaded + here (2KB).

  • @@ -1079,7 +1080,7 @@

    From wozniak at ci.uchicago.edu Thu Mar 5 13:59:49 2015 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Thu, 5 Mar 2015 13:59:49 -0600 (CST) Subject: [Swift-commit] r8388 - in www/Swift-T: . downloads Message-ID: <20150305195949.8C08A9CD26@svn.ci.uchicago.edu> Author: wozniak Date: 2015-03-05 13:59:49 -0600 (Thu, 05 Mar 2015) New Revision: 8388 Modified: www/Swift-T/downloads/gallery.tar.gz www/Swift-T/gallery.html Log: Updates Modified: www/Swift-T/downloads/gallery.tar.gz =================================================================== (Binary files differ) Modified: www/Swift-T/gallery.html =================================================================== --- www/Swift-T/gallery.html 2015-03-05 19:47:27 UTC (rev 8387) +++ www/Swift-T/gallery.html 2015-03-05 19:59:49 UTC (rev 8388) @@ -998,6 +998,7 @@
    import io;
     
    + at dispatch=WORKER
     (int o) add(int i, int j) "turbine" "0.0"
     [
     """
    @@ -1029,12 +1030,16 @@
     tcl: o=7
     o is: 7
    +

    The @dispatch=WORKER annotation allows add() to run on any worker +process in the run. Multiple add() s, or other functions, can run +concurrently. Swift automatically maintains ordering by managing data +dependencies from function outputs to inputs.

    8. Static executables

    -

    This section demonstrates a complete, concrete example of the +

    This section demonstrates a complete, concrete example of the optional static executable feature. First, compose a Swift script.

    File: static-exec/hello.swift

    @@ -1080,7 +1085,7 @@

    From wozniak at ci.uchicago.edu Thu Mar 5 14:27:38 2015 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Thu, 5 Mar 2015 14:27:38 -0600 (CST) Subject: [Swift-commit] r8389 - in www/papers: . pdfs Message-ID: <20150305202738.508BA9DA44@svn.ci.uchicago.edu> Author: wozniak Date: 2015-03-05 14:27:38 -0600 (Thu, 05 Mar 2015) New Revision: 8389 Added: www/papers/pdfs/Swift_MPI-IO_2014.pdf www/papers/pdfs/Swift_NAMD_2014.pdf Modified: www/papers/index.php Log: New papers Modified: www/papers/index.php =================================================================== --- www/papers/index.php 2015-03-05 19:59:49 UTC (rev 8388) +++ www/papers/index.php 2015-03-05 20:27:38 UTC (rev 8389) @@ -30,7 +30,7 @@ -
    @@ -255,7 +280,7 @@ Parallel High-resolution Climate Data Analysis using Swift. 4th Workshop on Many-Task Computing on Grids and Supercomputers 2011. - [ pdf ] + [ pdf ]
    @@ -287,7 +312,7 @@ Non-Rigid Registration for Image-Guided Neurosurgery on the TeraGrid: A Case Study College of William and Mary Technical Report 2009. - [ pdf ] + [ pdf ]
    @@ -295,7 +320,7 @@ Accelerating Medical Research using the Swift Workflow System Health Grid 2007. - [ pdf ] + [ pdf ]
    @@ -303,7 +328,7 @@ Accelerating solution of a moral hazard problem with Swift e-Social Science 2007. - [ pdf ] + [ pdf ]

    Virtual Data Language Applications

    @@ -312,7 +337,7 @@ Automating Climate Science: Large Ensemble Simulations on the TeraGrid with the GriPhyN Virtual Data System. 2nd IEEE International Conference on eScience and Grid Computing, 2006. - [ pdf ] + [ pdf ]
    @@ -327,7 +352,7 @@ Using Multiple Grid Resources for Bioinformatics Applications in GADU. IEEE/ACM International Symposium on Cluster Computing and Grid, 2006. - [ pdf ] + [ pdf ]
    @@ -335,7 +360,7 @@ GNARE: An Environment for Grid-Based High-Throughput Genome Analysis. Journal of Clinical Monitoring and Computing. 2005. - [ pdf ] + [ pdf ]
    @@ -352,7 +377,7 @@ Virtual Data in CMS Analysis. Computing in High Energy and Nuclear Physics, 2003. - [ pdf ] + [ pdf ]
    @@ -360,7 +385,7 @@ Virtual Data in CMS Production. Computing in High Energy and Nuclear Physics, 2003. - [ pdf ] + [ pdf ]
    @@ -394,7 +419,7 @@ Applying the Virtual Data Provenance Model. International Provenance and Annotation Workshop, Chicago, Illinois, 2006. - [ pdf ] + [ pdf ]
    @@ -402,7 +427,7 @@ Kickstarting Remote Applications. 2nd International Workshop on Grid Computing Environments, 2006. - [ pdf ] + [ pdf ]
    @@ -411,7 +436,7 @@ SIGMOD Record 34 (3) 37-43 2005. - [ pdf ] + [ pdf ]
    @@ -419,7 +444,7 @@ XDTM: XML Data Type and Mapping for Specifying Datasets. European Grid Conference, 2005. - [ pdf ] + [ pdf ]
    @@ -427,7 +452,7 @@ Grid Middleware Services for Virtual Data Discovery, Composition, and Integration. 2nd International Workshop on Middleware for Grid Computing, 2004. - [ pdf ] + [ pdf ]
    @@ -444,7 +469,7 @@ Chimera: A Virtual Data System for Representing, Querying, and Automating Data Derivation. 14th Intl. Conf. on Scientific and Statistical Database Management, Edinburgh, Scotland, 2002. - [ pdf ] + [ pdf ]
    @@ -460,7 +485,7 @@ Concurrency and Computation: Practice and Experience, 18 (6), 595-608. 2000. - [ pdf ] + [ pdf ]

     

    @@ -479,7 +504,7 @@ The First Provenance Challenge, Concurrency and Computation: Practice and Experience. 2008. - [ pdf ] + [ pdf ]
    @@ -487,7 +512,7 @@ Falkon: a Fast and Light-weight tasK executiON framework Supercomputing Conference 2007. - [ pdf ] + [ pdf ]
    @@ -504,7 +529,7 @@ An Opportunistic Algorithm for Scheduling Workflows on Grids. VECPAR'06, Rio De Janiero, 2006. - [ pdf ] + [ pdf ]
    @@ -512,7 +537,7 @@ A Tool for Prioritizing DAGMan Jobs and Its Evaluation. IEEE International Symposium on High Performance Distributed Computing, 2006. - [ pdf ] + [ pdf ]

     

    @@ -522,13 +547,13 @@ - +
    - + - + - +
    @@ -1066,13 +1094,14 @@

    Then, we bring the Tcl package into Swift (test-g-1.swift):

    -
    @dispatch=WORKER
    +
    import io;
    +
    + at dispatch=WORKER
     (int sum) g(int i1, int i2) "g" "0.0"
     [ "set <<sum>> [ g <<i1>> <<i2>> ]" ];
     
    -main {
    -  int sum = g(2, 3);
    -}
    +int sum = g(2, 3); +printf("Swift: sum: %i", sum);

    Compile and run (test-g-1.sh):

    @@ -1093,15 +1122,21 @@

    Write a foreach loop around the call to g (test-g-n.swift):

    -
    @dispatch=WORKER
    +
    import io;
    +import stats;
    +
    +
    + at dispatch=WORKER
     (int sum) g(int i1, int i2) "g" "0.0"
    -[ "set <<sum>> [ g <<i1>> <<i2>> ]" ];
    +  [ "set <<sum>> [ g <<i1>> <<i2>> ]" ];
     
    -main {
    -  foreach i in [0:5] {
    -    int sum = g(i, 5-i);
    -  }
    -}
    +int d[]; +foreach i in [0:5] { + d[i] = g(i, 5-i); +} + +y = sum_integer(d); +printf("y: %i", y);

    Compile and run (test-g-n.swift):

    @@ -1168,14 +1203,12 @@
    import blob;
     import io;
     
    -main {
    -  file data = input_file("input.data");
    -  blob b = blob_read(data);
    -  float v[] = floats_from_blob(b);
    -  printf("size(v) = %i", size(v));
    -  printf("v[0]=%0.2f", v[0]);
    -  printf("v[last]=%0.2f", v[size(v)-1]);
    -}
    +file data = input_file("input.data"); +blob b = blob_read(data); +float v[] = floats_from_blob(b); +printf("size(v) = %i", size(v)); +printf("v[0]=%0.2f", v[0]); +printf("v[last]=%0.2f", v[size(v)-1]);

    test-b-simple.sh:

    @@ -1239,9 +1272,9 @@
    rm *.o
    -swig -module b b.h
    +swig -module b b.i
     gcc -c -fPIC b.c
    -gcc -c -fPIC -I /usr/include/tcl8.5 b_wrap.c
    +gcc -c -fPIC $TCL_INCLUDE_SPEC b_wrap.c
     gcc -shared -o libb.so b_wrap.o b.o
     tclsh make-package.tcl > pkgIndex.tcl
    @@ -1293,13 +1326,11 @@ (blob sum) b(blob v) "b" "0.0" [ "set <<sum>> [ b::b_tcl <<v>> ]" ]; -main { - file data = input_file("input.data"); - blob v = blob_read(data); - blob s = b(v); - float sum[] = floats_from_blob(s); - printf("sum (swift): %f", sum[0]); -} +file data = input_file("input.data"); +blob v = blob_read(data); +blob s = b(v); +float sum[] = floats_from_blob(s); +printf("sum (swift): %f", sum[0]);

    test-b-swift.sh:

    @@ -1321,6 +1352,8 @@

    This example will demonstrate multiplying y=A*x using Swift to call a user Fortran library, which in turn calls the BLAS function dgemv. This demonstrates the generality of our model.

    +

    In these scripts, you must have the BLAS archive. Change the +appropriate shell variable to point to this file.

    leaf__1.png @@ -1383,18 +1416,17 @@

    It is built and run with (test-mvm.sh):

    -
    #!/bin/bash
    +
    #!/bin/bash -eu
     
     # Note Fortran memory layout:
     turbine-write-doubles A.data 1 3 2 4
     turbine-write-doubles x.data 5 6
     
    -BLAS="$HOME/Downloads/BLAS/blas_LINUX.a"
    -gfortran -c MVM.f &&
    -gfortran -c test-MVM.f &&
    -gfortran -o test-MVM.x test-MVM.o MVM.o $BLAS || exit 1
    +gfortran -c mvm.f
    +gfortran -c test-mvm.f
    +gfortran -o test-mvm.x test-mvm.o mvm.o ${BLAS}
     
    -./test-MVM.x
    +./test-mvm.x

    FortWrap will scan mvm.f and produce the C++ files FortFuncs.h and FortFuncs.cpp, which we wrap with SWIG to produce a Tcl function @@ -1413,15 +1445,13 @@ (blob y) mvm_blob(blob A, blob x, int n) "mvm" "0.0" [ "set <<y>> [ mvm::mvm <<A>> <<x>> <<n>> ]" ]; -main { - int n = 2; - blob A_blob = blob_read(input_file("A.data")); - blob x_blob = blob_read(input_file("x.data")); - blob y_blob = mvm_blob(A_blob, x_blob, n); - float y[] = floats_from_blob(y_blob); - foreach v, i in y { - printf("y[%i]=%f", i, v); - } +int n = 2; +blob A_blob = blob_read(input_file("A.data")); +blob x_blob = blob_read(input_file("x.data")); +blob y_blob = mvm_blob(A_blob, x_blob, n); +float y[] = floats_from_blob(y_blob); +foreach v, i in y { + printf("y[%i]=%f", i, v); }

    Our Tcl wrapper converts the Tcl call to mvm on blob arguments @@ -1433,17 +1463,12 @@

    Now, we build everything (build.sh):

    -
    #!/bin/bash -e
    +
    #!/bin/bash -eu
     
     TURBINE=$( which turbine )
     TURBINE_HOME=$( dirname $( dirname ${TURBINE} ) )
     source ${TURBINE_HOME}/scripts/turbine-config.sh
     
    -set -x
    -
    -BLAS="$HOME/Downloads/BLAS/blas_LINUX.a"
    -TCL_INCLUDE=${TCL}/include
    -
     # Wrap the Fortran in C++
     fortwrap.py --array-as-ptr --no-vector --no-fmat mvm.f
     # Wrap the C++ in Tcl
    @@ -1453,7 +1478,7 @@
     
     # Compile everything
     g++      -c -fPIC -I . FortFuncs.cpp
    -g++      -c -fPIC -I ${TCL_INCLUDE} FortFuncs_wrap.cxx
    +g++      -c -fPIC ${TCL_INCLUDE_SPEC} FortFuncs_wrap.cxx
     gfortran -c -fPIC mvm.f
     
     # Build the shared object
    @@ -1465,12 +1490,9 @@
     

    We run it in Swift with (test-mvm-swift.sh):

    -
    #!/bin/bash
    +
    #!/bin/sh
     
    -stc test-mvm.swift test-mvm.tcl
    -
    -export TURBINE_USER_LIB=$PWD
    -turbine test-mvm.tcl
    +swift-t -r $PWD test-mvm.swift

    This produces:

    @@ -1487,52 +1509,61 @@ case that the Swift program will construct an array of strings to pass to the C code. The C program is minimally modified by renaming its main() function and calling that from Swift/T.

    -

    Consider the user code (swift-main.c):

    +

    Consider the user code (main.c):

    #include <stdio.h>
    -#include "swift-main.h"
    -int swift_main(int argc, char* argv[]) {
    +#include "main.h"
    +int main(int argc, char* argv[]) {
       for (int i = 0; i < argc; i++)
         printf("arg[%i]: %s\n", i, argv[i]);
       return 0;
     }
    -

    This is essentially a simple C program except that its main() function -has been renamed to swift_main(). Also, the header swift-main.h -has been created to allow the Swift/T framework to call this code. -The function essentially acts like /bin/echo, reporting its +

    The function essentially acts like /bin/echo, reporting its arguments.

    -

    An example C wrapper to call this function is as follows (prog-c.c):

    +

    Consider the user code (main_leaf.c):

    -
    #include "swift-main.h"
    -int main(int argc, char* argv[]) {
    -  swift_main(argc, argv);
    +
    #include <stdio.h>
    +#include "main.h"
    +int main_leaf(int argc, char** argv) {
    +  for (int i = 0; i < argc; i++)
    +    printf("arg[%i]: %s\n", i, argv[i]);
       return 0;
     }
    -

    The equivalent Swift code is as follows (prog-swift.swift):

    +

    This is essentially the same program except that its main() function +has been renamed to swift_main(). Also, the header swift-main.h +has been created to allow the Swift/T framework to call this code.

    +

    The Swift/T distribution comes with functionality to make this easy to +call. The key program is genleaf.

    +

    The equivalent Swift code is as follows (test-main.swift):

    import io;
    -(int v) swift_main(string A[]) "swift_main" "0.0" "swift_main_wrap";
    -main {
    -  string A[] = [ "arg1", "arg2", "arg3" ];
    -  rc = swift_main(A);
    -  printf("exit code: %i", rc);
    -}
    + +mainapp; + +printf("Swift..."); +string A[] = [ "arg1", "arg2", "arg3" ]; +rc = main_leaf(A); +printf("exit code: %i", rc);

    The Swift version defines the extension function symbol swift_main, then calls it with the given array of strings. The "exit code" (actually just a return value) is available to Swift.

    This example may be compiled and run (from C, Tcl, and Swift) by using the provided Makefile.

    -

    The above process is semi-automated by the genleaf script. The script takes a C program and a header file as input and produces the required Tcl-C extension and Tcl leaf function. It also produces a simple source Swift and stc-compiled Tcl code ready to be run via turbine. Invoke genleaf as follows:

    +

    The above process is semi-automated by the genleaf script. The +script takes a C program and a header file as input and produces the +required Tcl-C extension and Tcl leaf function. It also produces a +simple source Swift and stc-compiled Tcl code ready to be run via +turbine. Invoke genleaf as follows (test-main.sh):

    -
    ./genleaf ex2.c ex2.h
    -turbine user-code.tcl
    +
    genleaf -vv main.c main.h test-main.swift
    +swift-t -r $PWD user-code.swift
    @@ -1614,8 +1645,8 @@

    -
    export TURBINE_USER_LIB=$PWD
    -turbine prog-swift.tcl
    +
    make
    +swift-t -r $PWD prog-swift.swift
    @@ -1691,12 +1722,10 @@ (int o) f(int i1, int i2) "my_pkg" "0.0" "f"; -main { - int x = 2; - int y = 3; - int z = f(x,y); - printf("sum: %i", z); -}
    +int x = 2; +int y = 3; +int z = f(x,y); +printf("sum: %i", z);

    In the declaration, @par allows us to call the function as a parallel function. At call time, we use @par=2.

    @@ -1773,7 +1801,7 @@ set k_value [ retrieve_integer $k ] # Look up MPI information set comm [ turbine::c::task_comm ] - set rank [ turbine::c::task_rank ] + set rank [ adlb::rank $comm ] # Run the user code set z_value [ f $comm $k_value ] # Store result @@ -1833,13 +1861,10 @@ import io; import sys; -main -{ - printf("args: %s", args()); - string s = "MY_OUTPUT"; - blob b = blob_from_string(s); - turbine_run_output_blob(b); -} +printf("args: %s", args()); +string s = "MY_OUTPUT"; +blob b = blob_from_string(s); +turbine_run_output_blob(b);

    The controller MPI program is run as: (run.sh):

    @@ -1866,7 +1891,7 @@

    From wozniak at ci.uchicago.edu Mon Mar 16 11:51:37 2015 From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu) Date: Mon, 16 Mar 2015 11:51:37 -0500 (CDT) Subject: [Swift-commit] r8394 - www/Swift-T Message-ID: <20150316165137.1047F9D670@svn.ci.uchicago.edu> Author: wozniak Date: 2015-03-16 11:51:36 -0500 (Mon, 16 Mar 2015) New Revision: 8394 Modified: www/Swift-T/gallery.html www/Swift-T/guide.html Log: Gallery updates Modified: www/Swift-T/gallery.html =================================================================== --- www/Swift-T/gallery.html 2015-03-13 20:03:05 UTC (rev 8393) +++ www/Swift-T/gallery.html 2015-03-16 16:51:36 UTC (rev 8394) @@ -791,6 +791,58 @@

    2. Swift/T for shell users

    +

    Swift/T has a powerful shell interface in its +app function syntax. Here are some +advanced examples:

    +

    To pass a whole command line into a generic app function, use:

    +

    File: sh-c/sh-1.swift

    +
    +
    +
    import string;
    +app f(string commandline)
    +{
    +  "sh" "-c" commandline;
    +}
    +tokens = ["/bin/echo","this","is","my","message"];
    +f(string_join(tokens," "));
    +
    +

    producing:

    +
    +
    +
    this is my message
    +
    +

    Programs that are found in different locations on different machines +can be accessed like this:

    +

    File: sh-c/sh-2.swift

    +
    +
    +
    import string;
    +import sys;
    +
    +// Program configuration
    +string program;
    +if (getenv("HOST") == "umbra")
    +{
    +  program = "/bin/echo";
    +}
    +else
    +{
    +  // something else
    +}
    +// End program configuration
    +
    +app f(string arguments)
    +{
    +  program arguments;
    +}
    +tokens = ["this","is","my","message"];
    +f(string_join(tokens," "));
    +
    +

    You can put the "program configuration" section in a separate file and +import it.

    +

    If you prefer, you could also put separate definitions of program in +separate files and conditionally #include +them with STC support for the C preprocessor.

    This script converts itself to octal in mtc.octal.

    File: mtc/mtc1.swift

    @@ -852,7 +904,7 @@

    A shorter, equivalent form of that command sequence is:

    -
    swift-t -t:f hosts.txt -n 4 mtc1.swift
    +
    swift-t -t f:hosts.txt -n 4 mtc1.swift

    On a PBS system, run with:

    @@ -881,7 +933,7 @@ "/bin/echo" s @stdout=o; } -string lines[] = file_lines(input("mtc2.swift")); +string lines[] = file_lines(input("mtc3.swift")); file fragments[]; foreach line,i in lines { @@ -1085,7 +1137,7 @@

    Modified: www/Swift-T/guide.html =================================================================== --- www/Swift-T/guide.html 2015-03-13 20:03:05 UTC (rev 8393) +++ www/Swift-T/guide.html 2015-03-16 16:51:36 UTC (rev 8394) @@ -744,7 +744,7 @@