[Swift-commit] r7057 - in SwiftApps/Swift-MapRed: . Paper patterns patterns/k patterns/t
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Wed Sep 11 16:09:45 CDT 2013
Author: ketan
Date: 2013-09-11 16:09:45 -0500 (Wed, 11 Sep 2013)
New Revision: 7057
Added:
SwiftApps/Swift-MapRed/patterns/
SwiftApps/Swift-MapRed/patterns/k/
SwiftApps/Swift-MapRed/patterns/k/clean.sh
SwiftApps/Swift-MapRed/patterns/k/mimo.sh
SwiftApps/Swift-MapRed/patterns/k/mimo.swift
SwiftApps/Swift-MapRed/patterns/k/miso.sh
SwiftApps/Swift-MapRed/patterns/k/miso.swift
SwiftApps/Swift-MapRed/patterns/k/pattern_template.swift
SwiftApps/Swift-MapRed/patterns/k/simo.sh
SwiftApps/Swift-MapRed/patterns/k/simo.swift
SwiftApps/Swift-MapRed/patterns/k/siso.sh
SwiftApps/Swift-MapRed/patterns/k/siso.swift
SwiftApps/Swift-MapRed/patterns/t/
SwiftApps/Swift-MapRed/patterns/t/mimo.sh
SwiftApps/Swift-MapRed/patterns/t/mimo.swift
SwiftApps/Swift-MapRed/patterns/t/miso.sh
SwiftApps/Swift-MapRed/patterns/t/miso.swift
SwiftApps/Swift-MapRed/patterns/t/pattern_template.swift
SwiftApps/Swift-MapRed/patterns/t/simo.sh
SwiftApps/Swift-MapRed/patterns/t/simo.swift
SwiftApps/Swift-MapRed/patterns/t/siso.sh
SwiftApps/Swift-MapRed/patterns/t/siso.swift
Modified:
SwiftApps/Swift-MapRed/Paper/swifthadoop.tex
Log:
adding patterns
Modified: SwiftApps/Swift-MapRed/Paper/swifthadoop.tex
===================================================================
--- SwiftApps/Swift-MapRed/Paper/swifthadoop.tex 2013-09-11 18:51:41 UTC (rev 7056)
+++ SwiftApps/Swift-MapRed/Paper/swifthadoop.tex 2013-09-11 21:09:45 UTC (rev 7057)
@@ -475,7 +475,16 @@
-- Portability of solution
--- Parallel find is a nice example
+-- Parallel find is a nice example especially on distributed Chirp servers
+-- Experiment platforms:
+ + Cray systems: Raven, Blue Waters, Beagle
+ + Bluegene/p
+
+-- Example workflow apps:
+ + DSSAT : 120k jobs, real science, high priority
+ + Synthetic workflow patterns: SISO, SIMO, MISO, MIMO
+
+
What is HDFS?
HDFS, inspired by Google's GFS.%TODO: citation
Added: SwiftApps/Swift-MapRed/patterns/k/clean.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/clean.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/clean.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo "cleaning up ..."
+rm -rf outdir/ *.d *.log *.rlog logs hi.* *.kml *.swiftx _concurrent output
+
Property changes on: SwiftApps/Swift-MapRed/patterns/k/clean.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/patterns/k/mimo.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/mimo.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/mimo.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cp $1 $5
+cp $2 $6
+cp $3 $7
+cp $4 $8
+
Property changes on: SwiftApps/Swift-MapRed/patterns/k/mimo.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/patterns/k/mimo.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/mimo.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/mimo.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,23 @@
+type file;
+type script;
+
+app (file _outB1, file _outB2, file _outB3, file _outB4) A (script _s, file _inA1, file _inA2, file _inA3, file _inA4){
+ sh @_s @_inA1 @_inA2 @_inA3 @_inA4 @_outB1 @_outB2 @_outB3 @_outB4;
+}
+
+file ina1<"sample1.txt">;
+file ina2<"sample2.txt">;
+file ina3<"sample3.txt">;
+file ina4<"sample4.txt">;
+
+script a<"mimo.sh">;
+
+foreach i in [0:1]{
+ file outb1 <single_file_mapper; file=@strcat("output/outb1.",i,".txt")>;
+ file outb2 <single_file_mapper; file=@strcat("output/outb2.",i,".txt")>;
+ file outb3 <single_file_mapper; file=@strcat("output/outb3.",i,".txt")>;
+ file outb4 <single_file_mapper; file=@strcat("output/outb4.",i,".txt")>;
+
+ (outb1, outb2, outb3, outb4) = A (a, ina1, ina2, ina3, ina4);
+}
+
Added: SwiftApps/Swift-MapRed/patterns/k/miso.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/miso.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/miso.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cat $1 $2 $3 $4 > $5
+
Property changes on: SwiftApps/Swift-MapRed/patterns/k/miso.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/patterns/k/miso.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/miso.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/miso.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,20 @@
+type file;
+type script;
+
+app (file _res) B (script _s, file _inB1, file _inB2, file _inB3, file _inB4){
+ sh @_s @_inB1 @_inB2 @_inB3 @_inB4 @_res;
+}
+
+file inb1<"sample1.txt">;
+file inb2<"sample2.txt">;
+file inb3<"sample3.txt">;
+file inb4<"sample4.txt">;
+
+script b<"miso.sh">;
+
+foreach i in [0:1]{
+ file res <single_file_mapper; file=@strcat("output/res.",i,".txt")>;
+
+ res = B (b, inb1, inb2, inb3, inb4);
+}
+
Added: SwiftApps/Swift-MapRed/patterns/k/pattern_template.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/pattern_template.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/pattern_template.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,44 @@
+type file;
+type script;
+
+app (file _outB1, file _outB2, file _outB3, file _outB4) A (script _s, file _inA1){
+ @_s @_inA1 @_outB1 @_outB2 @_outB3 @_outB4;
+}
+
+app (file _outC1) B (script _s, file _inB1){
+ @_s @_inB1 @_outC1;
+}
+
+app (file _res) C (script _s, file _inC1, file _inC2, file _inC3, file _inC4){
+ @_s @_inC1 @_inC2 @_inC3 @_inC4 @_res;
+}
+
+file ina1<"sample.txt">;
+script a<"a.sh">;
+script b<"b.sh">;
+script c<"c.sh">;
+
+foreach i in [0:1]{
+
+ file outb1 <single_file_mapper; file=@strcat("output/outb1.",i,".txt")>;
+ file outb2 <single_file_mapper; file=@strcat("output/outb2.",i,".txt")>;
+ file outb3 <single_file_mapper; file=@strcat("output/outb3.",i,".txt")>;
+ file outb4 <single_file_mapper; file=@strcat("output/outb4.",i,".txt")>;
+
+ file outc1 <single_file_mapper; file=@strcat("output/outc1.",i,".txt")>;
+ file outc2 <single_file_mapper; file=@strcat("output/outc2.",i,".txt")>;
+ file outc3 <single_file_mapper; file=@strcat("output/outc3.",i,".txt")>;
+ file outc4 <single_file_mapper; file=@strcat("output/outc4.",i,".txt")>;
+
+ file res <single_file_mapper; file=@strcat("output/res.",i,".txt")>;
+
+ (outb1, outb2, outb3, outb4) = A (a, ina1);
+
+ outc1 = B (b, outb1);
+ outc2 = B (b, outb2);
+ outc3 = B (b, outb3);
+ outc4 = B (b, outb4);
+
+ res = C (c, outc1, outc2, outc3, outc4);
+}
+
Added: SwiftApps/Swift-MapRed/patterns/k/simo.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/simo.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/simo.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cp $1 $2
+cp $1 $3
+cp $1 $4
+cp $1 $5
+
Added: SwiftApps/Swift-MapRed/patterns/k/simo.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/simo.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/simo.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,21 @@
+type file;
+type script;
+
+app (file _outB1, file _outB2, file _outB3, file _outB4) A (script _s, file _inA1){
+ sh @_s @_inA1 @_outB1 @_outB2 @_outB3 @_outB4;
+}
+
+
+file ina1<"sample1.txt">;
+script a<"simo.sh">;
+
+foreach i in [0:1]{
+
+ file outb1 <single_file_mapper; file=@strcat("output/outb1.",i,".txt")>;
+ file outb2 <single_file_mapper; file=@strcat("output/outb2.",i,".txt")>;
+ file outb3 <single_file_mapper; file=@strcat("output/outb3.",i,".txt")>;
+ file outb4 <single_file_mapper; file=@strcat("output/outb4.",i,".txt")>;
+
+ (outb1, outb2, outb3, outb4) = A (a, ina1);
+}
+
Added: SwiftApps/Swift-MapRed/patterns/k/siso.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/siso.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/siso.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+cp $1 $2
Property changes on: SwiftApps/Swift-MapRed/patterns/k/siso.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/patterns/k/siso.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/k/siso.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/k/siso.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,18 @@
+type file;
+type script;
+
+app (file _outB1) A (script _s, file _inA1){
+ sh @_s @_inA1 @_outB1;
+}
+
+
+file ina1<"sample1.txt">;
+script a<"siso.sh">;
+
+foreach i in [0:1]{
+
+ file outb1 <single_file_mapper; file=@strcat("output/outb1.",i,".txt")>;
+
+ outb1 = A (a, ina1);
+}
+
Added: SwiftApps/Swift-MapRed/patterns/t/mimo.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/mimo.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/mimo.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cp $1 $5
+cp $2 $6
+cp $3 $7
+cp $4 $8
+
Property changes on: SwiftApps/Swift-MapRed/patterns/t/mimo.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/patterns/t/mimo.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/mimo.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/mimo.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,31 @@
+import files;
+import io;
+import string;
+import sys;
+
+app (file _outB1, file _outB2, file _outB3, file _outB4) A (file _inA1, file _inA2, file _inA3, file _inA4){
+ "./mimo.sh" @_inA1 @_inA2 @_inA3 @_inA4 @_outB1 @_outB2 @_outB3 @_outB4;
+ }
+
+main{
+
+file in_a1=input_file("sample1.txt");
+file in_a2=input_file("sample2.txt");
+file in_a3=input_file("sample3.txt");
+file in_a4=input_file("sample4.txt");
+
+foreach i in [0:1]{
+ string outb1=sprintf("outb1-%i.dat", i);
+ string outb2=sprintf("outb2-%i.dat", i);
+ string outb3=sprintf("outb3-%i.dat", i);
+ string outb4=sprintf("outb4-%i.dat", i);
+
+ file o1 <outb1>;
+ file o2 <outb2>;
+ file o3 <outb3>;
+ file o4 <outb4>;
+
+ (o1, o2, o3, o4) = A (in_a1, in_a2, in_a3, in_a4);
+
+}
+}
Added: SwiftApps/Swift-MapRed/patterns/t/miso.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/miso.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/miso.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cat $1 $2 $3 $4 > $5
+
Property changes on: SwiftApps/Swift-MapRed/patterns/t/miso.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/patterns/t/miso.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/miso.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/miso.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,20 @@
+import files;
+import io;
+import string;
+import sys;
+
+app (file _res) C (file _inC1, file _inC2, file _inC3, file _inC4){
+ "./miso.sh" @_inC1 @_inC2 @_inC3 @_inC4 @_res;
+}
+
+main{
+file in_a1=input_file("sample.txt");
+file in_a2=input_file("sample.txt");
+file in_a3=input_file("sample.txt");
+file in_a4=input_file("sample.txt");
+
+foreach i in [0:1]{
+ string res=sprintf("/tmp/swift.work/res-%i.dat", i);
+ file r <res> = C (in_a1, in_a2, in_a3, in_a4);
+}
+}
Added: SwiftApps/Swift-MapRed/patterns/t/pattern_template.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/pattern_template.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/pattern_template.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,43 @@
+import files;
+import io;
+import string;
+import sys;
+
+app (file _outB1, file _outB2, file _outB3, file _outB4) A (file _inA1){
+ "a.sh" @_inA1 @_outB1 @_outB2 @_outB3 @_outB4;
+}
+
+app (file _outC1) B (file _inB1){
+ "b.sh" @_inB1 @_outC1;
+}
+
+app (file _res) C (file _inC1, file _inC2, file _inC3, file _inC4){
+ "c.sh" @_inC1 @_inC2 @_inC3 @_inC4 @_res;
+}
+
+file in_a=input_file("sample.txt");
+
+foreach i in [0:1]{
+
+ string outb1=sprintf("/tmp/swift.work/outb1-%i.dat", i);
+ string outb2=sprintf("/tmp/swift.work/outb2-%i.dat", i);
+ string outb3=sprintf("/tmp/swift.work/outb3-%i.dat", i);
+ string outb4=sprintf("/tmp/swift.work/outb4-%i.dat", i);
+
+ string outc1=sprintf("/tmp/swift.work/outc1-%i.dat", i);
+ string outc2=sprintf("/tmp/swift.work/outc2-%i.dat", i);
+ string outc3=sprintf("/tmp/swift.work/outc3-%i.dat", i);
+ string outc4=sprintf("/tmp/swift.work/outc4-%i.dat", i);
+
+ string res=sprintf("/tmp/swift.work/res-%i.dat", i);
+
+ (outb1, outb2, outb3, outb4) = A (ina1);
+
+ outc1 = B (outb1);
+ outc2 = B (outb2);
+ outc3 = B (outb3);
+ outc4 = B (outb4);
+
+ res = C (outc1, outc2, outc3, outc4);
+}
+
Added: SwiftApps/Swift-MapRed/patterns/t/simo.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/simo.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/simo.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cp $1 $2
+cp $1 $3
+cp $1 $4
+cp $1 $5
+
Property changes on: SwiftApps/Swift-MapRed/patterns/t/simo.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/patterns/t/simo.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/simo.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/simo.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,28 @@
+import files;
+import io;
+import string;
+import sys;
+
+app (file _outB1, file _outB2, file _outB3, file _outB4) A (file _inA1){
+ "./simo.sh" @_inA1 @_outB1 @_outB2 @_outB3 @_outB4;
+}
+
+main
+{
+
+file in_a=input_file("sample.txt");
+
+foreach i in [0:1]{
+
+ string outb1=sprintf("outb1-%i.dat", i);
+ string outb2=sprintf("outb2-%i.dat", i);
+ string outb3=sprintf("outb3-%i.dat", i);
+ string outb4=sprintf("outb4-%i.dat", i);
+ file o1 <outb1>;
+ file o2 <outb2>;
+ file o3 <outb3>;
+ file o4 <outb4>;
+
+ (o1, o2, o3, o4) = A (in_a);
+}
+}
Added: SwiftApps/Swift-MapRed/patterns/t/siso.sh
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/siso.sh (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/siso.sh 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+cp $1 $2
Property changes on: SwiftApps/Swift-MapRed/patterns/t/siso.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/patterns/t/siso.swift
===================================================================
--- SwiftApps/Swift-MapRed/patterns/t/siso.swift (rev 0)
+++ SwiftApps/Swift-MapRed/patterns/t/siso.swift 2013-09-11 21:09:45 UTC (rev 7057)
@@ -0,0 +1,20 @@
+import files;
+import io;
+import string;
+import sys;
+
+app (file _outC1) B (file _inB1){
+ "./siso.sh" @_inB1 @_outC1;
+}
+
+main{
+
+file in_a = input_file("sample.txt");
+
+foreach i in [0:1]{
+
+ string res=sprintf("res-%i.dat", i);
+ file r <res> = B (in_a);
+}
+}
+
More information about the Swift-commit
mailing list