[Swift-commit] r6840 - in SwiftApps/Swift-MapRed: . swiftreduce

[ketan at ci.uchicago.edu]

Author: ketan
Date: 2013-08-13 16:03:53 -0500 (Tue, 13 Aug 2013)
New Revision: 6840

Added:
   SwiftApps/Swift-MapRed/swiftreduce/
   SwiftApps/Swift-MapRed/swiftreduce/combiner.sh
   SwiftApps/Swift-MapRed/swiftreduce/teragen.swift
   SwiftApps/Swift-MapRed/swiftreduce/teragen_wrap.sh
Log:
swift reduce

Added: SwiftApps/Swift-MapRed/swiftreduce/combiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/swiftreduce/combiner.sh	                        (rev 0)
+++ SwiftApps/Swift-MapRed/swiftreduce/combiner.sh	2013-08-13 21:03:53 UTC (rev 6840)
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+FILES=$*
+SUM=0
+COUNT=0
+
+for file in $*
+do 
+    RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+    echo "${RES[0]} ${RES[1]}"
+    SUM=$(($SUM+${RES[0]}))
+    COUNT=$(($COUNT+${RES[1]}))
+done
+echo "SUM  : $SUM"
+echo "COUNT: $COUNT"
+exit 0
+


Property changes on: SwiftApps/Swift-MapRed/swiftreduce/combiner.sh
___________________________________________________________________
Added: svn:executable
   + *

Added: SwiftApps/Swift-MapRed/swiftreduce/teragen.swift
===================================================================
--- SwiftApps/Swift-MapRed/swiftreduce/teragen.swift	                        (rev 0)
+++ SwiftApps/Swift-MapRed/swiftreduce/teragen.swift	2013-08-13 21:03:53 UTC (rev 6840)
@@ -0,0 +1,46 @@
+import files;
+import io;
+import random;
+import string;
+import sys;
+
+app (file out, file err) gen_data (file run, int recsize){
+    "/bin/bash" run recsize @stdout=out @stderr=err
+}
+
+app (file out, file err) comb_data (file comb, file array[]){
+    "/bin/bash" comb array @stdout=out @stderr=err
+}
+
+/*
+file tgen_out[] <simple_mapper; prefix="tgen", suffix=".out">;
+file tgen_err[] <simple_mapper; prefix="tgen", suffix=".err">;
+string dir = @arg("dir", "./");
+*/
+main{
+file wrapper = input_file("teragen_wrap.sh");
+file tgen_out[];
+file tgen_err[];
+
+int loop = 2;
+int fsize = 10;
+
+foreach item,i in [0:loop-1] {
+    
+    file out <sprintf("tgenout-%i.dat", i)>;
+    file err <sprintf("tgenerr-%i.dat", i)>;
+
+	(out, err) = gen_data(wrapper, fsize);
+    tgen_out[i]=out;
+    tgen_err[i]=err;
+
+}
+
+file combine =input_file("combiner.sh");
+file final <"final_result">;
+file errs <"err_file">;
+
+(final, errs) = comb_data(combine, tgen_out);
+
+}
+

Added: SwiftApps/Swift-MapRed/swiftreduce/teragen_wrap.sh
===================================================================
--- SwiftApps/Swift-MapRed/swiftreduce/teragen_wrap.sh	                        (rev 0)
+++ SwiftApps/Swift-MapRed/swiftreduce/teragen_wrap.sh	2013-08-13 21:03:53 UTC (rev 6840)
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+
+# By default with ARG1:100 and SLICESIZE=10000, this script will generate
+# 10^6 records.  
+ARG1=1
+[ ! -z $1 ] && ARG1=$1
+
+FILE="input_$RANDOM.txt"
+LOWERLIMIT=0
+UPPERLIMIT=1000000000 # 10^9
+SLICESIZE=10000     # 10^4 records padded to 100B would result in 1MB file
+#SLICESIZE=1000     # 10^3  If padded to 100B would result 
+
+shuf -i $LOWERLIMIT-$UPPERLIMIT -n $(($SLICESIZE*$ARG1)) | awk '{printf "%-99s\n", $0}'
+exit 0
\ No newline at end of file


Property changes on: SwiftApps/Swift-MapRed/swiftreduce/teragen_wrap.sh
___________________________________________________________________
Added: svn:executable
   + *