[Swift-commit] r7240 - SwiftApps/Swift-MapRed/mapred_combiner_V4

[ketan at ci.uchicago.edu]

Author: ketan
Date: 2013-10-28 15:55:41 -0500 (Mon, 28 Oct 2013)
New Revision: 7240

Modified:
   SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
   SwiftApps/Swift-MapRed/mapred_combiner_V4/clean.sh
   SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh
   SwiftApps/Swift-MapRed/mapred_combiner_V4/tc.data
Log:
 

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift	2013-10-28 19:09:37 UTC (rev 7239)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift	2013-10-28 20:55:41 UTC (rev 7240)
@@ -41,7 +41,7 @@
 }
 
 app (fileptr out) aggregate (script a, fileptr map_results[]) {
-  bash "-c" @a @map_results stdout=@out;
+  bash @a @map_results stdout=@out;
 }
 
 app (file out, file err) reduce_mid (script wrapper, script reducer, string cmd, fileptr all_map){
@@ -60,7 +60,7 @@
 script red_wrapper    <"reduce_wrapper.sh">;
 script aggr           <"aggregate.sh">;
 script map_func       <"examples/blast/map_blast.sh">;
-script red_func       <"examples/reduce_sort.sh">;
+script red_func       <"examples/blast/reduce_blast.sh">;
 
 fileptr map_out[] <simple_mapper; prefix="map", suffix=".out">;
 file  map_err[] <simple_mapper; prefix="map", suffix=".err">;
@@ -76,12 +76,12 @@
 }
 
 fileptr all_map <"all_map.out">;
-#all_map = aggregate(aggr, map_out);
+all_map = aggregate(aggr, map_out);
 file redsite_out[] <simple_mapper; prefix="reduce", suffix=".out">;
 file redsite_err[] <simple_mapper; prefix="reduce", suffix=".err">;
-#(redsite_out[1], redsite_err[1]) = reduce_mid (red_wrapper, red_func, "local", all_map);
+(redsite_out[1], redsite_err[1]) = reduce_mid (red_wrapper, red_func, "local", all_map);
 // Final global reduce
 file final_out <"final_result">;
 file final_err <"final_errors">;
-#(final_out, final_err) = reduce_local (red_wrapper, red_func, "naive", redsite_out);
+(final_out, final_err) = reduce_local (red_wrapper, red_func, "naive", redsite_out);
 

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/clean.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/clean.sh	2013-10-28 19:09:37 UTC (rev 7239)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/clean.sh	2013-10-28 20:55:41 UTC (rev 7240)
@@ -6,4 +6,5 @@
 rm *{out,err,imd2}                 &> /dev/null
 rm final_errors final_result       &> /dev/null
 rm exec_client exec_server         &> /dev/null
-rm -rf swiftreduce-*-* &> /dev/null
+rm -rf blastreduce-*-* swiftreduce-*-* &> /dev/null
+

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh	2013-10-28 19:09:37 UTC (rev 7239)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh	2013-10-28 20:55:41 UTC (rev 7240)
@@ -12,13 +12,13 @@
 HOSTNAME=$(hostname -f)
 if   [[ "$HOSTNAME" == *midway* ]]; then # On midway node
     EXECSERVER=/scratch/midway/yadunand/bin/exec_server
-    LOGFOLDER=/scratch/midway/yadunand
+    LOGFOLDER=/scratch/midway/$USER
     FOLDER="/dev/shm"
     HOSTNAME="midway_$HOSTNAME"
 
 elif [[ "$HOSTNAME" == *nid* ]]; then    # On beagle node
     EXECSERVER=/lustre/beagle/yadunandb/bin/exec_server
-    LOGFOLDER=/lustre/beagle/yadunandb/Swift-MapRed/mapred_combiner_V5
+    LOGFOLDER=/lustre/beagle/$USER
     FOLDER="/dev/shm"
     HOSTNAME="beagle_$HOSTNAME"
 
@@ -30,7 +30,7 @@
 JOBS_LEVEL=`echo $PWD | grep -o ".*jobs"`
 if [ "$?" == "0" ]
 then
-    mkdir $JOBS_LEVEL/CHIRPING
+    mkdir -p $JOBS_LEVEL/CHIRPING
     if [ "$?" == "0" ]
     then
 	    killall -u $USER chirp_server;

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/tc.data
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/tc.data	2013-10-28 19:09:37 UTC (rev 7239)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/tc.data	2013-10-28 20:55:41 UTC (rev 7240)
@@ -1,5 +1,6 @@
 midway bash /bin/bash
 midway mid_bash /bin/bash
+midway loc_bash /bin/bash
 beagle bgl_bash /bin/bash
 local  loc_bash /bin/bash
 local  cat /bin/cat