[Swift-commit] r7372 - SwiftApps/Swift-MapRed/mapred_combiner_V4

[ketan at ci.uchicago.edu]

Author: ketan
Date: 2013-12-04 10:40:13 -0600 (Wed, 04 Dec 2013)
New Revision: 7372

Modified:
   SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
   SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh
   SwiftApps/Swift-MapRed/mapred_combiner_V4/reduce_wrapper.sh
Log:
remove last stage for single site

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift	2013-12-04 16:30:38 UTC (rev 7371)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift	2013-12-04 16:40:13 UTC (rev 7372)
@@ -87,8 +87,8 @@
 file redsite_out[] <simple_mapper; prefix="reduce", suffix=".out">;
 file redsite_err[] <simple_mapper; prefix="reduce", suffix=".err">;
 (redsite_out[1], redsite_err[1]) = reduce_mid (red_wrapper, red_func, "local", all_map);
-// Final global reduce
-file final_out <"final_result">;
-file final_err <"final_errors">;
-(final_out, final_err) = reduce_local (red_wrapper, red_func, "local", redsite_out);
+// Final global reduce (Not required on a single site)
+//file final_out <"final_result">;
+//file final_err <"final_errors">;
+//(final_out, final_err) = reduce_local (red_wrapper, red_func, "naive", redsite_out);
 

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh	2013-12-04 16:30:38 UTC (rev 7371)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh	2013-12-04 16:40:13 UTC (rev 7372)
@@ -15,15 +15,15 @@
 if   [[ "$HOSTNAME" == *midway* ]]; then # On midway node
     EXECSERVER=/scratch/midway/$MIDWAY_USERNAME/bin/exec_server
     LOGFOLDER=/scratch/midway/$USER
-    #FOLDER="/dev/shm"
-    FOLDER="/scratch/midway/$MIDWAY_USERNAME/mapdir"
+    FOLDER="/dev/shm"
+    #FOLDER="/scratch/midway/$MIDWAY_USERNAME/mapdir"
     HOSTNAME="midway_$HOSTNAME"
 
 elif [[ "$HOSTNAME" == *nid* ]]; then    # On beagle node
     EXECSERVER=/lustre/beagle/$BEAGLE_USERNAME/bin/exec_server
     LOGFOLDER=/lustre/beagle/$USER
-    #FOLDER="/dev/shm"
-    FOLDER="/lustre/beagle/$BEAGLE_USERNAME/mapdir"
+    FOLDER="/dev/shm"
+    #FOLDER="/lustre/beagle/$BEAGLE_USERNAME/mapdir"
     HOSTNAME="beagle_$HOSTNAME"
 fi
 

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/reduce_wrapper.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/reduce_wrapper.sh	2013-12-04 16:30:38 UTC (rev 7371)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/reduce_wrapper.sh	2013-12-04 16:40:13 UTC (rev 7372)
@@ -132,6 +132,7 @@
     log "Finished local_combiner_fetch"
     ls $FOLDER/comb.$ID*imd2 1>&2
     ./$USER_REDUCE_SCRIPT $FOLDER/comb.$ID*imd2
+    RC=$?
     log "Finished running ./$USER_REDUCE_SCRIPT $FOLDER/comb.$ID*imd2"
 }