[Swift-commit] r7370 - SwiftApps/Swift-MapRed/mapred_combiner_V4

[ketan at ci.uchicago.edu]

Author: ketan
Date: 2013-12-04 10:01:43 -0600 (Wed, 04 Dec 2013)
New Revision: 7370

Modified:
   SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
   SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh
   SwiftApps/Swift-MapRed/mapred_combiner_V4/reduce_wrapper.sh
Log:
username

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift	2013-12-04 02:33:02 UTC (rev 7369)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/blastreduce.swift	2013-12-04 16:01:43 UTC (rev 7370)
@@ -15,14 +15,15 @@
   seqfile psq;
 }
 
-app (database out) formatdb (fastaseq i){
- formatdb "-i" filename(i);
-}
 
 app (fastaseq out[]) split_database (fastaseq d, string n){
  fastasplitn filename(d) n;
 }
 
+app (database out) formatdb (fastaseq i){
+ formatdb "-i" filename(i);
+}
+
 app (fileptr out, file err) map_data (
 script wrapper,
 script map,
@@ -58,7 +59,7 @@
  loc_bash @wrapper @reducer cmd @site_results stdout=@out stderr=@err;
 }
 
-string num_partitions = arg("n", "200");
+string num_partitions = arg("n", "10");
 fastaseq partition[] <ext;exec = "examples/blast/splitmapper.sh", n = num_partitions>;
 database formatdbout[] <ext; exec = "examples/blast/formatdbmapper.sh", n = num_partitions>;
 

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh	2013-12-04 02:33:02 UTC (rev 7369)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/map_wrapper.sh	2013-12-04 16:01:43 UTC (rev 7370)
@@ -2,26 +2,29 @@
 
 # By default with ARG1:100 and SLICESIZE=10000, this script will generate
 # 10^6 records.
+
 if [ ! -z $1 ] && [ -f $1 ]; then
     USER_MAP_SCRIPT=$1;
     shift 1;
 fi
 MAP_ARGS=$*
-
+BEAGLE_USERNAME=ketan
+MIDWAY_USERNAME=maheshwari
 FILE="input_$RANDOM.txt"
 HOSTNAME=$(hostname -f)
 if   [[ "$HOSTNAME" == *midway* ]]; then # On midway node
-    EXECSERVER=/scratch/midway/yadunand/bin/exec_server
+    EXECSERVER=/scratch/midway/$MIDWAY_USERNAME/bin/exec_server
     LOGFOLDER=/scratch/midway/$USER
-    FOLDER="/dev/shm"
+    #FOLDER="/dev/shm"
+    FOLDER="/scratch/midway/$MIDWAY_USERNAME/mapdir"
     HOSTNAME="midway_$HOSTNAME"
 
 elif [[ "$HOSTNAME" == *nid* ]]; then    # On beagle node
-    EXECSERVER=/lustre/beagle/yadunandb/bin/exec_server
+    EXECSERVER=/lustre/beagle/$BEAGLE_USERNAME/bin/exec_server
     LOGFOLDER=/lustre/beagle/$USER
-    FOLDER="/dev/shm"
+    #FOLDER="/dev/shm"
+    FOLDER="/lustre/beagle/$BEAGLE_USERNAME/mapdir"
     HOSTNAME="beagle_$HOSTNAME"
-
 fi
 
 TIMEOUT=1200
@@ -34,6 +37,7 @@
     if [ "$?" == "0" ]
     then
 	    killall -u $USER chirp_server;
+        echo "first chirp server"
 	    echo "unix:$USER rwlds" >  $FOLDER/acl.conf
 	    echo "hostname:* rwl"   >> $FOLDER/acl.conf
         which chirp_server 1>&2

Modified: SwiftApps/Swift-MapRed/mapred_combiner_V4/reduce_wrapper.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_combiner_V4/reduce_wrapper.sh	2013-12-04 02:33:02 UTC (rev 7369)
+++ SwiftApps/Swift-MapRed/mapred_combiner_V4/reduce_wrapper.sh	2013-12-04 16:01:43 UTC (rev 7370)
@@ -116,7 +116,8 @@
 reduce ${FILES[*]} > $RESULT
 echo -e "$SITE"_"$(hostname -f) $RESULT"
 EOF
-        chmod a+x $Node.sh;
+
+        chmod a+x $Node.sh
 ############# END LOCAL COMBINER #####################
         #TODO: Move client with swift ?
 
@@ -139,22 +140,27 @@
     "naive")
         echo "Using naive reduce" 1>&2
         ./$USER_REDUCE_SCRIPT ${FILES[*]}
+        RC=$?
         ;;
     "plain")
         echo "Using plain reduce" 1>&2
         echo "plain $ID ${FILES[*]}" 1>&2
         plain $ID ${FILES[*]}
+        RC=$?
         ;;
     "local")
         echo "Using local combiners" 1>&2
         local_combine_fetch_plain $ID ${FILES[*]}
+        RC=$?
         ;;
     "local_k_way")
         echo "Using local combiners" 1>&2
         local_combine $ID ${FILES[*]}
+        RC=$?
         ;;
     *)
         echo "ERROR: Unknown reduction method requested" 1>&2
+        RC=$?
         ;;
 esac
 exit $RC