[Swift-commit] r6845 - in SwiftApps/Swift-MapRed: . TurboChirp mapred_chirp
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Tue Aug 13 16:59:30 CDT 2013
Author: yadunandb
Date: 2013-08-13 16:59:30 -0500 (Tue, 13 Aug 2013)
New Revision: 6845
Added:
SwiftApps/Swift-MapRed/TurboChirp/
SwiftApps/Swift-MapRed/TurboChirp/cleanup
SwiftApps/Swift-MapRed/TurboChirp/combiner.sh
SwiftApps/Swift-MapRed/TurboChirp/combiner2.sh
SwiftApps/Swift-MapRed/TurboChirp/hosts.txt
SwiftApps/Swift-MapRed/TurboChirp/recombiner.sh
SwiftApps/Swift-MapRed/TurboChirp/remote_combiner.sh
SwiftApps/Swift-MapRed/TurboChirp/result
SwiftApps/Swift-MapRed/TurboChirp/run.sh
SwiftApps/Swift-MapRed/TurboChirp/teragen.swift
SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl
SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh
SwiftApps/Swift-MapRed/TurboChirp/test
SwiftApps/Swift-MapRed/TurboChirp/turbo.swift
SwiftApps/Swift-MapRed/TurboChirp/wrap_test.sh
Modified:
SwiftApps/Swift-MapRed/mapred_chirp/LAB_RECORDS
SwiftApps/Swift-MapRed/mapred_chirp/anl.xml
SwiftApps/Swift-MapRed/mapred_chirp/err_file
SwiftApps/Swift-MapRed/mapred_chirp/final_result
SwiftApps/Swift-MapRed/mapred_chirp/run.sh
SwiftApps/Swift-MapRed/mapred_chirp/teragen_wrap.sh
Log:
TurboChirp
Added: SwiftApps/Swift-MapRed/TurboChirp/cleanup
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/cleanup (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/cleanup 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,3 @@
+rm -rfv *.dat
+rm *{imd2,imd}
+rm err_file final_result
\ No newline at end of file
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/cleanup
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/TurboChirp/combiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/combiner.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/combiner.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+FILES=$*
+SUM=0
+COUNT=0
+
+fetch_and_process()
+{
+ TOKEN=($(cat $1))
+ HOST=${TOKEN[0]}
+ FILE=${TOKEN[1]}
+ TARGET=$(basename $FILE)
+ chirp_get $HOST $TARGET $TARGET.imd
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $TARGET.imd))
+ echo "${RES[0]} ${RES[1]}" > "$2.$RANDOM.imd2"
+}
+
+ID=$RANDOM
+for file_token in $*
+do
+ fetch_and_process $file_token $ID &
+done
+wait
+
+SUM=$(awk '{ sum += $1 } END { print sum }' $ID*imd2)
+COUNT=$(awk '{ sum += $2 } END { print sum }' $ID*imd2)
+
+echo "SUM : $SUM"
+echo "COUNT: $COUNT"
+exit 0
+
+
+
+
+
+exit 0
+
+
+
+for file_token in $*
+do
+ TOKEN=($(cat $file_token))
+ HOST=${TOKEN[0]}
+ FILE=${TOKEN[1]}
+ TARGET=$(basename $FILE)
+ chirp_get $HOST $TARGET $TARGET.imd
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $TARGET.imd))
+ echo "$TARGET.imd"
+
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+done
+echo "SUM : $SUM"
+echo "COUNT: $COUNT"
+exit 0
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/combiner.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/TurboChirp/combiner2.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/combiner2.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/combiner2.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+FILES=$*
+SUM=0
+COUNT=0
+
+cat $* > RESULT_FILE
+# Find the unique sites in the RESULT_FILE to run a combiner there.
+#SITES=($(sort -u -t' ' -k1,1 RESULT_FILE))
+cat <<'EOF' > recombiner.sh
+#!/bin/bash
+#SITE=$1; shift 1
+echo "Recombiner on $HOSTNAME"
+FILES=$*
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+done;
+echo -e "$SUM $COUNT" > $HOSTNAME.imd
+EOF
+chmod 777 recombiner.sh
+
+STAGE2="";
+SITES=($(awk '{ print $1 }' RESULT_FILE | sort -u))
+for SITE in ${SITES[*]}
+do
+ echo $SITE
+ FILES=($(grep $SITE RESULT_FILE | awk '{ print $2 }'))
+ CHIRPDIR=$(dirname ${FILES[0]})
+ chirp_put ./recombiner.sh $SITE recombiner.sh
+# We will probably abandon parrot_run
+#parrot_run /chirp/$SITES/recombiner.sh ${FILES[*]}
+# ssh $SITE "cd $CHIRPDIR; ./recombiner.sh ${FILES[*]}"
+ ssh $SITE "cd /sandbox/$USER; ./recombiner.sh ${FILES[*]}"
+ chirp_get $SITE $SITE.imd $SITE.imd.txt
+ cat $SITE.imd.txt
+ STAGE2="$STAGE2 $SITE.imd.txt"
+done;
+
+cat ${STAGE2[*]} > TEMP
+SUM=($(awk '{ sum += $1 } END { print sum }' TEMP))
+COUNT=($(awk '{ sum += $2 } END { print sum }' TEMP))
+echo "Final results (on $HOSTNAME):"
+echo "$SUM $COUNT"
+
+exit 0;
+
+for file_token in $*
+do
+ TOKEN=($(cat $file_token))
+ HOST=${TOKEN[0]}
+ FILE=${TOKEN[1]}
+ TARGET=$(basename $FILE)
+ parrot_run /chirp/$HOST/FILE
+ chirp_get $HOST $TARGET $TARGET.imd
+ # TODO Try parrot_cp instead
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $TARGET.imd))
+
+
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+done
+
+echo "SUM : $SUM"
+echo "COUNT: $COUNT"
+
+
+exit 0
+
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/combiner2.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/TurboChirp/hosts.txt
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/hosts.txt (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/hosts.txt 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,7 @@
+thwomp
+stomp
+crush
+crank
+steamroller
+thrash
+vanquish
Added: SwiftApps/Swift-MapRed/TurboChirp/recombiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/recombiner.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/recombiner.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,11 @@
+#!/bin/bash
+#SITE=$1; shift 1
+FILES=$*
+SUM=0;COUNT=0;
+for file in ${FILES[*]}; do
+ [ ! -f $file ] && echo "$file is missing" >> $HOSTNAME.parrot_run.log
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+done;
+echo -e "$SUM $COUNT" > $HOSTNAME.imd
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/recombiner.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/TurboChirp/remote_combiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/remote_combiner.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/remote_combiner.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,4 @@
+#!/bin/bash
+SITE=$1; shift 1
+FILES=$*;
+echo $FILES
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/remote_combiner.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/TurboChirp/result
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/result (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/result 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1 @@
+crank /sandbox/yadunand/input_28748.txt
Added: SwiftApps/Swift-MapRed/TurboChirp/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/run.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/run.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,5 @@
+export TURBINE_LOG=0
+export TURBINE_LAUNCH_OPTS="-f ./hosts.txt"
+stc teragen.swift >teragen.tcl
+turbine -n 8 teragen.tcl
+
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/run.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/TurboChirp/teragen.swift
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen.swift (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen.swift 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,45 @@
+import files;
+import io;
+import random;
+import string;
+import sys;
+
+app (file out, file err) gen_data (file run, int recsize){
+ "/bin/bash" run recsize @stdout=out @stderr=err
+}
+
+app (file out, file err) comb_data (file comb, file array[]){
+ "/bin/bash" comb array @stdout=out @stderr=err
+}
+
+/*
+file tgen_out[] <simple_mapper; prefix="tgen", suffix=".out">;
+file tgen_err[] <simple_mapper; prefix="tgen", suffix=".err">;
+string dir = @arg("dir", "./");
+*/
+main{
+file wrapper = input_file("teragen_wrap.sh");
+file tgen_out[];
+file tgen_err[];
+
+int loop = 7;
+int fsize = 10;
+
+foreach item,i in [0:loop-1] {
+
+ file out <sprintf("tgenout-%i.dat", i)>;
+ file err <sprintf("tgenerr-%i.dat", i)>;
+
+ (out, err) = gen_data(wrapper, fsize);
+ tgen_out[i]=out;
+ tgen_err[i]=err;
+}
+
+file combine =input_file("combiner.sh");
+file final <"final_result">;
+file errs <"err_file">;
+
+(final, errs) = comb_data(combine, tgen_out);
+
+}
+
Added: SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,353 @@
+
+# Generated by stc version 0.2.0
+# date : 2013/08/13 16:55:59
+# Turbine version : 0.3.0
+# Input filename : /nfs2/yadunand/Swift-MapRed/TurboChirp/teragen.swift
+# Output filename : /nfs2/yadunand/Swift-MapRed/TurboChirp
+# STC home : /nfs2/yadunand/bin/exm/stc
+# Turbine home : /homes/yadunand/bin/exm/turbine
+# Compiler settings:
+# stc.auto-declare : true
+# stc.c_preprocess : true
+# stc.codegen.no-stack : true
+# stc.codegen.no-stack-vars : true
+# stc.compiler-debug : true
+# stc.debugging : COMMENTS
+# stc.exp.refcounting : false
+# stc.ic.output-file :
+# stc.input_filename : teragen.swift
+# stc.log.file :
+# stc.log.trace : false
+# stc.opt.algebra : false
+# stc.opt.cancel-refcounts : true
+# stc.opt.constant-fold : true
+# stc.opt.controlflow-fusion : true
+# stc.opt.dead-code-elim : true
+# stc.opt.disable-asserts : false
+# stc.opt.expand-dataflow-ops : true
+# stc.opt.expand-loop-threshold-insts: 256
+# stc.opt.expand-loop-threshold-iters: 16
+# stc.opt.expand-loops : true
+# stc.opt.flatten-nested : true
+# stc.opt.forward-dataflow : true
+# stc.opt.full-unroll : false
+# stc.opt.function-inline : false
+# stc.opt.function-inline-threshold: 500
+# stc.opt.function-signature : true
+# stc.opt.hoist : true
+# stc.opt.max-iterations : 10
+# stc.opt.piggyback-refcounts : true
+# stc.opt.pipeline : false
+# stc.opt.reorder-insts : false
+# stc.opt.shared-constants : true
+# stc.opt.unroll-loop-threshold-insts: 192
+# stc.opt.unroll-loop-threshold-iters: 8
+# stc.opt.unroll-loops : true
+# stc.opt.wait-coalesce : true
+# stc.output_filename :
+# stc.preproc.force-cpp : false
+# stc.preproc.force-gcc : false
+# stc.preprocess_only : false
+# stc.profile : false
+# stc.rpath :
+# stc.stc_home : /nfs2/yadunand/bin/exm/stc
+# stc.turbine.version : 0.3.0
+# stc.turbine_home : /homes/yadunand/bin/exm/turbine
+# stc.version : 0.2.0
+
+# Metadata:
+
+package require turbine 0.3.0
+namespace import turbine::*
+
+
+proc swift:constants { } {
+ turbine::c::log "function:swift:constants"
+ global c:i_10
+ turbine::allocate_custom c:i_10 integer 1 1 1
+ turbine::store_integer ${c:i_10} 10
+ global c:s_combiner_s
+ turbine::allocate_custom c:s_combiner_s string 1 1 1
+ turbine::store_string ${c:s_combiner_s} "combiner.sh"
+ global c:s_teragen_wr
+ turbine::allocate_custom c:s_teragen_wr string 1 1 1
+ turbine::store_string ${c:s_teragen_wr} "teragen_wrap.sh"
+}
+
+
+proc f:ternary { stack u:o u:b u:i1 u:i2 } {
+ turbine::c::log "enter function: ternary"
+ turbine::rule [ list ${u:b} ${u:i1} ${u:i2} ] "ternary-argwait ${stack} ${u:b} ${u:o} ${u:i2} ${u:i1}"
+}
+
+
+proc ternary-argwait { stack u:b u:o u:i2 u:i1 } {
+ # Value __v_b with type $boolean was defined
+ # Value __v_i1 with type $int was defined
+ # Value __v_i2 with type $int was defined
+ # Value __v_o with type $int was defined
+ set v:b [ turbine::retrieve_integer ${u:b} CACHED 1 ]
+ set v:i1 [ turbine::retrieve_integer ${u:i1} CACHED 1 ]
+ set v:i2 [ turbine::retrieve_integer ${u:i2} CACHED 1 ]
+ if { ${v:b} } { set v:o ${v:i1} } else { set v:o ${v:i2} }
+ turbine::store_integer ${u:o} ${v:o}
+}
+
+
+proc f:file_type { stack u:t u:f } {
+ turbine::c::log "enter function: file_type"
+ turbine::rule [ list [ turbine::get_file_status ${u:f} ] ] "file_type-argwait ${stack} {${u:f}} ${u:t}"
+}
+
+
+proc file_type-argwait { stack u:f u:t } {
+ # Value __v_f with type $file was defined
+ # Value __v_t with type $string was defined
+ set v:f [ turbine::get_file ${u:f} 1 ]
+ set v:t [ file type [ lindex ${v:f} 0 ] ]
+ turbine::store_string ${u:t} ${v:t}
+}
+
+
+proc f:file_exists { stack u:o u:f } {
+ turbine::c::log "enter function: file_exists"
+ turbine::rule [ list ${u:f} ] "file_exists-argwait ${stack} ${u:f} ${u:o}"
+}
+
+
+proc file_exists-argwait { stack u:f u:o } {
+ # Value __v_f with type $string was defined
+ # Value __v_o with type $boolean was defined
+ set v:f [ turbine::retrieve_string ${u:f} CACHED 1 ]
+ set v:o [ file exists ${v:f} ]
+ turbine::store_integer ${u:o} ${v:o}
+}
+
+
+proc f:file_mtime { stack u:o u:f } {
+ turbine::c::log "enter function: file_mtime"
+ turbine::rule [ list ${u:f} ] "file_mtime-argwait ${stack} ${u:f} ${u:o}"
+}
+
+
+proc file_mtime-argwait { stack u:f u:o } {
+ # Value __v_f with type $string was defined
+ # Value __v_o with type $int was defined
+ set v:f [ turbine::retrieve_string ${u:f} CACHED 1 ]
+ set v:o [ file mtime ${v:f} ]
+ turbine::store_integer ${u:o} ${v:o}
+}
+
+
+proc f:strlen { stack u:n u:s } {
+ turbine::c::log "enter function: strlen"
+ turbine::rule [ list ${u:s} ] "strlen-argwait ${stack} ${u:s} ${u:n}"
+}
+
+
+proc strlen-argwait { stack u:s u:n } {
+ # Value __v_s with type $string was defined
+ # Value __v_n with type $int was defined
+ set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
+ set v:n [ string length ${v:s} ]
+ turbine::store_integer ${u:n} ${v:n}
+}
+
+
+proc f:trim { stack u:o u:s } {
+ turbine::c::log "enter function: trim"
+ turbine::rule [ list ${u:s} ] "trim-argwait ${stack} ${u:s} ${u:o}"
+}
+
+
+proc trim-argwait { stack u:s u:o } {
+ # Value __v_s with type $string was defined
+ # Value __v_o with type $string was defined
+ set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
+ set v:o [ string trim ${v:s} ]
+ turbine::store_string ${u:o} ${v:o}
+}
+
+
+proc f:hash { stack u:h u:s } {
+ turbine::c::log "enter function: hash"
+ turbine::rule [ list ${u:s} ] "hash-argwait ${stack} ${u:s} ${u:h}"
+}
+
+
+proc hash-argwait { stack u:s u:h } {
+ # Value __v_s with type $string was defined
+ # Value __v_h with type $int was defined
+ set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
+ set v:h [ c_utils::hash ${v:s} ]
+ turbine::store_integer ${u:h} ${v:h}
+}
+
+
+proc f:clock_seconds { stack u:t } {
+ turbine::c::log "enter function: clock_seconds"
+ # Value __v_t with type $int was defined
+ set v:t [ clock seconds ]
+ turbine::store_integer ${u:t} ${v:t}
+}
+
+
+proc f:clock { stack u:t } {
+ turbine::c::log "enter function: clock"
+ # Value __v_t with type $float was defined
+ set v:t [ expr {[ clock clicks -milliseconds ] / 1000.0 } ]
+ turbine::store_float ${u:t} ${v:t}
+}
+
+
+proc f:gen_data { stack u:out u:err u:run u:recsize dr:location } {
+ turbine::c::log "enter function: gen_data"
+ # Alias __filename_run with type string was defined
+ # Alias __filename_out with type string was defined
+ # Alias __filename_err with type string was defined
+ set filename:run [ turbine::get_file_path ${u:run} ]
+ set filename:out [ turbine::get_output_file_path ${u:out} ]
+ set filename:err [ turbine::get_output_file_path ${u:err} ]
+ turbine::rule [ list ${filename:err} ${filename:out} ${filename:run} ${u:recsize} [ turbine::get_file_status ${u:run} ] ] "gen_data-app-leaf0 ${stack} {${u:err}} ${filename:run} ${filename:out} ${filename:err} {${u:run}} ${u:recsize} {${u:out}}" target ${dr:location} type ${::turbine::WORK}
+}
+
+
+proc gen_data-app-leaf0 { stack u:err filename:run filename:out filename:err u:run u:recsize u:out } {
+ # Value __v___filename_run with type $string was defined
+ # Value __v_recsize with type $int was defined
+ # Value __v___filename_out with type $string was defined
+ # Value __v___filename_err with type $string was defined
+ # Value __v_run with type $file was defined
+ # Value __v_out with type $file was defined
+ # Value __v_err with type $file was defined
+ set v:__filename_run [ turbine::retrieve_string ${filename:run} CACHED 1 ]
+ set v:recsize [ turbine::retrieve_integer ${u:recsize} CACHED 1 ]
+ set v:__filename_out [ turbine::retrieve_string ${filename:out} CACHED 1 ]
+ set v:__filename_err [ turbine::retrieve_string ${filename:err} CACHED 1 ]
+ set v:run [ turbine::get_file ${u:run} 1 ]
+ turbine::c::log [ list exec: /bin/bash ${v:__filename_run} ${v:recsize} [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ]
+ turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ${v:__filename_run} ${v:recsize}
+ set v:out [ turbine::create_local_file_ref ${v:__filename_out} ]
+ set v:err [ turbine::create_local_file_ref ${v:__filename_err} ]
+ turbine::set_file ${u:out} v:out
+ turbine::set_file ${u:err} v:err
+ turbine::decr_local_file_refcount v:out
+ turbine::decr_local_file_refcount v:err
+}
+
+
+proc f:comb_data { stack u:out u:err u:comb u:array dr:location } {
+ turbine::c::log "enter function: comb_data"
+ # Alias __filename_comb with type string was defined
+ # Alias __filename_out with type string was defined
+ # Alias __filename_err with type string was defined
+ set filename:comb [ turbine::get_file_path ${u:comb} ]
+ set filename:out [ turbine::get_output_file_path ${u:out} ]
+ set filename:err [ turbine::get_output_file_path ${u:err} ]
+ turbine::deeprule [ list ${filename:comb} ${filename:err} ${filename:out} ${u:array} [ turbine::get_file_status ${u:comb} ] ] [ list 0 0 0 1 0 ] [ list 0 0 0 1 1 ] "comb_data-app-leaf0 ${stack} {${u:err}} ${filename:out} ${filename:err} {${u:comb}} ${filename:comb} {${u:out}} ${u:array}" target ${dr:location} type ${::turbine::WORK}
+}
+
+
+proc comb_data-app-leaf0 { stack u:err filename:out filename:err u:comb filename:comb u:out u:array } {
+ # Value __v___filename_comb with type $string was defined
+ # Value __v___filename_out with type $string was defined
+ # Value __v___filename_err with type $string was defined
+ # Value __v_comb with type $file was defined
+ # Value __v_out with type $file was defined
+ # Value __v_err with type $file was defined
+ set v:__filename_comb [ turbine::retrieve_string ${filename:comb} CACHED 1 ]
+ set v:__filename_out [ turbine::retrieve_string ${filename:out} CACHED 1 ]
+ set v:__filename_err [ turbine::retrieve_string ${filename:err} CACHED 1 ]
+ set v:comb [ turbine::get_file ${u:comb} 1 ]
+ turbine::c::log [ list exec: /bin/bash ${v:__filename_comb} {*}[ turbine::unpack_args ${u:array} 1 1 ] [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ]
+ turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ${v:__filename_comb} {*}[ turbine::unpack_args ${u:array} 1 1 ]
+ set v:out [ turbine::create_local_file_ref ${v:__filename_out} ]
+ set v:err [ turbine::create_local_file_ref ${v:__filename_err} ]
+ turbine::set_file ${u:out} v:out
+ turbine::set_file ${u:err} v:err
+ turbine::decr_local_file_refcount v:out
+ turbine::decr_local_file_refcount v:err
+}
+
+
+proc swift:main { } {
+ turbine::c::log "enter function: main"
+ set stack 0
+ global c:s_combiner_s
+ global c:s_teragen_wr
+ lassign [ adlb::multicreate [ list ${::adlb::CONTAINER} ${::adlb::INTEGER} 1 1 ] [ list ${::adlb::CONTAINER} ${::adlb::INTEGER} 1 1 ] [ list ${::adlb::STRING} 1 ] [ list ${::adlb::STRING} 1 ] ] u:tgen_out t:1 t:10 t:11
+ turbine::c::log "allocated u:tgen_out=<${u:tgen_out}> t:1=<${t:1}> t:10=<${t:10}> t:11=<${t:11}>"
+ turbine::allocate_file2 u:wrapper "" 1
+ turbine::allocate_file2 u:combine "" 1
+ turbine::allocate_file2 u:final ${t:10} 1
+ turbine::allocate_file2 u:errs ${t:11} 1
+ # Swift l.21: assigning expression to wrapper
+ turbine::input_file [ list ${u:wrapper} ] [ list ${c:s_teragen_wr} ]
+ # Swift l.25: assigning expression to loop
+ # Swift l.26: assigning expression to fsize
+ turbine::range_work ${t:1} 0 6 1
+ # Swift l.38: assigning expression to combine
+ turbine::input_file [ list ${u:combine} ] [ list ${c:s_combiner_s} ]
+ turbine::store_string ${t:10} "final_result"
+ turbine::store_string ${t:11} "err_file"
+ # Swift l.42: assigning expression to final
+ # Swift l.42: assigning expression to errs
+ f:comb_data ${stack} ${u:final} ${u:errs} ${u:combine} ${u:tgen_out} -100
+ # Swift l.33: assigning expression to out
+ # Swift l.33: assigning expression to err
+ # Swift l.34: assigning expression to tgen_out[_]
+ # Swift l.35: assigning expression to tgen_err[_]
+ set tcltmp:container_sz [ adlb::enumerate ${t:1} count all 0 ]
+ set tcltmp:iters ${tcltmp:container_sz}
+ adlb::slot_create ${u:tgen_out} [ expr { ${tcltmp:iters} - 1 } ]
+ main-foreach0:outer ${stack} ${u:wrapper} ${u:tgen_out} ${t:1} 0 [ expr { ${tcltmp:container_sz} - 1 } ] 1
+}
+
+
+proc main-foreach0:outer { stack u:wrapper u:tgen_out t:1 tcltmp:lo tcltmp:hi tcltmp:inc } {
+ set tcltmp:itersleft [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
+ if { [ expr { ${tcltmp:itersleft} <= 0 } ] } {
+ return
+ }
+ if { [ expr { ${tcltmp:itersleft} <= 64 } ] } {
+ main-foreach0:inner ${stack} ${u:wrapper} ${u:tgen_out} ${t:1} ${tcltmp:lo} ${tcltmp:hi} ${tcltmp:inc}
+ } else {
+ set tcltmp:skip [ expr { ${tcltmp:inc} * max(64,((${tcltmp:itersleft} - 1) / 16) + 1) } ]
+ for { set tcltmp:splitstart ${tcltmp:lo} } { ${tcltmp:splitstart} <= ${tcltmp:hi} } { incr tcltmp:splitstart ${tcltmp:skip} } {
+ set tcltmp:splitend [ expr { min(${tcltmp:hi},${tcltmp:splitstart} + ${tcltmp:skip} - 1) } ]
+ set tcltmp:prio [ turbine::get_priority ]
+ turbine::set_priority ${tcltmp:prio}
+ adlb::spawn 1 "command main-foreach0:outer ${stack} {${u:wrapper}} ${u:tgen_out} ${t:1} ${tcltmp:splitstart} ${tcltmp:splitend} ${tcltmp:inc}"
+ turbine::reset_priority
+ }
+ }
+}
+
+
+proc main-foreach0:inner { stack u:wrapper u:tgen_out t:1 tcltmp:lo tcltmp:hi tcltmp:inc } {
+ set tcltmp:splitlen [ expr { ${tcltmp:hi} - ${tcltmp:lo} + 1 } ]
+ set tcltmp:contents [ adlb::enumerate ${t:1} dict ${tcltmp:splitlen} ${tcltmp:lo} ]
+ dict for {v:i u:item} ${tcltmp:contents} {
+ global c:i_10
+ # Value __ov___t5 with type $string was defined
+ # Value __ov___t7 with type $string was defined
+ lassign [ adlb::multicreate [ list ${::adlb::STRING} 1 ] [ list ${::adlb::STRING} 1 ] ] t:5 t:7
+ turbine::c::log "allocated t:5=<${t:5}> t:7=<${t:7}>"
+ turbine::allocate_file2 u:out ${t:5} 1
+ turbine::allocate_file2 u:err ${t:7} 1
+ set optv:__t5 [ eval format [ list "tgenout-%i.dat" ${v:i} ] ]
+ turbine::store_string ${t:5} ${optv:__t5}
+ set optv:__t7 [ eval format [ list "tgenerr-%i.dat" ${v:i} ] ]
+ turbine::store_string ${t:7} ${optv:__t7}
+ f:gen_data ${stack} ${u:out} ${u:err} ${u:wrapper} ${c:i_10} -100
+ turbine::container_insert ${u:tgen_out} ${v:i} ${u:out} 1
+ }
+ set tcltmp:iters [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
+}
+
+turbine::defaults
+turbine::init $engines $servers
+turbine::check_constants "WORKER" ${turbine::WORK_TASK} 0 "CONTROL" ${turbine::CONTROL_TASK} 1 "ADLB_RANK_ANY" ${adlb::RANK_ANY} -100
+turbine::start swift:main swift:constants
+turbine::finalize
+
Added: SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# By default with ARG1:100 and SLICESIZE=10000, this script will generate
+# 10^6 records.
+ARG1=1
+[ ! -z $1 ] && ARG1=$1
+
+FILE="input_$RANDOM.txt"
+LOWERLIMIT=0
+UPPERLIMIT=1000000000 # 10^9
+SLICESIZE=10000 # 10^4 records padded to 100B would result in 1MB file
+#SLICESIZE=1000 # 10^3 If padded to 100B would result
+
+
+FOLDER="/sandbox/$USER"
+TIMEOUT=120
+
+# clean_folder SLEEP_TIME FOLDER_NAME
+clean_folder()
+{
+ sleep $(($k1+1));
+ rm $2/* &> /dev/null
+}
+
+ps -u $USER | grep "chirp_server" &> /dev/null
+if [ "$?" != "0" ]
+then
+ killall -u $USER chirp_server;
+ echo "unix:$USER rwlda" > $FOLDER/acl.conf
+ echo "hostname:* rwlda" >> $FOLDER/acl.conf
+ timeout $TIMEOUT chirp_server -A $FOLDER/acl.conf -r $FOLDER &
+fi
+
+FILE=$FOLDER/$FILE;
+shuf -i $LOWERLIMIT-$UPPERLIMIT -n $(($SLICESIZE*$ARG1)) | awk '{printf "%-99s\n", $0}' > $FILE
+sleep 5;
+HOST=$(hostname -f)
+echo "$HOST $FILE"
+exit 0
\ No newline at end of file
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/TurboChirp/test
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/test (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/test 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,2 @@
+unix:yadunand rwlda
+hostname:* rwlda
Added: SwiftApps/Swift-MapRed/TurboChirp/turbo.swift
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/turbo.swift (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/turbo.swift 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,2 @@
+
+import io;
Added: SwiftApps/Swift-MapRed/TurboChirp/wrap_test.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/wrap_test.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/wrap_test.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+TIME=/usr/bin/time
+# GENERATE 1 MB
+
+/usr/bin/time -f "%e" ./teragen_wrap.sh 1 > tmp
+exit 0
+
+VAL=0
+RUNS=10
+for i in `seq 1 1 RUNS`
+do
+ T=`/usr/bin/time -f "%e" ./teragen_wrap.sh 1 > tmp)`
+ VAL=$(($VAL+$T))
+ rm tmp;
+done
+
+echo " TOTAL TIME : $VAL "
+echo " RUNS : $RUNS"
+echo " AVERAGE : "; bc <<< "scale=5; ($VAL / $RUNS)"
\ No newline at end of file
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/wrap_test.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/Swift-MapRed/mapred_chirp/LAB_RECORDS
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/LAB_RECORDS 2013-08-13 21:46:08 UTC (rev 6844)
+++ SwiftApps/Swift-MapRed/mapred_chirp/LAB_RECORDS 2013-08-13 21:59:30 UTC (rev 6845)
@@ -321,3 +321,167 @@
Progress: time: Sat, 10 Aug 2013 18:11:23 -0500 Stage in:2 Submitting:6 Active:7 Stage out:16 Finished successfully:19
Progress: time: Sat, 10 Aug 2013 18:11:24 -0500 Submitting:6 Active:6 Stage out:16 Finished successfully:22
Progress: time: Sat, 10 Aug 2013 18:11:26 -0500 Submitting:6 Active:5 Stage out:8 Finished successfully:31
+================================================================
+RUNTYPE :SWIFT+Chirp (No local combiners), SITE:anl, CHUNKS:20 X FILESIZE:20
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130812-1354-0yz2ut22
+Progress: time: Mon, 12 Aug 2013 13:54:07 -0500 Initializing:1
+Progress: time: Mon, 12 Aug 2013 13:54:27 -0500 Submitting:19 Submitted:1
+Progress: time: Mon, 12 Aug 2013 13:54:28 -0500 Submitting:19 Stage out:1
+Progress: time: Mon, 12 Aug 2013 13:54:30 -0500 Submitting:18 Submitted:1 Finished successfully:1
+Progress: time: Mon, 12 Aug 2013 13:54:34 -0500 Submitting:17 Submitted:1 Finished successfully:2
+Progress: time: Mon, 12 Aug 2013 13:54:37 -0500 Submitting:16 Finished successfully:4
+Progress: time: Mon, 12 Aug 2013 13:54:38 -0500 Submitting:15 Submitted:1 Finished successfully:4
+Progress: time: Mon, 12 Aug 2013 13:54:42 -0500 Submitting:13 Submitted:1 Finished successfully:6
+Progress: time: Mon, 12 Aug 2013 13:54:45 -0500 Submitting:9 Submitted:1 Finished successfully:10
+Progress: time: Mon, 12 Aug 2013 13:54:49 -0500 Submitting:6 Submitted:1 Finished successfully:13
+Progress: time: Mon, 12 Aug 2013 13:54:56 -0500 Submitting:1 Submitted:1 Finished successfully:18
+Progress: time: Mon, 12 Aug 2013 13:54:58 -0500 Active:1 Stage out:1 Finished successfully:18
+Progress: time: Mon, 12 Aug 2013 13:55:03 -0500 Submitted:1 Finished successfully:20
+Progress: time: Mon, 12 Aug 2013 13:55:05 -0500 Active:1 Finished successfully:20
+Progress: time: Mon, 12 Aug 2013 13:55:07 -0500 Active:1 Finished successfully:20
+Progress: time: Mon, 12 Aug 2013 13:55:14 -0500 Stage out:1 Finished successfully:20
+Final status: Mon, 12 Aug 2013 13:55:14 -0500 Finished successfully:21
+real 75.27
+user 11.00
+sys 3.65
+chunks 20 filesize 20 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 3600000
+EXPECTED COUNT: 4000000
+couldn't get crank:input_19771.txt: Permission denied
+couldn't get crank:input_4640.txt: Permission denied
+awk: cmd. line:1: fatal: cannot open file `input_4640.txt.imd' for reading (No such file or directory)
+awk: cmd. line:1: fatal: cannot open file `input_19771.txt.imd' for reading (No such file or directory)
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (No local combiners), SITE:anl, CHUNKS:50 X FILESIZE:20
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130812-1355-66zmb4l5
+Progress: time: Mon, 12 Aug 2013 13:55:22 -0500
+Progress: time: Mon, 12 Aug 2013 13:55:23 -0500 Selecting site:49 Submitting:1
+Progress: time: Mon, 12 Aug 2013 13:55:34 -0500 Submitting:49 Submitted:1
+Progress: time: Mon, 12 Aug 2013 13:55:36 -0500 Submitting:48 Submitted:1 Finished successfully:1
+Progress: time: Mon, 12 Aug 2013 13:55:39 -0500 Submitting:47 Submitted:1 Finished successfully:2
+Progress: time: Mon, 12 Aug 2013 13:55:42 -0500 Submitting:46 Submitted:1 Finished successfully:3
+Progress: time: Mon, 12 Aug 2013 13:55:45 -0500 Submitting:45 Submitted:1 Finished successfully:4
+Progress: time: Mon, 12 Aug 2013 13:55:48 -0500 Submitting:43 Submitted:1 Finished successfully:6
+Progress: time: Mon, 12 Aug 2013 13:55:49 -0500 Submitting:38 Active:2 Stage out:2 Finished successfully:8
+Progress: time: Mon, 12 Aug 2013 13:55:51 -0500 Submitting:37 Submitted:1 Finished successfully:12
+Progress: time: Mon, 12 Aug 2013 13:55:54 -0500 Submitting:35 Submitted:1 Finished successfully:14
+Progress: time: Mon, 12 Aug 2013 13:55:58 -0500 Submitting:24 Submitted:1 Finished successfully:25
+Progress: time: Mon, 12 Aug 2013 13:55:59 -0500 Submitting:1 Active:1 Stage out:7 Finished successfully:41
+Progress: time: Mon, 12 Aug 2013 13:56:07 -0500 Submitted:1 Finished successfully:49
+Progress: time: Mon, 12 Aug 2013 13:56:10 -0500 Stage out:1 Finished successfully:49
+Progress: time: Mon, 12 Aug 2013 13:56:14 -0500 Active:1 Finished successfully:50
+Progress: time: Mon, 12 Aug 2013 13:56:22 -0500 Active:1 Finished successfully:50
+Progress: time: Mon, 12 Aug 2013 13:56:26 -0500 Stage out:1 Finished successfully:50
+Final status: Mon, 12 Aug 2013 13:56:27 -0500 Finished successfully:51
+real 72.82
+user 9.94
+sys 2.68
+chunks 50 filesize 20 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 9800000
+EXPECTED COUNT: 10000000
+couldn't get crank:input_6730.txt: Permission denied
+awk: cmd. line:1: fatal: cannot open file `input_6730.txt.imd' for reading (No such file or directory)
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (No local combiners), SITE:anl, CHUNKS:20 X FILESIZE:20
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130812-1359-o5aton7d
+Progress: time: Mon, 12 Aug 2013 13:59:20 -0500
+Progress: time: Mon, 12 Aug 2013 13:59:36 -0500 Submitting:19 Submitted:1
+Progress: time: Mon, 12 Aug 2013 13:59:37 -0500 Submitting:19 Stage out:1
+Progress: time: Mon, 12 Aug 2013 13:59:39 -0500 Submitting:18 Submitted:1 Finished successfully:1
+Progress: time: Mon, 12 Aug 2013 13:59:43 -0500 Submitting:17 Submitted:1 Finished successfully:2
+Progress: time: Mon, 12 Aug 2013 13:59:46 -0500 Submitting:16 Submitted:1 Finished successfully:3
+Progress: time: Mon, 12 Aug 2013 13:59:47 -0500 Submitting:15 Stage out:1 Finished successfully:4
+Progress: time: Mon, 12 Aug 2013 13:59:50 -0500 Submitting:15 Finished successfully:5
+Progress: time: Mon, 12 Aug 2013 13:59:53 -0500 Submitting:10 Submitted:1 Finished successfully:9
+Progress: time: Mon, 12 Aug 2013 13:59:57 -0500 Submitting:8 Submitted:1 Finished successfully:11
+Progress: time: Mon, 12 Aug 2013 14:00:01 -0500 Submitting:2 Submitted:1 Finished successfully:17
+Progress: time: Mon, 12 Aug 2013 14:00:04 -0500 Submitted:1 Finished successfully:19
+Progress: time: Mon, 12 Aug 2013 14:00:07 -0500 Active:1 Finished successfully:20
+Progress: time: Mon, 12 Aug 2013 14:00:15 -0500 Stage out:1 Finished successfully:20
+Final status: Mon, 12 Aug 2013 14:00:16 -0500 Finished successfully:21
+real 65.01
+user 10.25
+sys 3.58
+chunks 20 filesize 20 site anl type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 4000000
+EXPECTED COUNT: 4000000
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (No local combiners), SITE:anl, CHUNKS:50 X FILESIZE:20
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130812-1400-eubrtzt6
+Progress: time: Mon, 12 Aug 2013 14:00:26 -0500
+Progress: time: Mon, 12 Aug 2013 14:00:27 -0500 Selecting site:49 Submitting:1
+Progress: time: Mon, 12 Aug 2013 14:00:38 -0500 Submitting:49 Submitted:1
+Progress: time: Mon, 12 Aug 2013 14:00:39 -0500 Submitting:48 Stage out:1 Finished successfully:1
+Progress: time: Mon, 12 Aug 2013 14:00:40 -0500 Stage in:1 Submitting:46 Active:1 Finished successfully:2
+Progress: time: Mon, 12 Aug 2013 14:00:43 -0500 Submitting:45 Submitted:1 Finished successfully:4
+Progress: time: Mon, 12 Aug 2013 14:00:46 -0500 Submitting:44 Submitted:1 Finished successfully:5
+Progress: time: Mon, 12 Aug 2013 14:00:49 -0500 Submitting:43 Submitted:1 Finished successfully:6
+Progress: time: Mon, 12 Aug 2013 14:00:52 -0500 Submitting:38 Submitted:1 Finished successfully:11
+Progress: time: Mon, 12 Aug 2013 14:00:55 -0500 Submitting:36 Submitted:1 Finished successfully:13
+Progress: time: Mon, 12 Aug 2013 14:00:58 -0500 Submitting:34 Submitted:1 Finished successfully:15
+Progress: time: Mon, 12 Aug 2013 14:00:59 -0500 Stage in:13 Submitting:2 Submitted:2 Active:13 Stage out:5 Finished successfully:15
+Progress: time: Mon, 12 Aug 2013 14:01:00 -0500 Submitting:2 Active:9 Stage out:3 Finished successfully:36
+Progress: time: Mon, 12 Aug 2013 14:01:04 -0500 Submitting:1 Submitted:1 Finished successfully:48
+Progress: time: Mon, 12 Aug 2013 14:01:09 -0500 Active:1 Finished successfully:50
+Progress: time: Mon, 12 Aug 2013 14:01:22 -0500 Stage out:1 Finished successfully:50
+Final status: Mon, 12 Aug 2013 14:01:22 -0500 Finished successfully:51
+real 66.36
+user 9.53
+sys 2.54
+chunks 50 filesize 20 site anl type Swift+Chirp
+---------------------------------------------
+COUNT does *NOT* match expected records
+ACTUAL COUNT : 9800000
+EXPECTED COUNT: 10000000
+couldn't get crank:input_6730.txt: Permission denied
+awk: cmd. line:1: fatal: cannot open file `input_6730.txt.imd' for reading (No such file or directory)
+---------------------------------------------
+================================================================
+RUNTYPE :SWIFT+Chirp (No local combiners), SITE:anl, CHUNKS:50 X FILESIZE:20
+Swift 0.94RC4 swift-r6303 cog-r3610
+
+RunID: 20130812-1403-7t6vd0n9
+Progress: time: Mon, 12 Aug 2013 14:03:19 -0500
+Progress: time: Mon, 12 Aug 2013 14:03:21 -0500 Selecting site:49 Submitting:1
+Progress: time: Mon, 12 Aug 2013 14:03:32 -0500 Submitting:49 Submitted:1
+Progress: time: Mon, 12 Aug 2013 14:03:33 -0500 Submitting:49 Finished successfully:1
+Progress: time: Mon, 12 Aug 2013 14:03:34 -0500 Submitting:48 Submitted:1 Finished successfully:1
+Progress: time: Mon, 12 Aug 2013 14:03:38 -0500 Submitting:44 Submitted:1 Finished successfully:5
+Progress: time: Mon, 12 Aug 2013 14:03:41 -0500 Submitting:37 Submitted:1 Finished successfully:12
+Progress: time: Mon, 12 Aug 2013 14:03:45 -0500 Submitting:35 Submitted:1 Finished successfully:14
+Progress: time: Mon, 12 Aug 2013 14:03:48 -0500 Submitting:33 Submitted:1 Finished successfully:16
+Progress: time: Mon, 12 Aug 2013 14:03:49 -0500 Submitting:33 Finished successfully:17
+Progress: time: Mon, 12 Aug 2013 14:03:51 -0500 Submitting:32 Submitted:1 Finished successfully:17
+Progress: time: Mon, 12 Aug 2013 14:03:52 -0500 Submitting:21 Stage out:3 Finished successfully:26
+Progress: time: Mon, 12 Aug 2013 14:03:55 -0500 Submitting:20 Submitted:1 Finished successfully:29
+Progress: time: Mon, 12 Aug 2013 14:03:56 -0500 Submitting:2 Active:6 Stage out:5 Finished successfully:37
+Progress: time: Mon, 12 Aug 2013 14:04:01 -0500 Submitting:1 Submitted:1 Finished successfully:48
+Progress: time: Mon, 12 Aug 2013 14:04:05 -0500 Active:1 Finished successfully:50
+Progress: time: Mon, 12 Aug 2013 14:04:19 -0500 Stage out:1 Finished successfully:50
+Final status: Mon, 12 Aug 2013 14:04:19 -0500 Finished successfully:51
+real 69.25
+user 9.71
+sys 2.76
+chunks 50 filesize 20 site anl type Swift+Chirp
+---------------------------------------------
+COUNT matches expected records
+ACTUAL COUNT : 10000000
+EXPECTED COUNT: 10000000
+---------------------------------------------
Modified: SwiftApps/Swift-MapRed/mapred_chirp/anl.xml
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/anl.xml 2013-08-13 21:46:08 UTC (rev 6844)
+++ SwiftApps/Swift-MapRed/mapred_chirp/anl.xml 2013-08-13 21:59:30 UTC (rev 6845)
@@ -1,5 +1,6 @@
<config>
+<!--
<pool handle="crank">
<execution provider="coaster" jobmanager="ssh-cl:local" url="crank.mcs.anl.gov"/>
<profile namespace="globus" key="jobsPerNode">8</profile>
@@ -11,7 +12,7 @@
<profile namespace="karajan" key="initialScore">10000</profile>
<workdirectory>/sandbox/{env.USER}/swiftwork</workdirectory>
</pool>
-
+-->
<pool handle="churn">
<execution provider="coaster" jobmanager="ssh-cl:local" url="churn.mcs.anl.gov"/>
<profile namespace="globus" key="jobsPerNode">8</profile>
Modified: SwiftApps/Swift-MapRed/mapred_chirp/err_file
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/err_file 2013-08-13 21:46:08 UTC (rev 6844)
+++ SwiftApps/Swift-MapRed/mapred_chirp/err_file 2013-08-13 21:59:30 UTC (rev 6845)
@@ -0,0 +1,2 @@
+couldn't get crank:input_6730.txt: Permission denied
+awk: cmd. line:1: fatal: cannot open file `input_6730.txt.imd' for reading (No such file or directory)
Modified: SwiftApps/Swift-MapRed/mapred_chirp/final_result
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/final_result 2013-08-13 21:46:08 UTC (rev 6844)
+++ SwiftApps/Swift-MapRed/mapred_chirp/final_result 2013-08-13 21:59:30 UTC (rev 6845)
@@ -1,2 +1,2 @@
-SUM : 4999568676887003
+SUM : 5000479383023296
COUNT: 10000000
Modified: SwiftApps/Swift-MapRed/mapred_chirp/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/run.sh 2013-08-13 21:46:08 UTC (rev 6844)
+++ SwiftApps/Swift-MapRed/mapred_chirp/run.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -25,6 +25,29 @@
}
#export COG_OPTS="-Dtcp.channel.log.io.performance=true"
+verify()
+{
+ LOOPS=$1
+ RECSIZE=$2
+ ACTUAL_RESULT=($(tail -n 1 final_result))
+ EXPECTED_RESULT=$(($LOOPS * $RECSIZE * 10000 ))
+ if [ "$EXPECTED_RESULT" == "${ACTUAL_RESULT[1]}" ]
+ then
+ echo "---------------------------------------------"
+ echo "COUNT matches expected records"
+ echo "ACTUAL COUNT : ${ACTUAL_RESULT[1]}"
+ echo "EXPECTED COUNT: $EXPECTED_RESULT"
+ echo "---------------------------------------------"
+ else
+ echo "---------------------------------------------"
+ echo "COUNT does *NOT* match expected records"
+ echo "ACTUAL COUNT : ${ACTUAL_RESULT[1]}"
+ echo "EXPECTED COUNT: $EXPECTED_RESULT"
+ cat err_file
+ echo "---------------------------------------------"
+ fi
+}
+
run_swift_chirp()
{
LOOPS=$1
@@ -34,10 +57,25 @@
echo "RUNTYPE :SWIFT+Chirp (No local combiners), SITE:$SITE, CHUNKS:$LOOPS X FILESIZE:$RECSIZE " | tee -a LAB_RECORDS
( time -p swift -tc.file tc.data -config cf -sites.file $SITE.xml teragen.swift -loops=$LOOPS -recsize=$RECSIZE ) 2>&1 | tee -a LAB_RECORDS
echo "chunks $LOOPS filesize $RECSIZE site $SITE type Swift+Chirp" >> LAB_RECORDS
+ verify $LOOPS $RECSIZE | tee -a LAB_RECORDS
echo "================================================================"
}
-run_swift_chirp 50 200;
+run_swift_chirp_combiner()
+{
+ LOOPS=$1
+ RECSIZE=$2
+ ./clean.sh
+ echo "================================================================" >> LAB_RECORDS
+ echo "RUNTYPE :SWIFT+Chirp (No local combiners), SITE:$SITE, CHUNKS:$LOOPS X FILESIZE:$RECSIZE " | tee -a LAB_RECORDS
+ ( time -p swift -tc.file tc.data -config cf -sites.file $SITE.xml teragen.swift -loops=$LOOPS -recsize=$RECSIZE ) 2>&1 | tee -a LAB_RECORDS
+ echo "chunks $LOOPS filesize $RECSIZE site $SITE type Swift+Chirp" >> LAB_RECORDS
+ verify $LOOPS $RECSIZE | tee -a LAB_RECORDS
+ echo "================================================================"
+}
+
+#run_swift_chirp 20 20;
+run_swift_chirp 10 100;
exit 0
####################SWIFT + CHIRP VERSION############################
Modified: SwiftApps/Swift-MapRed/mapred_chirp/teragen_wrap.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/teragen_wrap.sh 2013-08-13 21:46:08 UTC (rev 6844)
+++ SwiftApps/Swift-MapRed/mapred_chirp/teragen_wrap.sh 2013-08-13 21:59:30 UTC (rev 6845)
@@ -33,7 +33,7 @@
echo "unix:$USER rwlds" > $FOLDER/acl.conf
echo "hostname:* rwl" >> $FOLDER/acl.conf
timeout $TIMEOUT chirp_server -A $FOLDER/acl.conf -r $FOLDER &
- rm -rf
+
fi
FILE=$FOLDER/$FILE;
else
More information about the Swift-commit
mailing list