[Swift-commit] r6848 - SwiftApps/Swift-MapRed/TurboChirp
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Wed Aug 14 15:09:06 CDT 2013
Author: yadunandb
Date: 2013-08-14 15:09:06 -0500 (Wed, 14 Aug 2013)
New Revision: 6848
Added:
SwiftApps/Swift-MapRed/TurboChirp/jobs_per_site.sh
Modified:
SwiftApps/Swift-MapRed/TurboChirp/cleanup
SwiftApps/Swift-MapRed/TurboChirp/combiner.sh
SwiftApps/Swift-MapRed/TurboChirp/run.sh
SwiftApps/Swift-MapRed/TurboChirp/teragen.swift
SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl
SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh
Log:
Waiting for the correct usage of @location tag
Combiner code tested.
Modified: SwiftApps/Swift-MapRed/TurboChirp/cleanup
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/cleanup 2013-08-14 15:42:28 UTC (rev 6847)
+++ SwiftApps/Swift-MapRed/TurboChirp/cleanup 2013-08-14 20:09:06 UTC (rev 6848)
@@ -1,3 +1,2 @@
-rm -rfv *.dat
-rm *{imd2,imd}
-rm err_file final_result
\ No newline at end of file
+rm tgen*{err,out} *{imd2,imd} err_file final_result *~ &> /dev/null
+rm sitecomb*{err,out} &> /dev/null
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/TurboChirp/combiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/combiner.sh 2013-08-14 15:42:28 UTC (rev 6847)
+++ SwiftApps/Swift-MapRed/TurboChirp/combiner.sh 2013-08-14 20:09:06 UTC (rev 6848)
@@ -1,23 +1,46 @@
#!/bin/bash
+# The combiner is expecting to receive filenames as args
+# Each file would contains strings in the format HOSTNAME FILEPATH
-FILES=$*
+ARGS=($*)
+PASSED_SITE=${ARGS[0]}
+FILES=(${ARGS[*]:1})
+
+#CURRENT_HOST=$(hostname -f)
+CURRENT_HOST=$HOSTNAME
+
+# debug
+#[ "$CURRENT_HOST" == "$PASSED_HOST" ] && echo "Local combination";
+
SUM=0
COUNT=0
fetch_and_process()
{
TOKEN=($(cat $1))
- HOST=${TOKEN[0]}
+ HOST=${TOKEN[0]}
FILE=${TOKEN[1]}
+ echo "FILE: $FILE on $HOST"
TARGET=$(basename $FILE)
- chirp_get $HOST $TARGET $TARGET.imd
- RES=($(awk '{ sum += $1 } END { print sum,NR }' $TARGET.imd))
+ echo "HOST: $HOST, CURRENT_HOST=$HOSTNAME"
+ if [ "$HOST" != "$CURRENT_HOST" ]
+ then
+ # Fetch if file is remote
+ echo "chirp_get $HOST $TARGET $TARGET.imd"
+ chirp_get $HOST $TARGET /tmp/$TARGET.imd
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' /tmp/$TARGET.imd))
+ else
+ # Process directly if file is local
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $FILE))
+ fi
echo "${RES[0]} ${RES[1]}" > "$2.$RANDOM.imd2"
}
ID=$RANDOM
-for file_token in $*
+echo "DEBUG: ${FILES[*]}"
+for file_token in ${FILES[*]}
do
+ echo $file_token
fetch_and_process $file_token $ID &
done
wait
@@ -27,29 +50,6 @@
echo "SUM : $SUM"
echo "COUNT: $COUNT"
-exit 0
-
-
-
-
-exit 0
-
-
-
-for file_token in $*
-do
- TOKEN=($(cat $file_token))
- HOST=${TOKEN[0]}
- FILE=${TOKEN[1]}
- TARGET=$(basename $FILE)
- chirp_get $HOST $TARGET $TARGET.imd
- RES=($(awk '{ sum += $1 } END { print sum,NR }' $TARGET.imd))
- echo "$TARGET.imd"
-
- SUM=$(($SUM+${RES[0]}))
- COUNT=$(($COUNT+${RES[1]}))
-done
-echo "SUM : $SUM"
-echo "COUNT: $COUNT"
-exit 0
+#rm *imd2 *imd
+exit 0
\ No newline at end of file
Added: SwiftApps/Swift-MapRed/TurboChirp/jobs_per_site.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/jobs_per_site.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/jobs_per_site.sh 2013-08-14 20:09:06 UTC (rev 6848)
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+cat $* > tmp;
+FILES=$*
+
+SITES=($(awk '{ print $1 }' tmp | sort -u))
+
+for SITE in ${SITES[*]}
+do
+# FILES=$(grep $SITE tmp | awk '{print $2}')
+ FILES=$(grep -l $SITE tgen*out)
+ echo $SITE ${FILES[*]}
+done
+rm tmp;
\ No newline at end of file
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/jobs_per_site.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/Swift-MapRed/TurboChirp/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/run.sh 2013-08-14 15:42:28 UTC (rev 6847)
+++ SwiftApps/Swift-MapRed/TurboChirp/run.sh 2013-08-14 20:09:06 UTC (rev 6848)
@@ -1,5 +1,16 @@
+#!/bin/bash
+
+if [ "$1" == "" ]
+then
+ PROG=teragen.swift
+else
+ PROG=$1
+fi
+
+./cleanup
export TURBINE_LOG=0
-export TURBINE_LAUNCH_OPTS="-f ./hosts.txt"
-stc teragen.swift >teragen.tcl
-turbine -n 8 teragen.tcl
-
+export TURBINE_LAUNCH_OPTS="-f /homes/yadunand/Swift-MapRed/TurboChirp/hosts.txt"
+stc $PROG > ${PROG%.swift}.tcl
+[ "$?" == "0" ] && echo "Compile done"
+turbine -n 8 ${PROG%.swift}.tcl
+[ "$?" == "0" ] && echo "Execute done"
Modified: SwiftApps/Swift-MapRed/TurboChirp/teragen.swift
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen.swift 2013-08-14 15:42:28 UTC (rev 6847)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen.swift 2013-08-14 20:09:06 UTC (rev 6848)
@@ -3,8 +3,10 @@
import random;
import string;
import sys;
+import location;
+import assert;
-app (file out, file err) gen_data (file run, int recsize){
+app (file out, file err) gen_data (file run, int recsize){
"/bin/bash" run recsize @stdout=out @stderr=err
}
@@ -12,34 +14,55 @@
"/bin/bash" comb array @stdout=out @stderr=err
}
-/*
-file tgen_out[] <simple_mapper; prefix="tgen", suffix=".out">;
-file tgen_err[] <simple_mapper; prefix="tgen", suffix=".err">;
-string dir = @arg("dir", "./");
-*/
-main{
-file wrapper = input_file("teragen_wrap.sh");
-file tgen_out[];
-file tgen_err[];
+app (file out, file err) comb_data_local (file comb, string array){
+ "/bin/bash" comb array @stdout=out @stderr=err
+}
-int loop = 7;
-int fsize = 10;
+app (file out) get_uniq_sites (file sites, file array[]){
+ "/bin/bash" sites array @stdout=out
+}
-foreach item,i in [0:loop-1] {
+main
+{
+ file wrapper = input_file("teragen_wrap.sh");
+ file tgen_out[];
+
+ int loop = 10;
+ int fsize = 1;
+
+ foreach item,i in [0:loop-1] {
+ file out <sprintf("tgen_%i.out", i)>;
+ file err <sprintf("tgen_%i.err", i)>;
+ (out, err) = gen_data(wrapper, fsize);
+ tgen_out[i]=out;
+ }
+
+ file jobs_per_site = input_file("jobs_per_site.sh");
+ file combiner_list <"uniq_sites">;
+ combiner_list = get_uniq_sites(jobs_per_site, tgen_out);
- file out <sprintf("tgenout-%i.dat", i)>;
- file err <sprintf("tgenerr-%i.dat", i)>;
+ string sites[] = split(readFile(combiner_list), "\n");
+
- (out, err) = gen_data(wrapper, fsize);
- tgen_out[i]=out;
- tgen_err[i]=err;
-}
+ file combine = input_file("combiner.sh");
+ //Here we are forcing jobs to sites to run local combiners
+ file comb_out[];
+ foreach site,i in sites {
+ file out1 <sprintf("sitecomb_%i.out", i)>;
+ file err1 <sprintf("sitecomb_%i.err", i)>;
+ string site_info[] = split(site, " ");
+ printf("Site: %s String: %s\n", site_info[0], site);
+ location rank = hostmap_one(site_info[0]);
+ printf("Site: %s Rank: %i", site_info[0], rank);
+
+ //(out1, err1) = comb_data_local(combine, site);
+
+ // Todo : Find @location usage
+ (out1, err1) = @location=rank comb_data_local(combine, site);
+ comb_out[i] = out1;
+ }
-file combine =input_file("combiner.sh");
-file final <"final_result">;
-file errs <"err_file">;
+ // Here's where we do a K_way reduce of the comb_out array
-(final, errs) = comb_data(combine, tgen_out);
-
}
Modified: SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl 2013-08-14 15:42:28 UTC (rev 6847)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl 2013-08-14 20:09:06 UTC (rev 6848)
@@ -1,353 +0,0 @@
-
-# Generated by stc version 0.2.0
-# date : 2013/08/13 16:55:59
-# Turbine version : 0.3.0
-# Input filename : /nfs2/yadunand/Swift-MapRed/TurboChirp/teragen.swift
-# Output filename : /nfs2/yadunand/Swift-MapRed/TurboChirp
-# STC home : /nfs2/yadunand/bin/exm/stc
-# Turbine home : /homes/yadunand/bin/exm/turbine
-# Compiler settings:
-# stc.auto-declare : true
-# stc.c_preprocess : true
-# stc.codegen.no-stack : true
-# stc.codegen.no-stack-vars : true
-# stc.compiler-debug : true
-# stc.debugging : COMMENTS
-# stc.exp.refcounting : false
-# stc.ic.output-file :
-# stc.input_filename : teragen.swift
-# stc.log.file :
-# stc.log.trace : false
-# stc.opt.algebra : false
-# stc.opt.cancel-refcounts : true
-# stc.opt.constant-fold : true
-# stc.opt.controlflow-fusion : true
-# stc.opt.dead-code-elim : true
-# stc.opt.disable-asserts : false
-# stc.opt.expand-dataflow-ops : true
-# stc.opt.expand-loop-threshold-insts: 256
-# stc.opt.expand-loop-threshold-iters: 16
-# stc.opt.expand-loops : true
-# stc.opt.flatten-nested : true
-# stc.opt.forward-dataflow : true
-# stc.opt.full-unroll : false
-# stc.opt.function-inline : false
-# stc.opt.function-inline-threshold: 500
-# stc.opt.function-signature : true
-# stc.opt.hoist : true
-# stc.opt.max-iterations : 10
-# stc.opt.piggyback-refcounts : true
-# stc.opt.pipeline : false
-# stc.opt.reorder-insts : false
-# stc.opt.shared-constants : true
-# stc.opt.unroll-loop-threshold-insts: 192
-# stc.opt.unroll-loop-threshold-iters: 8
-# stc.opt.unroll-loops : true
-# stc.opt.wait-coalesce : true
-# stc.output_filename :
-# stc.preproc.force-cpp : false
-# stc.preproc.force-gcc : false
-# stc.preprocess_only : false
-# stc.profile : false
-# stc.rpath :
-# stc.stc_home : /nfs2/yadunand/bin/exm/stc
-# stc.turbine.version : 0.3.0
-# stc.turbine_home : /homes/yadunand/bin/exm/turbine
-# stc.version : 0.2.0
-
-# Metadata:
-
-package require turbine 0.3.0
-namespace import turbine::*
-
-
-proc swift:constants { } {
- turbine::c::log "function:swift:constants"
- global c:i_10
- turbine::allocate_custom c:i_10 integer 1 1 1
- turbine::store_integer ${c:i_10} 10
- global c:s_combiner_s
- turbine::allocate_custom c:s_combiner_s string 1 1 1
- turbine::store_string ${c:s_combiner_s} "combiner.sh"
- global c:s_teragen_wr
- turbine::allocate_custom c:s_teragen_wr string 1 1 1
- turbine::store_string ${c:s_teragen_wr} "teragen_wrap.sh"
-}
-
-
-proc f:ternary { stack u:o u:b u:i1 u:i2 } {
- turbine::c::log "enter function: ternary"
- turbine::rule [ list ${u:b} ${u:i1} ${u:i2} ] "ternary-argwait ${stack} ${u:b} ${u:o} ${u:i2} ${u:i1}"
-}
-
-
-proc ternary-argwait { stack u:b u:o u:i2 u:i1 } {
- # Value __v_b with type $boolean was defined
- # Value __v_i1 with type $int was defined
- # Value __v_i2 with type $int was defined
- # Value __v_o with type $int was defined
- set v:b [ turbine::retrieve_integer ${u:b} CACHED 1 ]
- set v:i1 [ turbine::retrieve_integer ${u:i1} CACHED 1 ]
- set v:i2 [ turbine::retrieve_integer ${u:i2} CACHED 1 ]
- if { ${v:b} } { set v:o ${v:i1} } else { set v:o ${v:i2} }
- turbine::store_integer ${u:o} ${v:o}
-}
-
-
-proc f:file_type { stack u:t u:f } {
- turbine::c::log "enter function: file_type"
- turbine::rule [ list [ turbine::get_file_status ${u:f} ] ] "file_type-argwait ${stack} {${u:f}} ${u:t}"
-}
-
-
-proc file_type-argwait { stack u:f u:t } {
- # Value __v_f with type $file was defined
- # Value __v_t with type $string was defined
- set v:f [ turbine::get_file ${u:f} 1 ]
- set v:t [ file type [ lindex ${v:f} 0 ] ]
- turbine::store_string ${u:t} ${v:t}
-}
-
-
-proc f:file_exists { stack u:o u:f } {
- turbine::c::log "enter function: file_exists"
- turbine::rule [ list ${u:f} ] "file_exists-argwait ${stack} ${u:f} ${u:o}"
-}
-
-
-proc file_exists-argwait { stack u:f u:o } {
- # Value __v_f with type $string was defined
- # Value __v_o with type $boolean was defined
- set v:f [ turbine::retrieve_string ${u:f} CACHED 1 ]
- set v:o [ file exists ${v:f} ]
- turbine::store_integer ${u:o} ${v:o}
-}
-
-
-proc f:file_mtime { stack u:o u:f } {
- turbine::c::log "enter function: file_mtime"
- turbine::rule [ list ${u:f} ] "file_mtime-argwait ${stack} ${u:f} ${u:o}"
-}
-
-
-proc file_mtime-argwait { stack u:f u:o } {
- # Value __v_f with type $string was defined
- # Value __v_o with type $int was defined
- set v:f [ turbine::retrieve_string ${u:f} CACHED 1 ]
- set v:o [ file mtime ${v:f} ]
- turbine::store_integer ${u:o} ${v:o}
-}
-
-
-proc f:strlen { stack u:n u:s } {
- turbine::c::log "enter function: strlen"
- turbine::rule [ list ${u:s} ] "strlen-argwait ${stack} ${u:s} ${u:n}"
-}
-
-
-proc strlen-argwait { stack u:s u:n } {
- # Value __v_s with type $string was defined
- # Value __v_n with type $int was defined
- set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
- set v:n [ string length ${v:s} ]
- turbine::store_integer ${u:n} ${v:n}
-}
-
-
-proc f:trim { stack u:o u:s } {
- turbine::c::log "enter function: trim"
- turbine::rule [ list ${u:s} ] "trim-argwait ${stack} ${u:s} ${u:o}"
-}
-
-
-proc trim-argwait { stack u:s u:o } {
- # Value __v_s with type $string was defined
- # Value __v_o with type $string was defined
- set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
- set v:o [ string trim ${v:s} ]
- turbine::store_string ${u:o} ${v:o}
-}
-
-
-proc f:hash { stack u:h u:s } {
- turbine::c::log "enter function: hash"
- turbine::rule [ list ${u:s} ] "hash-argwait ${stack} ${u:s} ${u:h}"
-}
-
-
-proc hash-argwait { stack u:s u:h } {
- # Value __v_s with type $string was defined
- # Value __v_h with type $int was defined
- set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
- set v:h [ c_utils::hash ${v:s} ]
- turbine::store_integer ${u:h} ${v:h}
-}
-
-
-proc f:clock_seconds { stack u:t } {
- turbine::c::log "enter function: clock_seconds"
- # Value __v_t with type $int was defined
- set v:t [ clock seconds ]
- turbine::store_integer ${u:t} ${v:t}
-}
-
-
-proc f:clock { stack u:t } {
- turbine::c::log "enter function: clock"
- # Value __v_t with type $float was defined
- set v:t [ expr {[ clock clicks -milliseconds ] / 1000.0 } ]
- turbine::store_float ${u:t} ${v:t}
-}
-
-
-proc f:gen_data { stack u:out u:err u:run u:recsize dr:location } {
- turbine::c::log "enter function: gen_data"
- # Alias __filename_run with type string was defined
- # Alias __filename_out with type string was defined
- # Alias __filename_err with type string was defined
- set filename:run [ turbine::get_file_path ${u:run} ]
- set filename:out [ turbine::get_output_file_path ${u:out} ]
- set filename:err [ turbine::get_output_file_path ${u:err} ]
- turbine::rule [ list ${filename:err} ${filename:out} ${filename:run} ${u:recsize} [ turbine::get_file_status ${u:run} ] ] "gen_data-app-leaf0 ${stack} {${u:err}} ${filename:run} ${filename:out} ${filename:err} {${u:run}} ${u:recsize} {${u:out}}" target ${dr:location} type ${::turbine::WORK}
-}
-
-
-proc gen_data-app-leaf0 { stack u:err filename:run filename:out filename:err u:run u:recsize u:out } {
- # Value __v___filename_run with type $string was defined
- # Value __v_recsize with type $int was defined
- # Value __v___filename_out with type $string was defined
- # Value __v___filename_err with type $string was defined
- # Value __v_run with type $file was defined
- # Value __v_out with type $file was defined
- # Value __v_err with type $file was defined
- set v:__filename_run [ turbine::retrieve_string ${filename:run} CACHED 1 ]
- set v:recsize [ turbine::retrieve_integer ${u:recsize} CACHED 1 ]
- set v:__filename_out [ turbine::retrieve_string ${filename:out} CACHED 1 ]
- set v:__filename_err [ turbine::retrieve_string ${filename:err} CACHED 1 ]
- set v:run [ turbine::get_file ${u:run} 1 ]
- turbine::c::log [ list exec: /bin/bash ${v:__filename_run} ${v:recsize} [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ]
- turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ${v:__filename_run} ${v:recsize}
- set v:out [ turbine::create_local_file_ref ${v:__filename_out} ]
- set v:err [ turbine::create_local_file_ref ${v:__filename_err} ]
- turbine::set_file ${u:out} v:out
- turbine::set_file ${u:err} v:err
- turbine::decr_local_file_refcount v:out
- turbine::decr_local_file_refcount v:err
-}
-
-
-proc f:comb_data { stack u:out u:err u:comb u:array dr:location } {
- turbine::c::log "enter function: comb_data"
- # Alias __filename_comb with type string was defined
- # Alias __filename_out with type string was defined
- # Alias __filename_err with type string was defined
- set filename:comb [ turbine::get_file_path ${u:comb} ]
- set filename:out [ turbine::get_output_file_path ${u:out} ]
- set filename:err [ turbine::get_output_file_path ${u:err} ]
- turbine::deeprule [ list ${filename:comb} ${filename:err} ${filename:out} ${u:array} [ turbine::get_file_status ${u:comb} ] ] [ list 0 0 0 1 0 ] [ list 0 0 0 1 1 ] "comb_data-app-leaf0 ${stack} {${u:err}} ${filename:out} ${filename:err} {${u:comb}} ${filename:comb} {${u:out}} ${u:array}" target ${dr:location} type ${::turbine::WORK}
-}
-
-
-proc comb_data-app-leaf0 { stack u:err filename:out filename:err u:comb filename:comb u:out u:array } {
- # Value __v___filename_comb with type $string was defined
- # Value __v___filename_out with type $string was defined
- # Value __v___filename_err with type $string was defined
- # Value __v_comb with type $file was defined
- # Value __v_out with type $file was defined
- # Value __v_err with type $file was defined
- set v:__filename_comb [ turbine::retrieve_string ${filename:comb} CACHED 1 ]
- set v:__filename_out [ turbine::retrieve_string ${filename:out} CACHED 1 ]
- set v:__filename_err [ turbine::retrieve_string ${filename:err} CACHED 1 ]
- set v:comb [ turbine::get_file ${u:comb} 1 ]
- turbine::c::log [ list exec: /bin/bash ${v:__filename_comb} {*}[ turbine::unpack_args ${u:array} 1 1 ] [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ]
- turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ${v:__filename_comb} {*}[ turbine::unpack_args ${u:array} 1 1 ]
- set v:out [ turbine::create_local_file_ref ${v:__filename_out} ]
- set v:err [ turbine::create_local_file_ref ${v:__filename_err} ]
- turbine::set_file ${u:out} v:out
- turbine::set_file ${u:err} v:err
- turbine::decr_local_file_refcount v:out
- turbine::decr_local_file_refcount v:err
-}
-
-
-proc swift:main { } {
- turbine::c::log "enter function: main"
- set stack 0
- global c:s_combiner_s
- global c:s_teragen_wr
- lassign [ adlb::multicreate [ list ${::adlb::CONTAINER} ${::adlb::INTEGER} 1 1 ] [ list ${::adlb::CONTAINER} ${::adlb::INTEGER} 1 1 ] [ list ${::adlb::STRING} 1 ] [ list ${::adlb::STRING} 1 ] ] u:tgen_out t:1 t:10 t:11
- turbine::c::log "allocated u:tgen_out=<${u:tgen_out}> t:1=<${t:1}> t:10=<${t:10}> t:11=<${t:11}>"
- turbine::allocate_file2 u:wrapper "" 1
- turbine::allocate_file2 u:combine "" 1
- turbine::allocate_file2 u:final ${t:10} 1
- turbine::allocate_file2 u:errs ${t:11} 1
- # Swift l.21: assigning expression to wrapper
- turbine::input_file [ list ${u:wrapper} ] [ list ${c:s_teragen_wr} ]
- # Swift l.25: assigning expression to loop
- # Swift l.26: assigning expression to fsize
- turbine::range_work ${t:1} 0 6 1
- # Swift l.38: assigning expression to combine
- turbine::input_file [ list ${u:combine} ] [ list ${c:s_combiner_s} ]
- turbine::store_string ${t:10} "final_result"
- turbine::store_string ${t:11} "err_file"
- # Swift l.42: assigning expression to final
- # Swift l.42: assigning expression to errs
- f:comb_data ${stack} ${u:final} ${u:errs} ${u:combine} ${u:tgen_out} -100
- # Swift l.33: assigning expression to out
- # Swift l.33: assigning expression to err
- # Swift l.34: assigning expression to tgen_out[_]
- # Swift l.35: assigning expression to tgen_err[_]
- set tcltmp:container_sz [ adlb::enumerate ${t:1} count all 0 ]
- set tcltmp:iters ${tcltmp:container_sz}
- adlb::slot_create ${u:tgen_out} [ expr { ${tcltmp:iters} - 1 } ]
- main-foreach0:outer ${stack} ${u:wrapper} ${u:tgen_out} ${t:1} 0 [ expr { ${tcltmp:container_sz} - 1 } ] 1
-}
-
-
-proc main-foreach0:outer { stack u:wrapper u:tgen_out t:1 tcltmp:lo tcltmp:hi tcltmp:inc } {
- set tcltmp:itersleft [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
- if { [ expr { ${tcltmp:itersleft} <= 0 } ] } {
- return
- }
- if { [ expr { ${tcltmp:itersleft} <= 64 } ] } {
- main-foreach0:inner ${stack} ${u:wrapper} ${u:tgen_out} ${t:1} ${tcltmp:lo} ${tcltmp:hi} ${tcltmp:inc}
- } else {
- set tcltmp:skip [ expr { ${tcltmp:inc} * max(64,((${tcltmp:itersleft} - 1) / 16) + 1) } ]
- for { set tcltmp:splitstart ${tcltmp:lo} } { ${tcltmp:splitstart} <= ${tcltmp:hi} } { incr tcltmp:splitstart ${tcltmp:skip} } {
- set tcltmp:splitend [ expr { min(${tcltmp:hi},${tcltmp:splitstart} + ${tcltmp:skip} - 1) } ]
- set tcltmp:prio [ turbine::get_priority ]
- turbine::set_priority ${tcltmp:prio}
- adlb::spawn 1 "command main-foreach0:outer ${stack} {${u:wrapper}} ${u:tgen_out} ${t:1} ${tcltmp:splitstart} ${tcltmp:splitend} ${tcltmp:inc}"
- turbine::reset_priority
- }
- }
-}
-
-
-proc main-foreach0:inner { stack u:wrapper u:tgen_out t:1 tcltmp:lo tcltmp:hi tcltmp:inc } {
- set tcltmp:splitlen [ expr { ${tcltmp:hi} - ${tcltmp:lo} + 1 } ]
- set tcltmp:contents [ adlb::enumerate ${t:1} dict ${tcltmp:splitlen} ${tcltmp:lo} ]
- dict for {v:i u:item} ${tcltmp:contents} {
- global c:i_10
- # Value __ov___t5 with type $string was defined
- # Value __ov___t7 with type $string was defined
- lassign [ adlb::multicreate [ list ${::adlb::STRING} 1 ] [ list ${::adlb::STRING} 1 ] ] t:5 t:7
- turbine::c::log "allocated t:5=<${t:5}> t:7=<${t:7}>"
- turbine::allocate_file2 u:out ${t:5} 1
- turbine::allocate_file2 u:err ${t:7} 1
- set optv:__t5 [ eval format [ list "tgenout-%i.dat" ${v:i} ] ]
- turbine::store_string ${t:5} ${optv:__t5}
- set optv:__t7 [ eval format [ list "tgenerr-%i.dat" ${v:i} ] ]
- turbine::store_string ${t:7} ${optv:__t7}
- f:gen_data ${stack} ${u:out} ${u:err} ${u:wrapper} ${c:i_10} -100
- turbine::container_insert ${u:tgen_out} ${v:i} ${u:out} 1
- }
- set tcltmp:iters [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
-}
-
-turbine::defaults
-turbine::init $engines $servers
-turbine::check_constants "WORKER" ${turbine::WORK_TASK} 0 "CONTROL" ${turbine::CONTROL_TASK} 1 "ADLB_RANK_ANY" ${adlb::RANK_ANY} -100
-turbine::start swift:main swift:constants
-turbine::finalize
-
Modified: SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh 2013-08-14 15:42:28 UTC (rev 6847)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh 2013-08-14 20:09:06 UTC (rev 6848)
@@ -34,6 +34,7 @@
FILE=$FOLDER/$FILE;
shuf -i $LOWERLIMIT-$UPPERLIMIT -n $(($SLICESIZE*$ARG1)) | awk '{printf "%-99s\n", $0}' > $FILE
sleep 5;
-HOST=$(hostname -f)
-echo "$HOST $FILE"
+#HOST=$(hostname -f)
+#echo "$HOST $FILE"
+echo "$HOSTNAME $FILE"
exit 0
\ No newline at end of file
More information about the Swift-commit
mailing list