[Swift-commit] r7133 - SwiftApps/Swift-MapRed/TurboChirp
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Fri Oct 4 11:35:12 CDT 2013
Author: yadunandb
Date: 2013-10-04 11:35:12 -0500 (Fri, 04 Oct 2013)
New Revision: 7133
Modified:
SwiftApps/Swift-MapRed/TurboChirp/combiner.sh
SwiftApps/Swift-MapRed/TurboChirp/local_combiner.sh
SwiftApps/Swift-MapRed/TurboChirp/run.sh
SwiftApps/Swift-MapRed/TurboChirp/teragen.swift
SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl
SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh
Log:
Committing latest working version of TurboChirp on Beagle
Modified: SwiftApps/Swift-MapRed/TurboChirp/combiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/combiner.sh 2013-10-04 16:33:52 UTC (rev 7132)
+++ SwiftApps/Swift-MapRed/TurboChirp/combiner.sh 2013-10-04 16:35:12 UTC (rev 7133)
@@ -20,9 +20,10 @@
if [ "$HOST" != "$CURRENT_HOST" ]
then
# Fetch if file is remote
- chirp_get $HOST $TARGET /tmp/$TARGET.imd
- RES=($(awk '{ sum += $1 } END { print sum,NR }' /tmp/$TARGET.imd))
- rm /tmp/$TARGET.imd &> /dev/null &
+ chirp_get $HOST $TARGET /dev/shm/$TARGET.imd
+ echo "$HOST $TARGET /dev/shm/$TARGET.imd : ";cat /dev/shm/$TARGET.imd
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' /dev/shm/$TARGET.imd))
+ rm /dev/shm/$TARGET.imd &> /dev/null &
else
# Process directly if file is local
RES=($(awk '{ sum += $1 } END { print sum,NR }' $FILE))
@@ -42,7 +43,8 @@
SUM=$(awk '{ sum += $1 } END { print sum }' $ID*imd2)
COUNT=$(awk '{ sum += $2 } END { print sum }' $ID*imd2)
-echo "$SUM $COUNT" > $CHIRP_FOLDER/$CURRENT_HOST.$ID.imd
-echo "$CURRENT_HOST $CHIRP_FOLDER/$CURRENT_HOST.$ID.imd"
+#echo "$SUM $COUNT" > $CHIRP_FOLDER/$CURRENT_HOST.$ID.imd
+echo "$SUM $COUNT"
+#echo "$CURRENT_HOST $CHIRP_FOLDER/$CURRENT_HOST.$ID.imd"
rm $ID*{imd,imd2} &> /dev/null &
exit 0
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/TurboChirp/local_combiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/local_combiner.sh 2013-10-04 16:33:52 UTC (rev 7132)
+++ SwiftApps/Swift-MapRed/TurboChirp/local_combiner.sh 2013-10-04 16:35:12 UTC (rev 7133)
@@ -6,8 +6,8 @@
PASSED_SITE=${ARGS[0]}
FILES=(${ARGS[*]:1})
-#CURRENT_HOST=$(hostname -f)
-CURRENT_HOST=$HOSTNAME
+CURRENT_HOST=$(hostname -f)
+#CURRENT_HOST=$HOSTNAME
SUM=0
COUNT=0
@@ -45,5 +45,6 @@
COUNT=$(awk '{ sum += $2 } END { print sum }' $ID*imd2)
echo "$SUM $COUNT" > $CHIRP_FOLDER/$CURRENT_HOST.$ID.imd
echo "$CURRENT_HOST $CHIRP_FOLDER/$CURRENT_HOST.$ID.imd"
+echo "$SUM $COUNT"
rm $ID*{imd,imd2} &> /dev/null &
exit 0
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/TurboChirp/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/run.sh 2013-10-04 16:33:52 UTC (rev 7132)
+++ SwiftApps/Swift-MapRed/TurboChirp/run.sh 2013-10-04 16:35:12 UTC (rev 7133)
@@ -7,6 +7,11 @@
PROG=$1
fi
+
+#LOGIN NODE
+#~wozniak/Public/turbine-trunk-beagle-login
+#export PATH=$PATH
+
./cleanup
export TURBINE_LOG=1
export ADLB_EXHAUST_TIME=0.1
Modified: SwiftApps/Swift-MapRed/TurboChirp/teragen.swift
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen.swift 2013-10-04 16:33:52 UTC (rev 7132)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen.swift 2013-10-04 16:35:12 UTC (rev 7133)
@@ -24,13 +24,13 @@
main
{
- file wrapper = input_file("teragen_wrap.sh");
+ file wrapper = input_file("/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/teragen_wrap.sh");
file tgen_out[];
- //int loop = 10;
- int loop=toint(argv("loop"));
- //int fsize = 1;
- int fsize = toint(argv("fsize"));
+ int loop = 10;
+ //int loop=toint(argv("loop"));
+ int fsize = 1;
+ //int fsize = toint(argv("fsize"));
foreach item,i in [0:loop-1] {
file out <sprintf("intermediate/tgen_%i.out", i)>;
@@ -39,37 +39,43 @@
tgen_out[i]=out;
}
- file jobs_per_site = input_file("jobs_per_site.sh");
- file combiner_list <"uniq_sites">;
+ file jobs_per_site = input_file("/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/jobs_per_site.sh");
+ file combiner_list <"intermediate/uniq_sites">;
combiner_list = get_uniq_sites(jobs_per_site, tgen_out);
+
string sites[] = split(read(combiner_list), "\n");
-
- file local_combine = input_file("local_combiner.sh");
+ file local_combine = input_file("/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/local_combiner.sh");
//Here we are forcing jobs to sites to run local combiners
file comb_out[];
foreach site,i in sites {
+ // TODO: Add check for site being empty string
+ if ( strlen(site) > 0 ){
file out1 <sprintf("intermediate/sitecomb_%i.out", i)>;
file err1 <sprintf("intermediate/sitecomb_%i.err", i)>;
string site_info[] = split(site, " ");
- location rank = hostmap_one(site_info[0]);
-
- //printf("Site: %s Rank: %i String: %s\n", site_info[0], rank, site);
+ //TODO: hostmap_one_worker
+ location rank = hostmap_one_worker(site_info[0]);
+
+ printf("Site: %s Rank: %i String: %s\n", site_info[0], rank, site);
+
(out1, err1) = @location=rank comb_data_local(local_combine, site);
comb_out[i] = out1;
+ }else{
+ printf("Empty string\n");
+ }
}
foreach file_item in comb_out {
printf("Item in comb_out : %s \n", filename(file_item));
}
-
- file combine = input_file("combiner.sh");
+
+ file combine = input_file("/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/combiner.sh");
// We are doing a simple single step reduce here
file final <"final_result">;
file errs <"final_errs">;
(final, errs) = comb_data (combine, comb_out );
-
}
Modified: SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl 2013-10-04 16:33:52 UTC (rev 7132)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl 2013-10-04 16:35:12 UTC (rev 7133)
@@ -1,11 +1,11 @@
# Generated by stc version 0.2.0
-# date : 2013/08/15 15:37:06
+# date : 2013/09/30 14:05:49
# Turbine version : 0.3.0
-# Input filename : /nfs2/yadunand/Swift-MapRed/TurboChirp/teragen.swift
-# Output filename : /nfs2/yadunand/Swift-MapRed/TurboChirp
-# STC home : /nfs2/wozniak/Public/stc
-# Turbine home : /homes/wozniak/Public/turbine
+# Input filename : /lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/teragen.swift
+# Output filename : /lustre/beagle/yadunandb/Swift-MapRed/TurboChirp
+# STC home : /lustre/beagle/yadunandb/bin/exm-install/stc
+# Turbine home : /lustre/beagle/yadunandb/bin/exm-install/turbine
# Compiler settings:
# stc.auto-declare : true
# stc.c_preprocess : true
@@ -20,6 +20,7 @@
# stc.log.trace : false
# stc.must_pass_wait_vars : true
# stc.opt.algebra : false
+# stc.opt.array-build : true
# stc.opt.cancel-refcounts : true
# stc.opt.constant-fold : true
# stc.opt.controlflow-fusion : true
@@ -51,9 +52,9 @@
# stc.preprocess_only : false
# stc.profile : false
# stc.rpath :
-# stc.stc_home : /nfs2/wozniak/Public/stc
+# stc.stc_home : /lustre/beagle/yadunandb/bin/exm-install/stc
# stc.turbine.version : 0.3.0
-# stc.turbine_home : /homes/wozniak/Public/turbine
+# stc.turbine_home : /lustre/beagle/yadunandb/bin/exm-install/turbine
# stc.version : 0.2.0
# Metadata:
@@ -64,243 +65,54 @@
proc swift:constants { } {
turbine::c::log "function:swift:constants"
+ global c:i_1
+ turbine::allocate_custom c:i_1 integer 1 1 1
+ turbine::store_integer ${c:i_1} 1
global c:s__
turbine::allocate_custom c:s__ string 1 1 1
- turbine::store_string ${c:s__} "\n"
+ turbine::store_string ${c:s__} " "
global c:s__-1
turbine::allocate_custom c:s__-1 string 1 1 1
- turbine::store_string ${c:s__-1} " "
- global c:s_combiner_s
- turbine::allocate_custom c:s_combiner_s string 1 1 1
- turbine::store_string ${c:s_combiner_s} "combiner.sh"
- global c:s_jobs_per_s
- turbine::allocate_custom c:s_jobs_per_s string 1 1 1
- turbine::store_string ${c:s_jobs_per_s} "jobs_per_site.sh"
- global c:s_local_comb
- turbine::allocate_custom c:s_local_comb string 1 1 1
- turbine::store_string ${c:s_local_comb} "local_combiner.sh"
- global c:s_teragen_wr
- turbine::allocate_custom c:s_teragen_wr string 1 1 1
- turbine::store_string ${c:s_teragen_wr} "teragen_wrap.sh"
+ turbine::store_string ${c:s__-1} "\n"
}
-proc f:ternary { stack u:o u:b u:i1 u:i2 } {
- turbine::c::log "enter function: ternary"
- turbine::read_refcount_incr ${u:i1} 1
- turbine::read_refcount_incr ${u:b} 1
- turbine::read_refcount_incr ${u:i2} 1
- turbine::rule [ list ${u:b} ${u:i1} ${u:i2} ] "ternary-argwait ${stack} ${u:b} ${u:o} ${u:i2} ${u:i1}"
-}
-
-
-proc ternary-argwait { stack u:b u:o u:i2 u:i1 } {
- # Value __v_b with type $boolean was defined
- # Value __v_i1 with type $int was defined
- # Value __v_i2 with type $int was defined
- # Value __v_o with type $int was defined
- set v:b [ turbine::retrieve_integer ${u:b} CACHED 1 ]
- set v:i1 [ turbine::retrieve_integer ${u:i1} CACHED 1 ]
- set v:i2 [ turbine::retrieve_integer ${u:i2} CACHED 1 ]
- if { ${v:b} } { set v:o ${v:i1} } else { set v:o ${v:i2} }
- turbine::store_integer ${u:o} ${v:o}
-}
-
-
-proc f:file_type { stack u:t u:f } {
- turbine::c::log "enter function: file_type"
- turbine::file_read_refcount_incr ${u:f} 1
- turbine::rule [ list [ turbine::get_file_status ${u:f} ] ] "file_type-argwait ${stack} {${u:f}} ${u:t}"
-}
-
-
-proc file_type-argwait { stack u:f u:t } {
- # Value __v_f with type $file was defined
- # Value __v_t with type $string was defined
- set v:f [ turbine::get_file ${u:f} 1 ]
- set v:t [ file type [ lindex ${v:f} 0 ] ]
- turbine::store_string ${u:t} ${v:t}
-}
-
-
-proc f:file_exists { stack u:o u:f } {
- turbine::c::log "enter function: file_exists"
- turbine::read_refcount_incr ${u:f} 1
- turbine::rule [ list ${u:f} ] "file_exists-argwait ${stack} ${u:f} ${u:o}"
-}
-
-
-proc file_exists-argwait { stack u:f u:o } {
- # Value __v_f with type $string was defined
- # Value __v_o with type $boolean was defined
- set v:f [ turbine::retrieve_string ${u:f} CACHED 1 ]
- set v:o [ file exists ${v:f} ]
- turbine::store_integer ${u:o} ${v:o}
-}
-
-
-proc f:file_mtime { stack u:o u:f } {
- turbine::c::log "enter function: file_mtime"
- turbine::read_refcount_incr ${u:f} 1
- turbine::rule [ list ${u:f} ] "file_mtime-argwait ${stack} ${u:f} ${u:o}"
-}
-
-
-proc file_mtime-argwait { stack u:f u:o } {
- # Value __v_f with type $string was defined
- # Value __v_o with type $int was defined
- set v:f [ turbine::retrieve_string ${u:f} CACHED 1 ]
- set v:o [ file mtime ${v:f} ]
- turbine::store_integer ${u:o} ${v:o}
-}
-
-
-proc f:strlen { stack u:n u:s } {
- turbine::c::log "enter function: strlen"
- turbine::read_refcount_incr ${u:s} 1
- turbine::rule [ list ${u:s} ] "strlen-argwait ${stack} ${u:s} ${u:n}"
-}
-
-
-proc strlen-argwait { stack u:s u:n } {
- # Value __v_s with type $string was defined
- # Value __v_n with type $int was defined
- set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
- set v:n [ string length ${v:s} ]
- turbine::store_integer ${u:n} ${v:n}
-}
-
-
-proc f:trim { stack u:o u:s } {
- turbine::c::log "enter function: trim"
- turbine::read_refcount_incr ${u:s} 1
- turbine::rule [ list ${u:s} ] "trim-argwait ${stack} ${u:s} ${u:o}"
-}
-
-
-proc trim-argwait { stack u:s u:o } {
- # Value __v_s with type $string was defined
- # Value __v_o with type $string was defined
- set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
- set v:o [ string trim ${v:s} ]
- turbine::store_string ${u:o} ${v:o}
-}
-
-
-proc f:hash { stack u:h u:s } {
- turbine::c::log "enter function: hash"
- turbine::read_refcount_incr ${u:s} 1
- turbine::rule [ list ${u:s} ] "hash-argwait ${stack} ${u:s} ${u:h}"
-}
-
-
-proc hash-argwait { stack u:s u:h } {
- # Value __v_s with type $string was defined
- # Value __v_h with type $int was defined
- set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
- set v:h [ c_utils::hash ${v:s} ]
- turbine::store_integer ${u:h} ${v:h}
-}
-
-
-proc f:clock_seconds { stack u:t } {
- turbine::c::log "enter function: clock_seconds"
- # Value __v_t with type $int was defined
- set v:t [ clock seconds ]
- turbine::store_integer ${u:t} ${v:t}
-}
-
-
-proc f:clock { stack u:t } {
- turbine::c::log "enter function: clock"
- # Value __v_t with type $float was defined
- set v:t [ expr {[ clock clicks -milliseconds ] / 1000.0 } ]
- turbine::store_float ${u:t} ${v:t}
-}
-
-
-proc f:location_from_rank { stack u:loc u:rank } {
- turbine::c::log "enter function: location_from_rank"
- turbine::read_refcount_incr ${u:rank} 1
- turbine::rule [ list ${u:rank} ] "location_from_rank-argwait ${stack} ${u:rank} ${u:loc}"
-}
-
-
-proc location_from_rank-argwait { stack u:rank u:loc } {
- # Value __v_rank with type $int was defined
- # Value __v_loc with type $int was defined
- set v:rank [ turbine::retrieve_integer ${u:rank} CACHED 1 ]
- set v:loc ${v:rank}
- turbine::store_integer ${u:loc} ${v:loc}
-}
-
-
-proc f:random_worker { stack u:loc } {
- turbine::c::log "enter function: random_worker"
- # Value __v_loc with type $int was defined
- set v:loc [ ::turbine::random_worker ]
- turbine::store_integer ${u:loc} ${v:loc}
-}
-
-
-proc f:random_engine { stack u:loc } {
- turbine::c::log "enter function: random_engine"
- # Value __v_loc with type $int was defined
- set v:loc [ ::turbine::random_engine ]
- turbine::store_integer ${u:loc} ${v:loc}
-}
-
-
-proc f:hostmap_one { stack u:rank u:name } {
- turbine::c::log "enter function: hostmap_one"
- turbine::read_refcount_incr ${u:name} 1
- turbine::rule [ list ${u:name} ] "hostmap_one-argwait ${stack} ${u:rank} ${u:name}"
-}
-
-
-proc hostmap_one-argwait { stack u:rank u:name } {
- # Value __v_name with type $string was defined
- # Value __v_rank with type $int was defined
- set v:name [ turbine::retrieve_string ${u:name} CACHED 1 ]
- set v:rank [ draw [ adlb::hostmap ${v:name} ] ]
- turbine::store_integer ${u:rank} ${v:rank}
-}
-
-
-proc f:hostmap_one_worker { stack u:rank u:name } {
- turbine::c::log "enter function: hostmap_one_worker"
- turbine::read_refcount_incr ${u:name} 1
- turbine::rule [ list ${u:name} ] "hostmap_one_worker-argwait ${stack} ${u:rank} ${u:name}"
-}
-
-
-proc hostmap_one_worker-argwait { stack u:rank u:name } {
- # Value __v_name with type $string was defined
- # Value __v_rank with type $int was defined
- set v:name [ turbine::retrieve_string ${u:name} CACHED 1 ]
- set v:rank [ ::turbine::random_rank WORKER [ adlb::hostmap ${v:name} ] ]
- turbine::store_integer ${u:rank} ${v:rank}
-}
-
-
proc f:gen_data { stack u:out u:err u:run u:recsize dr:location } {
turbine::c::log "enter function: gen_data"
# Alias __filename_run with type string was defined
# Alias __filename_out with type string was defined
+ # Value __ov_mapped_out with type $boolean was defined
# Alias __filename_err with type string was defined
+ # Value __ov_mapped_err with type $boolean was defined
turbine::file_read_refcount_incr ${u:run} 1
turbine::read_refcount_incr ${u:recsize} 1
set filename:run [ turbine::get_file_path ${u:run} ]
turbine::read_refcount_incr ${filename:run} 1
- set filename:out [ turbine::get_output_file_path ${u:out} ]
+ set optv:mapped_out [ turbine::is_file_mapped ${u:out} ]
+ if { ${optv:mapped_out} } {
+ set filename:out [ turbine::get_file_path ${u:out} ]
+ } else {
+ # Value __of_out with type $string was defined
+ set optf:out [ turbine::mktemp ]
+ turbine::set_filename_val ${u:out} ${optf:out}
+ set filename:out [ turbine::get_file_path ${u:out} ]
+ }
turbine::read_refcount_incr ${filename:out} 1
- set filename:err [ turbine::get_output_file_path ${u:err} ]
+ set optv:mapped_err [ turbine::is_file_mapped ${u:err} ]
+ if { ${optv:mapped_err} } {
+ set filename:err [ turbine::get_file_path ${u:err} ]
+ } else {
+ # Value __of_err with type $string was defined
+ set optf:err [ turbine::mktemp ]
+ turbine::set_filename_val ${u:err} ${optf:err}
+ set filename:err [ turbine::get_file_path ${u:err} ]
+ }
turbine::read_refcount_incr ${filename:err} 1
- turbine::rule [ list ${filename:err} ${filename:out} ${filename:run} ${u:recsize} [ turbine::get_file_status ${u:run} ] ] "gen_data-app-leaf0 ${stack} {${u:err}} ${filename:run} ${filename:out} ${filename:err} {${u:run}} ${u:recsize} {${u:out}}" target ${dr:location} type ${::turbine::WORK}
+ turbine::rule [ list ${filename:err} ${filename:out} ${filename:run} ${u:recsize} [ turbine::get_file_status ${u:run} ] ] "gen_data-app-leaf0 ${stack} ${optv:mapped_out} {${u:err}} ${filename:out} ${filename:run} ${optv:mapped_err} ${filename:err} {${u:run}} ${u:recsize} {${u:out}}" target ${dr:location} type ${::turbine::WORK}
}
-proc gen_data-app-leaf0 { stack u:err filename:run filename:out filename:err u:run u:recsize u:out } {
+proc gen_data-app-leaf0 { stack optv:mapped_out u:err filename:out filename:run optv:mapped_err filename:err u:run u:recsize u:out } {
# Value __v___filename_run with type $string was defined
# Value __v_recsize with type $int was defined
# Value __v___filename_out with type $string was defined
@@ -313,10 +125,20 @@
set v:__filename_out [ turbine::retrieve_string ${filename:out} CACHED 1 ]
set v:__filename_err [ turbine::retrieve_string ${filename:err} CACHED 1 ]
set v:run [ turbine::get_file ${u:run} 1 ]
+ if { ${optv:mapped_out} } {
+ set tcltmp:init_rc 2
+ } else {
+ set tcltmp:init_rc 1
+ }
+ set v:out [ turbine::create_local_file_ref ${v:__filename_out} ${tcltmp:init_rc} ]
+ if { ${optv:mapped_err} } {
+ set tcltmp:init_rc 2
+ } else {
+ set tcltmp:init_rc 1
+ }
+ set v:err [ turbine::create_local_file_ref ${v:__filename_err} ${tcltmp:init_rc} ]
turbine::c::log [ list exec: /bin/bash ${v:__filename_run} ${v:recsize} [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ]
turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ${v:__filename_run} ${v:recsize}
- set v:out [ turbine::create_local_file_ref ${v:__filename_out} ]
- set v:err [ turbine::create_local_file_ref ${v:__filename_err} ]
turbine::set_file ${u:out} v:out
turbine::set_file ${u:err} v:err
turbine::decr_local_file_refcount v:out
@@ -328,20 +150,38 @@
turbine::c::log "enter function: comb_data"
# Alias __filename_comb with type string was defined
# Alias __filename_out with type string was defined
+ # Value __ov_mapped_out with type $boolean was defined
# Alias __filename_err with type string was defined
+ # Value __ov_mapped_err with type $boolean was defined
turbine::file_read_refcount_incr ${u:comb} 1
turbine::read_refcount_incr ${u:array} 1
set filename:comb [ turbine::get_file_path ${u:comb} ]
turbine::read_refcount_incr ${filename:comb} 1
- set filename:out [ turbine::get_output_file_path ${u:out} ]
+ set optv:mapped_out [ turbine::is_file_mapped ${u:out} ]
+ if { ${optv:mapped_out} } {
+ set filename:out [ turbine::get_file_path ${u:out} ]
+ } else {
+ # Value __of_out with type $string was defined
+ set optf:out [ turbine::mktemp ]
+ turbine::set_filename_val ${u:out} ${optf:out}
+ set filename:out [ turbine::get_file_path ${u:out} ]
+ }
turbine::read_refcount_incr ${filename:out} 1
- set filename:err [ turbine::get_output_file_path ${u:err} ]
+ set optv:mapped_err [ turbine::is_file_mapped ${u:err} ]
+ if { ${optv:mapped_err} } {
+ set filename:err [ turbine::get_file_path ${u:err} ]
+ } else {
+ # Value __of_err with type $string was defined
+ set optf:err [ turbine::mktemp ]
+ turbine::set_filename_val ${u:err} ${optf:err}
+ set filename:err [ turbine::get_file_path ${u:err} ]
+ }
turbine::read_refcount_incr ${filename:err} 1
- turbine::deeprule [ list ${filename:comb} ${filename:err} ${filename:out} ${u:array} [ turbine::get_file_status ${u:comb} ] ] [ list 0 0 0 1 0 ] [ list 0 0 0 1 1 ] "comb_data-app-leaf0 ${stack} {${u:err}} ${filename:out} ${filename:err} {${u:comb}} ${filename:comb} {${u:out}} ${u:array}" target ${dr:location} type ${::turbine::WORK}
+ turbine::deeprule [ list ${filename:comb} ${filename:err} ${filename:out} ${u:array} [ turbine::get_file_status ${u:comb} ] ] [ list 0 0 0 1 0 ] [ list 0 0 0 1 1 ] "comb_data-app-leaf0 ${stack} ${optv:mapped_out} {${u:err}} ${filename:out} ${optv:mapped_err} ${filename:err} {${u:comb}} ${filename:comb} {${u:out}} ${u:array}" target ${dr:location} type ${::turbine::WORK}
}
-proc comb_data-app-leaf0 { stack u:err filename:out filename:err u:comb filename:comb u:out u:array } {
+proc comb_data-app-leaf0 { stack optv:mapped_out u:err filename:out optv:mapped_err filename:err u:comb filename:comb u:out u:array } {
# Value __v___filename_comb with type $string was defined
# Value __v___filename_out with type $string was defined
# Value __v___filename_err with type $string was defined
@@ -352,10 +192,21 @@
set v:__filename_out [ turbine::retrieve_string ${filename:out} CACHED 1 ]
set v:__filename_err [ turbine::retrieve_string ${filename:err} CACHED 1 ]
set v:comb [ turbine::get_file ${u:comb} 1 ]
- turbine::c::log [ list exec: /bin/bash ${v:__filename_comb} {*}[ turbine::unpack_args ${u:array} 1 1 ] [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ]
- turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ${v:__filename_comb} {*}[ turbine::unpack_args ${u:array} 1 1 ]
- set v:out [ turbine::create_local_file_ref ${v:__filename_out} ]
- set v:err [ turbine::create_local_file_ref ${v:__filename_err} ]
+ if { ${optv:mapped_out} } {
+ set tcltmp:init_rc 2
+ } else {
+ set tcltmp:init_rc 1
+ }
+ set v:out [ turbine::create_local_file_ref ${v:__filename_out} ${tcltmp:init_rc} ]
+ if { ${optv:mapped_err} } {
+ set tcltmp:init_rc 2
+ } else {
+ set tcltmp:init_rc 1
+ }
+ set v:err [ turbine::create_local_file_ref ${v:__filename_err} ${tcltmp:init_rc} ]
+ set tcltmp:unpacked1 [ turbine::unpack_args ${u:array} 1 1 ]
+ turbine::c::log [ list exec: /bin/bash ${v:__filename_comb} {*}${tcltmp:unpacked1} [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ]
+ turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ${v:__filename_comb} {*}${tcltmp:unpacked1}
turbine::set_file ${u:out} v:out
turbine::set_file ${u:err} v:err
turbine::decr_local_file_refcount v:out
@@ -368,20 +219,38 @@
turbine::c::log "enter function: comb_data_local"
# Alias __filename_comb with type string was defined
# Alias __filename_out with type string was defined
+ # Value __ov_mapped_out with type $boolean was defined
# Alias __filename_err with type string was defined
+ # Value __ov_mapped_err with type $boolean was defined
turbine::file_read_refcount_incr ${u:comb} 1
turbine::read_refcount_incr ${u:array} 1
set filename:comb [ turbine::get_file_path ${u:comb} ]
turbine::read_refcount_incr ${filename:comb} 1
- set filename:out [ turbine::get_output_file_path ${u:out} ]
+ set optv:mapped_out [ turbine::is_file_mapped ${u:out} ]
+ if { ${optv:mapped_out} } {
+ set filename:out [ turbine::get_file_path ${u:out} ]
+ } else {
+ # Value __of_out with type $string was defined
+ set optf:out [ turbine::mktemp ]
+ turbine::set_filename_val ${u:out} ${optf:out}
+ set filename:out [ turbine::get_file_path ${u:out} ]
+ }
turbine::read_refcount_incr ${filename:out} 1
- set filename:err [ turbine::get_output_file_path ${u:err} ]
+ set optv:mapped_err [ turbine::is_file_mapped ${u:err} ]
+ if { ${optv:mapped_err} } {
+ set filename:err [ turbine::get_file_path ${u:err} ]
+ } else {
+ # Value __of_err with type $string was defined
+ set optf:err [ turbine::mktemp ]
+ turbine::set_filename_val ${u:err} ${optf:err}
+ set filename:err [ turbine::get_file_path ${u:err} ]
+ }
turbine::read_refcount_incr ${filename:err} 1
- turbine::rule [ list ${filename:comb} ${filename:err} ${filename:out} ${u:array} [ turbine::get_file_status ${u:comb} ] ] "comb_data_local-app-leaf0 ${stack} {${u:err}} ${filename:out} ${filename:err} {${u:comb}} ${filename:comb} {${u:out}} ${u:array}" target ${dr:location} type ${::turbine::WORK}
+ turbine::rule [ list ${filename:comb} ${filename:err} ${filename:out} ${u:array} [ turbine::get_file_status ${u:comb} ] ] "comb_data_local-app-leaf0 ${stack} ${optv:mapped_out} {${u:err}} ${filename:out} ${optv:mapped_err} ${filename:err} {${u:comb}} ${filename:comb} {${u:out}} ${u:array}" target ${dr:location} type ${::turbine::WORK}
}
-proc comb_data_local-app-leaf0 { stack u:err filename:out filename:err u:comb filename:comb u:out u:array } {
+proc comb_data_local-app-leaf0 { stack optv:mapped_out u:err filename:out optv:mapped_err filename:err u:comb filename:comb u:out u:array } {
# Value __v___filename_comb with type $string was defined
# Value __v_array with type $string was defined
# Value __v___filename_out with type $string was defined
@@ -394,10 +263,20 @@
set v:__filename_out [ turbine::retrieve_string ${filename:out} CACHED 1 ]
set v:__filename_err [ turbine::retrieve_string ${filename:err} CACHED 1 ]
set v:comb [ turbine::get_file ${u:comb} 1 ]
+ if { ${optv:mapped_out} } {
+ set tcltmp:init_rc 2
+ } else {
+ set tcltmp:init_rc 1
+ }
+ set v:out [ turbine::create_local_file_ref ${v:__filename_out} ${tcltmp:init_rc} ]
+ if { ${optv:mapped_err} } {
+ set tcltmp:init_rc 2
+ } else {
+ set tcltmp:init_rc 1
+ }
+ set v:err [ turbine::create_local_file_ref ${v:__filename_err} ${tcltmp:init_rc} ]
turbine::c::log [ list exec: /bin/bash ${v:__filename_comb} ${v:array} [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ]
turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} "stderr" ${v:__filename_err} ] ${v:__filename_comb} ${v:array}
- set v:out [ turbine::create_local_file_ref ${v:__filename_out} ]
- set v:err [ turbine::create_local_file_ref ${v:__filename_err} ]
turbine::set_file ${u:out} v:out
turbine::set_file ${u:err} v:err
turbine::decr_local_file_refcount v:out
@@ -409,17 +288,26 @@
turbine::c::log "enter function: get_uniq_sites"
# Alias __filename_sites with type string was defined
# Alias __filename_out with type string was defined
+ # Value __ov_mapped_out with type $boolean was defined
turbine::read_refcount_incr ${u:array} 1
turbine::file_read_refcount_incr ${u:sites} 1
set filename:sites [ turbine::get_file_path ${u:sites} ]
turbine::read_refcount_incr ${filename:sites} 1
- set filename:out [ turbine::get_output_file_path ${u:out} ]
+ set optv:mapped_out [ turbine::is_file_mapped ${u:out} ]
+ if { ${optv:mapped_out} } {
+ set filename:out [ turbine::get_file_path ${u:out} ]
+ } else {
+ # Value __of_out with type $string was defined
+ set optf:out [ turbine::mktemp ]
+ turbine::set_filename_val ${u:out} ${optf:out}
+ set filename:out [ turbine::get_file_path ${u:out} ]
+ }
turbine::read_refcount_incr ${filename:out} 1
- turbine::deeprule [ list ${filename:out} ${filename:sites} ${u:array} [ turbine::get_file_status ${u:sites} ] ] [ list 0 0 1 0 ] [ list 0 0 1 1 ] "get_uniq_sites-app-leaf0 ${stack} {${u:sites}} ${filename:out} {${u:out}} ${filename:sites} ${u:array}" target ${dr:location} type ${::turbine::WORK}
+ turbine::deeprule [ list ${filename:out} ${filename:sites} ${u:array} [ turbine::get_file_status ${u:sites} ] ] [ list 0 0 1 0 ] [ list 0 0 1 1 ] "get_uniq_sites-app-leaf0 ${stack} {${u:sites}} ${optv:mapped_out} ${filename:out} {${u:out}} ${filename:sites} ${u:array}" target ${dr:location} type ${::turbine::WORK}
}
-proc get_uniq_sites-app-leaf0 { stack u:sites filename:out u:out filename:sites u:array } {
+proc get_uniq_sites-app-leaf0 { stack u:sites optv:mapped_out filename:out u:out filename:sites u:array } {
# Value __v___filename_sites with type $string was defined
# Value __v___filename_out with type $string was defined
# Value __v_sites with type $file was defined
@@ -427,9 +315,15 @@
set v:__filename_sites [ turbine::retrieve_string ${filename:sites} CACHED 1 ]
set v:__filename_out [ turbine::retrieve_string ${filename:out} CACHED 1 ]
set v:sites [ turbine::get_file ${u:sites} 1 ]
- turbine::c::log [ list exec: /bin/bash ${v:__filename_sites} {*}[ turbine::unpack_args ${u:array} 1 1 ] [ dict create "stdout" ${v:__filename_out} ] ]
- turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} ] ${v:__filename_sites} {*}[ turbine::unpack_args ${u:array} 1 1 ]
- set v:out [ turbine::create_local_file_ref ${v:__filename_out} ]
+ if { ${optv:mapped_out} } {
+ set tcltmp:init_rc 2
+ } else {
+ set tcltmp:init_rc 1
+ }
+ set v:out [ turbine::create_local_file_ref ${v:__filename_out} ${tcltmp:init_rc} ]
+ set tcltmp:unpacked1 [ turbine::unpack_args ${u:array} 1 1 ]
+ turbine::c::log [ list exec: /bin/bash ${v:__filename_sites} {*}${tcltmp:unpacked1} [ dict create "stdout" ${v:__filename_out} ] ]
+ turbine::exec_external "/bin/bash" [ dict create "stdout" ${v:__filename_out} ] ${v:__filename_sites} {*}${tcltmp:unpacked1}
turbine::set_file ${u:out} v:out
turbine::decr_local_file_refcount v:out
turbine::read_refcount_decr ${u:array} 1
@@ -439,66 +333,66 @@
proc swift:main { } {
turbine::c::log "enter function: main"
set stack 0
- global c:s_jobs_per_s
- global c:s_local_comb
- global c:s_teragen_wr
- global c:s_combiner_s
- global c:s__
- # Value __ov___t1 with type $string was defined
- # Value __ov_loop with type $int was defined
- # Value __ov___t3 with type $string was defined
- # Value __ov_fsize with type $int was defined
- # Value __ov___t7 with type $int was defined
- lassign [ adlb::multicreate [ list container integer file_ref 1 1 ] [ list integer 1 ] [ list container integer ref 1 1 ] [ list string 2 ] [ list container integer ref 1 1 ] [ list string 1 ] [ list container integer file_ref 1 1 ] [ list string 2 ] [ list string 2 ] ] u:tgen_out u:fsize t:5 t:14 u:sites t:15 u:comb_out t:29 t:30
- turbine::c::log "allocated u:tgen_out=<${u:tgen_out}> u:fsize=<${u:fsize}> t:5=<${t:5}> t:14=<${t:14}> u:sites=<${u:sites}>"
- turbine::c::log "allocated t:15=<${t:15}> u:comb_out=<${u:comb_out}> t:29=<${t:29}> t:30=<${t:30}>"
+ global c:s__-1
+ # Value __ov_wrapper with type $file was defined
+ # Value __ov_jobs_per_site with type $file was defined
+ # Value __ov_local_combine with type $file was defined
+ # Value __ov_combine with type $file was defined
+ lassign [ adlb::multicreate [ list container integer file_ref 1 1 ] [ list container integer ref 1 1 ] [ list string 2 ] [ list container integer ref 1 1 ] [ list string 1 ] [ list container integer file_ref 1 1 ] [ list string 2 ] [ list string 2 ] ] u:tgen_out t:1 t:10 u:sites t:11 u:comb_out t:34 t:35
+ turbine::c::log "allocated u:tgen_out=<${u:tgen_out}> t:1=<${t:1}> t:10=<${t:10}> u:sites=<${u:sites}> t:11=<${t:11}>"
+ turbine::c::log "allocated u:comb_out=<${u:comb_out}> t:34=<${t:34}> t:35=<${t:35}>"
turbine::allocate_file2 u:wrapper "" 1
turbine::allocate_file2 u:jobs_per_site "" 1
- turbine::allocate_file2 u:combiner_list ${t:14} 1
+ turbine::allocate_file2 u:combiner_list ${t:10} 1
turbine::allocate_file2 u:local_combine "" 1
turbine::allocate_file2 u:combine "" 1
- turbine::allocate_file2 u:final ${t:29} 1
- turbine::allocate_file2 u:errs ${t:30} 1
+ turbine::allocate_file2 u:final ${t:34} 1
+ turbine::allocate_file2 u:errs ${t:35} 1
# Swift l.27: assigning expression to wrapper
- turbine::input_file [ list ${u:wrapper} ] [ list ${c:s_teragen_wr} ]
- # Swift l.31: assigning expression to loop
- set optv:__t1 [ turbine::argv_get_impl "loop" ]
- set optv:loop [ turbine::check_str_int ${optv:__t1} ]
- # Swift l.33: assigning expression to fsize
- set optv:__t3 [ turbine::argv_get_impl "fsize" ]
- set optv:fsize [ turbine::check_str_int ${optv:__t3} ]
- turbine::store_integer ${u:fsize} ${optv:fsize}
- set optv:__t7 [ expr { ${optv:loop} - 1 } ]
- turbine::range_work ${t:5} 0 ${optv:__t7} 1
+ set optv:wrapper [ turbine::input_file_local "/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/teragen_wrap.sh" ]
+ turbine::set_filename_val ${u:wrapper} "/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/teragen_wrap.sh"
+ turbine::set_file ${u:wrapper} optv:wrapper
+ # Swift l.30: assigning expression to loop
+ # Swift l.32: assigning expression to fsize
+ turbine::range_work ${t:1} 0 9 1
# Swift l.42: assigning expression to jobs_per_site
- turbine::input_file [ list ${u:jobs_per_site} ] [ list ${c:s_jobs_per_s} ]
- turbine::store_string ${t:14} "uniq_sites"
+ set optv:jobs_per_site [ turbine::input_file_local "/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/jobs_per_site.sh" ]
+ turbine::set_filename_val ${u:jobs_per_site} "/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/jobs_per_site.sh"
+ turbine::set_file ${u:jobs_per_site} optv:jobs_per_site
+ turbine::store_string ${t:10} "intermediate/uniq_sites"
# Swift l.44: assigning expression to combiner_list
f:get_uniq_sites ${stack} ${u:combiner_list} ${u:jobs_per_site} ${u:tgen_out} -100
- # Swift l.46: assigning expression to sites
- turbine::split [ list ${u:sites} ] [ list ${t:15} ${c:s__} ]
+ # Swift l.47: assigning expression to sites
+ turbine::split [ list ${u:sites} ] [ list ${t:11} ${c:s__-1} ]
# Swift l.49: assigning expression to local_combine
- turbine::input_file [ list ${u:local_combine} ] [ list ${c:s_local_comb} ]
- # Swift l.68: assigning expression to combine
- turbine::input_file [ list ${u:combine} ] [ list ${c:s_combiner_s} ]
- turbine::store_string ${t:29} "final_result"
- turbine::store_string ${t:30} "final_errs"
- # Swift l.72: assigning expression to final
- # Swift l.72: assigning expression to errs
+ set optv:local_combine [ turbine::input_file_local "/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/local_combiner.sh" ]
+ turbine::set_filename_val ${u:local_combine} "/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/local_combiner.sh"
+ turbine::set_file ${u:local_combine} optv:local_combine
+ # Swift l.75: assigning expression to combine
+ set optv:combine [ turbine::input_file_local "/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/combiner.sh" ]
+ turbine::set_filename_val ${u:combine} "/lustre/beagle/yadunandb/Swift-MapRed/TurboChirp/combiner.sh"
+ turbine::set_file ${u:combine} optv:combine
+ turbine::store_string ${t:34} "final_result"
+ turbine::store_string ${t:35} "final_errs"
+ # Swift l.79: assigning expression to final
+ # Swift l.79: assigning expression to errs
f:comb_data ${stack} ${u:final} ${u:errs} ${u:combine} ${u:comb_out} -100
# Swift l.38: assigning expression to out
# Swift l.38: assigning expression to err
# Swift l.39: assigning expression to tgen_out[_]
turbine::rule [ list ${u:sites} ] "main-foreach-wait1 ${stack} ${u:sites} {${u:local_combine}} ${u:comb_out}" type ${::turbine::LOCAL}
turbine::rule [ list ${u:comb_out} ] "main-foreach-wait2 ${stack} ${u:comb_out}" type ${::turbine::LOCAL}
- set tcltmp:container_sz [ adlb::enumerate ${t:5} count all 0 ]
+ set tcltmp:container_sz [ adlb::enumerate ${t:1} count all 0 ]
set tcltmp:iters ${tcltmp:container_sz}
turbine::file_read_refcount_incr ${u:wrapper} [ expr { ${tcltmp:iters} - 1 } ]
- turbine::read_refcount_incr ${t:5} [ expr { ${tcltmp:iters} - 1 } ]
- turbine::read_refcount_incr ${u:fsize} [ expr { ${tcltmp:iters} - 1 } ]
+ turbine::read_refcount_incr ${t:1} [ expr { ${tcltmp:iters} - 1 } ]
adlb::write_refcount_incr ${u:tgen_out} [ expr { ${tcltmp:iters} - 1 } ]
- main-foreach0:outer ${stack} ${u:wrapper} ${u:fsize} ${u:tgen_out} ${t:5} 0 [ expr { ${tcltmp:container_sz} - 1 } ] 1
- turbine::rule [ list [ turbine::get_file_status ${u:combiner_list} ] ] "main-call_builtin-read ${stack} ${t:15} {${u:combiner_list}}" type ${::turbine::WORK}
+ main-foreach0:outer ${stack} ${u:wrapper} ${u:tgen_out} ${t:1} 0 [ expr { ${tcltmp:container_sz} - 1 } ] 1
+ turbine::rule [ list [ turbine::get_file_status ${u:combiner_list} ] ] "main-call_foreign-read ${stack} ${t:11} {${u:combiner_list}}" type ${::turbine::WORK}
+ turbine::decr_local_file_refcount optv:wrapper
+ turbine::decr_local_file_refcount optv:jobs_per_site
+ turbine::decr_local_file_refcount optv:local_combine
+ turbine::decr_local_file_refcount optv:combine
turbine::read_refcount_decr ${u:tgen_out} 1
turbine::file_read_refcount_decr ${u:errs} 1
turbine::file_read_refcount_decr ${u:jobs_per_site} 1
@@ -508,11 +402,6 @@
proc main-foreach-wait1 { stack u:sites u:local_combine u:comb_out } {
- # Swift l.55: assigning expression to site_info
- # Swift l.56: assigning expression to rank
- # Swift l.60: assigning expression to out1
- # Swift l.60: assigning expression to err1
- # Swift l.61: assigning expression to comb_out[_]
set tcltmp:container_sz [ adlb::enumerate ${u:sites} count all 0 ]
set tcltmp:iters ${tcltmp:container_sz}
turbine::read_refcount_incr ${u:sites} [ expr { ${tcltmp:iters} - 1 } ]
@@ -546,42 +435,79 @@
set tcltmp:splitlen [ expr { ${tcltmp:hi} - ${tcltmp:lo} + 1 } ]
set tcltmp:contents [ adlb::enumerate ${u:sites} dict ${tcltmp:splitlen} ${tcltmp:lo} ]
dict for {v:i0 u:site} ${tcltmp:contents} {
- global c:s__-1
- # Value __ov___t18 with type $string was defined
- # Value __ov___t20 with type $string was defined
- lassign [ adlb::multicreate [ list string 2 ] [ list string 2 ] [ list container integer ref 1 1 ] [ list string 1 ] [ list ref 1 ] ] t:18 t:20 u:site_info t:23 t:24
- turbine::c::log "allocated t:18=<${t:18}> t:20=<${t:20}> u:site_info=<${u:site_info}> t:23=<${t:23}> t:24=<${t:24}>"
- turbine::allocate_file2 u:out1 ${t:18} 1
- turbine::allocate_file2 u:err1 ${t:20} 0
- turbine::read_refcount_incr ${u:site} 2
- set optv:__t18 [ eval format [ list "intermediate/sitecomb_%i.out" ${v:i0} ] ]
- turbine::store_string ${t:18} ${optv:__t18}
- set optv:__t20 [ eval format [ list "intermediate/sitecomb_%i.err" ${v:i0} ] ]
- turbine::store_string ${t:20} ${optv:__t20}
- turbine::split [ list ${u:site_info} ] [ list ${u:site} ${c:s__-1} ]
- turbine::container_reference ${u:site_info} 0 ${t:24} ref
- turbine::dereference_string ${t:23} ${t:24}
- turbine::container_insert ${u:comb_out} ${v:i0} ${u:out1} file_ref 1
- turbine::rule [ list ${t:23} ] "main-call_sync-hostmap_one ${stack} ${u:site} ${t:23} {${u:local_combine}} {${u:out1}} {${u:err1}}"
+ lassign [ adlb::multicreate [ list integer 1 ] ] t:14
+ turbine::c::log "allocated t:14=<${t:14}>"
+ turbine::read_refcount_incr ${u:site} 1
+ turbine::rule [ list ${u:site} ] "main-call_sync-strlen ${stack} ${u:site} ${v:i0} {${u:local_combine}} ${u:comb_out} ${t:14}"
}
set tcltmp:iters [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
turbine::read_refcount_decr ${u:sites} [ expr { ${tcltmp:iters} } ]
}
-proc main-call_sync-hostmap_one { stack u:site t:23 u:local_combine u:out1 u:err1 } {
- # Value __ov___t23 with type $string was defined
+proc main-call_sync-strlen { stack u:site v:i0 u:local_combine u:comb_out t:14 } {
+ # Value __ov_site with type $string was defined
+ # Value __ov___t15 with type $int was defined
+ # Value __ov___t14 with type $boolean was defined
+ # Value __v___t14 with type $boolean was defined
+ set optv:site [ turbine::retrieve_string ${u:site} ]
+ set optv:__t15 [ string length ${optv:site} ]
+ set optv:__t14 [ expr { ${optv:__t15} > 0 } ]
+ turbine::store_integer ${t:14} ${optv:__t14}
+ set v:__t14 [ turbine::retrieve_integer ${t:14} CACHED 1 ]
+ if { ${v:__t14} } {
+ global c:s__
+ # Value __ov___t17 with type $string was defined
+ # Value __ov___t19 with type $string was defined
+ lassign [ adlb::multicreate [ list string 2 ] [ list string 2 ] [ list container integer ref 1 1 ] [ list string 1 ] [ list ref 1 ] ] t:17 t:19 u:site_info t:22 t:23
+ turbine::c::log "allocated t:17=<${t:17}> t:19=<${t:19}> u:site_info=<${u:site_info}> t:22=<${t:22}> t:23=<${t:23}>"
+ turbine::allocate_file2 u:out1 ${t:17} 2
+ turbine::allocate_file2 u:err1 ${t:19} 1
+ turbine::file_read_refcount_incr ${u:local_combine} 1
+ turbine::read_refcount_incr ${u:site} 2
+ set optv:__t17 [ eval format [ list "intermediate/sitecomb_%i.out" ${v:i0} ] ]
+ turbine::store_string ${t:17} ${optv:__t17}
+ set optv:__t19 [ eval format [ list "intermediate/sitecomb_%i.err" ${v:i0} ] ]
+ turbine::store_string ${t:19} ${optv:__t19}
+ # Swift l.57: assigning expression to site_info
+ turbine::split [ list ${u:site_info} ] [ list ${u:site} ${c:s__} ]
+ # Swift l.60: assigning expression to rank
+ turbine::container_reference ${u:site_info} 0 ${t:23} ref
+ turbine::dereference_string ${t:22} ${t:23}
+ # Swift l.62 evaluating expression and throwing away 1 results
+ # Swift l.64: assigning expression to out1
+ # Swift l.64: assigning expression to err1
+ # Swift l.65: assigning expression to comb_out[_]
+ turbine::container_insert ${u:comb_out} ${v:i0} ${u:out1} file_ref 0
+ turbine::rule [ list ${t:22} ] [ list main-call_sync-hostmap_one_worker ${stack} ${u:site} ${t:22} ${u:local_combine} ${optv:site} ${u:out1} ${u:err1} ]
+ } else {
+ # Value __ov___t28 with type $void was defined
+ # Swift l.67 evaluating expression and throwing away 1 results
+ set optv:__t28 [ turbine::printf_local "Empty string\n" ]
+ }
+ turbine::read_refcount_decr ${u:site} 1
+ turbine::file_read_refcount_decr ${u:local_combine} 1
+ adlb::write_refcount_decr ${u:comb_out} 1
+}
+
+
+proc main-call_sync-hostmap_one_worker { stack u:site t:22 u:local_combine optv:site u:out1 u:err1 } {
+ # Value __ov___t22 with type $string was defined
# Value __ov_rank with type $int was defined
- set optv:__t23 [ turbine::retrieve_string ${t:23} CACHED 1 ]
- set optv:rank [ draw [ adlb::hostmap ${optv:__t23} ] ]
+ # Value __ov___t24 with type $void was defined
+ set optv:__t22 [ turbine::retrieve_string ${t:22} CACHED 1 ]
+ set optv:rank [ ::turbine::random_rank WORKER [ adlb::hostmap_lookup ${optv:__t22} ] ]
f:comb_data_local ${stack} ${u:out1} ${u:err1} ${u:local_combine} ${u:site} ${optv:rank}
+ set optv:__t24 [ turbine::printf_local "Site: %s Rank: %i String: %s\n" ${optv:__t22} ${optv:rank} ${optv:site} ]
turbine::read_refcount_decr ${u:site} 1
turbine::file_read_refcount_decr ${u:local_combine} 1
+ turbine::file_read_refcount_decr ${u:err1} 1
+ turbine::file_read_refcount_decr ${u:out1} 1
}
proc main-foreach-wait2 { stack u:comb_out } {
- # Swift l.65 evaluating expression and throwing away 1 results
+ # Swift l.72 evaluating expression and throwing away 1 results
set tcltmp:container_sz [ adlb::enumerate ${u:comb_out} count all 0 ]
set tcltmp:iters ${tcltmp:container_sz}
turbine::read_refcount_incr ${u:comb_out} [ expr { ${tcltmp:iters} - 1 } ]
@@ -613,78 +539,109 @@
set tcltmp:splitlen [ expr { ${tcltmp:hi} - ${tcltmp:lo} + 1 } ]
set tcltmp:contents [ adlb::enumerate ${u:comb_out} members ${tcltmp:splitlen} ${tcltmp:lo} ]
foreach u:file_item ${tcltmp:contents} {
- lassign [ adlb::multicreate [ list string 1 ] ] t:27
- turbine::c::log "allocated t:27=<${t:27}>"
- turbine::file_read_refcount_incr ${u:file_item} 1
- turbine::filename2 [ list ${t:27} ] [ list ${u:file_item} ]
- turbine::rule [ list ${t:27} ] "main-call_builtin-printf ${stack} ${t:27}"
+ # Alias __of_file_item with type string was defined
+ set optf:file_item [ turbine::get_file_path ${u:file_item} ]
+ turbine::read_refcount_incr ${optf:file_item} 1
+ turbine::rule [ list ${optf:file_item} ] "main-call_foreign-printf ${stack} ${optf:file_item}"
}
set tcltmp:iters [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
turbine::read_refcount_decr ${u:comb_out} [ expr { ${tcltmp:iters} } ]
}
-proc main-call_builtin-printf { stack t:27 } {
- # Value __ov___t27 with type $string was defined
- # Value __ov___t25 with type $void was defined
- set optv:__t27 [ turbine::retrieve_string ${t:27} CACHED 1 ]
- set optv:__t25 [ turbine::printf_local "Item in comb_out : %s \n" ${optv:__t27} ]
+proc main-call_foreign-printf { stack optf:file_item } {
+ # Value __ov___of_file_item with type $string was defined
+ # Value __ov___t30 with type $void was defined
+ set optv:__of_file_item [ turbine::retrieve_string ${optf:file_item} CACHED 1 ]
+ set optv:__t30 [ turbine::printf_local "Item in comb_out : %s \n" ${optv:__of_file_item} ]
}
-proc main-foreach0:outer { stack u:wrapper u:fsize u:tgen_out t:5 tcltmp:lo tcltmp:hi tcltmp:inc } {
+proc main-foreach0:outer { stack u:wrapper u:tgen_out t:1 tcltmp:lo tcltmp:hi tcltmp:inc } {
set tcltmp:itersleft [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
if { [ expr { ${tcltmp:itersleft} <= 0 } ] } {
return
}
if { [ expr { ${tcltmp:itersleft} <= 64 } ] } {
- main-foreach0:inner ${stack} ${u:wrapper} ${u:fsize} ${u:tgen_out} ${t:5} ${tcltmp:lo} ${tcltmp:hi} ${tcltmp:inc}
+ main-foreach0:inner ${stack} ${u:wrapper} ${u:tgen_out} ${t:1} ${tcltmp:lo} ${tcltmp:hi} ${tcltmp:inc}
} else {
set tcltmp:skip [ expr { ${tcltmp:inc} * max(64,((${tcltmp:itersleft} - 1) / 16) + 1) } ]
for { set tcltmp:splitstart ${tcltmp:lo} } { ${tcltmp:splitstart} <= ${tcltmp:hi} } { incr tcltmp:splitstart ${tcltmp:skip} } {
set tcltmp:splitend [ expr { min(${tcltmp:hi},${tcltmp:splitstart} + ${tcltmp:skip} - 1) } ]
set tcltmp:prio [ turbine::get_priority ]
turbine::set_priority ${tcltmp:prio}
- adlb::spawn 1 "command main-foreach0:outer ${stack} {${u:wrapper}} ${u:fsize} ${u:tgen_out} ${t:5} ${tcltmp:splitstart} ${tcltmp:splitend} ${tcltmp:inc}"
+ adlb::spawn 1 "command main-foreach0:outer ${stack} {${u:wrapper}} ${u:tgen_out} ${t:1} ${tcltmp:splitstart} ${tcltmp:splitend} ${tcltmp:inc}"
turbine::reset_priority
}
}
}
-proc main-foreach0:inner { stack u:wrapper u:fsize u:tgen_out t:5 tcltmp:lo tcltmp:hi tcltmp:inc } {
+proc main-foreach0:inner { stack u:wrapper u:tgen_out t:1 tcltmp:lo tcltmp:hi tcltmp:inc } {
set tcltmp:splitlen [ expr { ${tcltmp:hi} - ${tcltmp:lo} + 1 } ]
- set tcltmp:contents [ adlb::enumerate ${t:5} dict ${tcltmp:splitlen} ${tcltmp:lo} ]
+ set tcltmp:contents [ adlb::enumerate ${t:1} dict ${tcltmp:splitlen} ${tcltmp:lo} ]
dict for {v:i u:item} ${tcltmp:contents} {
- # Value __ov___t9 with type $string was defined
- # Value __ov___t11 with type $string was defined
- lassign [ adlb::multicreate [ list string 2 ] [ list string 2 ] ] t:9 t:11
- turbine::c::log "allocated t:9=<${t:9}> t:11=<${t:11}>"
- turbine::allocate_file2 u:out ${t:9} 1
- turbine::allocate_file2 u:err ${t:11} 1
- set optv:__t9 [ eval format [ list "intermediate/tgen_%i.out" ${v:i} ] ]
- turbine::store_string ${t:9} ${optv:__t9}
- set optv:__t11 [ eval format [ list "intermediate/tgen_%i.err" ${v:i} ] ]
- turbine::store_string ${t:11} ${optv:__t11}
- f:gen_data ${stack} ${u:out} ${u:err} ${u:wrapper} ${u:fsize} -100
+ global c:i_1
+ # Value __ov___t5 with type $string was defined
+ # Value __ov___t7 with type $string was defined
+ lassign [ adlb::multicreate [ list string 2 ] [ list string 2 ] ] t:5 t:7
+ turbine::c::log "allocated t:5=<${t:5}> t:7=<${t:7}>"
+ turbine::allocate_file2 u:out ${t:5} 1
+ turbine::allocate_file2 u:err ${t:7} 1
+ set optv:__t5 [ eval format [ list "intermediate/tgen_%i.out" ${v:i} ] ]
+ turbine::store_string ${t:5} ${optv:__t5}
+ set optv:__t7 [ eval format [ list "intermediate/tgen_%i.err" ${v:i} ] ]
+ turbine::store_string ${t:7} ${optv:__t7}
+ f:gen_data ${stack} ${u:out} ${u:err} ${u:wrapper} ${c:i_1} -100
turbine::container_insert ${u:tgen_out} ${v:i} ${u:out} file_ref 1
turbine::file_read_refcount_decr ${u:err} 1
}
set tcltmp:iters [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
- turbine::read_refcount_decr ${t:5} [ expr { ${tcltmp:iters} } ]
- turbine::read_refcount_decr ${u:fsize} [ expr { ${tcltmp:iters} } ]
turbine::file_read_refcount_decr ${u:wrapper} [ expr { ${tcltmp:iters} } ]
+ turbine::read_refcount_decr ${t:1} [ expr { ${tcltmp:iters} } ]
}
-proc main-call_builtin-read { stack t:15 u:combiner_list } {
+proc main-call_foreign-read { stack t:11 u:combiner_list } {
# Value __ov_combiner_list with type $file was defined
- # Value __ov___t15 with type $string was defined
+ # Value __ov___t11 with type $string was defined
set optv:combiner_list [ turbine::get_file ${u:combiner_list} 1 ]
- set optv:__t15 [ turbine::file_read_local ${optv:combiner_list} ]
- turbine::store_string ${t:15} ${optv:__t15}
+ set optv:__t11 [ turbine::file_read_local ${optv:combiner_list} ]
+ turbine::store_string ${t:11} ${optv:__t11}
}
+
+proc f:strlen { stack u:n u:s } {
+ turbine::c::log "enter function: strlen"
+ turbine::read_refcount_incr ${u:s} 1
+ turbine::rule [ list ${u:s} ] "strlen-argwait ${stack} ${u:s} ${u:n}"
+}
+
+
+proc strlen-argwait { stack u:s u:n } {
+ # Value __v_s with type $string was defined
+ # Value __v_n with type $int was defined
+ set v:s [ turbine::retrieve_string ${u:s} CACHED 1 ]
+ set v:n [ string length ${v:s} ]
+ turbine::store_integer ${u:n} ${v:n}
+}
+
+
+proc f:hostmap_one_worker { stack u:rank u:name } {
+ turbine::c::log "enter function: hostmap_one_worker"
+ turbine::read_refcount_incr ${u:name} 1
+ turbine::rule [ list ${u:name} ] "hostmap_one_worker-argwait ${stack} ${u:rank} ${u:name}"
+}
+
+
+proc hostmap_one_worker-argwait { stack u:rank u:name } {
+ # Value __v_name with type $string was defined
+ # Value __v_rank with type $int was defined
+ set v:name [ turbine::retrieve_string ${u:name} CACHED 1 ]
+ set v:rank [ ::turbine::random_rank WORKER [ adlb::hostmap_lookup ${v:name} ] ]
+ turbine::store_integer ${u:rank} ${v:rank}
+}
+
turbine::defaults
turbine::init $engines $servers
turbine::enable_read_refcount
Modified: SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh 2013-10-04 16:33:52 UTC (rev 7132)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen_wrap.sh 2013-10-04 16:35:12 UTC (rev 7133)
@@ -12,7 +12,7 @@
#SLICESIZE=1000 # 10^3 If padded to 100B would result
-FOLDER="/sandbox/$USER"
+FOLDER="/dev/shm/$USER"
TIMEOUT=120
# clean_folder SLEEP_TIME FOLDER_NAME
@@ -22,6 +22,8 @@
rm $2/* &> /dev/null
}
+HOST=$(hostname -f)
+
ps -u $USER | grep "chirp_server" &> /dev/null
if [ "$?" != "0" ]
then
@@ -29,12 +31,11 @@
echo "unix:$USER rwlda" > $FOLDER/acl.conf
echo "hostname:* rwlda" >> $FOLDER/acl.conf
timeout $TIMEOUT chirp_server -A $FOLDER/acl.conf -r $FOLDER &
+ sleep 3;
fi
FILE=$FOLDER/$FILE;
shuf -i $LOWERLIMIT-$UPPERLIMIT -n $(($SLICESIZE*$ARG1)) | awk '{printf "%-99s\n", $0}' > $FILE
-sleep 5;
-#HOST=$(hostname -f)
-#echo "$HOST $FILE"
-echo "$HOSTNAME $FILE"
+echo "$HOST $FILE"
+#echo "$HOSTNAME $FILE"
exit 0
\ No newline at end of file
More information about the Swift-commit
mailing list