[Swift-commit] r6875 - SwiftApps/Swift-MapRed/TurboChirp
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Fri Aug 16 11:23:08 CDT 2013
Author: yadunandb
Date: 2013-08-16 11:23:07 -0500 (Fri, 16 Aug 2013)
New Revision: 6875
Added:
SwiftApps/Swift-MapRed/TurboChirp/client.sh
SwiftApps/Swift-MapRed/TurboChirp/server.sh
Modified:
SwiftApps/Swift-MapRed/TurboChirp/run.sh
SwiftApps/Swift-MapRed/TurboChirp/teragen.swift
SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl
Log:
Adding netcat based server and client for remote execution
Added: SwiftApps/Swift-MapRed/TurboChirp/client.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/client.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/client.sh 2013-08-16 16:23:07 UTC (rev 6875)
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+REMOTE_PORT=20033
+
+if [ "$1" == "" ]
+then
+ echo "Client needs an arg for remote"
+ exit 0
+fi
+
+EXEC_FILE="tmp.$RANDOM"
+
+PORT=$(($REMOTE_PORT + 10))
+echo "$(hostname -f) $PORT" > $EXEC_FILE
+cat <<'EOF' >> $EXEC_FILE
+#!/bin/bash
+echo "Recombiner on $HOSTNAME"
+echo $(hostname -f)
+EOF
+
+listen_for_reply()
+{
+ nc -l $PORT > reply;
+ echo "Connection received back on $PORT"
+}
+
+listen_for_reply &
+nc $1 $REMOTE_PORT < $EXEC_FILE
+echo "nc $1 $PORT;"
+
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/client.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/Swift-MapRed/TurboChirp/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/run.sh 2013-08-15 20:15:38 UTC (rev 6874)
+++ SwiftApps/Swift-MapRed/TurboChirp/run.sh 2013-08-16 16:23:07 UTC (rev 6875)
@@ -8,11 +8,11 @@
fi
./cleanup
-export TURBINE_LOG=0
+export TURBINE_LOG=1
export TURBINE_LAUNCH_OPTS="-f ./hosts.txt"
stc $PROG > ${PROG%.swift}.tcl
[ "$?" == "0" ] && echo "Compile done"
#turbine -n 8 ${PROG%.swift}.tcl
#turbine -l -V -n 10 -f hosts.txt ${PROG%.swift}.tcl -loop=10 -fsize=1 >& ${PROG%.swift}.out
-turbine -n 10 -f hosts.txt ${PROG%.swift}.tcl -loop=10 -fsize=1
+time turbine -n 10 -f hosts.txt ${PROG%.swift}.tcl -loop=10 -fsize=1
[ "$?" == "0" ] && echo "Execute done"
Added: SwiftApps/Swift-MapRed/TurboChirp/server.sh
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/server.sh (rev 0)
+++ SwiftApps/Swift-MapRed/TurboChirp/server.sh 2013-08-16 16:23:07 UTC (rev 6875)
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+PORT=20033
+TMP="tmp.$RANDOM.$HOSTNAME"
+nc -l $PORT > $TMP
+chmod +x $TMP
+RETURN_ADDRESS=$(head -n 1 $TMP)
+sed -i -e "1d" $TMP
+./$TMP &> $TMP.reply
+nc $RETURN_ADDRESS < $TMP.reply
Property changes on: SwiftApps/Swift-MapRed/TurboChirp/server.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/Swift-MapRed/TurboChirp/teragen.swift
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen.swift 2013-08-15 20:15:38 UTC (rev 6874)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen.swift 2013-08-16 16:23:07 UTC (rev 6875)
@@ -57,7 +57,7 @@
//printf("Site: %s Rank: %i String: %s\n", site_info[0], rank, site);
- (out1, err1) = @location=rank comb_data_local(local_combine, site);
+ (out1, err1) = @location=rank comb_data_local(local_combine, site);
comb_out[i] = out1;
}
Modified: SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl
===================================================================
--- SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl 2013-08-15 20:15:38 UTC (rev 6874)
+++ SwiftApps/Swift-MapRed/TurboChirp/teragen.tcl 2013-08-16 16:23:07 UTC (rev 6875)
@@ -1,6 +1,6 @@
# Generated by stc version 0.2.0
-# date : 2013/08/15 15:10:03
+# date : 2013/08/15 15:37:06
# Turbine version : 0.3.0
# Input filename : /nfs2/yadunand/Swift-MapRed/TurboChirp/teragen.swift
# Output filename : /nfs2/yadunand/Swift-MapRed/TurboChirp
@@ -449,16 +449,16 @@
# Value __ov___t3 with type $string was defined
# Value __ov_fsize with type $int was defined
# Value __ov___t7 with type $int was defined
- lassign [ adlb::multicreate [ list container integer file_ref 1 1 ] [ list integer 1 ] [ list container integer ref 1 1 ] [ list string 2 ] [ list container integer ref 1 1 ] [ list string 1 ] [ list container integer file_ref 1 1 ] [ list string 2 ] [ list string 2 ] ] u:tgen_out u:fsize t:5 t:14 u:sites t:15 u:comb_out t:37 t:38
+ lassign [ adlb::multicreate [ list container integer file_ref 1 1 ] [ list integer 1 ] [ list container integer ref 1 1 ] [ list string 2 ] [ list container integer ref 1 1 ] [ list string 1 ] [ list container integer file_ref 1 1 ] [ list string 2 ] [ list string 2 ] ] u:tgen_out u:fsize t:5 t:14 u:sites t:15 u:comb_out t:29 t:30
turbine::c::log "allocated u:tgen_out=<${u:tgen_out}> u:fsize=<${u:fsize}> t:5=<${t:5}> t:14=<${t:14}> u:sites=<${u:sites}>"
- turbine::c::log "allocated t:15=<${t:15}> u:comb_out=<${u:comb_out}> t:37=<${t:37}> t:38=<${t:38}>"
+ turbine::c::log "allocated t:15=<${t:15}> u:comb_out=<${u:comb_out}> t:29=<${t:29}> t:30=<${t:30}>"
turbine::allocate_file2 u:wrapper "" 1
turbine::allocate_file2 u:jobs_per_site "" 1
turbine::allocate_file2 u:combiner_list ${t:14} 1
turbine::allocate_file2 u:local_combine "" 1
turbine::allocate_file2 u:combine "" 1
- turbine::allocate_file2 u:final ${t:37} 1
- turbine::allocate_file2 u:errs ${t:38} 1
+ turbine::allocate_file2 u:final ${t:29} 1
+ turbine::allocate_file2 u:errs ${t:30} 1
# Swift l.27: assigning expression to wrapper
turbine::input_file [ list ${u:wrapper} ] [ list ${c:s_teragen_wr} ]
# Swift l.31: assigning expression to loop
@@ -479,12 +479,12 @@
turbine::split [ list ${u:sites} ] [ list ${t:15} ${c:s__} ]
# Swift l.49: assigning expression to local_combine
turbine::input_file [ list ${u:local_combine} ] [ list ${c:s_local_comb} ]
- # Swift l.71: assigning expression to combine
+ # Swift l.68: assigning expression to combine
turbine::input_file [ list ${u:combine} ] [ list ${c:s_combiner_s} ]
- turbine::store_string ${t:37} "final_result"
- turbine::store_string ${t:38} "final_errs"
- # Swift l.75: assigning expression to final
- # Swift l.75: assigning expression to errs
+ turbine::store_string ${t:29} "final_result"
+ turbine::store_string ${t:30} "final_errs"
+ # Swift l.72: assigning expression to final
+ # Swift l.72: assigning expression to errs
f:comb_data ${stack} ${u:final} ${u:errs} ${u:combine} ${u:comb_out} -100
# Swift l.38: assigning expression to out
# Swift l.38: assigning expression to err
@@ -509,12 +509,10 @@
proc main-foreach-wait1 { stack u:sites u:local_combine u:comb_out } {
# Swift l.55: assigning expression to site_info
- # Swift l.56 evaluating expression and throwing away 1 results
- # Swift l.57: assigning expression to rank
- # Swift l.58 evaluating expression and throwing away 1 results
- # Swift l.63: assigning expression to out1
- # Swift l.63: assigning expression to err1
- # Swift l.64: assigning expression to comb_out[_]
+ # Swift l.56: assigning expression to rank
+ # Swift l.60: assigning expression to out1
+ # Swift l.60: assigning expression to err1
+ # Swift l.61: assigning expression to comb_out[_]
set tcltmp:container_sz [ adlb::enumerate ${u:sites} count all 0 ]
set tcltmp:iters ${tcltmp:container_sz}
turbine::read_refcount_incr ${u:sites} [ expr { ${tcltmp:iters} - 1 } ]
@@ -551,8 +549,8 @@
global c:s__-1
# Value __ov___t18 with type $string was defined
# Value __ov___t20 with type $string was defined
- lassign [ adlb::multicreate [ list string 2 ] [ list string 2 ] [ list container integer ref 1 1 ] [ list string 1 ] [ list ref 1 ] ] t:18 t:20 u:site_info t:25 t:26
- turbine::c::log "allocated t:18=<${t:18}> t:20=<${t:20}> u:site_info=<${u:site_info}> t:25=<${t:25}> t:26=<${t:26}>"
+ lassign [ adlb::multicreate [ list string 2 ] [ list string 2 ] [ list container integer ref 1 1 ] [ list string 1 ] [ list ref 1 ] ] t:18 t:20 u:site_info t:23 t:24
+ turbine::c::log "allocated t:18=<${t:18}> t:20=<${t:20}> u:site_info=<${u:site_info}> t:23=<${t:23}> t:24=<${t:24}>"
turbine::allocate_file2 u:out1 ${t:18} 1
turbine::allocate_file2 u:err1 ${t:20} 0
turbine::read_refcount_incr ${u:site} 2
@@ -561,39 +559,29 @@
set optv:__t20 [ eval format [ list "intermediate/sitecomb_%i.err" ${v:i0} ] ]
turbine::store_string ${t:20} ${optv:__t20}
turbine::split [ list ${u:site_info} ] [ list ${u:site} ${c:s__-1} ]
- turbine::container_reference ${u:site_info} 0 ${t:26} ref
- turbine::dereference_string ${t:25} ${t:26}
+ turbine::container_reference ${u:site_info} 0 ${t:24} ref
+ turbine::dereference_string ${t:23} ${t:24}
turbine::container_insert ${u:comb_out} ${v:i0} ${u:out1} file_ref 1
- turbine::rule [ list ${t:25} ] "main-optmerged ${stack} ${u:site} {${u:local_combine}} ${t:25} {${u:out1}} {${u:err1}}"
+ turbine::rule [ list ${t:23} ] "main-call_sync-hostmap_one ${stack} ${u:site} ${t:23} {${u:local_combine}} {${u:out1}} {${u:err1}}"
}
set tcltmp:iters [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
turbine::read_refcount_decr ${u:sites} [ expr { ${tcltmp:iters} } ]
}
-proc main-optmerged { stack u:site u:local_combine t:25 u:out1 u:err1 } {
- # Value __ov___t25:1 with type $string was defined
+proc main-call_sync-hostmap_one { stack u:site t:23 u:local_combine u:out1 u:err1 } {
+ # Value __ov___t23 with type $string was defined
# Value __ov_rank with type $int was defined
- # Value __ov___t29 with type $void was defined
- set optv:__t25:1 [ turbine::retrieve_string ${t:25} CACHED 1 ]
- set optv:rank [ draw [ adlb::hostmap ${optv:__t25:1} ] ]
+ set optv:__t23 [ turbine::retrieve_string ${t:23} CACHED 1 ]
+ set optv:rank [ draw [ adlb::hostmap ${optv:__t23} ] ]
f:comb_data_local ${stack} ${u:out1} ${u:err1} ${u:local_combine} ${u:site} ${optv:rank}
- set optv:__t29 [ turbine::printf_local "Site: %s Rank: %i" ${optv:__t25:1} ${optv:rank} ]
- turbine::rule [ list ${u:site} ] [ list main-call_builtin-printf ${stack} ${u:site} ${optv:__t25:1} ]
+ turbine::read_refcount_decr ${u:site} 1
turbine::file_read_refcount_decr ${u:local_combine} 1
}
-proc main-call_builtin-printf { stack u:site optv:__t25:1 } {
- # Value __ov_site with type $string was defined
- # Value __ov___t23 with type $void was defined
- set optv:site [ turbine::retrieve_string ${u:site} CACHED 1 ]
- set optv:__t23 [ turbine::printf_local "Site: %s String: %s\n" ${optv:__t25:1} ${optv:site} ]
-}
-
-
proc main-foreach-wait2 { stack u:comb_out } {
- # Swift l.68 evaluating expression and throwing away 1 results
+ # Swift l.65 evaluating expression and throwing away 1 results
set tcltmp:container_sz [ adlb::enumerate ${u:comb_out} count all 0 ]
set tcltmp:iters ${tcltmp:container_sz}
turbine::read_refcount_incr ${u:comb_out} [ expr { ${tcltmp:iters} - 1 } ]
@@ -625,22 +613,22 @@
set tcltmp:splitlen [ expr { ${tcltmp:hi} - ${tcltmp:lo} + 1 } ]
set tcltmp:contents [ adlb::enumerate ${u:comb_out} members ${tcltmp:splitlen} ${tcltmp:lo} ]
foreach u:file_item ${tcltmp:contents} {
- lassign [ adlb::multicreate [ list string 1 ] ] t:35
- turbine::c::log "allocated t:35=<${t:35}>"
+ lassign [ adlb::multicreate [ list string 1 ] ] t:27
+ turbine::c::log "allocated t:27=<${t:27}>"
turbine::file_read_refcount_incr ${u:file_item} 1
- turbine::filename2 [ list ${t:35} ] [ list ${u:file_item} ]
- turbine::rule [ list ${t:35} ] "main-call_builtin-printf-1 ${stack} ${t:35}"
+ turbine::filename2 [ list ${t:27} ] [ list ${u:file_item} ]
+ turbine::rule [ list ${t:27} ] "main-call_builtin-printf ${stack} ${t:27}"
}
set tcltmp:iters [ expr { max(0,(${tcltmp:hi} - ${tcltmp:lo}) / ${tcltmp:inc} + 1) } ]
turbine::read_refcount_decr ${u:comb_out} [ expr { ${tcltmp:iters} } ]
}
-proc main-call_builtin-printf-1 { stack t:35 } {
- # Value __ov___t35 with type $string was defined
- # Value __ov___t33 with type $void was defined
- set optv:__t35 [ turbine::retrieve_string ${t:35} CACHED 1 ]
- set optv:__t33 [ turbine::printf_local "Item in comb_out : %s \n" ${optv:__t35} ]
+proc main-call_builtin-printf { stack t:27 } {
+ # Value __ov___t27 with type $string was defined
+ # Value __ov___t25 with type $void was defined
+ set optv:__t27 [ turbine::retrieve_string ${t:27} CACHED 1 ]
+ set optv:__t25 [ turbine::printf_local "Item in comb_out : %s \n" ${optv:__t27} ]
}
More information about the Swift-commit
mailing list