[Swift-commit] r3610 - SwiftApps/SwiftR/Swift/exec
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Tue Sep 7 20:01:21 CDT 2010
Author: wilde
Date: 2010-09-07 20:01:21 -0500 (Tue, 07 Sep 2010)
New Revision: 3610
Added:
SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
SwiftApps/SwiftR/Swift/exec/passive-coaster-swift
Modified:
SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh
SwiftApps/SwiftR/Swift/exec/swiftapply.swift
Log:
Initial versions of scripts for passive coasters and persisten R eval processes.
Added: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-09-08 01:01:21 UTC (rev 3610)
@@ -0,0 +1,80 @@
+#! /bin/bash
+
+# Arguments: inputBatchSaveFile outputBatchSaveFile
+
+# Find our bin dir (to use for running utility scripts)
+
+SWIFTRBIN=$(cd $(dirname $0); pwd)
+
+# This script expects to be called (typically from Swift worker.pl)
+# with the following env vars set:
+# SWIFT_WORKER_PID - process ID of the worker.pl that launched this script
+# SWIFT_JOB_SLOT - slot ID of the worker job that launched this script (0..maxWorkers-1)
+
+# Ensure that the dir for this slot exists.
+
+BASEDIR=/tmp/SwiftR/fifos
+WORKERDIR=$BASEDIR/worker.$SWIFT_WORKER_PID
+SLOTDIR=$WORKERDIR/${SWIFT_JOB_SLOT}
+
+mkdir -p $WORKERDIR
+
+# Try to make slotdir.
+# If the mkdir succeeds, this is the first request to the slot,
+# so we create a new R server and send the current request to it;
+# else we send the current request to an existing R server.
+
+RPIDS=
+
+TRAPS="EXIT 1 2 3 15"
+
+function onexit { # FIXME: move this logic into worker.pl, or try to use R timeout option if it works on fifos - need to test.
+ trap - $TRAPS
+ echo "Terminating R processes $RPIDS"
+ if [ "_$RPIDS" != _ ]; then
+ kill $RPIDS >& /dev/null
+ fi
+}
+
+# trap onexit $TRAPS
+
+function idletimer {
+ cd $1
+ while true; do
+ touch idle
+ sleep 60
+ if [ ! -f lastwrite -o \( idle -nt lastwrite \) ]; then
+ # FIXME: should work, but does not: echo quit x x >toR.fifo # fifo problem?
+ RPID=$(cat R.pid)
+ echo killing idle R process $RPID
+ kill $RPID
+ exit 0
+ fi
+ done
+}
+
+mkdir $SLOTDIR >& /dev/null
+if [ $? = 0 ]; then
+ mkfifo $SLOTDIR/toR.fifo
+ mkfifo $SLOTDIR/fromR.fifo
+ $SWIFTRBIN/SwiftRServer.sh $SLOTDIR >& $SLOTDIR/R.log & # launch R server
+ idletimer $SLOTDIR </dev/null >/dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it
+ echo "$0: INFO: Launched $SWIFTRBIN/SwiftServer.sh $SLOTDIR Rscript"
+else
+ # wait to make sure fifo exists
+ # fromR fifo is created last, so wait for that one
+ if [ ! -p $SLOTDIR/fromR.fifo ]; then
+ sleep 5 # FIXME: try a few times, or wait longer?
+ if [ ! -p $SLOTDIR/fromR.fifo ]; then
+ echo "$0: ERROR: FIFO $SLOTDIR/fromR.fifo did not appear within 5 seconds.\n"
+ exit 1
+ fi
+ fi
+fi
+
+# Ready to talk to the server: send request and read response
+
+echo run $(pwd)/$1 $(pwd)/$2 > $SLOTDIR/toR.fifo
+touch $SLOTDIR/lastwrite
+
+head -3 < $SLOTDIR/fromR.fifo # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit")
Modified: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-09-07 19:36:39 UTC (rev 3609)
+++ SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-09-08 01:01:21 UTC (rev 3610)
@@ -1,9 +1,10 @@
rundir=$1
site=$2
script=$3
-# runR=$4
+workerhosts=$4
-cp $script $runR $rundir
+SWIFTRBIN=$(cd $(dirname $0); pwd)
+cp $script $SWIFTRBIN/passive-coaster-swift $rundir
# FIXME: rework this script to transfer all shells and rscripts
# needed, and to copy in the R prelude for the R server processes (to
@@ -15,13 +16,19 @@
# use /bin/sh and count on users PATH?
# then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename)
+app=/bin/bash
+
cat >tc <<END
-$site Rscript Rscript null null ENV::R_TESTS="";ENV::PATH="$PATH"
-# $site RunR $runR null null ENV::R_TESTS="";ENV::PATH="$PATH"
+$site bash $app null null
+#$site bash $app null null ENV::R_TESTS=""
+#$site bash /bin/bash null null ENV::R_TESTS="";ENV::PATH="$PATH"
+#$site Rscript Rscript null null ENV::R_TESTS="";ENV::PATH="$PATH"
+# $site RunR $runR null null ENV::R_TESTS="";ENV::PATH="$PATH"
END
cat >sites.xml <<END
<config>
+
<pool handle="local">
<execution provider="local" url="none" />
<profile namespace="karajan" key="initialScore">10000</profile>
@@ -29,6 +36,17 @@
<filesystem provider="local"/>
<workdirectory>$(pwd)</workdirectory>
</pool>
+
+ <pool handle="passive">
+ <execution provider="coaster" url="none" jobmanager="local:local"/>
+ <profile namespace="globus" key="workerManager">passive</profile>
+ <profile namespace="globus" key="workersPerNode">8</profile>
+ <profile key="jobThrottle" namespace="karajan">.07</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local" url="none" />
+ <workdirectory>/home/wilde/swiftwork</workdirectory>
+ </pool>
+
<pool handle="pbs">
<profile namespace="globus" key="maxwalltime">00:00:10</profile>
<profile namespace="globus" key="maxtime">1800</profile>
@@ -39,11 +57,18 @@
<filesystem provider="local"/>
<workdirectory>$(pwd)</workdirectory>
</pool>
+
</config>
END
-cat >RunR.R <<END # Note: \$ escapes in "here document" below
+cat >RunR.sh <<ENDX # Note: \$ escapes and nested "<<" in "here document" below
+echo PRINTENV:
+printenv
+echo ENV:
+env
+cat >serverscript.R <<END
+
# User initialization statements (if specified):
$(cat initialize.R 2>/dev/null)
# End of user initialization statements
@@ -53,14 +78,25 @@
load(argv[1]);
result=list()
-for(c in 1:length(rcall\$arglistbatch)) {
+for(c in 1:length(rcall\\\$arglistbatch)) {
# FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status)
- result[[c]] = do.call( rcall\$func, rcall\$arglistbatch[[c]] )
+ result[[c]] = do.call( rcall\\\$func, rcall\\\$arglistbatch[[c]] )
}
save(result,file=argv[2])
END
+Rscript serverscript.R \$*
+ENDX
+
+if [ _$site = _passive ]; then
+ swiftcmd=./passive-coaster-swift
+ # workhosts passed in as argument
+else
+ swiftcmd=swift
+ workerhosts=""
+fi
+
script=$(basename $script)
-swift -tc.file tc -sites.file sites.xml $script
+$swiftcmd "$workerhosts" -tc.file tc -sites.file sites.xml $script
Added: SwiftApps/SwiftR/Swift/exec/passive-coaster-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/passive-coaster-swift (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/passive-coaster-swift 2010-09-08 01:01:21 UTC (rev 3610)
@@ -0,0 +1,70 @@
+#! /bin/bash
+
+COMPUTEHOSTS=$1
+# COMPUTEHOSTS='crush thwomp stomp crank steamroller grind churn trounce thrash vanquish'
+# COMPUTEHOSTS='communicado'
+
+shift
+
+function wait-and-start-workers
+{
+ # Look for:
+ # Passive queue processor initialized. Callback URI is http://140.221.8.62:55379
+
+ for try in $(seq 1 20); do
+ uriline=$(grep "Passive queue processor initialized. Callback URI is" $out 2> /dev/null)
+ if [ "_$uriline" = _ ]; then
+ sleep 1
+ else
+ break;
+ fi
+ done
+ CONTACT=$(echo $uriline | sed -e 's/^.*http:/http:/')
+ echo Coaster contact: $CONTACT
+
+ LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env?
+ mkdir -p $LOGDIR
+
+ workerscript=/home/wilde/swift/lab/worker.pl # FIXME: Try to send this inline or via scp; cp from exec/ dir
+ # CA="X509_CERT_DIR=$CERTDIR X509_CADIR=$CERTDIR"
+ for host in $(echo $COMPUTEHOSTS); do
+ timestamp=$(date "+%Y.%m%d.%H%M%S")
+ random=$(awk "BEGIN {printf \"%0.5d\", $RANDOM}")
+ ID=$timestamp.$random
+ ssh $host WORKER_LOGGING_ENABLED=true $workerscript $CONTACT $ID $LOGDIR &
+ sshpids="$sshpids $!"
+ done
+
+ echo Started workers from these ssh processes: $sshpids
+ echo $sshpids > $sshpidfile
+ # ps $sshpids
+}
+
+if ! which swift >& /dev/null; then
+ echo $0: \'swift\' command not found in PATH
+ exit 1
+fi
+
+out=`mktemp swift.stdouterr.XXXX`
+touch $out
+sshpidfile=${out/stdouterr/workerpids}
+
+echo swift output is in: $out, pids in $sshpidfile
+
+TRAPS="EXIT 1 2 3 15" # Signals and conditions to trap
+
+function onexit {
+ trap - $TRAPS
+ sshpids=$(cat $sshpidfile)
+ echo "Terminating worker processes $sshpids and starter $starterpid"
+ if [ "_$sshpids$starterpid" != _ ]; then
+ kill $sshpids $starterpid >& /dev/null
+ fi
+}
+
+trap onexit $TRAPS
+
+wait-and-start-workers &
+starterpid=$!
+
+swift $* 2>&1 | tee $out
Property changes on: SwiftApps/SwiftR/Swift/exec/passive-coaster-swift
___________________________________________________________________
Name: svn:executable
+ *
Modified: SwiftApps/SwiftR/Swift/exec/swiftapply.swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-09-07 19:36:39 UTC (rev 3609)
+++ SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-09-08 01:01:21 UTC (rev 3610)
@@ -1,10 +1,9 @@
type file;
type RData;
-type RScript;
-app (RData result, file stout, file sterr) RunR (RScript rscript, RData rcall)
+app (RData result, file stout, file sterr) RunR (file shellscript, RData rcall)
{
- Rscript @rscript @rcall @result stdout=@stout stderr=@sterr;
+ bash @shellscript @rcall @result stdout=@stout stderr=@sterr;
# RunR @rcall @result stdout=@stout stderr=@sterr;
}
@@ -12,8 +11,8 @@
RData results[] <simple_mapper; prefix="rbatch.", suffix=".Rdata", padding=0>;
file stout[] <simple_mapper; prefix="stdout.", suffix=".txt", padding=0>;
file sterr[] <simple_mapper; prefix="stderr.", suffix=".txt", padding=0>;
-RScript rs <"RunR.R">;
+file runRscript <"RunR.sh">;
foreach c, i in rcalls {
- (results[i],stout[i], sterr[i]) = RunR(rs,c);
+ (results[i],stout[i], sterr[i]) = RunR(runRscript,c);
}
More information about the Swift-commit
mailing list