[Swift-commit] r4116 - SwiftApps/SwiftR/Swift/exec

[noreply at svn.ci.uchicago.edu]

Author: tga
Date: 2011-02-18 15:59:30 -0600 (Fri, 18 Feb 2011)
New Revision: 4116

Modified:
   SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
   SwiftApps/SwiftR/Swift/exec/start-swift
Log:
Set cobalt to use provider staging.
Fixed bug where cobalt submit file was overwritten before it could be used.


Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt	2011-02-17 23:50:37 UTC (rev 4115)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt	2011-02-18 21:59:30 UTC (rev 4116)
@@ -30,7 +30,8 @@
     <profile namespace="karajan" key="jobThrottle">$throttle</profile>
     <profile namespace="karajan" key="initialScore">10000</profile>
     <filesystem provider="local" url="none"/>
-    <workdirectory>$HOME/swiftwork</workdirectory>
+    <workdirectory>$(pwd)/swiftwork</workdirectory>
+    <scratch>$(pwd)/swiftscratch</scratch>
   </pool>
 
 </config>
@@ -42,8 +43,8 @@
 execution.retries=0
 lazy.errors=false
 status.mode=provider
-use.provider.staging=false
-provider.staging.pin.swiftfiles=false
+use.provider.staging=true
+provider.staging.pin.swiftfiles=true
 #throttle.host.submit=1
 END
 

Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift	2011-02-17 23:50:37 UTC (rev 4115)
+++ SwiftApps/SwiftR/Swift/exec/start-swift	2011-02-18 21:59:30 UTC (rev 4116)
@@ -1,6 +1,6 @@
 #! /bin/bash
 
-#set -x
+set -x
 
 export TRAPEVENTS="EXIT 1 2 3 15"  # Signals and conditions to trap
 
@@ -81,12 +81,13 @@
 # FIXME: does PBS need same workers-per-node logic as SGE?
 make-pbs-submit-file()
 {
+  SUBMIT_FILE=$1
   if [ $queue != NONE ]; then
     queueDirective="#PBS -q $queue"
   else
     queueDirective=""
   fi
-cat >batch.sub <<END
+cat > $SUBMIT_FILE <<END
 #PBS -S /bin/sh
 #PBS -N SwiftR-workers
 #PBS -m n
@@ -114,6 +115,7 @@
 
 make-pbsf-submit-file()
 {
+  SUBMIT_FILE=$1
   if [ _$GLOBUS_HOSTNAME = _ ]; then
     echo GLOBUS_HOSTNAME must be set to worker-reachable address of submit host for pbsf server mode.
     usage
@@ -124,7 +126,7 @@
   else
     queueDirective=""
   fi
-cat >batch.sub <<END
+cat > $SUBMIT_FILE <<END
 #PBS -S /bin/sh
 #PBS -N SwiftR-workers
 #PBS -m n
@@ -164,10 +166,12 @@
 
 make-cobalt-submit-file() 
 {
+  SUBMIT_FILE=$1
 # Most cobalt options are passed through the command line, we just need to
 # include relevant arguments for worker.pl in here.
-
-cat > $HOME/batch.sub <<END
+# Create a batch file in home, as the file needs to be readable
+# by cluster nodes
+cat > $SUBMIT_FILE <<END
 #!/bin/sh
 export WORKER_LOGGING_LEVEL=$workerLogging
 echo '***' COBALT_NODEFILE file: \$COBALT_NODEFILE CONTACT:$CONTACT
@@ -180,13 +184,14 @@
 wait
 
 END
-chmod +x $HOME/batch.sub
+chmod +x $SUBMIT_FILE 
 }
 
 # FIXME: for big systems like Ranger, need to use ssh_tree to avoid socket FD exhastion?
 
 make-sge-submit-file()
 {
+  SUBMIT_FILE=$1
   if [ $queue != NONE ]; then
     queueDirective="#$ -q $queue"
   else
@@ -209,7 +214,7 @@
 
 #  rcmd="qrsh" # FIXME - need to set on system basis; qrsh works for siraf
 
-cat >batch.sub <<END
+cat > $SUBMIT_FILE <<END
 #!/bin/bash
 #$ -S /bin/bash
 #$ -o $HOME
@@ -275,7 +280,13 @@
   IDLETIMEOUT=$((60*60*240)) # 10 days: FIXME: make this a command line arg
 
   # FIXME: set up for capturing batch job id: rm -rf remotepid.* # FIXME: should not be needed if we start in a new dir each time
-  make-${server}-submit-file
+
+  if [ "${server}" = "cobalt" ] ; then
+      SUBMIT_FILE=$(mktemp -p $HOME SwiftR-batch.XXXXXX)
+  else
+      SUBMIT_FILE=batch.sub
+  fi
+  make-${server}-submit-file $SUBMIT_FILE
   #FIXME: doesn't work for SGE on IBI cluster as there is additional text
   # returned by qsub
   
@@ -286,7 +297,7 @@
     # directly from qsub.  We will parse out the first number in this text
     # and assume this is the job ID (this is true for versions of SGE
     # this was tested on).
-    qsub batch.sub | sed 's/[^0-9 ]//g' | awk '{ print $1 }' > $jobidfile
+    qsub $SUBMIT_FILE | sed 's/[^0-9 ]//g' | awk '{ print $1 }' > $jobidfile
     succ=$?
   elif [ "${server}" = "cobalt" ]
   then
@@ -317,10 +328,10 @@
     qsub -t "$time" -n $nodes $queueDirective $kernDirective \
 	$projDirective -e $wkdir/SwiftR-workers.err \
         -o $wkdir/SwiftR-workers.out --debuglog $wkdir/cobalt.log \
-        --cwd / $HOME/batch.sub > $jobidfile
+        --cwd / $SUBMIT_FILE > $jobidfile
     succ=$?
   else
-    qsub batch.sub > $jobidfile
+    qsub $SUBMIT_FILE > $jobidfile
     succ=$?
   fi
 
@@ -501,6 +512,10 @@
     if [ "$keepdir" = "FALSE" ]; then
         cd /
         rm -rf "$trundir"
+        if [ "$server" = "cobalt" ]; then
+            # Submit file in different directory
+            rm $SUBMIT_FILE
+        fi 
     fi
 }