[Swift-commit] r4127 - in SwiftApps/SwiftR/Swift: R exec

[noreply at svn.ci.uchicago.edu]

Author: tga
Date: 2011-02-21 16:15:00 -0600 (Mon, 21 Feb 2011)
New Revision: 4127

Modified:
   SwiftApps/SwiftR/Swift/R/Workers.R
   SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
   SwiftApps/SwiftR/Swift/exec/configure-server-pbs
   SwiftApps/SwiftR/Swift/exec/configure-server-sge
   SwiftApps/SwiftR/Swift/exec/configure-server-ssh
   SwiftApps/SwiftR/Swift/exec/start-swift
Log:
Revamped throttle setting to be based on core and node count rather than hardwired.


Modified: SwiftApps/SwiftR/Swift/R/Workers.R
===================================================================
--- SwiftApps/SwiftR/Swift/R/Workers.R	2011-02-21 21:22:50 UTC (rev 4126)
+++ SwiftApps/SwiftR/Swift/R/Workers.R	2011-02-21 22:15:00 UTC (rev 4127)
@@ -118,8 +118,14 @@
         }
     }
     
-    if(is.null(throttle))
+    if(is.null(throttle)) {
         throttle <- getOption("swift.throttle")
+        if (is.null(throttle)) {
+            # number of simultaneous jobs / 10:
+            #   Round up to nearest integer
+            throttle <- as.integer((cores * nodes / 10.0) + 1)
+        }
+    }
     if(! is.null(throttle) )  {
         cmdString <- paste(cmdString, "-p", shQuote(throttle)) 
     }

Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt	2011-02-21 21:22:50 UTC (rev 4126)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt	2011-02-21 22:15:00 UTC (rev 4127)
@@ -3,9 +3,8 @@
 # configuration for cobalt with manually-started Swift workers (passive coasters)
 
 cores=$1
+throttle=$2
 
-throttle=5.0 # allow approximately 500 concurrent jobs
-
 cat >tc <<END
 fork      bashlocal /bin/bash null null GLOBUS::maxwalltime="00:00:10"
 cobalt       bash      /bin/bash null null ENV::PATH="$PATH";GLOBUS::maxwalltime="00:01:00"

Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbs
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-pbs	2011-02-21 21:22:50 UTC (rev 4126)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-pbs	2011-02-21 22:15:00 UTC (rev 4127)
@@ -4,7 +4,7 @@
 
 cores=$1
 
-throttlePBS=5.0
+throttlePBS=$2
 
 cat >tc <<END
 fork      bashlocal /bin/bash null null null

Modified: SwiftApps/SwiftR/Swift/exec/configure-server-sge
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-sge	2011-02-21 21:22:50 UTC (rev 4126)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-sge	2011-02-21 22:15:00 UTC (rev 4127)
@@ -3,7 +3,7 @@
 # Generate Swift configuration files for SGE with manually-started Swift workers (passive coasters)
 
 cores=$1
-throttle=5.0 # allow approximately 500 concurrent jobs
+throttle=$2
 
 cat >tc <<END
 fork      bashlocal /bin/bash null null GLOBUS::maxwalltime="00:00:10"
@@ -11,7 +11,6 @@
 END
 
 # FIXME: examine effect of 1-min default maxwalltime above
-# FIXME: determine best value for throttle below
 
 cat >sites.xml <<END
 <config>

Modified: SwiftApps/SwiftR/Swift/exec/configure-server-ssh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-ssh	2011-02-21 21:22:50 UTC (rev 4126)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-ssh	2011-02-21 22:15:00 UTC (rev 4127)
@@ -1,8 +1,10 @@
 #! /bin/bash
 
 cores=$1
-time=$2
+throttle=$2
+time=$3
 
+
 cat >tc <<END
 fork      bashlocal /bin/bash null null null
 sshcoast  bash      /bin/bash null null ENV::PATH="$PATH";GLOBUS::maxwalltime="$time"

Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift	2011-02-21 21:22:50 UTC (rev 4126)
+++ SwiftApps/SwiftR/Swift/exec/start-swift	2011-02-21 22:15:00 UTC (rev 4127)
@@ -403,7 +403,7 @@
    -h hosts    1           list of hosts, quoted as one argument, space separated
    -m workmode node        node: start one worker for all slots on a node; slot (one worker on each slot) (Currently ignored)
    -n nodes    1
-   -p throttle 10          >= 1
+   -p throttle 10          >= 1 integer
    -q queue                site speific (PBS, SGE, Cobalt)
    -r rcmd     ssh         site specific, SGE only, typically ssh. qrsh for siraf cluster
    -s server   local       local, pbs, sge, ssh, pbsf,cobalt (for firewalled worker nodes)
@@ -433,8 +433,6 @@
 time="00:30:00"
 nodes=1
 queue=short
-#throttleOneCore="-0.001"  FIXME: Remove
-#throttleOneCore="0.00"    FIXME: Remove
 cores=0
 defaultLocalCores=2
 defaultSshCores=4
@@ -581,7 +579,7 @@
   fi
   echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle
 
-  source $SWIFTRBIN/configure-server-ssh $cores $time
+  source $SWIFTRBIN/configure-server-ssh $cores $throttle $time
 
   sshpidfile=${out/stdouterr/workerpids}
 
@@ -623,9 +621,9 @@
   echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle
   if [ $server = pbsf ]
   then
-    source $SWIFTRBIN/configure-server-pbs $cores
+    source $SWIFTRBIN/configure-server-pbs $cores $throttle
   else
-    source $SWIFTRBIN/configure-server-${server} $cores
+    source $SWIFTRBIN/configure-server-${server} $cores $throttle
   fi
 
   jobidfile=${out/stdouterr/jobid}