[Swift-commit] cog r3645

swift at ci.uchicago.edu swift at ci.uchicago.edu
Mon Mar 18 16:20:08 CDT 2013


------------------------------------------------------------------------
r3645 | davidkelly999 | 2013-03-18 16:16:10 -0500 (Mon, 18 Mar 2013) | 2 lines

Some fixes to allow MPI to work well with Slurm

------------------------------------------------------------------------
Index: modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java
===================================================================
--- modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java	(revision 3644)
+++ modules/provider-localscheduler/src/org/globus/cog/abstraction/impl/scheduler/slurm/SlurmExecutor.java	(working copy)
@@ -23,16 +23,17 @@
 	// Number of program invocations
 	private int count = 1;
 	private static NumberFormat IDF = new DecimalFormat("000000");
-	
+
 	// Used for task name generation
 	private static int unique = 0;
-	
+
 	public SlurmExecutor(Task task, ProcessListener listener) {
 		super(task, listener);
 	}
 
 	/**
 	 * Write attribute if non-null
+	 * 
 	 * @throws IOException
 	 */
 	protected void writeAttr(String attrName, String arg, Writer wr)
@@ -44,7 +45,8 @@
 	}
 
 	/**
-	 * Write attribute if non-null and non-empty 
+	 * Write attribute if non-null and non-empty
+	 * 
 	 * @throws IOException
 	 */
 	protected void writeNonEmptyAttr(String attrName, String arg, Writer wr)
@@ -59,6 +61,7 @@
 
 	/**
 	 * Write walltime in hh:mm:ss
+	 * 
 	 * @param wr
 	 * @throws IOException
 	 */
@@ -90,9 +93,10 @@
 			logger.debug("Slurm name: for: " + task.getIdentity() + " is: " + name);
 		}
 	}
-	
+
 	/**
 	 * Verify that an object contains a valid int
+	 * 
 	 * @param obj
 	 * @param name
 	 * @return
@@ -101,8 +105,7 @@
 		try {
 			assert (obj != null);
 			return Integer.parseInt(obj.toString());
-		} 
-		catch (NumberFormatException e) {
+		} catch (NumberFormatException e) {
 			throw new IllegalArgumentException("Illegal value for " + name + ". Must be an integer.");
 		}
 	}
@@ -110,79 +113,93 @@
 	@Override
 	protected void writeScript(Writer wr, String exitcodefile, String stdout, String stderr) 
 			throws IOException {
+		
 		Task task = getTask();
 		JobSpecification spec = getSpec();
 		Properties properties = Properties.getProperties();
-                boolean exclusive_defined=false;
+		boolean exclusive_defined = false;
 
 		validate(task);
 		writeHeader(wr);
 
+		String type = (String) spec.getAttribute("jobType");
 		Object countValue = getSpec().getAttribute("count");
-		if (countValue != null)
+		
+		if (countValue != null) {
 			count = parseAndValidateInt(countValue, "count");
+		}
+		
 		wr.write("#SBATCH --job-name=" + task.getName() + '\n');
 		wr.write("#SBATCH --output=" + quote(stdout) + '\n');
 		wr.write("#SBATCH --error=" + quote(stderr) + '\n');
 		wr.write("#SBATCH --nodes=" + count + '\n');
-		wr.write("#SBATCH --ntasks-per-node=1\n");
-		writeNonEmptyAttr("ppn", "--cpus-per-task", wr);
 		writeNonEmptyAttr("project", "--account", wr);
 		writeNonEmptyAttr("queue", "--partition", wr);
 		writeWallTime(wr);
 		
+	    if("single".equalsIgnoreCase(type)) {
+			writeNonEmptyAttr("ppn", "--ntasks-per-node", wr);
+	    } else { 
+	    	wr.write("#SBATCH --ntasks-per-node=1\n");
+	    	writeNonEmptyAttr("jobsPerNode", "--cpus-per-task", wr);
+	    }
+
 		// Handle all slurm attributes specified by the user
 		for (String a : spec.getAttributeNames()) {
 			if (a != null && a.startsWith("slurm.")) {
 				String attributeName[] = a.split("slurm.");
-                                 if (attributeName[1].equals("exclusive")) { 
-					exclusive_defined=true;
-					if(spec.getAttribute(a).equals("true")) {
+				if (attributeName[1].equals("exclusive")) {
+					exclusive_defined = true;
+					if (spec.getAttribute(a).equals("true")) {
 						wr.write("#SBATCH --exclusive");
 					} else {
 						wr.write("#SBATCH --share");
 					}
-                                
-                                 } else {
-				 wr.write("#SBATCH --" + attributeName[1] + "=" + spec.getAttribute(a) + '\n');
-                        	 }
+				} else {
+					wr.write("#SBATCH --" + attributeName[1] + "="
+							+ spec.getAttribute(a) + '\n');
+				}
 			}
 		}
- 
-                if(!exclusive_defined) {
-                   wr.write("#SBATCH --exclusive\n");
-                }		
 
+		// Default to exclusive mode
+		if (!exclusive_defined) {
+			wr.write("#SBATCH --exclusive\n");
+		}
+
+		// Environment variables
 		wr.write("\n");
 		for (String name : spec.getEnvironmentVariableNames()) {
-			wr.write("export " + name + '=' + quote(spec.getEnvironmentVariable(name)) + '\n');
+			wr.write("export " + name + '='
+					+ quote(spec.getEnvironmentVariable(name)) + '\n');
 		}
 
-
-
-		String type = (String) spec.getAttribute("jobType");
-		if (logger.isDebugEnabled())
-			logger.debug("Job type: " + type);
-
+		// Determine wrapper
 		if (type != null) {
 			String wrapper = properties.getProperty("wrapper." + type);
 			if (logger.isDebugEnabled()) {
 				logger.debug("Wrapper: " + wrapper);
 			}
-			
+
 			if (wrapper != null) {
 				wrapper = replaceVars(wrapper);
 				wr.write(wrapper);
 				wr.write(' ');
 			}
-			
-			if (logger.isDebugEnabled()) { 
+
+			if (logger.isDebugEnabled()) {
 				logger.debug("Wrapper after variable substitution: " + wrapper);
 			}
 		}
-	        
-                wr.write("RUNCOMMAND=$( command -v ibrun || command -v srun )\n");
-		wr.write("$RUNCOMMAND /bin/bash -c \'");
+
+		// Don't use srun for MPI jobs
+		if("single".equalsIgnoreCase(type)) {
+			wr.write("/bin/bash -c \'");
+		} else {
+			wr.write("RUNCOMMAND=$( command -v ibrun || command -v srun )\n");
+			wr.write("$RUNCOMMAND /bin/bash -c \'");			
+		}
+		
 		if (spec.getDirectory() != null) {
 			wr.write("cd " + quote(spec.getDirectory()) + " && ");
 		}
@@ -193,8 +210,8 @@
 		if (spec.getStdInput() != null) {
 			wr.write(" < " + quote(spec.getStdInput()));
 		}
-		
-		if("multiple".equals(type)) {
+
+		if ("multiple".equals(type)) {
 			wr.write("; /bin/echo $? >" + exitcodefile + ".$SLURM_PROCID\'\n");
 		} else {
 			wr.write("; /bin/echo $? >" + exitcodefile + "\'\n");
@@ -240,14 +257,14 @@
 			return poller;
 		}
 	}
-	
-    protected String parseSubmitCommandOutput(String out) throws IOException {
-        if ("".equals(out)) {
-            throw new IOException(getProperties().getSubmitCommandName()
-                    + " returned an empty job ID");
-        }
-        String outArray[] = out.split(" ");
-        return outArray[outArray.length-1].trim();
-    }
-    
+
+	protected String parseSubmitCommandOutput(String out) throws IOException {
+		if ("".equals(out)) {
+			throw new IOException(getProperties().getSubmitCommandName()
+					+ " returned an empty job ID");
+		}
+		String outArray[] = out.split(" ");
+		return outArray[outArray.length - 1].trim();
+	}
+
 }



More information about the Swift-commit mailing list