[Swift-commit] r2051 - in trunk: . docs libexec tests/misc

[noreply at svn.ci.uchicago.edu]

Author: benc
Date: 2008-06-06 17:40:15 -0500 (Fri, 06 Jun 2008)
New Revision: 2051

Added:
   trunk/tests/misc/workernode-local.sh
Modified:
   trunk/CHANGES.txt
   trunk/docs/userguide.xml
   trunk/libexec/wrapper.sh
   trunk/tests/misc/run
Log:
worker-node local copying of temp files

Modified: trunk/CHANGES.txt
===================================================================
--- trunk/CHANGES.txt	2008-06-06 16:55:20 UTC (rev 2050)
+++ trunk/CHANGES.txt	2008-06-06 22:40:15 UTC (rev 2051)
@@ -1,3 +1,9 @@
+(06/06/08)
+*** SWIFT_JOBDIR_PATH environment variable can be use in profiles (or
+    anywhere else that will get it to the worker script) to cause the
+    worker to copy input files to a possibly-worker-node-local working
+    directory.
+
 (04/24/08)
 *** PATHPREFIX environment variable is prefixed to the path before execution
     in wrapper.sh; this variable can be set in profile entries using the usual

Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml	2008-06-06 16:55:20 UTC (rev 2050)
+++ trunk/docs/userguide.xml	2008-06-06 22:40:15 UTC (rev 2051)
@@ -2587,6 +2587,14 @@
 Typing 'java -help' will sometimes give a list of commands. The Sun Java
 1.4.2 command line options are <ulink url="http://java.sun.com/j2se/1.4.2/docs/tooldocs/windows/java.html">documented here</ulink>.
 		</para>
+		<para>
+SWIFT_JOBDIR_PATH - set in env namespace profiles. If set, then Swift will
+use the path specified here as a worker-node local temporary directory to
+copy input files to before running a job. If unset, Swift will keep input
+files on the site-shared filesystem. In some cases, copying to a worker-node
+local directory can be much faster than having applications access the
+site-shared filesystem directly.
+		</para>
 	</section>
 
 </article>

Modified: trunk/libexec/wrapper.sh
===================================================================
--- trunk/libexec/wrapper.sh	2008-06-06 16:55:20 UTC (rev 2050)
+++ trunk/libexec/wrapper.sh	2008-06-06 22:40:15 UTC (rev 2051)
@@ -144,7 +144,15 @@
 	fail 254 "Missing arguments (-a option)"
 fi
 
-DIR=jobs/$JOBDIR/$ID
+if [ "X$SWIFT_JOBDIR_PATH" != "X" ]; then
+  log "Job directory mode is: local copy"
+  DIR=${SWIFT_JOBDIR_PATH}/$JOBDIR/$ID
+  COPYNOTLINK=1
+else
+  log "Job directory mode is: link on shared filesystem"
+  DIR=jobs/$JOBDIR/$ID
+  COPYNOTLINK=0
+fi
 
 PATH=$PATH:/bin:/usr/bin
 
@@ -180,9 +188,15 @@
 
 logstate "LINK_INPUTS"
 for L in $INF ; do
-	ln -s "$PWD/shared/$L" "$DIR/$L" 2>&1 >& $INFO
-	checkError 254 "Failed to link input file $L"
-	log "Linked input: $PWD/shared/$L to $DIR/$L"
+	if [ $COPYNOTLINK = 1 ]; then
+		cp "$PWD/shared/$L" "$DIR/$L" 2>&1 >& $INFO
+		checkError 254 "Failed to copy input file $L"
+		log "Copied input: $PWD/shared/$L to $DIR/$L"
+	else
+		ln -s "$PWD/shared/$L" "$DIR/$L" 2>&1 >& $INFO
+		checkError 254 "Failed to link input file $L"
+		log "Linked input: $PWD/shared/$L to $DIR/$L"
+	fi
 done
 
 logstate "EXECUTE"

Modified: trunk/tests/misc/run
===================================================================
--- trunk/tests/misc/run	2008-06-06 16:55:20 UTC (rev 2050)
+++ trunk/tests/misc/run	2008-06-06 22:40:15 UTC (rev 2051)
@@ -1,5 +1,5 @@
 #!/bin/sh
-for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5; do
+for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 workernode-local; do
  ./${a}.sh
  R=$?
  echo test $a ended with return value $R

Added: trunk/tests/misc/workernode-local.sh
===================================================================
--- trunk/tests/misc/workernode-local.sh	                        (rev 0)
+++ trunk/tests/misc/workernode-local.sh	2008-06-06 22:40:15 UTC (rev 2051)
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+rm -rf wn-local.d/ wn-local-timestamp
+
+mkdir wn-local.d
+sleep 5s
+touch wn-local-timestamp
+sleep 5s
+
+export SWIFT_JOBDIR_PATH=`pwd`/wn-local.d/
+
+swift ../language-behaviour/001-echo.swift
+
+if [ wn-local.d -nt wn-local-timestamp ]; then
+ echo something happened in the worker node directory - PASS
+ exit 0
+else
+ echo nothing happened in the worker node directory - FAIL
+ exit 1
+fi


Property changes on: trunk/tests/misc/workernode-local.sh
___________________________________________________________________
Name: svn:executable
   + *