[Swift-commit] r4962 - branches/release-0.93/libexec

[hategan at ci.uchicago.edu]

Author: hategan
Date: 2011-08-07 21:23:12 -0500 (Sun, 07 Aug 2011)
New Revision: 4962

Modified:
   branches/release-0.93/libexec/_swiftwrap
   branches/release-0.93/libexec/_swiftwrap.staging
   branches/release-0.93/libexec/vdl-int-staging.k
   branches/release-0.93/libexec/vdl-int.k
Log:
use error files even when using provider status for better clarity

Modified: branches/release-0.93/libexec/_swiftwrap
===================================================================
--- branches/release-0.93/libexec/_swiftwrap	2011-08-08 00:07:35 UTC (rev 4961)
+++ branches/release-0.93/libexec/_swiftwrap	2011-08-08 02:23:12 UTC (rev 4962)
@@ -58,9 +58,13 @@
 fail() {
 	EC=$1
 	shift
-	if [ "$STATUSMODE" = "files" ]; then
-		echo $@ >"$WFDIR/status/$JOBDIR/${ID}-error"
+	
+	if [ "$STATUSMODE" != "files" ]; then
+		mkdir -p $WFDIR/status/$JOBDIR
 	fi
+	
+	echo $@ >"$WFDIR/status/$JOBDIR/${ID}-error"
+		
 	log $@
 	info
 	if [ "$STATUSMODE" = "files" ]; then

Modified: branches/release-0.93/libexec/_swiftwrap.staging
===================================================================
--- branches/release-0.93/libexec/_swiftwrap.staging	2011-08-08 00:07:35 UTC (rev 4961)
+++ branches/release-0.93/libexec/_swiftwrap.staging	2011-08-08 02:23:12 UTC (rev 4962)
@@ -61,19 +61,11 @@
 fail() {
 	EC=$1
 	shift
-	if [ "$STATUSMODE" = "files" ]; then
-		echo $@ >"$WFDIR/status/$JOBDIR/${ID}-error"
-	fi
+
 	log $@
 	info
-	if [ "$STATUSMODE" = "files" ]; then
-		exit 0
-	else
-		if [ "X$STDERR" != "X" ]; then
-			echo "$@" >>$STDERR
-		fi
-		exit $EC
-	fi
+	echo "$@" >> wrapper.error
+	exit $EC
 }
 
 checkError() {
@@ -308,7 +300,7 @@
 	fi
 	"$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN"
 fi
-checkError $? "Exit code $?"
+checkError $? "Exit code $?" <<$STDERR
 
 if [ ! -s "$STDOUT" ]; then
 	log "Removing empty stdout"

Modified: branches/release-0.93/libexec/vdl-int-staging.k
===================================================================
--- branches/release-0.93/libexec/vdl-int-staging.k	2011-08-08 00:07:35 UTC (rev 4961)
+++ branches/release-0.93/libexec/vdl-int-staging.k	2011-08-08 02:23:12 UTC (rev 4962)
@@ -70,7 +70,12 @@
 		element(cleanups, [cleanup]
 			log(LOG:INFO, "START cleanups={cleanup}")
 		)
-
+		
+		element(readErrorFile, [dir, jobid]
+			str:strip(file:read("{dir}/{jobid}.error"))
+			file:remove("{dir}/{jobid}.error")
+		)
+		
 		element(execute2, [tr, optional(arguments, stdin, stdout, stderr), stagein, stageout,  restartout,
 			replicationGroup, replicationChannel]
 			stagein := list(unique(each(stagein)))
@@ -150,6 +155,8 @@
 								mode = WRAPPER_TRANSFER_MODE)
 							//stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}")
 							//stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}")
+							stageOut("wrapper.error", "{stagingMethod}://localhost/{ddir}/{jobid}.error", 
+								mode = STAGING_MODE:IF_PRESENT)
 							appStageouts(jobid, stageout, ".", stagingMethod)
 
 							task:cleanUp(".") //the whole job directory
@@ -163,6 +170,7 @@
 					)
 					catch("^(?!Abort$).*"
 						vdl:setprogress("Failed but can retry")
+						exception := try(exception(readErrorFile(ddir, jobid)), exception)
 						log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception)
 
 						throw(

Modified: branches/release-0.93/libexec/vdl-int.k
===================================================================
--- branches/release-0.93/libexec/vdl-int.k	2011-08-08 00:07:35 UTC (rev 4961)
+++ branches/release-0.93/libexec/vdl-int.k	2011-08-08 02:23:12 UTC (rev 4962)
@@ -46,24 +46,37 @@
 					file:remove("{wfdir}/status/{jobdir}/{jobid}-success", host=rhost)
 					log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found")
 				)
-				if(
-					file:exists("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost) then(
-						log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found")
-						task:transfer(srchost=rhost, srcdir="{wfdir}/status/{jobdir}", srcfile="{jobid}-error")
-						error := parallel(
-							file:remove("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost)
-							sequential(
-								str:strip(file:read("{jobid}-error"))
-								file:remove("{jobid}-error")
-							)
+				sequential(
+					try (
+						msg = checkErrorFile(rhost, wfdir, jobid, jobdir)
+						sequential (
+							log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing")
+							throw("No status file was found. Check the shared filesystem on {rhost}")
 						)
-						throw(error)
 					)
-					else (
-						log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Both status files are missing")
-						throw("No status file was found. Check the shared filesystem on {rhost}")
+				)
+				throw(checkErrorFile(rhost, wfdir, jobid, tr, jobdir))
+			)
+		)
+		
+		element(checkErrorFile, [rhost, wfdir, jobid, jobdir]
+			if (
+				file:exists("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost) then(
+					log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found")
+					task:transfer(srchost=rhost, srcdir="{wfdir}/status/{jobdir}", srcfile="{jobid}-error")
+					error := parallel(
+						file:remove("{wfdir}/status/{jobdir}/{jobid}-error", host=rhost)
+						sequential(
+							str:strip(file:read("{jobid}-error"))
+							file:remove("{jobid}-error")
+						)
 					)
+					error
 				)
+				else (
+					log(LOG:INFO, "NO_STATUS_FILE jobid={jobid} - Error file missing")
+					throw("No status file was found. Check the shared filesystem on {rhost}")
+				)
 			)
 		)
 
@@ -555,6 +568,8 @@
 					)
 					catch("^(?!Abort$).*"
 						vdl:setprogress("Failed but can retry")
+						exception := try(exception(checkErrorFile(rhost, wfdir, jobid, jobdir)), exception)
+						
 						log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception)
 
 						if(matches(exception,".*executable bit.*")