[Swift-commit] r3578 - trunk/libexec

noreply at svn.ci.uchicago.edu noreply at svn.ci.uchicago.edu
Fri Aug 27 17:15:46 CDT 2010


Author: wozniak
Date: 2010-08-27 17:15:46 -0500 (Fri, 27 Aug 2010)
New Revision: 3578

Modified:
   trunk/libexec/_swiftwrap
   trunk/libexec/_swiftwrap.staging
   trunk/libexec/cdm_lib.sh
   trunk/libexec/vdl-int-staging.k
Log:
Enable CDM DIRECT for input with provider staging


Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap	2010-08-27 22:15:23 UTC (rev 3577)
+++ trunk/libexec/_swiftwrap	2010-08-27 22:15:46 UTC (rev 3578)
@@ -6,7 +6,7 @@
         echo >& "$INFO"
 	echo "_____________________________________________________________________________" >& "$INFO"
 	echo >& "$INFO"
-	echo "        $1" >& "$INFO" 
+	echo "        $1" >& "$INFO"
 	echo "_____________________________________________________________________________" >& "$INFO"
 	echo >& "$INFO"
 }
@@ -121,104 +121,23 @@
 
 	for a in ${!ARRAY}
 	do
-		if [[ ${a} == ${X} ]]; then 
+		if [[ ${a} == ${X} ]]; then
 			return 0
 		fi
 	done
 	return 1
 }
 
-# Output: 
-cdm_lookup() {
-	FILE=$1
-	CDM_FILE=$2
-
-	RESULT="DEFAULT"
-	if [ -f shared/cdm.pl ]; then 
-		RESULT=$( perl shared/cdm.pl lookup $FILE < $CDM_FILE 2> $INFO )
-		checkError 254 "CDM[LOOKUP]: failed! (malformed CDM file?)"
-	fi
-	echo $RESULT
-}
-
-cdm_action() {
-	log "CDM_ACTION: $@"
-
-	local JOBDIR=$1  # Given jobdir
-	local MODE=$2    # INPUT or OUTPUT
-	local FILE=$3    # User file
-	local POLICY=$4  # DIRECT, BROADCAST, ... 
-	shift 4
-	local ARGS=$@
-
-	case $POLICY in 
-		DIRECT)
-			DIRECT_DIR=${ARGS[0]}
-			log "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE via $JOBDIR/$FILE"
-			if [ $MODE == "INPUT" ]; then 
-				[ -f "$DIRECT_DIR/$FILE" ]
-				checkError 254 "CDM[DIRECT]: $DIRECT_DIR/$FILE does not exist!"
-				ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE
-				checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!"
-			elif [ $MODE == "OUTPUT" ]; then 
-				mkdir -p $DIRECT_DIR
-				checkError 254 "CDM[DIRECT]: mkdir -p $DIRECT_DIR failed!"
-				mkdir -p $( dirname $DIRECT_DIR/$FILE ) 
-				checkError 254 "CDM[DIRECT]: mkdir -p $( dirname $FILE ) failed!"
-				touch $DIRECT_DIR/$FILE
-				checkError 254 "CDM[DIRECT]: Touching $DIRECT_DIR/$FILE failed!"
-				ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE
-				checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!"
-			else 
-				fail 254 "Unknown MODE: $MODE"
-			fi
-			;;
- 		LOCAL)
- 			TOOL=${ARGS[0]}
- 			REMOTE_DIR=${ARGS[1]}
- 			FLAGS=${ARGS[3]}
- 			log "CDM[LOCAL]: Copying $DIRECT_DIR/$FILE to $JOBDIR/$FILE"
- 			if [ $MODE == "INPUT" ]; then 
- 				[ -f "$DIRECT_DIR/$FILE" ]
- 				checkError 254 "CDM[LOCAL]: $REMOTE_DIR/$FILE does not exist!"
- 				$TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE
- 				checkError 254 "CDM[LOCAL]: Tool failed!"
- 			elif [ $MODE == "OUTPUT" ]; then 
- 				log "CDM[LOCAL]..."
- 			else 
- 				fail 254 "Unknown MODE: $MODE"
- 			fi
- 			;;
-		BROADCAST)
-			BROADCAST_DIR=${ARGS[0]}
-			
-			if [ $MODE == "INPUT" ]; then 
-				log "CDM[BROADCAST]: Linking $JOBDIR/$FILE to $BROADCAST_DIR/$FILE"
-				[ -f "$BROADCAST_DIR/$FILE" ]
-				checkError 254 "CDM[BROADCAST]: $BROADCAST_DIR/$FILE does not exist!"
-				ln -s $BROADCAST_DIR/$FILE $JOBDIR/$FILE
-				checkError 254 "CDM[BROADCAST]: Linking to $BROADCAST_DIR/$FILE failed!"
-			else 
-				echo "CDM[BROADCAST]: Skipping output file: ${FILE}"
-			fi
-			;;
-		GATHER)
-			if [ $MODE == "INPUT" ]; then 
-				fail 254 "Cannot GATHER an input file!"
-			fi
-	esac
-}
-
 cdm_local_output()
 {
  	L=$1
 
-	if [[ $CDM_FILE == "" ]]; then 
+	if [[ $CDM_FILE == "" ]]; then
 		return
 	fi
 
- 	CDM_POLICY=$( cdm_lookup $L $CDM_FILE )
-	if [[ $CDM_POLICY == "LOCAL" ]]; then 
+ 	CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+	if [[ $CDM_POLICY == "LOCAL" ]]; then
 		cdm_local_output_perform $L $CDM_POLICY
 	fi
 }
@@ -239,13 +158,13 @@
 cdm_gather()
 {
 	GATHER_OUTPUT=${*}
-	if [[ $CDM_FILE == "" ]]; then 
+	if [[ $CDM_FILE == "" ]]; then
 		return
 	fi
-	if [[ $GATHER_OUTPUT == "" ]]; then 
+	if [[ $GATHER_OUTPUT == "" ]]; then
 		return
 	fi
-	
+
 	cdm_gather_action $GATHER_MAX $GATHER_OUTPUT
 }
 
@@ -333,7 +252,7 @@
 
 getarg "-cdmfile" "$@"
 CDM_FILE=
-if [ "X$VALUE" != "X" ]; then 
+if [ "X$VALUE" != "X" ]; then
 	CDM_FILE=shared/$VALUE
 fi
 shift $SHIFTCOUNT
@@ -429,10 +348,10 @@
 
 logstate "LINK_INPUTS"
 for L in $INF ; do
-        CDM_POLICY=$( cdm_lookup $L $CDM_FILE )
+        CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
 	if [ $CDM_POLICY != "DEFAULT" ]; then
 		log "CDM_POLICY: $L -> $CDM_POLICY"
-    	        eval cdm_action $DIR "INPUT" $L $CDM_POLICY 
+    	        eval cdm_action $DIR "INPUT" $L $CDM_POLICY
 		continue
 	fi
 	if [ $COPYNOTLINK = 1 ]; then
@@ -441,7 +360,7 @@
 		log "Copied input: $WFDIR/shared/$L to $DIR/$L"
 	else
 		[ -f $WFDIR/shared/$L ]
-		checkError 254 "Could not locate input file: $L" 
+		checkError 254 "Could not locate input file: $L"
 		ln -s "$WFDIR/shared/$L" "$DIR/$L" 2>&1 >& $INFO
 		checkError 254 "Failed to link input file $L"
 		log "Linked input: $WFDIR/shared/$L to $DIR/$L"
@@ -452,16 +371,16 @@
 SKIPPED_OUTPUT=()
 GATHER_OUTPUT=()
 for L in $OUTF ; do
-	CDM_POLICY=$( cdm_lookup $L $CDM_FILE )
-	if [[ $CDM_POLICY != "DEFAULT" && 
+	CDM_POLICY=$( cdm_lookup shared/cdm.pl $CDM_FILE $L )
+	if [[ $CDM_POLICY != "DEFAULT" &&
               $CDM_POLICY != "BROADCAST"* ]]; then
-		log "CDM_POLICY: $L -> $CDM_POLICY" 
+		log "CDM_POLICY: $L -> $CDM_POLICY"
     	        eval cdm_action $DIR "OUTPUT" $L $CDM_POLICY
 		SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L )
 	fi
-	if [ $CDM_POLICY == "GATHER" ]; then 
+	if [ $CDM_POLICY == "GATHER" ]; then
 		GATHER_OUTPUT=( $GATHER_OUTPUT $L )
-	elif [ $CDM_POLICY == "LOCAL" ]; then 
+	elif [ $CDM_POLICY == "LOCAL" ]; then
 		CDM_LOCAL_OUTPUT=( $CDM_LOCAL_OUTPUT $L )
 	fi
 done
@@ -496,7 +415,7 @@
 else
 	if [ ! -f "$KICKSTART" ]; then
 		log "Kickstart executable ($KICKSTART) not found"
-		fail 254 "The Kickstart executable ($KICKSTART) was not found"		
+		fail 254 "The Kickstart executable ($KICKSTART) was not found"
 	elif [ ! -x "$KICKSTART" ]; then
 		log "Kickstart executable ($KICKSTART) is not executable"
 		fail 254 "The Kickstart executable ($KICKSTART) does not have the executable bit set"
@@ -550,7 +469,7 @@
 
 logstate "RM_JOBDIR"
 rm -rf "$DIR" 2>&1 >& "$INFO"
-checkError 254 "Failed to remove job directory $DIR" 
+checkError 254 "Failed to remove job directory $DIR"
 
 if [ "$STATUSMODE" = "files" ]; then
 	logstate "TOUCH_SUCCESS"
@@ -569,7 +488,7 @@
 # ensure we exit with a 0 after a successful execution
 exit 0
 
-# Local Variables: 
+# Local Variables:
 # mode: sh
 # sh-basic-offset: 8
 # End:

Modified: trunk/libexec/_swiftwrap.staging
===================================================================
--- trunk/libexec/_swiftwrap.staging	2010-08-27 22:15:23 UTC (rev 3577)
+++ trunk/libexec/_swiftwrap.staging	2010-08-27 22:15:46 UTC (rev 3578)
@@ -4,7 +4,7 @@
 	echo >& "$INFO"
 	echo "_____________________________________________________________________________" >& "$INFO"
 	echo >& "$INFO"
-	echo "        $1" >& "$INFO" 
+	echo "        $1" >& "$INFO"
 	echo "_____________________________________________________________________________" >& "$INFO"
 	echo >& "$INFO"
 }
@@ -164,6 +164,14 @@
 KICKSTART=$VALUE
 shift $SHIFTCOUNT
 
+getarg "-cdmfile" "$@"
+CDM_FILE=
+if [ "X$VALUE" != "X" ]; then
+	CDM_FILE=$VALUE
+	log "CDM_FILE: $CDM_FILE"
+fi
+shift $SHIFTCOUNT
+
 getarg "-status" "$@"
 STATUSMODE=$VALUE
 shift $SHIFTCOUNT
@@ -187,6 +195,8 @@
 	fail 254 "Missing arguments (-a option)"
 fi
 
+[[ $CDM_FILE != "" ]] && source cdm_lib.sh
+
 PATH=$PATH:/bin:/usr/bin
 
 if [ "$PATHPREFIX" != "" ]; then
@@ -205,6 +215,7 @@
 	fi
 fi
 
+log "PWD=$PWD"
 log "EXEC=$EXEC"
 log "STDIN=$STDIN"
 log "STDOUT=$STDOUT"
@@ -225,6 +236,14 @@
 	log "Created output directory: $D"
 done
 
+if [[ $CDM_FILE != "" ]] ; then
+	for F in $INF ; do
+		CDM_POLICY=$( cdm_lookup cdm.pl $CDM_FILE $F )
+		log "CDM_POLICY: $F -> $CDM_POLICY"
+		eval cdm_action $PWD "INPUT" $F $CDM_POLICY
+	done
+fi
+
 logstate "EXECUTE"
 
 #ls >>$WRAPPERLOG
@@ -285,3 +304,9 @@
 # ensure we exit with a 0 after a successful execution
 exit 0
 
+# Local Variables:
+# mode: sh
+# sh-basic-offset: 4
+# tab-width: 4
+# indent-tabs-mode: 1
+# End:

Modified: trunk/libexec/cdm_lib.sh
===================================================================
--- trunk/libexec/cdm_lib.sh	2010-08-27 22:15:23 UTC (rev 3577)
+++ trunk/libexec/cdm_lib.sh	2010-08-27 22:15:46 UTC (rev 3578)
@@ -1,6 +1,90 @@
 
 # Source this for CDM shell functions
 
+log "Reading cdm_lib.sh ..."
+
+# Do a CDM lookup
+cdm_lookup() {
+	CDM_PL=$1
+	CDM_FILE=$2
+	FILE=$3
+
+	RESULT="DEFAULT"
+	if [ -f $CDM_PL ]; then
+		RESULT=$( perl $CDM_PL lookup $FILE < $CDM_FILE 2> $INFO )
+		checkError 254 "cdm_lookup(): failed! (malformed CDM file?)"
+	fi
+	echo $RESULT
+}
+
+cdm_action() {
+	log "CDM_ACTION: $@"
+
+	local JOBDIR=$1  # Given jobdir
+	local MODE=$2    # INPUT or OUTPUT
+	local FILE=$3    # User file
+	local POLICY=$4  # DIRECT, BROADCAST, ...
+	shift 4
+	local ARGS=$@
+
+	log "POLICY=$POLICY"
+	case $POLICY in
+		DIRECT)
+			DIRECT_DIR=${ARGS[0]}
+			log "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE via $JOBDIR/$FILE"
+			if [ $MODE == "INPUT" ]; then
+				[ -f "$DIRECT_DIR/$FILE" ]
+				checkError 254 "CDM[DIRECT]: $DIRECT_DIR/$FILE does not exist!"
+				ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE
+				checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!"
+			elif [ $MODE == "OUTPUT" ]; then
+				mkdir -p $DIRECT_DIR
+				checkError 254 "CDM[DIRECT]: mkdir -p $DIRECT_DIR failed!"
+				mkdir -p $( dirname $DIRECT_DIR/$FILE )
+				checkError 254 "CDM[DIRECT]: mkdir -p $( dirname $FILE ) failed!"
+				touch $DIRECT_DIR/$FILE
+				checkError 254 "CDM[DIRECT]: Touching $DIRECT_DIR/$FILE failed!"
+				ln -s $DIRECT_DIR/$FILE $JOBDIR/$FILE
+				checkError 254 "CDM[DIRECT]: Linking to $DIRECT_DIR/$FILE failed!"
+			else
+				fail 254 "Unknown MODE: $MODE"
+			fi
+			;;
+ 		LOCAL)
+ 			TOOL=${ARGS[0]}
+ 			REMOTE_DIR=${ARGS[1]}
+ 			FLAGS=${ARGS[3]}
+ 			log "CDM[LOCAL]: Copying $DIRECT_DIR/$FILE to $JOBDIR/$FILE"
+ 			if [ $MODE == "INPUT" ]; then
+ 				[ -f "$DIRECT_DIR/$FILE" ]
+ 				checkError 254 "CDM[LOCAL]: $REMOTE_DIR/$FILE does not exist!"
+ 				$TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE
+ 				checkError 254 "CDM[LOCAL]: Tool failed!"
+ 			elif [ $MODE == "OUTPUT" ]; then
+ 				log "CDM[LOCAL]..."
+ 			else
+ 				fail 254 "Unknown MODE: $MODE"
+ 			fi
+ 			;;
+		BROADCAST)
+			BROADCAST_DIR=${ARGS[0]}
+			if [ $MODE == "INPUT" ]; then
+				log "CDM[BROADCAST]: Linking $JOBDIR/$FILE to $BROADCAST_DIR/$FILE"
+				[ -f "$BROADCAST_DIR/$FILE" ]
+				checkError 254 "CDM[BROADCAST]: $BROADCAST_DIR/$FILE does not exist!"
+				ln -s $BROADCAST_DIR/$FILE $JOBDIR/$FILE
+				checkError 254 "CDM[BROADCAST]: Linking to $BROADCAST_DIR/$FILE failed!"
+			else
+				echo "CDM[BROADCAST]: Skipping output file: ${FILE}"
+			fi
+			;;
+		GATHER)
+			if [ $MODE == "INPUT" ]; then
+				fail 254 "Cannot GATHER an input file!"
+			fi
+    esac
+}
+
 # Setup GATHER_DIR and some variables
 cdm_gather_setup() {
 	GATHER_DIR=$1
@@ -8,7 +92,7 @@
 	logstate "GATHER_DIR $GATHER_DIR"
 	mkdir -p $GATHER_DIR
 	checkError 254 "Could not create: $GATHER_DIR"
-	
+
 	GATHER_LOCKFILE=$GATHER_DIR/.cdm.lock
 	GATHER_MY_FILE=$GATHER_DIR/.cdm-$ID.lock
 	GATHER_MY_OUTBOX=$GATHER_DIR/.cdm-outgoing-$ID
@@ -72,7 +156,7 @@
 	unlink $GATHER_LOCKFILE
 }
 
-# Move files from (LFS) OUTBOX to (GFS) GATHER_TARGET 
+# Move files from (LFS) OUTBOX to (GFS) GATHER_TARGET
 cdm_gather_flush() {
 	pushd ${GATHER_MY_OUTBOX}
 	logstate "GATHER_TARGET $GATHER_TARGET"
@@ -93,10 +177,10 @@
 	GATHER_MAX=$(    perl shared/cdm.pl property GATHER_LIMIT  < $CDM_FILE )
 	GATHER_TARGET=$( perl shared/cdm.pl property GATHER_TARGET < $CDM_FILE )
 
-	cdm_gather_setup $GATHER_DIR 
-	cdm_gather_lock_acquire 
+	cdm_gather_setup $GATHER_DIR
+	cdm_gather_lock_acquire
 
-	cdm_gather_import 
+	cdm_gather_import
 
 	USAGE=$( du -s --exclude=".cdm*" -B 1 $GATHER_DIR )
 	USAGE=${USAGE%	*} # Chop off filename in output
@@ -104,22 +188,22 @@
 	logstate "USAGE_CHECK $USAGE / $GATHER_MAX"
 
 	FLUSH="no"
-	if (( USAGE > GATHER_MAX )); then 
+	if (( USAGE > GATHER_MAX )); then
 		FLUSH="yes"
 		cdm_gather_export
 	fi
-	
+
 	cdm_gather_lock_release
-	if [ $FLUSH == "yes" ]; then 
-		cdm_gather_flush 
+	if [ $FLUSH == "yes" ]; then
+		cdm_gather_flush
 	fi
 }
 
 # Called by cdm_cleanup.sh at the end of the workflow
 cdm_gather_cleanup() {
 	declare -p PWD DIR GATHER_DIR GATHER_TARGET ID
-	cdm_gather_setup $GATHER_DIR 
-	cdm_gather_lock_acquire 
+	cdm_gather_setup $GATHER_DIR
+	cdm_gather_lock_acquire
 	cdm_gather_export
 	EXPORT_RESULT=$?
 	cdm_gather_lock_release
@@ -128,7 +212,10 @@
 	fi
 }
 
-# Local Variables: 
+
+# Local Variables:
 # mode: sh
-# sh-basic-offset: 8
+# sh-basic-offset: 4
+# tab-width: 4
+# indent-tabs-mode: 1
 # End:

Modified: trunk/libexec/vdl-int-staging.k
===================================================================
--- trunk/libexec/vdl-int-staging.k	2010-08-27 22:15:23 UTC (rev 3577)
+++ trunk/libexec/vdl-int-staging.k	2010-08-27 22:15:46 UTC (rev 3578)
@@ -22,11 +22,11 @@
 			)
 			ddir
 		)
-		
+
 		element(inFiles, [stageins]
 			pathnames(stageins)
 		)
-		
+
 		element(fileDirs, [stageins, stageouts]
 			list(
 				unique(
@@ -35,13 +35,13 @@
 				)
 			)
 		)
-		
+
 		element(graphStuff, [tr, stagein, stageout, err, optional(args)]
 			if(
 				vdl:configProperty("pgraph") != "false" then(
 					errprops := if(err ",color=lightsalmon" ",color=lightsteelblue1")
 					tp := vdl:threadPrefix()
-					to(graph, 
+					to(graph,
 						concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]")
 					)
 					for(si, stagein
@@ -57,7 +57,7 @@
 						file := basename(file)
 						label := vdl:niceName(var, path = path)
 						to(graph
-							concat(str:quote(file), " [shape=parallelogram,label=", 
+							concat(str:quote(file), " [shape=parallelogram,label=",
 								str:quote(label), "]")
 							concat(str:quote(tp), " -> ", str:quote(file))
 						)
@@ -65,13 +65,13 @@
 				)
 			)
 		)
-		
+
 		element(fileSizes, [files]
 			math:sum(
 				for(f, files, file:size(file))
 			)
 		)
-		
+
 		element(cleanups, [cleanup]
 			log(LOG:INFO, "START cleanups={cleanup}")
 		)
@@ -80,17 +80,17 @@
 			replicationGroup, replicationChannel]
 			stagein := list(unique(each(stagein)))
 			stageout := list(unique(each(stageout)))
-			
+
 			allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein)
-				
+
 				ddir := initDDir()
-				
+
 				uid := uid()
 				jobdir := substring(uid, from=0, to=1)
 				jobid := "{tr}-{uid}"
-				
+
 				fileDirs := fileDirs(stagein, stageout)
-				
+
 				log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}")
 
 				wrapper := "_swiftwrap.staging"
@@ -102,11 +102,12 @@
 				vdl:setprogress("Stage in")
 				wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}"
 				tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid)
-				
+                                cdmfile := cdm:file()
+
 				try(
 					sequential(
 						log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}")
-				
+
 						vdl:setprogress("Submitting")
 
 						vdl:execute(
@@ -114,14 +115,15 @@
 							list(
 								vdl:siteprofile(rhost, "swift:wrapperInterpreterOptions"),
 								wrapper,
-								"-e", vdl:executable(tr, rhost), 
+								"-e", vdl:executable(tr, rhost),
 								"-out", stdout,
-								"-err", stderr, 
+								"-err", stderr,
 								"-i", maybe(stdin),
 								"-d", flatten(each(fileDirs)),
-								"-if", flatten(infiles(stagein)), 
+								"-if", flatten(infiles(stagein)),
 								"-of", flatten(outfiles(stageout)),
 								"-k",
+								"-cdmfile", cdmfile,
 								"-status", "provider"
 								"-a", maybe(each(arguments))
 							)
@@ -132,17 +134,27 @@
 							replicationGroup = replicationGroup
 							replicationChannel = replicationChannel
 							jobid = jobid
-							
-							stagingMethod := vdl:siteProfile(rhost, "swift:stagingMethod", default="proxy") 	
-							
+
+							stagingMethod := vdl:siteProfile(rhost, "swift:stagingMethod", default="proxy")
+
 							stageIn("{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper)
+
+							if( cdmfile != "" then(
+								d := vdl:dirname(cdmfile)
+								f := basename(cdmfile)
+								echo("d: {d} f: {f}")
+								stageIn("{stagingMethod}://localhost/{d}/{f}", cdmfile)
+								stageIn("{stagingMethod}://localhost/{swift.home}/libexec/cdm.pl", "cdm.pl")
+								stageIn("{stagingMethod}://localhost/{swift.home}/libexec/cdm_lib.sh", "cdm_lib.sh")
+								))
+
 							appStageins(jobid, stagein, ".", stagingMethod)
-							
+
 							stageOut("wrapper.log", "{stagingMethod}://localhost/{ddir}/{jobid}.info")
 							//stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}")
 							//stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}")
 							appStageouts(jobid, stageout, ".", stagingMethod)
-							
+
 							task:cleanUp(".") //the whole job directory
 						)
 						doRestartlog(restartout)
@@ -155,7 +167,7 @@
 					catch("^(?!Abort$).*"
 						vdl:setprogress("Failed but can retry")
 						log(LOG:DEBUG, "APPLICATION_EXCEPTION jobid={jobid} - Application exception: ", exception)
-						
+
 						throw(
 							exception(
 								concat(
@@ -173,7 +185,7 @@
 				)
 			)
 		)
-	
+
 		element(generateProvenanceGraph, [gdata]
 			pgraph := vdl:configProperty("pgraph")
 			gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph)
@@ -181,7 +193,7 @@
 				"digraph SwiftProvenance {{", nl()
 				"	graph [", vdl:configProperty("pgraph.graph.options"), "];", nl()
 				"	node [", vdl:configProperty("pgraph.node.options"), "];", nl()
-						
+
 				for(i, gdata
 					"	", i, nl()
 				)




More information about the Swift-commit mailing list