[Swift-commit] r2940 - in trunk: . docs etc libexec src/org/griphyn/vdl/util tests/misc tests/sites
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Sun May 17 07:03:40 CDT 2009
Author: benc
Date: 2009-05-17 07:03:39 -0500 (Sun, 17 May 2009)
New Revision: 2940
Added:
trunk/tests/misc/arg-mode.sh
trunk/tests/sites/local-wrapper-args-file.xml
Modified:
trunk/CHANGES.txt
trunk/docs/userguide.xml
trunk/etc/swift.properties
trunk/libexec/_swiftwrap
trunk/libexec/vdl-int.k
trunk/src/org/griphyn/vdl/util/VDL2Config.java
trunk/tests/misc/run
Log:
Allow wrapper parameters to be specific in a file rather than on the
commandline. This trades commandline length for an extra file to stage-in
per job.
A new parameter, wrapper.parameter.mode, is introduced to control this.
Modified: trunk/CHANGES.txt
===================================================================
--- trunk/CHANGES.txt 2009-05-17 12:00:03 UTC (rev 2939)
+++ trunk/CHANGES.txt 2009-05-17 12:03:39 UTC (rev 2940)
@@ -1,3 +1,10 @@
+(05/17/09)
+*** Allow wrapper parameters to be specific in a file rather than on the
+ commandline. This trades commandline length for an extra file to stage-in
+ per job.
+ A new parameter, wrapper.parameter.mode, is introduced to control this.
+ This was feature request bug 201.
+
(05/07/09)
*** swift-log-to-cedps is removed. The functionality of generating
.cedps format files is available by specifying the desired .cedps files
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2009-05-17 12:00:03 UTC (rev 2939)
+++ trunk/docs/userguide.xml 2009-05-17 12:03:39 UTC (rev 2940)
@@ -3039,6 +3039,25 @@
<varlistentry>
<term>
+ <property>wrapper.parameter.mode</property>
+ </term>
+ <listitem>
+ <para>
+Controls how Swift will supply parameters to the remote wrapper script.
+<literal>args</literal> mode will pass parameters on the command line. Some
+execution systems do not pass commandline parameters sufficiently cleanly
+for Swift to operate correctly.
+<literal>files</literal> mode will pass parameters through an additional
+input file (since Swift 0.95). This provides a cleaner communication channel
+for parameters, at the expense of transferring an additional file for each
+job invocation.
+ </para>
+ </listitem>
+
+ </varlistentry>
+
+ <varlistentry>
+ <term>
<property>wrapperlog.always.transfer</property>
</term>
<listitem>
Modified: trunk/etc/swift.properties
===================================================================
--- trunk/etc/swift.properties 2009-05-17 12:00:03 UTC (rev 2939)
+++ trunk/etc/swift.properties 2009-05-17 12:03:39 UTC (rev 2940)
@@ -295,6 +295,14 @@
#
# status.mode=files
+# Controls how swift will supply parameters to the remote wrapper script.
+# 'args' mode will pass parameters on the command line
+# 'files' mode will pass parameters through an additional input file
+#
+# valid values: args, files
+# Default: files
+#
+# wrapper.parameter.mode=args
# Determines if Swift remote wrappers will be executed by specifying an
# absolute path, or a path relative to the job initial working directory
Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap 2009-05-17 12:00:03 UTC (rev 2939)
+++ trunk/libexec/_swiftwrap 2009-05-17 12:03:39 UTC (rev 2940)
@@ -74,12 +74,21 @@
fi
}
+checkparamfile() {
+ log "checking for paramfile"
+ if [ "$1" == "-p" ]; then
+ JOBDIR=$2
+ PARAMFILE=${WFDIR}/parameters/${JOBDIR}/param-${ID}
+ fi
+ log "paramfile is: $PARAMFILE"
+}
+
getarg() {
NAME=$1
shift
VALUE=""
SHIFTCOUNT=0
- if [ "$1" == "$NAME" ]; then
+ if [ "$PARAMFILE" == "" ] && [ "$1" == "$NAME" ]; then
shift
let "SHIFTCOUNT=$SHIFTCOUNT+1"
while [ "${1:0:1}" != "-" ] && [ "$#" != "0" ]; do
@@ -87,10 +96,12 @@
shift
let "SHIFTCOUNT=$SHIFTCOUNT+1"
done
+ VALUE="${VALUE:1}"
+ elif [ "$PARAMFILE" != "" ] && grep -E "^$NAME " $PARAMFILE ; then
+ VALUE=$(grep -E "^$NAME " $PARAMFILE | cut -d ' ' -f 2-)
else
fail 254 "Missing $NAME argument"
fi
- VALUE="${VALUE:1}"
}
openinfo() {
@@ -115,6 +126,7 @@
# make the WFDIR absolute
WFDIR=$(pwd)
+PARAMFILE=
openinfo "wrapper.log"
ID=$1
@@ -122,10 +134,16 @@
shift
-getarg "-jobdir" "$@"
-JOBDIR=$VALUE
-shift $SHIFTCOUNT
+checkparamfile "$@"
+# JOBDIR might have been assigned through the -p option, or might
+# be a parameter here
+if [ "$JOBDIR" == "" ] ; then
+ getarg "-jobdir" "$@"
+ JOBDIR=$VALUE
+ shift $SHIFTCOUNT
+fi
+
checkEmpty "$JOBDIR" "Missing job directory prefix"
mkdir -p $WFDIR/info/$JOBDIR
closeinfo
@@ -172,15 +190,25 @@
STATUSMODE=$VALUE
shift $SHIFTCOUNT
-if [ "$1" == "-a" ]; then
+declare -a CMDARGS
+if [ "$PARAMFILE" == "" ] && [ "$1" == "-a" ] ; then
shift
+ CMDARGS=("$@")
+elif [ "$PARAMFILE" != "" ] ; then
+ CMDARGS=()
+ FIRST=1
+ while read line ; do
+ if [ "$FIRST" == "1" ] ; then
+ CMDARGS=("$line")
+ FIRST=0
+ else
+ CMDARGS=("${CMDARGS[*]}" "$line")
+ fi
+ done < <(grep -E "^-a " $PARAMFILE | cut -d " " -f 2-)
else
fail 254 "Missing arguments (-a option)"
fi
-declare -a CMDARGS
-CMDARGS=("$@")
-
if [ "$STATUSMODE" = "files" ]; then
mkdir -p $WFDIR/status/$JOBDIR
fi
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2009-05-17 12:00:03 UTC (rev 2939)
+++ trunk/libexec/vdl-int.k 2009-05-17 12:03:39 UTC (rev 2940)
@@ -113,6 +113,11 @@
dir:make(dircat(wfdir, "status"), host=rhost)
)
+ wrapperMode := configProperty("wrapper.parameter.mode",host=rhost)
+ if(wrapperMode == "files"
+ dir:make(dircat(wfdir, "parameters"), host=rhost)
+ )
+
dir:make(dircat(wfdir, "info"), host=rhost)
wfdir, sharedDir
//we send the cleanup data to vdl:main()
@@ -204,6 +209,24 @@
)
)
+
+ element(stageWrapperParams, [jobid, jobdir, wrapfile, dir, host]
+ log(LOG:INFO, "START jobid={jobid} - staging in wrapper params")
+ provider := provider(wrapfile)
+ srchost := hostname(wrapfile)
+ srcdir := dirname(wrapfile)
+ destdir := dircat(dir, "/parameters/{jobdir}/")
+ filename := basename(wrapfile)
+
+ cacheOn(list(destdir, host)
+ dir:make(destdir, host=host, provider=provider)
+ )
+
+ task:transfer(srcprovider=provider, srchost=srchost, srcfile=filename, srcdir=srcdir, desthost=host, destdir=destdir)
+ log(LOG:INFO, "END jobid={jobid}")
+
+ )
+
element(doStagein, [jobid, files, dir, host]
log(LOG:INFO, "START jobid={jobid} - Staging in files")
uParallelFor(file, files
@@ -381,27 +404,71 @@
log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}")
statusMode := configProperty("status.mode",host=rhost)
+ wrapperMode := configProperty("wrapper.parameter.mode",host=rhost)
+ ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d"
+ if(sys:not(file:exists(ddir))
+ task:dir:make(ddir)
+ )
+ wrapfile := "{ddir}/param-{jobid}"
+
+ stdout := try(stdout, "stdout.txt")
+ stderr := try(stderr, "stderr.txt")
+ kickstart := vdl:kickstart(rhost)
+ fileDirs := fileDirs(stagein, stageout)
+
+ if(wrapperMode == "files"
+ sequential(
+ sys:file:write(wrapfile,
+ "-e ",vdl:executable(tr, rhost), nl(),
+ "-out ",stdout,nl(),
+ "-err ",stderr,nl(),
+ "-i ",maybe(stdin),nl(),
+ "-d ",flatten(each(fileDirs)),nl(),
+ "-if ",flatten(infiles(stagein)),nl(),
+ "-of ",flatten(outfiles(stageout)),nl(),
+ "-k ",kickstart,nl(),
+ "-status ",statusMode,nl())
+ for(argiterator, arguments
+ sys:file:write(wrapfile,append=true,"-a ",argiterator,nl())
+ )
+ )
+ )
+
vdl:setprogress("Stage in")
tmpdir := dircat(concat(wfdir, "/jobs/", jobdir), jobid)
- stdout := try(stdout, "stdout.txt")
- stderr := try(stderr, "stderr.txt")
- kickstart := vdl:kickstart(rhost)
try(
sequential(
- fileDirs := fileDirs(stagein, stageout)
createDirSet(jobid, sharedDir, rhost, fileDirs)
doStagein(jobid, stagein, sharedDir, rhost)
+ if(wrapperMode == "files"
+ stageWrapperParams(jobid, jobdir, wrapfile, wfdir, rhost)
+ )
+
log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}")
vdl:setprogress("Submitting")
- vdl:execute("/bin/bash",
- list("shared/_swiftwrap", jobid,
+
+ if(wrapperMode == "files"
+ vdl:execute("/bin/bash",
+ list("shared/_swiftwrap", jobid, "-p", jobdir)
+ directory=wfdir
+ redirect=false
+ host=rhost
+ vdl:tcprofile(tr, rhost) //this gets various app params from the tc, such as environment, walltime, etc
+ replicationGroup=replicationGroup
+ replicationChannel=replicationChannel
+ jobid=jobid
+ )
+ )
+ if(wrapperMode == "args"
+ vdl:execute("/bin/bash",
+ list("shared/_swiftwrap", jobid,
"-jobdir", jobdir,
"-e", vdl:executable(tr, rhost),
"-out", stdout,
@@ -412,7 +479,8 @@
"-of", flatten(outfiles(stageout)),
"-k", kickstart,
"-status", statusMode
- "-a", maybe(each(arguments)))
+ "-a", maybe(each(arguments))
+ )
directory=wfdir
redirect=false
host=rhost
@@ -420,13 +488,18 @@
replicationGroup=replicationGroup
replicationChannel=replicationChannel
jobid=jobid
+ )
)
vdl:setprogress("Checking status")
if(statusMode == "files"
checkJobStatus(rhost, wfdir, jobid, tr, jobdir)
)
-
+
+ if(wrapperMode == "files"
+ file:remove(wrapfile)
+ )
+
log(LOG:DEBUG, "STAGING_OUT jobid={jobid}")
@@ -497,7 +570,7 @@
)
)
)
-
+
element(generateProvenanceGraph, [gdata]
pgraph := vdl:configProperty("pgraph")
gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph)
Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2009-05-17 12:00:03 UTC (rev 2939)
+++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2009-05-17 12:03:39 UTC (rev 2940)
@@ -89,6 +89,7 @@
put("replication.min.queue.time", "60");
put("replication.limit", "3");
put("status.mode", "files");
+ put("wrapper.parameter.mode", "args");
put("wrapper.invocation.mode", "absolute");
}
Added: trunk/tests/misc/arg-mode.sh
===================================================================
--- trunk/tests/misc/arg-mode.sh (rev 0)
+++ trunk/tests/misc/arg-mode.sh 2009-05-17 12:03:39 UTC (rev 2940)
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+INITDIR=$(pwd)
+
+cd ../language-behaviour
+
+./generate-tc.data
+
+cat tc.data ../sites/tc.data > tmp.tc.data.sites
+
+SITE=local-wrapper-args-file.xml
+
+echo testing site configuration: $SITE
+
+export SWIFT_TEST_PARAMS="-sites.file ../sites/${SITE} -tc.file tmp.tc.data.sites"
+
+./run 001-echo 066-many 141-space-in-filename 142-space-and-quotes 1421-space-and-quotes
+
+
Property changes on: trunk/tests/misc/arg-mode.sh
___________________________________________________________________
Name: svn:executable
+ *
Modified: trunk/tests/misc/run
===================================================================
--- trunk/tests/misc/run 2009-05-17 12:00:03 UTC (rev 2939)
+++ trunk/tests/misc/run 2009-05-17 12:03:39 UTC (rev 2940)
@@ -2,7 +2,7 @@
for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 restart-iterate workernode-local \
ordering-extern-notlazy restart-extern ordering-extern \
external-mapper-args extract-int-delayed asserts \
-relative-wrapper arg \
+relative-wrapper arg arg-mode \
; do
./${a}.sh
R=$?
Added: trunk/tests/sites/local-wrapper-args-file.xml
===================================================================
--- trunk/tests/sites/local-wrapper-args-file.xml (rev 0)
+++ trunk/tests/sites/local-wrapper-args-file.xml 2009-05-17 12:03:39 UTC (rev 2940)
@@ -0,0 +1,12 @@
+<config>
+
+ <pool handle="localhost">
+ <execution provider="local" />
+ <gridftp url="local://localhost" />
+ <workdirectory >/var/tmp</workdirectory>
+ <profile namespace="karajan" key="jobThrottle">0</profile>
+ <profile namespace="karajan" key="delayBase">0</profile>
+ <profile namespace="karajan" key="wrapper.parameter.mode">files</profile>
+ </pool>
+
+</config>
More information about the Swift-commit
mailing list