[Swift-commit] r3237 - trunk/libexec
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Fri Feb 12 14:32:49 CST 2010
Author: wozniak
Date: 2010-02-12 14:32:49 -0600 (Fri, 12 Feb 2010)
New Revision: 3237
Modified:
trunk/libexec/_swiftwrap
trunk/libexec/cdm.pl
trunk/libexec/vdl-int.k
trunk/libexec/vdl-lib.xml
Log:
Basic CDM GATHER policy.
Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap 2010-02-10 18:34:11 UTC (rev 3236)
+++ trunk/libexec/_swiftwrap 2010-02-12 20:32:49 UTC (rev 3237)
@@ -1,4 +1,5 @@
# this script must be invoked inside of bash, not plain sh
+# note that this script modifies $IFS
infosection() {
echo >& "$INFO"
@@ -132,7 +133,7 @@
RESULT="DEFAULT"
if [ -f shared/cdm.pl ]; then
- RESULT=$( perl shared/cdm.pl $FILE < $CDM_FILE )
+ RESULT=$( perl shared/cdm.pl lookup $FILE < $CDM_FILE )
fi
echo $RESULT
}
@@ -177,9 +178,23 @@
fail 254 "Cannot BROADCAST an output file!"
fi
;;
+ GATHER)
+ if [ $MODE == "INPUT" ]; then
+ fail 254 "Cannot GATHER an input file!"
+ fi
esac
}
+cdm_gather()
+{
+ GATHER_OUTPUT=${*}
+ if [[ $CDM_FILE == "" ]]; then
+ return
+ fi
+
+ cdm_gather_action $GATHER_MAX $GATHER_OUTPUT
+}
+
COMMANDLINE=$@
# get the parent directory of the directory containing _swiftwrap, to use
@@ -263,7 +278,10 @@
shift $SHIFTCOUNT
getarg "-cdmfile" "$@"
-CDM_FILE=$VALUE
+CDM_FILE=
+if [ "X$VALUE" != "X" ]; then
+ CDM_FILE=shared/$VALUE
+fi
shift $SHIFTCOUNT
getarg "-status" "$@"
@@ -293,6 +311,12 @@
mkdir -p $WFDIR/status/$JOBDIR
fi
+if [ "X$CDM_FILE" != "X" ]; then
+ logstate "SOURCE_CDM_LIB $WFDIR/shared/cdm_lib.sh"
+ source $WFDIR/shared/cdm_lib.sh
+ checkError 254 "Could not source: $WFDIR/shared/cdm_lib.sh"
+fi
+
if [ "X$SCRATCH" != "X" ]; then
log "Job directory mode is: local copy"
DIR=$SCRATCH/$JOBDIR/$ID
@@ -348,14 +372,11 @@
log "Created output directory: $DIR/$D"
done
-log "PWD: $( pwd )"
-log "FIND: $( find . )"
-
logstate "LINK_INPUTS"
for L in $INF ; do
- CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE )
- log "CDM_POLICY: $L -> $CDM_POLICY"
+ CDM_POLICY=$( cdm_lookup $L $CDM_FILE )
if [ $CDM_POLICY != "DEFAULT" ]; then
+ log "CDM_POLICY: $L -> $CDM_POLICY"
eval cdm_action $DIR "INPUT" $L $CDM_POLICY
continue
fi
@@ -374,13 +395,17 @@
logstate "LINK_CDM_OUTPUTS"
SKIPPED_OUTPUT=()
+GATHER_OUTPUT=()
for L in $OUTF ; do
- CDM_POLICY=$( cdm_lookup $L shared/$CDM_FILE )
- log "CDM_POLICY: $L -> $CDM_POLICY"
+ CDM_POLICY=$( cdm_lookup $L $CDM_FILE )
if [ $CDM_POLICY != "DEFAULT" ]; then
+ log "CDM_POLICY: $L -> $CDM_POLICY"
eval cdm_action $DIR "OUTPUT" $L $CDM_POLICY
SKIPPED_OUTPUT=( $SKIPPED_OUTPUT $L )
fi
+ if [ $CDM_POLICY == "GATHER" ]; then
+ GATHER_OUTPUT=( $GATHER_OUTPUT $L )
+ fi
done
logstate "EXECUTE"
@@ -456,23 +481,21 @@
logstate "MOVING_OUTPUTS $OUTF"
for O in $OUTF ; do
- log "output: $O"
if ! contains SKIPPED_OUTPUT $O ; then
mv -v "$DIR/$O" "$WFDIR/shared/$O" 2>&1 >& "$INFO"
checkError 254 "Failed to move output file $O to shared directory"
- else
- log "CDM: SKIPPED_OUTPUT: $O"
fi
done
+cdm_gather $GATHER_OUTPUT
+
logstate "RM_JOBDIR"
-# rm -rf "$DIR" 2>&1 >& "$INFO"
+rm -rf "$DIR" 2>&1 >& "$INFO"
checkError 254 "Failed to remove job directory $DIR"
if [ "$STATUSMODE" = "files" ]; then
logstate "TOUCH_SUCCESS"
touch $WFDIR/status/${JOBDIR}/${ID}-success
- log "TOUCH CODE: $WFDIR/status/${JOBDIR}/${ID}-success $?"
fi
logstate "END"
Modified: trunk/libexec/cdm.pl
===================================================================
--- trunk/libexec/cdm.pl 2010-02-10 18:34:11 UTC (rev 3236)
+++ trunk/libexec/cdm.pl 2010-02-12 20:32:49 UTC (rev 3237)
@@ -29,22 +29,74 @@
print "$result\n";
}
+sub cdm_property
+{
+ my $hash = $_[0];
+ my $name = $_[1];
+
+ $result = "UNSET";
+ foreach (keys %$hash)
+ {
+ $property = $_;
+ if ($name eq $property)
+ {
+ $result = "$$hash{$property}\n";
+ last;
+ }
+ }
+ print "$result\n";
+}
+
# Command-line arguments:
-$file = $ARGV[0];
+$task = $ARGV[0];
+$arg = $ARGV[1];
# Read fs.data off of stdin:
@keys = ();
%map = ();
+%properties = ();
while (<STDIN>)
{
chomp;
+
+ my $comment = index $_, "#";
+ if ($comment >= 0) {
+ $_ = substr($_, 0, $comment);
+ }
+
next if $_ eq "";
@tokens = split(/ /, $_);
- $key = shift(@tokens);
- $rest = join(' ', @tokens);
- @keys = (@keys, $key);
- $map{$key} = $rest;
+ $type = shift(@tokens);
+ if ($type eq "rule")
+ {
+ $key = shift(@tokens);
+ $rest = join(' ', @tokens);
+ @keys = (@keys, $key);
+ $map{$key} = $rest;
+ }
+ elsif ($type eq "property")
+ {
+ $property = shift(@tokens);
+ $rest = join(' ', @tokens);
+ $properties{$property} = $rest;
+ }
+ else
+ {
+ die "Unknown directive: $type";
+ }
}
-cdm_lookup(\@keys, \%map, $file);
+# Do the user task:
+if ($task eq "lookup")
+{
+ cdm_lookup(\@keys, \%map, $arg);
+}
+elsif ($task eq "property")
+{
+ cdm_property(\%properties, $arg);
+}
+else
+{
+ die "Unknown task: $task";
+}
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2010-02-10 18:34:11 UTC (rev 3236)
+++ trunk/libexec/vdl-int.k 2010-02-12 20:32:49 UTC (rev 3237)
@@ -191,6 +191,25 @@
element(cleanup, [dir, host]
log(LOG:INFO, "START dir={dir} host={host}")
+ cdmfile := cdm:file()
+ log(LOG:INFO, "cdmfile {cdmfile}")
+ if(cdmfile != "" then(
+ log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}")
+ task:transfer(srcfile="cdm_cleanup.sh",
+ srcdir="{swift.home}/libexec",
+ desthost=host, destdir=dir)
+ task:transfer(srcfile="cdm_lib.sh",
+ srcdir="{swift.home}/libexec",
+ desthost=host, destdir=dir)
+ log(LOG:INFO, "execute: cdm_cleanup.sh")
+ task:execute(
+ executable="/bin/bash",
+ arguments=list("{dir}/cdm_cleanup.sh",
+ cdm:get("GATHER_DIR"), cdm:get("GATHER_TARGET")
+ sys:uid() )
+ host=host, batch=true, tcprofile(host))
+ )
+ )
if(vdl:configProperty("sitedir.keep") == "false"
task:execute(
vdl:siteprofile(host, "swift:cleanupCommand"),
@@ -259,6 +278,9 @@
doStageinFile(provider="file", srchost="localhost", srcfile="cdm.pl",
srcdir=libexec, desthost=host, destdir=dir,
size=file:size("{libexec}/cdm.pl}"), policy="DEFAULT")
+ doStageinFile(provider="file", srchost="localhost", srcfile="cdm_lib.sh",
+ srcdir=libexec, desthost=host, destdir=dir,
+ size=file:size("{libexec}/cdm_lib.sh}"), policy="DEFAULT")
))
uParallelFor(file, files
Modified: trunk/libexec/vdl-lib.xml
===================================================================
--- trunk/libexec/vdl-lib.xml 2010-02-10 18:34:11 UTC (rev 3236)
+++ trunk/libexec/vdl-lib.xml 2010-02-12 20:32:49 UTC (rev 3237)
@@ -100,7 +100,7 @@
<namespace prefix="cdm">
<export name="query"><elementDef classname="org.globus.swift.data.Query"/></export>
- <export name="property"><elementDef classname="org.globus.swift.data.Query"/></export>
+ <export name="get"><elementDef classname="org.globus.swift.data.Query"/></export>
<export name="file"><elementDef classname="org.globus.swift.data.Query"/></export>
<export name="broadcast"><elementDef classname="org.globus.swift.data.Action"/></export>
</namespace>
More information about the Swift-commit
mailing list