[Swift-commit] r6639 - in trunk: . bin
davidk at ci.uchicago.edu
davidk at ci.uchicago.edu
Fri Jul 12 10:34:52 CDT 2013
Author: davidk
Date: 2013-07-12 10:34:52 -0500 (Fri, 12 Jul 2013)
New Revision: 6639
Added:
trunk/bin/nswift
Removed:
trunk/bin/chxml
Modified:
trunk/bin/swiftrun
trunk/build.xml
Log:
Swiftrun and nswift scripts
Deleted: trunk/bin/chxml
===================================================================
--- trunk/bin/chxml 2013-07-11 02:30:21 UTC (rev 6638)
+++ trunk/bin/chxml 2013-07-12 15:34:52 UTC (rev 6639)
@@ -1,99 +0,0 @@
-#!/usr/bin/env python
-
-# This does not run under python3.2
-
-# ensures that that there is a sites file specified and that it contains
-# well-formed xml as well as essential attributes for the specified provider
-
-import xml.parsers.expat,sys,os,commands,pprint
-from glob import glob
-
-class siteProps:
-
- def __init__(self,argstr):
- self.sites_file = ""
- self.startTags= []
- self.endTags=[]
- self.globusParams = 0
- self.execProvider = ""
-
- if len(argstr)<2:
- print "please specify a sites file"
- sys.exit(1)
- i = 0
- for o in argstr:
- if o == "'-sites.file'":
- self.sites_file = argstr[i+1].split("'")[1]
- elif o == "'-config'":
- try:
- config_filename = argstr[i+1].split("'")[1]
- config_file = open(config_filename,"r")
- except IOError, (errno, strerror):
- print strerror
- sys.exit("Could not read: " + config_filename)
-
- for line in config_file.readlines():
- if line.strip().startswith('#'):
- continue
- if line.find("sites.file")>-1:
- self.sites_file = line.split("=")[1].strip()
- i = i+1
- if (self.sites_file == ""):
- swift_loc = commands.getoutput("which swift")
- swift_home = swift_loc.split("bin")[0]
- self.sites_file = swift_home+"etc/sites.xml"
- print "no sites file specified, setting to default: "+self.sites_file
-
-
- def parsefile(self):
- parser = xml.parsers.expat.ParserCreate()
- parser.ParseFile(open(self.sites_file, "r"))
-
- def start_element(self, name, attrs):
- self.startTags.append(name)
- for x, y in attrs.iteritems():
- if (name=="execution" and x=="provider"):
- self.execProvider = y
- if (y=="globus"):
- self.globusParams = 1
- if (mysite.globusParams == 1 and mysite.execProvider == "local"):
- pp = pprint.PrettyPrinter()
- print "Warning: "+mysite.sites_file+" contains globus parameters that do not apply to the local execution provider ("+name+" "+pp.pformat(attrs)+")"
- sys.exit()
- def end_element(self, name):
- self.endTags.append(name)
-
-
-#------------------------main----------------------------------
-
-mysite = siteProps(sys.argv)
-
-try:
- mysite.parsefile()
-except Exception, e:
- print "Error: XML parsing error in "+mysite.sites_file
- print "%s is %s" % (mysite.sites_file, e)
- sys.exit()
-
-p = xml.parsers.expat.ParserCreate()
-p.StartElementHandler = mysite.start_element
-p.EndElementHandler = mysite.end_element
-p.ParseFile(open(mysite.sites_file,"r"))
-
-config = 0
-pool = 0
-
-for tag in mysite.startTags:
- if tag=="config":
- config = 1
- else:
- if tag=="pool":
- pool= 1
-
-if (pool<1):
- print "Warning: Your sites file, "+mysite.sites_file+" is missing a pool entry"
- sys.exit()
-if (config<1):
- print "Warning: Your site file, "+mysite.sites_file+" is missing a config entry"
- sys.exit()
-
Added: trunk/bin/nswift
===================================================================
--- trunk/bin/nswift (rev 0)
+++ trunk/bin/nswift 2013-07-12 15:34:52 UTC (rev 6639)
@@ -0,0 +1,175 @@
+#!/bin/bash
+
+# Referenced environment variables (default values):
+# SWIFT_ASSERTS
+# SWIFT_HEAP_MAX (1024M)
+# SWIFT_USAGE_STATS
+
+EXEC=org.griphyn.vdl.karajan.Loader
+OPTIONS="-XX:+HeapDumpOnOutOfMemoryError"
+
+CYGWIN=
+CPDELIM=":"
+HEAPMAX=1024M
+
+if echo `uname` | grep -i "cygwin"; then
+ CYGWIN="yes"
+ CPDELIM=";"
+fi
+
+updateOptions() {
+
+ if [ "X$1" != "X" ] ; then
+ OPTIONS="$OPTIONS -D$2=$1"
+ fi
+
+}
+
+####### MAIN BODY ##########
+
+if [ ! -d "$SWIFT_HOME" ] ; then
+ SWIFT_HOME=`dirname $0`"/.."
+ if [ ! -f "$SWIFT_HOME/lib/cog-swift-svn.jar" ] && [ ! -f "$SWIFT_HOME/lib/cog.jar" ] ; then
+ MY_PATH=`which $0`
+ SWIFT_HOME=`dirname $MY_PATH`"/.."
+ if [ ! -f "$SWIFT_HOME/lib/cog-swift-svn.jar" ] && [ ! -f "$SWIFT_HOME/lib/cog.jar" ] ; then
+ echo "Error: SWIFT_HOME is not set, and all attempts at guessing it failed"
+ exit 1
+ fi
+ fi
+fi
+
+OPTIONS="$OPTIONS -Djava.endorsed.dirs=$SWIFT_HOME/lib/endorsed"
+
+LOCALCLASSPATH=$SWIFT_HOME/etc$CPDELIM$SWIFT_HOME/libexec
+for JAR in $SWIFT_HOME/lib/*.jar; do
+ LOCALCLASSPATH="$LOCALCLASSPATH$CPDELIM$JAR"
+done
+LOCALCLASSPATH=$LOCALCLASSPATH$CPDELIM$CLASSPATH
+
+CMDLINE="$@"
+
+### SETUP OTHER ENV VARIABLES ####
+
+USERID=`id | awk '{printf("%s\n", substr($1, 5, index($1, "(")-5 ) ) }'`
+
+if [ "X$GLOBUS_HOSTNAME" = "X" ] && [ "X$HOSTNAME" != "X" ]; then
+ GLOBUS_HOSTNAME=$HOSTNAME
+fi
+
+updateOptions "$USERID" "UID"
+updateOptions "$GLOBUS_INSTALL_PATH" "GLOBUS_INSTALL_PATH"
+updateOptions "$GLOBUS_DEPLOY_PATH" "GLOBUS_DEPLOY_PATH"
+updateOptions "$GLOBUS_TCP_PORT_RANGE" "GLOBUS_TCP_PORT_RANGE"
+updateOptions "$GLOBUS_TCP_SOURCE_PORT_RANGE" "GLOBUS_TCP_SOURCE_PORT_RANGE"
+updateOptions "$GLOBUS_UDP_SOURCE_PORT_RANGE" "GLOBUS_UDP_SOURCE_PORT_RANGE"
+updateOptions "$GLOBUS_HOSTNAME" "GLOBUS_HOSTNAME"
+updateOptions "$X509_USER_CERT" "X509_USER_CERT"
+updateOptions "$X509_USER_KEY" "X509_USER_KEY"
+updateOptions "$X509_CERT_DIR" "X509_CERT_DIR"
+updateOptions "$X509_USER_PROXY" "X509_USER_PROXY"
+updateOptions "$SWIFT_HOME" "COG_INSTALL_PATH"
+updateOptions "$SWIFT_HOME" "swift.home"
+updateOptions "$SWIFT_USERHOME" "user.home"
+
+#Use /dev/urandom instead of /dev/random for seeding RNGs
+#This will lower the randomness of the seed, but avoid
+#large delays if /dev/random does not have enough entropy collected
+updateOptions "file:///dev/urandom" "java.security.egd"
+
+if [ "X$SWIFT_HEAP_MAX" != "X" ]; then
+ HEAPMAX=$SWIFT_HEAP_MAX
+fi
+if [ "X$HEAPMAX" != "X" ]; then
+ OPTIONS="-Xmx$HEAPMAX $OPTIONS"
+fi
+
+if [ "X$SWIFT_ASSERTS" != "X" ]; then
+ OPTIONS="-ea $OPTIONS"
+fi
+
+# Anonymous usage statistics
+LISTENER="128.135.125.18"
+PORT="9999"
+
+if [ -z "$SWIFT_USAGE_STATS" ]; then
+ SWIFT_USAGE_STATS=1
+fi
+
+# To disable usage statistics, uncomment the line below
+#SWIFT_USAGE_STATS=0
+
+PATH=$PATH:/sbin
+IFCONFIG=`which ifconfig 2>&1`
+if [ -x "$IFCONFIG" ]; then
+ MAC=`$IFCONFIG 2>/dev/null |grep HWaddr`
+fi
+
+# Check if we are running on a mac, if so look for md5 and not md5sum
+if [ "`uname`" = "Darwin" ]; then
+ MD5SUM=`which md5 2>&1`
+ if [ ! -x "$MD5SUM" ]; then
+ echo "warning: unable to find md5"
+ SWIFT_USAGE_STATS=0
+ fi
+else
+ MD5SUM=`which md5sum 2>&1`
+ if [ ! -x "$MD5SUM" ]; then
+ echo "warning: unable to find md5sum"
+ SWIFT_USAGE_STATS=0
+ fi
+fi
+
+BASH="/bin/bash"
+if [ ! -x "$BASH" ]; then
+ BASH=`which bash 2>&1`
+ if [ ! -x "$BASH" ]; then
+ echo "warning: unable to find bash"
+ SWIFT_USAGE_STATS=0
+ fi
+fi
+
+if [ "$SWIFT_USAGE_STATS" = 1 ]; then
+ for var in "$@"
+ do
+ SCRIPT=`echo $var|grep -i .swift`
+ if [ -e "$SCRIPT" ]; then
+ SCRIPT=$var
+ SCRIPTID=`$MD5SUM $SCRIPT|cut -d ' ' -f1`
+ SCRIPTLENGTH=`wc -l $SCRIPT|cut -d ' ' -f1`
+ fi
+ done
+ HOSTNAME=`hostname`
+ USERID=`echo $USERID $HOSTNAME $MAC | $MD5SUM |cut -d ' ' -f1`
+ DATE=`date`
+ KEY=`echo $USERID $DATE $$ | $MD5SUM | cut -d ' ' -f1`
+ $BASH -c "echo $KEY $USERID $SCRIPTID $SCRIPTLENGTH -1 0 > /dev/udp/$LISTENER/$PORT" > /dev/null 2>&1 &
+fi
+
+# Swiftrun
+CMDLINE="$( swiftrun $CMDLINE )"
+
+### EXECUTE ############
+# Note: killing Swift may be briefly delayed due to Coasters shutdown
+if test -n "$CYGWIN"; then
+ set CLASSPATHSAVE=$CLASSPATH
+ export CLASSPATH="$LOCALCLASSPATH"
+ eval java ${OPTIONS} ${COG_OPTS} ${EXEC} ${CMDLINE}
+ export CLASSPATH=$CLASSPATHSAVE
+else
+ eval java ${OPTIONS} ${COG_OPTS} -classpath ${LOCALCLASSPATH} ${EXEC} ${CMDLINE}
+fi
+return_code=$?
+
+if [ "$SWIFT_USAGE_STATS" = 1 ]; then
+ $BASH -c "echo $KEY $USERID $SCRIPTID $SCRIPTLENGTH $return_code 1 > /dev/udp/$LISTENER/$PORT" > /dev/null 2>&1 &
+fi
+
+mv *.d/ $( ls -1rtd run*/ |tail -1 )
+rm *.swiftx *.kml
+exit $return_code
+
+# Local Variables:
+# tab-width: 4
+# indent-tabs-mode: t
+# End:
Property changes on: trunk/bin/nswift
___________________________________________________________________
Added: svn:executable
+ *
Modified: trunk/bin/swiftrun
===================================================================
--- trunk/bin/swiftrun 2013-07-11 02:30:21 UTC (rev 6638)
+++ trunk/bin/swiftrun 2013-07-12 15:34:52 UTC (rev 6639)
@@ -1,116 +1,214 @@
-#!/bin/bash
+#!/usr/bin/perl -w
-# Verify an argument is not null
-verify_not_null()
-{
- argname=$1; shift
- if [ _$1 != _ ]; then
- return 0;
- else
- echo $0: value for $argname can not be null
- exit 1
- fi
+use strict;
+use warnings;
+use File::Basename;
+use Getopt::Long qw(GetOptionsFromArray);
+use Switch;
+use Cwd;
+
+Getopt::Long::Configure("pass_through");
+
+my %properties = (); # Hash storing all swift properties
+my @property_files = (); # List of swift property files to be read
+my $option_tcfile; # Value of -tc.file option
+my $option_sitesfile; # Value of -sites.file option
+my $option_siteslist; # Value of -sites option, comma separated
+my $option_config; # Value of -config option
+my @sites; # List of sites from -sites in array
+my $run_directory; # Run directory to be created, run.NNNN
+my @new_swift_args; # New command line arguments
+my @original_swift_args; # Original arguments
+
+# List of xml shortcuts. Keep in lower case.
+my %xml_aliases = (
+ 'provider' => 'provider.coaster',
+ 'jobspernode' => 'globus.jobspernode',
+ 'maxwalltime' => 'globus.maxwalltime',
+ 'queue' => 'globus.queue',
+ 'initialscore' => 'karajan.initialscore',
+);
+
+# Verify a property file exists and add it to the search path
+sub add_property_file {
+ my $property_file = $_[0];
+ if( -e "$property_file" ) {
+ push( @property_files, $property_file );
+ }
}
-# crash: Report a problem and exit
-crash()
-{
- MSG=$1
- echo ${MSG} >&2
- exit 1
+# Create a string with a single pool entry
+sub create_pool_entry {
+ my $site = $_[0];
+ my $pool = "\n<pool handle=\"" . $site . "\">\n";
+ while( my( $key, $value ) = each( %properties ) ) {
+ if( $key =~ /^\s*site.$site/ ) {
+ $pool .= &property_to_xml( $key, $value );
+ }
+ }
+ $pool .= "</pool>\n";
+ return $pool;
}
-# Create .swift structure
-create_dotswift()
-{
- if [ ! -d "$HOME/.swift" ]; then
- mkdir $HOME/.swift || crash "Unable to mkdir $HOME/.swift"
- fi
+# Given a string in format site.beagle.globus.jobsPerNode=12, convert to xml
+sub property_to_xml {
+ my $key = $_[0];
+ my $value = $_[1];
+ my ( $site, $sitename, $namespace, $property ) = split(/\./, $key);
- if [ ! -d "$HOME/.swift/runs" ]; then
- mkdir $HOME/.swift/runs || crash "Unable to mkdir $HOME/.swift/runs"
- fi
+ # In the case of a shortened definition, check for aliases
+ if ( !defined( $property ) ) {
+ if ( defined( $xml_aliases{ lc( $namespace )})) {
+ ( $namespace, $property ) = split(/\./, $xml_aliases{ lc($namespace) });
+ }
+ }
+
+ # Make sure to handle special cases that are not in the namespace=x, key=y, format
+ switch($namespace) {
+ case "provider" { return "<execution provider=\"" . $property . "\" jobmanager=\"" . $value . "\"/>\n"; }
+ case "workdirectory" { return "<workdirectory>$value</workdirectory>\n"; }
+ case "filesystem" { return "<filesystem provider=\"" . $value . "\"/>\n"; }
+ else { return "<profile namespace=\"" . $namespace . "\" key=\"" . $property . "\">" . $value . "</profile>\n"; }
+ }
+}
- if [ ! -d "$HOME/.swift/runs/current" ]; then
- mkdir $HOME/.swift/runs/current || crash "Unable to mkdir $HOME/.swift/runs/current"
- fi
+# Write apps to file in tc.data format
+sub write_apps {
+ my $app_filename = $_[0];
+ open( APP_FILE, ">$run_directory/$app_filename" ) || die "Unable to open $run_directory/$app_filename\n";
- if [ ! -d "$HOME/.swift/runs/completed" ]; then
- mkdir $HOME/.swift/runs/completed || crash "Unable to mkdir $HOME/.swift/runs/completed"
- fi
+ # Find app definitions
+ while( my( $key, $value ) = each( %properties ) ) {
+ if( $key =~ /^\s*app./ ) {
+ my ( $app, $site, $command ) = split(/\./, $key);
+
+ # Handle wildcards for site names
+ if ( $site eq "*" ) {
+ foreach my $nsite( @sites ) {
+ print APP_FILE "$nsite $command $value\n";
+ }
+ }
+
+ # Handle apps for defined sites
+ elsif ( grep( /^$site$/, @sites ) ) {
+ print APP_FILE "$site $command $value\n";
+ }
+ }
+ }
+ # Update command line args
+ push( @new_swift_args, "-tc.file $run_directory/$app_filename" );
}
-# Parse command line arguments
-args=""
-script=""
-while [ $# -gt 0 ]
-do
- case "$1" in
- -site|-sites|-s) SITES=$2; verify_not_null sites $SITES; shift ;;
- -config|-c) CONFIG=$2; verify_not_null config $CONFIG; shift ;;
- *) if [ -z "$script" ] && [[ $1 == *.swift ]]; then script="$1"
- else args+="$1 "
- fi ;;
- esac
- shift
-done
+# Write properties to file
+sub write_properties {
+ my $property_filename = $_[0];
+ open( PROPERTY_FILE, ">$run_directory/$property_filename" ) || die "Unable to open $run_directory/$property_filename\n";
+ while( my( $key, $value ) = each( %properties ) ) {
+ # Ignore properties that start with app. or site.
+ next if $key =~ /^\s*app./;
+ next if $key =~ /^\s*site./;
+ print PROPERTY_FILE "$key=$value\n";
+ }
+ push( @new_swift_args, "-config $run_directory/$property_filename" );
+}
-# Create run directory
-create_dotswift
-rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
-mkdir $rundir
+# Write sites.xml to disk
+sub write_sites {
+ my $filename = $_[0];
+ open(SITES, ">$run_directory/$filename") || die "Unable to open $run_directory/$filename\n";
+ print SITES "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
+ print SITES "<config xmlns=\"http://www.ci.uchicago.edu/swift/SwiftSites\">\n";
-# Link input_files
-if [ -d "input_files" ]; then
- for input_file in input_files/*
- do
- ln -s $PWD/$input_file $rundir
- done
-fi
+ foreach my $site(@sites) {
+ print SITES &create_pool_entry($site);
+ }
-# Swift script
-if [ -f "$script" ]; then
- cp $script $rundir
-fi
+ print SITES "\n</config>\n";
+ close(SITES);
+ push(@new_swift_args, "-sites.file $run_directory/$filename");
+}
-# Copy config file if specified
-if [ -f "$CONFIG" ]; then
- cp $CONFIG $rundir
- CONFIG=$( basename $CONFIG )
-fi
+# Getopt::Long can't accept dots in arguments, work around to make things easy
+foreach my $arg(@ARGV) {
+ $arg =~ s/sites.file/sitesfile/g;
+ $arg =~ s/tc.file/tcfile/g;
+}
-# Create sites.xml
-IFS=","
-echo "<config>" > $rundir/sites.xml
-for site in $SITES
-do
- # If config is not defined, but there is a config in conf/ that matches, use that
- if [ -z "$CONFIG" ] && [ -f "conf/$site.cf" ]; then
- cp conf/$site.cf $rundir
- config=$site.cf
- fi
+# Command line arguments we care about
+ at original_swift_args = @ARGV;
+GetOptionsFromArray(
+ \@original_swift_args,
+ 'sitesfile=s' => \$option_sitesfile,
+ 'tcfile=s' => \$option_tcfile,
+ 'config=s' => \$option_config,
+ 'site|sites=s' => \$option_siteslist,
+);
- if [ -f "conf/$config" ]; then
- gensites -p conf/$config $site >> $rundir/sites.xml
- else
- gensites $site >> $rundir/sites.xml
- fi
+# Set @sites array from comma separated list of sites
+if(defined($option_siteslist)) {
+ @sites = split(',', $option_siteslist);
+}
-done
-echo "</config>" >> $rundir/sites.xml
-mv tc.data $rundir
-cd $rundir
+# Verify Swift is found in PATH
+my $swift_etc_directory = dirname(dirname(`which swift`)) . "/etc";
+if( ! -d $swift_etc_directory ) { die "Unable to find a valid Swift installation"; }
-if [ -z "$CONFIG" ]; then
- CONFIG=$config
-fi
+# Set the search order for properties
+&add_property_file("$swift_etc_directory/swift.properties");
+&add_property_file("$ENV{SWIFT_CONFIG}/swift.properties") if defined($ENV{SWIFT_CONFIG});
+&add_property_file("$ENV{HOME}/.swift/swift.properties");
+&add_property_file("swift.properties");
-# Run
-timestamp=$( date +%s )
-ln -s $PWD $HOME/.swift/runs/current/$rundir.$timestamp
-if [ -n "$CONFIG" ]; then
- eval time swift -sites.file sites.xml -tc.file tc.data -config $CONFIG $( basename $script ) $args 2>&1 | tee swift.out
-else
- eval time swift -sites.file sites.xml -tc.file tc.data $( basename $script ) $args 2>&1 | tee swift.out
-fi
-mv $HOME/.swift/runs/current/$rundir.$timestamp $HOME/.swift/runs/completed
+# Set property values
+foreach my $property_file(@property_files) {
+ open(PROPERTIES, $property_file) || die "Unable to open $property_file";
+ while( <PROPERTIES> ) {
+ chomp;
+ next if /^\s*#/ || /^(\s)*$/; # Ignore blank lines and comments
+ my ($key, $value) = split('=', $_, 2);
+ $value =~ s/\$(\w+)/$ENV{$1}/g; # Substitute environment variables
+ $properties{ $key } = $value;
+ }
+}
+
+# Create/increment run directory
+my $run_number = 0;
+my @run_directories = <run????>;
+
+if(@run_directories) {
+ $run_number = $run_directories[-1];
+ $run_number =~ s/run//g;
+ $run_number++;
+}
+
+$run_directory = getcwd() . "/" . sprintf "run%04s", $run_number;
+mkdir($run_directory) || die "Unable to create run directory $run_directory\n";
+
+# Write swift configuration files
+&write_sites("sites.xml") if defined( $option_siteslist );
+&write_apps("tc.data") if ( !defined( $option_tcfile ) && defined( $option_siteslist ));
+&write_properties("cf") if !defined( $option_config );
+
+# Repair args with dots
+foreach my $arg(@ARGV) {
+ $arg =~ s/sitesfile/sites.file/g;
+ $arg =~ s/tcfile/tc.file/g;
+}
+
+# Make sure -sites isn't in the arguments we send back to swift
+while( @ARGV ) {
+ my $option = shift;
+ switch( $option ) {
+ case "-site" { shift; }
+ case "-sites" { shift; }
+ else { push( @new_swift_args, $option ); }
+ }
+}
+
+push( @new_swift_args, sprintf( "-logfile $run_directory/run%04s.log", $run_number ));
+
+foreach my $arg( @new_swift_args ) {
+ print "$arg ";
+}
+
+print "\n";
Modified: trunk/build.xml
===================================================================
--- trunk/build.xml 2013-07-11 02:30:21 UTC (rev 6638)
+++ trunk/build.xml 2013-07-12 15:34:52 UTC (rev 6639)
@@ -88,6 +88,7 @@
<include name="swift-plot-log"/>
<include name="swiftconfig"/>
<include name="swiftrun"/>
+ <include name="nswift"/>
<include name="gensites"/>
<include name="chxml"/>
<include name="start-coaster-service"/>
More information about the Swift-commit
mailing list