[Swift-commit] r7337 - trunk/bin
davidk at ci.uchicago.edu
davidk at ci.uchicago.edu
Tue Nov 26 16:08:37 CST 2013
Author: davidk
Date: 2013-11-26 16:08:36 -0600 (Tue, 26 Nov 2013)
New Revision: 7337
Added:
trunk/bin/prop2scs.pl
Removed:
trunk/bin/swift-workers
Modified:
trunk/bin/swift-service
Log:
Add utility to export shell variables from swift.property service properties
Merge swift-workers
Added: trunk/bin/prop2scs.pl
===================================================================
--- trunk/bin/prop2scs.pl (rev 0)
+++ trunk/bin/prop2scs.pl 2013-11-26 22:08:36 UTC (rev 7337)
@@ -0,0 +1,82 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;
+use File::Basename;
+
+my %properties = (); # Hash storing all swift properties
+my @property_files = (); # List of swift property files to be read
+my $service = $ARGV[0];
+if (!defined($service) || $service eq "-help") {
+ print STDERR "Usage: $0 service\n";
+ exit 1;
+}
+
+# How to convert SCS options to environment variables
+my %conversionTable = (
+ 'ip' => 'IPADDR',
+ 'localport' => 'LOCAL_PORT',
+ 'mode' => 'WORKER_MODE',
+ 'serviceport' => 'SERVICE_PORT',
+ 'workers' => 'WORKER_HOSTS',
+);
+
+# Verify a property file exists and add it to the search path
+sub add_property_file {
+ my $property_file = $_[0];
+ if( -e "$property_file" ) {
+ push( @property_files, $property_file );
+ }
+}
+
+# Verify Swift is found in PATH
+my $swift_etc_directory = dirname(dirname(`which swift`)) . "/etc";
+if( ! -d $swift_etc_directory ) { die "Unable to find a valid Swift installation"; }
+
+# Set the search order for properties
+&add_property_file("$swift_etc_directory/swift.properties");
+&add_property_file("$ENV{SWIFT_SITE_CONF}/swift.properties") if defined($ENV{SWIFT_SITE_CONF});
+&add_property_file("$ENV{HOME}/.swift/swift.properties");
+&add_property_file("swift.properties");
+
+# Set property values
+foreach my $property_file(@property_files) {
+ open(PROPERTIES, $property_file) || die "Unable to open $property_file";
+
+ while( <PROPERTIES> ) {
+ chomp;
+ next if /^\s*#/ || /^(\s)*$/; # Ignore blank lines and comments
+ $_ =~ s/^\s+//; # Remove leading whitespace
+
+ # Handle brackets
+ if( /^site\.|^service\./ && /{/ ) {
+ my $prefix = (split /\s+{/)[0];
+ while( <PROPERTIES> ) {
+ chomp;
+ next if /^\s*#/ || /^(\s)*$/;
+ $_ =~ s/^\s+//;
+ if( /^}/ ) { last; }
+ my ($key, $value) = split('=', ($prefix . ".$_"), 2);
+ if($key eq "sites") { $key = "site"; }
+ $value =~ s/\$(\w+)/$ENV{$1}/g;
+ $properties{ $key } = $value;
+ }
+ }
+
+ else {
+ my ($key, $value) = split('=', $_, 2);
+ if($key eq "sites") { $key = "site"; }
+ $value =~ s/\$(\w+)/$ENV{$1}/g; # Substitute environment variables
+ $properties{ $key } = $value;
+ }
+ }
+}
+
+foreach my $key (sort keys %properties) {
+ if($key =~ m/service\.$service/ ) {
+ my $val = (split /\./, $key)[-1];
+ if ( defined( $conversionTable{ lc( $val )})) {
+ print "export $conversionTable{lc($val)}=$properties{$key}\n";
+ }
+ }
+}
Property changes on: trunk/bin/prop2scs.pl
___________________________________________________________________
Added: svn:executable
+ *
Modified: trunk/bin/swift-service
===================================================================
--- trunk/bin/swift-service 2013-11-26 17:29:13 UTC (rev 7336)
+++ trunk/bin/swift-service 2013-11-26 22:08:36 UTC (rev 7337)
@@ -3,14 +3,11 @@
# Allows a user to start, stop, name, and view the status of coaster services
#
-# :set debug
-
export SWIFT_BIN="$( cd "$( dirname "$0" )" && pwd )"
-export WORKER="$SWIFT_BIN/worker.pl"
+export SERVICE_DIR="$HOME/.swift/service"
export COASTER_SERVICE="$SWIFT_BIN/coaster-service"
export LOG="swift-service.log"
-export SERVICE_DIR="$HOME/.swift/service"
-export IPADDR=127.0.0.1
+export WORKER="$SWIFT_BIN/worker.pl"
mkdir -p "$SERVICE_DIR" || crash "Unable to create $SERVICE_DIR"
@@ -22,12 +19,24 @@
exit 1
}
+# Verify an argument is not null
+verify_not_null()
+{
+ argname=$1; shift
+ if [ _$1 != _ ]; then
+ return 0;
+ else
+ usage
+ fi
+}
+
# Print usage
usage()
{
- crash "$( basename $0 ) [ -start | -stop | -status | -xml | -debug ] [ -name servicename ] [ -serviceport port ] [ -workerport port ]"
+ crash "$( basename $0 ) [ -start service | -stop service | -status | -xml | -debug ]"
}
+# Verify list of files exist, or crash
verify_files_exist()
{
for file in "$@"
@@ -172,8 +181,17 @@
crash "Service named $SERVICE_NAME already exists!"
fi
+ mkdir $COASTER_DIR || crash "Unable to create $COASTER_DIR"
+
+ # Generate environment variables from swift.properties
+ $SWIFT_BIN/prop2scs.pl $SERVICE_NAME > $COASTER_DIR/swift-service.conf 2>/dev/null
+ if [ ! -s "$COASTER_DIR/swift-service.conf" ]; then
+ rm -rf $COASTER_DIR
+ crash "Service $SERVICE_NAME not defined! Please add service definition to swift.properties"
+ fi
+ source $COASTER_DIR/swift-service.conf
+
echo -e "Starting Swift service..\n"
- mkdir $COASTER_DIR || crash "Unable to create $COASTER_DIR"
LOG="$COASTER_DIR/swift-service.log"
# Verify we can find coaster service
@@ -242,7 +260,6 @@
# Generate sites.xml
export EXECUTIONURL="http://$IPADDR:$SERVICE_PORT"
export WORKERURL=$( java -jar $SWIFT_BIN/listcoasterurls.jar $LOCAL_PORT )
-
echo Execution URL: $EXECUTIONURL
echo Worker URL: $WORKERURL
echo $EXECUTIONURL >> $COASTER_DIR/execution.url
@@ -329,62 +346,68 @@
fi
}
-# Generate XML
-generate_xml()
+# Display debug info
+display_debug()
{
- count=0
- echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
- echo -e "<config xmlns=\"http://www.ci.uchicago.edu/swift/SwiftSites\">\n"
-
- services=""
- if [ -n "$SERVICE_NAME" ]; then
- NAME=$( echo $SERVICE_NAME | sed s/,/' '/g )
- for n in $SERVICE_NAME
- do
- services="$services $SERVICE_DIR/$n "
- done
- else
- services=$( get_services )
- fi
-
- for service in $services
+ SERVICES=$( get_services )
+ for service in $SERVICES
do
- if [ ! -d "$service" ]; then
- crash "Service $( basename $service ) does not exist"
- fi
- verify_files_exist $service/sites.xml
- cat $service/sites.xml | grep -v "<?xml" | grep -v "<config" | grep -v "</config>"
+ echo -e "Displaying debug info for service $( basename $service ) ($service/swift-service.log)\n"
+ cat $service/swift-service.log
echo
done
+}
- echo "</config>"
+# Start SSH workers
+start-workers-ssh()
+{
+ for MACHINE in $WORKER_HOSTS
+ do
+ # Enable ssh tunneling if needed
+ if [ "$SSH_TUNNELING" != "no" ]; then
+ run_command_bg ssh -N -T -R *:$LOCAL_PORT:localhost:$LOCAL_PORT "$WORKER_USERNAME@$MACHINE"
+ fi
+ # Connect directly
+ run_command ssh $WORKER_USERNAME@$MACHINE mkdir -p $WORKER_LOCATION
+ run_command scp $WORKER $WORKER_USERNAME@$MACHINE:$WORKER_LOCATION
+ echo Starting worker on $MACHINE
+ run_command_bg ssh $WORKER_USERNAME@$MACHINE "WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL $WORKER_LOCATION/worker.pl $WORKERURL $MACHINE $WORKER_LOG_DIR"
+ done
+ return 0
}
-# Display debug info
-display_debug()
+# Start local workers
+start-workers-local()
{
- SERVICES=$( get_services )
- for service in $SERVICES
+ for service in $SERVICE
do
- echo -e "Displaying debug info for service $( basename $service ) ($service/swift-service.log)\n"
- cat $service/swift-service.log
- echo
+ echo Connecting local workers to service $service
+ service="$SERVICE_DIR/$service"
+ verify_files_exist $service/worker.path $service/worker.url
+ LOG=$service/workers.log
+ WORKER=$( cat $service/worker.path )
+ WORKERURL=$( cat $service/worker.url )
+ WORKERPID=$( run_command_bg $WORKER $WORKERURL $WORKER_LOG $WORKER_LOG_DIR )
+ echo $WORKERPID >> $service/worker.pid
done
}
+# Start condor workers
+start-workers-scheduler()
+{
+ echo Starting workers
+ run_command $SCHEDULER_COMMAND
+}
+
# Parse command line arguments
START=0; STOP=0; STATUS=0; XML=0; DEBUG=0
while [ $# -gt 0 ]; do
case $1 in
-debug) DEBUG=1; shift;;
- -localport|-workerport) LOCAL_PORT=$2; shift 2;;
- -name) SERVICE_NAME=$2; shift 2;;
- -serviceport) SERVICE_PORT=$2; shift 2;;
- -start) START=1; shift;;
+ -start) START=1; SERVICE_NAME=$2; verify_not_null service_name $SERVICE_NAME; shift 2;;
-status) STATUS=1; shift;;
-stop) STOP=1; shift;;
- -xml) XML=1; shift;;
*) echo "Do not recognize command line option: $1" 1>&2; usage;;
esac
done
@@ -399,8 +422,6 @@
stop_service
elif [ "$STATUS" == 1 ]; then
display_status
-elif [ "$XML" == 1 ]; then
- generate_xml
elif [ "$DEBUG" == 1 ]; then
display_debug
fi
Deleted: trunk/bin/swift-workers
===================================================================
--- trunk/bin/swift-workers 2013-11-26 17:29:13 UTC (rev 7336)
+++ trunk/bin/swift-workers 2013-11-26 22:08:36 UTC (rev 7337)
@@ -1,188 +0,0 @@
-#!/bin/bash
-
-export SWIFT_BIN="$( cd "$( dirname "$0" )" && pwd )"
-export SERVICE_DIR="$HOME/.swift/service"
-
-# Report a problem and exit
-crash()
-{
- MSG=$1
- echo ${MSG} >&2
- exit 1
-}
-
-# Print usage
-usage()
-{
- crash "$( basename $0 ) [ -start | -stop ] [ -service servicename ]"
-}
-
-# Return current timestamp
-get_timestamp()
-{
- date +"%b %d %H:%M:%S "
-}
-
-# Write command to log, run command, wait for completion
-run_command()
-{
- command="$@"
- echo "$( get_timestamp ) Running command $command" >> $LOG
- $command 2>&1 | tee -a $LOG
-}
-
-# Write command to log, run in background, return PID
-run_command_bg()
-{
- command="$@"
- echo "$( get_timestamp ) Running background command $command" >> $LOG
- $command >> $LOG 2>&1 &
- echo $!
-}
-
-verify_files_exist()
-{
- for file in "$@"
- do
- if [ ! -f "$file" ]; then
- crash "File $file does not exist. Service is in a corrupt state, please stop and restart"
- fi
- done
-}
-
-# Kill all children processes
-nicely_kill_all_children()
-{
- PID=$1
- ps -u $USER -o "pid,ppid"|sed 1d | while read PROC
- do
- PROC_PID=$( echo $PROC | awk '{print $1}' )
- PROC_PPID=$( echo $PROC | awk '{print $2}' )
- if [ $PROC_PPID == $PID ]; then
- nice_kill $PROC_PID 5
- fi
- done
- nice_kill $PID 5
-}
-
-# Try to nicely kill a process before giving it a SIGKILL
-nice_kill()
-{
- pid=$1
- grace=$2
- run_command kill $pid &>/dev/null
-
- count=0
- while ps -p $pid &>/dev/null
- do
- sleep 1
- (( count++ ))
- if [ "$count" -ge "$grace" ]; then
- run_command kill -9 $pid
- fi
- done
-}
-
-# Start SSH workers
-start-workers-ssh()
-{
- for MACHINE in $WORKER_HOSTS
- do
- # Enable ssh tunneling if needed
- if [ "$SSH_TUNNELING" != "no" ]; then
- run_command_bg ssh -N -T -R *:$LOCAL_PORT:localhost:$LOCAL_PORT "$WORKER_USERNAME@$MACHINE"
- fi
-
- # Connect directly
- run_command ssh $WORKER_USERNAME@$MACHINE mkdir -p $WORKER_LOCATION
- run_command scp $WORKER $WORKER_USERNAME@$MACHINE:$WORKER_LOCATION
- echo Starting worker on $MACHINE
- run_command_bg ssh $WORKER_USERNAME@$MACHINE "WORKER_LOGGING_LEVEL=$WORKER_LOGGING_LEVEL $WORKER_LOCATION/worker.pl $WORKERURL $MACHINE $WORKER_LOG_DIR"
- done
- return 0
-}
-
-# Start local workers
-start-workers-local()
-{
- for service in $SERVICE
- do
- echo Connecting local workers to service $service
- service="$SERVICE_DIR/$service"
- verify_files_exist $service/worker.path $service/worker.url
- LOG=$service/workers.log
- WORKER=$( cat $service/worker.path )
- WORKERURL=$( cat $service/worker.url )
- WORKERPID=$( run_command_bg $WORKER $WORKERURL $WORKER_LOG $WORKER_LOG_DIR )
- echo $WORKERPID >> $service/worker.pid
- done
-}
-
-# Start condor workers
-start-workers-scheduler()
-{
- echo Starting workers
- run_command $SCHEDULER_COMMAND
-}
-
-# Print a list of services
-get_services()
-{
- find $SERVICE_DIR -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort -n
-}
-
-# Stop worker processes
-stop_workers()
-{
- if [ ! -d "$SERVICE_DIR/$SERVICE" ]; then
- crash "Service $SERVICE does not work"
- fi
- echo Stopping service $SERVICE
- verify_files_exist "$SERVICE_DIR/$SERVICE/worker.pid"
- nicely_kill_all_children $( cat $SERVICE_DIR/$SERVICE/worker.pid )
- rm $SERVICE_DIR/$SERVICE/worker.pid
-}
-
-# Parse command line arguments
-START=0;STOP=0
-while [ $# -gt 0 ]; do
- case $1 in
- -conf) CMDLN_CONF=$2; shift 2;;
- -start) START=1; shift;;
- -stop) STOP=1; shift;;
- -service) SERVICE=$2; shift 2;;
- *) echo "Do not recognize command line option: $1" 1>&2; exit 1;;
- esac
-done
-
-if [ -z "$SERVICE" ] || [ $(($START+$STOP)) != 1 ]; then
- usage
-fi
-
-# Defaults for worker logging
-WORKER_LOG="worker.log"
-WORKER_LOG_DIR="NOLOGGING"
-WORKER_LOGGING_LEVEL="OFF"
-
-# Import settings
-if [ -f "$CMDLN_CONF" ]; then
- CONFIG_FILE=$CMDLN_CONF
-elif [ -f "coaster-service.conf" ]; then
- CONFIG_FILE="coaster-service.conf"
-fi
-
-if [ -f "$CONFIG_FILE" ]; then
- echo "Configuration: $CONFIG_FILE"
- source $CONFIG_FILE
-fi
-
-if [ "$START" -eq 1 ]; then
- case $WORKER_MODE in
- ssh) start-workers-ssh;;
- local) start-workers-local;;
- scheduler) start-workers-scheduler;;
- *) crash "Unknown WORKER_MODE";;
- esac
-elif [ "$STOP" -eq 1 ]; then
- stop_workers
-fi
More information about the Swift-commit
mailing list