[Swift-commit] r3501 - usertools/swift/swiftconfig/bin
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Mon Aug 9 21:11:28 CDT 2010
Author: davidk
Date: 2010-08-09 21:11:28 -0500 (Mon, 09 Aug 2010)
New Revision: 3501
Modified:
usertools/swift/swiftconfig/bin/swiftconfig
usertools/swift/swiftconfig/bin/swiftrun
Log:
Better handling of auth.defaults
Modified: usertools/swift/swiftconfig/bin/swiftconfig
===================================================================
--- usertools/swift/swiftconfig/bin/swiftconfig 2010-08-08 21:06:36 UTC (rev 3500)
+++ usertools/swift/swiftconfig/bin/swiftconfig 2010-08-10 02:11:28 UTC (rev 3501)
@@ -1,5 +1,6 @@
-#!/usr/bin/perl
+#!/usr/bin/perl -w
+use strict;
use Cwd;
use Cwd qw(abs_path);
use File::Copy;
@@ -11,7 +12,7 @@
}
my @execution_path = split( '/', abs_path($0) );
-my @execution_path = splice( @execution_path, 0, $#execution_path - 1 );
+ at execution_path = splice( @execution_path, 0, $#execution_path - 1 );
my $parent_directory = join( '/', @execution_path );
if ( !$swift_home ) {
@@ -37,12 +38,12 @@
use Data::Dumper;
# Variables used for translation catalog
-my $tc_host = 'localhost'; # Host name
-my $tc_name = q{}; # Translation name
-my $tc_path = "/usr/bin/$name"; # Full path of executable location
-my $tc_status = 'INSTALLED'; # Installation status (not used)
-my $tc_platform = 'INTEL32::LINUX'; # Platform (not used)
-my $tc_profile = 'null'; # Profile entries
+#my $tc_host = 'localhost'; # Host name
+#my $tc_name = q{}; # Translation name
+#my $tc_path = "/usr/bin/$name"; # Full path of executable location
+#my $tc_status = 'INSTALLED'; # Installation status (not used)
+#my $tc_platform = 'INTEL32::LINUX'; # Platform (not used)
+#my $tc_profile = 'null'; # Profile entries
# Variables used for sites.xml
my $option_templates = q{}; # List all templates
@@ -50,9 +51,9 @@
my $option_modify = q{}; # Modify mode
my $option_add = q{}; # Add entry
my $option_sites = q{}; # List all usable sites
+my $option_describe = q{}; # Print raw XML configuration
my @execution_providers =
( "gt4", "local", "pbs", "condor", "ssh", "coaster", );
-my $key_file = "$ENV{'HOME'}/.ssh/id_rsa";
# Assign variables from command line options
GetOptions(
@@ -67,7 +68,7 @@
# Create a new directory if it does not exist
sub create_directory {
- $directory = $_[0];
+ my $directory = $_[0];
if ( !-d "$directory" ) {
mkdir "$directory", 0700
or die "Unable to create directory $directory\n";
@@ -101,7 +102,7 @@
print 'Invalid value selected. Please select from: '
. "$error_message\n";
$entry_value = get_entry( $entry_description, $entry_value,
- @allowable_entry_values, );
+ @allowable_values, );
}
}
@@ -118,72 +119,91 @@
create_directory("$dotswift_directory/sites");
# Determine the template directory
-my @execution_path = split( '/', abs_path($0) );
+ at execution_path = split( '/', abs_path($0) );
@execution_path = splice( @execution_path, 0, $#execution_path - 1 );
my $template_directory = join( '/', @execution_path ) . '/etc/sites';
# Add new entry to auths.default
sub add_ssh {
my ( $ssh_site, $ssh_url) = @_;
-
+
# Open authfile
- my $auth_file = "$dotswift_directory/sites/$ssh_site/auth.defaults";
- create_directory("$dotswift_directory/sites/$ssh_site");
- if ( -e "$auth_file" ) {
- open( AUTH_FILE, "$auth_file" )
- || die "Unable to open auth file $auth_file for reading!\n";
+ my $auth_file = "$ENV{'HOME'}/.ssh/auth.defaults";
+ if( -e "$auth_file") {
+ open( AUTH_FILE, "<$auth_file")
+ || die "Unable to open $auth_file for reading\n";
}
else {
- open( AUTH_FILE, ">$auth_file" )
- || die "Unable to open $auth_file for writing!\n";
+ open( AUTH_FILE, "+>$auth_file" )
+ || die "Unable to open $auth_file for read/write!\n";
}
my @auth_data = <AUTH_FILE>;
close(AUTH_FILE);
- # Create RSA keypair if needed
- #my $keyfile = "$ENV{'HOME'}/.ssh/auth.defaults";
- #if ( !-e "$keyfile" ) {
- # system("ssh-keygen -t rsa -f $keyfile");
- #}
-
- # Get existing username and passphrase values
+ # Get existing values and modify if found
my $ssh_username;
- my $ssh_passphrase;
+ my $ssh_password;
+ my $found = 0;
foreach (@auth_data) {
- if (/username/i) {
- ( my $not_used, $ssh_username, ) = split( '=', $_ );
+ if (/username/i && /^$ssh_url/i) {
+ ( my $setting, $ssh_username, ) = split( '=', $_ );
chomp($ssh_username);
+ $ssh_username = get_entry( 'Username', $ssh_username );
+ $_ = "$ssh_url.username=$ssh_username\n";
+ $found = 1;
}
- if (/passphrase/i) {
- ( my $not_used, $ssh_passphrase, ) = split( '=', $_ );
- chomp($ssh_passphrase);
+ if (/\.password/i && /^$ssh_url/i) {
+ ( my $setting, $ssh_password, ) = split( '=', $_ );
+ chomp($ssh_password);
+ $ssh_password = get_entry( 'Password', $ssh_password );
+ $_ = "$ssh_url.password=$ssh_password\n";
+ $found = 1;
}
}
- if ( !$ssh_username ) {
+
+ # Add new entry if needed
+ if(!$found) {
$ssh_username = getlogin();
- }
- $ssh_username = get_entry( 'Username', $ssh_username );
- $ssh_passphrase = get_entry( 'Passphrase', $ssh_passphrase );
+ $ssh_username = get_entry( 'Username', $ssh_username );
+ $ssh_password = get_entry( 'Password', '');
- # Remove old auth.defaults if it already exists
- if ( -e "$auth_file" ) {
- unlink($auth_file) || die "Unable to remove $auth_file\n";
- @auth_data = ();
+ # Password based authentication
+ push( @auth_data, "$ssh_url.type=password\n" );
+ push( @auth_data, "$ssh_url.username=$ssh_username\n" );
+ push( @auth_data, "$ssh_url.password=$ssh_password\n" );
}
-
- # Key based authentication
- #push( @authdata, "$site.type=key\n" );
- #push( @authdata, "$site.username=$username\n" );
- #push( @authdata, "$site.key=$keyfile\n" );
- #push( @authdata, "$site.passphrase=$passphrase\n" );
-
- # Password based authentication
- push( @auth_data, "$ssh_url.type=password\n" );
- push( @auth_data, "$ssh_url.username=$ssh_username\n" );
- push( @auth_data, "$ssh_url.password=$ssh_passphrase\n" );
+
+ # Write data
write_file( $auth_file, @auth_data );
}
+# Remove an entry from auth.defaults
+sub remove_ssh {
+ my ( $ssh_url, ) = @_;
+
+ # Open auth.defaults
+ my $auth_file = "$ENV{'HOME'}/.ssh/auth.defaults";
+ if( -e "$auth_file") {
+ open( AUTH_FILE, "<$auth_file")
+ || die "Unable to open $auth_file for reading\n";
+ }
+ else {
+ return;
+ }
+ my @auth_data = <AUTH_FILE>;
+ close(AUTH_FILE);
+
+ # Remove
+ foreach (@auth_data) {
+ if (/^$ssh_url/i) {
+ $_ = '';
+ }
+ }
+
+ # Write data
+ write_file( $auth_file, @auth_data );
+}
+
# Update TC with correct hostname
sub update_tc_hostname {
my ( $tc_filename, $tc_host ) = @_;
@@ -191,10 +211,10 @@
# Store TC data
open( TC_FILESTREAM, "$tc_filename" )
or die("Unable to open tc file $tc_filename!");
- @tc_data = <TC_FILESTREAM>;
+ my @tc_data = <TC_FILESTREAM>;
close(TC_FILESTREAM);
- foreach $line (@tc_data) {
+ foreach my $line (@tc_data) {
# Ignore comments
my $first_character = substr( $line, 0, 1 );
@@ -277,7 +297,7 @@
$xml_ref->{execution}{provider} = get_entry(
'Execution Provider',
$xml_ref->{execution}{provider},
- @validproviders
+ @execution_providers,
);
}
if ( $xml_ref->{execution}{jobmanager} ) {
@@ -298,7 +318,7 @@
$xml_ref->{filesystem}{provider} = get_entry(
'Filesystem Provider',
$xml_ref->{filesystem}{provider},
- @validproviders
+ @execution_providers,
);
}
if ( $xml_ref->{filesystem}{url} ) {
@@ -331,7 +351,7 @@
chdir($directory)
|| die "Unable to change directories to $directory\n";
my @files = <*>;
- foreach $file (@files) {
+ foreach my $file (@files) {
( my $basename, my $ext ) = split( /\./, $file );
my @path = split( '/', $basename );
print "$basename\n";
@@ -361,9 +381,9 @@
# Update and copy files to reflect input
my @files_to_copy = <$from_directory/*>;
create_directory("$output_directory");
- foreach $file_to_copy (@files_to_copy) {
+ foreach my $file_to_copy (@files_to_copy) {
copy( $file_to_copy, "$output_directory" )
- || die "Unable to copy to $file to $output_directory\n";
+ || die "Unable to copy to $file_to_copy to $output_directory\n";
}
my $xml_out_ref = $xml->XMLout(
$xml_ref,
@@ -394,11 +414,22 @@
# Remove an entry
if ($option_remove) {
+ my $xml_filename = "$dotswift_directory/sites/$option_remove/sites.xml";
+ my $xml_ref = $xml->XMLin(
+ $xml_filename,
+ ForceArray => [qw(workdirectory profile)],
+ KeyAttr => [],
+ );
+ # Remove SSH entry if needed
+ if($xml_ref->{execution}{provider} eq "ssh") {
+ remove_ssh($xml_ref->{execution}{url});
+ }
+
+ # Remove directory
if ( !-d "$dotswift_directory/sites/$option_remove" ) {
- "Unable to find site entry for $option_remove\n";
+ die "Unable to find site entry for $option_remove\n";
}
-
rmtree("$dotswift_directory/sites/$option_remove")
|| die 'Unable to remove directory '
. "$dotswift_directory/sites/$option_remove\n";
@@ -417,15 +448,25 @@
# Modify a site entry
if ($option_modify) {
my $output_directory = "$dotswift_directory/sites/$option_modify";
- my $sites_file = "$output_directory/sites.xml";
+ create_directory($output_directory);
+
+ my $sites_file = "$output_directory/sites.xml";
if ( !-e "$sites_file" ) {
die "Unable to find configuration for $option_modify\n";
}
-
- my $xml_ref = update_xml("$sites_file");
+
+ my $xml_filename = "$dotswift_directory/sites/$option_modify/sites.xml";
+ my $xml_ref = $xml->XMLin(
+ $xml_filename,
+ ForceArray => [qw(workdirectory profile)],
+ KeyAttr => [],
+ );
+
+ my $initial_exurl = $xml_ref->{execution}{url};
+ $xml_ref = update_xml("$sites_file");
+ my $new_exurl = $xml_ref->{execution}{url};
my $new_sitename = $xml_ref->{handle};
-
- create_directory($output_directory);
+
my $xml_out_ref = $xml->XMLout(
$xml_ref,
RootName => 'pool',
@@ -447,7 +488,11 @@
update_tc_hostname( "$tc_filename", $new_sitename );
}
+ # Add/modify/remove ssh entries as needed
if ( $xml_ref->{execution}{provider} eq 'ssh' ) {
+ if ( $initial_exurl ne $new_exurl ) {
+ remove_ssh( $initial_exurl );
+ }
add_ssh( $xml_ref->{handle}, $xml_ref->{execution}{url} );
}
}
Modified: usertools/swift/swiftconfig/bin/swiftrun
===================================================================
--- usertools/swift/swiftconfig/bin/swiftrun 2010-08-08 21:06:36 UTC (rev 3500)
+++ usertools/swift/swiftconfig/bin/swiftrun 2010-08-10 02:11:28 UTC (rev 3501)
@@ -1,5 +1,6 @@
-#!/usr/bin/perl
+#!/usr/bin/perl -w
+use strict;
use Cwd;
use Cwd qw(abs_path);
use File::Copy;
@@ -11,7 +12,7 @@
}
my @path = split( '/', abs_path($0) );
-my @path = splice( @path, 0, $#path - 1 );
+ at path = splice( @path, 0, $#path - 1 );
my $parent_directory = join( '/', @path );
if ( !$swifthome ) {
@@ -31,37 +32,42 @@
# Create a new directory if it does not exist
sub create_directory {
- $directory = $_[0];
+ my $directory = $_[0];
if ( !-d "$directory" ) {
mkdir "$directory", 0700
- or die "Unable to create directory $directory\n";
+ or die "Unable to create directory $directory\n";
}
}
# Append one file to another
sub cat_file {
- my ($src, $dst) = @_;
- if(-e $dst){ open(DSTFILE, ">>$dst") or die "Unable to open $dst for append\n"; }
- else { open(DSTFILE, ">$dst") or die "Unable to open $dst for writing\n"; }
- open(SRCFILE, $src) || die "Unable to open $src for reading\n";
- foreach $line(<SRCFILE>) {
+ my ( $src, $dst ) = @_;
+ if ( -e $dst ) {
+ open( DSTFILE, ">>$dst" ) or die "Unable to open $dst for append\n";
+ }
+ else {
+ open( DSTFILE, ">$dst" )
+ or die "Unable to open $dst for writing\n";
+ }
+ open( SRCFILE, $src ) || die "Unable to open $src for reading\n";
+ foreach my $line (<SRCFILE>) {
print DSTFILE $line;
}
}
# Copy a file to a new location
sub copy_file {
- my ($src, $dst) = @_;
- if(!-e $src) { die "Unable to access $src\n"; }
- if(!-e $dst) { die "Unable to access $dst\n"; }
- copy($src, $dst) or die "Unable to copy $src to $dst\n";
+ my ( $src, $dst ) = @_;
+ if ( !-e $src ) { die "Unable to access $src\n"; }
+ if ( !-e $dst ) { die "Unable to access $dst\n"; }
+ copy( $src, $dst ) or die "Unable to copy $src to $dst\n";
}
# Strip directory name out of script
sub strip_directory {
my ($fullpath) = @_;
- my @path = split( '/', $fullpath);
- my $filename = @path[$#path];
+ my @path = split( '/', $fullpath );
+ my $filename = $path[$#path];
return $filename;
}
@@ -72,56 +78,61 @@
use XML::Simple;
use Data::Dumper;
+my $site; # Site name to execute upon
+my $script; # Script name to run
+
GetOptions(
- 'site=s' => \$site,
- 'script=s' => \$script,
+ 'site=s' => \$site,
+ 'script=s' => \$script,
);
# Verify user-created profile info exists
-my ($seconds, $minutes, $hours, $days, $months, $years, @junk) = localtime(time);
-$datestring = sprintf "%04s%02s%02s%02s%02s%02s%s", $years+1900, $months+1, $days, $hours, $minutes, $seconds, getpgrp(0);
+my ( $seconds, $minutes, $hours, $days, $months, $years, @junk ) =
+ localtime(time);
+my $datestring = sprintf "%04s%02s%02s%02s%02s%02s%s", $years + 1900,
+ $months + 1, $days, $hours, $minutes, $seconds, getpgrp(0);
my $dotswift = "$ENV{'HOME'}/.swift";
-
# Run job
- at sites = split(/,/, $site);
+my @sites = split( /,/, $site );
my $script_filename = strip_directory($script);
create_directory("$dotswift/jobs");
create_directory("$dotswift/jobs/$script_filename");
create_directory("$dotswift/jobs/$script_filename/$datestring");
-copy_file("$script", "$dotswift/jobs/$script_filename/$datestring");
+copy_file( "$script", "$dotswift/jobs/$script_filename/$datestring" );
-# Temporarily rename existing auth.defaults
-if(-e "$ENV{'HOME'}/.ssh/auth.defaults") {
- move("$ENV{'HOME'}/.ssh/auth.defaults", "$ENV{'HOME'}/.ssh/auth.defaults.$datestring") || die "Unable to rename $ENV{'HOME'}/.ssh/auth.defaults";
+# Create configuration file for run
+foreach my $s (@sites) {
+ if ( !-d "$dotswift/sites/$s" ) {
+ print "$dotswift/sites/$site\n";
+ die "Unable to find entry for $s\n";
+ }
+ if ( !-e "$dotswift/sites/$s/sites.xml" ) {
+ die "Unable to find sites.xml for $s\n";
+ }
+ if ( !-e "$dotswift/sites/$s/tc.data" ) {
+ die "Unable to find tc.data for $s\n";
+ }
+ cat_file( "$dotswift/sites/$s/sites.xml",
+ "$dotswift/jobs/$script_filename/$datestring/sites.xml" );
+ cat_file( "$dotswift/sites/$s/tc.data",
+ "$dotswift/jobs/$script_filename/$datestring/tc.data" );
}
-foreach my $s(@sites) {
- if(!-d "$dotswift/sites/$s"){ print "$dotswift/sites/$site\n"; die "Unable to find entry for $s\n"; }
- if(!-e "$dotswift/sites/$s/sites.xml"){ die "Unable to find sites.xml for $s\n"; }
- if(!-e "$dotswift/sites/$s/tc.data"){ die "Unable to find tc.data for $s\n"; }
- cat_file("$dotswift/sites/$s/sites.xml", "$dotswift/jobs/$script_filename/$datestring/sites.xml");
- cat_file("$dotswift/sites/$s/tc.data", "$dotswift/jobs/$script_filename/$datestring/tc.data");
- if(-e "$dotswift/sites/$s/auth.defaults"){
- cat_file("$dotswift/sites/$s/auth.defaults", "$ENV{'HOME'}/.ssh/auth.defaults");
- }
-}
-
# Update XML tags
-open(SITEFILE, "$dotswift/jobs/$script_filename/$datestring/sites.xml") || die "Unable to open $dotswift/jobs/$script_filename/$datestring/sites.xml\n";
+my $site_filename = "$dotswift/jobs/$script_filename/$datestring/sites.xml";
+open( SITEFILE, "$site_filename" ) || die "Unable to open $site_filename\n";
my @sitedata = <SITEFILE>;
close(SITEFILE);
-open(SITEFILE, ">$dotswift/jobs/$script_filename/$datestring/sites.xml") || die "Unable to open $dotswift/jobs/$script_filename/$datestring/sites.xml\n";
+open( SITEFILE, ">$site_filename" ) || die "Unable to open $site_filename\n";
print SITEFILE "<config>\n at sitedata</config>\n";
close(SITEFILE);
# Run swift
-chdir("$dotswift/jobs/$script_filename/$datestring") || die "Unable to change directories to $dotswift/job/$script_filename/$datestring\n";
-print "Submitting job. Check $dotswift/jobs/$script_filename/$datestring for details\n";
-system("swift -sites.file $dotswift/jobs/$script_filename/$datestring/sites.xml -tc.file $dotswift/jobs/$script_filename/$datestring/tc.data $script");
+my $run_directory = "$dotswift/jobs/$script_filename/$datestring";
+chdir("$run_directory") || die "Unable to chdir to $run_directory\n";
+print "Submitting job. Check $run_directory for details\n";
+system(
+ "swift -sites.file $site_filename -tc.file $run_directory/tc.data $script"
+);
-# Rename auths.default back to original
-if(-e "$ENV{'HOME'}/.ssh/auth.defaults.$datestring") {
- move("$ENV{'HOME'}/.ssh/auth.defaults.$datestring", "$ENV{'HOME'}/.ssh/auth.defaults") || die "Unable to rename $ENV{'HOME'}/.ssh/auth.defaults.$datestring";
-}
-
More information about the Swift-commit
mailing list