[Swift-commit] r3530 - in usertools/swift/swiftconfig: bin etc
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Fri Aug 13 10:43:27 CDT 2010
Author: davidk
Date: 2010-08-13 10:43:27 -0500 (Fri, 13 Aug 2010)
New Revision: 3530
Modified:
usertools/swift/swiftconfig/bin/swiftconfig
usertools/swift/swiftconfig/bin/swiftrun
usertools/swift/swiftconfig/etc/TODO
Log:
Swiftconfig: edit, copy, template
pod for swiftrun
Help option displays pod for both swiftconfig and swiftrun
editing modes
Modified: usertools/swift/swiftconfig/bin/swiftconfig
===================================================================
--- usertools/swift/swiftconfig/bin/swiftconfig 2010-08-13 05:12:44 UTC (rev 3529)
+++ usertools/swift/swiftconfig/bin/swiftconfig 2010-08-13 15:43:27 UTC (rev 3530)
@@ -5,6 +5,7 @@
use Cwd qw(abs_path);
use File::Copy;
use File::Path;
+use Pod::Usage;
my $swift_home = q{};
if ( $ENV{'SWIFT_HOME'} && -e "$ENV{'SWIFT_HOME'}/bin/swift" ) {
@@ -37,37 +38,32 @@
use XML::Simple;
use Data::Dumper;
-# Variables used for translation catalog
-#my $tc_host = 'localhost'; # Host name
-#my $tc_name = q{}; # Translation name
-#my $tc_path = "/usr/bin/$name"; # Full path of executable location
-#my $tc_status = 'INSTALLED'; # Installation status (not used)
-#my $tc_platform = 'INTEL32::LINUX'; # Platform (not used)
-#my $tc_profile = 'null'; # Profile entries
-
# Variables used for sites.xml
my $option_templates = q{}; # List all templates
my $option_remove = q{}; # Remove entry from sites.xml
-my $option_modify = q{}; # Modify mode
-my $option_add = q{}; # Add entry
+my $option_edit = q{}; # Edit mode
my $option_sites = q{}; # List all usable sites
my $option_describe = q{}; # Print raw XML configuration
my $option_default = q{}; # When adding a site, use all defaults
+my $option_template = q{}; # Create config from template specified
+my $option_copy = q{}; # Copy an existing configuration
my @execution_providers =
( "gt4", "local", "pbs", "condor", "ssh", "coaster", );
# Assign variables from command line options
GetOptions(
- 'add=s' => \$option_add,
'remove=s' => \$option_remove,
'templates' => \$option_templates,
- 'modify=s' => \$option_modify,
'swift_home=s' => \$swift_home,
'sites' => \$option_sites,
'describe=s' => \$option_describe,
'default|defaults' => \$option_default,
'full' => sub { $option_default = 0; },
+ 'template=s' => \$option_template,
+ 'edit=s' => \$option_edit,
+ 'copy=s' => \$option_copy,
+ 'help' => sub { pod2usage(-verbose => 2); },
);
# Create a new directory if it does not exist
@@ -83,6 +79,7 @@
sub get_entry {
my ( $entry_description, $entry_value, $use_default, @allowable_values, ) = @_;
+ # If use_default is specified, automatically plug in the default value
if($use_default) {
return $entry_value;
}
@@ -250,7 +247,12 @@
sub update_xml {
# Set up data
- my ($xml_filename) = @_;
+ my ($xml_filename, $edit_mode) = @_;
+ # Values for edit mode
+ # 0 - Edit all
+ # 1 - All but name
+ # 2 - Customized.. to do
+
if ( !-e $xml_filename ) {
die "Unable to update xml file $xml_filename\n";
}
@@ -262,8 +264,10 @@
);
# Handle
- $xml_ref->{handle} = get_entry( 'Site Entry Name', $xml_ref->{handle}, $option_default);
-
+ if($edit_mode == 0) {
+ $xml_ref->{handle} = get_entry( 'Site Entry Name', $xml_ref->{handle}, $option_default);
+ }
+
# Grid FTP
if ( $xml_ref->{gridftp} ) {
$xml_ref->{gridftp}{url} =
@@ -368,24 +372,20 @@
}
}
-# If a template is specified, find the correct one and modify as needed
-if ($option_add) {
+# Add configuration from a template
+if ($option_template) {
# Update XML info
- my $from_directory = "$template_directory/$option_add";
+ my $from_directory = "$template_directory/$option_template";
my $sites_input_file = "$from_directory/sites.xml";
if ( !-e "$sites_input_file" ) {
- $from_directory = "$dotswift_directory/sites/$option_add";
- $sites_input_file = "$from_directory/sites.xml";
- if ( !-e "$sites_input_file" ) {
- die "Unable to find $sites_input_file\n";
- }
+ die "Unable to find configuration for $sites_input_file\n";
}
- my $xml_ref = update_xml("$sites_input_file");
+ my $xml_ref = update_xml("$sites_input_file", 0);
my $new_sitename = $xml_ref->{handle};
my $output_directory = "$dotswift_directory/sites/$new_sitename";
if ( -d "$output_directory" ) {
- die "Entry for $new_sitename already exists\n";
+ die "Configuration for $new_sitename already exists\n";
}
# Update and copy files to reflect input
@@ -402,7 +402,7 @@
);
write_file( "$output_directory/sites.xml", $xml_out_ref );
- if ( $option_add ne $new_sitename ) {
+ if ( $option_template ne $new_sitename ) {
my $tc_filename = "$output_directory/tc.data";
update_tc_hostname( "$tc_filename", $new_sitename );
}
@@ -456,16 +456,16 @@
}
# Modify a site entry
-if ($option_modify) {
- my $output_directory = "$dotswift_directory/sites/$option_modify";
+if ($option_edit) {
+ my $output_directory = "$dotswift_directory/sites/$option_edit";
create_directory($output_directory);
my $sites_file = "$output_directory/sites.xml";
if ( !-e "$sites_file" ) {
- die "Unable to find configuration for $option_modify\n";
+ die "Unable to find configuration for $option_edit\n";
}
- my $xml_filename = "$dotswift_directory/sites/$option_modify/sites.xml";
+ my $xml_filename = "$output_directory/sites.xml";
my $xml_ref = $xml->XMLin(
$xml_filename,
ForceArray => [qw(workdirectory profile)],
@@ -473,9 +473,9 @@
);
my $initial_exurl = $xml_ref->{execution}{url};
- $xml_ref = update_xml("$sites_file");
+ $xml_ref = update_xml("$sites_file", 1);
my $new_exurl = $xml_ref->{execution}{url};
- my $new_sitename = $xml_ref->{handle};
+ #my $new_sitename = $xml_ref->{handle};
my $xml_out_ref = $xml->XMLout(
$xml_ref,
@@ -484,29 +484,74 @@
);
write_file( "$output_directory/sites.xml", $xml_out_ref );
- # If the sitename has changed, move files
- if ( $new_sitename ne $option_modify ) {
- $output_directory = "$dotswift_directory/sites/$new_sitename";
- if ( -d "$output_directory" ) {
- die "Directory for $new_sitename already exists\n";
- }
- move( "$dotswift_directory/sites/$option_modify",
- "$output_directory" )
- || die "Unable to rename $dotswift_directory/sites/$option_modify"
- . " to $output_directory";
- my $tc_filename = "$output_directory/tc.data";
- update_tc_hostname( "$tc_filename", $new_sitename );
- }
-
# Add/modify/remove ssh entries as needed
if ( $xml_ref->{execution}{provider} eq 'ssh' ) {
- if ( $initial_exurl ne $new_exurl ) {
+ if ( $initial_exurl && $initial_exurl ne $new_exurl ) {
remove_ssh( $initial_exurl );
}
add_ssh( $xml_ref->{handle}, $xml_ref->{execution}{url} );
}
}
+# Copy a configuration
+if($option_copy) {
+ my $input_directory = "$dotswift_directory/sites/$option_copy";
+ my $input_site = "$input_directory/sites.xml";
+ if( !-e $input_site) {
+ die "Unable to find configuration for $option_copy\n";
+ }
+
+ my $new_config_name = get_entry( 'New Configuration Name', '', 0);
+ my $output_directory = "$dotswift_directory/sites/$new_config_name";
+ if(-d "$output_directory") {
+ die "Configuration for $new_config_name already exists\n";
+ }
+ create_directory($output_directory);
+
+ my @files_to_copy = <$input_directory/*>;
+ foreach my $file_to_copy (@files_to_copy) {
+ copy( $file_to_copy, "$output_directory" )
+ || die "Unable to copy $file_to_copy to $output_directory\n";
+ }
+
+ # Rename and save
+ my $xml_filename = "$output_directory/sites.xml";
+ my $xml_ref = $xml->XMLin(
+ $xml_filename,
+ ForceArray => [qw(workdirectory profile)],
+ KeyAttr => [],
+ );
+ $xml_ref->{handle} = $new_config_name;
+ my $xml_out_ref = $xml->XMLout(
+ $xml_ref,
+ RootName => 'pool',
+ SuppressEmpty => 1,
+ );
+ write_file( "$output_directory/sites.xml", $xml_out_ref );
+
+ # Edit sites
+ my $initial_exurl = $xml_ref->{execution}{url};
+ $xml_ref = update_xml( "$output_directory/sites.xml", 1);
+ my $new_exurl = $xml_ref->{execution}{url};
+
+ $xml_out_ref = $xml->XMLout(
+ $xml_ref,
+ RootName => 'pool',
+ SuppressEmpty => 1,
+ );
+
+ write_file( "$output_directory/sites.xml", $xml_out_ref );
+
+ # Update tc
+ my $tc_filename = "$output_directory/tc.data";
+ update_tc_hostname( "$tc_filename", $new_config_name );
+
+ # Add ssh entries as needed
+ if ( $xml_ref->{execution}{provider} eq 'ssh' ) {
+ add_ssh( $xml_ref->{handle}, $xml_ref->{execution}{url} );
+ }
+}
+
__END__
=head1 NAME
@@ -527,33 +572,27 @@
=head1 DESCRIPTION
General operations:
- -add sitename add a site from template
- -remove site removes a site from sites.xml
- -templates display all available sites in template
- -modify site Specifies the name of a site to modify
- -describe site Prints the raw XML site configuration
+ -template sitename Add a configuration based on a template
+ -remove site Removes a site configuration
+ -templates Display all available templates
+ -sites Displays all existing configurations
+ -edit site Specifies the name of a configuration to edit
+ -describe site Prints the raw XML site configuration
+ -copy site Copies a configuration
+ -default Do not prompt for values, always use defaults
+ -full Edit every available configuration option
=head1 EXAMPLES
List all templates available for adding
swiftconfig -templates
-Add a site from template into working sites.xml
- swiftconfig -add teraport
+Add a site from template into a working configuration
+ swiftconfig -template ssh
-Modify the work directory of a site
- swiftconfig -modify teraport
+Edit a configuration
+ swiftconfig -edit teraport
Remove a site
swiftconfig -remove teraport
-
-=head1 CAVEATS
-
-Swiftconfig will attempt to automatically determine the location of swift
-configuration files. It first checks for an environment variable called
-$SWIFT_HOME. If that is not found, it will look for the location of "swift"
-in the path.
-
-The XML library that swiftconfig uses ignores comments in XML. All comments
-will be stripped from sites.xml as it gets modified.
=cut
Modified: usertools/swift/swiftconfig/bin/swiftrun
===================================================================
--- usertools/swift/swiftconfig/bin/swiftrun 2010-08-13 05:12:44 UTC (rev 3529)
+++ usertools/swift/swiftconfig/bin/swiftrun 2010-08-13 15:43:27 UTC (rev 3530)
@@ -10,6 +10,7 @@
use Getopt::Long qw(:config permute pass_through);
use File::Which qw(which where);
use XML::Simple;
+use Pod::Usage;
# Figure out the path to Swift
my $swift_home = q{};
@@ -83,6 +84,7 @@
'site|sites=s{,}' => \@option_sites,
'script=s' => \$option_script,
'input|inputs=s{,}' => \@option_inputs,
+ 'help' => sub { pod2usage(-verbose => 2); },
);
@@ -184,3 +186,41 @@
# Execute
system($system_command);
+__END__
+
+=head1 NAME
+
+swiftrun - Run swift using configurations generated by swiftconfig
+
+=head1 SYNOPSIS
+
+swiftrun [B<-option> value]
+
+=head1 OVERVIEW
+
+The swiftrun program allows you to execute swift using configurations
+generated by swiftconfig.
+
+=head1 DESCRIPTION
+
+General operations:
+ -site <site1 site2> Execute swift on given sitename(s)
+ -script file.swift Specify the swift script to run
+ -input <file1 file2> Link to data required for execution
+
+=head1 EXAMPLES
+
+Run a script on a single site
+ swiftrun -site local -script my.script
+
+Run a script on multiple sites
+ swiftrun -site host1 host2 -script my.script
+
+Run a script on all available configurations
+ swiftrun -site all -script my.script
+
+Link multiple files
+ swiftrun -site local -script my.script -input /foo/file* /foo2/file*
+
+=cut
+
Modified: usertools/swift/swiftconfig/etc/TODO
===================================================================
--- usertools/swift/swiftconfig/etc/TODO 2010-08-13 05:12:44 UTC (rev 3529)
+++ usertools/swift/swiftconfig/etc/TODO 2010-08-13 15:43:27 UTC (rev 3530)
@@ -2,9 +2,6 @@
In the short term, re-add ability to edit TC file with an interactive editor
Longer term, eliminate TC and search PATH instead
-Non-Swiftrun related command line arguments should be passed on to be handled
-by swift
-
Running and testing on more setups, OSG, teragrid, others
Job tagging with search capabilities
@@ -12,9 +9,6 @@
Ability to create groups of related sites for runs, ability to run on all
sites (-site all, -site mygroup, etc)
-Bypass swift shell script
Have swiftrun be flexible enough to work for new users, as well as advanced users
-Add a -link or similar option to create symlinks to input data within run directory
-
Add -help or similar
More information about the Swift-commit
mailing list