From noreply at svn.ci.uchicago.edu Fri Sep 3 14:25:50 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 3 Sep 2010 14:25:50 -0500 (CDT) Subject: [Swift-commit] r3595 - in trunk: bin lib/perl Message-ID: <20100903192550.9E1799CC94@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-09-03 14:25:50 -0500 (Fri, 03 Sep 2010) New Revision: 3595 Modified: trunk/bin/swiftconfig trunk/bin/swiftrun trunk/lib/perl/SwiftConfig.pm Log: Initial implementation of site groupings Modified: trunk/bin/swiftconfig =================================================================== --- trunk/bin/swiftconfig 2010-09-01 01:25:26 UTC (rev 3594) +++ trunk/bin/swiftconfig 2010-09-03 19:25:50 UTC (rev 3595) @@ -15,16 +15,17 @@ use File::Which qw(which where); use Data::Dumper; -# Variables used for sites.xml -my $option_list = q{}; # List information about config -my $option_remove = q{}; # Remove entry from sites.xml -my $option_edit = q{}; # Edit mode -my $option_default = q{}; # When adding a site, use all defaults -my @option_templates = (); # Create config from template specified -my $option_copy = q{}; # Copy an existing configuration -my $option_applications = q{}; # Edit applications -my $option_import = q{}; # Import existing configuration -my @option_profiles = (); # Specify one or more profiles to edit +# Variables for command line options +my $option_list = q{}; # List information about config +my $option_remove = q{}; # Remove entry from sites.xml +my $option_edit = q{}; # Edit mode +my $option_default = q{}; # When adding a site, use all defaults +my @option_templates = (); # Create config from template specified +my $option_copy = q{}; # Copy an existing configuration +my $option_applications = q{}; # Edit applications +my $option_import = q{}; # Import existing configuration +my @option_profiles = (); # Specify one or more profiles to edit +my $option_groups = q{}; # Edit site groupings # Assign variables from command line options GetOptions( @@ -37,8 +38,9 @@ 'apps|applications' => \$option_applications, 'import=s' => \$option_import, 'list=s' => \$option_list, - 'help|man' => sub { pod2usage(-verbose => 2); }, + 'help|man|h' => sub { pod2usage(-verbose => 2); }, 'profile|profiles=s{,}' => \@option_profiles, + 'group|groups' => \$option_groups, ); # Prepare data @@ -46,13 +48,14 @@ initialize_swiftconfig(); # Check command line options -if( !$option_remove +if( !$option_remove && !@option_templates && !$option_edit && !$option_copy && !$option_applications && !$option_import - && !$option_list) { + && !$option_list + && !$option_groups) { print "Incorrect usage. See swiftconfig -help for more information\n"; exit; } @@ -78,7 +81,7 @@ if ( !-e "$sites_input_file" ) { die "Unable to find template for $option_template\n"; } - my $xml_ref = update_xml("$sites_input_file", 0, $option_default); + my $xml_ref = update_xml("$sites_input_file", 0, $option_default); my $new_sitename = $xml_ref->{handle}; my $output_directory = "$dotswift_directory/sites/$new_sitename"; if ( -d "$output_directory" ) { @@ -98,7 +101,6 @@ SuppressEmpty => 1, ); write_file( "$output_directory/sites.xml", $xml_out_ref ); - update_site_applications($new_sitename, $option_default); # Handle SSH @@ -143,7 +145,7 @@ } else { $xml_ref = update_xml("$xml_filename", 1, $option_default); - update_site_applications($option_edit, $option_default); + update_site_applications($option_edit, $option_default); } # Save @@ -214,6 +216,54 @@ } } +# Edit site groupings +if($option_groups) { + my @groups = glob("$dotswift_directory/groups/*"); + foreach(@groups) { + $_ = strip_directory($_); + my ($base, $ext) = split('\.'); + $_ = $base; + } + push(@groups, 'new'); + my $group_selection = get_entry('Select group', 'new', 0, @groups); + if($group_selection eq 'new') { + $group_selection = get_entry('Name of new group', '', 0); + if(-e "$dotswift_directory/groups/$group_selection.group") { + print "Group named $group_selection already exists!\n"; + exit; + } + open(GROUPFILE, ">$dotswift_directory/groups/$group_selection.group") + || die "Unable to create group file $dotswift_directory/groups/$group_selection.group"; + print "Group $group_selection created\n"; + close(GROUPFILE); + } + + my $quit = 0; + my $group_filename = "$dotswift_directory/groups/$group_selection.group"; + + while($quit == 0) { + print "\n(A)dd, (r)emove, (l)ist, or (q)uit: "; + my $command = ; + chomp($command); + + if(lc($command) eq 'a') { + add_to_group($group_filename); + } + elsif(lc($command) eq 'q') { + $quit = 1; + } + elsif(lc($command) eq 'r') { + remove_from_group($group_filename); + } + elsif(lc($command) eq 'l') { + list_group($group_filename); + } + else { + print "Unknown option: $command\n"; + } + } +} + # Add/Edit applications if($option_applications) { my @applications = glob("$dotswift_directory/apps/*"); @@ -311,6 +361,8 @@ } } } + + __END__ =head1 NAME @@ -339,6 +391,7 @@ -import Import a sites.xml file into swiftconfig -list Options: templates, configs, or a config -apps Edit or create application sets + -groups Edit site groupings =head1 EXAMPLES Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2010-09-01 01:25:26 UTC (rev 3594) +++ trunk/bin/swiftrun 2010-09-03 19:25:50 UTC (rev 3595) @@ -17,17 +17,19 @@ my @option_sites; # Site name to execute upon my $option_script; # Script name to run my @option_inputs; # Name of data to include with execution +my @option_groups; # Name of group/groups to run script on GetOptions( 'site|sites=s{,}' => \@option_sites, 'script=s' => \$option_script, 'input|inputs=s{,}' => \@option_inputs, - 'help|man' => sub { pod2usage(-verbose => 2); }, + 'help|man|h' => sub { pod2usage(-verbose => 2); }, + 'group|groups=s{,}' => \@option_groups, ); # Check arguments initialize_swiftconfig(); -if(!$option_script || !@option_sites) { +if(!$option_script || (!@option_sites && !@option_groups)) { print "Incorrect usage. Try swiftrun -help\n"; exit; } @@ -43,109 +45,98 @@ $run_number++; my $run_directory = getcwd() . "/" . sprintf "run.%04s", $run_number; create_directory($run_directory); -my @sites_xml; # Copy over needed files copy_file( "$option_script", "$run_directory" ); -# Sites.xml for 'all' -if( grep /all/, @option_sites ) { - foreach my $all_site(<$dotswift_directory/sites/*>) { - if ( !-d "$all_site" ) { - die "Unable to find directory for $all_site\n"; - } - if ( !-e "$all_site/sites.xml" ) { - die "Unable to find sites.xml for $all_site\n"; - } - my $properties_file = "$dotswift_directory/sites/$all_site/swift.properties"; - if( -e "$properties_file") { - copy_file( $properties_file, "$run_directory/swift.properties"); - } +# Determine list of sites. Store each swiftconfig directory we need in @sites +my @sites = (); - open(CURRENT_SITE, "$all_site/sites.xml") - || die "Unable to open $all_site/sites.xml"; - my @current_site_data = ; - push(@sites_xml, @current_site_data); - close(CURRENT_SITE); - # Add applications - my @all_apps; - if(-e "$all_site/apps") { - open(APPFILE, "$all_site/apps") - || die "Unable to open $all_site/apps"; - @all_apps = ; - close(APPFILE); - foreach(@all_apps) { - chomp($_); - $_ = "$dotswift_directory/apps/$_.apps\n"; - } - } - else { - @all_apps = glob("$dotswift_directory/apps/*.apps"); - } - foreach my $current_app_file(@all_apps) { - chomp($current_app_file); - open(CURRENT_APP, $current_app_file) || die "Unable to open app file $current_app_file\n"; - my @current_app_data = ; - close(CURRENT_APP); - $all_site = strip_directory($all_site); - foreach(@current_app_data) { - s/SWIFTCONFIGURATION/$all_site/ig; - } - open(RUNTC, ">>$run_directory/tc.data") || die "Unable to write to $run_directory/tc.data\n"; - print RUNTC @current_app_data; - close(RUNTC); - } +# Check for -sites all +foreach my $option_site (@option_sites) { + if(lc($option_site) eq 'all') { + @sites = glob("$dotswift_directory/sites/*"); } } - -# Sites.xml for non-'all' -else { - foreach my $site (@option_sites) { - if ( !-d "$dotswift_directory/sites/$site" ) { - die "Unable to find configuration for $site\n"; - } - if ( !-e "$dotswift_directory/sites/$site/sites.xml" ) { - die "Unable to find sites.xml for $site\n"; - } - open(CURRENT_SITE, "$dotswift_directory/sites/$site/sites.xml") - || die "Unable to open $dotswift_directory/sites/$site/sites.xml"; - my @current_site_data = ; - close(CURRENT_SITE); - push(@sites_xml, @current_site_data); - my $properties_file = "$dotswift_directory/sites/$site/swift.properties"; - if( -e "$properties_file") { - copy_file( $properties_file, "$run_directory"); - } - # Add applications - my @all_apps; - if(-e "$dotswift_directory/sites/$site/apps") { - open(APPFILE, "$dotswift_directory/sites/$site/apps") - || die "Unable to open $site/apps"; - @all_apps = ; - close(APPFILE); - foreach(@all_apps) { - chomp($_); - $_ = "$dotswift_directory/apps/$_.apps"; + +# Everything else specified by -sites +foreach my $option_site (@option_sites) { + if(lc($option_site) eq 'all') { + next; + } + if(-d "$dotswift_directory/sites/$option_site") { + push(@sites, "$dotswift_directory/sites/$option_site"); + } + else { + die "Unable to find configuration for $option_site\n"; + } +} + +# Groups +foreach my $group(@option_groups) { + open(GROUPFILE, "$dotswift_directory/groups/$group.group") + || die "Unable to open file for $group\n"; + while() { + chomp($_); + push(@sites, "$dotswift_directory/sites/$_"); + } + close(GROUPFILE); +} + +# Create configuration files for run +my @sites_xml; +my @apps_data; +foreach my $site (@sites) { + if ( !-d "$site" ) { + die "Unable to find configuration for $site\n"; + } + + # Sites.xml + if ( !-e "$site/sites.xml" ) { + die "Unable to find sites.xml for $site\n"; + } + open(CURRENT_SITE, "$site/sites.xml") + || die "Unable to open $site/sites.xml"; + push(@sites_xml, ); + close(CURRENT_SITE); + + # Properties + if( -e "$site/swift.properties") { + copy_file( "$site/swift.properties", "$run_directory"); + } + + # Applications + if(-e "$site/apps") { + open(APPFILE, "$site/apps") || die "Unable to open $site/apps"; + while() { + chomp($_); + if(-e "$dotswift_directory/apps/$_.apps") { + open(APPDATA, "$dotswift_directory/apps/$_.apps") + || die "Unable to open $dotswift_directory/apps/$_.apps"; + my @temp_app_data = ; + close(APPDATA); + foreach(@temp_app_data) { + my $simple_site_name = strip_directory($site); + s/SWIFTCONFIGURATION/$simple_site_name/ig; + } + push(@apps_data, @temp_app_data); } } - else { - @all_apps = glob("$dotswift_directory/apps/*.apps"); - } - foreach my $current_app_file(@all_apps) { - open(CURRENT_APP, $current_app_file) || die "Unable to open app file $current_app_file\n"; - my @current_app_data = ; - close(CURRENT_APP); - foreach(@current_app_data) { - s/SWIFTCONFIGURATION/$site/ig; - } - open(RUNTC, ">>$run_directory/tc.data") || die "Unable to write to $run_directory/tc.data\n"; - print RUNTC @current_app_data; - close(RUNTC); - } } } +# Save configuration information +open(TC, ">>$run_directory/tc.data") + || die "Unable to write to $run_directory/tc.data\n"; +print TC @apps_data; +close(TC); +open( SITEFILE, ">$run_directory/sites.xml" ) + || die "Unable to open $run_directory/sites.xml for writing\n"; +print SITEFILE "\n at sites_xml\n"; +close(SITEFILE); +exit; + # Link input data foreach my $input_file (@option_inputs) { # Check for relative path @@ -162,11 +153,6 @@ || die "Unable to create symlink to $new_path\n"; } -open( SITEFILE, ">$run_directory/sites.xml" ) - || die "Unable to open $run_directory/sites.xml for writing\n"; -print SITEFILE "\n at sites_xml\n"; -close(SITEFILE); - # Run swift chdir("$run_directory") || die "Unable to chdir to $run_directory\n"; print "Executing script $option_script\n"; @@ -213,10 +199,10 @@ =head1 DESCRIPTION General operations: - -site Execute swift on given sitename(s) - -script file.swift Specify the swift script to run - -input Link to data required for execution - + -site Execute swift on given sitename(s) + -script file.swift Specify the swift script to run + -input Link to data required for execution + -groups Run script on a site grouping =head1 EXAMPLES Run a script on a single site Modified: trunk/lib/perl/SwiftConfig.pm =================================================================== --- trunk/lib/perl/SwiftConfig.pm 2010-09-01 01:25:26 UTC (rev 3594) +++ trunk/lib/perl/SwiftConfig.pm 2010-09-03 19:25:50 UTC (rev 3595) @@ -21,7 +21,7 @@ update_tc_hostname update_xml write_file print_directory strip_directory copy_file cat_file add_application edit_application remove_application list_applications initialize_swiftconfig update_site_applications import_tc -import_xml edit_profile +import_xml edit_profile add_to_group list_group remove_from_group ); our $VERSION = '0.01'; @@ -542,6 +542,7 @@ sub initialize_swiftconfig { create_directory("$ENV{'HOME'}/.swift"); create_directory("$ENV{'HOME'}/.swift/sites"); + create_directory("$ENV{'HOME'}/.swift/groups"); if(!-e "$ENV{'HOME'}/.swift/apps") { create_directory("$ENV{'HOME'}/.swift/apps"); my @app_files = glob("$FindBin::Bin/../etc/apps/*.apps"); @@ -703,6 +704,59 @@ return $xml_ref; } +sub add_to_group { + my ($group_filename) = @_; + if(!-e $group_filename) { + print "Unable to find file $group_filename for editing\n"; + return; + } + open(GROUPFILE, ">>$group_filename") || die "Error opening $group_filename\n"; + my @all_sites = glob("$ENV{'HOME'}/.swift/sites/*"); + foreach(@all_sites) { + $_ = strip_directory($_); + } + my $new_site = get_entry("Enter name of site to add", '', 0, @all_sites); + + print GROUPFILE "$new_site\n"; + close(GROUPFILE); +} + +sub list_group { + my ($group_filename) = @_; + if(!-e $group_filename) { + print "Unable to find $group_filename for editing\n"; + return; + } + system("cat $group_filename"); +} + +sub remove_from_group { + my ($group_filename) = @_; + if(!-e $group_filename) { + print "Unable to find $group_filename for editing\n"; + return; + } + + open(GROUPFILE, $group_filename) || die "Error opening $group_filename\n"; + my @group_data = ; + close(GROUPFILE); + + my $remove_group = get_entry("Enter name of site to remove", '', 0); + $remove_group .= "\n"; + + my $found=0; + foreach(@group_data) { + if($_ eq $remove_group) + { + $found=1; + $_ = ''; + } + } + if(!$found) { + print "Unable to find site $remove_group. Use 'l' to get a list\n"; + } + write_file($group_filename, @group_data); +} 1; __END__ From noreply at svn.ci.uchicago.edu Fri Sep 3 14:50:16 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 3 Sep 2010 14:50:16 -0500 (CDT) Subject: [Swift-commit] r3596 - trunk/bin Message-ID: <20100903195016.BDF469CC94@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-09-03 14:50:16 -0500 (Fri, 03 Sep 2010) New Revision: 3596 Modified: trunk/bin/swiftrun Log: Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2010-09-03 19:25:50 UTC (rev 3595) +++ trunk/bin/swiftrun 2010-09-03 19:50:16 UTC (rev 3596) @@ -135,7 +135,6 @@ || die "Unable to open $run_directory/sites.xml for writing\n"; print SITEFILE "\n at sites_xml\n"; close(SITEFILE); -exit; # Link input data foreach my $input_file (@option_inputs) { From noreply at svn.ci.uchicago.edu Fri Sep 3 16:57:20 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 3 Sep 2010 16:57:20 -0500 (CDT) Subject: [Swift-commit] r3597 - SwiftApps/SwiftR/Swift/R Message-ID: <20100903215720.B35E49CD30@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-03 16:57:20 -0500 (Fri, 03 Sep 2010) New Revision: 3597 Modified: SwiftApps/SwiftR/Swift/R/Swift.R Log: Fix to retain names of input and result lists. Added manual mode. Prelim changes to option handling. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-03 19:50:16 UTC (rev 3596) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-03 21:57:20 UTC (rev 3597) @@ -1,4 +1,4 @@ -swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL ) +swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL, manualmode=NULL ) { # Move swiftprops into a Swift namespace @@ -8,13 +8,15 @@ # swiftprops$callsperbatch <<- 1 # } + # Set Swift default options if not pre-set by user + if (is.null(getOption("swift.site"))) - options(swift.site="local"); + options(swift.site="local") if (is.null(getOption("swift.callsperbatch"))) options(swift.callsperbatch=1) + if (is.null(getOption("swift.manualmode"))) + options(swift.manualmode = FALSE) - # Set Swift properties - # if(is.null(site)) # if( is.null(swiftprops) || is.null(swiftprops$site)) # site <- "local" @@ -29,7 +31,9 @@ if(is.null(site)) site <- getOption("swift.site") if(is.null(callsperbatch)) - callsperbatch <- getOption("swift.callsperbatch") + callsperbatch <- getOption("swift.callsperbatch") + if(is.null(manualmode)) + manualmode <- getOption("swift.manualmode") cat("\nSwift R properties:\n") cat(" site =",site,"\n"); @@ -64,8 +68,22 @@ RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh") RunRScript <- system.file(package="Swift","exec/RunR.sh") swiftapplyScript <- system.file(package="Swift","exec/swiftapply.swift") - system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript,sep=" ")) + if( manualmode ) { # Prompt for return (empty line) to continue; assumes user ran a manual R to process the call. + cat("Manual Swift Run:\n run dir: ", getwd(), "/", rundir,"\n\n") + cat(" swift script: ", RunSwiftScript, "\n") + cat(" site: ", site,"\n") + cat(" swiftapplyScript: ", swiftapplyScript,"\n") + cat(" RunRScript: ", RunRScript,"\n\n") + cat(" Use RunAllR.sh to process and press return when complete:") + system(paste("cp ", system.file(package="Swift","exec/RunAllR.sh"), rundir)) + readLines(n=1) + } + else { + system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript)) +# system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript,sep=" ")) + } + # Fetch the batch results rno <- 1 @@ -76,19 +94,24 @@ nresults <- length(result) for(r in 1:nresults) { rlist[[rno]] <- result[[r]] +#DB cat("swiftapply: result rno=",rno,":\n") +#DB cat(rlist[[rno]]@output$gradient,"\n") rno <- rno + 1 } } + names(rlist) = names(arglists) return(rlist) } swiftLapply <- function( tlist, func, ... ) { + if (length(tlist) == 0) return(tlist) arglists <- list() narglists <- length(tlist) for(i in 1 : narglists) { arglists[[i]] <- list(tlist[[i]], ...); } + names(arglists) = names(tlist) swiftapply(func, arglists) } ##### From noreply at svn.ci.uchicago.edu Fri Sep 3 21:25:15 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 3 Sep 2010 21:25:15 -0500 (CDT) Subject: [Swift-commit] r3598 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100904022515.251739CC94@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-09-03 21:25:14 -0500 (Fri, 03 Sep 2010) New Revision: 3598 Modified: trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java Log: skip empty dirs Modified: trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2010-09-03 21:57:20 UTC (rev 3597) +++ trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2010-09-04 02:25:14 UTC (rev 3598) @@ -28,10 +28,14 @@ for (Object f : files) { String path = (String) f; String dir = new AbsFile(path).getDir(); - if (dir.startsWith("/")) { + // there could be a clash here since + // "/a/b/c.txt" would be remotely the same + // as "a/b/c.txt". Perhaps absolute paths + // should have a unique prefix. + if (dir.startsWith("/") && dir.length() != 1) { ret.append(dir.substring(1)); } - else { + else if (dir.length() != 0) { ret.append(dir); } } From noreply at svn.ci.uchicago.edu Fri Sep 3 21:25:42 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 3 Sep 2010 21:25:42 -0500 (CDT) Subject: [Swift-commit] r3599 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100904022542.0D6FA9CC94@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-09-03 21:25:41 -0500 (Fri, 03 Sep 2010) New Revision: 3599 Modified: trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java Log: skip empty dirs Modified: trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java 2010-09-04 02:25:14 UTC (rev 3598) +++ trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java 2010-09-04 02:25:41 UTC (rev 3599) @@ -34,7 +34,12 @@ DSHandle leaf = handle.getField(p); String fname = VDLFunction.filename(leaf)[0]; String dir = new AbsFile(fname).getDir(); - ret.append(dir); + if (dir.startsWith("/") && dir.length() != 1) { + ret.append(dir.substring(1)); + } + else if (dir.length() != 0) { + ret.append(dir); + } } } catch (Exception e) { From noreply at svn.ci.uchicago.edu Fri Sep 3 21:31:42 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 3 Sep 2010 21:31:42 -0500 (CDT) Subject: [Swift-commit] r3600 - trunk/libexec Message-ID: <20100904023142.898789CC94@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-09-03 21:31:42 -0500 (Fri, 03 Sep 2010) New Revision: 3600 Modified: trunk/libexec/vdl-int-staging.k Log: no need to create the input file dirs since by the time we get there, the input files should already be staged in in those directories Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-09-04 02:25:41 UTC (rev 3599) +++ trunk/libexec/vdl-int-staging.k 2010-09-04 02:31:42 UTC (rev 3600) @@ -27,15 +27,6 @@ pathnames(stageins) ) - element(fileDirs, [stageins, stageouts] - list( - unique( - inFileDirs(stageins) - outFileDirs(stageouts) - ) - ) - ) - element(graphStuff, [tr, stagein, stageout, err, optional(args)] if( vdl:configProperty("pgraph") != "false" then( @@ -89,10 +80,8 @@ jobdir := substring(uid, from=0, to=1) jobid := "{tr}-{uid}" - fileDirs := fileDirs(stagein, stageout) - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") - + wrapper := "_swiftwrap.staging" wrapfile := "{ddir}/param-{jobid}" @@ -119,7 +108,7 @@ "-out", stdout, "-err", stderr, "-i", maybe(stdin), - "-d", flatten(each(fileDirs)), + "-d", flatten(unique(outFileDirs(stageout))), "-if", flatten(infiles(stagein)), "-of", flatten(outfiles(stageout)), "-k", @@ -130,7 +119,8 @@ directory = "{wfdir}-{jobdir}-{jobid}" redirect = false host = rhost - vdl:tcprofile(rhost, tr = tr) //this gets various app params from the tc, such as environment, walltime, etc + + vdl:tcprofile(rhost, tr = tr) //this gets various app params from the tc, such as environment, walltime, etc replicationGroup = replicationGroup replicationChannel = replicationChannel jobid = jobid From noreply at svn.ci.uchicago.edu Sat Sep 4 19:47:13 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Sep 2010 19:47:13 -0500 (CDT) Subject: [Swift-commit] r3601 - SwiftApps/SwiftR/Swift/tests Message-ID: <20100905004713.4A3A39CD30@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-04 19:47:13 -0500 (Sat, 04 Sep 2010) New Revision: 3601 Modified: SwiftApps/SwiftR/Swift/tests/TestSwift.R Log: Considerable reworking of the tests, into their first really useful state. Modified: SwiftApps/SwiftR/Swift/tests/TestSwift.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-09-04 02:31:42 UTC (rev 3600) +++ SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-09-05 00:47:13 UTC (rev 3601) @@ -1,41 +1,284 @@ -require(boot) -#source("Swift.R") + +##### Setup + require(Swift) -sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } +cat("\n*** Starting test group 1 - functions on simple data structures ***\n\n") -args=list(ducks,dogs) -arglist = rep(list(args),9) +##### Test 1.1 -### -require("boot") -sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } +cat("\n*** Starting test 1.1 ***\n\n") -args=list(ducks,dogs) -arglist = rep(list(args),9) -res = swiftapply(sumcrits,arglist) +sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } +data(cars) +data(trees) -res = swiftapply(sumcrits,arglist,callsperbatch=10) +args=list(trees,cars) +arglist = rep(list(args),1) -res = swiftapply(sumcrits,arglist,callsperbatch=2) +cat("Test of local do.call(sumstuff)\n") +localres = do.call(sumstuff,args) +cat("local result=\n") +print(localres) -res = swiftapply(sumcrits,arglist,callsperbatch=3) +cat("\nTest of swiftapply(sumstuff,arglist)\n") +swiftres = swiftapply(sumstuff,arglist) +cat("Swift result:\n") +print(swiftres) -# res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs") -### +if(identical(localres,swiftres[[1]])) { + cat("\n==> test 1.1 passed\n") +} else { + cat("\n==> test 1.1 FAILED !!!!!\n") +} -if(TRUE) { # Basic tests +##### Test 1.2 - res = do.call(sumcrits,args) - cat("Test of do.call(sumcrits)\n") - print(res) +# test 10 remote calls - cat("\nTest of swiftapply(sumcrits,arglist)\n") - res = swiftapply(sumcrits,arglist) - print(res) +sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } +data(cars) +data(trees) + +args=list(trees,cars) +arglist <- rep(list(args),10) + +cat("\n*** Test 1.2.1: 10 calls to substuff()\n") +swiftres <- swiftapply(sumstuff,arglist) +cat("Swift result:\n") +format(swiftres) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i,swiftres[[i]])) + } } +if(diffs == 0) { + cat("\n==> test 1.2.1 passed\n") +} else { + cat("\n!!!==> test 1.2.1 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +cat("\n*** Test 1.2.2: 10 calls to substuff() - callsperbatch=10\n") +swiftres = swiftapply(sumstuff,arglist,callsperbatch=10) +cat("Swift result:\n") +format(swiftres) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i,swiftres[[i]])) + } +} + +if(diffs == 0) { + cat("\n==> test 1.2.2 passed\n") +} else { + cat("\n!!!==> test 1.2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +cat("\n*** Test 1.2.3: 10 calls to substuff() - callsperbatch=2\n") +swiftres = swiftapply(sumstuff,arglist,callsperbatch=2) +cat("Swift result:\n") +format(swiftres) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i,swiftres[[i]])) + } +} + +if(diffs == 0) { + cat("\n==> test 1.2.3 passed\n") +} else { + cat("\n!!!==> test 1.2.3 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +cat("\n*** Test 1.2.4: 10 calls to substuff() - callsperbatch=3\n") +swiftres = swiftapply(sumstuff,arglist,callsperbatch=3) +swiftres <- swiftapply(sumstuff,arglist) +cat("Swift result:\n") +format(swiftres) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !identical(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i,swiftres[[i]])) + } +} + +if(diffs == 0) { + cat("\n==> test 1.2.4 passed\n") +} else { + cat("\n!!!==> test 1.2.4 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +# swiftres = swiftapply(sumstuff,arglist,callsperbatch=2,site="pbs") +# test variations on local vs ssh vs pbs; coasters vs non; etc. + + + +##### Test Group 2 + +cat("\n*** Starting test group 2 - test matrix passing***\n") + +matfunc <- function( m1, m2 ) +{ + (1/m1) %*% m2 +} + +n <- 5 +m1 <- array(sin(1:n**2), dim=c(n,n)) +m2 <- t(m1) + +localres = matfunc(m1,m2) + +cat("\n*** Test 2.1: 100 calls to matfunc(dim=5x5) - callsperbatch=9\n") + +args=list(m1,m2) +arglist <- rep(list(args),100) + +swiftres = swiftapply(matfunc,arglist,callsperbatch=9) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 2.1 passed\n") +} else { + cat("\n!!!==> test 2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +n <- 237 +m1 <- array(sin(1:n**2), dim=c(n,n)) +m2 <- t(m1) + +localres = matfunc(m1,m2) + +cat("\n*** Test 2.2: 123 calls to matfunc(dim=500x500) - callsperbatch=7\n") + +args=list(m1,m2) +arglist <- rep(list(args),123) + +swiftres = swiftapply(matfunc,arglist,callsperbatch=7) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 2.2 passed\n") +} else { + cat("\n!!!==> test 2.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + + +##### Test Group 3 + +cat("\n*** Starting test group 3 - test list element and name passing***\n") + +# Test if list element names are being sent and returned correctly + +n <- 5 +m1 <- array(sin(1:n**2), dim=c(n,n)) +m2 <- t(m1) + +inlist = list() +inlist[[1]]=123 +inlist[[2]]=456 +inlist$name1=789 +inlist$name2=987 +inlist$mat1 = m1 +inlist[[99]] = m2 + +listfunc <- function(ilist) +{ + olist = ilist + olist$sum = ilist[[1]] + ilist[[2]] + ilist$name1 + ilist$name2 + olist$names = names(ilist) + olist$mprod = ilist$mat1 %*% ilist[[99]] + return(olist) +} +localres = listfunc(inlist) + +cat("\n*** Starting test 3.1 - 4 calls in one batch of 5 ***\n") + +args=list(inlist) +arglist <- rep(list(args),4) + +swiftres = swiftapply(listfunc,arglist,callsperbatch=5) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 3.1 passed\n") +} else { + cat("\n!!!==> test 3.1 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + +cat("\n*** Starting test 3.2 - 99 calls in batches of 11 ***\n") + +args=list(inlist) +arglist <- rep(list(args),99) + +swiftres = swiftapply(listfunc,arglist,callsperbatch=11) + +diffs <- 0 +for(i in 1:length(swiftres) ) { + if( !all.equal(swiftres[[i]],localres) ) { + diffs <- diffs + 1 + if( diffs < 10 ) cat(sprintf("res[%d]=%d\n",i,format(swiftres[[i]]))) + } +} + +if(diffs == 0) { + cat("\n==> test 3.2 passed\n") +} else { + cat("\n!!!==> test 3.2 failed.\n") + cat(sprintf(" %d result elements failed to match.\n",diffs)); + failures=failures+1 +} + + + + if(FALSE) { # Test various batch sizes cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") From noreply at svn.ci.uchicago.edu Sat Sep 4 21:38:34 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Sep 2010 21:38:34 -0500 (CDT) Subject: [Swift-commit] r3602 - in trunk: . bin etc etc/certificate Message-ID: <20100905023834.387949CD31@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-09-04 21:38:34 -0500 (Sat, 04 Sep 2010) New Revision: 3602 Added: trunk/etc/certificate/ trunk/etc/certificate/b0cb24a8.0 trunk/etc/certificate/b0cb24a8.signing_policy trunk/etc/certificate/swift.usercert.pem trunk/etc/certificate/swift.userkey.pem trunk/etc/certificate/x509up_swift Modified: trunk/bin/swiftrun trunk/build.xml Log: Dummy certificate for coasters. Swiftrun will set appropriate variables when -certificate is specified. Modified: trunk/bin/swiftrun =================================================================== --- trunk/bin/swiftrun 2010-09-05 00:47:13 UTC (rev 3601) +++ trunk/bin/swiftrun 2010-09-05 02:38:34 UTC (rev 3602) @@ -14,10 +14,11 @@ use SwiftConfig; # Command line processing -my @option_sites; # Site name to execute upon -my $option_script; # Script name to run -my @option_inputs; # Name of data to include with execution -my @option_groups; # Name of group/groups to run script on +my @option_sites; # Site name to execute upon +my $option_script; # Script name to run +my @option_inputs; # Name of data to include with execution +my @option_groups; # Name of group/groups to run script on +my $option_certificate; # Use a default certificate instead of your own GetOptions( 'site|sites=s{,}' => \@option_sites, @@ -25,6 +26,7 @@ 'input|inputs=s{,}' => \@option_inputs, 'help|man|h' => sub { pod2usage(-verbose => 2); }, 'group|groups=s{,}' => \@option_groups, + 'certificate' => \$option_certificate, ); # Check arguments @@ -152,6 +154,14 @@ || die "Unable to create symlink to $new_path\n"; } +# Handle certificates if needed +if($option_certificate) { + $ENV{'X509_CERT_DIR'} = "$FindBin::Bin/../etc/certificate"; + $ENV{'X509_CADIR'} = "$FindBin::Bin/../etc/certificate"; + $ENV{'X509_USER_PROXY'} = "$FindBin::Bin/../etc/certificate/x509up_swift"; +} + + # Run swift chdir("$run_directory") || die "Unable to chdir to $run_directory\n"; print "Executing script $option_script\n"; Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2010-09-05 00:47:13 UTC (rev 3601) +++ trunk/build.xml 2010-09-05 02:38:34 UTC (rev 3602) @@ -223,6 +223,9 @@ + + + Added: trunk/etc/certificate/b0cb24a8.0 =================================================================== --- trunk/etc/certificate/b0cb24a8.0 (rev 0) +++ trunk/etc/certificate/b0cb24a8.0 2010-09-05 02:38:34 UTC (rev 3602) @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIICgTCCAeqgAwIBAgIJANcvv4SNybEAMA0GCSqGSIb3DQEBBQUAMF8xDTALBgNV +BAoTBEdyaWQxEzARBgNVBAsTCkdsb2J1c1Rlc3QxHjAcBgNVBAsTFXNpbXBsZUNB +LXN3aWZ0LWxhcHRvcDEZMBcGA1UEAxMQR2xvYnVzIFNpbXBsZSBDQTAeFw0xMDA4 +MzExOTQxMzRaFw0xNTA4MzAxOTQxMzRaMF8xDTALBgNVBAoTBEdyaWQxEzARBgNV +BAsTCkdsb2J1c1Rlc3QxHjAcBgNVBAsTFXNpbXBsZUNBLXN3aWZ0LWxhcHRvcDEZ +MBcGA1UEAxMQR2xvYnVzIFNpbXBsZSBDQTCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEAsx+GyaNvx9mH8UZRgM4jgQq9h3Us8qJD+/AgsbRB3JPF3M+aKGVnrcfP +pBOo9+ykk5zCF6X7hS9BtJnhniaYQLBoHFE1YQqNKoP4+J77KmmkZYAYjJfGvyBn +Ac2SAqR+7l25LmIGL+Sg3+ykPqFEhTXtGZEgUDYYPRZuePLLpbUCAwEAAaNFMEMw +DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUjd4qyt3WgOqh65/Tkq40wtJMGsAw +EQYJYIZIAYb4QgEBBAQDAgAHMA0GCSqGSIb3DQEBBQUAA4GBAA4YawTmBK58kESD +QtaNmoIRlw3UCW6gyACJlwbrUts5W6j7VA+MiBOFaLNbTf5XZA8cKYqD7xa/05FQ +TLGWd6iFhx65sAj6/ravGxrepUWAqwmTyvFvDoUmlKAuQs8GftsYOcvmnBTzxtQt +AkaWrwmGFR7a12k4MjXygJt1Ggy3 +-----END CERTIFICATE----- Added: trunk/etc/certificate/b0cb24a8.signing_policy =================================================================== --- trunk/etc/certificate/b0cb24a8.signing_policy (rev 0) +++ trunk/etc/certificate/b0cb24a8.signing_policy 2010-09-05 02:38:34 UTC (rev 3602) @@ -0,0 +1,33 @@ +# ca-signing-policy.conf, see ca-signing-policy.doc for more information +# +# This is the configuration file describing the policy for what CAs are +# allowed to sign whoses certificates. +# +# This file is parsed from start to finish with a given CA and subject +# name. +# subject names may include the following wildcard characters: +# * Matches any number of characters. +# ? Matches any single character. +# +# CA names must be specified (no wildcards). Names containing whitespaces +# must be included in single quotes, e.g. 'Certification Authority'. +# Names must not contain new line symbols. +# The value of condition attribute is represented as a set of regular +# expressions. Each regular expression must be included in double quotes. +# +# This policy file dictates the following policy: +# -The Globus CA can sign Globus certificates +# +# Format: +#------------------------------------------------------------------------ +# token type | def.authority | value +#--------------|---------------|----------------------------------------- +# EACL entry #1| + + access_id_CA X509 '/O=Grid/OU=GlobusTest/OU=simpleCA-swift-laptop/CN=Globus Simple CA' + + pos_rights globus CA:sign + + cond_subjects globus '"/O=Grid/OU=GlobusTest/OU=simpleCA-swift-laptop/*"' + +# end of EACL Added: trunk/etc/certificate/swift.usercert.pem =================================================================== --- trunk/etc/certificate/swift.usercert.pem (rev 0) +++ trunk/etc/certificate/swift.usercert.pem 2010-09-05 02:38:34 UTC (rev 3602) @@ -0,0 +1,51 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 1 (0x1) + Signature Algorithm: sha1WithRSAEncryption + Issuer: O=Grid, OU=GlobusTest, OU=simpleCA-swift-laptop, CN=Globus Simple CA + Validity + Not Before: Aug 31 20:07:52 2010 GMT + Not After : Aug 31 20:07:52 2011 GMT + Subject: O=Grid, OU=GlobusTest, OU=simpleCA-swift-laptop, CN=Swift Laptop + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public Key: (1024 bit) + Modulus (1024 bit): + 00:c1:9a:15:08:a5:47:f1:05:04:cc:26:a5:a9:58: + 24:40:67:04:27:45:05:2e:84:34:16:96:1a:b5:0a: + 06:15:a1:fd:e4:c0:ef:aa:d5:8a:6f:3f:ba:64:ab: + 9c:dd:e7:9e:73:23:5a:f3:64:cb:d3:14:a8:d4:3e: + 65:68:ff:75:60:c7:a4:6a:f2:24:09:81:67:59:8c: + 71:64:12:28:be:14:e6:00:fe:7a:ea:56:a4:65:91: + af:31:5c:f1:34:2a:22:0c:63:57:1f:bc:f8:85:b5: + 08:5c:41:70:6c:d2:7b:d4:55:7d:b3:f8:20:c4:91: + 57:17:f5:5c:fe:87:a0:bc:a1 + Exponent: 65537 (0x10001) + X509v3 extensions: + Netscape Cert Type: + SSL Client, SSL Server, S/MIME, Object Signing + Signature Algorithm: sha1WithRSAEncryption + 4e:23:cd:8c:81:1b:2c:b2:3a:41:ba:7a:ee:83:a7:6a:24:ae: + 28:74:25:fb:58:8e:8e:7c:ed:f0:43:a8:11:c4:1c:06:f7:e1: + 1b:f0:cf:24:64:2c:93:ab:da:a3:93:a4:01:06:b6:33:ec:96: + dd:92:f3:1c:c3:67:98:56:f2:ea:da:0d:3c:6c:b8:83:da:3b: + 46:74:64:56:a5:e6:65:5e:c4:9c:5b:7a:fc:f3:1b:46:13:2b: + bc:76:32:c1:80:08:90:f1:c2:87:95:5c:a6:37:da:ae:d2:38: + 6c:35:d7:be:e2:92:fe:16:a9:a0:2f:b3:49:f4:8d:82:00:16: + 03:fb +-----BEGIN CERTIFICATE----- +MIICRTCCAa6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBfMQ0wCwYDVQQKEwRHcmlk +MRMwEQYDVQQLEwpHbG9idXNUZXN0MR4wHAYDVQQLExVzaW1wbGVDQS1zd2lmdC1s +YXB0b3AxGTAXBgNVBAMTEEdsb2J1cyBTaW1wbGUgQ0EwHhcNMTAwODMxMjAwNzUy +WhcNMTEwODMxMjAwNzUyWjBbMQ0wCwYDVQQKEwRHcmlkMRMwEQYDVQQLEwpHbG9i +dXNUZXN0MR4wHAYDVQQLExVzaW1wbGVDQS1zd2lmdC1sYXB0b3AxFTATBgNVBAMT +DFN3aWZ0IExhcHRvcDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwZoVCKVH +8QUEzCalqVgkQGcEJ0UFLoQ0FpYatQoGFaH95MDvqtWKbz+6ZKuc3eeecyNa82TL +0xSo1D5laP91YMekavIkCYFnWYxxZBIovhTmAP566lakZZGvMVzxNCoiDGNXH7z4 +hbUIXEFwbNJ71FV9s/ggxJFXF/Vc/oegvKECAwEAAaMVMBMwEQYJYIZIAYb4QgEB +BAQDAgTwMA0GCSqGSIb3DQEBBQUAA4GBAE4jzYyBGyyyOkG6eu6Dp2okrih0JftY +jo587fBDqBHEHAb34RvwzyRkLJOr2qOTpAEGtjPslt2S8xzDZ5hW8uraDTxsuIPa +O0Z0ZFal5mVexJxbevzzG0YTK7x2MsGACJDxwoeVXKY32q7SOGw1177ikv4WqaAv +s0n0jYIAFgP7 +-----END CERTIFICATE----- Added: trunk/etc/certificate/swift.userkey.pem =================================================================== --- trunk/etc/certificate/swift.userkey.pem (rev 0) +++ trunk/etc/certificate/swift.userkey.pem 2010-09-05 02:38:34 UTC (rev 3602) @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,55AAA5E005750907 + +SD3JZeEdG1iLwFbFmScMZcLncTp4iTuNsNodF9jrHw5SenkE0l6VMJPwaW9eqF6c +U/BzY1maxuf6cU0zjldBe75OzuEzWygx4v/yNVjw2zyBPuhe5l+xc1RLhXSNzlP6 +10F5jmu+fBdMMTqbRGTPSaWMWc36RQrsypBt0DWIO3kc7Lgf+FUix3dKnHnCrT9b +lcaZhbZcCzlMl6UDWxqErerkNYoCo8i9n2RYMo8mvFOn7toeWzHOEXB+mSh73mi/ +goBeV2ezDPF5GJgeBaDv91jbzj5riy3Cnigvo9ClHwWAbG/VSdMgsB65/a1YjdBO +4vA297LduALKMwENfEBaKiQ0FBlkUsQt+c0QRA9TOov3MsiASZzTdagKx2Obb8EG +d26XXFA/DX7slZiJgQ7kFo0wmQsEuIkCE9hn1mQHud42u4WZptagvWzCQ5Ik8wtr +qAacDlcAMuG4HUHtpOQT8ucpKnIRGEOvc7FKz03BGrGttY+l+eJf6i+nKFyRh3KM +RnJ2GtSboZhE4ptnWDbgDdzxnwqJT3c9PQUbDMIwgII/licvJsIZjwSrzzVfY+HA +UtCwt3safnxXNt55MVw7+42pwXM5gfrFDcMetsCZYitXp8hulmUV4viDrJyYVyPh +UGmJ0qTjwiVBRiVLYtTYtvixSHZJtRWISQR5PQkAO+xAULpkDxTtZ36rm82sqKU3 +ej+LO16iFlzZIxX2j+Z5vcSn28nsJzFHffNQCs7j4AAx9oTSLJOyTzJc4ZAw5L/x +KnXesrYfkjonYcxTBDMr7VrLqSPR8frIetF3SK3ppysChppGA82tig== +-----END RSA PRIVATE KEY----- Added: trunk/etc/certificate/x509up_swift =================================================================== --- trunk/etc/certificate/x509up_swift (rev 0) +++ trunk/etc/certificate/x509up_swift 2010-09-05 02:38:34 UTC (rev 3602) @@ -0,0 +1,38 @@ +-----BEGIN CERTIFICATE----- +MIICITCCAYqgAwIBAgIEc5HGqjANBgkqhkiG9w0BAQQFADBbMQ0wCwYDVQQKEwRH +cmlkMRMwEQYDVQQLEwpHbG9idXNUZXN0MR4wHAYDVQQLExVzaW1wbGVDQS1zd2lm +dC1sYXB0b3AxFTATBgNVBAMTDFN3aWZ0IExhcHRvcDAeFw0xMDA4MzEyMDM5NDVa +Fw0xMDA5MTAyMDQ0NDVaMHAxDTALBgNVBAoTBEdyaWQxEzARBgNVBAsTCkdsb2J1 +c1Rlc3QxHjAcBgNVBAsTFXNpbXBsZUNBLXN3aWZ0LWxhcHRvcDEVMBMGA1UEAxMM +U3dpZnQgTGFwdG9wMRMwEQYDVQQDEwoxOTM4OTMzNDE4MFwwDQYJKoZIhvcNAQEB +BQADSwAwSAJBAOAcIkT0dPXb40p+z7NI4gPOEKx4wAFpuFe4vz6qPvtXlv0rdLLb +8uiHycAg3mf6tIAFRA8MA5WAmafoNG4oNnsCAwEAAaMhMB8wHQYIKwYBBQUHAQ4B +Af8EDjAMMAoGCCsGAQUFBxUBMA0GCSqGSIb3DQEBBAUAA4GBAFHJrxlgNwg+W7tA +Lqp/eG1FK7AJJDNMY6uuPN7G8YuTAL0nPZPuRjRJQEJK1ntXosU2oCn/SXjxr4km +IGnKiWtvQWFyqzLLsitvcaynD4mBGjAk/GDa9+B+c+Kpy1R+E+z680OXXZlS9rID +vqj20EfRfRvm/QB62dubanJElCOG +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIBOgIBAAJBAOAcIkT0dPXb40p+z7NI4gPOEKx4wAFpuFe4vz6qPvtXlv0rdLLb +8uiHycAg3mf6tIAFRA8MA5WAmafoNG4oNnsCAwEAAQJBAIVe+N4EhV6Tn66DzWOq +6ibNiYq6ybmNixFZdyb59kVo+2alRnqHVL+GQX1eNSqWRRXbr7Yhs9VMbAWVuM4G +rIECIQD+tDASzbqqEUHAGorOuo7lQtit7a3mY0PTzIxHqSMCIQIhAOFAFxZVkY5R +vzK5r7jYwkGoXle1ypZKiOdlzcBLkV0bAiAyaSFWp2c5f77esaIldmEP2SIfknon +rh/fjI17JWurYQIgQwnhFSuYa4QW5JDxJJGWlxX2JiKbzckXH89jwhR7ifsCIBAd +DtI2chvsfpQdhXtPBVNUN4NQzmSS+5FqbZpsTDxQ +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICRTCCAa6gAwIBAgIBATANBgkqhkiG9w0BAQUFADBfMQ0wCwYDVQQKEwRHcmlk +MRMwEQYDVQQLEwpHbG9idXNUZXN0MR4wHAYDVQQLExVzaW1wbGVDQS1zd2lmdC1s +YXB0b3AxGTAXBgNVBAMTEEdsb2J1cyBTaW1wbGUgQ0EwHhcNMTAwODMxMjAwNzUy +WhcNMTEwODMxMjAwNzUyWjBbMQ0wCwYDVQQKEwRHcmlkMRMwEQYDVQQLEwpHbG9i +dXNUZXN0MR4wHAYDVQQLExVzaW1wbGVDQS1zd2lmdC1sYXB0b3AxFTATBgNVBAMT +DFN3aWZ0IExhcHRvcDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwZoVCKVH +8QUEzCalqVgkQGcEJ0UFLoQ0FpYatQoGFaH95MDvqtWKbz+6ZKuc3eeecyNa82TL +0xSo1D5laP91YMekavIkCYFnWYxxZBIovhTmAP566lakZZGvMVzxNCoiDGNXH7z4 +hbUIXEFwbNJ71FV9s/ggxJFXF/Vc/oegvKECAwEAAaMVMBMwEQYJYIZIAYb4QgEB +BAQDAgTwMA0GCSqGSIb3DQEBBQUAA4GBAE4jzYyBGyyyOkG6eu6Dp2okrih0JftY +jo587fBDqBHEHAb34RvwzyRkLJOr2qOTpAEGtjPslt2S8xzDZ5hW8uraDTxsuIPa +O0Z0ZFal5mVexJxbevzzG0YTK7x2MsGACJDxwoeVXKY32q7SOGw1177ikv4WqaAv +s0n0jYIAFgP7 +-----END CERTIFICATE----- From noreply at svn.ci.uchicago.edu Tue Sep 7 11:48:43 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 11:48:43 -0500 (CDT) Subject: [Swift-commit] r3603 - SwiftApps/SwiftR/Swift/exec Message-ID: <20100907164843.545DF9CC80@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 11:48:43 -0500 (Tue, 07 Sep 2010) New Revision: 3603 Modified: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh Log: Move swift script to the run dir. Modified: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-09-05 02:38:34 UTC (rev 3602) +++ SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-09-07 16:48:43 UTC (rev 3603) @@ -3,10 +3,19 @@ script=$3 runR=$4 +cp $script $rundir + +# FIXME: rework this script to transfer all shells and rscripts +# needed, and to copy in the R prelude for the R server processes (to +# incude for example the OpenMx library) + cd $rundir +# FIXME: how to get the right PATH for R in here? +# use /bin/sh and count on users PATH? +# then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename) + cat >tc < EOF +script=$(basename $script) swift -tc.file tc -sites.file sites.xml $script From noreply at svn.ci.uchicago.edu Tue Sep 7 11:50:26 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 11:50:26 -0500 (CDT) Subject: [Swift-commit] r3604 - SwiftApps/SwiftR/Swift/exec Message-ID: <20100907165026.873DC9CC80@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 11:50:26 -0500 (Tue, 07 Sep 2010) New Revision: 3604 Modified: SwiftApps/SwiftR/Swift/exec/swiftapply.swift Log: Return stdout and stderr from R batch execution back to user. Modified: SwiftApps/SwiftR/Swift/exec/swiftapply.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-09-07 16:48:43 UTC (rev 3603) +++ SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-09-07 16:50:26 UTC (rev 3604) @@ -1,13 +1,16 @@ +type file; type RFile; -app (RFile result) RunR (RFile rcall) +app (RFile result, file stout, file sterr) RunR (RFile rcall) { - RunR @rcall @result; + RunR @rcall @result stdout=@stout stderr=@sterr; } RFile rcalls[] ; RFile results[] ; +file stout[] ; +file sterr[] ; foreach c, i in rcalls { - results[i] = RunR(c); + (results[i],stout[i], sterr[i]) = RunR(c); } From noreply at svn.ci.uchicago.edu Tue Sep 7 11:54:44 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 11:54:44 -0500 (CDT) Subject: [Swift-commit] r3605 - SwiftApps/SwiftR/Swift/exec Message-ID: <20100907165444.73D849CC80@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 11:54:44 -0500 (Tue, 07 Sep 2010) New Revision: 3605 Added: SwiftApps/SwiftR/Swift/exec/RunAllR.sh Log: Add script to manually execute Swift call batches for manualmode processing. Added: SwiftApps/SwiftR/Swift/exec/RunAllR.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/RunAllR.sh (rev 0) +++ SwiftApps/SwiftR/Swift/exec/RunAllR.sh 2010-09-07 16:54:44 UTC (rev 3605) @@ -0,0 +1,24 @@ +#! /usr/bin/env Rscript + +# Use this script to run batches manually for testing. See "manualmode" notes in R/Swift.R. + +require(OpenMx) + +argv = commandArgs(TRUE) + +# load(argv[1]); + +for(infile in argv) { + cat("Processing file: ", infile, "\n") + rcall <- NULL # Needed? + load(infile) + ofile <- sub('^c','r',infile) + result <- list() + cat("Number of calls: ", length(rcall$arglistbatch),"\n") + for(c in 1:length(rcall$arglistbatch)) { + # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) + cat("Running call # ", c, "\n"); + result[[c]] <- do.call( rcall$func, rcall$arglistbatch[[c]] ) + } + save(result,file=ofile) +} Property changes on: SwiftApps/SwiftR/Swift/exec/RunAllR.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Tue Sep 7 11:59:21 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 11:59:21 -0500 (CDT) Subject: [Swift-commit] r3606 - SwiftApps/SwiftR Message-ID: <20100907165921.360699CC80@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 11:59:21 -0500 (Tue, 07 Sep 2010) New Revision: 3606 Added: SwiftApps/SwiftR/README Modified: SwiftApps/SwiftR/TODO Log: Update notes. Added: SwiftApps/SwiftR/README =================================================================== --- SwiftApps/SwiftR/README (rev 0) +++ SwiftApps/SwiftR/README 2010-09-07 16:59:21 UTC (rev 3606) @@ -0,0 +1,132 @@ +TIPS AND REMINDERS FOR THE FORGETFUL DEVELOPER :) + + +* NOTES on where everything lives: + +Am testing on PADS + + ~/SwiftR is my "project" main working dir + +R is under ~/R and ~/R/pads (compiled for PADS; the ~/R/bin/R +executable gets a library error on pads) + +To build R: + http://cran.r-project.org/src/base/R-2/R-2.11.1.tar.gz (or desired release) + tar zxf *gz + cd ddd + ./configure --prefix=/home/wilde/R/R-2.11.0 + make + make install + +Need to put the right R in the PATH for building and testing. +Using R 2.11 as of 2010.0903 + +OpenMx source tree checked out under: ~/SwiftR/OpenMx + +Repo is: https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SwiftR + +R packages are installed under: ~/RPackages +(These are the release-independent packages? +FIXME: how does this relate to packages under the current R relesae? + +#----- +bri$ pwd +/home/wilde/R/R-2.11.0 +bri$ cd library/ +bri$ pwd +/home/wilde/R/R-2.11.0/library +bri$ ls +KernSmooth/ OpenMx/ base/ cluster/ foreign/ grid/ mgcv/ rpart/ stats/ tcltk/ +MASS/ R.css boot/ codetools/ grDevices/ lattice/ nlme/ spatial/ stats4/ tools/ +Matrix/ Swift/ class/ datasets/ graphics/ methods/ nnet/ splines/ survival/ utils/ +bri$ +#----- + +* Note: dont yet know if we do or do not need separate compiles between +communicado, bridled, pads. teraport, and other systems on the CI +net. Hopefully not; if we do, weill need to create a tree to R +releases each with a separate subtree for user-installed packages) +Seems that we do, at least for PADS, we get this error: + +login1$ ~/R/bin/R +/home/wilde/R/lib64/R/bin/exec/R: + error while loading shared libraries: libreadline.so.4: + cannot open shared object file: No such file or directory +login1$ + +*-- + +-- Swift package *source* (tbd) is under: + + ~/SwiftR/Swift (the "Swift" package) + +-- Swift package is installed under: ???????? + +* Setup + +Get latest R in path for installs etc. + + PATH=/home/wilde/R/R-2.11.0/bin:$PATH + +* Fetch, Build, Install OpenMx + +* Make Swift Package + + +FIXME: document R CMD BUILD and R CMD INSTALL here + + + cd ~/SwiftR/Swift? + +* How to build swift package into a source archive: + + R CMD build Swift + +* package can also be fetched from svn abd built/installed from source I think? Isnt this how OpenMx itself works? + +* How to build swift package right into source code + +* How to test the package + + R CMD check Swift + +* Install Swift Package + + R CMD INSTALL Swift # Puts package under current R release: + + $ R CMD INSTALL Swift | tee INSTALL.out + +#--- Example: + +login1$ R CMD INSTALL Swift +* installing to library '/home/wilde/R/R-2.11.0/library' +* installing *source* package 'Swift' ... +...etc + +#--- + + R CMD INSTALL -l ~/RLibrary Swift # Puts package under user lib; then need to add this to R search path? + R CMD INSTALL Swift_0.1.tar.gz # Installs from a pre-built source archive instead of from Swift/ source dir + +* Run tests Swift package tests manually + +$ cd ~/SwiftR/Swift +$ R +> require(Swift) +> source("tests/TestSwift.R") + +################## INTERNAL DESIGN LOGIC + +the core R function is switapply() + +Its a bit more general then the lApply/sApply/etc functions: + +For each call it specfies a full set of arguments + +lApply calls swiftapply + +in swiftapply the remote calls are batched + +option processing: + + Modified: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO 2010-09-07 16:54:44 UTC (rev 3605) +++ SwiftApps/SwiftR/TODO 2010-09-07 16:59:21 UTC (rev 3606) @@ -1,73 +1,138 @@ +*** TO DO LIST: -*** NOTES on where everything lives: -Am testing on PADS -~/SwiftR is my "project" main working dir + need persistent coaster service -R is under ~/R and ~/R/pads (compiled for PADS; the ~/R/bin/R executable gets a library error on pads) -OpenMx source tree checked out under: ~/SwiftR/OpenMx + output issues from swift worker to clean up (see bottom of this file): + "Directory no empty" worker.pl line 1001 + unlink messages -R packages are installed under: ~/RPackages + clean up of R procs on remote side --- -(Note: dont yet know if we do or do not need separate compiles between -communicado, bridles, pads. teraport, and other systems on the CI -net. Hopefully nit; if we do, weill need to create a tree to R -releases each with a separate subtree for user-installed packages) -Seems that we do, at least for PADS, we get this error: +[] debug version + create version of swiftapply etc that runs by saving file, printing + message, and waiting for user to do "swiftrun" in a separate + window. This will mimic the exec path and enable a developer to debug + serially, and under a debugger, on both the client and server side. + --> did i do this already? -login1$ ~/R/bin/R -/home/wilde/R/lib64/R/bin/exec/R: - error while loading shared libraries: libreadline.so.4: - cannot open shared object file: No such file or directory -login1$ +x check snow/fall code to see if we are missing anything for + transparent remote exec + (This was list propoerties; fixed now; anything else?) --- +x n args +x batch +x into svn +x unique dirs + clean up source code formatting: Use OpenMx conventions? + (spaces, tabbing?) -Swift package *source* (tbd) is under: +For OpenMx 1.0: -~/SwiftR/Swift (the "Swift" package) + coasters for persistent R Servers + handle entire apply family + determine if we should improve the way we handle options (ask OpenMx team) -Swift package is installed under: + use match.fun ( ??? ) + ensure (and test) that all expected object attributes are passed and preserved on inputs and outputs (names, dims, etc) + capture stdout/err, perhaps with length limits, into R return vars + control over Swift properties; set defaults to "no retries": + Progress: Active:7 Failed but can retry:2 ==> should not happen + R docs +x R package (SwiftR) + Swift docs + pass the func as val ( ??? ) + pass extra funcs and packages required + pass extra vals + pass extra files + better download and install of Swift (compliant with CRAN regs) + args as alists vs args as list( ??? ) + return good error messages including messages from R eval and from Swift + Name pbs jobs mnemonically + options for cleaning up and/or retaining intermidiate save/load and stdout/err files + tools for displaying what was sent / received in calls (easy to pick apart from retained Rcall and Rret files) +_ select sites and swift args (throttles etc) + Improve tests - see list below + + runids, output logging + select exec sites and swift params etc + make polymorphic to *apply and snow -*** TO DO LIST: +For General R usage: -x n args -x batch -x into svn -x unique dirs -_ select sites and swift args (throttles etc) - R docs - R package (SwiftR) - Swift docs + integration with snowfall/snow? + async exec clean up boot: fix all calls to statistics; update boot package w/ pboot() or swiftboot() error handling and null and missing values: ensure res#s correspond to arg#s status specify swift scripts + specify data files to be passed to remote side + setup the R envs (???) + specify initial R code to run on remote side: (if !initialized_swift) { initialRCodeHere } run async and grab status (track 'runs' in R) increm result collect - pass the func as val - pass extra funcs and packages required - pass extra vals - pass extra files specifiy unique swift scritps ala Dirk's tools - setup the R envs (???) - coasters for persistent R Servers - test suites use littleR - args as alists vs args as list - runids, output logging - select exec sites and swift params etc - make polymorphic to *apply and snow stream results back to R (so use can inspect as they arrive) - (pull them in with a Swift.poll() func) - handle discontiguous results - return good error messages including messages from R eval and from Swift - TUI - Name pbs jobs mnemonically + (pull them in with a Swift.poll() func) + handle discontiguous results (??) and incomplete results (not every element of apply() completes) + TUI (in various forms) + +*** TESTS NEEDED + +more tests +test with better data that makes it easier to cross check the results +dont require boot() for the test datasets +but test (p)boot() if available +add output data comparisons (or other output checks) to all tests +test across a bigger range of types and R weirnesses like local funcs, namespaces, non local assigns, closures, data types etc. +test on matrices +test on big matrixes etc to assure that we have no length issues +tests to ensure all data object attrs passed in and out OK + +test strategy: +- can we part data of arbitrary structure back and forth? +- can we pass functions back and forth? +- do the remote calls work the same regardless of whether they go to an R server or a pre-call R invocation + +flag to control # of tests and degree + +---- + +OUTPUT and SWIFT EXEC ISSUES TO CLEAN UP: + +Issues below was that we created a long-lived process that hung on to open files, preventing jobdir from being removed. + +bri$ ./TestSwiftScript.sh run03 +swift output is in: swift.stdouterr.7124, pids in swift.workerpids.7124 +Swift svn swift-r3591 cog-r2868 (cog modified locally) + +RunID: 20100902-1723-etxm22pg +Progress: +Passive queue processor initialized. Callback URI is http://128.135.125.18:50003 +Coaster contact: http://128.135.125.18:50003 +Started workers from these ssh processes: 27297 +logilename: /home/wilde/SwiftR/run03/swiftworkerlogs/worker-2010.0902.172334.09027.log +Progress: Active:1 +Can't remove directory /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj (Directory not empty) at /home/wilde/swift/lab/worker.pl line 1001 +Final status: Finished successfully:1 +Cleaning up... +Shutting down service at https://128.135.125.18:50002 +Got channel MetaChannel: 570110481[1922304900: {}] -> null[1922304900: {}] ++ Done +unlink /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj/_swiftwrap.staging +unlink /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj/outdir/r.0001.out +rmdir /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj/outdir +unlink /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj/3 +unlink /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj/stdout.txt +unlink /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj/wrapper.log +unlink /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj/cb1 +rmdir /home/wilde/SwiftR/run03/swiftwork/rtest-20100902-1723-etxm22pg-f-runr-f90rr6yj +Terminating worker processes 27297 and starter 27126 +bri$ From noreply at svn.ci.uchicago.edu Tue Sep 7 12:20:51 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 12:20:51 -0500 (CDT) Subject: [Swift-commit] r3607 - SwiftApps/SwiftR/Swift/R Message-ID: <20100907172051.20E309CC80@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 12:20:50 -0500 (Tue, 07 Sep 2010) New Revision: 3607 Modified: SwiftApps/SwiftR/Swift/R/Swift.R Log: Cleaned up argument/option processing. Added stub for specifiying initializer R statement for remote R processes. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-07 16:59:21 UTC (rev 3606) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-07 17:20:50 UTC (rev 3607) @@ -1,49 +1,47 @@ -swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL, manualmode=NULL ) +swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL, manualmode=NULL, initialize=NULL ) { - # Move swiftprops into a Swift namespace + # Set Swift default options if not passed as keyword paramets or pre-set by user -# if(!exists("swiftprops")) { -# swiftprops <<-list() -# swiftprops$site <<- "local" -# swiftprops$callsperbatch <<- 1 -# } - - # Set Swift default options if not pre-set by user - - if (is.null(getOption("swift.site"))) - options(swift.site="local") if (is.null(getOption("swift.callsperbatch"))) options(swift.callsperbatch=1) if (is.null(getOption("swift.manualmode"))) options(swift.manualmode = FALSE) + if (is.null(getOption("swift.initialize"))) + options(swift.initialize = "") -# if(is.null(site)) -# if( is.null(swiftprops) || is.null(swiftprops$site)) -# site <- "local" -# else -# site <- swiftprops$site -# if(is.null(callsperbatch)) -# if( is.null(swiftprops) || is.null(swiftprops$callsperbatch)) -# callsperbatch <- 1 -# else -# Callsperbatch <- swiftprops$callsperbatch - if(is.null(site)) site <- getOption("swift.site") + if(is.null(site)) + site<-"local" + if(is.null(callsperbatch)) callsperbatch <- getOption("swift.callsperbatch") + if(is.null(callsperbatch)) + callsperbatch <- 1 + if(is.null(manualmode)) manualmode <- getOption("swift.manualmode") + if(is.null(manualmode)) + manualmode <- FALSE - cat("\nSwift R properties:\n") + if(is.null(initialize)) + initialExpr <- getOption("swift.initialize") + # else leave NULL + + cat("\nSwift properties:\n") cat(" site =",site,"\n"); - cat(" callsperbatch =",callsperbatch,"\n") + cat(" callsperbatch =", callsperbatch,"\n") + cat(" manualmode =", manualmode,"\n") + cat(" initialize =", initialize,"\n") cat("\nCurrent dir: ",getwd(),"\n"); # Execute the calls in batches rundir <- system("mktemp -d SwiftR.run.XXX",intern=TRUE) cat("Swift running in",rundir,"\n") + if(! is.null(initialize) ) { + cat(initialize,file=paste(rundir,"/initialize.R",sep="")) + } narglists <- length(arglists) # number of arglists to process batch <- 1 # Next arglist batch number to fill arglist <- 1 # Next arglist number to insert From noreply at svn.ci.uchicago.edu Tue Sep 7 12:22:04 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 12:22:04 -0500 (CDT) Subject: [Swift-commit] r3608 - SwiftApps/SwiftR/Swift/man Message-ID: <20100907172204.E91449CC80@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 12:22:04 -0500 (Tue, 07 Sep 2010) New Revision: 3608 Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd Log: Adjusted examples to get test data from data() function rather than boot package. Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-09-07 17:20:50 UTC (rev 3607) +++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-09-07 17:22:04 UTC (rev 3608) @@ -45,18 +45,17 @@ %%} \examples{ -require("boot") -sumcrits <- function(duckdata,dogdata) { sum( duckdata$plumage, dogdata$mvo ) } +myfunc <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } +data(cars) +data(trees) -args=list(ducks,dogs) -arglist = rep(list(args),9) -res = swiftapply(sumcrits,arglist) +args=list(trees,cars) +arglist = rep(list(args),10) # Create 10 calls -res = swiftapply(sumcrits,arglist,callsperbatch=10) +res = swiftapply(myfunc,arglist) -res = swiftapply(sumcrits,arglist,callsperbatch=2) +res = swiftapply(myfunc,arglist,callsperbatch=5) -res = swiftapply(sumcrits,arglist,callsperbatch=3) +# res = swiftapply(myfunc,arglist,callsperbatch=2,site="pbs") -# res = swiftapply(sumcrits,arglist,callsperbatch=2,site="pbs") } From noreply at svn.ci.uchicago.edu Tue Sep 7 14:36:39 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 14:36:39 -0500 (CDT) Subject: [Swift-commit] r3609 - in SwiftApps/SwiftR/Swift: R exec tests Message-ID: <20100907193639.466BD9CC80@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 14:36:39 -0500 (Tue, 07 Sep 2010) New Revision: 3609 Modified: SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh SwiftApps/SwiftR/Swift/exec/swiftapply.swift SwiftApps/SwiftR/Swift/tests/TestSwift.R Log: Added initialize= string option to execute initialization statements in the remote R service. WIth this change we pass the RunR.R script as a Swift aergument and include the initialization statements in that script. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-07 17:22:04 UTC (rev 3608) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-07 19:36:39 UTC (rev 3609) @@ -2,12 +2,14 @@ { # Set Swift default options if not passed as keyword paramets or pre-set by user +if(FALSE){ if (is.null(getOption("swift.callsperbatch"))) options(swift.callsperbatch=1) if (is.null(getOption("swift.manualmode"))) options(swift.manualmode = FALSE) if (is.null(getOption("swift.initialize"))) options(swift.initialize = "") +} # FALSE if(is.null(site)) site <- getOption("swift.site") @@ -25,8 +27,9 @@ manualmode <- FALSE if(is.null(initialize)) - initialExpr <- getOption("swift.initialize") - # else leave NULL + initialize <- getOption("swift.initialize") + if(is.null(initialize)) + initialize <- ""; cat("\nSwift properties:\n") cat(" site =",site,"\n"); @@ -39,9 +42,8 @@ rundir <- system("mktemp -d SwiftR.run.XXX",intern=TRUE) cat("Swift running in",rundir,"\n") - if(! is.null(initialize) ) { - cat(initialize,file=paste(rundir,"/initialize.R",sep="")) - } + cat(initialize,file=paste(rundir,"/initialize.R",sep="")) + narglists <- length(arglists) # number of arglists to process batch <- 1 # Next arglist batch number to fill arglist <- 1 # Next arglist number to insert @@ -64,7 +66,8 @@ } nbatches <- batch - 1 RunSwiftScript <- system.file(package="Swift","exec/RunSwiftScript.sh") - RunRScript <- system.file(package="Swift","exec/RunR.sh") +# RunRScript <- system.file(package="Swift","exec/RunR.sh") +# RunRScript <- system.file(package="Swift","exec/RunR.R") swiftapplyScript <- system.file(package="Swift","exec/swiftapply.swift") if( manualmode ) { # Prompt for return (empty line) to continue; assumes user ran a manual R to process the call. @@ -72,13 +75,14 @@ cat(" swift script: ", RunSwiftScript, "\n") cat(" site: ", site,"\n") cat(" swiftapplyScript: ", swiftapplyScript,"\n") - cat(" RunRScript: ", RunRScript,"\n\n") +# cat(" RunRScript: ", RunRScript,"\n\n") cat(" Use RunAllR.sh to process and press return when complete:") system(paste("cp ", system.file(package="Swift","exec/RunAllR.sh"), rundir)) readLines(n=1) } else { - system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript)) + system(paste(RunSwiftScript,rundir,site,swiftapplyScript)) +# system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript)) # system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript,sep=" ")) } Modified: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-09-07 17:22:04 UTC (rev 3608) +++ SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-09-07 19:36:39 UTC (rev 3609) @@ -1,9 +1,9 @@ rundir=$1 site=$2 script=$3 -runR=$4 +# runR=$4 -cp $script $rundir +cp $script $runR $rundir # FIXME: rework this script to transfer all shells and rscripts # needed, and to copy in the R prelude for the R server processes (to @@ -15,11 +15,12 @@ # use /bin/sh and count on users PATH? # then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename) -cat >tc <tc <sites.xml <sites.xml < @@ -39,7 +40,27 @@ $(pwd) -EOF +END +cat >RunR.R </dev/null) +# End of user initialization statements + +argv = commandArgs(TRUE) + +load(argv[1]); + +result=list() +for(c in 1:length(rcall\$arglistbatch)) { + # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) + result[[c]] = do.call( rcall\$func, rcall\$arglistbatch[[c]] ) +} + +save(result,file=argv[2]) + +END + script=$(basename $script) swift -tc.file tc -sites.file sites.xml $script Modified: SwiftApps/SwiftR/Swift/exec/swiftapply.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-09-07 17:22:04 UTC (rev 3608) +++ SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-09-07 19:36:39 UTC (rev 3609) @@ -1,16 +1,19 @@ type file; -type RFile; +type RData; +type RScript; -app (RFile result, file stout, file sterr) RunR (RFile rcall) +app (RData result, file stout, file sterr) RunR (RScript rscript, RData rcall) { - RunR @rcall @result stdout=@stout stderr=@sterr; + Rscript @rscript @rcall @result stdout=@stout stderr=@sterr; +# RunR @rcall @result stdout=@stout stderr=@sterr; } -RFile rcalls[] ; -RFile results[] ; +RData rcalls[] ; +RData results[] ; file stout[] ; file sterr[] ; +RScript rs <"RunR.R">; foreach c, i in rcalls { - (results[i],stout[i], sterr[i]) = RunR(c); + (results[i],stout[i], sterr[i]) = RunR(rs,c); } Modified: SwiftApps/SwiftR/Swift/tests/TestSwift.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-09-07 17:22:04 UTC (rev 3608) +++ SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-09-07 19:36:39 UTC (rev 3609) @@ -276,9 +276,37 @@ failures=failures+1 } +##### Test Group 4 # test initializer string +cat("\n*** Starting test group 4 - test remote R service initialization string ***\n") +sumivars <- function() { initVar1+initVar2 } +initcmds <- " +initVar1 <- 19 +initVar2 <- sqrt(400)+3 +" + +options(swift.initialize=initcmds) + +args=list() +arglist = rep(list(args),1) + +localres = 42 + +cat("\nTest of swiftapply(sumivars,arglist)\n") +swiftres = swiftapply(sumivars,arglist) +cat("Swift result:\n") +print(swiftres) + +if(identical(localres,swiftres[[1]])) { + cat("\n==> test 4.1 passed\n") +} else { + cat("\n==> test 4.1 FAILED !!!!!\n") +} + + + if(FALSE) { # Test various batch sizes cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") From noreply at svn.ci.uchicago.edu Tue Sep 7 20:01:21 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 20:01:21 -0500 (CDT) Subject: [Swift-commit] r3610 - SwiftApps/SwiftR/Swift/exec Message-ID: <20100908010121.306609CD19@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 20:01:21 -0500 (Tue, 07 Sep 2010) New Revision: 3610 Added: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/passive-coaster-swift Modified: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh SwiftApps/SwiftR/Swift/exec/swiftapply.swift Log: Initial versions of scripts for passive coasters and persisten R eval processes. Added: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh (rev 0) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-09-08 01:01:21 UTC (rev 3610) @@ -0,0 +1,80 @@ +#! /bin/bash + +# Arguments: inputBatchSaveFile outputBatchSaveFile + +# Find our bin dir (to use for running utility scripts) + +SWIFTRBIN=$(cd $(dirname $0); pwd) + +# This script expects to be called (typically from Swift worker.pl) +# with the following env vars set: +# SWIFT_WORKER_PID - process ID of the worker.pl that launched this script +# SWIFT_JOB_SLOT - slot ID of the worker job that launched this script (0..maxWorkers-1) + +# Ensure that the dir for this slot exists. + +BASEDIR=/tmp/SwiftR/fifos +WORKERDIR=$BASEDIR/worker.$SWIFT_WORKER_PID +SLOTDIR=$WORKERDIR/${SWIFT_JOB_SLOT} + +mkdir -p $WORKERDIR + +# Try to make slotdir. +# If the mkdir succeeds, this is the first request to the slot, +# so we create a new R server and send the current request to it; +# else we send the current request to an existing R server. + +RPIDS= + +TRAPS="EXIT 1 2 3 15" + +function onexit { # FIXME: move this logic into worker.pl, or try to use R timeout option if it works on fifos - need to test. + trap - $TRAPS + echo "Terminating R processes $RPIDS" + if [ "_$RPIDS" != _ ]; then + kill $RPIDS >& /dev/null + fi +} + +# trap onexit $TRAPS + +function idletimer { + cd $1 + while true; do + touch idle + sleep 60 + if [ ! -f lastwrite -o \( idle -nt lastwrite \) ]; then + # FIXME: should work, but does not: echo quit x x >toR.fifo # fifo problem? + RPID=$(cat R.pid) + echo killing idle R process $RPID + kill $RPID + exit 0 + fi + done +} + +mkdir $SLOTDIR >& /dev/null +if [ $? = 0 ]; then + mkfifo $SLOTDIR/toR.fifo + mkfifo $SLOTDIR/fromR.fifo + $SWIFTRBIN/SwiftRServer.sh $SLOTDIR >& $SLOTDIR/R.log & # launch R server + idletimer $SLOTDIR /dev/null 2>&1 & # R saves pid in R.pid for idletimer to kill it + echo "$0: INFO: Launched $SWIFTRBIN/SwiftServer.sh $SLOTDIR Rscript" +else + # wait to make sure fifo exists + # fromR fifo is created last, so wait for that one + if [ ! -p $SLOTDIR/fromR.fifo ]; then + sleep 5 # FIXME: try a few times, or wait longer? + if [ ! -p $SLOTDIR/fromR.fifo ]; then + echo "$0: ERROR: FIFO $SLOTDIR/fromR.fifo did not appear within 5 seconds.\n" + exit 1 + fi + fi +fi + +# Ready to talk to the server: send request and read response + +echo run $(pwd)/$1 $(pwd)/$2 > $SLOTDIR/toR.fifo +touch $SLOTDIR/lastwrite + +head -3 < $SLOTDIR/fromR.fifo # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit") Modified: SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-09-07 19:36:39 UTC (rev 3609) +++ SwiftApps/SwiftR/Swift/exec/RunSwiftScript.sh 2010-09-08 01:01:21 UTC (rev 3610) @@ -1,9 +1,10 @@ rundir=$1 site=$2 script=$3 -# runR=$4 +workerhosts=$4 -cp $script $runR $rundir +SWIFTRBIN=$(cd $(dirname $0); pwd) +cp $script $SWIFTRBIN/passive-coaster-swift $rundir # FIXME: rework this script to transfer all shells and rscripts # needed, and to copy in the R prelude for the R server processes (to @@ -15,13 +16,19 @@ # use /bin/sh and count on users PATH? # then add ENV::PATH as an option, e.g., from options(swift.remotepath and swift.remotepath.sitename) +app=/bin/bash + cat >tc <sites.xml < + 10000 @@ -29,6 +36,17 @@ $(pwd) + + + + passive + 8 + .07 + 10000 + + /home/wilde/swiftwork + + 00:00:10 1800 @@ -39,11 +57,18 @@ $(pwd) + END -cat >RunR.R <RunR.sh <serverscript.R </dev/null) # End of user initialization statements @@ -53,14 +78,25 @@ load(argv[1]); result=list() -for(c in 1:length(rcall\$arglistbatch)) { +for(c in 1:length(rcall\\\$arglistbatch)) { # FIXME: run this under try/catch and save error status in results object (need to make it a list: rval + error status) - result[[c]] = do.call( rcall\$func, rcall\$arglistbatch[[c]] ) + result[[c]] = do.call( rcall\\\$func, rcall\\\$arglistbatch[[c]] ) } save(result,file=argv[2]) END +Rscript serverscript.R \$* +ENDX + +if [ _$site = _passive ]; then + swiftcmd=./passive-coaster-swift + # workhosts passed in as argument +else + swiftcmd=swift + workerhosts="" +fi + script=$(basename $script) -swift -tc.file tc -sites.file sites.xml $script +$swiftcmd "$workerhosts" -tc.file tc -sites.file sites.xml $script Added: SwiftApps/SwiftR/Swift/exec/passive-coaster-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/passive-coaster-swift (rev 0) +++ SwiftApps/SwiftR/Swift/exec/passive-coaster-swift 2010-09-08 01:01:21 UTC (rev 3610) @@ -0,0 +1,70 @@ +#! /bin/bash + +COMPUTEHOSTS=$1 +# COMPUTEHOSTS='crush thwomp stomp crank steamroller grind churn trounce thrash vanquish' +# COMPUTEHOSTS='communicado' + +shift + +function wait-and-start-workers +{ + # Look for: + # Passive queue processor initialized. Callback URI is http://140.221.8.62:55379 + + for try in $(seq 1 20); do + uriline=$(grep "Passive queue processor initialized. Callback URI is" $out 2> /dev/null) + if [ "_$uriline" = _ ]; then + sleep 1 + else + break; + fi + done + CONTACT=$(echo $uriline | sed -e 's/^.*http:/http:/') + echo Coaster contact: $CONTACT + + LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env? + mkdir -p $LOGDIR + + workerscript=/home/wilde/swift/lab/worker.pl # FIXME: Try to send this inline or via scp; cp from exec/ dir + # CA="X509_CERT_DIR=$CERTDIR X509_CADIR=$CERTDIR" + for host in $(echo $COMPUTEHOSTS); do + timestamp=$(date "+%Y.%m%d.%H%M%S") + random=$(awk "BEGIN {printf \"%0.5d\", $RANDOM}") + ID=$timestamp.$random + ssh $host WORKER_LOGGING_ENABLED=true $workerscript $CONTACT $ID $LOGDIR & + sshpids="$sshpids $!" + done + + echo Started workers from these ssh processes: $sshpids + echo $sshpids > $sshpidfile + # ps $sshpids +} + +if ! which swift >& /dev/null; then + echo $0: \'swift\' command not found in PATH + exit 1 +fi + +out=`mktemp swift.stdouterr.XXXX` +touch $out +sshpidfile=${out/stdouterr/workerpids} + +echo swift output is in: $out, pids in $sshpidfile + +TRAPS="EXIT 1 2 3 15" # Signals and conditions to trap + +function onexit { + trap - $TRAPS + sshpids=$(cat $sshpidfile) + echo "Terminating worker processes $sshpids and starter $starterpid" + if [ "_$sshpids$starterpid" != _ ]; then + kill $sshpids $starterpid >& /dev/null + fi +} + +trap onexit $TRAPS + +wait-and-start-workers & +starterpid=$! + +swift $* 2>&1 | tee $out Property changes on: SwiftApps/SwiftR/Swift/exec/passive-coaster-swift ___________________________________________________________________ Name: svn:executable + * Modified: SwiftApps/SwiftR/Swift/exec/swiftapply.swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-09-07 19:36:39 UTC (rev 3609) +++ SwiftApps/SwiftR/Swift/exec/swiftapply.swift 2010-09-08 01:01:21 UTC (rev 3610) @@ -1,10 +1,9 @@ type file; type RData; -type RScript; -app (RData result, file stout, file sterr) RunR (RScript rscript, RData rcall) +app (RData result, file stout, file sterr) RunR (file shellscript, RData rcall) { - Rscript @rscript @rcall @result stdout=@stout stderr=@sterr; + bash @shellscript @rcall @result stdout=@stout stderr=@sterr; # RunR @rcall @result stdout=@stout stderr=@sterr; } @@ -12,8 +11,8 @@ RData results[] ; file stout[] ; file sterr[] ; -RScript rs <"RunR.R">; +file runRscript <"RunR.sh">; foreach c, i in rcalls { - (results[i],stout[i], sterr[i]) = RunR(rs,c); + (results[i],stout[i], sterr[i]) = RunR(runRscript,c); } From noreply at svn.ci.uchicago.edu Tue Sep 7 20:03:05 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 20:03:05 -0500 (CDT) Subject: [Swift-commit] r3611 - SwiftApps/SwiftR/Swift/R Message-ID: <20100908010305.381859CD19@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 20:03:05 -0500 (Tue, 07 Sep 2010) New Revision: 3611 Modified: SwiftApps/SwiftR/Swift/R/Swift.R Log: Added options for passive worker host list; removed fossil option code. Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-08 01:01:21 UTC (rev 3610) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-08 01:03:05 UTC (rev 3611) @@ -1,16 +1,7 @@ -swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL, manualmode=NULL, initialize=NULL ) +swiftapply <- function( func, arglists, site=NULL, callsperbatch=NULL, manualmode=NULL, initialize=NULL, workerhosts=NULL ) { # Set Swift default options if not passed as keyword paramets or pre-set by user -if(FALSE){ - if (is.null(getOption("swift.callsperbatch"))) - options(swift.callsperbatch=1) - if (is.null(getOption("swift.manualmode"))) - options(swift.manualmode = FALSE) - if (is.null(getOption("swift.initialize"))) - options(swift.initialize = "") -} # FALSE - if(is.null(site)) site <- getOption("swift.site") if(is.null(site)) @@ -31,11 +22,17 @@ if(is.null(initialize)) initialize <- ""; + if(is.null(workerhosts)) + workerhosts <- getOption("swift.workerhosts") + if(is.null(workerhosts)) + workerhosts <- "localhost"; + cat("\nSwift properties:\n") cat(" site =",site,"\n"); cat(" callsperbatch =", callsperbatch,"\n") cat(" manualmode =", manualmode,"\n") cat(" initialize =", initialize,"\n") + cat(" workerhosts =", workerhosts,"\n") cat("\nCurrent dir: ",getwd(),"\n"); # Execute the calls in batches @@ -81,7 +78,7 @@ readLines(n=1) } else { - system(paste(RunSwiftScript,rundir,site,swiftapplyScript)) + system(paste(RunSwiftScript,rundir,site,swiftapplyScript,"\"",workerhosts,"\"")) # system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript)) # system(paste(RunSwiftScript,rundir,site,swiftapplyScript,RunRScript,sep=" ")) } From noreply at svn.ci.uchicago.edu Tue Sep 7 20:04:16 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Sep 2010 20:04:16 -0500 (CDT) Subject: [Swift-commit] r3612 - SwiftApps/SwiftR/Swift/tests Message-ID: <20100908010416.4DAA39CD19@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-07 20:04:16 -0500 (Tue, 07 Sep 2010) New Revision: 3612 Modified: SwiftApps/SwiftR/Swift/tests/TestSwift.R Log: Added tests for passive coaster worker option. Modified: SwiftApps/SwiftR/Swift/tests/TestSwift.R =================================================================== --- SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-09-08 01:03:05 UTC (rev 3611) +++ SwiftApps/SwiftR/Swift/tests/TestSwift.R 2010-09-08 01:04:16 UTC (rev 3612) @@ -305,8 +305,38 @@ cat("\n==> test 4.1 FAILED !!!!!\n") } +#### Test Group 5: test passive mode (make this start of multiple config tests?) +cat("\n*** Starting test 5.1 ***\n\n") +options(swift.site="passive") + +sumstuff <- function(treedata,cardata) { sum( treedata$Height, cardata$dist ) } +data(cars) +data(trees) + +args=list(trees,cars) +arglist = rep(list(args),1) + +cat("Test of local do.call(sumstuff)\n") +localres = do.call(sumstuff,args) +cat("local result=\n") +print(localres) + +cat("\nTest of swiftapply(sumstuff,arglist)\n") +swiftres = swiftapply(sumstuff,arglist) +cat("Swift result:\n") +print(swiftres) + +if(identical(localres,swiftres[[1]])) { + cat("\n==> test 1.1 passed\n") +} else { + cat("\n==> test 1.1 FAILED !!!!!\n") +} + + + + if(FALSE) { # Test various batch sizes cat("\nTest of swiftapply(sumcrits,arglist,callsperbatch=10)\n") From noreply at svn.ci.uchicago.edu Wed Sep 8 13:56:26 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 13:56:26 -0500 (CDT) Subject: [Swift-commit] r3613 - trunk/src/org/griphyn/vdl/util Message-ID: <20100908185626.CFBB39CD19@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-08 13:56:26 -0500 (Wed, 08 Sep 2010) New Revision: 3613 Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java Log: Minor fixes to warnings Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2010-09-08 01:04:16 UTC (rev 3612) +++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2010-09-08 18:56:26 UTC (rev 3613) @@ -18,8 +18,10 @@ public class VDL2Config extends Properties { - public static final Logger logger = Logger.getLogger(VDL2Config.class); + private static final long serialVersionUID = 1L; + public static final Logger logger = Logger.getLogger(VDL2Config.class); + public static final String CONFIG_FILE_NAME = "swift.properties"; public static final String[] CONFIG_FILE_SEARCH_PATH = new String[] { System.getProperty("swift.home") + File.separator + "etc" + File.separator @@ -62,11 +64,11 @@ return config.check(); } - private List files, tried; + private List files, tried; private VDL2Config() { - files = new LinkedList(); - tried = new LinkedList(); + files = new LinkedList(); + tried = new LinkedList(); put(VDL2ConfigProperties.POOL_FILE, "${vds.home}/etc/sites.xml"); put(VDL2ConfigProperties.TC_FILE, "${vds.home}/var/tc.data"); put(VDL2ConfigProperties.LAZY_ERRORS, "false"); @@ -131,7 +133,7 @@ } /** - * Overriden to do variable expansion. Variables will be expanded if there + * Overridden to do variable expansion. Variables will be expanded if there * is a system property with that name. Otherwise, the expansion will not * occur. */ @@ -228,12 +230,11 @@ String prop = (String) bc.getProperty(name); if(prop != null) { return prop; - } + } } if(logger.isDebugEnabled()) { logger.debug("Getting property "+name+" from global configuration"); } return getProperty(name); } - } From noreply at svn.ci.uchicago.edu Wed Sep 8 13:57:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 13:57:55 -0500 (CDT) Subject: [Swift-commit] r3614 - in trunk: etc libexec tests/cdm-ps Message-ID: <20100908185755.018CE9CD19@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-08 13:57:54 -0500 (Wed, 08 Sep 2010) New Revision: 3614 Modified: trunk/etc/swift.properties trunk/libexec/vdl-int-staging.k trunk/tests/cdm-ps/swift.properties Log: Add provider.staging.pin.swiftfiles property Modified: trunk/etc/swift.properties =================================================================== --- trunk/etc/swift.properties 2010-09-08 18:56:26 UTC (rev 3613) +++ trunk/etc/swift.properties 2010-09-08 18:57:54 UTC (rev 3614) @@ -4,10 +4,10 @@ # # The host name of the submit machine is used by GRAM as a callback # address to report the status of submitted jobs. In general, Swift -# can automatically detect the host name of the local machine. +# can automatically detect the host name of the local machine. # However, if the machine host name is improperly configured or if # it does not represent a valid DNS entry, certain services (such as -# GRAM) will not be able to send job status notifications back to +# GRAM) will not be able to send job status notifications back to # the client. The value of this property can be an IP address. # # Format: @@ -20,7 +20,7 @@ # # A TCP port range can be specified to restrict the ports on which GRAM # callback services are started. This is likely needed if your submit -# host is behind a firewall, in which case the firewall should be +# host is behind a firewall, in which case the firewall should be # configured to allow incoming connections on ports in the range. # # Format: @@ -33,10 +33,10 @@ # false - means an error will be immediately reported and cause the # workflow to abort. At this time remote jobs that are already # running will not be canceled -# true - means that Swift will try to do as much work as possible and +# true - means that Swift will try to do as much work as possible and # report all errors encountered at the end. However, "errors" # here only applies to job execution errors. Certain errors -# that are related to the Swift implementation (should such +# that are related to the Swift implementation (should such # errors occur) will still be reported eagerly. # # Default: false @@ -46,7 +46,7 @@ # # What algorithm to use for caching of remote files. LRU (as in what # files to purge) is the only implementation right now. One can set -# a target size (in bytes) for a host by using the swift:storagesize +# a target size (in bytes) for a host by using the swift:storagesize # profile for a host in sites.xml # # Default: LRU @@ -56,7 +56,7 @@ # # true - generate a provenance graph in .dot format (Swift will # choose a random file name) -# false - do not generate a provenance graph +# false - do not generate a provenance graph # - generate a provenange graph in the give file name # # Default: false @@ -65,7 +65,7 @@ # -# graph properties for the provenance graph (.dot specific) +# graph properties for the provenance graph (.dot specific) # # Default: splines="compound", rankdir="TB" # @@ -73,25 +73,25 @@ # -# node properties for the provenance graph (.dot specific) +# node properties for the provenance graph (.dot specific) # # Default: color="seagreen", style="filled" # pgraph.node.options=color="seagreen", style="filled" # -# true - clustering of small jobs is enabled. Clustering works in the +# true - clustering of small jobs is enabled. Clustering works in the # following way: If a job is clusterable (meaning that it has the # GLOBUS::maxwalltime profile specified in tc.data and its value # is less than the value of the "clustering.min.time" property) it will -# be put in a clustering queue. The queue is processed at intervals +# be put in a clustering queue. The queue is processed at intervals # specified by the "clustering.queue.delay" property. The processing # of the clustering queue consists of selecting compatible jobs and # grouping them in clusters whose max wall time does not exceed twice -# the value of the "clustering.min.time" property. Two or more jobs are +# the value of the "clustering.min.time" property. Two or more jobs are # considered compatible if they share the same site and do not have # conflicting profiles (e.g. different values for the same environment -# variable). +# variable). # false - clustering of small jobs is disabled. # # Default: false @@ -123,7 +123,7 @@ # true - use Kickstart. If a job is scheduled on a site that does not have # Kickstart installed, that job will fail. # maybe - Use Kickstart if installed (i.e. the entry is present in the sites -# file) +# file) # # Default: maybe # @@ -164,9 +164,9 @@ # throttle only limits the number of concurrent tasks (jobs) that are being # sent to sites, not the total number of concurrent jobs that can be run. # The submission stage in GRAM is one of the most CPU expensive stages (due -# mostly to the mutual authentication and delegation). Having too many +# mostly to the mutual authentication and delegation). Having too many # concurrent submissions can overload either or both the submit host CPU -# and the remote host/head node causing degraded performance. +# and the remote host/head node causing degraded performance. # # Default: 4 # @@ -176,7 +176,7 @@ # # Limits the number of concurrent submissions for any of the sites Swift will -# try to send jobs to. In other words it guarantees that no more than the +# try to send jobs to. In other words it guarantees that no more than the # value of this throttle jobs sent to any site will be concurrently in a state # of being submitted. # @@ -192,7 +192,7 @@ # is assigned a score (initially 1), which can increase or decrease based # on whether the site yields successful or faulty job runs. The score for a # site can take values in the (0.1, 100) interval. The number of allowed jobs -# is calculated using the following formula: +# is calculated using the following formula: # 2 + score*throttle.score.job.factor # This means a site will always be allowed at least two concurrent jobs and # at most 2 + 100*throttle.score.job.factor. With a default of 4 this means @@ -220,11 +220,11 @@ # Limits the total number of concurrent file operations that can happen at any # given time. File operations (like transfers) require an exclusive connection # to a site. These connections can be expensive to establish. A large number -# of concurrent file operations may cause Swift to attempt to establish many +# of concurrent file operations may cause Swift to attempt to establish many # such expensive connections to various sites. Limiting the number of concurrent # file operations causes Swift to use a small number of cached connections and -# achieve better overall performance. -# +# achieve better overall performance. +# # Default: 8 # @@ -240,7 +240,7 @@ sitedir.keep=false -# number of time a job will be retried if it fails (giving a maximum of +# number of time a job will be retried if it fails (giving a maximum of # 1 + execution.retries attempts at execution) # @@ -256,7 +256,7 @@ replication.enabled=false # If replication is enabled, this value specifies the minimum time, in seconds, -# a job needs to be queued in a batch queue in order to be considered for +# a job needs to be queued in a batch queue in order to be considered for # replication # @@ -271,7 +271,7 @@ # # The IP address of the submit machine is used by GRAM as a callback # address to report the status of submitted jobs. In general, Swift -# can automatically detect the IP address of the local machine. +# can automatically detect the IP address of the local machine. # However, if the machine has more than one network interface, Swift # will pick the first one, which may not be the right choice. It is # recommended that this property is set properly before attempting to @@ -312,7 +312,7 @@ # # Limits the number of concurrent iterations that each foreach statement -# can have at one time. This conserves memory for swift programs that +# can have at one time. This conserves memory for swift programs that # have large numbers of iterations (which would otherwise all be executed # in parallel). # @@ -327,11 +327,11 @@ provenance.log=false -# Controls whether file staging is done by swift or by the execution +# Controls whether file staging is done by swift or by the execution # provider. If set to false, the standard swift staging mechanism is -# used. If set to true, swift does not stage files. Instead, the +# used. If set to true, swift does not stage files. Instead, the # execution provider is instructed to stage files in and out. -# +# # Provider staging is experimental. # # When enabled, and when coasters are used as an execution provider, @@ -339,14 +339,15 @@ # using the swift:stagingMethod site profile in sites.xml. The # following is a list of accepted mechanisms: # -# * file: Staging is done from a filesystem accessible to the -# coaster service (typically running on the head node) +# * file: Staging is done from a filesystem accessible to the +# coaster service (typically running on the head node) # * proxy: Staging is done from a filesystem accessible to the # client machine that swift is running on, and is proxied # through the coaster service # * sfs: (short for "shared filesystem") Staging is done by # copying files to and from a filesystem accessible -# by the compute node (such as an NFS or GPFS mount). - +# by the compute node (such as an NFS or GPFS mount). -use.provider.staging=false \ No newline at end of file + +use.provider.staging=false +provider.staging.pin.swiftfiles=false Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-09-08 18:56:26 UTC (rev 3613) +++ trunk/libexec/vdl-int-staging.k 2010-09-08 18:57:54 UTC (rev 3614) @@ -81,7 +81,7 @@ jobid := "{tr}-{uid}" log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") - + wrapper := "_swiftwrap.staging" wrapfile := "{ddir}/param-{jobid}" @@ -91,7 +91,7 @@ vdl:setprogress("Stage in") wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) - cdmfile := cdm:file() + cdmfile := cdm:file() try( sequential( @@ -119,24 +119,26 @@ directory = "{wfdir}-{jobdir}-{jobid}" redirect = false host = rhost - - vdl:tcprofile(rhost, tr = tr) //this gets various app params from the tc, such as environment, walltime, etc + + vdl:tcprofile(rhost, tr = tr) //this gets various app params from the tc, such as environment, walltime, etc replicationGroup = replicationGroup replicationChannel = replicationChannel jobid = jobid stagingMethod := vdl:siteProfile(rhost, "swift:stagingMethod", default="proxy") + pinOption := configProperty("provider.staging.pin.swiftfiles") + pin := if (pinOption == "" then ("pinned:") else ("")) - stageIn("{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper) + stageIn("{pin}{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper) - if( cdmfile != "" then( - d := vdl:dirname(cdmfile) - f := basename(cdmfile) - echo("d: {d} f: {f}") - stageIn("{stagingMethod}://localhost/{d}/{f}", cdmfile) + if ( cdmfile != "" then( + d := vdl:dirname(cdmfile) + file := basename(cdmfile) + dir := if ( d == "" then("./") else(str:concat(d,"/"))) + stageIn("{stagingMethod}://localhost/{dir}{file}", cdmfile) stageIn("{stagingMethod}://localhost/{swift.home}/libexec/cdm.pl", "cdm.pl") stageIn("{stagingMethod}://localhost/{swift.home}/libexec/cdm_lib.sh", "cdm_lib.sh") - )) + )) appStageins(jobid, stagein, ".", stagingMethod) @@ -193,3 +195,8 @@ ) ) ) + +// Local variables: +// tab-width: 4 +// indent-tabs-mode: 1 +// End: Modified: trunk/tests/cdm-ps/swift.properties =================================================================== --- trunk/tests/cdm-ps/swift.properties 2010-09-08 18:56:26 UTC (rev 3613) +++ trunk/tests/cdm-ps/swift.properties 2010-09-08 18:57:54 UTC (rev 3614) @@ -350,3 +350,4 @@ use.provider.staging=true +provider.staging.pin.swiftfiles=false From noreply at svn.ci.uchicago.edu Wed Sep 8 13:59:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 13:59:32 -0500 (CDT) Subject: [Swift-commit] r3615 - in trunk/tests: . local Message-ID: <20100908185932.0B00B9CD19@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-08 13:59:31 -0500 (Wed, 08 Sep 2010) New Revision: 3615 Modified: trunk/tests/local/range.check.sh trunk/tests/nightly.sh Log: Minor improvements Modified: trunk/tests/local/range.check.sh =================================================================== --- trunk/tests/local/range.check.sh 2010-09-08 18:57:54 UTC (rev 3614) +++ trunk/tests/local/range.check.sh 2010-09-08 18:59:31 UTC (rev 3615) @@ -4,7 +4,7 @@ ls stdout.txt || exit 1 -COUNT=$( grep -c "trace:" stdout.txt ) +COUNT=$( grep -c "num:" stdout.txt ) (( $COUNT == 11 )) || exit 1 exit 0 Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-08 18:57:54 UTC (rev 3614) +++ trunk/tests/nightly.sh 2010-09-08 18:59:31 UTC (rev 3615) @@ -129,6 +129,16 @@ mkdir -p $RUNDIR [ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1 +checkfail() { + ERR=$? + shift + MSG=${*} + if [[ $ERR != 0 ]]; then + echo "FAILED($ERR): $MSG" + exit $ERR + fi +} + crash() { MSG=$1 echo $MSG @@ -288,6 +298,13 @@ html_~html } +printlist() { + while [[ $1 != "" ]]; do + echo $1 + shift + done +} + outecho() { TYPE=$1 shift @@ -427,7 +444,6 @@ } process_exec() { - declare -p PWD printf "\nExecuting: $@" >>$LOG rm -fv $OUTPUT "$@" > $OUTPUT 2>&1 @@ -585,9 +601,6 @@ # ptest } -#ptest() { -#} - build_package() { TEST="Package" test_exec cd $SWIFT_HOME/lib @@ -631,10 +644,10 @@ group_swift_properties() { if [ -f $GROUP/swift.properties ]; then - cp -uv $GROUP/swift.properties . + cp -v $GROUP/swift.properties . [ $? != 0 ] && crash "Could not copy swift.properties!" else - cp -uv $SWIFT_HOME/etc/swift.properties . + cp -v $SWIFT_HOME/etc/swift.properties . [ $? != 0 ] && crash "Could not copy swift.properties!" fi } @@ -654,12 +667,15 @@ group_fs_data group_swift_properties - for TEST in $( ls $GROUP/*.swift ); do + SWIFTS=$( ls $GROUP/*.swift ) + checkfail "Could not ls: $GROUP" + for TEST in $SWIFTS; do + (( SKIP_COUNTER++ < SKIP_TESTS )) && continue TESTNAME=$( basename $TEST) - cp -uv $GROUP/$TESTNAME . + cp -v $GROUP/$TESTNAME . TESTLINK=$TESTNAME start_row @@ -670,7 +686,6 @@ done } - date > $LOG header @@ -734,8 +749,6 @@ $TESTDIR/cdm \ $TESTDIR/cdm-ps ) -echo ${GROUPLIST[@]} - GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do GROUP=$G @@ -769,6 +782,8 @@ footer +exit 0 + # Local Variables: # sh-basic-offset: 2 # End: From noreply at svn.ci.uchicago.edu Wed Sep 8 14:52:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 14:52:25 -0500 (CDT) Subject: [Swift-commit] r3616 - in trunk/tests: . cdm cdm/ps Message-ID: <20100908195225.A905D9CD19@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-08 14:52:25 -0500 (Wed, 08 Sep 2010) New Revision: 3616 Added: trunk/tests/cdm/ps/ trunk/tests/cdm/ps/swift.properties Removed: trunk/tests/cdm-ps/ trunk/tests/cdm/ps/swift.properties Modified: trunk/tests/nightly.sh Log: Move cdm-ps to cdm/ps Copied: trunk/tests/cdm/ps (from rev 3612, trunk/tests/cdm-ps) Deleted: trunk/tests/cdm/ps/swift.properties =================================================================== --- trunk/tests/cdm-ps/swift.properties 2010-09-08 01:04:16 UTC (rev 3612) +++ trunk/tests/cdm/ps/swift.properties 2010-09-08 19:52:25 UTC (rev 3616) @@ -1,352 +0,0 @@ -sites.file=${swift.home}/etc/sites.xml -tc.file=${swift.home}/etc/tc.data - -# -# The host name of the submit machine is used by GRAM as a callback -# address to report the status of submitted jobs. In general, Swift -# can automatically detect the host name of the local machine. -# However, if the machine host name is improperly configured or if -# it does not represent a valid DNS entry, certain services (such as -# GRAM) will not be able to send job status notifications back to -# the client. The value of this property can be an IP address. -# -# Format: -# hostname=string -# - - -#hostname=localhost - -# -# A TCP port range can be specified to restrict the ports on which GRAM -# callback services are started. This is likely needed if your submit -# host is behind a firewall, in which case the firewall should be -# configured to allow incoming connections on ports in the range. -# -# Format: -# tcp.port.range=start,end -# - -#tcp.port.range=50000,50100 - -# -# false - means an error will be immediately reported and cause the -# workflow to abort. At this time remote jobs that are already -# running will not be canceled -# true - means that Swift will try to do as much work as possible and -# report all errors encountered at the end. However, "errors" -# here only applies to job execution errors. Certain errors -# that are related to the Swift implementation (should such -# errors occur) will still be reported eagerly. -# -# Default: false -# -lazy.errors=false - -# -# What algorithm to use for caching of remote files. LRU (as in what -# files to purge) is the only implementation right now. One can set -# a target size (in bytes) for a host by using the swift:storagesize -# profile for a host in sites.xml -# -# Default: LRU -# -caching.algorithm=LRU - -# -# true - generate a provenance graph in .dot format (Swift will -# choose a random file name) -# false - do not generate a provenance graph -# - generate a provenange graph in the give file name -# -# Default: false -# -pgraph=false - - -# -# graph properties for the provenance graph (.dot specific) -# -# Default: splines="compound", rankdir="TB" -# -pgraph.graph.options=splines="compound", rankdir="TB" - - -# -# node properties for the provenance graph (.dot specific) -# -# Default: color="seagreen", style="filled" -# -pgraph.node.options=color="seagreen", style="filled" - -# -# true - clustering of small jobs is enabled. Clustering works in the -# following way: If a job is clusterable (meaning that it has the -# GLOBUS::maxwalltime profile specified in tc.data and its value -# is less than the value of the "clustering.min.time" property) it will -# be put in a clustering queue. The queue is processed at intervals -# specified by the "clustering.queue.delay" property. The processing -# of the clustering queue consists of selecting compatible jobs and -# grouping them in clusters whose max wall time does not exceed twice -# the value of the "clustering.min.time" property. Two or more jobs are -# considered compatible if they share the same site and do not have -# conflicting profiles (e.g. different values for the same environment -# variable). -# false - clustering of small jobs is disabled. -# -# Default: false -# -clustering.enabled=false - - -# -# - the intervals at which the clustering queue is processed -# -# Default: 4 -# -clustering.queue.delay=4 - -# -# - the threshold time for clustering -# -# Default: 60 -# -clustering.min.time=60 - -# -# Kickstart is a useful tool that can be used to gather various information -# about a remote process. Before it can be used it must be installed on the -# remote site and the corresponding entry be set in the sites file. -# This option allows controlling of how Swift uses Kickstart. The following -# values are possible: -# false - do not use Kickstart -# true - use Kickstart. If a job is scheduled on a site that does not have -# Kickstart installed, that job will fail. -# maybe - Use Kickstart if installed (i.e. the entry is present in the sites -# file) -# -# Default: maybe -# - -kickstart.enabled=maybe - -# -# Indicates when Kickstart records should be fetched from the remote site: -# true - always transfer Kickstart records if Kickstart was used (see -# kickstart.enabled) -# false - only transfer Kickstart records if the job fails -# -# Default: false -# - -kickstart.always.transfer=false - -# -# Indicates when wrapper logs should be fetched from the remote site: -# true - always transfer wrapper logs -# false - only transfer wrapper logs if the job fails -# -# Default: false -# - -wrapperlog.always.transfer=false - -########################################################################### -# Throttling options # -########################################################################### -# -# For the throttling parameters, valid values are either a positive integer -# or "off" (without the quotes). -# - -# -# Limits the number of concurrent submissions for a workflow instance. This -# throttle only limits the number of concurrent tasks (jobs) that are being -# sent to sites, not the total number of concurrent jobs that can be run. -# The submission stage in GRAM is one of the most CPU expensive stages (due -# mostly to the mutual authentication and delegation). Having too many -# concurrent submissions can overload either or both the submit host CPU -# and the remote host/head node causing degraded performance. -# -# Default: 4 -# - -throttle.submit=4 -#throttle.submit=off - -# -# Limits the number of concurrent submissions for any of the sites Swift will -# try to send jobs to. In other words it guarantees that no more than the -# value of this throttle jobs sent to any site will be concurrently in a state -# of being submitted. -# -# Default: 2 -# - -throttle.host.submit=2 -#throttle.host.submit=off - -# -# The Swift scheduler has the ability to limit the number of concurrent jobs -# allowed on a site based on the performance history of that site. Each site -# is assigned a score (initially 1), which can increase or decrease based -# on whether the site yields successful or faulty job runs. The score for a -# site can take values in the (0.1, 100) interval. The number of allowed jobs -# is calculated using the following formula: -# 2 + score*throttle.score.job.factor -# This means a site will always be allowed at least two concurrent jobs and -# at most 2 + 100*throttle.score.job.factor. With a default of 4 this means -# at least 2 jobs and at most 402. -# -# Default: 4 -# - -throttle.score.job.factor=0.2 -#throttle.score.job.factor=off - - -# -# Limits the total number of concurrent file transfers that can happen at any -# given time. File transfers consume bandwidth. Too many concurrent transfers -# can cause the network to be overloaded preventing various other signalling -# traffic from flowing properly. -# -# Default: 4 -# - -throttle.transfers=4 -#throttle.transfers=off - -# Limits the total number of concurrent file operations that can happen at any -# given time. File operations (like transfers) require an exclusive connection -# to a site. These connections can be expensive to establish. A large number -# of concurrent file operations may cause Swift to attempt to establish many -# such expensive connections to various sites. Limiting the number of concurrent -# file operations causes Swift to use a small number of cached connections and -# achieve better overall performance. -# -# Default: 8 -# - -throttle.file.operations=8 -#throttle.file.operations=off - -# Indicates whether the working directory on the remote site should be -# left intact even when the workflow completes successfully. This can be -# used to inspect the site working directory for debugging purposes. -# -# Default: false -# - -sitedir.keep=false - -# number of time a job will be retried if it fails (giving a maximum of -# 1 + execution.retries attempts at execution) -# - -execution.retries=2 - - -# Enables/disables replication. Replication is used to deal with jobs sitting -# in batch queues for abnormally large amounts of time. If replication is enabled -# and certain conditions are met, Swift creates and submits replicas of jobs, and -# allows multiple instances of a job to compete. -# - -replication.enabled=false - -# If replication is enabled, this value specifies the minimum time, in seconds, -# a job needs to be queued in a batch queue in order to be considered for -# replication -# - -replication.min.queue.time=60 - -# The maximum number of replicas that Swift should attempt. - -replication.limit=3 - -# -# WARNING: This option is deprecated. Please use the hostname option. -# -# The IP address of the submit machine is used by GRAM as a callback -# address to report the status of submitted jobs. In general, Swift -# can automatically detect the IP address of the local machine. -# However, if the machine has more than one network interface, Swift -# will pick the first one, which may not be the right choice. It is -# recommended that this property is set properly before attempting to -# run jobs through GRAM. -# -# Format: -# ip.address=x.y.z.w -# - -#ip.address=127.0.0.1 - - -# Controls how Swift will communicate the result code of running user programs -# from workers to the submit side. In files mode, a file -# indicating success or failure will be created on the site shared filesystem. -# In provider mode, the execution provider job status will -# be used. Notably, GRAM2 does not return job statuses correctly, and so -# provider mode will not work with GRAM2. With other -# providers, it can be used to reduce the amount of filesystem access compared -# to files mode. -# -# status.mode=files - -# Controls how swift will supply parameters to the remote wrapper script. -# 'args' mode will pass parameters on the command line -# 'files' mode will pass parameters through an additional input file -# -# valid values: args, files -# Default: files -# -# wrapper.parameter.mode=args - -# Determines if Swift remote wrappers will be executed by specifying an -# absolute path, or a path relative to the job initial working directory -# -# valid values: absolute, relative -# wrapper.invocation.mode=absolute - -# -# Limits the number of concurrent iterations that each foreach statement -# can have at one time. This conserves memory for swift programs that -# have large numbers of iterations (which would otherwise all be executed -# in parallel). -# -# Default: 1024 -# - -foreach.max.threads=16384 - -# controls whether the log file will contain provenance information -# enabling this will increase the size of log files, sometimes -# significantly. - -provenance.log=false - -# Controls whether file staging is done by swift or by the execution -# provider. If set to false, the standard swift staging mechanism is -# used. If set to true, swift does not stage files. Instead, the -# execution provider is instructed to stage files in and out. -# -# Provider staging is experimental. -# -# When enabled, and when coasters are used as an execution provider, -# a staging mechanism can be selected for each site -# using the swift:stagingMethod site profile in sites.xml. The -# following is a list of accepted mechanisms: -# -# * file: Staging is done from a filesystem accessible to the -# coaster service (typically running on the head node) -# * proxy: Staging is done from a filesystem accessible to the -# client machine that swift is running on, and is proxied -# through the coaster service -# * sfs: (short for "shared filesystem") Staging is done by -# copying files to and from a filesystem accessible -# by the compute node (such as an NFS or GPFS mount). - - -use.provider.staging=true Copied: trunk/tests/cdm/ps/swift.properties (from rev 3614, trunk/tests/cdm-ps/swift.properties) =================================================================== --- trunk/tests/cdm/ps/swift.properties (rev 0) +++ trunk/tests/cdm/ps/swift.properties 2010-09-08 19:52:25 UTC (rev 3616) @@ -0,0 +1,353 @@ +sites.file=${swift.home}/etc/sites.xml +tc.file=${swift.home}/etc/tc.data + +# +# The host name of the submit machine is used by GRAM as a callback +# address to report the status of submitted jobs. In general, Swift +# can automatically detect the host name of the local machine. +# However, if the machine host name is improperly configured or if +# it does not represent a valid DNS entry, certain services (such as +# GRAM) will not be able to send job status notifications back to +# the client. The value of this property can be an IP address. +# +# Format: +# hostname=string +# + + +#hostname=localhost + +# +# A TCP port range can be specified to restrict the ports on which GRAM +# callback services are started. This is likely needed if your submit +# host is behind a firewall, in which case the firewall should be +# configured to allow incoming connections on ports in the range. +# +# Format: +# tcp.port.range=start,end +# + +#tcp.port.range=50000,50100 + +# +# false - means an error will be immediately reported and cause the +# workflow to abort. At this time remote jobs that are already +# running will not be canceled +# true - means that Swift will try to do as much work as possible and +# report all errors encountered at the end. However, "errors" +# here only applies to job execution errors. Certain errors +# that are related to the Swift implementation (should such +# errors occur) will still be reported eagerly. +# +# Default: false +# +lazy.errors=false + +# +# What algorithm to use for caching of remote files. LRU (as in what +# files to purge) is the only implementation right now. One can set +# a target size (in bytes) for a host by using the swift:storagesize +# profile for a host in sites.xml +# +# Default: LRU +# +caching.algorithm=LRU + +# +# true - generate a provenance graph in .dot format (Swift will +# choose a random file name) +# false - do not generate a provenance graph +# - generate a provenange graph in the give file name +# +# Default: false +# +pgraph=false + + +# +# graph properties for the provenance graph (.dot specific) +# +# Default: splines="compound", rankdir="TB" +# +pgraph.graph.options=splines="compound", rankdir="TB" + + +# +# node properties for the provenance graph (.dot specific) +# +# Default: color="seagreen", style="filled" +# +pgraph.node.options=color="seagreen", style="filled" + +# +# true - clustering of small jobs is enabled. Clustering works in the +# following way: If a job is clusterable (meaning that it has the +# GLOBUS::maxwalltime profile specified in tc.data and its value +# is less than the value of the "clustering.min.time" property) it will +# be put in a clustering queue. The queue is processed at intervals +# specified by the "clustering.queue.delay" property. The processing +# of the clustering queue consists of selecting compatible jobs and +# grouping them in clusters whose max wall time does not exceed twice +# the value of the "clustering.min.time" property. Two or more jobs are +# considered compatible if they share the same site and do not have +# conflicting profiles (e.g. different values for the same environment +# variable). +# false - clustering of small jobs is disabled. +# +# Default: false +# +clustering.enabled=false + + +# +# - the intervals at which the clustering queue is processed +# +# Default: 4 +# +clustering.queue.delay=4 + +# +# - the threshold time for clustering +# +# Default: 60 +# +clustering.min.time=60 + +# +# Kickstart is a useful tool that can be used to gather various information +# about a remote process. Before it can be used it must be installed on the +# remote site and the corresponding entry be set in the sites file. +# This option allows controlling of how Swift uses Kickstart. The following +# values are possible: +# false - do not use Kickstart +# true - use Kickstart. If a job is scheduled on a site that does not have +# Kickstart installed, that job will fail. +# maybe - Use Kickstart if installed (i.e. the entry is present in the sites +# file) +# +# Default: maybe +# + +kickstart.enabled=maybe + +# +# Indicates when Kickstart records should be fetched from the remote site: +# true - always transfer Kickstart records if Kickstart was used (see +# kickstart.enabled) +# false - only transfer Kickstart records if the job fails +# +# Default: false +# + +kickstart.always.transfer=false + +# +# Indicates when wrapper logs should be fetched from the remote site: +# true - always transfer wrapper logs +# false - only transfer wrapper logs if the job fails +# +# Default: false +# + +wrapperlog.always.transfer=false + +########################################################################### +# Throttling options # +########################################################################### +# +# For the throttling parameters, valid values are either a positive integer +# or "off" (without the quotes). +# + +# +# Limits the number of concurrent submissions for a workflow instance. This +# throttle only limits the number of concurrent tasks (jobs) that are being +# sent to sites, not the total number of concurrent jobs that can be run. +# The submission stage in GRAM is one of the most CPU expensive stages (due +# mostly to the mutual authentication and delegation). Having too many +# concurrent submissions can overload either or both the submit host CPU +# and the remote host/head node causing degraded performance. +# +# Default: 4 +# + +throttle.submit=4 +#throttle.submit=off + +# +# Limits the number of concurrent submissions for any of the sites Swift will +# try to send jobs to. In other words it guarantees that no more than the +# value of this throttle jobs sent to any site will be concurrently in a state +# of being submitted. +# +# Default: 2 +# + +throttle.host.submit=2 +#throttle.host.submit=off + +# +# The Swift scheduler has the ability to limit the number of concurrent jobs +# allowed on a site based on the performance history of that site. Each site +# is assigned a score (initially 1), which can increase or decrease based +# on whether the site yields successful or faulty job runs. The score for a +# site can take values in the (0.1, 100) interval. The number of allowed jobs +# is calculated using the following formula: +# 2 + score*throttle.score.job.factor +# This means a site will always be allowed at least two concurrent jobs and +# at most 2 + 100*throttle.score.job.factor. With a default of 4 this means +# at least 2 jobs and at most 402. +# +# Default: 4 +# + +throttle.score.job.factor=0.2 +#throttle.score.job.factor=off + + +# +# Limits the total number of concurrent file transfers that can happen at any +# given time. File transfers consume bandwidth. Too many concurrent transfers +# can cause the network to be overloaded preventing various other signalling +# traffic from flowing properly. +# +# Default: 4 +# + +throttle.transfers=4 +#throttle.transfers=off + +# Limits the total number of concurrent file operations that can happen at any +# given time. File operations (like transfers) require an exclusive connection +# to a site. These connections can be expensive to establish. A large number +# of concurrent file operations may cause Swift to attempt to establish many +# such expensive connections to various sites. Limiting the number of concurrent +# file operations causes Swift to use a small number of cached connections and +# achieve better overall performance. +# +# Default: 8 +# + +throttle.file.operations=8 +#throttle.file.operations=off + +# Indicates whether the working directory on the remote site should be +# left intact even when the workflow completes successfully. This can be +# used to inspect the site working directory for debugging purposes. +# +# Default: false +# + +sitedir.keep=false + +# number of time a job will be retried if it fails (giving a maximum of +# 1 + execution.retries attempts at execution) +# + +execution.retries=2 + + +# Enables/disables replication. Replication is used to deal with jobs sitting +# in batch queues for abnormally large amounts of time. If replication is enabled +# and certain conditions are met, Swift creates and submits replicas of jobs, and +# allows multiple instances of a job to compete. +# + +replication.enabled=false + +# If replication is enabled, this value specifies the minimum time, in seconds, +# a job needs to be queued in a batch queue in order to be considered for +# replication +# + +replication.min.queue.time=60 + +# The maximum number of replicas that Swift should attempt. + +replication.limit=3 + +# +# WARNING: This option is deprecated. Please use the hostname option. +# +# The IP address of the submit machine is used by GRAM as a callback +# address to report the status of submitted jobs. In general, Swift +# can automatically detect the IP address of the local machine. +# However, if the machine has more than one network interface, Swift +# will pick the first one, which may not be the right choice. It is +# recommended that this property is set properly before attempting to +# run jobs through GRAM. +# +# Format: +# ip.address=x.y.z.w +# + +#ip.address=127.0.0.1 + + +# Controls how Swift will communicate the result code of running user programs +# from workers to the submit side. In files mode, a file +# indicating success or failure will be created on the site shared filesystem. +# In provider mode, the execution provider job status will +# be used. Notably, GRAM2 does not return job statuses correctly, and so +# provider mode will not work with GRAM2. With other +# providers, it can be used to reduce the amount of filesystem access compared +# to files mode. +# +# status.mode=files + +# Controls how swift will supply parameters to the remote wrapper script. +# 'args' mode will pass parameters on the command line +# 'files' mode will pass parameters through an additional input file +# +# valid values: args, files +# Default: files +# +# wrapper.parameter.mode=args + +# Determines if Swift remote wrappers will be executed by specifying an +# absolute path, or a path relative to the job initial working directory +# +# valid values: absolute, relative +# wrapper.invocation.mode=absolute + +# +# Limits the number of concurrent iterations that each foreach statement +# can have at one time. This conserves memory for swift programs that +# have large numbers of iterations (which would otherwise all be executed +# in parallel). +# +# Default: 1024 +# + +foreach.max.threads=16384 + +# controls whether the log file will contain provenance information +# enabling this will increase the size of log files, sometimes +# significantly. + +provenance.log=false + +# Controls whether file staging is done by swift or by the execution +# provider. If set to false, the standard swift staging mechanism is +# used. If set to true, swift does not stage files. Instead, the +# execution provider is instructed to stage files in and out. +# +# Provider staging is experimental. +# +# When enabled, and when coasters are used as an execution provider, +# a staging mechanism can be selected for each site +# using the swift:stagingMethod site profile in sites.xml. The +# following is a list of accepted mechanisms: +# +# * file: Staging is done from a filesystem accessible to the +# coaster service (typically running on the head node) +# * proxy: Staging is done from a filesystem accessible to the +# client machine that swift is running on, and is proxied +# through the coaster service +# * sfs: (short for "shared filesystem") Staging is done by +# copying files to and from a filesystem accessible +# by the compute node (such as an NFS or GPFS mount). + + +use.provider.staging=true +provider.staging.pin.swiftfiles=false Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-08 18:59:31 UTC (rev 3615) +++ trunk/tests/nightly.sh 2010-09-08 19:52:25 UTC (rev 3616) @@ -747,7 +747,7 @@ $TESTDIR/local \ $TESTDIR/language/should-not-work \ $TESTDIR/cdm \ - $TESTDIR/cdm-ps ) + $TESTDIR/cdm/ps ) GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do From noreply at svn.ci.uchicago.edu Wed Sep 8 14:53:36 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 14:53:36 -0500 (CDT) Subject: [Swift-commit] r3617 - trunk/tests/cdm/ps Message-ID: <20100908195336.6AF7A9CD19@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-08 14:53:36 -0500 (Wed, 08 Sep 2010) New Revision: 3617 Added: trunk/tests/cdm/ps/pinned/ Log: Test pinned: prefix for CDM w/ provider staging From noreply at svn.ci.uchicago.edu Wed Sep 8 14:55:10 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 14:55:10 -0500 (CDT) Subject: [Swift-commit] r3618 - trunk/tests/cdm/ps/pinned Message-ID: <20100908195510.7B7889CD19@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-08 14:55:10 -0500 (Wed, 08 Sep 2010) New Revision: 3618 Added: trunk/tests/cdm/ps/pinned/201-direct-output.check.sh trunk/tests/cdm/ps/pinned/201-direct-output.clean.sh trunk/tests/cdm/ps/pinned/201-direct-output.setup.sh trunk/tests/cdm/ps/pinned/201-direct-output.swift trunk/tests/cdm/ps/pinned/202-direct-io.check.sh trunk/tests/cdm/ps/pinned/202-direct-io.clean.sh trunk/tests/cdm/ps/pinned/202-direct-io.setup.sh trunk/tests/cdm/ps/pinned/202-direct-io.swift trunk/tests/cdm/ps/pinned/README.txt trunk/tests/cdm/ps/pinned/fs.template.data trunk/tests/cdm/ps/pinned/merge.sh trunk/tests/cdm/ps/pinned/sites.template.xml trunk/tests/cdm/ps/pinned/swift.properties trunk/tests/cdm/ps/pinned/tc.template.data trunk/tests/cdm/ps/pinned/title.txt Log: Copy pinned tests from cdm/ps Copied: trunk/tests/cdm/ps/pinned/201-direct-output.check.sh (from rev 3616, trunk/tests/cdm/ps/201-direct-output.check.sh) =================================================================== --- trunk/tests/cdm/ps/pinned/201-direct-output.check.sh (rev 0) +++ trunk/tests/cdm/ps/pinned/201-direct-output.check.sh 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,8 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 201-input.txt || exit 1 +grep $( uname -m ) 201/201-output.txt || exit 1 + +exit 0 Copied: trunk/tests/cdm/ps/pinned/201-direct-output.clean.sh (from rev 3616, trunk/tests/cdm/ps/201-direct-output.clean.sh) =================================================================== --- trunk/tests/cdm/ps/pinned/201-direct-output.clean.sh (rev 0) +++ trunk/tests/cdm/ps/pinned/201-direct-output.clean.sh 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -rv 201-input.txt 201 || exit 1 + +exit 0 Copied: trunk/tests/cdm/ps/pinned/201-direct-output.setup.sh (from rev 3616, trunk/tests/cdm/ps/201-direct-output.setup.sh) =================================================================== --- trunk/tests/cdm/ps/pinned/201-direct-output.setup.sh (rev 0) +++ trunk/tests/cdm/ps/pinned/201-direct-output.setup.sh 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,10 @@ +#!/bin/sh + +set -x + +{ + uname -a + date +} > 201-input.txt + +exit 0 Copied: trunk/tests/cdm/ps/pinned/201-direct-output.swift (from rev 3616, trunk/tests/cdm/ps/201-direct-output.swift) =================================================================== --- trunk/tests/cdm/ps/pinned/201-direct-output.swift (rev 0) +++ trunk/tests/cdm/ps/pinned/201-direct-output.swift 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,14 @@ + +type file; + +app (file o) copy (file i) +{ + cp @i @o; +} + +file f1<"201-input.txt">; + +// Will be transformed by CDM to 201/output.txt: +file f2<"201-output.txt">; + +f2 = copy(f1); Copied: trunk/tests/cdm/ps/pinned/202-direct-io.check.sh (from rev 3616, trunk/tests/cdm/ps/202-direct-io.check.sh) =================================================================== --- trunk/tests/cdm/ps/pinned/202-direct-io.check.sh (rev 0) +++ trunk/tests/cdm/ps/pinned/202-direct-io.check.sh 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,11 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 202/202-input-1.txt || exit 1 +grep $( hostname ) 202/202-input-1.txt || exit 1 + +grep $( uname -m ) 202/202-output.txt || exit 1 +grep $( hostname ) 202/202-output.txt || exit 1 + +exit 0 Copied: trunk/tests/cdm/ps/pinned/202-direct-io.clean.sh (from rev 3616, trunk/tests/cdm/ps/202-direct-io.clean.sh) =================================================================== --- trunk/tests/cdm/ps/pinned/202-direct-io.clean.sh (rev 0) +++ trunk/tests/cdm/ps/pinned/202-direct-io.clean.sh 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -rv 202 || exit 1 + +exit 0 Copied: trunk/tests/cdm/ps/pinned/202-direct-io.setup.sh (from rev 3616, trunk/tests/cdm/ps/202-direct-io.setup.sh) =================================================================== --- trunk/tests/cdm/ps/pinned/202-direct-io.setup.sh (rev 0) +++ trunk/tests/cdm/ps/pinned/202-direct-io.setup.sh 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,17 @@ +#!/bin/sh + +set -x + +mkdir -v 202 + +{ + uname -a + date +} > 202/202-input-1.txt + +{ + hostname + pwd +} > 202/202-input-2.txt + +exit 0 Copied: trunk/tests/cdm/ps/pinned/202-direct-io.swift (from rev 3616, trunk/tests/cdm/ps/202-direct-io.swift) =================================================================== --- trunk/tests/cdm/ps/pinned/202-direct-io.swift (rev 0) +++ trunk/tests/cdm/ps/pinned/202-direct-io.swift 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,16 @@ + +type file; + +app (file o) merge (file i, file j) +{ + merge @i @j @o; +} + +// Will be transformed by CDM to 202/input-[12].txt +file f1<"202-input-1.txt">; +file f2<"202-input-2.txt">; + +// Will be transformed by CDM to 202/output.txt: +file f3<"202-output.txt">; + +f3 = merge(f1, f2); Copied: trunk/tests/cdm/ps/pinned/README.txt (from rev 3616, trunk/tests/cdm/ps/README.txt) =================================================================== --- trunk/tests/cdm/ps/pinned/README.txt (rev 0) +++ trunk/tests/cdm/ps/pinned/README.txt 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,5 @@ + +000 series : noop tests +200 series : DIRECT tests +300 series : BROADCAST test + Copied: trunk/tests/cdm/ps/pinned/fs.template.data (from rev 3616, trunk/tests/cdm/ps/fs.template.data) =================================================================== --- trunk/tests/cdm/ps/pinned/fs.template.data (rev 0) +++ trunk/tests/cdm/ps/pinned/fs.template.data 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,4 @@ +rule .*201-output.txt DIRECT _PWD_/201 + +rule .*202-input.*.txt DIRECT _PWD_/202 +rule .*202-output.txt DIRECT _PWD_/202 Copied: trunk/tests/cdm/ps/pinned/merge.sh (from rev 3616, trunk/tests/cdm/ps/merge.sh) =================================================================== --- trunk/tests/cdm/ps/pinned/merge.sh (rev 0) +++ trunk/tests/cdm/ps/pinned/merge.sh 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,7 @@ +#!/bin/sh + +F1=$1 +F2=$2 +F3=$3 + +cat $F1 $F2 > $F3 Copied: trunk/tests/cdm/ps/pinned/sites.template.xml (from rev 3616, trunk/tests/cdm/ps/sites.template.xml) =================================================================== --- trunk/tests/cdm/ps/pinned/sites.template.xml (rev 0) +++ trunk/tests/cdm/ps/pinned/sites.template.xml 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,24 @@ + + + + + + _WORK_ + file + + + + + + _HOST_ + 2.55 + 10000 + 4 + 8 + 1000 + 1 + 4 + _WORK_ + + + Copied: trunk/tests/cdm/ps/pinned/swift.properties (from rev 3616, trunk/tests/cdm/ps/swift.properties) =================================================================== --- trunk/tests/cdm/ps/pinned/swift.properties (rev 0) +++ trunk/tests/cdm/ps/pinned/swift.properties 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,353 @@ +sites.file=${swift.home}/etc/sites.xml +tc.file=${swift.home}/etc/tc.data + +# +# The host name of the submit machine is used by GRAM as a callback +# address to report the status of submitted jobs. In general, Swift +# can automatically detect the host name of the local machine. +# However, if the machine host name is improperly configured or if +# it does not represent a valid DNS entry, certain services (such as +# GRAM) will not be able to send job status notifications back to +# the client. The value of this property can be an IP address. +# +# Format: +# hostname=string +# + + +#hostname=localhost + +# +# A TCP port range can be specified to restrict the ports on which GRAM +# callback services are started. This is likely needed if your submit +# host is behind a firewall, in which case the firewall should be +# configured to allow incoming connections on ports in the range. +# +# Format: +# tcp.port.range=start,end +# + +#tcp.port.range=50000,50100 + +# +# false - means an error will be immediately reported and cause the +# workflow to abort. At this time remote jobs that are already +# running will not be canceled +# true - means that Swift will try to do as much work as possible and +# report all errors encountered at the end. However, "errors" +# here only applies to job execution errors. Certain errors +# that are related to the Swift implementation (should such +# errors occur) will still be reported eagerly. +# +# Default: false +# +lazy.errors=false + +# +# What algorithm to use for caching of remote files. LRU (as in what +# files to purge) is the only implementation right now. One can set +# a target size (in bytes) for a host by using the swift:storagesize +# profile for a host in sites.xml +# +# Default: LRU +# +caching.algorithm=LRU + +# +# true - generate a provenance graph in .dot format (Swift will +# choose a random file name) +# false - do not generate a provenance graph +# - generate a provenange graph in the give file name +# +# Default: false +# +pgraph=false + + +# +# graph properties for the provenance graph (.dot specific) +# +# Default: splines="compound", rankdir="TB" +# +pgraph.graph.options=splines="compound", rankdir="TB" + + +# +# node properties for the provenance graph (.dot specific) +# +# Default: color="seagreen", style="filled" +# +pgraph.node.options=color="seagreen", style="filled" + +# +# true - clustering of small jobs is enabled. Clustering works in the +# following way: If a job is clusterable (meaning that it has the +# GLOBUS::maxwalltime profile specified in tc.data and its value +# is less than the value of the "clustering.min.time" property) it will +# be put in a clustering queue. The queue is processed at intervals +# specified by the "clustering.queue.delay" property. The processing +# of the clustering queue consists of selecting compatible jobs and +# grouping them in clusters whose max wall time does not exceed twice +# the value of the "clustering.min.time" property. Two or more jobs are +# considered compatible if they share the same site and do not have +# conflicting profiles (e.g. different values for the same environment +# variable). +# false - clustering of small jobs is disabled. +# +# Default: false +# +clustering.enabled=false + + +# +# - the intervals at which the clustering queue is processed +# +# Default: 4 +# +clustering.queue.delay=4 + +# +# - the threshold time for clustering +# +# Default: 60 +# +clustering.min.time=60 + +# +# Kickstart is a useful tool that can be used to gather various information +# about a remote process. Before it can be used it must be installed on the +# remote site and the corresponding entry be set in the sites file. +# This option allows controlling of how Swift uses Kickstart. The following +# values are possible: +# false - do not use Kickstart +# true - use Kickstart. If a job is scheduled on a site that does not have +# Kickstart installed, that job will fail. +# maybe - Use Kickstart if installed (i.e. the entry is present in the sites +# file) +# +# Default: maybe +# + +kickstart.enabled=maybe + +# +# Indicates when Kickstart records should be fetched from the remote site: +# true - always transfer Kickstart records if Kickstart was used (see +# kickstart.enabled) +# false - only transfer Kickstart records if the job fails +# +# Default: false +# + +kickstart.always.transfer=false + +# +# Indicates when wrapper logs should be fetched from the remote site: +# true - always transfer wrapper logs +# false - only transfer wrapper logs if the job fails +# +# Default: false +# + +wrapperlog.always.transfer=false + +########################################################################### +# Throttling options # +########################################################################### +# +# For the throttling parameters, valid values are either a positive integer +# or "off" (without the quotes). +# + +# +# Limits the number of concurrent submissions for a workflow instance. This +# throttle only limits the number of concurrent tasks (jobs) that are being +# sent to sites, not the total number of concurrent jobs that can be run. +# The submission stage in GRAM is one of the most CPU expensive stages (due +# mostly to the mutual authentication and delegation). Having too many +# concurrent submissions can overload either or both the submit host CPU +# and the remote host/head node causing degraded performance. +# +# Default: 4 +# + +throttle.submit=4 +#throttle.submit=off + +# +# Limits the number of concurrent submissions for any of the sites Swift will +# try to send jobs to. In other words it guarantees that no more than the +# value of this throttle jobs sent to any site will be concurrently in a state +# of being submitted. +# +# Default: 2 +# + +throttle.host.submit=2 +#throttle.host.submit=off + +# +# The Swift scheduler has the ability to limit the number of concurrent jobs +# allowed on a site based on the performance history of that site. Each site +# is assigned a score (initially 1), which can increase or decrease based +# on whether the site yields successful or faulty job runs. The score for a +# site can take values in the (0.1, 100) interval. The number of allowed jobs +# is calculated using the following formula: +# 2 + score*throttle.score.job.factor +# This means a site will always be allowed at least two concurrent jobs and +# at most 2 + 100*throttle.score.job.factor. With a default of 4 this means +# at least 2 jobs and at most 402. +# +# Default: 4 +# + +throttle.score.job.factor=0.2 +#throttle.score.job.factor=off + + +# +# Limits the total number of concurrent file transfers that can happen at any +# given time. File transfers consume bandwidth. Too many concurrent transfers +# can cause the network to be overloaded preventing various other signalling +# traffic from flowing properly. +# +# Default: 4 +# + +throttle.transfers=4 +#throttle.transfers=off + +# Limits the total number of concurrent file operations that can happen at any +# given time. File operations (like transfers) require an exclusive connection +# to a site. These connections can be expensive to establish. A large number +# of concurrent file operations may cause Swift to attempt to establish many +# such expensive connections to various sites. Limiting the number of concurrent +# file operations causes Swift to use a small number of cached connections and +# achieve better overall performance. +# +# Default: 8 +# + +throttle.file.operations=8 +#throttle.file.operations=off + +# Indicates whether the working directory on the remote site should be +# left intact even when the workflow completes successfully. This can be +# used to inspect the site working directory for debugging purposes. +# +# Default: false +# + +sitedir.keep=false + +# number of time a job will be retried if it fails (giving a maximum of +# 1 + execution.retries attempts at execution) +# + +execution.retries=2 + + +# Enables/disables replication. Replication is used to deal with jobs sitting +# in batch queues for abnormally large amounts of time. If replication is enabled +# and certain conditions are met, Swift creates and submits replicas of jobs, and +# allows multiple instances of a job to compete. +# + +replication.enabled=false + +# If replication is enabled, this value specifies the minimum time, in seconds, +# a job needs to be queued in a batch queue in order to be considered for +# replication +# + +replication.min.queue.time=60 + +# The maximum number of replicas that Swift should attempt. + +replication.limit=3 + +# +# WARNING: This option is deprecated. Please use the hostname option. +# +# The IP address of the submit machine is used by GRAM as a callback +# address to report the status of submitted jobs. In general, Swift +# can automatically detect the IP address of the local machine. +# However, if the machine has more than one network interface, Swift +# will pick the first one, which may not be the right choice. It is +# recommended that this property is set properly before attempting to +# run jobs through GRAM. +# +# Format: +# ip.address=x.y.z.w +# + +#ip.address=127.0.0.1 + + +# Controls how Swift will communicate the result code of running user programs +# from workers to the submit side. In files mode, a file +# indicating success or failure will be created on the site shared filesystem. +# In provider mode, the execution provider job status will +# be used. Notably, GRAM2 does not return job statuses correctly, and so +# provider mode will not work with GRAM2. With other +# providers, it can be used to reduce the amount of filesystem access compared +# to files mode. +# +# status.mode=files + +# Controls how swift will supply parameters to the remote wrapper script. +# 'args' mode will pass parameters on the command line +# 'files' mode will pass parameters through an additional input file +# +# valid values: args, files +# Default: files +# +# wrapper.parameter.mode=args + +# Determines if Swift remote wrappers will be executed by specifying an +# absolute path, or a path relative to the job initial working directory +# +# valid values: absolute, relative +# wrapper.invocation.mode=absolute + +# +# Limits the number of concurrent iterations that each foreach statement +# can have at one time. This conserves memory for swift programs that +# have large numbers of iterations (which would otherwise all be executed +# in parallel). +# +# Default: 1024 +# + +foreach.max.threads=16384 + +# controls whether the log file will contain provenance information +# enabling this will increase the size of log files, sometimes +# significantly. + +provenance.log=false + +# Controls whether file staging is done by swift or by the execution +# provider. If set to false, the standard swift staging mechanism is +# used. If set to true, swift does not stage files. Instead, the +# execution provider is instructed to stage files in and out. +# +# Provider staging is experimental. +# +# When enabled, and when coasters are used as an execution provider, +# a staging mechanism can be selected for each site +# using the swift:stagingMethod site profile in sites.xml. The +# following is a list of accepted mechanisms: +# +# * file: Staging is done from a filesystem accessible to the +# coaster service (typically running on the head node) +# * proxy: Staging is done from a filesystem accessible to the +# client machine that swift is running on, and is proxied +# through the coaster service +# * sfs: (short for "shared filesystem") Staging is done by +# copying files to and from a filesystem accessible +# by the compute node (such as an NFS or GPFS mount). + + +use.provider.staging=true +provider.staging.pin.swiftfiles=false Copied: trunk/tests/cdm/ps/pinned/tc.template.data (from rev 3616, trunk/tests/cdm/ps/tc.template.data) =================================================================== --- trunk/tests/cdm/ps/pinned/tc.template.data (rev 0) +++ trunk/tests/cdm/ps/pinned/tc.template.data 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1,20 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null + +coasterslocal cp /bin/cp INSTALLED INTEL32::LINUX null +coasterslocal merge _DIR_/merge.sh INSTALLED INTEL32::LINUX null Copied: trunk/tests/cdm/ps/pinned/title.txt (from rev 3616, trunk/tests/cdm/ps/title.txt) =================================================================== --- trunk/tests/cdm/ps/pinned/title.txt (rev 0) +++ trunk/tests/cdm/ps/pinned/title.txt 2010-09-08 19:55:10 UTC (rev 3618) @@ -0,0 +1 @@ +CDM tests with provider staging From noreply at svn.ci.uchicago.edu Wed Sep 8 14:56:15 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 14:56:15 -0500 (CDT) Subject: [Swift-commit] r3619 - trunk/tests/cdm/ps/pinned Message-ID: <20100908195615.207E39CD19@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-08 14:56:15 -0500 (Wed, 08 Sep 2010) New Revision: 3619 Modified: trunk/tests/cdm/ps/pinned/swift.properties Log: For these tests set provider.staging.pin.swiftfiles=true Modified: trunk/tests/cdm/ps/pinned/swift.properties =================================================================== --- trunk/tests/cdm/ps/pinned/swift.properties 2010-09-08 19:55:10 UTC (rev 3618) +++ trunk/tests/cdm/ps/pinned/swift.properties 2010-09-08 19:56:15 UTC (rev 3619) @@ -350,4 +350,4 @@ use.provider.staging=true -provider.staging.pin.swiftfiles=false +provider.staging.pin.swiftfiles=true From noreply at svn.ci.uchicago.edu Wed Sep 8 15:13:21 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 15:13:21 -0500 (CDT) Subject: [Swift-commit] r3620 - trunk/libexec Message-ID: <20100908201321.5238D9CD19@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-08 15:13:21 -0500 (Wed, 08 Sep 2010) New Revision: 3620 Modified: trunk/libexec/vdl-int-staging.k Log: Bug fix Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-09-08 19:56:15 UTC (rev 3619) +++ trunk/libexec/vdl-int-staging.k 2010-09-08 20:13:21 UTC (rev 3620) @@ -127,7 +127,7 @@ stagingMethod := vdl:siteProfile(rhost, "swift:stagingMethod", default="proxy") pinOption := configProperty("provider.staging.pin.swiftfiles") - pin := if (pinOption == "" then ("pinned:") else ("")) + pin := if (pinOption == "true" then ("pinned:") else ("")) stageIn("{pin}{stagingMethod}://localhost/{swift.home}/libexec/{wrapper}", wrapper) From noreply at svn.ci.uchicago.edu Wed Sep 8 16:39:47 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Sep 2010 16:39:47 -0500 (CDT) Subject: [Swift-commit] r3621 - SwiftApps/SwiftR/Swift/exec Message-ID: <20100908213947.35FDC9CC90@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-08 16:39:47 -0500 (Wed, 08 Sep 2010) New Revision: 3621 Added: SwiftApps/SwiftR/Swift/exec/start-swift-workers SwiftApps/SwiftR/Swift/exec/worker.pl Log: Interim verion of modified worker for R and script to start persistent passive worker enemble. Added: SwiftApps/SwiftR/Swift/exec/start-swift-workers =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-workers (rev 0) +++ SwiftApps/SwiftR/Swift/exec/start-swift-workers 2010-09-08 21:39:47 UTC (rev 3621) @@ -0,0 +1,140 @@ +#! /bin/bash + +COMPUTEHOSTS=$1 +# COMPUTEHOSTS='crush thwomp stomp crank steamroller grind churn trounce thrash vanquish' +# COMPUTEHOSTS='communicado' + +shift + +# Find our bin dir (to use for running utility scripts) + +SWIFTRBIN=$(cd $(dirname $0); pwd) +echo SWIFTRBIN=$SWIFTRBIN +workerscript=$SWIFTRBIN/worker.pl + +function wait-and-start-workers +{ + # Look for: + # Passive queue processor initialized. Callback URI is http://140.221.8.62:55379 + + for try in $(seq 1 20); do + uriline=$(grep "Passive queue processor initialized. Callback URI is" $out 2> /dev/null) + if [ "_$uriline" = _ ]; then + sleep 1 + else + break; + fi + done + CONTACT=$(echo $uriline | sed -e 's/^.*http:/http:/') + echo Coaster contact: $CONTACT + + LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env? + mkdir -p $LOGDIR + + IDLETIMEOUT=$((60*60)) # 1 hour + + # CA="X509_CERT_DIR=$CERTDIR X509_CADIR=$CERTDIR" + for host in $(echo $COMPUTEHOSTS); do + timestamp=$(date "+%Y.%m%d.%H%M%S") + random=$(awk "BEGIN {printf \"%0.5d\", $RANDOM}") + ID=$timestamp.$random +# ssh $host WORKER_LOGGING_ENABLED=false $workerscript $CONTACT $ID $LOGDIR $IDLETIMEOUT & +# ssh $host '/bin/sh -c "'"WORKER_LOGGING_ENABLED=true $workerscript $CONTACT $ID $LOGDIR $IDLETIMEOUT"' & echo \$!"' >remotepid.$host &1 & echo PID=\$!"\' >remotepid.$host $sshpidfile + # ps $sshpids +} + +if ! which swift >& /dev/null; then + echo $0: \'swift\' command not found in PATH + exit 1 +fi + + +mkdir -p service +servicedir=service +cd service +out=`mktemp swift.stdouterr.XXXX` + +coaster-service -nosec >& coaster-service.log & +coasterservicepid=$! + +cat >sites.xml < + + + passive + 4 + .03 + 10000 + + /home/wilde/swiftwork + + +END + +cat >tc <cf <passivate.swift <& /dev/null + fi + for rpfile in $(ls -1 remotepid.*); do + rpid=$(cat $rpfile) + rhost=$(echo $rpfile | sed -e 's/remotepid.//') + echo from $rpfile: ssh $rhost kill $rpid + done +} + +trap onexit $TRAPS + +wait-and-start-workers & +starterpid=$! + +swift -config cf -tc.file tc -sites.file sites.xml passivate.swift 2>&1 | tee $out + +echo "==> Service started." + +wait Property changes on: SwiftApps/SwiftR/Swift/exec/start-swift-workers ___________________________________________________________________ Name: svn:executable + * Added: SwiftApps/SwiftR/Swift/exec/worker.pl =================================================================== --- SwiftApps/SwiftR/Swift/exec/worker.pl (rev 0) +++ SwiftApps/SwiftR/Swift/exec/worker.pl 2010-09-08 21:39:47 UTC (rev 3621) @@ -0,0 +1,1372 @@ +#!/usr/bin/perl +# Args: +# +# where: +# - comma separated list of URIs for the coaster service; they +# will be tried in order +# - some block id (the log file will be named based on this) +# - some directory in which the logs should go +# + +use IO::Socket; +use File::Basename; +use File::Path; +use File::Copy; +use Time::HiRes qw(time); +use Cwd; +use POSIX ":sys_wait_h"; +use strict; +use warnings; + +# Create a stack of job slot ids (for auxiliary services): +# each slot has a small integer id +# take a slot from the stack when starting a job +# return the slot to the stack (making it available) when a job ends +# pass the slot id to the job via env var SWIFT_JOB_SLOT +# jobs can use this to reach a persistent process asscoaited with the "slot" +# initial use is to send work to persistent R service workers. + +use constant MAXJOBSLOTS => 128; # FIXME: determine maxworkers dynamically? +my @jobslots=(); +for( my $jobslot=MAXJOBSLOTS-1; $jobslot>=0; $jobslot--) { + push @jobslots, $jobslot; +} + +# If ASYNC is on, the following will be done: +# 1. Stageouts will be done in parallel +# 2. The job status will be set to "COMPLETED" as soon as the last +# file is staged out (and before any cleanup is done). +use constant ASYNC => 1; + +use constant { + TRACE => 0, + DEBUG => 1, + INFO => 2, + WARN => 3, + ERROR => 4, +}; + +use constant { + CONTINUE => 0, + YIELD => 1, +}; + +my $LOGLEVEL = DEBUG; + +my @LEVELS = ("TRACE", "DEBUG", "INFO ", "WARN ", "ERROR"); + +use constant { + REPLY_FLAG => 0x00000001, + FINAL_FLAG => 0x00000002, + ERROR_FLAG => 0x00000004, + PROGRESSIVE_FLAG => 0x00000008 +}; + +use constant { + COMPLETED => 0x07, + FAILED => 0x05, + ACTIVE => 0x02, + STAGEIN => 0x10, + STAGEOUT => 0x11, +}; + +my $TAG = 0; +use constant RETRIES => 3; +use constant REPLYTIMEOUT => 180; +use constant MAXFRAGS => 16; +use constant MAX_RECONNECT_ATTEMPTS => 3; + +my $LASTRECV = 0; +my $JOBS_RUNNING = 0; + +my $JOB_COUNT = 0; + +use constant BUFSZ => 2048; + +# 60 seconds by default. Note that since there is no configuration handshake +# this would have to match the default interval in the service in order to avoid +# "lost heartbeats". +use constant HEARTBEAT_INTERVAL => 2 * 60; + +my $ID = "-"; + +sub wlog { + my $msg; + my $level = shift; + if ($level >= $LOGLEVEL) { + foreach $msg (@_) { + my $timestamp = timestring(); + my $msgline = sprintf("%s %s %s %s", + $timestamp, + $LEVELS[$level], + $ID, $msg); + print LOG $msgline; + } + } + return 1; +} + +# Command-line arguments: + +my $URISTR=$ARGV[0]; +my $BLOCKID=$ARGV[1]; +my $LOGDIR=$ARGV[2]; +my $IDLETIMEOUT = ( $#ARGV <= 2 ) ? (4 * 60) : $ARGV[3]; + + +# REQUESTS holds a map of incoming requests +my %REQUESTS = (); + +# REPLIES stores the state of (outgoing) commands for which replies are expected +my %REPLIES = (); + +my $LOG = logfilename($LOGDIR, $BLOCKID); + +my %HANDLERS = ( + "SHUTDOWN" => \&shutdownw, + "SUBMITJOB" => \&submitjob, + "REGISTER" => \®ister, + "HEARTBEAT" => \&heartbeat, + "WORKERSHELLCMD" => \&workershellcmd, +); + +my @CMDQ = (); + +my @URIS = split(/,/, $URISTR); +my @SCHEME; +my @HOSTNAME; +my @PORT; +my $URI; +foreach $URI (@URIS) { + if ($URI =~ /(.*):\/\//) { push(@SCHEME, $1); } else { die "Could not parse url scheme: $URI"; } + if ($URI =~ /.*:\/\/(.*):/) { push(@HOSTNAME, $1); } else { die "Could not parse url hostname: $URI"; } + if ($URI =~ /.*:\/\/.*:(.*)/) { push(@PORT, $1); } else { die "Could not parse url port: $URI"; } +} +my $SOCK; +my $LAST_HEARTBEAT = 0; + +my %JOBWAITDATA = (); +my %JOBDATA = (); + +sub logfilename { + $LOGDIR = shift; + $BLOCKID = shift; + my $result = undef; + my $uci; + if (-r "/proc/personality.sh") { + $uci = get_bg_uci(); + $result = "$LOGDIR/worker-$BLOCKID-$uci.log"; + } + else { + $result = "$LOGDIR/worker-$BLOCKID.log"; + } + print STDERR "logfilename: $result\n"; + return $result; +} + +# Get the BlueGene Universal Component Identifier from Zepto +sub get_bg_uci() { + my %vars = file2hash("/proc/personality.sh"); + my $uci = $vars{"BG_UCI"}; + return $uci; +} + +# Read a file into a hash, with file formatted as: +# KEY=VALUE +sub file2hash() { + my $file = shift; + my %hash; + open FILE, "<$file"; + while () + { + chomp; + my ($key, $val) = split /=/; + $hash{$key} = $val; + } + close FILE; + return %hash; +} + +sub timestring() { + my $t = sprintf("%.3f", time()); + #my @d = localtime(time()); + #my $t = sprintf("%i/%02i/%02i %02i:%02i", + # $d[5]+1900, $d[4], $d[3], $d[2], $d[1]); + return $t; +} + +sub hts { + my ($H) = @_; + + my $k; + my $s = "{"; + my $first = 1; + + for $k (keys %$H) { + if (!$first) { + $s = $s.", "; + } + else { + $first = 0; + } + $s = $s."$k = $$H{$k}"; + } + + return $s."}"; +} + +sub reconnect() { + my $fail = 0; + my $success; + my $i; + my $j; + for ($i = 0; $i < MAX_RECONNECT_ATTEMPTS; $i++) { + wlog INFO, "Connecting ($i)...\n"; + my $sz = @HOSTNAME; + $success = 0; + for ($j = 0; $j < $sz; $j++) { + wlog DEBUG, "Trying $HOSTNAME[$j]:$PORT[$j]...\n"; + $SOCK = IO::Socket::INET->new(Proto=>'tcp', PeerAddr=>$HOSTNAME[$j], PeerPort=>$PORT[$j], Blocking=>1) || ($fail = 1); + if (!$fail) { + $success = 1; + last; + } + else { + wlog DEBUG, "Connection failed: $!. Trying other addresses\n"; + } + } + if ($success) { + $SOCK->setsockopt(SOL_SOCKET, SO_RCVBUF, 16384); + $SOCK->setsockopt(SOL_SOCKET, SO_SNDBUF, 32768); + wlog INFO, "Connected\n"; + $SOCK->blocking(0); + queueCmd(registerCB(), "REGISTER", $BLOCKID, ""); + last; + } + else { + my $delay = 2 ** $i; + wlog ERROR, "Connection failed for all addresses. Retrying in $delay seconds\n"; + select(undef, undef, undef, $delay); + } + } + if (!$success) { + die "Failed to connect: $!"; + } + $LAST_HEARTBEAT = time(); +} + +sub initlog() { + if (defined $ENV{"WORKER_LOGGING_ENABLED"}) { + open(LOG, ">>$LOG") or die "Failed to open log file: $!"; + my $b = select(LOG); + $| = 1; + select($b); + my $date = localtime; + wlog INFO, "$BLOCKID Logging started: $date\n"; + } + else { + $LOGLEVEL = 999; + } +} + + +sub init() { + logsetup(); + reconnect(); +} + +sub logsetup() { + my $schemes = join(", ", @SCHEME); + my $hosts = join(", ", @HOSTNAME); + my $ports = join(", ", @PORT); + wlog DEBUG, "uri=$URISTR\n"; + wlog DEBUG, "scheme=$schemes\n"; + wlog DEBUG, "host=$hosts\n"; + wlog DEBUG, "port=$ports\n"; + wlog DEBUG, "blockid=$BLOCKID\n"; + wlog DEBUG, "idletimeout=$IDLETIMEOUT\n"; +} + +sub sendm { + my ($tag, $flags, $msg) = @_; + my $len = length($msg); + my $buf = pack("VVV", $tag, $flags, $len); + $buf = $buf.$msg; + + wlog(DEBUG, "OUT: len=$len, tag=$tag, flags=$flags\n"); + wlog(TRACE, "$msg\n"); + + #($SOCK->send($buf) == length($buf)) || reconnect(); + $SOCK->blocking(1); + eval {defined($SOCK->send($buf))} or wlog(WARN, "Send failed: $!\n") and die "Send failed: $!"; + #eval {defined($SOCK->send($buf))} or wlog(WARN, "Send failed: $!\n"); +} + +sub sendFrags { + my ($tag, $flg, $data) = @_; + + my $flg2; + my $msg; + my $yield; + if (defined($$data{"tag"})) { + $tag = $$data{"tag"}; + } + do { + ($flg2, $msg, $yield) = $$data{"nextData"}($data); + sendm($tag, $flg | $flg2, $msg); + } while (($flg2 & FINAL_FLAG) == 0 && !$yield); + + if (($flg2 & FINAL_FLAG) == 0) { + # final flag not set; put it back in the queue + wlog DEBUG, "$tag yielding\n"; + $$data{"tag"} = $tag; + queueCmdCustomDataHandling($REPLIES{$tag}, $data); + } + else { + if (exists($REPLIES{$tag})) { + my $record = $REPLIES{$tag}; + my ($cont, $start) = ($$record[0], $$record[1]); + if (defined($$cont{"dataSent"})) { + $$cont{"dataSent"}($cont, $tag); + } + } + wlog(DEBUG, "done sending frags for $tag\n"); + } +} + +sub nextArrayData { + my ($state) = @_; + + my $index = $$state{"index"}; + $$state{"index"} = $index + 1; + my $data = $$state{"data"}; + if ($index > $#$data) { + die "Index out of bounds"; + } + return ($index >= $#$data ? FINAL_FLAG : 0, $$data[$index], CONTINUE); +} + +sub arrayData { + return { + "index" => 0, + "nextData" => \&nextArrayData, + "data" => \@_ + }; +} + +sub nextFileData { + my ($state) = @_; + + my $s = $$state{"state"}; + if ($s == 0) { + $$state{"state"} = $s + 1; + return (0, $$state{"cmd"}, CONTINUE); + } + elsif ($s == 1) { + $$state{"state"} = $s + 1; + return (0, pack("VV", $$state{"size"}, 0), CONTINUE); + } + elsif ($s == 2) { + $$state{"state"} = $s + 1; + return (0, $$state{"lname"}, CONTINUE); + } + elsif ($s == 3) { + $$state{"state"} = $s + 1; + $$state{"sent"} = 0; + return ($$state{"size"} == 0 ? FINAL_FLAG : 0, $$state{"rname"}, CONTINUE); + } + else { + my $handle = $$state{"handle"}; + my $buffer; + my $sz = read($handle, $buffer, 8192); + if (!defined $sz) { + wlog INFO, "Failed to read data from file: $!\n"; + return (FINAL_FLAG + ERROR_FLAG, "$!", CONTINUE); + } + elsif ($sz == 0 && $$state{"sent"} < $$state{"size"}) { + wlog INFO, "File size mismatch. $$state{'size'} vs. $$state{'sent'}\n"; + return (FINAL_FLAG + ERROR_FLAG, "File size mismatch. Expected $$state{'size'}, got $$state{'sent'}", CONTINUE); + } + $$state{"sent"} += $sz; + wlog DEBUG, "size: $$state{'size'}, sent: $$state{'sent'}\n"; + if ($$state{"sent"} == $$state{"size"}) { + close $handle; + } + return (($$state{"sent"} < $$state{"size"}) ? 0 : FINAL_FLAG, $buffer, YIELD); + } +} + +sub fileData { + my ($cmd, $lname, $rname) = @_; + + my $desc; + if (!open($desc, "<", "$lname")) { + wlog WARN, "Failed to open $lname\n"; + # let it go on for now. The next read from the descriptor will fail + } + return { + "cmd" => $cmd, + "state" => 0, + "handle" => $desc, + "nextData" => \&nextFileData, + "size" => -s $lname, + "lname" => $lname, + "rname" => $rname + }; +} + + +sub sendCmdInt { + my ($cont, $state) = @_; + my $ctag = $$state{"tag"}; + if (!defined $ctag) { + $ctag = $TAG++; + registerCmd($ctag, $cont); + } + sendFrags($ctag, 0, $state); + return $ctag; +} + +sub sendCmd { + my @cmd = @_; + my $cont = shift(@cmd); + return sendCmdInt($cont, arrayData(@cmd)); +} + +sub queueCmd { + my @cmd = @_; + my $cont = shift(@cmd); + push @CMDQ, [$cont, arrayData(@cmd)]; +} + +sub queueCmdCustomDataHandling { + my ($cont, $state) = @_; + push @CMDQ, [$cont, $state]; +} + +sub sendReply { + my ($tag, @msgs) = @_; + sendFrags($tag, REPLY_FLAG, arrayData(@msgs)); +} + +sub sendError { + my ($tag, @msgs) = @_; + sendFrags($tag, REPLY_FLAG | ERROR_FLAG, arrayData(@msgs)); +} + +sub unpackData { + my ($data) = @_; + + my $lendata = length($data); + if ($lendata < 12) { + wlog WARN, "Received faulty message (length < 12: $lendata)\n"; + die "Received faulty message (length < 12: $lendata)"; + } + my $tag = unpack("V", substr($data, 0, 4)); + my $flg = unpack("V", substr($data, 4, 4)); + my $len = unpack("V", substr($data, 8, 4)); + my $msg; + my $frag; + my $alen = 0; + while ($alen < $len) { + $SOCK->recv($frag, $len - $alen); + $alen = $alen + length($frag); + $msg = $msg.$frag; + } + + my $actuallen = length($msg); + wlog(TRACE, " IN: len=$len, actuallen=$actuallen, tag=$tag, flags=$flg, $msg\n"); + if ($len != $actuallen) { + wlog(WARN, "len != actuallen\n"); + } + return ($tag, $flg, $msg); +} + +sub processRequest { + my ($state, $tag, $timeout, $err, $fin, $msg) = @_; + + my $request = $$state{"request"}; + if (!defined($request)) { + $request = []; + $$state{"request"} = $request; + } + push(@$request, $msg); + + if ($timeout) { + sendError($tag, ("Timed out waiting for all fragments")); + } + elsif (!$fin) { + return; + } + else { + wlog DEBUG, "Processing request\n"; + my $cmd = shift(@$request); + wlog DEBUG, "Cmd is $cmd\n"; + if (exists($HANDLERS{$cmd})) { + $HANDLERS{$cmd}->($tag, 0, $request); + } + else { + sendError($tag, ("Unknown command: $cmd")); + } + } +} + +sub process { + my ($tag, $flg, $msg) = @_; + + + my $reply = $flg & REPLY_FLAG; + my ($record, $cont, $start); + + if ($reply) { + if (exists($REPLIES{$tag})) { + $record = $REPLIES{$tag}; + ($cont, $start) = ($$record[0], $$record[1]); + } + else { + wlog(WARN, "received reply to unregistered command (tag=$tag). Discarding.\n"); + return; + } + } + else { + $LASTRECV = time(); + if (!exists($REQUESTS{$tag})) { + $REQUESTS{$tag} = [{"dataIn" => \&processRequest}, time()]; + wlog DEBUG, "New request ($tag)\n"; + } + $record = $REQUESTS{$tag}; + ($cont, $start) = ($$record[0], $$record[1]); + } + + my $fin = $flg & FINAL_FLAG; + my $err = $flg & ERROR_FLAG; + + + if ($fin) { + if ($reply) { + # A reply for a command sent by us has been received, which means that + # the lifecycle of the command is complete, therefore the state of + # that command can be deleted. + delete($REPLIES{$tag}); + } + else { + # All fragments of a request have been received. Since the record is + # stored in $cont, $tag, $err, $fin, $msg, we can remove it from the + # table of (partial) incoming requests + delete($REQUESTS{$tag}); + } + wlog DEBUG, "Fin flag set\n"; + } + + $$cont{"dataIn"}($cont, $tag, 0, $err, $fin, $msg); + + return 1; +} + +sub checkTimeouts2 { + my ($hash) = @_; + + my $now = time(); + my @del = (); + + my $k; + my $v; + + while (($k, $v) = each(%$hash)) { + if ($now - $$v[1] > REPLYTIMEOUT) { + push(@del, $k); + my $cont = $$v[0]; + $$cont{"dataIn"}($cont, $k, 1, 0, 0, "Reply timeout"); + } + } + + foreach $k (@del) { + delete $$hash{$k}; + } +} + +my $LASTTIMEOUTCHECK = 0; + +sub checkTimeouts { + my $time = time(); + if ($time - $LASTTIMEOUTCHECK < 1) { + return; + } + $LASTTIMEOUTCHECK = $time; + checkTimeouts2(\%REQUESTS); + checkTimeouts2(\%REPLIES); + if ($LASTRECV != 0) { + my $dif = $time - $LASTRECV; + wlog TRACE, "time: $time, lastrecv: $LASTRECV, dif: $dif\n"; + if ($dif >= $IDLETIMEOUT && $JOBS_RUNNING == 0 ) { + wlog INFO, "Idle time exceeded (time=$time, LASTRECV=$LASTRECV, dif=$dif)\n"; + die "Idle time exceeded"; + } + } +} + +sub recvOne { + my $data; + $SOCK->blocking(0); + $SOCK->recv($data, 12); + if (length($data) > 0) { + # wlog DEBUG, "Received " . unpackData($data) . "\n"; + eval { process(unpackData($data)); } || (wlog ERROR, "Failed to process data: $@\n" && die "Failed to process data: $@"); + } + else { + #sleep 1ms + select(undef, undef, undef, 0.001); + checkTimeouts(); + } +} + +sub registerCmd { + my ($tag, $cont) = @_; + + wlog DEBUG, "Replies: ".hts(\%REPLIES)."\n"; + + $REPLIES{$tag} = [$cont, time(), ()]; +} + + +sub mainloop { + while(1) { + loopOne(); + } +} + +sub loopOne { + my $cmd; + if (time() - $LAST_HEARTBEAT > HEARTBEAT_INTERVAL) { + queueCmd(heartbeatCB(), "HEARTBEAT"); + $LAST_HEARTBEAT = time(); + } + # send whatever is now queued; don't clear the queue, since + # things may be added to it while stuff is being sent + my $sz = scalar(@CMDQ); + for (my $i = 0; $i < $sz; $i++) { + $cmd = pop(@CMDQ); + sendCmdInt(@$cmd); + } + checkJobs(); + recvOne(); +} + +sub printreply { + my ($tag, $timeout, $err, $fin, $reply) = @_; + if ($timeout) { + wlog WARN, "Timed out waiting for reply to $tag\n"; + } + else { + wlog DEBUG, "$$reply[0]\n"; + } +} + +sub nullCB { + return { + "dataIn" => sub {} + }; +} + +sub registerCB { + return { + "dataIn" => \®isterCBDataIn + }; +} + +sub registerCBDataIn { + my ($state, $tag, $timeout, $err, $fin, $reply) = @_; + + if ($timeout) { + die "Failed to register (timeout)\n"; + } + elsif ($err) { + die "Failed to register (service returned error: ".join("\n", $reply).")"; + } + else { + $ID = $reply; + wlog INFO, "Registration successful. ID=$ID\n"; + } +} + +sub heartbeatCB { + return { + "dataIn" => \&heartbeatCBDataIn + }; +} + +sub heartbeatCBDataIn { + my ($state, $tag, $timeout, $err, $fin, $reply) = @_; + + if ($timeout) { + if (time() - $LAST_HEARTBEAT > 2 * HEARTBEAT_INTERVAL) { + wlog WARN, "No heartbeat replies in a while. Dying.\n"; + die "No response to heartbeat\n"; + } + } + elsif ($err) { + wlog WARN, "Heartbeat failed: $reply\n"; + die "Heartbeat failed: $reply\n"; + } + else { + wlog DEBUG, "Heartbeat acknowledged\n"; + } +} + + +sub register { + my ($tag, $timeout, $reply) = @_; + sendReply($tag, ("OK")); +} + + +sub shutdownw { + my ($tag, $timeout, $msgs) = @_; + wlog DEBUG, "Shutdown command received\n"; + sendReply($tag, ("OK")); + select(undef, undef, undef, 1); + wlog INFO, "Acknowledged shutdown. Exiting\n"; + wlog INFO, "Ran a total of $JOB_COUNT jobs\n"; + exit 0; +} + +sub heartbeat { + my ($tag, $timeout, $msgs) = @_; + sendReply($tag, ("OK")); +} + +sub workershellcmd { + my ($tag, $timeout, $msgs) = @_; + my $cmd = $$msgs[1]; + my $out; + if ($cmd =~ m/cd\s*(.*)/) { + wlog DEBUG, "chdir $1\n"; + chdir $1; + if ($! ne '') { + sendError($tag, ("$!")); + } + else { + sendReply($tag, ("OK", "")); + } + } + elsif ($cmd =~ m/mls\s*(.*)/) { + wlog DEBUG, "mls $1\n"; + $out = `ls -d $1 2>/dev/null`; + sendReply($tag, ("OK", "$out")); + } + else { + $out = `$cmd 2>&1`; + sendReply($tag, ("OK", "$out")); + } +} + +sub urisplit { + my ($name) = @_; + + if (index($name, ":") == -1) { + return ("file", $name); + } + + my ($protocol, $path) = split(/:\/\//, $name, 2); + + return ($protocol, $path); +} + +sub getFileCB { + my ($jobid, $src, $dst) = @_; + + my ($protocol, $path) = urisplit($src); + wlog DEBUG, "$jobid src: $src, protocol: $protocol, path: $path\n"; + + if (($protocol eq "file") || ($protocol eq "proxy")) { + wlog DEBUG, "Opening $dst...\n"; + my $dir = dirname($dst); + if (-f $dir) { + die "$jobid Cannot create directory $dir. A file with this name already exists"; + } + if (!-d $dir) { + if (!mkpath($dir)) { + die "Cannot create directory $dir. $!"; + } + } + # don't try open(DESC, ...) (as I did). It will use the same reference + # and concurrent operations will fail. + my $desc; + if (!open($desc, ">", "$dst")) { + die "Failed to open $dst: $!"; + } + else { + wlog DEBUG, "$jobid Opened $dst\n"; + return { + "jobid" => $jobid, + "dataIn" => \&getFileCBDataIn, + "state" => 0, + "lfile" => $dst, + "desc" => $desc + }; + } + } + else { + return { + "jobid" => $jobid, + "dataIn" => \&getFileCBDataInIndirect, + "lfile" => $dst, + }; + } +} + +sub getFileCBDataInIndirect { + my ($state, $tag, $timeout, $err, $fin, $reply) = @_; + + my $jobid = $$state{"jobid"}; + wlog DEBUG, "$jobid getFileCBDataInIndirect jobid: $jobid, tag: $tag, err: $err, fin: $fin\n"; + if ($err) { + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "520", "Error staging in file: $reply")); + delete($JOBDATA{$jobid}); + return; + } + elsif ($timeout) { + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "521", "Timeout staging in file")); + delete($JOBDATA{$jobid}); + return; + } + if ($fin) { + stagein($jobid); + } +} + + +sub getFileCBDataIn { + my ($state, $tag, $timeout, $err, $fin, $reply) = @_; + + my $s = $$state{"state"}; + my $jobid = $$state{"jobid"}; + wlog DEBUG, "$jobid getFileCBDataIn jobid: $jobid, state: $s, tag: $tag, err: $err, fin: $fin\n"; + if ($err) { + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "520", "Error staging in file: $reply")); + delete($JOBDATA{$jobid}); + return; + } + elsif ($timeout) { + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "521", "Timeout staging in file")); + delete($JOBDATA{$jobid}); + return; + } + elsif ($s == 0) { + $$state{"state"} = 1; + $$state{"size"} = unpack("V", $reply); + my $lfile = $$state{"lfile"}; + } + else { + my $desc = $$state{"desc"}; + if (!(print {$desc} $reply)) { + close $desc; + wlog DEBUG, "$jobid Could not write to file: $!. Descriptor was $desc; lfile: $$state{'lfile'}\n"; + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "522", "Could not write to file: $!")); + delete($JOBDATA{$jobid}); + return; + } + } + if ($fin) { + my $desc = $$state{"desc"}; + close $desc; + wlog DEBUG, "$jobid Closed $$state{'lfile'}\n"; + stagein($jobid); + } +} + +sub stagein { + my ($jobid) = @_; + + my $STAGE = $JOBDATA{$jobid}{"stagein"}; + my $STAGED = $JOBDATA{$jobid}{"stageind"}; + my $STAGEINDEX = $JOBDATA{$jobid}{"stageindex"}; + + if (scalar @$STAGE <= $STAGEINDEX) { + wlog DEBUG, "$jobid Done staging in files ($STAGEINDEX, $STAGE)\n"; + $JOBDATA{$jobid}{"stageindex"} = 0; + sendCmd((nullCB(), "JOBSTATUS", $jobid, ACTIVE, "0", "workerid=$ID")); + forkjob($jobid); + } + else { + if ($STAGEINDEX == 0) { + sendCmd((nullCB(), "JOBSTATUS", $jobid, STAGEIN, "0", "workerid=$ID")); + } + wlog DEBUG, "$jobid Staging in $$STAGE[$STAGEINDEX]\n"; + $JOBDATA{$jobid}{"stageindex"} = $STAGEINDEX + 1; + my ($protocol, $path) = urisplit($$STAGE[$STAGEINDEX]); + if ($protocol eq "sfs") { + if (!copy($path, $$STAGED[$STAGEINDEX])) { + wlog DEBUG, "$jobid Error staging in $path: $!\n"; + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "524", "$@")); + } + else { + stagein($jobid); + } + } + else { + my $state; + eval { + $state = getFileCB($jobid, $$STAGE[$STAGEINDEX], $$STAGED[$STAGEINDEX]); + }; + if ($@) { + wlog DEBUG, "$jobid Error staging in file: $@\n"; + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "524", "$@")); + } + else { + sendCmd(($state, "GET", $$STAGE[$STAGEINDEX], $$STAGED[$STAGEINDEX])); + } + } + } +} + + +sub stageout { + my ($jobid) = @_; + + wlog DEBUG, "$jobid Staging out\n"; + my $STAGE = $JOBDATA{$jobid}{"stageout"}; + my $STAGED = $JOBDATA{$jobid}{"stageoutd"}; + my $STAGEINDEX = $JOBDATA{$jobid}{"stageindex"}; + + my $sz = scalar @$STAGE; + wlog DEBUG, "sz: $sz, STAGEINDEX: $STAGEINDEX\n"; + if (scalar @$STAGE <= $STAGEINDEX) { + $JOBDATA{$jobid}{"stageindex"} = 0; + wlog DEBUG, "$jobid No more stageouts. Doing cleanup.\n"; + cleanup($jobid); + } + else { + my $lfile = $$STAGE[$STAGEINDEX]; + if (-e $lfile) { + if ($STAGEINDEX == 0) { + wlog DEBUG, "$jobid Sending STAGEOUT status\n"; + sendCmd((nullCB(), "JOBSTATUS", $jobid, STAGEOUT, "0", "workerid=$ID")); + } + my $rfile = $$STAGED[$STAGEINDEX]; + $JOBDATA{$jobid}{"stageindex"} = $STAGEINDEX + 1; + wlog DEBUG, "$jobid Staging out $lfile.\n"; + my ($protocol, $path) = urisplit($rfile); + if ($protocol eq "file" || $protocol eq "proxy") { + queueCmdCustomDataHandling(putFileCB($jobid), fileData("PUT", $lfile, $rfile)); + } + elsif ($protocol eq "sfs") { + if (!copy($lfile, $path)) { + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "528", "$!")); + return; + } + else { + stageout($jobid); + } + } + else { + queueCmd((putFileCB($jobid), "PUT", pack("VV", 0, 0), $lfile, $rfile)); + } + wlog DEBUG, "$jobid PUT sent.\n"; + } + else { + wlog INFO, "$jobid Skipping stageout of missing file ($lfile)\n"; + $JOBDATA{$jobid}{"stageindex"} = $STAGEINDEX + 1; + stageout($jobid); + } + } +} + +sub cleanup { + my ($jobid) = @_; + + my $ec = $JOBDATA{$jobid}{"exitcode"}; + if (ASYNC) { + if ($ec == 0) { + queueCmd((nullCB(), "JOBSTATUS", $jobid, COMPLETED, "0", "")); + } + else { + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "$ec", "Job failed with an exit code of $ec")); + } + } + + if ($ec != 0) { + wlog DEBUG, "$jobid Job data: ".hts($JOBDATA{$jobid})."\n"; + wlog DEBUG, "$jobid Job: ".hts($JOBDATA{$jobid}{'job'})."\n"; + wlog DEBUG, "$jobid Job dir ".`ls -al $JOBDATA{$jobid}{'job'}{'directory'}`."\n"; + } + + my $CLEANUP = $JOBDATA{$jobid}{"cleanup"}; + my $c; + if ($ec == 0) { + for $c (@$CLEANUP) { + if ($c =~ /\/\.$/) { + chop $c; + chop $c; + } + wlog DEBUG, "$jobid Removing $c\n"; + rmtree($c, {safe => 1, verbose => 0}); + wlog DEBUG, "$jobid Removed $c\n"; + } + } + + if (!ASYNC) { + if ($ec == 0) { + queueCmd((nullCB(), "JOBSTATUS", $jobid, COMPLETED, "0", "")); + } + else { + wlog DEBUG, "$jobid Sending failure.\n"; + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "$ec", "Job failed with and exit code of $ec")); + } + } +} + +sub putFileCB { + my ($jobid) = @_; + return { + "jobid" => $jobid, + "dataIn" => \&putFileCBDataIn, + "dataSent" => \&putFileCBDataSent + }; +} + +sub putFileCBDataSent { + my ($state, $tag) = @_; + + if (ASYNC) { + wlog DEBUG, "putFileCBDataSent\n"; + my $jobid = $$state{"jobid"}; + if ($jobid != -1) { + wlog DEBUG, "Data sent, async is on. Staging out next file\n"; + stageout($jobid); + } + } +} + +sub putFileCBDataIn { + my ($state, $tag, $timeout, $err, $fin, $reply) = @_; + + wlog DEBUG, "putFileCBDataIn: $reply\n"; + + my $jobid = $$state{"jobid"}; + + if ($err || $timeout) { + if ($JOBDATA{$jobid}) { + wlog DEBUG, "Stage out failed ($reply)\n"; + queueCmd((nullCB(), "JOBSTATUS", $jobid, FAILED, "515", "Stage out failed ($reply)")); + delete($JOBDATA{$jobid}); + } + return; + } + elsif ($jobid != -1) { + if (!ASYNC) { + wlog DEBUG, "Stageout done; staging out next file\n"; + stageout($jobid); + } + } +} + +sub isabsolute { + my ($fn) = @_; + + return substr($fn, 0, 1) eq "/"; +} + + +sub submitjob { + my ($tag, $timeout, $msgs) = @_; + my $desc = $$msgs[0]; + my @lines = split(/\n/, $desc); + my $line; + my $JOBID = undef; + my %JOB = (); + my @JOBARGS = (); + my %JOBENV = (); + my @STAGEIN = (); + my @STAGEIND = (); + my @STAGEOUT = (); + my @STAGEOUTD = (); + my @CLEANUP = (); + foreach $line (@lines) { + $line =~ s/\\n/\n/; + $line =~ s/\\\\/\\/; + my @pair = split(/=/, $line, 2); + if ($pair[0] eq "arg") { + push @JOBARGS, $pair[1]; + } + elsif ($pair[0] eq "env") { + my @ep = split(/=/, $pair[1], 2); + $JOBENV{"$ep[0]"} = $ep[1]; + } + elsif ($pair[0] eq "identity") { + $JOBID = $pair[1]; + } + elsif ($pair[0] eq "stagein") { + my @pp = split(/\n/, $pair[1], 2); + push @STAGEIN, $pp[0]; + if (isabsolute($pp[1])) { + push @STAGEIND, $pp[1]; + } + else { + # there's the assumption here that the directory is sent before + # the stagein/out data. + push @STAGEIND, $JOB{directory}."/".$pp[1]; + } + } + elsif ($pair[0] eq "stageout") { + my @pp = split(/\n/, $pair[1], 2); + if (isabsolute($pp[0])) { + push @STAGEOUT, $pp[0]; + } + else { + push @STAGEOUT, $JOB{directory}."/".$pp[0]; + } + push @STAGEOUTD, $pp[1]; + } + elsif ($pair[0] eq "cleanup") { + if (isabsolute($pair[1])) { + push @CLEANUP, $pair[1]; + } + else { + push @CLEANUP, $JOB{directory}."/".$pair[1]; + } + } + else { + $JOB{$pair[0]} = $pair[1]; + } + } + if (checkJob($tag, $JOBID, \%JOB)) { + $JOBDATA{$JOBID} = { + stagein => \@STAGEIN, + stageind => \@STAGEIND, + stageindex => 0, + job => \%JOB, + jobargs => \@JOBARGS, + jobenv => \%JOBENV, + stageout => \@STAGEOUT, + stageoutd => \@STAGEOUTD, + cleanup => \@CLEANUP, + }; + + stagein($JOBID); + } +} + +sub checkJob() { + my ($tag, $JOBID, $JOB) = @_; + + wlog INFO, "$JOBID Job info received (tag=$tag)\n"; + my $executable = $$JOB{"executable"}; + if (!(defined $JOBID)) { + my $ds = hts($JOB); + + wlog DEBUG, "$JOBID Job details $ds\n"; + + sendError($tag, ("Missing job identity")); + return 0; + } + elsif (!(defined $executable)) { + sendError($tag, ("Missing executable")); + return 0; + } + else { + my $dir = $$JOB{directory}; + if (!defined $dir) { + $dir = "."; + } + my $dirlen = length($dir); + my $cleanup = $$JOB{"cleanup"}; + my $c; + foreach $c (@$cleanup) { + if (substr($c, 0, $dirlen) ne $dir) { + sendError($tag, ("Cannot clean up outside of the job directory (cleanup: $c, jobdir: $dir)")); + return 0; + } + } + chdir $dir; + wlog DEBUG, "$JOBID Job check ok (dir: $dir)\n"; + wlog INFO, "$JOBID Sending submit reply (tag=$tag)\n"; + sendReply($tag, ("OK")); + wlog INFO, "$JOBID Submit reply sent (tag=$tag)\n"; + return 1; + } +} + +sub forkjob { + my ($JOBID) = @_; + my ($pid, $status); + + my $JOB = $JOBDATA{$JOBID}{"job"}; + my $JOBARGS = $JOBDATA{$JOBID}{"jobargs"}; + my $JOBENV = $JOBDATA{$JOBID}{"jobenv"}; + my $WORKERPID = $$; + + # allocate a jobslot here because we know we are starting exactly one job here + # if we verify that we dont have more stageins than slots taking place at once, + # we can move this to where the rest of the job options are set. Or we can place + # the slot in the JOBWAITDATA. (FIXME: remove when validated) + + my $JOBSLOT = pop(@jobslots); + if( ! defined($JOBSLOT) ) { + wlog DEBUG, "Job $JOBID has undefined jobslot\n"; + } + $JOBDATA{$JOBID}{"jobslot"} = $JOBSLOT; + + pipe(PARENT_R, CHILD_W); + $pid = fork(); + if (defined($pid)) { + if ($pid == 0) { + close PARENT_R; + runjob(\*CHILD_W, $JOB, $JOBARGS, $JOBENV, $JOBSLOT, $WORKERPID); + close CHILD_W; + } + else { + wlog DEBUG, "$JOBID Forked process $pid. Waiting for its completion\n"; + close CHILD_W; + $JOBS_RUNNING++; + $JOBWAITDATA{$JOBID} = { + pid => $pid, + pipe => \*PARENT_R, + }; + } + } + else { + queueCmd(nullCB(), "JOBSTATUS", $JOBID, FAILED, "512", "Could not fork child process"); + } + $LASTRECV = time(); +} + +my $LASTJOBCHECK = 0; + +sub checkJobs { + my $time = time(); + if ($time - $LASTJOBCHECK < 0.100) { + return; + } + $LASTJOBCHECK = $time; + if (!%JOBWAITDATA) { + return; + } + + wlog DEBUG, "Checking jobs status ($JOBS_RUNNING active)\n"; + + my @DELETEIDS = (); + + for my $JOBID (keys %JOBWAITDATA) { + if (checkJobStatus($JOBID)) { + push @DELETEIDS, $JOBID; + } + } + for my $i (@DELETEIDS) { + delete $JOBWAITDATA{$i}; + } +} + +sub checkJobStatus { + my ($JOBID) = @_; + + + my $pid = $JOBWAITDATA{$JOBID}{"pid"}; + my $RD = $JOBWAITDATA{$JOBID}{"pipe"}; + + my $tid; + my $status; + + wlog DEBUG, "$JOBID Checking pid $pid\n"; + + $tid = waitpid($pid, &WNOHANG); + if ($tid != $pid) { + # not done + wlog DEBUG, "$JOBID Job $pid still running\n"; + return 0; + } + else { + # exit code is in MSB and signal in LSB, so + # switch them such that status & 0xff is the + # exit code + $status = $? >> 8 + (($? & 0xff) << 8); + } + + wlog DEBUG, "$JOBID Child process $pid terminated. Status is $status.\n"; + my $s; + if (!eof($RD)) { + $s = <$RD>; + } + wlog DEBUG, "$JOBID Got output from child. Closing pipe.\n"; + close $RD; + $JOBDATA{$JOBID}{"exitcode"} = $status; + + my $JOBSLOT = $JOBDATA{$JOBID}{"jobslot"}; + if ( defined $JOBSLOT ) { + push @jobslots,$JOBSLOT; + } + + if (defined $s) { + queueCmd(nullCB(), "JOBSTATUS", $JOBID, FAILED, "$status", $s); + } + else { + #queueCmd(nullCB(), "JOBSTATUS", $JOBID, COMPLETED, "$status", ""); + stageout($JOBID); + } + $JOB_COUNT++; + $JOBS_RUNNING--; + + return 1; +} + +sub runjob { + my ($WR, $JOB, $JOBARGS, $JOBENV, $JOBSLOT, $WORKERPID) = @_; + my $executable = $$JOB{"executable"}; + my $stdout = $$JOB{"stdout"}; + my $stderr = $$JOB{"stderr"}; + + my $cwd = getcwd(); + wlog DEBUG, "CWD: $cwd\n"; + wlog DEBUG, "Running $executable\n"; + if (defined $$JOB{directory}) { + wlog DEBUG, "Directory: $$JOB{directory}\n"; + } + my $ename; + foreach $ename (keys %$JOBENV) { + $ENV{$ename} = $$JOBENV{$ename}; + } + $ENV{"SWIFT_JOB_SLOT"} = $JOBSLOT; + $ENV{"SWIFT_WORKER_PID"} = $WORKERPID; + wlog DEBUG, "Command: @$JOBARGS\n"; + unshift @$JOBARGS, $executable; + if (defined $$JOB{directory}) { + chdir $$JOB{directory}; + } + if (defined $stdout) { + wlog DEBUG, "STDOUT: $stdout\n"; + close STDOUT; + open STDOUT, ">$stdout" or die "Cannot redirect STDOUT"; + } + if (defined $stderr) { + wlog DEBUG, "STDERR: $stderr\n"; + close STDERR; + open STDERR, ">$stderr" or die "Cannot redirect STDERR"; + } + close STDIN; + wlog DEBUG, "Command: @$JOBARGS\n"; + exec { $executable } @$JOBARGS or print $WR "Could not execute $executable: $!\n"; + die "Could not execute $executable: $!"; +} + +initlog(); + +my $MSG="0"; + +my $myhost=`hostname`; +$myhost =~ s/\s+$//; + +wlog(INFO, "Running on node $myhost\n"); +# wlog(INFO, "New log name: $LOGNEW \n"); + +init(); + +mainloop(); +wlog INFO, "Worker finished. Exiting.\n"; +exit(0); + +# This file works well with cperl-mode in the latest emacs +# Local Variables: +# indent-tabs-mode: t +# tab-width: 8 +# cperl-indent-level: 8 +# End: Property changes on: SwiftApps/SwiftR/Swift/exec/worker.pl ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Tue Sep 14 12:23:44 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Sep 2010 12:23:44 -0500 (CDT) Subject: [Swift-commit] r3622 - trunk/tests/cdm/ps/pinned Message-ID: <20100914172344.533199CC86@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-14 12:23:44 -0500 (Tue, 14 Sep 2010) New Revision: 3622 Modified: trunk/tests/cdm/ps/pinned/title.txt Log: Test title Modified: trunk/tests/cdm/ps/pinned/title.txt =================================================================== --- trunk/tests/cdm/ps/pinned/title.txt 2010-09-08 21:39:47 UTC (rev 3621) +++ trunk/tests/cdm/ps/pinned/title.txt 2010-09-14 17:23:44 UTC (rev 3622) @@ -1 +1 @@ -CDM tests with provider staging +CDM tests with provider staging and pinned From noreply at svn.ci.uchicago.edu Tue Sep 14 13:06:54 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Sep 2010 13:06:54 -0500 (CDT) Subject: [Swift-commit] r3623 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100914180655.011899CC86@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-14 13:06:54 -0500 (Tue, 14 Sep 2010) New Revision: 3623 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java Log: Usage note Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-09-14 17:23:44 UTC (rev 3622) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-09-14 18:06:54 UTC (rev 3623) @@ -21,6 +21,7 @@ 4) does not impose any formatting (commas, etc.).

Format specifiers:
%%: % sign.
+ %M: Filename output: waits for close %p: Not typechecked, output as in trace().
%f: Typechecked float output.
%i: Typechecked int output.
From noreply at svn.ci.uchicago.edu Tue Sep 14 13:07:37 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Sep 2010 13:07:37 -0500 (CDT) Subject: [Swift-commit] r3624 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20100914180737.13D349CC86@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-14 13:07:36 -0500 (Tue, 14 Sep 2010) New Revision: 3624 Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java Log: Add logging Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2010-09-14 18:06:54 UTC (rev 3623) +++ trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2010-09-14 18:07:36 UTC (rev 3624) @@ -8,6 +8,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.log4j.Logger; + import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.HandleOpenException; @@ -17,7 +19,11 @@ import org.griphyn.vdl.mapping.PhysicalFormat; public class StructuredRegularExpressionMapper extends AbstractFileMapper { - public static final MappingParam PARAM_SOURCE = new MappingParam("source"); + + public static final Logger logger = + Logger.getLogger(StructuredRegularExpressionMapper.class); + + public static final MappingParam PARAM_SOURCE = new MappingParam("source"); public static final MappingParam PARAM_MATCH = new MappingParam("match"); public static final MappingParam PARAM_TRANSFORM = new MappingParam("transform"); @@ -57,6 +63,9 @@ } public PhysicalFormat map(Path path) { + + logger.debug("map(): path: " + path); + String match = PARAM_MATCH.getStringValue(this); String transform = PARAM_TRANSFORM.getStringValue(this); From noreply at svn.ci.uchicago.edu Tue Sep 14 14:05:58 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Sep 2010 14:05:58 -0500 (CDT) Subject: [Swift-commit] r3625 - trunk/tests Message-ID: <20100914190558.8D9BA9CC9A@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-14 14:05:58 -0500 (Tue, 14 Sep 2010) New Revision: 3625 Added: trunk/tests/functions/ Log: Functions tests From noreply at svn.ci.uchicago.edu Thu Sep 16 15:14:27 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 16 Sep 2010 15:14:27 -0500 (CDT) Subject: [Swift-commit] r3626 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100916201427.641069CC8F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-09-16 15:14:27 -0500 (Thu, 16 Sep 2010) New Revision: 3626 Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: use proper test instead of class cast exception to determine if a value is of a primitive type Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-09-14 19:05:58 UTC (rev 3625) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-09-16 20:14:27 UTC (rev 3626) @@ -84,16 +84,15 @@ if (value != null) { final VariableArguments vret = ArgUtil.getVariableReturn(stack); if (value.getClass().isArray()) { - try { + if (value.getClass().getComponentType().isPrimitive()) { + vret.append(value); + } + else { Object[] array = (Object[]) value; for (int i = 0; i < array.length; i++) { vret.append(array[i]); } } - catch (ClassCastException e) { - // array of primitives; return as is - vret.append(value); - } } else { vret.append(value); From noreply at svn.ci.uchicago.edu Thu Sep 16 15:26:47 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 16 Sep 2010 15:26:47 -0500 (CDT) Subject: [Swift-commit] r3627 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100916202647.CA07E9CC8F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-09-16 15:26:47 -0500 (Thu, 16 Sep 2010) New Revision: 3627 Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: optimized mapper access a bit; do not hold lock for the whole variable when it's only needed to get the mapper Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-09-16 20:14:27 UTC (rev 3626) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-09-16 20:26:47 UTC (rev 3627) @@ -44,6 +44,7 @@ import org.griphyn.vdl.mapping.GeneralizedFileFormat; import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; +import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; import org.griphyn.vdl.type.Type; @@ -175,29 +176,31 @@ public static final String[] EMPTY_STRING_ARRAY = new String[0]; public static String[] filename(VariableStack stack) throws ExecutionException { - DSHandle ovar = (DSHandle)PA_VAR.getValue(stack); - synchronized(ovar.getRoot()) { - try { - return filename(ovar); - } - catch(VDL2FutureException ve) { - throw new FutureNotYetAvailable(addFutureListener(stack, ve.getHandle())); - } - catch (HandleOpenException e) { - throw new FutureNotYetAvailable(addFutureListener(stack, e.getSource())); - } - } - } + DSHandle handle = (DSHandle)PA_VAR.getValue(stack); + return filename(stack, handle); + } + + public static String[] filename(VariableStack stack, DSHandle handle) throws ExecutionException { + try { + return filename(handle); + } + catch(VDL2FutureException ve) { + throw new FutureNotYetAvailable(addFutureListener(stack, ve.getHandle())); + } + catch (HandleOpenException e) { + throw new FutureNotYetAvailable(addFutureListener(stack, e.getSource())); + } + } - /** The caller is expected to have synchronized on the root of var. */ public static String[] filename(DSHandle var) throws ExecutionException, HandleOpenException { - assert Thread.holdsLock(var.getRoot()); try { if (var.getType().isArray()) { return leavesFileNames(var); - } else if(var.getType().getFields().size()>0) { + } + else if(var.getType().getFields().size() > 0) { return leavesFileNames(var); - } else { + } + else { return new String[] { leafFileName(var) }; } } @@ -220,9 +223,13 @@ } Collections.sort(src, new PathComparator()); i = src.iterator(); + Mapper mapper; + synchronized (var.getRoot()) { + mapper = var.getMapper(); + } while (i.hasNext()) { Path p = (Path) i.next(); - l.add(leafFileName(var.getField(p))); + l.add(leafFileName(var.getField(p), mapper)); } } catch (InvalidPathException e) { @@ -276,13 +283,23 @@ return Integer.parseInt(i1) - Integer.parseInt(i2); } } + + private static String leafFileName(DSHandle var) throws ExecutionException { + Mapper mapper; + synchronized (var.getRoot()) { + mapper = var.getMapper(); + } + return leafFileName(var, mapper); + } - private static String leafFileName(DSHandle var) throws ExecutionException { + private static String leafFileName(DSHandle var, Mapper mapper) throws ExecutionException { if (Types.STRING.equals(var.getType())) { return relativize(String.valueOf(var.getValue())); } else { - PhysicalFormat f = var.getMapper().map(var.getPathFromRoot()); + PhysicalFormat f; + Path pathFromRoot = var.getPathFromRoot(); + f = mapper.map(pathFromRoot); if (f instanceof GeneralizedFileFormat) { String filename = ((GeneralizedFileFormat) f).getURIAsString(); if (filename == null) { @@ -491,7 +508,7 @@ getFutureWrapperMap(stack).markAsAvailable(handle, key); } - protected final Path parsePath(Object o, VariableStack stack) throws ExecutionException { + public static Path parsePath(Object o, VariableStack stack) throws ExecutionException { Path q = Path.EMPTY_PATH; Path p; if (o instanceof Path) { From noreply at svn.ci.uchicago.edu Thu Sep 16 15:30:37 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 16 Sep 2010 15:30:37 -0500 (CDT) Subject: [Swift-commit] r3628 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100916203037.406B99CC8F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-09-16 15:30:37 -0500 (Thu, 16 Sep 2010) New Revision: 3628 Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: get the mapper before doing potentially expensive operations Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-09-16 20:26:47 UTC (rev 3627) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-09-16 20:30:37 UTC (rev 3628) @@ -210,6 +210,10 @@ } private static String[] leavesFileNames(DSHandle var) throws ExecutionException, HandleOpenException { + Mapper mapper; + synchronized (var.getRoot()) { + mapper = var.getMapper(); + } List l = new ArrayList(); Iterator i; try { @@ -223,10 +227,6 @@ } Collections.sort(src, new PathComparator()); i = src.iterator(); - Mapper mapper; - synchronized (var.getRoot()) { - mapper = var.getMapper(); - } while (i.hasNext()) { Path p = (Path) i.next(); l.add(leafFileName(var.getField(p), mapper)); From noreply at svn.ci.uchicago.edu Fri Sep 17 16:28:58 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 17 Sep 2010 16:28:58 -0500 (CDT) Subject: [Swift-commit] r3630 - SwiftApps/SwiftR/Swift/exec Message-ID: <20100917212858.D2CD29D77C@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-17 16:28:58 -0500 (Fri, 17 Sep 2010) New Revision: 3630 Modified: SwiftApps/SwiftR/Swift/exec/start-swift-workers Log: Put remote worker log on local /tmp. Modified: SwiftApps/SwiftR/Swift/exec/start-swift-workers =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-workers 2010-09-17 20:37:40 UTC (rev 3629) +++ SwiftApps/SwiftR/Swift/exec/start-swift-workers 2010-09-17 21:28:58 UTC (rev 3630) @@ -26,7 +26,8 @@ echo Coaster contact: $CONTACT LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env? - mkdir -p $LOGDIR + LOGDIR=/tmp/$USER/SwiftR/swiftworkerlogs +# mkdir -p $LOGDIR # moved this to the ssh command IDLETIMEOUT=$((60*60*240)) # 10 days: FIXME: make this a command line arg @@ -35,7 +36,7 @@ random=$(awk "BEGIN {printf \"%0.5d\", $RANDOM}") ID=$timestamp.$random # FIXME: make logging an argument; set false by default - ssh $host '/bin/sh -c '\'"WORKER_LOGGING_ENABLED=true $SWIFTBIN/worker.pl $CONTACT $ID $LOGDIR $IDLETIMEOUT 2>&1 & echo PID=\$!"\' >remotepid.$host &1 & echo PID=\$!"\' >remotepid.$host Author: wilde Date: 2010-09-20 11:37:35 -0500 (Mon, 20 Sep 2010) New Revision: 3631 Modified: SwiftApps/SwiftR/Swift/exec/start-swift-workers SwiftApps/SwiftR/TODO SwiftApps/SwiftR/UserGuide Log: Improve and correct cleanup: all services should now shut down when start command is terminated. start-swift-workers no starts the Swift R service. Modified: SwiftApps/SwiftR/Swift/exec/start-swift-workers =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-workers 2010-09-17 21:28:58 UTC (rev 3630) +++ SwiftApps/SwiftR/Swift/exec/start-swift-workers 2010-09-20 16:37:35 UTC (rev 3631) @@ -31,6 +31,7 @@ IDLETIMEOUT=$((60*60*240)) # 10 days: FIXME: make this a command line arg + rm -rf remotepid.* # FIXME: should not be needed if we start in a new dir each time for host in $(echo $COMPUTEHOSTS); do timestamp=$(date "+%Y.%m%d.%H%M%S") random=$(awk "BEGIN {printf \"%0.5d\", $RANDOM}") @@ -106,16 +107,16 @@ trap - $TRAPS sshpids=$(cat $sshpidfile) echo "Terminating worker processes $sshpids, starter $starterpid, and coaster-service pid $coasterservicepid" - if [ "_$sshpids$starterpid$coasterservicepid" != _ ]; then - echo kill $sshpids $starterpid $coasterservicepid >& /dev/null - fi for rpfile in $(ls -1 remotepid.*); do rpid=$(grep PID= $rpfile | sed -e 's/PID=//') rhost=$(echo $rpfile | sed -e 's/remotepid.//') echo from $rpfile: doing ssh $rhost kill $rpid - ssh $rhost kill $rpid + ssh $rhost kill -s TERM -- '-$(' ps -p $rpid -o pgid --no-headers ')' done - kill 0 # Kill + if [ "_$sshpids$starterpid$coasterservicepid" != _ ]; then + echo kill $sshpids $starterpid $coasterservicepid >& /dev/null + fi + kill 0 # Kill # FIXME: what was this for???? } trap onexit $TRAPS @@ -123,8 +124,10 @@ wait-and-start-workers & starterpid=$! -$SWIFTBIN/swift -config cf -tc.file tc -sites.file sites.xml passivate.swift 2>&1 | tee $out +$SWIFTBIN/swift -config cf -tc.file tc -sites.file sites.xml passivate.swift 2>&1 Service started and set to passive mode. Use ^C to terminate all services and workers." +$SWIFTRBIN/start-swift-Rserver service + wait Modified: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO 2010-09-17 21:28:58 UTC (rev 3630) +++ SwiftApps/SwiftR/TODO 2010-09-20 16:37:35 UTC (rev 3631) @@ -20,10 +20,17 @@ ----- + find why TestSwift is faster with local internal coaster service (13 secs) than with external service (19 seconds) + If thats true, maybe use local internal ervice for local exec if possible. + Ensure that shell startup overhead is not impacting swiftapply() latency + Do R functions to start stop list and manage service processes, both ocal and remote + Get traces into R vars of swiftapply performance + + See if swift can write to pipe without using an app() Bundle start-swift-workers and RunRServer into one command; Modified: SwiftApps/SwiftR/UserGuide =================================================================== --- SwiftApps/SwiftR/UserGuide 2010-09-17 21:28:58 UTC (rev 3630) +++ SwiftApps/SwiftR/UserGuide 2010-09-20 16:37:35 UTC (rev 3631) @@ -37,6 +37,7 @@ # do this outside of R SWIFT=/Swift +$SWIFT/exec/start-swift-workers hostname $SWIFT/exec/start-swift-server HELLO WORLD TEST From noreply at svn.ci.uchicago.edu Mon Sep 20 23:12:48 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 20 Sep 2010 23:12:48 -0500 (CDT) Subject: [Swift-commit] r3632 - SwiftApps/SwiftR/Swift/exec Message-ID: <20100921041248.DEE489CC9D@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-20 23:12:48 -0500 (Mon, 20 Sep 2010) New Revision: 3632 Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh SwiftApps/SwiftR/Swift/exec/start-swift-Rserver SwiftApps/SwiftR/Swift/exec/start-swift-workers Log: Permit selection of provider staging vs. local staging. Provider staging works at modest scales but still frequently hangs. Fix some problems in process termination cleanup. Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh =================================================================== --- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-09-20 16:37:35 UTC (rev 3631) +++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2010-09-21 04:12:48 UTC (rev 3632) @@ -98,6 +98,8 @@ echo run $(pwd)/$callFile $(pwd)/$resultFile > $SLOTDIR/toR.fifo touch $SLOTDIR/lastwrite +echo dummy stderr response 1>&2 # FIXME - testing if this is the provider staging problem (not xfering zero len stderr) + head -3 < $SLOTDIR/fromR.fifo # FIXME: Trim this down to 1 line for each call (or same # lines for each, in particular, for "quit") # Fixme: how to get exceptions and stdout/stderr text from R server ??? \ No newline at end of file Modified: SwiftApps/SwiftR/Swift/exec/start-swift-Rserver =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-Rserver 2010-09-20 16:37:35 UTC (rev 3631) +++ SwiftApps/SwiftR/Swift/exec/start-swift-Rserver 2010-09-21 04:12:48 UTC (rev 3632) @@ -2,10 +2,15 @@ SWIFTRBIN=$(cd $(dirname $0); pwd) +serviceport=1985 + #rundir=/tmp/SwiftR/swiftserver rundir=/tmp/$USER/SwiftR/swiftserver # FIXME: handle multiple concurent independent swift servers per user #site=local + location=$1 +stagingmethod=$2 + script=$SWIFTRBIN/rserver.swift trundir=$(mktemp -d $rundir.XXXX) @@ -38,6 +43,8 @@ fork bash /bin/bash null null null END +if [ $stagingmethod = local-staging ]; then + cat >sites.xml < @@ -56,7 +63,85 @@ 4 .03 10000 +# + file + /tmp/$USER/SwiftR/swiftserver + + + + + 10000 + .03 + + /tmp/$USER/SwiftR/swiftserver + + + + + passive + 8 + .07 + 10000 + /home/wilde/swiftwork + + + + + passive + 4 + .03 + 10000 + + /home/wilde/swiftwork + + + + 00:00:10 + 1800 + + 1 + 10000 + 5.99 + + $(pwd) + + + +END + +cat >cf <sites.xml < + + + + passive + 4 + .03 + 10000 + + $rundir/swiftwork + + + + + 4 + .03 + 10000 +# + file /tmp/$USER/SwiftR/swiftserver @@ -64,6 +149,14 @@ 10000 .03 + file + /tmp/$USER/SwiftR/swiftserver + + + + + 10000 + .03 /tmp/$USER/SwiftR/swiftserver @@ -79,6 +172,16 @@ + + passive + 4 + .03 + 10000 + proxy + /tmp/wilde/SwiftR/swiftwork + + + passive 4 @@ -102,12 +205,27 @@ END +cat >cf <& swift.stdouterr & swift.stdouterr & swift.stdouterr &1 & echo PID=\$!"\' >remotepid.$host &1 & echo PID=\$!"\' >remotepid.$host &1 & echo PID=\$!"\' >remotepid.$host & coaster-service.log & +$SWIFTBIN/coaster-service -nosec -p $SERVICEPORT >& coaster-service.log & coasterservicepid=$! +cat >tc <sites.xml < - + passive 4 .03 @@ -68,21 +92,44 @@ END -cat >tc <cf <sites.xml < + + + passive + 4 + .03 + 10000 + proxy + /tmp/$USER/SwiftR/swiftwork + + END +cat >cf <passivate.swift <& /dev/null @@ -128,6 +175,6 @@ echo "==> Service started and set to passive mode. Use ^C to terminate all services and workers." -$SWIFTRBIN/start-swift-Rserver service +$SWIFTRBIN/start-swift-Rserver service $STAGINGMETHOD wait From noreply at svn.ci.uchicago.edu Tue Sep 21 13:57:45 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Sep 2010 13:57:45 -0500 (CDT) Subject: [Swift-commit] r3633 - trunk/tests Message-ID: <20100921185745.DDFB49CCC2@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-21 13:57:45 -0500 (Tue, 21 Sep 2010) New Revision: 3633 Modified: trunk/tests/nightly.sh Log: Test line numbers, better process killing, export GROUP Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-21 04:12:48 UTC (rev 3632) +++ trunk/tests/nightly.sh 2010-09-21 18:57:45 UTC (rev 3633) @@ -35,6 +35,8 @@ # stdout.txt retains stdout from the previous test (for *.clean.sh) # output_*.txt is the HTML-linked permanent output from a test +# WARNING: On timeout, this script will call killall -9 java + printhelp() { echo "nightly.sh " echo "" @@ -125,6 +127,8 @@ SCRIPTDIR=$( dirname $0 ) +SWIFTCOUNT=0 + cd $TOPDIR mkdir -p $RUNDIR [ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1 @@ -349,7 +353,7 @@ start_part() { PART=$1 html_tr part - html_th 2 + html_th 3 html "$PART" html_~th html_~tr @@ -358,6 +362,9 @@ start_row() { html_tr testline html_td right + html "$SWIFTCOUNT" + html_~td + html_td right if [[ -n $TESTLINK ]]; then html_a_href $TESTLINK $TESTNAME else @@ -446,7 +453,13 @@ process_exec() { printf "\nExecuting: $@" >>$LOG rm -fv $OUTPUT - "$@" > $OUTPUT 2>&1 + + "$@" > $OUTPUT 2>&1 & + EXEC_PID=$! + + trap "process_exec_trap $EXEC_PID" SIGTERM + + wait $EXEC_PID EXITCODE=$? if [ "$EXITCODE" == "127" ]; then echo "Command not found: $@" > $OUTPUT @@ -457,6 +470,16 @@ return $EXITCODE } +# Ensure we kill the tested process and any subordinate java processes +# in case of monitor() timeout +# Rationale: Killing bin/swift does not kill the Swift java process +process_exec_trap() { + EXEC_PID=$1 + echo "process_exec_trap()" + kill -KILL $EXEC_PID + killall -9 java +} + # Execute as part of test set # Equivalent to monitored_exec() (but w/o monitoring) test_exec() { @@ -489,17 +512,19 @@ sleep $TIMEOUT EXITCODE=1 +# /bin/kill -TERM $PID +# KILLCODE=$? +# if [ $KILLCODE == 0 ]; then +# echo "monitor(): killed process (TERM)" +# sleep 1 +# fi /bin/kill -TERM $PID KILLCODE=$? if [ $KILLCODE == 0 ]; then - echo "monitor(): killed process (TERM)" - sleep 1 + echo "monitor(): killed process_exec (TERM)" fi - /bin/kill -KILL $PID - if [ $KILLCODE == 0 ]; then - echo "monitor(): killed process (KILL)" - fi + sleep 1 MSG="nightly.sh: monitor(): killed: exceeded $TIMEOUT seconds" echo "$MSG" >> $OUTPUT } @@ -526,7 +551,7 @@ # If EXITCODE != 0, monitor() may have work to do (( $EXITCODE != 0 )) && sleep 5 - kill -TERM $MONITOR_PID + /bin/kill -TERM $MONITOR_PID echo "TOOK: $(( STOP-START ))" @@ -567,6 +592,8 @@ CDM= [ -r fs.data ] && CDM="-cdm.file fs.data" + (( SWIFTCOUNT++ )) + monitored_exec swift -wrapperlog.always.transfer true \ -config swift.properties \ -sites.file sites.xml \ @@ -743,15 +770,19 @@ SKIP_COUNTER=0 -GROUPLIST=( $TESTDIR/language/working \ +# GROUPLIST=( $TESTDIR/local $TESTDIR/cdm $TESTDIR/cdm/ps $TESTDIR/cdm/ps/pinned ) + +GROUPLIST=( $TESTDIR/language-behaviour + $TESTDIR/language/working \ $TESTDIR/local \ $TESTDIR/language/should-not-work \ $TESTDIR/cdm \ - $TESTDIR/cdm/ps ) + $TESTDIR/cdm/ps \ + $TESTDIR/cdm/ps/pinned ) GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do - GROUP=$G + export GROUP=$G TITLE=$( group_title ) start_part "Part $GROUPCOUNT: $TITLE" test_group From noreply at svn.ci.uchicago.edu Tue Sep 21 14:00:50 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Sep 2010 14:00:50 -0500 (CDT) Subject: [Swift-commit] r3634 - in trunk/tests: . language-behaviour Message-ID: <20100921190050.222579CCC2@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-21 14:00:46 -0500 (Tue, 21 Sep 2010) New Revision: 3634 Added: trunk/tests/language-behaviour/00261-function-param-file.clean.sh trunk/tests/language-behaviour/00261-function-param-file.setup.sh trunk/tests/language-behaviour/00262-function-param-file-many.clean.sh trunk/tests/language-behaviour/00262-function-param-file-many.setup.sh trunk/tests/language-behaviour/060-duplicate.check.sh trunk/tests/language-behaviour/060-duplicate.clean.sh trunk/tests/language-behaviour/060-duplicate.setup.sh trunk/tests/language-behaviour/061-cattwo.clean.sh trunk/tests/language-behaviour/061-cattwo.setup.sh trunk/tests/language-behaviour/066-many.check.sh trunk/tests/language-behaviour/066-many.clean.sh trunk/tests/language-behaviour/066-many.time trunk/tests/language-behaviour/tc.template.data trunk/tests/language-behaviour/title.txt Modified: trunk/tests/language-behaviour/065-delay.swift trunk/tests/language-behaviour/0651-several-delay.swift trunk/tests/language-behaviour/066-many.swift trunk/tests/nightly.sh Log: Test fixes Added: trunk/tests/language-behaviour/00261-function-param-file.clean.sh =================================================================== --- trunk/tests/language-behaviour/00261-function-param-file.clean.sh (rev 0) +++ trunk/tests/language-behaviour/00261-function-param-file.clean.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -v 00261-function-param-file.in || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/00261-function-param-file.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/00261-function-param-file.setup.sh =================================================================== --- trunk/tests/language-behaviour/00261-function-param-file.setup.sh (rev 0) +++ trunk/tests/language-behaviour/00261-function-param-file.setup.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,5 @@ +#!/bin/bash + +cp -v ${GROUP}/00261-function-param-file.in . || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/00261-function-param-file.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/00262-function-param-file-many.clean.sh =================================================================== --- trunk/tests/language-behaviour/00262-function-param-file-many.clean.sh (rev 0) +++ trunk/tests/language-behaviour/00262-function-param-file-many.clean.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -v 00262-function-param-file-many.in || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/00262-function-param-file-many.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/00262-function-param-file-many.setup.sh =================================================================== --- trunk/tests/language-behaviour/00262-function-param-file-many.setup.sh (rev 0) +++ trunk/tests/language-behaviour/00262-function-param-file-many.setup.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,5 @@ +#!/bin/bash + +cp -v ${GROUP}/00262-function-param-file-many.in . || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/00262-function-param-file-many.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/060-duplicate.check.sh =================================================================== --- trunk/tests/language-behaviour/060-duplicate.check.sh (rev 0) +++ trunk/tests/language-behaviour/060-duplicate.check.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,5 @@ +#!/bin/bash + +grep 060-duplicate.in 060-duplicate.out || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/060-duplicate.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/060-duplicate.clean.sh =================================================================== --- trunk/tests/language-behaviour/060-duplicate.clean.sh (rev 0) +++ trunk/tests/language-behaviour/060-duplicate.clean.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,6 @@ +#!/bin/bash + +rm -v 060-duplicate.in || exit 1 +rm -v 060-duplicate.out || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/060-duplicate.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/060-duplicate.setup.sh =================================================================== --- trunk/tests/language-behaviour/060-duplicate.setup.sh (rev 0) +++ trunk/tests/language-behaviour/060-duplicate.setup.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,5 @@ +#!/bin/bash + +cp -v ${GROUP}/060-duplicate.in . || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/060-duplicate.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/061-cattwo.clean.sh =================================================================== --- trunk/tests/language-behaviour/061-cattwo.clean.sh (rev 0) +++ trunk/tests/language-behaviour/061-cattwo.clean.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +rm 061-cattwo.[12].in || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/061-cattwo.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/061-cattwo.setup.sh =================================================================== --- trunk/tests/language-behaviour/061-cattwo.setup.sh (rev 0) +++ trunk/tests/language-behaviour/061-cattwo.setup.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +cp ${GROUP}/061-cattwo.[12].in . || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/061-cattwo.setup.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/language-behaviour/065-delay.swift =================================================================== --- trunk/tests/language-behaviour/065-delay.swift 2010-09-21 18:57:45 UTC (rev 3633) +++ trunk/tests/language-behaviour/065-delay.swift 2010-09-21 19:00:46 UTC (rev 3634) @@ -1,9 +1,6 @@ -delay() { - app { - sleep "90s"; - } +app delay() { + sleep "3"; } delay(); - Modified: trunk/tests/language-behaviour/0651-several-delay.swift =================================================================== --- trunk/tests/language-behaviour/0651-several-delay.swift 2010-09-21 18:57:45 UTC (rev 3633) +++ trunk/tests/language-behaviour/0651-several-delay.swift 2010-09-21 19:00:46 UTC (rev 3634) @@ -1,10 +1,8 @@ -delay() { - app { - sleep "90s"; - } +app delay(int i) { + sleep i; } -foreach i in [1:10] { -delay(); +foreach i in [1:3] { + delay(i); } Added: trunk/tests/language-behaviour/066-many.check.sh =================================================================== --- trunk/tests/language-behaviour/066-many.check.sh (rev 0) +++ trunk/tests/language-behaviour/066-many.check.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +[ -f nop ] || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/066-many.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/066-many.clean.sh =================================================================== --- trunk/tests/language-behaviour/066-many.clean.sh (rev 0) +++ trunk/tests/language-behaviour/066-many.clean.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +rm -v nop || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/066-many.clean.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/language-behaviour/066-many.swift =================================================================== --- trunk/tests/language-behaviour/066-many.swift 2010-09-21 18:57:45 UTC (rev 3633) +++ trunk/tests/language-behaviour/066-many.swift 2010-09-21 19:00:46 UTC (rev 3634) @@ -1,8 +1,7 @@ -p() { - app { - touch "nop"; - } +app p() +{ + touch "nop"; } foreach i in [1:3000] { Added: trunk/tests/language-behaviour/066-many.time =================================================================== --- trunk/tests/language-behaviour/066-many.time (rev 0) +++ trunk/tests/language-behaviour/066-many.time 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1 @@ +300 Added: trunk/tests/language-behaviour/tc.template.data =================================================================== --- trunk/tests/language-behaviour/tc.template.data (rev 0) +++ trunk/tests/language-behaviour/tc.template.data 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1,20 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null +localhost cp /bin/cp INSTALLED INTEL32::LINUX null +localhost sleep /bin/sleep INSTALLED INTEL32::LINUX null +localhost touch /bin/touch INSTALLED INTEL32::LINUX null Added: trunk/tests/language-behaviour/title.txt =================================================================== --- trunk/tests/language-behaviour/title.txt (rev 0) +++ trunk/tests/language-behaviour/title.txt 2010-09-21 19:00:46 UTC (rev 3634) @@ -0,0 +1 @@ +Language-behaviour tests Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-21 18:57:45 UTC (rev 3633) +++ trunk/tests/nightly.sh 2010-09-21 19:00:46 UTC (rev 3634) @@ -22,14 +22,16 @@ # Each *.swift test may be accompanied by a # *.setup.sh, *.check.sh, and/or *.clean.sh script # These may setup and inspect files in RUNDIR including exec.out +# The GROUP scripts can read the GROUP variable # Tests are GROUPed into directories # Each GROUP directory has: # 1) a list of *.swift tests (plus *.sh scripts) # 2) optionally a tc.template.data -# 3) optionally a swift.properties -# 4) optionally a title.txt -# 5) preferably a README.txt +# 3) optionally a fs.template.data +# 4) optionally a swift.properties +# 5) optionally a title.txt +# 6) preferably a README.txt # OUTPUT is the stdout of the current test # stdout.txt retains stdout from the previous test (for *.clean.sh) From noreply at svn.ci.uchicago.edu Tue Sep 21 15:57:47 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Sep 2010 15:57:47 -0500 (CDT) Subject: [Swift-commit] r3635 - in trunk/tests: . language-behaviour Message-ID: <20100921205747.70AC49CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-21 15:57:47 -0500 (Tue, 21 Sep 2010) New Revision: 3635 Added: trunk/tests/language-behaviour/066-many.setup.sh trunk/tests/language-behaviour/066-many.timeout Modified: trunk/tests/language-behaviour/066-many.swift trunk/tests/nightly.sh Log: Process management improvements Added: trunk/tests/language-behaviour/066-many.setup.sh =================================================================== --- trunk/tests/language-behaviour/066-many.setup.sh (rev 0) +++ trunk/tests/language-behaviour/066-many.setup.sh 2010-09-21 20:57:47 UTC (rev 3635) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +touch nop || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/066-many.setup.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/language-behaviour/066-many.swift =================================================================== --- trunk/tests/language-behaviour/066-many.swift 2010-09-21 19:00:46 UTC (rev 3634) +++ trunk/tests/language-behaviour/066-many.swift 2010-09-21 20:57:47 UTC (rev 3635) @@ -1,9 +1,12 @@ -app p() +type file; + +app p(file f) { - touch "nop"; + touch @f; } foreach i in [1:3000] { - p(); + file f<"nop">; + p(f); } Added: trunk/tests/language-behaviour/066-many.timeout =================================================================== --- trunk/tests/language-behaviour/066-many.timeout (rev 0) +++ trunk/tests/language-behaviour/066-many.timeout 2010-09-21 20:57:47 UTC (rev 3635) @@ -0,0 +1 @@ +600 Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-21 19:00:46 UTC (rev 3634) +++ trunk/tests/nightly.sh 2010-09-21 20:57:47 UTC (rev 3635) @@ -21,8 +21,11 @@ # Each *.swift test may be accompanied by a # *.setup.sh, *.check.sh, and/or *.clean.sh script -# These may setup and inspect files in RUNDIR including exec.out +# and a *.timeout specifier +# The scripts may setup and inspect files in RUNDIR including exec.out # The GROUP scripts can read the GROUP variable +# The timeout number in the *.timeout file overrides the default +# timeout # Tests are GROUPed into directories # Each GROUP directory has: @@ -37,8 +40,20 @@ # stdout.txt retains stdout from the previous test (for *.clean.sh) # output_*.txt is the HTML-linked permanent output from a test -# WARNING: On timeout, this script will call killall -9 java +# WARNING: On timeout, this script will call killall on java and sleep +# All timeouts in this script are in seconds + +# PID TREE: +# Background processes are used so that hung Swift jobs can be killed +# These are the background processes (PIDs are tracked) +# nightly.sh +# +-monitor() +# +-sleep +# +-process_exec() +# +-bin/swift +# +-java + printhelp() { echo "nightly.sh " echo "" @@ -56,6 +71,7 @@ } # Defaults: +DEFAULT_TIMEOUT=30 # seconds RUN_ANT=1 CLEAN=1 SKIP_TESTS=0 @@ -511,30 +527,43 @@ TIMEOUT=$2 # seconds OUTPUT=$3 - sleep $TIMEOUT - EXITCODE=1 + V=$SWIFTCOUNT -# /bin/kill -TERM $PID -# KILLCODE=$? -# if [ $KILLCODE == 0 ]; then -# echo "monitor(): killed process (TERM)" -# sleep 1 -# fi + # Use background so kill/trap is immediate + sleep $TIMEOUT & + SLEEP_PID=$! + trap "monitor_trap $SLEEP_PID $V" SIGTERM + wait $SLEEP_PID + [ $? != 0 ] && echo "monitor($V) cancelled" && return 0 + /bin/kill -TERM $PID KILLCODE=$? if [ $KILLCODE == 0 ]; then - echo "monitor(): killed process_exec (TERM)" + echo "monitor($V): killed process_exec (TERM)" fi sleep 1 - MSG="nightly.sh: monitor(): killed: exceeded $TIMEOUT seconds" + MSG="nightly.sh: monitor($V): killed: exceeded $TIMEOUT seconds" echo "$MSG" >> $OUTPUT + trap + return 1 } +monitor_trap() { + SLEEP_PID=$1 + V=$2 + echo "monitor_trap($V)" + /bin/kill -9 $SLEEP_PID +} + # Execute given command line in background with monitor # Otherwise equivalent to test_exec() +# usage: monitored_exec * monitored_exec() { + TIMEOUT=$1 + shift + banner "$TEST (part $SEQ)" echo "Executing $TEST (part $SEQ)" @@ -543,7 +572,7 @@ process_exec "$@" & PROCESS_PID=$! - monitor $PROCESS_PID 30 $OUTPUT & + monitor $PROCESS_PID $TIMEOUT $OUTPUT & MONITOR_PID=$! wait $PROCESS_PID @@ -553,6 +582,7 @@ # If EXITCODE != 0, monitor() may have work to do (( $EXITCODE != 0 )) && sleep 5 + echo "Killing monitor..." /bin/kill -TERM $MONITOR_PID echo "TOOK: $(( STOP-START ))" @@ -587,6 +617,7 @@ SETUPSCRIPT=${SWIFTSCRIPT%.swift}.setup.sh CHECKSCRIPT=${SWIFTSCRIPT%.swift}.check.sh CLEANSCRIPT=${SWIFTSCRIPT%.swift}.clean.sh + TIMEOUTFILE=${SWIFTSCRIPT%.swift}.timeout if [ -x $GROUP/$SETUPSCRIPT ]; then script_exec $GROUP/$SETUPSCRIPT "S" fi @@ -596,10 +627,13 @@ (( SWIFTCOUNT++ )) - monitored_exec swift -wrapperlog.always.transfer true \ - -config swift.properties \ - -sites.file sites.xml \ - -tc.file tc.data \ + TIMEOUT=$( gettimeout $GROUP/$TIMEOUTFILE ) + + monitored_exec $TIMEOUT swift \ + -wrapperlog.always.transfer true \ + -config swift.properties \ + -sites.file sites.xml \ + -tc.file tc.data \ $CDM $SWIFTSCRIPT if [ -x $GROUP/$CHECKSCRIPT ]; then @@ -610,6 +644,18 @@ fi } +# All timeouts in this script are in seconds +gettimeout() { + FILE=$1 + + if [ -f $FILE ]; then + cat $FILE + else + echo $DEFAULT_TIMEOUT + fi + return 0 +} + ssexec() { SEQSAVE=$SEQ SEQ=$1 From noreply at svn.ci.uchicago.edu Tue Sep 21 16:23:50 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Sep 2010 16:23:50 -0500 (CDT) Subject: [Swift-commit] r3636 - trunk/tests Message-ID: <20100921212350.6C4CB9CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-21 16:23:49 -0500 (Tue, 21 Sep 2010) New Revision: 3636 Modified: trunk/tests/nightly.sh Log: Log fix in error case Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-21 20:57:47 UTC (rev 3635) +++ trunk/tests/nightly.sh 2010-09-21 21:23:49 UTC (rev 3636) @@ -22,7 +22,9 @@ # Each *.swift test may be accompanied by a # *.setup.sh, *.check.sh, and/or *.clean.sh script # and a *.timeout specifier -# The scripts may setup and inspect files in RUNDIR including exec.out +# The scripts may setup and inspect files in RUNDIR including exec.out, +# which must be accessed in stdout.txt, because the currently running +# tested process writes to exec.out, stdout.txt is a copy # The GROUP scripts can read the GROUP variable # The timeout number in the *.timeout file overrides the default # timeout @@ -348,7 +350,7 @@ html_a_href $TEST_LOG "$LABEL" else echo "FAILED" - cat $TEST_LOG < /dev/null + cat $RUNDIR/$TEST_LOG < /dev/null html "" html_a_href $TEST_LOG $LABEL fi @@ -416,7 +418,6 @@ # TEST_LOG = test log test_log() { TEST_LOG="output_$LOGCOUNT.txt" - rm -fv $TEST_LOG banner "$LASTCMD" $RUNDIR/$TEST_LOG if [ -f $OUTPUT ]; then cp -v $OUTPUT $RUNDIR/$TEST_LOG 2>>$LOG From noreply at svn.ci.uchicago.edu Wed Sep 22 09:53:50 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 22 Sep 2010 09:53:50 -0500 (CDT) Subject: [Swift-commit] r3637 - trunk/tests Message-ID: <20100922145350.13CF49CCBC@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-22 09:53:49 -0500 (Wed, 22 Sep 2010) New Revision: 3637 Modified: trunk/tests/nightly.sh Log: Minor fix Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-21 21:23:49 UTC (rev 3636) +++ trunk/tests/nightly.sh 2010-09-22 14:53:49 UTC (rev 3637) @@ -421,7 +421,7 @@ banner "$LASTCMD" $RUNDIR/$TEST_LOG if [ -f $OUTPUT ]; then cp -v $OUTPUT $RUNDIR/$TEST_LOG 2>>$LOG - cp -v $OUTPUT stdout.txt + cp -v $OUTPUT $RUNDIR/stdout.txt fi let "LOGCOUNT=$LOGCOUNT+1" } From noreply at svn.ci.uchicago.edu Wed Sep 22 12:36:51 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 22 Sep 2010 12:36:51 -0500 (CDT) Subject: [Swift-commit] r3638 - trunk/libexec Message-ID: <20100922173651.3B1889CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-22 12:36:51 -0500 (Wed, 22 Sep 2010) New Revision: 3638 Modified: trunk/libexec/vdl-int-staging.k Log: Enable provider.staging.pin for Swift CDM files Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-09-22 14:53:49 UTC (rev 3637) +++ trunk/libexec/vdl-int-staging.k 2010-09-22 17:36:51 UTC (rev 3638) @@ -135,9 +135,10 @@ d := vdl:dirname(cdmfile) file := basename(cdmfile) dir := if ( d == "" then("./") else(str:concat(d,"/"))) - stageIn("{stagingMethod}://localhost/{dir}{file}", cdmfile) - stageIn("{stagingMethod}://localhost/{swift.home}/libexec/cdm.pl", "cdm.pl") - stageIn("{stagingMethod}://localhost/{swift.home}/libexec/cdm_lib.sh", "cdm_lib.sh") + loc := "{pin}{stagingMethod}://localhost/" + stageIn("{loc}{dir}{file}", cdmfile) + stageIn("{loc}{swift.home}/libexec/cdm.pl", "cdm.pl") + stageIn("{loc}{swift.home}/libexec/cdm_lib.sh", "cdm_lib.sh") )) appStageins(jobid, stagein, ".", stagingMethod) From noreply at svn.ci.uchicago.edu Wed Sep 22 12:37:18 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 22 Sep 2010 12:37:18 -0500 (CDT) Subject: [Swift-commit] r3639 - trunk/src/org/griphyn/vdl/util Message-ID: <20100922173718.92D819CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-22 12:37:18 -0500 (Wed, 22 Sep 2010) New Revision: 3639 Modified: trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java Log: Use generics Modified: trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2010-09-22 17:36:51 UTC (rev 3638) +++ trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2010-09-22 17:37:18 UTC (rev 3639) @@ -25,10 +25,10 @@ public static final String WRAPPERLOG_ALWAYS_TRANSFER = "wrapperlog.always.transfer"; public static final String SITEDIR_KEEP = "sitedir.keep"; public static final String PROVENANCE_LOG = "provenance.log"; - public static final Map PROPERTIES; + public static final Map PROPERTIES; static { - PROPERTIES = new TreeMap(); + PROPERTIES = new TreeMap(); PROPERTIES.put(POOL_FILE, new PropInfo("file", "Points to the location of the sites.xml file")); PROPERTIES.put(TC_FILE, new PropInfo("file", "Points to the location of the tc.data file")); @@ -118,12 +118,12 @@ } - public static Map getPropertyDescriptions() { + public static Map getPropertyDescriptions() { return PROPERTIES; } public static String getPropertyDescription(String name) { - return (String) PROPERTIES.get(name); + return PROPERTIES.get(name).desc; } public static class PropInfo { From noreply at svn.ci.uchicago.edu Wed Sep 22 17:33:52 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 22 Sep 2010 17:33:52 -0500 (CDT) Subject: [Swift-commit] r3640 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20100922223352.9CBFF9CCBC@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-22 17:33:52 -0500 (Wed, 22 Sep 2010) New Revision: 3640 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: New swift -minimal.logging option Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-09-22 17:37:18 UTC (rev 3639) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-09-22 22:33:52 UTC (rev 3640) @@ -70,6 +70,7 @@ public static final String ARG_TUI = "tui"; public static final String ARG_RECOMPILE = "recompile"; public static final String ARG_REDUCED_LOGGING = "reduced.logging"; + public static final String ARG_MINIMAL_LOGGING = "minimal.logging"; public static final String CONST_VDL_OPERATION = "vdl:operation"; public static final String VDL_OPERATION_RUN = "run"; @@ -427,6 +428,8 @@ ap.addFlag(ARG_TUI); ap.addFlag(ARG_REDUCED_LOGGING, "Makes logging more terse by disabling provenance " + "information and low-level task messages"); + ap.addFlag(ARG_MINIMAL_LOGGING, "Makes logging much more terse: " + + "reports warnings only"); Map desc = VDL2ConfigProperties.getPropertyDescriptions(); Iterator i = desc.entrySet().iterator(); @@ -488,6 +491,9 @@ Level.INFO); ca.setThreshold(Level.FATAL); } + else if (ap.isPresent(ARG_MINIMAL_LOGGING)) { + fa.setThreshold(Level.WARN); + } else if (ap.isPresent(ARG_REDUCED_LOGGING)) { Logger.getLogger(AbstractDataNode.class).setLevel(Level.WARN); Logger.getLogger(New.class).setLevel(Level.WARN); From noreply at svn.ci.uchicago.edu Fri Sep 24 13:44:00 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 24 Sep 2010 13:44:00 -0500 (CDT) Subject: [Swift-commit] r3641 - in SwiftApps/SwiftR: . Swift/R Swift/exec Message-ID: <20100924184400.C9C419CC9D@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2010-09-24 13:44:00 -0500 (Fri, 24 Sep 2010) New Revision: 3641 Modified: SwiftApps/SwiftR/README SwiftApps/SwiftR/Swift/R/Swift.R SwiftApps/SwiftR/Swift/exec/start-swift-Rserver SwiftApps/SwiftR/Swift/exec/start-swift-workers SwiftApps/SwiftR/TODO Log: Revisions to configuration and cleanup logic. Additional misc fixes. Modified: SwiftApps/SwiftR/README =================================================================== --- SwiftApps/SwiftR/README 2010-09-22 22:33:52 UTC (rev 3640) +++ SwiftApps/SwiftR/README 2010-09-24 18:44:00 UTC (rev 3641) @@ -21,8 +21,14 @@ Need to put the right R in the PATH for building and testing. Using R 2.11 as of 2010.0903 -OpenMx source tree checked out under: ~/SwiftR/OpenMx +OpenMx source tree checked out under: ~/OpenMx +working in .../trunk +~/SwiftR/OpenMx is a fossil and should be moved and then removed (when certain) +test under: trunk/demo +source("ParallelBootrapDemo") +... FIXME ^^^ + Repo is: https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/SwiftR R packages are installed under: ~/RPackages @@ -128,5 +134,29 @@ in swiftapply the remote calls are batched option processing: +# describe here: func kw args; def vals mostly NULL; options() values override if kw not specified. +Processing mode selects a set of sites, a cf, and a tc. +Site selects a site within a tc for a secific exec. + +rserver.swift runs an app() whose name encodes the site (eg: bashservice, bashlocal) + +Provider staging is a problem: its all or none. How to do provider staging for local? I think thats OK: hopefully its fast. + +Probem: would like to do multiple sites fomr the internal coaster service rather than an external coaster service + +--- + +cd ~/SwiftR +R CMD INSTALL Swift +R CMD check Swift +R CMD build Swift # produces ./Swift_0.1.tar.gz + +wget http://www.ci.uchicago.edu/~wilde/Swift_0.1.tar.hz +R CMD INSTALL -l ~/RLibrary Swift_0.1.tar.gz +export R_LIBS=~/RLibrary +R +require(Swift) + + Modified: SwiftApps/SwiftR/Swift/R/Swift.R =================================================================== --- SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-22 22:33:52 UTC (rev 3640) +++ SwiftApps/SwiftR/Swift/R/Swift.R 2010-09-24 18:44:00 UTC (rev 3641) @@ -123,6 +123,9 @@ nresults <- length(result) for(r in 1:nresults) { rlist[[rno]] <- result[[r]] +if(class(result[[r]]) == "try-error") { +cat("ERROR in eval: ", result[[r]], "\n"); +} #DB cat("swiftapply: result rno=",rno,":\n") # FIXME: for logging #DB cat(rlist[[rno]]@output$gradient,"\n") rno <- rno + 1 Modified: SwiftApps/SwiftR/Swift/exec/start-swift-Rserver =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-Rserver 2010-09-22 22:33:52 UTC (rev 3640) +++ SwiftApps/SwiftR/Swift/exec/start-swift-Rserver 2010-09-24 18:44:00 UTC (rev 3641) @@ -55,7 +55,7 @@ .03 10000 - $rundir/swiftwork + $HOME/swiftwork @@ -83,7 +83,7 @@ .07 10000 - /home/wilde/swiftwork + /$HOME/swiftwork @@ -93,7 +93,7 @@ .03 10000 - /home/wilde/swiftwork + /$HOME/swiftwork @@ -104,7 +104,7 @@ 10000 5.99 - $(pwd) + /$HOME/swiftwork @@ -168,7 +168,7 @@ .07 10000 - /home/wilde/swiftwork + $HOME/swiftwork @@ -178,7 +178,7 @@ .03 10000 proxy - /tmp/wilde/SwiftR/swiftwork + /tmp/$USER/SwiftR/swiftwork @@ -188,7 +188,7 @@ .03 10000 - /home/wilde/swiftwork + /$HOME/swiftwork @@ -199,7 +199,7 @@ 10000 5.99 - $(pwd) + $HOME/swiftwork Modified: SwiftApps/SwiftR/Swift/exec/start-swift-workers =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift-workers 2010-09-22 22:33:52 UTC (rev 3640) +++ SwiftApps/SwiftR/Swift/exec/start-swift-workers 2010-09-24 18:44:00 UTC (rev 3641) @@ -1,19 +1,47 @@ #! /bin/bash +Usage() +{ + echo $0: 'Usage: start-swift-workers [-ls|--local-staging|-ps|--provider-staging|-sp|--service-port portnum] site1 ... siteN' +} + # Command arguments -STAGINGMETHOD=${1:-local-staging} -COMPUTEHOSTS=${2:-localhost} +STAGINGMETHOD=local-staging +COMPUTEHOSTS=localhost +SERVICEPORT=1985 -# COMPUTEHOSTS='crush thwomp stomp crank steamroller grind churn trounce thrash vanquish' -# COMPUTEHOSTS='communicado' +while [ $# -gt 0 -a $(expr "$1" : -) = 1 ]; do + case "$1" in + -ls|--local-staging) STAGINGMETHOD=local-staging; shift ;; + -ps|--provider-staging) STAGINGMETHOD=provider-staging; shift ;; + -sp|--service-port) shift; SERVICEPORT=$1; shift ;; + *) echo $0: Invalid argument "$1"; Usage; exit 1 + esac +done -# Parameters # FIXME: Add to command line options +if [ $# -gt 0 ]; then + COMPUTEHOSTS=$* +fi -SERVICEPORT=1985 +if [ $(expr "$SERVICEPORT" : '[0-9]*' ) = 0 ]; then + echo $0: Error: invalid service port $SERVICEPORT; Usage; exit 1 +fi -# shift # FIXME: Why? Fossil? +echo Staging method: $STAGINGMETHOD +echo Service port: $SERVICEPORT +for h in $COMPUTEHOSTS; do + echo Host: $h + if [ $(expr $h : -) != 0 ]; then + echo $0: Error: invalid argument or host name: $h + Usage + exit 1 + fi +done +# COMPUTEHOSTS='crush thwomp stomp crank steamroller grind churn trounce thrash vanquish' +# COMPUTEHOSTS='communicado' + SWIFTRBIN=$(cd $(dirname $0); pwd) # Find our bin dir (to use for running utility scripts) SWIFTBIN=$SWIFTRBIN/../swift/bin @@ -58,13 +86,17 @@ echo $sshpids > $sshpidfile } +# make swiftworkers.XXXX temp dir and link swiftworkers/ to it + rundir=/tmp/$USER/SwiftR/swiftworkers - +mkdir -p $(basename $rundir) trundir=$(mktemp -d $rundir.XXXX) mkdir -p $trundir rm -rf $rundir ln -s $trundir $rundir +echo "Logging to $trundir" + servicedir=service cd $rundir out=swift.stdouterr @@ -87,7 +119,7 @@ .03 10000 - /home/wilde/swiftwork + $HOME/swiftwork END @@ -158,7 +190,7 @@ rpid=$(grep PID= $rpfile | sed -e 's/PID=//') rhost=$(echo $rpfile | sed -e 's/remotepid.//') echo Based on $rpfile: terminating process group of process $rpid on $rhost - ssh $rhost sh -c \'kill -s TERM -- '-$(' ps -p $rpid -o pgid --no-headers ')'\' + ssh $rhost sh -c \''PGID=$(ps -p '$rpid' -o pgid --no-headers|sed -e "s/ //g"); kill -s TERM -- -$PGID'\' done if [ "_$sshpids$starterpid$coasterservicepid" != _ ]; then echo kill $sshpids $starterpid $coasterservicepid >& /dev/null Modified: SwiftApps/SwiftR/TODO =================================================================== --- SwiftApps/SwiftR/TODO 2010-09-22 22:33:52 UTC (rev 3640) +++ SwiftApps/SwiftR/TODO 2010-09-24 18:44:00 UTC (rev 3641) @@ -29,8 +29,41 @@ Get traces into R vars of swiftapply performance +Init perf #s (from Crush) +11 runs, ~17 secs real time each. CPU times are below: +wilde 5804 5794 5804 5804 0 09:15 pts/12 00:00:00 bash +wilde 7182 5804 7182 5804 0 09:23 pts/12 00:00:00 /bin/bash /homes/wilde/RLibrary/Swift/exec/start-swift-workers +wilde 7193 7182 7182 5804 0 09:23 pts/12 00:00:00 /bin/sh /homes/wilde/RLibrary/Swift/exec/../swift/bin/coaster +wilde 7219 7193 7182 5804 0 09:23 pts/12 00:00:07 java -Djava.endorsed.dirs=/homes/wilde/RLibrary/Swift/exec/ +wilde 7625 7182 7182 5804 0 09:23 pts/12 00:00:00 /bin/bash /homes/wilde/RLibrary/Swift/exec/start-swift-Rserve +wilde 7643 7625 7182 5804 0 09:23 pts/12 00:00:00 /bin/sh /homes/wilde/RLibrary/Swift/exec/../swift/bin/swift +wilde 7706 7643 7182 5804 3 09:23 pts/12 00:00:34 java -Xmx256M -Djava.endorsed.dirs=/homes/wilde/RLibrary/ +wilde 8268 5804 8268 5804 9 09:29 pts/12 00:01:01 /usr/lib64/R/bin/exec/R +wilde 15581 5804 15581 5804 0 09:40 pts/12 00:00:00 ps -fjH -u wilde +wilde 28836 28769 28769 28769 0 Sep20 ? 00:00:00 sshd: wilde at pts/1 +wilde 28837 28836 28837 28837 0 Sep20 pts/1 00:00:00 -bash +wilde 7581 7513 7513 7513 0 09:23 ? 00:00:00 sshd: wilde at notty +wilde 8926 1 7582 7582 2 09:34 ? 00:00:07 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --a +wilde 8924 1 7582 7582 1 09:34 ? 00:00:06 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --a +wilde 8918 1 7582 7582 1 09:34 ? 00:00:07 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --a +wilde 8758 1 7582 7582 2 09:34 ? 00:00:07 /usr/lib64/R/bin/exec/R --slave --no-restore --file=./SwiftRServer.sh --a +wilde 7587 1 7582 7582 1 09:23 ? 00:00:16 /usr/bin/perl /tmp/wilde/SwiftR/swiftworkerlogs/worker.pl http://140.221. +wilde 7510 1 7182 5804 0 09:23 pts/12 00:00:00 ssh localhost /bin/sh -c 'WORKER_LOGGING_ENABLED=true /tmp/wilde/SwiftR/s +7s coaster service +34s swift +1:01 R +27s R servers +16 worker.pl + +R cli may have some overhead for processing answer each time. + +187 secs total real time. + + + + See if swift can write to pipe without using an app() Bundle start-swift-workers and RunRServer into one command; From noreply at svn.ci.uchicago.edu Mon Sep 27 09:30:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Sep 2010 09:30:32 -0500 (CDT) Subject: [Swift-commit] r3642 - trunk/tests/language-behaviour Message-ID: <20100927143032.2DB719CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-27 09:30:31 -0500 (Mon, 27 Sep 2010) New Revision: 3642 Added: trunk/tests/language-behaviour/071-singlefilemapper-input.check.sh trunk/tests/language-behaviour/071-singlefilemapper-input.clean.sh trunk/tests/language-behaviour/071-singlefilemapper-input.setup.sh trunk/tests/language-behaviour/07511-fixed-array-mapper-input.check.sh trunk/tests/language-behaviour/07511-fixed-array-mapper-input.clean.sh trunk/tests/language-behaviour/07511-fixed-array-mapper-input.setup.sh Modified: trunk/tests/language-behaviour/071-singlefilemapper-input.swift trunk/tests/language-behaviour/07511-fixed-array-mapper-input.swift Log: language-behaviour fixes Added: trunk/tests/language-behaviour/071-singlefilemapper-input.check.sh =================================================================== --- trunk/tests/language-behaviour/071-singlefilemapper-input.check.sh (rev 0) +++ trunk/tests/language-behaviour/071-singlefilemapper-input.check.sh 2010-09-27 14:30:31 UTC (rev 3642) @@ -0,0 +1,8 @@ +#!/bin/bash + +set -x + +[ -f 071-singlefilemapper-input.in ] || exit 1 +grep 071-singlefilemapper-input.in 071-singlefilemapper-input.out || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/071-singlefilemapper-input.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/071-singlefilemapper-input.clean.sh =================================================================== --- trunk/tests/language-behaviour/071-singlefilemapper-input.clean.sh (rev 0) +++ trunk/tests/language-behaviour/071-singlefilemapper-input.clean.sh 2010-09-27 14:30:31 UTC (rev 3642) @@ -0,0 +1,8 @@ +#!/bin/bash + +set -x + +rm -v 071-singlefilemapper-input.in \ + 071-singlefilemapper-input.out || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/071-singlefilemapper-input.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/071-singlefilemapper-input.setup.sh =================================================================== --- trunk/tests/language-behaviour/071-singlefilemapper-input.setup.sh (rev 0) +++ trunk/tests/language-behaviour/071-singlefilemapper-input.setup.sh 2010-09-27 14:30:31 UTC (rev 3642) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +cp -v $GROUP/071-singlefilemapper-input.in . || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/071-singlefilemapper-input.setup.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/language-behaviour/071-singlefilemapper-input.swift =================================================================== --- trunk/tests/language-behaviour/071-singlefilemapper-input.swift 2010-09-24 18:44:00 UTC (rev 3641) +++ trunk/tests/language-behaviour/071-singlefilemapper-input.swift 2010-09-27 14:30:31 UTC (rev 3642) @@ -1,9 +1,7 @@ type messagefile; -(messagefile o) write(messagefile i) { - app { - echo @filename(i) stdout=@filename(o); - } +app (messagefile o) write(messagefile i) { + echo @filename(i) stdout=@filename(o); } messagefile infile <"071-singlefilemapper-input.in">; Added: trunk/tests/language-behaviour/07511-fixed-array-mapper-input.check.sh =================================================================== --- trunk/tests/language-behaviour/07511-fixed-array-mapper-input.check.sh (rev 0) +++ trunk/tests/language-behaviour/07511-fixed-array-mapper-input.check.sh 2010-09-27 14:30:31 UTC (rev 3642) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +cat stdout.txt + +X=$( grep -c "file: 07511" stdout.txt ) +[ $? == 0 ] || exit 1 +echo $X +(( $X == 3 )) || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/07511-fixed-array-mapper-input.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/07511-fixed-array-mapper-input.clean.sh =================================================================== --- trunk/tests/language-behaviour/07511-fixed-array-mapper-input.clean.sh (rev 0) +++ trunk/tests/language-behaviour/07511-fixed-array-mapper-input.clean.sh 2010-09-27 14:30:31 UTC (rev 3642) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +rm -v $GROUP/07511-fixed-array-mapper-input.*.in || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/07511-fixed-array-mapper-input.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/language-behaviour/07511-fixed-array-mapper-input.setup.sh =================================================================== --- trunk/tests/language-behaviour/07511-fixed-array-mapper-input.setup.sh (rev 0) +++ trunk/tests/language-behaviour/07511-fixed-array-mapper-input.setup.sh 2010-09-27 14:30:31 UTC (rev 3642) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +cp -v $GROUP/07511-fixed-array-mapper-input.*.in . || exit 1 + +exit 0 Property changes on: trunk/tests/language-behaviour/07511-fixed-array-mapper-input.setup.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/language-behaviour/07511-fixed-array-mapper-input.swift =================================================================== --- trunk/tests/language-behaviour/07511-fixed-array-mapper-input.swift 2010-09-24 18:44:00 UTC (rev 3641) +++ trunk/tests/language-behaviour/07511-fixed-array-mapper-input.swift 2010-09-27 14:30:31 UTC (rev 3642) @@ -1,16 +1,19 @@ type messagefile; -p(messagefile t) { - app { - echo "foo" stdin=@filename(t); - } +/* +app p(messagefile t) { + echo "foo" stdin=@filename(t); } - +*/ string fns="07511-fixed-array-mapper-input.first.in 07511-fixed-array-mapper-input.second.in 07511-fixed-array-mapper-input.third.in"; messagefile infile[] ; +/* p(infile[0]); p(infile[1]); p(infile[2]); - +*/ +tracef("file: %s\n", @filename(infile[0])); +tracef("file: %s\n", @filename(infile[1])); +tracef("file: %s\n", @filename(infile[2])); From noreply at svn.ci.uchicago.edu Mon Sep 27 11:28:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Sep 2010 11:28:25 -0500 (CDT) Subject: [Swift-commit] r3643 - trunk/tests/functions Message-ID: <20100927162825.1AE479CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-27 11:28:24 -0500 (Mon, 27 Sep 2010) New Revision: 3643 Added: trunk/tests/functions/title.txt Log: Title for functions tests Added: trunk/tests/functions/title.txt =================================================================== --- trunk/tests/functions/title.txt (rev 0) +++ trunk/tests/functions/title.txt 2010-09-27 16:28:24 UTC (rev 3643) @@ -0,0 +1 @@ +Swift functions From noreply at svn.ci.uchicago.edu Mon Sep 27 11:30:59 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Sep 2010 11:30:59 -0500 (CDT) Subject: [Swift-commit] r3644 - in trunk: libexec src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan src/org/griphyn/vdl/karajan/lib/swiftscript tests tests/functions Message-ID: <20100927163059.E1E609CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-27 11:30:59 -0500 (Mon, 27 Sep 2010) New Revision: 3644 Added: trunk/src/org/griphyn/vdl/karajan/AssertFailedException.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java trunk/tests/functions/100-assert-bool.check.sh trunk/tests/functions/100-assert-bool.swift trunk/tests/functions/100-assert-int.check.sh trunk/tests/functions/100-assert-int.swift Modified: trunk/libexec/vdl-lib.xml trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java trunk/tests/nightly.sh Log: New assert() function and tests Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-09-27 16:28:24 UTC (rev 3643) +++ trunk/libexec/vdl-lib.xml 2010-09-27 16:30:59 UTC (rev 3644) @@ -16,6 +16,7 @@ + Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-09-27 16:28:24 UTC (rev 3643) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-09-27 16:30:59 UTC (rev 3644) @@ -143,6 +143,11 @@ tracef.setInvocationMode(INVOCATION_INTERNAL); proceduresMap.put("tracef", tracef); + ProcedureSignature assrt = new ProcedureSignature("assert"); + assrt.setAnyNumOfInputArgs(); + assrt.setInvocationMode(INVOCATION_INTERNAL); + proceduresMap.put("assert", assrt); + ProcedureSignature writeData = new ProcedureSignature("writeData"); FormalArgumentSignature wdInputArg = new FormalArgumentSignature(true); writeData.addInputArg(wdInputArg); Added: trunk/src/org/griphyn/vdl/karajan/AssertFailedException.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/AssertFailedException.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/AssertFailedException.java 2010-09-27 16:30:59 UTC (rev 3644) @@ -0,0 +1,19 @@ + +package org.griphyn.vdl.karajan; + +/** + * Generated only by SwiftScript @assert(). + * + * Created on September 27, 2010 + * @author wozniak + */ +public class AssertFailedException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + String message = null; + + public AssertFailedException(String message) { + super(message); + } +} Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java 2010-09-27 16:30:59 UTC (rev 3644) @@ -0,0 +1,69 @@ +package org.griphyn.vdl.karajan.lib.swiftscript; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.karajan.AssertFailedException; +import org.griphyn.vdl.karajan.lib.SwiftArg; +import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.type.Types; + +/** + Throw AssertionException if input is false or 0. + */ +public class Assert extends VDLFunction { + + private static final Logger logger = + Logger.getLogger(Assert.class); + + static { + setArguments(Assert.class, new Arg[] { Arg.VARGS }); + } + + @Override + protected Object function(VariableStack stack) + throws ExecutionException { + DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); + String message = ""; + + for (int i = 0; i < args.length; i++) { + DSHandle handle = args[i]; + VDLFunction.waitFor(stack, handle); + } + + logger.debug("check: "); + + if (args.length == 2) + if (args[1].getType() == Types.STRING) + message = args[1].toString(); + else + throw new ExecutionException + ("Second argument to assert() must be a String!"); + + checkAssert(args[0], message); + + return null; + } + + private void checkAssert(DSHandle value, String message) + throws ExecutionException + { + boolean success = true; + if (value.getType() == Types.BOOLEAN) { + if (! (Boolean) value.getValue()) + success = false; + } + else if (value.getType() == Types.INT) { + double d = ((Double) value.getValue()).doubleValue(); + if (d == 0.0) + success = false; + } + else + throw new ExecutionException + ("First argument to assert() must be boolean or int!"); + if (! success) + throw new AssertFailedException(message); + } +} Added: trunk/tests/functions/100-assert-bool.check.sh =================================================================== --- trunk/tests/functions/100-assert-bool.check.sh (rev 0) +++ trunk/tests/functions/100-assert-bool.check.sh 2010-09-27 16:30:59 UTC (rev 3644) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +grep ASSERT_MESSAGE stdout.txt || exit 1 + +exit 0 Property changes on: trunk/tests/functions/100-assert-bool.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/functions/100-assert-bool.swift =================================================================== --- trunk/tests/functions/100-assert-bool.swift (rev 0) +++ trunk/tests/functions/100-assert-bool.swift 2010-09-27 16:30:59 UTC (rev 3644) @@ -0,0 +1,4 @@ + +// THIS-SCRIPT-SHOULD-FAIL + +assert(false, "ASSERT_MESSAGE"); Added: trunk/tests/functions/100-assert-int.check.sh =================================================================== --- trunk/tests/functions/100-assert-int.check.sh (rev 0) +++ trunk/tests/functions/100-assert-int.check.sh 2010-09-27 16:30:59 UTC (rev 3644) @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x + +grep AssertFailedException stdout.txt || exit 1 + +exit 0 Property changes on: trunk/tests/functions/100-assert-int.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/functions/100-assert-int.swift =================================================================== --- trunk/tests/functions/100-assert-int.swift (rev 0) +++ trunk/tests/functions/100-assert-int.swift 2010-09-27 16:30:59 UTC (rev 3644) @@ -0,0 +1,4 @@ + +// THIS-SCRIPT-SHOULD-FAIL + +assert(2-2); Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-27 16:28:24 UTC (rev 3643) +++ trunk/tests/nightly.sh 2010-09-27 16:30:59 UTC (rev 3644) @@ -56,6 +56,12 @@ # +-bin/swift # +-java +# FAILURE CASES +# Some cases are designed to cause Swift to crash. These +# SwiftScripts contain the token THIS-SCRIPT-SHOULD-FAIL somewhere. +# The response of nightly.sh to the exit code of these Swift +# executions is reversed. + printhelp() { echo "nightly.sh " echo "" @@ -486,6 +492,7 @@ if [ -f $OUTPUT ]; then cat $OUTPUT >> $LOG fi + (( $TEST_SHOULD_FAIL )) && EXITCODE=$(( ! $EXITCODE )) return $EXITCODE } @@ -619,6 +626,8 @@ CHECKSCRIPT=${SWIFTSCRIPT%.swift}.check.sh CLEANSCRIPT=${SWIFTSCRIPT%.swift}.clean.sh TIMEOUTFILE=${SWIFTSCRIPT%.swift}.timeout + + TEST_SHOULD_FAIL=0 if [ -x $GROUP/$SETUPSCRIPT ]; then script_exec $GROUP/$SETUPSCRIPT "S" fi @@ -630,6 +639,9 @@ TIMEOUT=$( gettimeout $GROUP/$TIMEOUTFILE ) + grep THIS-SCRIPT-SHOULD-FAIL $SWIFTSCRIPT > /dev/null + TEST_SHOULD_FAIL=$(( ! $? )) + monitored_exec $TIMEOUT swift \ -wrapperlog.always.transfer true \ -config swift.properties \ @@ -637,6 +649,7 @@ -tc.file tc.data \ $CDM $SWIFTSCRIPT + TEST_SHOULD_FAIL=0 if [ -x $GROUP/$CHECKSCRIPT ]; then script_exec $GROUP/$CHECKSCRIPT "√" fi @@ -819,15 +832,15 @@ SKIP_COUNTER=0 -# GROUPLIST=( $TESTDIR/local $TESTDIR/cdm $TESTDIR/cdm/ps $TESTDIR/cdm/ps/pinned ) +GROUPLIST=( $TESTDIR/functions $TESTDIR/local $TESTDIR/cdm $TESTDIR/cdm/ps $TESTDIR/cdm/ps/pinned ) -GROUPLIST=( $TESTDIR/language-behaviour - $TESTDIR/language/working \ - $TESTDIR/local \ - $TESTDIR/language/should-not-work \ - $TESTDIR/cdm \ - $TESTDIR/cdm/ps \ - $TESTDIR/cdm/ps/pinned ) +# GROUPLIST=( $TESTDIR/language-behaviour +# $TESTDIR/language/working \ +# $TESTDIR/local \ +# $TESTDIR/language/should-not-work \ +# $TESTDIR/cdm \ +# $TESTDIR/cdm/ps \ +# $TESTDIR/cdm/ps/pinned ) GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do From noreply at svn.ci.uchicago.edu Mon Sep 27 11:31:57 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Sep 2010 11:31:57 -0500 (CDT) Subject: [Swift-commit] r3645 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100927163157.82F7C9CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-27 11:31:57 -0500 (Mon, 27 Sep 2010) New Revision: 3645 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java Log: Corrections Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java 2010-09-27 16:30:59 UTC (rev 3644) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Assert.java 2010-09-27 16:31:57 UTC (rev 3645) @@ -12,6 +12,7 @@ /** Throw AssertionException if input is false or 0. + Optional second argument is string message printed on failure. */ public class Assert extends VDLFunction { @@ -33,8 +34,6 @@ VDLFunction.waitFor(stack, handle); } - logger.debug("check: "); - if (args.length == 2) if (args[1].getType() == Types.STRING) message = args[1].toString(); From noreply at svn.ci.uchicago.edu Mon Sep 27 11:32:10 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Sep 2010 11:32:10 -0500 (CDT) Subject: [Swift-commit] r3646 - trunk/src/org/griphyn/vdl/type Message-ID: <20100927163210.31FC29CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-27 11:32:09 -0500 (Mon, 27 Sep 2010) New Revision: 3646 Modified: trunk/src/org/griphyn/vdl/type/Types.java Log: Use generics Modified: trunk/src/org/griphyn/vdl/type/Types.java =================================================================== --- trunk/src/org/griphyn/vdl/type/Types.java 2010-09-27 16:31:57 UTC (rev 3645) +++ trunk/src/org/griphyn/vdl/type/Types.java 2010-09-27 16:32:09 UTC (rev 3646) @@ -1,13 +1,13 @@ package org.griphyn.vdl.type; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; public abstract class Types { //TODO: check namespace references in type definitions - private static Map types = new HashMap(); + private static Map types = + new HashMap(); public synchronized static Type getType(String name) throws NoSuchTypeException { Type type = (Type) types.get(name); @@ -61,14 +61,11 @@ } public synchronized static void resolveTypes() throws NoSuchTypeException { - Map typesCopy = new HashMap(types); - Iterator i = typesCopy.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); + Map typesCopy = + new HashMap(types); + for (Map.Entry e : typesCopy.entrySet()) { Type type = (Type) e.getValue(); - Iterator j = type.getFields().iterator(); - while (j.hasNext()) { - Field field = (Field) j.next(); + for (Field field : type.getFields()) { Type resolved = getType(field.getType().getName()); field.setType(resolved); } From noreply at svn.ci.uchicago.edu Mon Sep 27 12:56:09 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Sep 2010 12:56:09 -0500 (CDT) Subject: [Swift-commit] r3647 - in trunk: libexec src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib/swiftscript tests/functions Message-ID: <20100927175609.814299CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-27 12:56:09 -0500 (Mon, 27 Sep 2010) New Revision: 3647 Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java trunk/tests/functions/200-sprintf.swift trunk/tests/functions/README.txt Modified: trunk/libexec/vdl-lib.xml trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java Log: New @sprintf() and test Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-09-27 16:32:09 UTC (rev 3646) +++ trunk/libexec/vdl-lib.xml 2010-09-27 17:56:09 UTC (rev 3647) @@ -15,6 +15,7 @@ + Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-09-27 16:32:09 UTC (rev 3646) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-09-27 17:56:09 UTC (rev 3647) @@ -224,6 +224,12 @@ strcat.addOutputArg(strcatOut1); functionsMap.put(strcat.getName(), strcat); + ProcedureSignature sprintf = new ProcedureSignature("sprintf"); + sprintf.setAnyNumOfInputArgs(); + FormalArgumentSignature sprintfOut1 = new FormalArgumentSignature("string"); + sprintf.addOutputArg(sprintfOut1); + functionsMap.put(sprintf.getName(), sprintf); + ProcedureSignature strcut = new ProcedureSignature("strcut"); FormalArgumentSignature strcutIn1 = new FormalArgumentSignature("string"); strcut.addInputArg(strcutIn1); Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java 2010-09-27 17:56:09 UTC (rev 3647) @@ -0,0 +1,245 @@ +package org.griphyn.vdl.karajan.lib.swiftscript; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.karajan.lib.SwiftArg; +import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.mapping.ArrayDataNode; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.mapping.RootDataNode; +import org.griphyn.vdl.type.Types; + +/** + Formatted string generation.
+ Example: sprintf("\t%s\n", "hello");
+ Differences from trace(): + 1) respects \t, \n and \\; + 2) allows for typechecked format specifiers; + 3) allows for consumption of variables without display (%k) + Format specifiers:
+ %%: % sign.
+ %M: Filename output: waits for close + %p: Not typechecked, output as in trace().
+ %f: Typechecked float output.
+ %i: Typechecked int output.
+ %s: Typechecked string output.
+ %k: Variable sKipped, no output.
+ %q: Array output + */ +public class Sprintf extends VDLFunction { + + private static final Logger logger = + Logger.getLogger(Sprintf.class); + + static { + setArguments(Sprintf.class, new Arg[] { Arg.VARGS }); + } + + @Override + protected Object function(VariableStack stack) + throws ExecutionException { + DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); + + for (int i = 0; i < args.length; i++) { + DSHandle handle = args[i]; + VDLFunction.waitFor(stack, handle); + } + String msg = format(args); + logger.debug("generated: " + msg); + + DSHandle result = new RootDataNode(Types.STRING); + result.setValue(msg); + return result; + } + + private String format(DSHandle[] args) + throws ExecutionException { + if (! (args[0].getType() == Types.STRING)) + throw new ExecutionException + ("First argument to sprintf() must be a string!"); + + String spec = args[0].toString(); + DSHandle[] vars = copyArray(args, 1, args.length-1); + + StringBuilder output = new StringBuilder(); + format(spec, vars, output); + + return output.toString(); + } + + public static DSHandle[] copyArray(DSHandle[] src, + int offset, int length) + { + DSHandle[] result = new DSHandle[length]; + + for (int i = 0; i < length; i++) + result[i] = src[i+offset]; + + return result; + } + + /** + This method can be targeted as a helper function + (by @sprintf(), etc.) + */ + public static void format(String spec, DSHandle[] vars, + StringBuilder output) + throws ExecutionException + { + int i = 0; + int arg = 0; + while (i < spec.length()) { + char c = spec.charAt(i); + if (c == '%') { + char d = spec.charAt(++i); + arg = append(d, arg, vars, output); + } + else if (c == '\\') { + char d = spec.charAt(++i); + escape(d, output); + } + else { + output.append(c); + } + i++; + } + } + + private static int append(char c, int arg, DSHandle[] vars, + StringBuilder output) + throws ExecutionException { + if (c == '%') { + output.append('%'); + return arg; + } + if (arg >= vars.length) { + throw new ExecutionException + ("tracef(): too many specifiers!"); + } + if (c == 'M') { + append_M(vars[arg], output); + } + else if (c == 'f') { + append_f(vars[arg], output); + } + else if (c == 'i') { + append_i(vars[arg], output); + } + else if (c == 'p') { + output.append(vars[arg].toString()); + } + else if (c == 's') { + append_s(vars[arg], output); + } + else if (c == 'q') { + append_q(vars[arg], output); + } + else if (c == 'k') { + ; + } + else { + throw new ExecutionException + ("tracef(): Unknown format: %" + c); + } + return arg+1; + } + + private static void append_M(DSHandle arg, StringBuilder output) + throws ExecutionException { + try { + synchronized (arg.getRoot()) { + String[] names = VDLFunction.filename(arg); + if (names.length > 1) + output.append(names); + else + output.append(names[0]); + } + } + catch (Exception e) { + throw new ExecutionException + ("tracef(%M): Could not lookup: " + arg); + } + } + + private static void append_f(DSHandle arg, StringBuilder output) + throws ExecutionException { + if (arg.getType() == Types.FLOAT) { + output.append(arg).toString(); + } + else { + throw new ExecutionException + ("tracef(): %f requires a float!"); + } + } + + private static void append_i(DSHandle arg, StringBuilder output) + throws ExecutionException { + if (arg.getType() == Types.INT) { + output.append(arg).toString(); + } + else { + throw new ExecutionException + ("tracef(): %i requires an int!"); + } + } + + private static void append_q(DSHandle arg, StringBuilder output) + throws ExecutionException { + if (arg instanceof ArrayDataNode) { + ArrayDataNode node = (ArrayDataNode) arg; + output.append("["); + try { + int size = node.size(); + for (int i = 0; i < size; i++) { + String entry = ""+i; + DSHandle handle = + node.getField(Path.parse(entry)); + output.append(handle); + if (i < size-1) + output.append(","); + } + } + catch (Exception e) { + e.printStackTrace(); + throw new ExecutionException + ("trace(%q): Could not get children of: " + arg); + } + output.append("]"); + } + else { + throw new ExecutionException + ("tracef(): %q requires an array!"); + } + } + + private static void append_s(DSHandle arg, StringBuilder output) + throws ExecutionException { + if (arg.getType() == Types.STRING) { + output.append(arg).toString(); + } + else { + throw new ExecutionException + ("tracef(): %s requires a string!"); + } + } + + private static void escape(char c, StringBuilder output) + throws ExecutionException { + if (c == '\\') { + output.append('\\'); + } + else if (c == 'n') { + output.append('\n'); + } + else if (c == 't') { + output.append('\t'); + } + else { + throw new ExecutionException + ("tracef(): unknown backslash escape sequence!"); + } + } +} Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-09-27 16:32:09 UTC (rev 3646) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-09-27 17:56:09 UTC (rev 3647) @@ -6,28 +6,17 @@ import org.globus.cog.karajan.workflow.ExecutionException; import org.griphyn.vdl.karajan.lib.SwiftArg; import org.griphyn.vdl.karajan.lib.VDLFunction; -import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.type.Types; /** Formatted trace output.
Example: tracef("\t%s\n", "hello");
Differences from trace(): 1) respects \t, \n and \\; - 2) allows for typechecked format specifiers; + 2) allows for typechecked format specifiers + (cf. {@link Sprintf}); 3) allows for consumption of variables without display (%k); 4) does not impose any formatting (commas, etc.).

- Format specifiers:
- %%: % sign.
- %M: Filename output: waits for close - %p: Not typechecked, output as in trace().
- %f: Typechecked float output.
- %i: Typechecked int output.
- %s: Typechecked string output.
- %k: Variable sKipped, no output.
- %q: Array output */ public class Tracef extends VDLFunction { @@ -47,175 +36,15 @@ DSHandle handle = args[i]; VDLFunction.waitFor(stack, handle); } - String msg = format(args); + + String spec = args[0].toString(); + DSHandle[] vars = Sprintf.copyArray(args, 1, args.length-1); + + StringBuilder output = new StringBuilder(); + Sprintf.format(spec, vars, output); + String msg = output.toString(); logger.info(msg); System.out.print(msg); return null; } - - private String format(DSHandle[] args) - throws ExecutionException { - if (! (args[0].getType() == Types.STRING)) - throw new ExecutionException - ("First argument to tracef() must be a string!"); - - String spec = args[0].toString(); - StringBuffer output = new StringBuffer(); - int i = 0; - int arg = 1; - while (i < spec.length()) { - char c = spec.charAt(i); - if (c == '%') { - char d = spec.charAt(++i); - arg = append(d, arg, args, output); - } - else if (c == '\\') { - char d = spec.charAt(++i); - escape(d, output); - } - else { - output.append(c); - } - i++; - } - String result = output.toString(); - return result; - } - - private int append(char c, int arg, DSHandle[] args, - StringBuffer output) - throws ExecutionException { - if (c == '%') { - output.append('%'); - return arg; - } - if (arg >= args.length) { - throw new ExecutionException - ("tracef(): too many specifiers!"); - } - if (c == 'M') { - append_M(args[arg], output); - } - else if (c == 'f') { - append_f(args[arg], output); - } - else if (c == 'i') { - append_i(args[arg], output); - } - else if (c == 'p') { - output.append(args[arg].toString()); - } - else if (c == 's') { - append_s(args[arg], output); - } - else if (c == 'q') { - append_q(args[arg], output); - } - else if (c == 'k') { - ; - } - else { - throw new ExecutionException - ("tracef(): Unknown format: %" + c); - } - return arg+1; - } - - private void append_M(DSHandle arg, StringBuffer output) - throws ExecutionException { - try { - synchronized (arg.getRoot()) { - String[] names = VDLFunction.filename(arg); - if (names.length > 1) - output.append(names); - else - output.append(names[0]); - } - } - catch (Exception e) { - throw new ExecutionException - ("tracef(%M): Could not lookup: " + arg); - } - } - - private void append_f(DSHandle arg, StringBuffer output) - throws ExecutionException { - if (arg.getType() == Types.FLOAT) { - output.append(arg).toString(); - } - else { - throw new ExecutionException - ("tracef(): %f requires a float!"); - } - } - - private void append_i(DSHandle arg, StringBuffer output) - throws ExecutionException { - if (arg.getType() == Types.INT) { - output.append(arg).toString(); - } - else { - throw new ExecutionException - ("tracef(): %i requires an int!"); - } - } - - private void append_q(DSHandle arg, StringBuffer output) - throws ExecutionException { - if (arg instanceof ArrayDataNode) { - ArrayDataNode node = (ArrayDataNode) arg; - output.append("["); - try { - int size = node.size(); - for (int i = 0; i < size; i++) { - String entry = ""+i; - DSHandle handle = - node.getField(Path.parse(entry)); - output.append(handle); - if (i < size-1) - output.append(","); - } - } - catch (Exception e) { - e.printStackTrace(); - throw new ExecutionException - ("trace(%q): Could not get children of: " + arg); - } - output.append("]"); - } - else { - throw new ExecutionException - ("tracef(): %q requires an array!"); - } - } - - private void append_s(DSHandle arg, StringBuffer output) - throws ExecutionException { - if (arg.getType() == Types.STRING) { - output.append(arg).toString(); - } - else { - throw new ExecutionException - ("tracef(): %s requires a string!"); - } - } - - private void escape(char c, StringBuffer output) - throws ExecutionException { - if (c == '\\') { - output.append('\\'); - } - else if (c == 'n') { - output.append('\n'); - } - else if (c == 't') { - output.append('\t'); - } - else { - throw new ExecutionException - ("tracef(): unknown backslash escape sequence!"); - } - } - - } Added: trunk/tests/functions/200-sprintf.swift =================================================================== --- trunk/tests/functions/200-sprintf.swift (rev 0) +++ trunk/tests/functions/200-sprintf.swift 2010-09-27 17:56:09 UTC (rev 3647) @@ -0,0 +1,7 @@ + +string d = "-"; +string b = "bye"; + +string f = @sprintf("hi%s%s", d, b); + +assert(f == "hi-bye"); Added: trunk/tests/functions/README.txt =================================================================== --- trunk/tests/functions/README.txt (rev 0) +++ trunk/tests/functions/README.txt 2010-09-27 17:56:09 UTC (rev 3647) @@ -0,0 +1,6 @@ + +Tests for Swift's functions library + +100 series: Fundamentals +200 series: String processing +400 series: I/O From noreply at svn.ci.uchicago.edu Mon Sep 27 13:51:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Sep 2010 13:51:55 -0500 (CDT) Subject: [Swift-commit] r3648 - in trunk: libexec src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib/swiftscript tests/functions Message-ID: <20100927185155.DB81A9CCA4@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-27 13:51:54 -0500 (Mon, 27 Sep 2010) New Revision: 3648 Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java trunk/tests/functions/400-fprintf.check.sh trunk/tests/functions/400-fprintf.clean.sh trunk/tests/functions/400-fprintf.setup.sh trunk/tests/functions/400-fprintf.swift Modified: trunk/libexec/vdl-lib.xml trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java Log: New fprintf() and test Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-09-27 17:56:09 UTC (rev 3647) +++ trunk/libexec/vdl-lib.xml 2010-09-27 18:51:54 UTC (rev 3648) @@ -20,6 +20,8 @@ + + Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-09-27 17:56:09 UTC (rev 3647) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-09-27 18:51:54 UTC (rev 3648) @@ -143,6 +143,11 @@ tracef.setInvocationMode(INVOCATION_INTERNAL); proceduresMap.put("tracef", tracef); + ProcedureSignature fprintf = new ProcedureSignature("ftracef"); + fprintf.setAnyNumOfInputArgs(); + fprintf.setInvocationMode(INVOCATION_INTERNAL); + proceduresMap.put("fprintf", fprintf); + ProcedureSignature assrt = new ProcedureSignature("assert"); assrt.setAnyNumOfInputArgs(); assrt.setInvocationMode(INVOCATION_INTERNAL); Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Fprintf.java 2010-09-27 18:51:54 UTC (rev 3648) @@ -0,0 +1,107 @@ +package org.griphyn.vdl.karajan.lib.swiftscript; + +import java.io.FileWriter; +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.karajan.lib.SwiftArg; +import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.type.Types; + +/** + Formatted file output.
+ Example: fprintf("tmp.log", "\t%s\n", "hello");
+ Appends to file. + @see Tracef, Sprintf + @author wozniak + */ +public class Fprintf extends VDLFunction { + + private static final Logger logger = + Logger.getLogger(Fprintf.class); + + static { + setArguments(Fprintf.class, new Arg[] { Arg.VARGS }); + } + + static ConcurrentHashMap openFiles = + new ConcurrentHashMap(); + + @Override + protected Object function(VariableStack stack) + throws ExecutionException { + DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); + + for (int i = 0; i < args.length; i++) { + DSHandle handle = args[i]; + VDLFunction.waitFor(stack, handle); + } + + check(args); + + String filename = args[0].toString(); + String spec = args[1].toString(); + DSHandle[] vars = Sprintf.copyArray(args, 2, args.length-2); + + StringBuilder output = new StringBuilder(); + Sprintf.format(spec, vars, output); + String msg = output.toString(); + + logger.debug("file: " + filename + " msg: " + msg); + write(filename, msg); + return null; + } + + private static void check(DSHandle[] args) + throws ExecutionException { + if (args.length < 2) + throw new ExecutionException + ("fprintf(): requires at least 2 arguments!"); + if (! args[0].getType().equals(Types.STRING)) + throw new ExecutionException + ("fprintf(): first argument is a string filename!"); + if (! args[0].getType().equals(Types.STRING)) + throw new ExecutionException + ("fprintf(): second argument is a string specifier!"); + } + + private static void write(String filename, String msg) + throws ExecutionException { + acquire(filename); + + try { + FileWriter writer = new FileWriter(filename, true); + writer.write(msg); + writer.close(); + } + catch (IOException e) { + throw new ExecutionException + ("write(): problem writing to: " + filename, e); + } + + openFiles.remove(filename); + } + + private static void acquire(String filename) + throws ExecutionException { + int count = 0; + Object marker = new Object(); + while (openFiles.putIfAbsent(filename, marker) != null && + count < 10) { + try { + Thread.sleep(count); + } + catch (InterruptedException e) + {} + count++; + } + if (count == 10) + throw new ExecutionException + ("write(): could not acquire: " + filename); + } +} Added: trunk/tests/functions/400-fprintf.check.sh =================================================================== --- trunk/tests/functions/400-fprintf.check.sh (rev 0) +++ trunk/tests/functions/400-fprintf.check.sh 2010-09-27 18:51:54 UTC (rev 3648) @@ -0,0 +1,15 @@ +#!/bin/bash + +set -x + +FILES=( $( ls 400-fprintf-*.out ) ) +(( ${#FILES[@]} == 3 )) || exit 1 + +grep hello 400-fprintf-1.out || exit 1 + +LINES=$( wc -l 400-fprintf-2.out | cut -d ' ' -f 1 ) +(( ${LINES} == 3 )) || exit 1 + +grep "hello: 32" 400-fprintf-3.out || exit 1 + +exit 0 Property changes on: trunk/tests/functions/400-fprintf.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/functions/400-fprintf.clean.sh =================================================================== --- trunk/tests/functions/400-fprintf.clean.sh (rev 0) +++ trunk/tests/functions/400-fprintf.clean.sh 2010-09-27 18:51:54 UTC (rev 3648) @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -v 400-fprintf-*.out || exit 1 + +exit 0 Property changes on: trunk/tests/functions/400-fprintf.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/functions/400-fprintf.setup.sh =================================================================== --- trunk/tests/functions/400-fprintf.setup.sh (rev 0) +++ trunk/tests/functions/400-fprintf.setup.sh 2010-09-27 18:51:54 UTC (rev 3648) @@ -0,0 +1,7 @@ +#!/bin/bash + +# fprintf() uses append mode, so ensure target files do not exist + +rm -fv 400-fprintf-*.out + +exit 0 Property changes on: trunk/tests/functions/400-fprintf.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/functions/400-fprintf.swift =================================================================== --- trunk/tests/functions/400-fprintf.swift (rev 0) +++ trunk/tests/functions/400-fprintf.swift 2010-09-27 18:51:54 UTC (rev 3648) @@ -0,0 +1,4 @@ + +fprintf("400-fprintf-1.out", "hello\n"); +fprintf("400-fprintf-2.out", "%s\n%s\n%s\n", "hi", "ok", "bye"); +fprintf("400-fprintf-3.out", "hello: %i\n", 32); From noreply at svn.ci.uchicago.edu Mon Sep 27 16:19:48 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Sep 2010 16:19:48 -0500 (CDT) Subject: [Swift-commit] r3649 - trunk/tests Message-ID: <20100927211948.B06049CCA4@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-27 16:19:48 -0500 (Mon, 27 Sep 2010) New Revision: 3649 Modified: trunk/tests/nightly.sh Log: Output readability improvements Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-09-27 18:51:54 UTC (rev 3648) +++ trunk/tests/nightly.sh 2010-09-27 21:19:48 UTC (rev 3649) @@ -155,6 +155,8 @@ SWIFTCOUNT=0 +echo "HTML_OUTPUT: $HTML" + cd $TOPDIR mkdir -p $RUNDIR [ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1 @@ -175,6 +177,12 @@ exit 1 } +verbose() { + MSG="${*}" + (( $VERBOSE )) && echo $MSG + return 0 +} + header() { CURRENT=$SCRIPTDIR/html/current.html sed "s at _HTMLBASE_@$HTMLPATH@" < $CURRENT > $TOPDIR/current.html @@ -426,8 +434,8 @@ TEST_LOG="output_$LOGCOUNT.txt" banner "$LASTCMD" $RUNDIR/$TEST_LOG if [ -f $OUTPUT ]; then - cp -v $OUTPUT $RUNDIR/$TEST_LOG 2>>$LOG - cp -v $OUTPUT $RUNDIR/stdout.txt + cp $OUTPUT $RUNDIR/$TEST_LOG 2>>$LOG + cp $OUTPUT $RUNDIR/stdout.txt fi let "LOGCOUNT=$LOGCOUNT+1" } @@ -477,7 +485,7 @@ process_exec() { printf "\nExecuting: $@" >>$LOG - rm -fv $OUTPUT + rm -f $OUTPUT "$@" > $OUTPUT 2>&1 & EXEC_PID=$! @@ -538,12 +546,13 @@ V=$SWIFTCOUNT # Use background so kill/trap is immediate - sleep $TIMEOUT & + sleep $TIMEOUT > /dev/null 2>&1 & SLEEP_PID=$! - trap "monitor_trap $SLEEP_PID $V" SIGTERM + trap "monitor_trap $SLEEP_PID $V > /dev/null 2>&1" SIGTERM wait $SLEEP_PID - [ $? != 0 ] && echo "monitor($V) cancelled" && return 0 + [ $? != 0 ] && verbose "monitor($V) cancelled" && return 0 + echo "monitor($V): killing test process..." /bin/kill -TERM $PID KILLCODE=$? if [ $KILLCODE == 0 ]; then @@ -553,14 +562,13 @@ sleep 1 MSG="nightly.sh: monitor($V): killed: exceeded $TIMEOUT seconds" echo "$MSG" >> $OUTPUT - trap + # trap return 1 } monitor_trap() { SLEEP_PID=$1 V=$2 - echo "monitor_trap($V)" /bin/kill -9 $SLEEP_PID } @@ -590,7 +598,7 @@ # If EXITCODE != 0, monitor() may have work to do (( $EXITCODE != 0 )) && sleep 5 - echo "Killing monitor..." + verbose "Killing monitor..." /bin/kill -TERM $MONITOR_PID echo "TOOK: $(( STOP-START ))" @@ -706,9 +714,11 @@ if [ -f $TEMPLATE ]; then sed "s at _WORK_@$PWD/work@;s at _HOST_@$GLOBUS_HOSTNAME@" < $TEMPLATE > sites.xml [ $? != 0 ] && crash "Could not create sites.xml!" + echo "Using: $GROUP/sites.template.xml" else sed "s at _WORK_@$PWD/work@" < $TESTDIR/sites/localhost.xml > sites.xml [ $? != 0 ] && crash "Could not create sites.xml!" + echo "Using: $TESTDIR/sites/localhost.xml" fi } @@ -716,6 +726,7 @@ if [ -f $GROUP/tc.template.data ]; then sed "s at _DIR_@$GROUP@" < $GROUP/tc.template.data > tc.data [ $? != 0 ] && crash "Could not create tc.data!" + echo "Using: $GROUP/tc.template.data" else cp -v $SWIFT_HOME/etc/tc.data . [ $? != 0 ] && crash "Could not copy tc.data!" @@ -726,6 +737,7 @@ if [ -f $GROUP/fs.template.data ]; then sed "s at _PWD_@$PWD@" < $GROUP/fs.template.data > fs.data [ $? != 0 ] && crash "Could not create fs.data!" + echo "Using: $GROUP/fs.template.data" else rm -f fs.data fi From noreply at svn.ci.uchicago.edu Wed Sep 29 12:05:12 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 29 Sep 2010 12:05:12 -0500 (CDT) Subject: [Swift-commit] r3650 - trunk/tests/functions Message-ID: <20100929170512.ED8AB9CC9D@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-09-29 12:05:12 -0500 (Wed, 29 Sep 2010) New Revision: 3650 Added: trunk/tests/functions/201-sprintf-k.check.sh trunk/tests/functions/201-sprintf-k.swift trunk/tests/functions/201-sprintf-k.timeout trunk/tests/functions/202-sprintf-k-array.check.sh trunk/tests/functions/202-sprintf-k-array.swift trunk/tests/functions/202-sprintf-k-array.timeout trunk/tests/functions/swift.properties trunk/tests/functions/tc.template.data Log: Tests for properties of %k in @sprintf() Added: trunk/tests/functions/201-sprintf-k.check.sh =================================================================== --- trunk/tests/functions/201-sprintf-k.check.sh (rev 0) +++ trunk/tests/functions/201-sprintf-k.check.sh 2010-09-29 17:05:12 UTC (rev 3650) @@ -0,0 +1,14 @@ +#!/bin/bash + +set -x + +OUTPUT=$( cat stdout.txt ) +[[ $? == 0 ]] || exit 1 +echo $OUTPUT grep "ready.*delayed" +[[ $? == 0 ]] || exit 1 + +OUTPUT=$( grep delayed: stdout.txt | cut -d ' ' -f 2 ) +[[ $? == 0 ]] || exit 1 +[[ ${OUTPUT[@]} == "4 6 8 10 12" ]] || exit 1 + +exit 0 Property changes on: trunk/tests/functions/201-sprintf-k.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/functions/201-sprintf-k.swift =================================================================== --- trunk/tests/functions/201-sprintf-k.swift (rev 0) +++ trunk/tests/functions/201-sprintf-k.swift 2010-09-29 17:05:12 UTC (rev 3650) @@ -0,0 +1,21 @@ + +app (external e) sleep (int i) +{ + sleep i; +} + +(string t) mkstring (string s, int i, external e) +{ + t = @sprintf("%s: %i%k", s, i, e); +} + +foreach i in [1:5] +{ + int delay = (7-i)*2; + external e1 = sleep(delay); + string s1 = mkstring("delayed", delay, e1); + tracef("%s\n", s1); +} + +string s2 = @sprintf("ready"); +tracef("%s\n", s2); Added: trunk/tests/functions/201-sprintf-k.timeout =================================================================== --- trunk/tests/functions/201-sprintf-k.timeout (rev 0) +++ trunk/tests/functions/201-sprintf-k.timeout 2010-09-29 17:05:12 UTC (rev 3650) @@ -0,0 +1 @@ +60 Added: trunk/tests/functions/202-sprintf-k-array.check.sh =================================================================== --- trunk/tests/functions/202-sprintf-k-array.check.sh (rev 0) +++ trunk/tests/functions/202-sprintf-k-array.check.sh 2010-09-29 17:05:12 UTC (rev 3650) @@ -0,0 +1,21 @@ +#!/bin/bash + +set -x + +# Grab partial output +OUTPUT=$( grep delayed: stdout.txt | head -4 | cut -d ' ' -f 2 ) +[[ $? == 0 ]] || exit 1 + +# NOTE: we cannot guarantee that the "12" is before "array" + +# First four outputs are in order +[[ ${OUTPUT[@]} == "4 6 8 10" ]] || exit 1 + +# Grab whole output +OUTPUT=$( grep delayed: stdout.txt | cut -d ' ' -f 2 ) + +# Output "10" is before "array" +echo ${OUTPUT[@]} | grep "10.*array" +[[ $? == 0 ]] || exit 1 + +exit 0 Property changes on: trunk/tests/functions/202-sprintf-k-array.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/functions/202-sprintf-k-array.swift =================================================================== --- trunk/tests/functions/202-sprintf-k-array.swift (rev 0) +++ trunk/tests/functions/202-sprintf-k-array.swift 2010-09-29 17:05:12 UTC (rev 3650) @@ -0,0 +1,26 @@ + +app (external e) sleep (int i) +{ + sleep i; +} + +(string t) mkstring (string s, int i, external e) +{ + t = @sprintf("%s: %i%k", s, i, e); +} + +string a[]; + +string s1 = @sprintf("delayed: array%k", a); +tracef("%s\n", s1); + +foreach i in [1:5] +{ + int delay = (7-i)*2; + external e1 = sleep(delay); + a[i] = mkstring("delayed", delay, e1); + tracef("%s\n", a[i]); +} + +string s2 = @sprintf("ready"); +tracef("%s\n", s2); Added: trunk/tests/functions/202-sprintf-k-array.timeout =================================================================== --- trunk/tests/functions/202-sprintf-k-array.timeout (rev 0) +++ trunk/tests/functions/202-sprintf-k-array.timeout 2010-09-29 17:05:12 UTC (rev 3650) @@ -0,0 +1 @@ +60 Added: trunk/tests/functions/swift.properties =================================================================== --- trunk/tests/functions/swift.properties (rev 0) +++ trunk/tests/functions/swift.properties 2010-09-29 17:05:12 UTC (rev 3650) @@ -0,0 +1,353 @@ +sites.file=${swift.home}/etc/sites.xml +tc.file=${swift.home}/etc/tc.data + +# +# The host name of the submit machine is used by GRAM as a callback +# address to report the status of submitted jobs. In general, Swift +# can automatically detect the host name of the local machine. +# However, if the machine host name is improperly configured or if +# it does not represent a valid DNS entry, certain services (such as +# GRAM) will not be able to send job status notifications back to +# the client. The value of this property can be an IP address. +# +# Format: +# hostname=string +# + + +#hostname=localhost + +# +# A TCP port range can be specified to restrict the ports on which GRAM +# callback services are started. This is likely needed if your submit +# host is behind a firewall, in which case the firewall should be +# configured to allow incoming connections on ports in the range. +# +# Format: +# tcp.port.range=start,end +# + +#tcp.port.range=50000,50100 + +# +# false - means an error will be immediately reported and cause the +# workflow to abort. At this time remote jobs that are already +# running will not be canceled +# true - means that Swift will try to do as much work as possible and +# report all errors encountered at the end. However, "errors" +# here only applies to job execution errors. Certain errors +# that are related to the Swift implementation (should such +# errors occur) will still be reported eagerly. +# +# Default: false +# +lazy.errors=false + +# +# What algorithm to use for caching of remote files. LRU (as in what +# files to purge) is the only implementation right now. One can set +# a target size (in bytes) for a host by using the swift:storagesize +# profile for a host in sites.xml +# +# Default: LRU +# +caching.algorithm=LRU + +# +# true - generate a provenance graph in .dot format (Swift will +# choose a random file name) +# false - do not generate a provenance graph +# - generate a provenange graph in the give file name +# +# Default: false +# +pgraph=false + + +# +# graph properties for the provenance graph (.dot specific) +# +# Default: splines="compound", rankdir="TB" +# +pgraph.graph.options=splines="compound", rankdir="TB" + + +# +# node properties for the provenance graph (.dot specific) +# +# Default: color="seagreen", style="filled" +# +pgraph.node.options=color="seagreen", style="filled" + +# +# true - clustering of small jobs is enabled. Clustering works in the +# following way: If a job is clusterable (meaning that it has the +# GLOBUS::maxwalltime profile specified in tc.data and its value +# is less than the value of the "clustering.min.time" property) it will +# be put in a clustering queue. The queue is processed at intervals +# specified by the "clustering.queue.delay" property. The processing +# of the clustering queue consists of selecting compatible jobs and +# grouping them in clusters whose max wall time does not exceed twice +# the value of the "clustering.min.time" property. Two or more jobs are +# considered compatible if they share the same site and do not have +# conflicting profiles (e.g. different values for the same environment +# variable). +# false - clustering of small jobs is disabled. +# +# Default: false +# +clustering.enabled=false + + +# +# - the intervals at which the clustering queue is processed +# +# Default: 4 +# +clustering.queue.delay=4 + +# +# - the threshold time for clustering +# +# Default: 60 +# +clustering.min.time=60 + +# +# Kickstart is a useful tool that can be used to gather various information +# about a remote process. Before it can be used it must be installed on the +# remote site and the corresponding entry be set in the sites file. +# This option allows controlling of how Swift uses Kickstart. The following +# values are possible: +# false - do not use Kickstart +# true - use Kickstart. If a job is scheduled on a site that does not have +# Kickstart installed, that job will fail. +# maybe - Use Kickstart if installed (i.e. the entry is present in the sites +# file) +# +# Default: maybe +# + +kickstart.enabled=maybe + +# +# Indicates when Kickstart records should be fetched from the remote site: +# true - always transfer Kickstart records if Kickstart was used (see +# kickstart.enabled) +# false - only transfer Kickstart records if the job fails +# +# Default: false +# + +kickstart.always.transfer=false + +# +# Indicates when wrapper logs should be fetched from the remote site: +# true - always transfer wrapper logs +# false - only transfer wrapper logs if the job fails +# +# Default: false +# + +wrapperlog.always.transfer=false + +########################################################################### +# Throttling options # +########################################################################### +# +# For the throttling parameters, valid values are either a positive integer +# or "off" (without the quotes). +# + +# +# Limits the number of concurrent submissions for a workflow instance. This +# throttle only limits the number of concurrent tasks (jobs) that are being +# sent to sites, not the total number of concurrent jobs that can be run. +# The submission stage in GRAM is one of the most CPU expensive stages (due +# mostly to the mutual authentication and delegation). Having too many +# concurrent submissions can overload either or both the submit host CPU +# and the remote host/head node causing degraded performance. +# +# Default: 4 +# + +throttle.submit=4 +#throttle.submit=off + +# +# Limits the number of concurrent submissions for any of the sites Swift will +# try to send jobs to. In other words it guarantees that no more than the +# value of this throttle jobs sent to any site will be concurrently in a state +# of being submitted. +# +# Default: 2 +# + +throttle.host.submit=100 +#throttle.host.submit=off + +# +# The Swift scheduler has the ability to limit the number of concurrent jobs +# allowed on a site based on the performance history of that site. Each site +# is assigned a score (initially 1), which can increase or decrease based +# on whether the site yields successful or faulty job runs. The score for a +# site can take values in the (0.1, 100) interval. The number of allowed jobs +# is calculated using the following formula: +# 2 + score*throttle.score.job.factor +# This means a site will always be allowed at least two concurrent jobs and +# at most 2 + 100*throttle.score.job.factor. With a default of 4 this means +# at least 2 jobs and at most 402. +# +# Default: 4 +# + +throttle.score.job.factor=4 +#throttle.score.job.factor=off + + +# +# Limits the total number of concurrent file transfers that can happen at any +# given time. File transfers consume bandwidth. Too many concurrent transfers +# can cause the network to be overloaded preventing various other signalling +# traffic from flowing properly. +# +# Default: 4 +# + +throttle.transfers=4 +#throttle.transfers=off + +# Limits the total number of concurrent file operations that can happen at any +# given time. File operations (like transfers) require an exclusive connection +# to a site. These connections can be expensive to establish. A large number +# of concurrent file operations may cause Swift to attempt to establish many +# such expensive connections to various sites. Limiting the number of concurrent +# file operations causes Swift to use a small number of cached connections and +# achieve better overall performance. +# +# Default: 8 +# + +throttle.file.operations=8 +#throttle.file.operations=off + +# Indicates whether the working directory on the remote site should be +# left intact even when the workflow completes successfully. This can be +# used to inspect the site working directory for debugging purposes. +# +# Default: false +# + +sitedir.keep=false + +# number of time a job will be retried if it fails (giving a maximum of +# 1 + execution.retries attempts at execution) +# + +execution.retries=2 + + +# Enables/disables replication. Replication is used to deal with jobs sitting +# in batch queues for abnormally large amounts of time. If replication is enabled +# and certain conditions are met, Swift creates and submits replicas of jobs, and +# allows multiple instances of a job to compete. +# + +replication.enabled=false + +# If replication is enabled, this value specifies the minimum time, in seconds, +# a job needs to be queued in a batch queue in order to be considered for +# replication +# + +replication.min.queue.time=60 + +# The maximum number of replicas that Swift should attempt. + +replication.limit=3 + +# +# WARNING: This option is deprecated. Please use the hostname option. +# +# The IP address of the submit machine is used by GRAM as a callback +# address to report the status of submitted jobs. In general, Swift +# can automatically detect the IP address of the local machine. +# However, if the machine has more than one network interface, Swift +# will pick the first one, which may not be the right choice. It is +# recommended that this property is set properly before attempting to +# run jobs through GRAM. +# +# Format: +# ip.address=x.y.z.w +# + +#ip.address=127.0.0.1 + + +# Controls how Swift will communicate the result code of running user programs +# from workers to the submit side. In files mode, a file +# indicating success or failure will be created on the site shared filesystem. +# In provider mode, the execution provider job status will +# be used. Notably, GRAM2 does not return job statuses correctly, and so +# provider mode will not work with GRAM2. With other +# providers, it can be used to reduce the amount of filesystem access compared +# to files mode. +# +# status.mode=files + +# Controls how swift will supply parameters to the remote wrapper script. +# 'args' mode will pass parameters on the command line +# 'files' mode will pass parameters through an additional input file +# +# valid values: args, files +# Default: files +# +# wrapper.parameter.mode=args + +# Determines if Swift remote wrappers will be executed by specifying an +# absolute path, or a path relative to the job initial working directory +# +# valid values: absolute, relative +# wrapper.invocation.mode=absolute + +# +# Limits the number of concurrent iterations that each foreach statement +# can have at one time. This conserves memory for swift programs that +# have large numbers of iterations (which would otherwise all be executed +# in parallel). +# +# Default: 1024 +# + +foreach.max.threads=16384 + +# controls whether the log file will contain provenance information +# enabling this will increase the size of log files, sometimes +# significantly. + +provenance.log=false + +# Controls whether file staging is done by swift or by the execution +# provider. If set to false, the standard swift staging mechanism is +# used. If set to true, swift does not stage files. Instead, the +# execution provider is instructed to stage files in and out. +# +# Provider staging is experimental. +# +# When enabled, and when coasters are used as an execution provider, +# a staging mechanism can be selected for each site +# using the swift:stagingMethod site profile in sites.xml. The +# following is a list of accepted mechanisms: +# +# * file: Staging is done from a filesystem accessible to the +# coaster service (typically running on the head node) +# * proxy: Staging is done from a filesystem accessible to the +# client machine that swift is running on, and is proxied +# through the coaster service +# * sfs: (short for "shared filesystem") Staging is done by +# copying files to and from a filesystem accessible +# by the compute node (such as an NFS or GPFS mount). + + +use.provider.staging=false +provider.staging.pin.swiftfiles=false Added: trunk/tests/functions/tc.template.data =================================================================== --- trunk/tests/functions/tc.template.data (rev 0) +++ trunk/tests/functions/tc.template.data 2010-09-29 17:05:12 UTC (rev 3650) @@ -0,0 +1,9 @@ +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null +localhost cp /bin/cp INSTALLED INTEL32::LINUX null +localhost sleep /bin/sleep INSTALLED INTEL32::LINUX null From noreply at svn.ci.uchicago.edu Thu Sep 30 15:54:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 30 Sep 2010 15:54:25 -0500 (CDT) Subject: [Swift-commit] r3651 - SwiftApps/Montage/scripts Message-ID: <20100930205425.59D819CC8F@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-09-30 15:54:24 -0500 (Thu, 30 Sep 2010) New Revision: 3651 Added: SwiftApps/Montage/scripts/mDiff_wrap.sh SwiftApps/Montage/scripts/mFitplane_wrap.sh Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift SwiftApps/Montage/scripts/Swift_Montage_Batch.swift SwiftApps/Montage/scripts/Swift_Montage_Types.swift Log: o Montage/scripts/Swift_Montage_Apps.swift o Montage/scripts/Swift_Montage_Batch.swift o Montage/scripts/Swift_Montage_Types.swift -- Removed all tracefs from code. Seemed to exacerbate hanging problem that is being encountered due to low Java heap memory. o Montage/scripts/mFitplane_wrap.sh o Montage/scripts/mDiff_wrap.sh -- Wrote wrapper script to always return exit code 0. If these Montage functions encounter a problem them return exit code 1 which in turn causes Swift to fail. I do not need Swift to fail, merely continue on. Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-09-29 17:05:12 UTC (rev 3650) +++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-09-30 20:54:24 UTC (rev 3651) @@ -1,157 +1,83 @@ -/*! \fn app Image mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) - \brief Adds all images in imgs[] into one image file based off the information in img_tbl and hdr. - - \param imgs[] List of images to be added into one image. All images but reside in same directory. - \param img_tbl Table containing information about images being added - \param hdr MosaicData file containing information about how mosaic will look. - \return mos Image file containing data about mosaic. -*/ app ( Image mos ) mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) { mAdd "-p" @dirname( imgs[0] ) "-n" @img_tbl @hdr @mos; } -/*! \fn app Image mBackground( Image img, Table img_tbl, Table corr_tbl ) - \brief Generates a corrected image from imgs and the information from corr_tbl - - \param img Image to be corrected - \param img_tbl Table containing information about images being added - \param corr_tbl Correction table containing correction coefficients to be applied - \return mos bg_img corrected image -*/ app ( Image bg_img ) mBackground( Image img, float a, float b, float c ) { mBackground "-n" @img @bg_img a b c; } -/*! \fn app Table mBgModel( Table img_tbl, Table fits_tbl ) - \brief Generates the corrected image table from the coefficeints data from fits_tbl and data from the projected image table - - \param img_tbl Table containing information about the projected images. - \param fits_tbl Table containing information about how to correct an image - \return corr_tbl Correction table containing correction coefficients to be applied -*/ app ( Table corr_tbl ) mBgModel( Table img_tbl, Table fits_tbl ) { mBgModel @img_tbl @fits_tbl @corr_tbl; } -/*! \fn app Table mConcatFit( Table status_tbl, Status stats[] ) - \brief Takes all the status files created by mFitplane and creates the fits table. - - \param status_tbl Table containing the file names for the status files - \param stats[] Array of all the status files - \return fits_tbl Table contianing information about how to correct and image. -*/ app ( Table fits_tbl ) mConcatFit( Table status_tbl, Status stats[] ) { mConcatFit @status_tbl @fits_tbl @dirname( stats[0] ); } -/*! \fn app Image mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) - \brief Calculates the difference image from the two projected images and the header file. - - \param proj_img_1 The first image to calculate how much is overlapped - \param proj_img_2 The second image to calculate how much is overlapped - \param hdr MosaicData file that tells how the mosaic is set up - \return diff_img Difference image that contains overlapped information -*/ app ( Image diff_img ) mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) { - mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; +// mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; + mDiff_wrap "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; } -/*! \fn app Status mFitplane( Image diff_img ) - \brief Creates a file with all the information on how to fit the particular difference image - - \param diff_img The image to calculate best fit for. - \return stat File containing coefficents and mosaic data on how to fit diff_img. -*/ app ( Status stat ) mFitplane( Image diff_img ) { - mFitplane "-s" @stat @diff_img; +// mFitplane "-s" @stat @diff_img; + mFitplane_wrap "-s" @stat @diff_img; } -/*! \fn app Table mImgtbl( Image imgs[] ) - \brief Take a list of of images and create a table out of them. All images in the list must be located in the same directory. - - \param imgs[] List of files to create an table out of. - \return img_tbl Table of information of the list of images. -*/ app ( Table img_tbl ) mImgtbl( Image imgs[] ) { mImgtbl @dirname( imgs[0] ) @img_tbl; } -/*! \fn app JPEG mJPEG( Image mos_img ) - \brief Creates a viewable jpeg image from the mosaic data image - - \param mos_img Image file to create a jpeg out of. - \return mos_img_jpg Jpeg picture. -*/ app ( JPEG mos_img_jpg ) mJPEG( Image mos_img ) { mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg; } -/*! \fn app Table mOverlaps( Table img_tbl ) - \brief Calculates which images overlaps each other and creates a table out of. - - \param img_tbl Table of projected images. - \return diff_tbl Table of which overlapped images -*/ app ( Table diff_tbl ) mOverlaps( Table img_tbl ) { mOverlaps @img_tbl @diff_tbl; } -/*! \fn app Image mProjectPP( Image raw_img, MosaicData hdr ) - \brief Projects an image based off of the header file. - - \param raw_img Raw image to project. - \param hdr MosaicData file telling how the mosaic will be set up in the end. - \return proj_img Projected image. -*/ app ( Image proj_img ) mProjectPP( Image raw_img, MosaicData hdr ) { mProjectPP "-X" @raw_img @proj_img @hdr; } +app ( Image proj_img ) mProject( Image raw_img, MosaicData hdr ) +{ + mProject "-X" @raw_img @proj_img @hdr; +} + + /*----------------------------------------------------------------------------------*/ /* */ /* Util Scripts */ /* */ /*----------------------------------------------------------------------------------*/ -/*! \fn app Table Background_list( Table imgs_tbl, Table corrs_tbl - \brief Generates a list of coefficients that need to be applied and the files to apply them to. - - \param imgs_tbl Table generated by mImgtbl containing information on the set of images - \param corrs_tbl Table containing the coefficients and the id of the image to apply them to - \return back_tbl Table containing the Image file name and the coefficients that need to be applied to the image. Data is set up so readDatat2 can read the data. -*/ app ( Table back_tbl ) Background_list( Table imgs_tbl, Table corrs_tbl ) { Background_list @imgs_tbl @corrs_tbl @back_tbl; } -/*! \fn app Table create_status_table( Table diff_tbl ) - \brief Generates a Table of the status files created by mFitplane - - \param diff_tbl Difference table of the files that mFitplane will run on. - \return stat_tbl Table that contains the information of the status files that will be generated using mFitplane. -*/ app ( Table stat_tbl ) create_status_table( Table diff_tbl ) { create_status_table @diff_tbl @stat_tbl; Modified: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2010-09-29 17:05:12 UTC (rev 3650) +++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2010-09-30 20:54:24 UTC (rev 3651) @@ -1,19 +1,3 @@ -/*! \file Swift_Montage_Batch.swift - \brief File containing Batch functions for the Swift Montage wrappers - - Each batch function allows basic Montage functions to be called on a list of - images. -*/ - - -/*! \fn Image mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) - \brief Runs the mBackground wrapper on a list of images in parallel. - - \param bg_imgs[] The list of images to run the mBackground wrapper on. - \param img_tbl Table containing information on original images. - \param corr_tbl Table containing correction coefficients to be applied to the images. - \return corr_imgs[] List of the corrected images to be added to create image mosaic. -*/ ( Image corr_imgs[] ) mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) { Table tmp <"back_tmp.tbl">; @@ -36,16 +20,9 @@ } -/*! \fn Image mDiffBatch( Table diff_tbl, MosaicData hdr ) - \brief Runs the mDiff wrapper on a list of images in parallel. - - \param diff_tbl Table containing a list of images that overlap and must calculate by how much they overlap. - \param hdr File containing information on how the final mosaic will be set up. - \return diff_imgs[] List of the difference images that must have correction coefficients calculated. -*/ ( Image diff_imgs[] ) mDiffBatch( Table diff_tbl, MosaicData hdr ) { - DiffStruct diffs[] ; + DiffStruct diffs[] ; foreach d_entry, i in diffs { @@ -57,18 +34,12 @@ Image diff_img ; diff_img = mDiff( proj_1, proj_2, hdr ); + diff_imgs[ i ] = diff_img; } } -/*! \fn app Table mFitBatch( Table diff_tbl, Image diff_imgs[] ) - \brief Generates a corrected image from imgs and the information from corr_tbl - - \param diff_tbl - \param diff_imgs - \return fits_tbl -*/ ( Table fits_tbl ) mFitBatch( Image diff_imgs[], Table diff_tbl ) { Status stats[] ; @@ -78,27 +49,20 @@ foreach img, i in stats { - stats[i] = mFitplane ( diff_imgs[i] ); + stats[ i ] = mFitplane ( diff_imgs[i] ); } fits_tbl = mConcatFit( status_tbl, stats ); } -/*! \fn Image mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) - \brief Projects the raw images from raw_imgs to a new plane that was declared in hdr. - - \param raw_imgs[] List of raw images that need to be projected to new plane. - \param hdr File containing information on how the final mosaic will be set up. - \return proj_imgs[] List of newly reprojected images. -*/ -( Image proj_imgs[] ) mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) +( Image proj_imgs[] ) mProjectBatch( Image raw_imgs[], MosaicData hdr ) { foreach img, i in raw_imgs { - Image proj_img ; - + Image proj_img ; proj_img = mProjectPP( img, hdr ); +// proj_img = mProject( img, hdr ); proj_imgs[ i ] = proj_img; } Modified: SwiftApps/Montage/scripts/Swift_Montage_Types.swift =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Types.swift 2010-09-29 17:05:12 UTC (rev 3650) +++ SwiftApps/Montage/scripts/Swift_Montage_Types.swift 2010-09-30 20:54:24 UTC (rev 3651) @@ -1,39 +1,9 @@ -/*! \file Swift_Montage_Types.swift - \brief File containing declared Swift types for the Swift Montage wrappers -*/ - -/*! \typdef type Image - \brief Declared Image Swift type. -*/ type Image; - - -/*! \typdef type MosaicData - \brief Declared MosaicData Swift type. -*/ type MosaicData; - - -/*! \typdef type Table - \brief Declared Table Swift type. -*/ type Table; - - -/*! \typdef type JPEG - \brief Declared JPEG Swift type. -*/ type JPEG; - -/*! \typdef type Status - \brief Declared Status Swift type. -*/ type Status; - -/*! \struct BackgroundStruct - \brief Stuct to hold data found in the tables generated my mImgtbl and mBgModel -*/ type BackgroundStruct { string fname; @@ -43,9 +13,6 @@ }; -/*! \struct DiffStruct - \brief Stuct to hold data found in the table that is generated from the mOverlaps wrapper. -*/ type DiffStruct { int cntr1; Added: SwiftApps/Montage/scripts/mDiff_wrap.sh =================================================================== --- SwiftApps/Montage/scripts/mDiff_wrap.sh (rev 0) +++ SwiftApps/Montage/scripts/mDiff_wrap.sh 2010-09-30 20:54:24 UTC (rev 3651) @@ -0,0 +1,7 @@ +#!/bin/bash + +mDiff $1 $2 $3 $4 $5 +if [ $? == 1 ]; then + touch $4 +fi +exit 0 Property changes on: SwiftApps/Montage/scripts/mDiff_wrap.sh ___________________________________________________________________ Name: svn:executable + * Added: SwiftApps/Montage/scripts/mFitplane_wrap.sh =================================================================== --- SwiftApps/Montage/scripts/mFitplane_wrap.sh (rev 0) +++ SwiftApps/Montage/scripts/mFitplane_wrap.sh 2010-09-30 20:54:24 UTC (rev 3651) @@ -0,0 +1,7 @@ +#!/bin/bash + +mFitplane $1 $2 $3 +if [ $? == 1 ]; then + touch $2 +fi +exit 0 Property changes on: SwiftApps/Montage/scripts/mFitplane_wrap.sh ___________________________________________________________________ Name: svn:executable + *