From noreply at svn.ci.uchicago.edu Thu Jul 1 14:31:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 1 Jul 2010 14:31:24 -0500 (CDT) Subject: [Swift-commit] r3422 - trunk Message-ID: <20100701193124.B63E99CC94@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-01 14:31:24 -0500 (Thu, 01 Jul 2010) New Revision: 3422 Modified: trunk/.project Log: Remove bad builders Modified: trunk/.project =================================================================== --- trunk/.project 2010-06-30 22:33:17 UTC (rev 3421) +++ trunk/.project 2010-07-01 19:31:24 UTC (rev 3422) @@ -1,34 +1,14 @@ - - - swift - - - abstraction-common - karajan - - - - org.eclipse.ui.externaltools.ExternalToolBuilder - full,incremental, - - - LaunchConfigHandle - <project>/.externalToolBuilders/org.eclipse.jdt.core.javabuilder.launch - - - - - org.eclipse.ui.externaltools.ExternalToolBuilder - full,incremental, - - - LaunchConfigHandle - <project>/.externalToolBuilders/Swift_Builder.launch - - - - - - org.eclipse.jdt.core.javanature - - + + + swift + + + abstraction-common + karajan + + + + + org.eclipse.jdt.core.javanature + + From noreply at svn.ci.uchicago.edu Thu Jul 1 15:31:40 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 1 Jul 2010 15:31:40 -0500 (CDT) Subject: [Swift-commit] r3423 - trunk/libexec Message-ID: <20100701203140.4B8939CC94@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-01 15:31:40 -0500 (Thu, 01 Jul 2010) New Revision: 3423 Modified: trunk/libexec/vdl-int-staging.k trunk/libexec/vdl-int.k Log: Update .k files: dirname() now needs namespace qualifier Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-07-01 19:31:24 UTC (rev 3422) +++ trunk/libexec/vdl-int-staging.k 2010-07-01 20:31:40 UTC (rev 3423) @@ -64,7 +64,7 @@ file := vdl:filename(vdl:getfield(var, path = path)) - dirname(file) + vdl:dirname(file) ) ) Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-07-01 19:31:24 UTC (rev 3422) +++ trunk/libexec/vdl-int.k 2010-07-01 20:31:40 UTC (rev 3423) @@ -148,7 +148,7 @@ file := vdl:filename(vdl:getfield(var, path = path)) - dirname(file) + vdl:dirname(file) ) ) @@ -284,7 +284,7 @@ uParallelFor(file, files provider := provider(file) srchost := hostname(file) - srcdir := dirname(file) + srcdir := vdl:dirname(file) destdir := dircat(dir, reldirname(file)) filename := basename(file) size := file:size("{srcdir}/{filename}", host=srchost, provider=provider) @@ -331,7 +331,7 @@ dhost := vdl:hostname(file) rdir := dircat(dir, reldirname(file)) bname := basename(file) - ldir := dirname(file) + ldir := vdl:dirname(file) fullLocal := dircat(ldir, bname) fullRemote := dircat(rdir, bname) From noreply at svn.ci.uchicago.edu Fri Jul 2 10:57:22 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 2 Jul 2010 10:57:22 -0500 (CDT) Subject: [Swift-commit] r3425 - trunk/libexec Message-ID: <20100702155722.06BC79CCBD@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-02 10:57:21 -0500 (Fri, 02 Jul 2010) New Revision: 3425 Modified: trunk/libexec/vdl-int-staging.k trunk/libexec/vdl-int.k Log: More dirname fixes Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-07-02 04:21:03 UTC (rev 3424) +++ trunk/libexec/vdl-int-staging.k 2010-07-02 15:57:21 UTC (rev 3425) @@ -97,7 +97,7 @@ protocol := vdl:provider(file) provider := if(protocol == "file", stagingMethod, protocol) srchost := hostname(file) - srcdir := dirname(file) + srcdir := vdl:dirname(file) destdir := dircat(dir, reldirname(file)) filename := basename(file) @@ -117,7 +117,7 @@ dhost := vdl:hostname(file) rdir := dircat(dir, reldirname(file)) bname := basename(file) - ldir := dirname(file) + ldir := vdl:dirname(file) fullLocal := dircat(ldir, bname) fullRemote := dircat(rdir, bname) Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-07-02 04:21:03 UTC (rev 3424) +++ trunk/libexec/vdl-int.k 2010-07-02 15:57:21 UTC (rev 3425) @@ -250,7 +250,7 @@ log(LOG:INFO, "START jobid={jobid} - staging in wrapper params") provider := provider(wrapfile) srchost := hostname(wrapfile) - srcdir := dirname(wrapfile) + srcdir := vdl:dirname(wrapfile) destdir := dircat(dir, "/parameters/{jobdir}/") filename := basename(wrapfile) @@ -271,7 +271,7 @@ if (cdmfile != "" then( doStageinFile(provider="file", srchost="localhost", srcfile=basename(cdmfile), - srcdir=dirname(cdmfile), desthost=host, destdir=dir, + srcdir=vdl:dirname(cdmfile), desthost=host, destdir=dir, size=file:size(cdmfile), policy="DEFAULT") doStageinFile(provider="file", srchost="localhost", srcfile="cdm.pl", srcdir=libexec, desthost=host, destdir=dir, From noreply at svn.ci.uchicago.edu Mon Jul 5 03:31:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 5 Jul 2010 03:31:55 -0500 (CDT) Subject: [Swift-commit] r3426 - in usertools/swift/swiftconfig: bin etc Message-ID: <20100705083155.1CFA39CC90@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-05 03:31:54 -0500 (Mon, 05 Jul 2010) New Revision: 3426 Modified: usertools/swift/swiftconfig/bin/swiftconfig usertools/swift/swiftconfig/etc/sites-template.xml Log: /home/david replacement in templates, site profiles, tc command removals Modified: usertools/swift/swiftconfig/bin/swiftconfig =================================================================== --- usertools/swift/swiftconfig/bin/swiftconfig 2010-07-02 15:57:21 UTC (rev 3425) +++ usertools/swift/swiftconfig/bin/swiftconfig 2010-07-05 08:31:54 UTC (rev 3426) @@ -1,223 +1,255 @@ #!/usr/bin/perl # Find swift configuration files and libraries -if($ENV{'SWIFT_HOME'}) -{ - $swifthome = $ENV{'SWIFT_HOME'}; +if ( $ENV{'SWIFT_HOME'} ) { + $swifthome = $ENV{'SWIFT_HOME'}; } -else -{ - @path = split('/', which("swift")); - @path = splice(@path, 0, $#path-1); - $swifthome = join('/', @path); +else { + @path = split( '/', which("swift") ); + @path = splice( @path, 0, $#path - 1 ); + $swifthome = join( '/', @path ); } -use FindBin; -use lib "$FindBin::Bin/../lib/perl"; # Use libraries in $swifthome/lib/perl +use FindBin; +use lib "$FindBin::Bin/../lib/perl"; # Use libraries in $swifthome/lib/perl use Getopt::Long; use File::Which qw(which where); use XML::Simple; - # Variables used for translation catalog -my $host = 'localhost'; # Host name -my $name = ''; # Translation name -my $path = 'null'; # Full path of executable location -my $status = 'INSTALLED'; # Installation status (not used) -my $platform = 'INTEL32::LINUX'; # Platform (not used) -my $profile = 'null'; # Profile entries -my $tcfile = "$swifthome/etc/tc.data"; # Location of TC file +my $host = 'localhost'; # Host name +my $name = ''; # Translation name +my $path = "/usr/bin/$name"; # Full path of executable location +my $status = 'INSTALLED'; # Installation status (not used) +my $platform = 'INTEL32::LINUX'; # Platform (not used) +my $profile = 'null'; # Profile entries +my $tcfile = "$swifthome/etc/tc.data"; # Location of TC file # Variables used for sites.xml -my $templatefile = "$swifthome/etc/sites-template.xml"; # Template file -my $template = ''; # Use template entry -my $templates = 0; # List all templates -my $sitesfile = "$swifthome/etc/sites.xml"; # Working sites.xml file -my $entry = ''; # Name of new entry (pool handle) -my $gridftp = ''; # Specify gridftp URL -my $jobuniverse = ''; # Specify jobmanager universe -my $joburl = ''; # Specify jobmanager URL -my $jobmajor = ''; # Jobmanager major value -my $jobminor = ''; # Jobmanager minor value -my $directory = ''; # Work directory -my $exprovider = ''; # Execution provider -my $exmanager = ''; # Execution job manager -my $exurl = ''; # Execution URL -my $remove = ''; # Remove entry from sites.xml -my $modify = ''; # Modify mode +my $templatefile = "$swifthome/etc/sites-template.xml"; # Template file +my $template = ''; # Use template entry +my $templates = 0; # List all templates +my $sitesfile = "$swifthome/etc/sites.xml"; # Working sites.xml file +my $entry = ''; # Name of new entry (pool handle) +my $gridftp = ''; # Specify gridftp URL +my $jobuniverse = ''; # Specify jobmanager universe +my $joburl = ''; # Specify jobmanager URL +my $jobmajor = ''; # Jobmanager major value +my $jobminor = ''; # Jobmanager minor value +my $directory = ''; # Work directory +my $exprovider = ''; # Execution provider +my $exmanager = ''; # Execution job manager +my $exurl = ''; # Execution URL +my $remove = ''; # Remove entry from sites.xml +my $modify = ''; # Modify mode # Assign variables from command line options GetOptions( - 'host=s' => \$host, - 'name=s' => \$name, - 'path=s' => \$path, - 'status=s' => \$status, - 'platform=s' => \$platform, - 'profile=s' => \$profile, - 'tcfile=s' => \$tcfile, - 'overwrite' => \$overwrite, - 'templatefile=s' => \$templatefile, - 'add=s' => \$add, - 'sitesfile=s' => \$sitesfile, - 'gridftp=s' => \$gridftp, - 'jobuniverse=s' => \$jobuniverse, - 'joburl=s' => \$joburl, - 'jobmajor=i' => \$jobmajor, - 'jobminor=i' => \$jobminor, - 'directory=s' => \$directory, - 'exprovider=s' => \$exprovider, - 'exmanager=s' => \$exmanager, - 'exurl=s' => \$exurl, - 'remove=s' => \$remove, - 'templates' => \$templates, - 'modify=s' => \$modify + 'host=s' => \$host, + 'name=s' => \$name, + 'path=s' => \$path, + 'status=s' => \$status, + 'platform=s' => \$platform, + 'profile=s' => \$profile, + 'tcfile=s' => \$tcfile, + 'overwrite' => \$overwrite, + 'templatefile=s' => \$templatefile, + 'add=s' => \$add, + 'sitesfile=s' => \$sitesfile, + 'gridftp=s' => \$gridftp, + 'jobuniverse=s' => \$jobuniverse, + 'joburl=s' => \$joburl, + 'jobmajor=i' => \$jobmajor, + 'jobminor=i' => \$jobminor, + 'directory=s' => \$directory, + 'exprovider=s' => \$exprovider, + 'exmanager=s' => \$exmanager, + 'exurl=s' => \$exurl, + 'remove=s' => \$remove, + 'templates' => \$templates, + 'modify=s' => \$modify, + 'profile=s' => \$profile, + 'key=s' => \$key, + 'namespace=s' => \$namespace, + 'value=s' => \$value ); -$xml = new XML::Simple(); +# Prepare data +$xml = new XML::Simple(); +$template_data = $xml->XMLin( + "$templatefile", + ForceArray => [qw(workdirectory pool profile)], + KeyAttr => [] +); +$sites_data = $xml->XMLin( + "$sitesfile", + ForceArray => [qw(workdirectory pool profile)], + KeyAttr => [] +); +open( TCFILESTREAM, "$tcfile" ) or die("Unable to open tc file $tcfile!"); + at tcdata = ; +close(TCFILESTREAM); # Update XML hash with values from command line -sub update_xml_entry -{ - my $xml_data = @_[0]; - my $name = @_[1]; - foreach my $e (@{$xml_data->{pool}}) - { - if($name eq $e->{handle}) - { - if($gridftp){ $e->{gridftp}{url} = $gridftp; } - if($jobuniverse){ $e->{jobmanager}{universe} = $jobuniverse; } - if($joburl){ $e->{jobmanager}{url} = $joburl; } - if($jobmajor){ $e->{jobmanager}{major} = $jobmajor; } - if($jobminor){ $e->{jobmanager}{minor} = $jobminor; } - if($directory){ $e->{workdirectory} = [$directory]; } - if($exprovider){ $e->{execution}{provider} = $exprovider; } - if($exmanager){ $e->{execution}{jobmanager} = $exmanager; } - if($exurl){ $e->{execution}{url} = $exurl; } - return $e; - } - } +sub update_xml_entry { + my $xml_data = @_[0]; + my $name = @_[1]; + foreach my $e ( @{ $xml_data->{pool} } ) { + if ( $name eq $e->{handle} ) { + $e->{workdirectory}[0] =~ s/\$HOME/$ENV{'HOME'}/; + if ($gridftp) { $e->{gridftp}{url} = $gridftp; } + if ($jobuniverse) { $e->{jobmanager}{universe} = $jobuniverse; } + if ($joburl) { $e->{jobmanager}{url} = $joburl; } + if ($jobmajor) { $e->{jobmanager}{major} = $jobmajor; } + if ($jobminor) { $e->{jobmanager}{minor} = $jobminor; } + if ($directory) { $e->{workdirectory} = [$directory]; } + if ($exprovider) { $e->{execution}{provider} = $exprovider; } + if ($exmanager) { $e->{execution}{jobmanager} = $exmanager; } + if ($exurl) { $e->{execution}{url} = $exurl; } + + if ( $key && $namespace && $value ) { + my $found = 0; + foreach my $profile ( @{ $e->{profile} } ) { + if ( $profile->{key} eq $key ) { + $profile->{key} = $key; + $profile->{namespace} = $namespace; + $profile->{content} = $value; + $found = 1; + } + } + if ( !$found ) { + push @{ $e->{profile} }, + { + key => $key, + namespace => $namespace, + content => $value + }; + } + } + return $e; + } + } } # Write a file given variable and filename -sub write_file -{ - my ($filename, @data) = @_; - open(TEMPFILESTREAM, ">$filename") or die("Unable to open $filename!\n"); - print TEMPFILESTREAM @data; - close(TEMPFILESTREAM); +sub write_file { + my ( $filename, @data ) = @_; + open( TEMPFILESTREAM, ">$filename" ) or die("Unable to open $filename!\n"); + print TEMPFILESTREAM @data; + close(TEMPFILESTREAM); } # If a template is specified, find the correct one and modify as needed -if($add) -{ - $data = $xml->XMLin("$templatefile", ForceArray => [qw(workdirectory pool)]); - $data = update_xml_entry($data, $add); +if ($add) { + my $data = update_xml_entry( $template_data, $add ); - # Modify entry if one already exists - $isfound = 0; - $data2 = $xml->XMLin("$sitesfile", ForceArray => [qw(workdirectory pool)]); - foreach my $e (@{$data2->{pool}}) - { - if($add eq $e->{handle}) - { - $e = $data; - $isfound = 1; - } - } + # Modify existing entry + my $isfound = 0; + foreach my $e ( @{ $sites_data->{pool} } ) { + if ( $add eq $e->{handle} ) { + $e = $data; + $isfound = 1; + } + } - # Add a new entry if needed - if(!$isfound) - { - push @{$data2->{pool}}, $data; - } + # Add new entry + if ( !$isfound ) { + push @{ $sites_data->{pool} }, $data; + } - # Write to sites file - write_file($sitesfile, $xml->XMLout($data2, RootName => 'config')); + # Write to sites file + write_file( $sitesfile, $xml->XMLout( $sites_data, RootName => 'config' ) ); } # Remove an entry -if($remove) -{ - $isfound=0; - $data = $xml->XMLin("$sitesfile", ForceArray => [qw(workdirectory pool)], SuppressEmpty => 1); - - foreach my $e (@{$data->{pool}}) - { - if($remove eq $e->{handle}) - { - undef $e; - $isfound = 1; - } - } +if ($remove) { - # Write to sites file - if($isfound) - { - write_file($sitesfile, $xml->XMLout($data, RootName => 'config', SuppressEmpty => 1)); - } + # Look first for a site to remove + $isfound = 0; + + foreach my $e ( @{ $sites_data->{pool} } ) { + if ( $remove eq $e->{handle} ) { + undef $e; + $isfound = 1; + } + } + + # Write to sites file + if ($isfound) { + write_file( + $sitesfile, + $xml->XMLout( + $sites_data, + RootName => 'config', + SuppressEmpty => 1 + ) + ); + } + + # If no site found, check for a TC entry + else { + foreach $line (@tcdata) { + + # Ignore comments + if ( substr( $line, 0, 1 ) eq '#' ) { + next; + } + ( $tmphost, $tmpname, @junk ) = split( /\s+/, $line ); + + # Replace old entry with new entry + if ( $remove eq $tmpname ) { + $line = ""; + } + } + write_file( $tcfile, @tcdata ); + } } # List all available templates -if($templates) -{ - $data = $xml->XMLin("$templatefile", ForceArray => [qw(workdirectory pool)]); - foreach my $e (@{$data->{pool}}) - { - print $e->{handle} . "\n"; - } +if ($templates) { + foreach my $e ( @{ $template_data->{pool} } ) { + print $e->{handle} . "\n"; + } } -# Modify an entry -if($modify) -{ - $data = $xml->XMLin("$sitesfile", ForceArray => [qw(workdirectory pool)]); - $data2 = update_xml_entry($data, $modify); +# Modify a site entry +if ($modify) { + my $data = update_xml_entry( $sites_data, $modify ); - # Modify entry if one already exists - foreach my $e (@{$data->{pool}}) - { - if($modify eq $e->{handle}) - { - $e = $data2; - $isfound = 1; - } - } + # Modify entry if one already exists + foreach my $e ( @{ $sites_data->{pool} } ) { + if ( $modify eq $e->{handle} ) { + $e = $data; + } + } - write_file($sitesfile, $xml->XMLout($data, RootName => 'config', SuppressEmpty => 1)); + write_file( $sitesfile, + $xml->XMLout( $sites_data, RootName => 'config', SuppressEmpty => 1 ) ); } -# Add an entry to tc.data -if($name) -{ - open(TCFILESTREAM, "$tcfile") or die("Unable to open tc file $tcfile!"); - @TCDATA = ; - close(TCFILESTREAM); +# Add or modify a TC entry +if ($name) { + foreach $line (@tcdata) { - foreach $line (@TCDATA) - { - # Ignore comments - if(substr($line, 0, 1) eq '#') - { - $line = ""; - next; - } - ($tmphost, $tmpname, @junk) = split(/\s+/, $line); - - # Replace old entry with new entry - if($name eq $tmpname && $host eq $tmphost) - { - $isfound=1; - $line = "\n$host\t$name\t$path\t$status\t$platform\t$profile\n"; - } - } + # Ignore comments + if ( substr( $line, 0, 1 ) eq '#' ) { + next; + } + ( $tmphost, $tmpname, @junk ) = split( /\s+/, $line ); - # Add new entry - if(!$isfound) - { - push(@TCDATA, "\n$host\t$name\t$path\t$status\t$platform\t$profile\n") - } + # Replace old entry with new entry + if ( $name eq $tmpname && $host eq $tmphost ) { + $isfound = 1; + $line = "$host\t$name\t$path\t$status\t$platform\t$profile\n"; + } + } - # Write to tc.data - write_file($tcfile, @TCDATA); + # Add new entry + if ( !$isfound ) { + push( @tcdata, "$host\t$name\t$path\t$status\t$platform\t$profile\n" ); + } + + # Write to tc.data + write_file( $tcfile, @tcdata ); } Modified: usertools/swift/swiftconfig/etc/sites-template.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites-template.xml 2010-07-02 15:57:21 UTC (rev 3425) +++ usertools/swift/swiftconfig/etc/sites-template.xml 2010-07-05 08:31:54 UTC (rev 3426) @@ -2,14 +2,14 @@ - /home/arjun/swiftwork + $HOME/swiftwork 0 - /home/arjun/swiftwork + $HOME/swiftwork 0 @@ -24,7 +24,7 @@ 0.5 10000 - /home/arjun/swiftwork + $HOME/swiftwork @@ -38,13 +38,13 @@ 0.5 10000 - /home/arjun/swiftwork + $HOME/swiftwork - /home/arjun/swiftwork + $HOME/swiftwork 3000 8 1 @@ -58,7 +58,7 @@ - /home/arjun/swiftwork + $HOME/swiftwork 3000 8 1 @@ -72,28 +72,28 @@ - /home/arjun/swiftwork + $HOME/swiftwork 0 - /home/arjun/swiftwork + $HOME/swiftwork 0 - /home/arjun/swiftwork + $HOME/swiftwork 0 - /home/arjun/swiftwork + $HOME/swiftwork 0 @@ -108,7 +108,7 @@ 0.5 10000 - /home/arjun/swiftwork + $HOME/swiftwork @@ -122,6 +122,6 @@ 0.5 10000 - /home/arjun/swiftwork + $HOME/swiftwork From noreply at svn.ci.uchicago.edu Tue Jul 6 03:22:26 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 6 Jul 2010 03:22:26 -0500 (CDT) Subject: [Swift-commit] r3427 - usertools/swift/swiftconfig/bin Message-ID: <20100706082226.C38359CCB1@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-06 03:22:26 -0500 (Tue, 06 Jul 2010) New Revision: 3427 Modified: usertools/swift/swiftconfig/bin/swiftconfig Log: perldoc Modified: usertools/swift/swiftconfig/bin/swiftconfig =================================================================== --- usertools/swift/swiftconfig/bin/swiftconfig 2010-07-05 08:31:54 UTC (rev 3426) +++ usertools/swift/swiftconfig/bin/swiftconfig 2010-07-06 08:22:26 UTC (rev 3427) @@ -52,7 +52,6 @@ 'platform=s' => \$platform, 'profile=s' => \$profile, 'tcfile=s' => \$tcfile, - 'overwrite' => \$overwrite, 'templatefile=s' => \$templatefile, 'add=s' => \$add, 'sitesfile=s' => \$sitesfile, @@ -68,7 +67,6 @@ 'remove=s' => \$remove, 'templates' => \$templates, 'modify=s' => \$modify, - 'profile=s' => \$profile, 'key=s' => \$key, 'namespace=s' => \$namespace, 'value=s' => \$value @@ -253,3 +251,79 @@ write_file( $tcfile, @tcdata ); } +__END__ +=head1 NAME + +swiftconfig - Utility for managing Swift configuration + +=head1 SYNOPSIS + +swiftconfig [B<-option> value] + +=head1 OVERVIEW + +The swiftconfig program allows users to configure Swift. It allows for the adding, removing, and modification of remote sites by utilizing a set of standard templates. It also provides a way to quickly add, remove and modify translation catalog entries without having to manually edit files. + +=head1 DESCRIPTION + +General operations: + -add sitename add a site from template + -remove site removes a site from sites.xml + -remove command removes a command from the catalog + -templates display all available sites in template + -modify site Specifies the name of a site to modify + +Translation catalog settings: + -host hostname hostname of the translation catalog entry + -name name translation name + -path path full pathname to location of program + -status status installation status (deprecated) + -profile setting define the profile value for an entry + -tcfile filename explicitly specify a translation file + +Sites settings: + -templatefile file explicitly set the template file to use + -sitesfile file explicitly set the sites file to use + -gridftp GridFTPURL GridFTP URL + -jobuniverse universe job manager universe + -joburl URL job manager URL + -jobmajor major job mager number + -jobminor minor job minor number + -directory dir work directory + -exprovider name execution provider + -exmanager name execution job manager + -exurl URL execution URL + -key key profile key + -value value profile value + -namespace name profile namespace + + +=head1 EXAMPLES + +List all templates available for adding + swiftconfig -templates + +Add a site from template into working sites.xml + swiftconfig -add teraport + +Modify the work directory of a site + swiftconfig -modify teraport -directory /var/tmp + +Remove a site + swiftconfig -remove teraport + +Add a new command to translation catalog + swiftconfig -name convert -path /usr/local/bin/convert + +Modify an existing command in the translation catalog + swiftconfig -name convert -path /usr/bin/convert + +Remove a command from the translation catalog + swiftconfig -remove convert + +=head1 CAVEATS + +Swiftconfig will attempt to automatically determine the location of swift configuration files. It first checks for an environment variable called $SWIFT_HOME. If that is not found, it will look for the location of "swift" in the path, and try to find the configuration files from there. This default behavior can be overwridden by manually specifying the location of files with -templatefile, -sitesfile, and -tcfile. + +The XML library that swiftconfig uses ignores comments in XML. All comments will be stripped from sites.xml as it gets modified. +=cut From noreply at svn.ci.uchicago.edu Tue Jul 6 16:54:12 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 6 Jul 2010 16:54:12 -0500 (CDT) Subject: [Swift-commit] r3428 - SwiftApps/Montage Message-ID: <20100706215412.984F39CC90@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-06 16:54:12 -0500 (Tue, 06 Jul 2010) New Revision: 3428 Added: SwiftApps/Montage/Swift_Montage_Apps.swift SwiftApps/Montage/Swift_Montage_Batch.swift SwiftApps/Montage/Swift_Montage_Types.swift Modified: SwiftApps/Montage/montage.swift Log: o SwiftApps/Montage/Swift_Montage_Batch.swift o SwiftApps/Montage/Swift_Montage_Apps.swift o SwiftApps/Montage/Swift_Montage_Types.swift -- Moved Batch functions, app functions, and declared types into appropriate files o Montage/montage.swift -- Now only import above files into script file. This change was for organization. Added: SwiftApps/Montage/Swift_Montage_Apps.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Apps.swift (rev 0) +++ SwiftApps/Montage/Swift_Montage_Apps.swift 2010-07-06 21:54:12 UTC (rev 3428) @@ -0,0 +1,135 @@ +/*! \file Swift_Montage_Apps.swift + \brief File containing Swift wrappers to basic Montage functions + + Each app function is a wrapper to a scalar Montage function. + Using the word scalar becuase each function takes a single image to + work on unlike the Swift batch functions. +*/ + + +/*! \fn app Image mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) + \brief Adds all images in imgs[] into one image file based off the information in img_tbl and hdr. + + \param imgs[] List of images to be added into one image. All images but reside in same directory. + \param img_tbl Table containing information about images being added + \param hdr MosaicData file containing information about how mosaic will look. + \return mos Image file containing data about mosaic. +*/ +( Image mos ) mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) +{ + app + { + mAdd "-p" @dirname( imgs[0] ) "-n" @img_tbl @hdr @mos; + } +} + + +/*! \fn app Image mBackground( Image img, Table img_tbl, Table corr_tbl ) + \brief Generates a corrected image from imgs and the information from corr_tbl + + \param img Image to be corrected + \param img_tbl Table containing information about images being added + \param corr_tbl Correction table containing correction coefficients to be applied + \return mos bg_img corrected image +*/ +( Image bg_img ) mBackground( Image img, Table img_tbl, Table corr_tbl ) +{ + app + { + mBackground "-n" @img @bg_img @img_tbl @corr_tbl; + } +} + + +/*! \fn app Table mBgModel( Table img_tbl, Table fits_tbl ) + \brief Generates the corrected image table from the coefficeints data from fits_tbl and data from the projected image table + + \param img_tbl Table containing information about the projected images. + \param fits_tbl Table containing information about how to correct an image + \return corr_tbl Correction table containing correction coefficients to be applied +*/ +( Table corr_tbl ) mBgModel( Table img_tbl, Table fits_tbl ) +{ + app + { + mBgModel @img_tbl @fits_tbl @corr_tbl; + } +} + + +/*! \fn app Image mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) + \brief Calculates the difference image from the two projected images and the header file. + + \param proj_img_1 The first image to calculate how much is overlapped + \param proj_img_2 The second image to calculate how much is overlapped + \param hdr MosaicData file that tells how the mosaic is set up + \return diff_img Difference image that contains overlapped information +*/ +( Image diff_img ) mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) +{ + app + { + mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; + } +} + + +/*! \fn app Table mImgtbl( Image imgs[] ) + \brief Take a list of of images and create a table out of them. All images in the list must be located in the same directory. + + \param imgs[] List of files to create an table out of. + \return img_tbl Table of information of the list of images. +*/ +( Table img_tbl ) mImgtbl( Image imgs[] ) +{ + app + { + mImgtbl @dirname( imgs[0] ) @img_tbl; + } +} + + +/*! \fn app JPEG mJPEG( Image mos_img ) + \brief Creates a viewable jpeg image from the mosaic data image + + \param mos_img Image file to create a jpeg out of. + \return mos_img_jpg Jpeg picture. +*/ +( JPEG mos_img_jpg ) mJPEG( Image mos_img ) +{ + app + { + mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg; + } +} + + +/*! \fn app Table mOverlaps( Table img_tbl ) + \brief Calculates which images overlaps each other and creates a table out of. + + \param img_tbl Table of projected images. + \return diff_tbl Table of which overlapped images +*/ +( Table diff_tbl ) mOverlaps( Table img_tbl ) +{ + app + { + mOverlaps @img_tbl @diff_tbl; + } +} + + +/*! \fn app Image mProjectPP( Image raw_img, MosaicData hdr ) + \brief Projects an image based off of the header file. + + \param raw_img Raw image to project. + \param hdr MosaicData file telling how the mosaic will be set up in the end. + \return proj_img Projected image. +*/ +( Image proj_img ) mProjectPP( Image raw_img, MosaicData hdr ) +{ + app + { + mProjectPP "-X" @raw_img @proj_img @hdr; + } +} Added: SwiftApps/Montage/Swift_Montage_Batch.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Batch.swift (rev 0) +++ SwiftApps/Montage/Swift_Montage_Batch.swift 2010-07-06 21:54:12 UTC (rev 3428) @@ -0,0 +1,103 @@ +/*! \file Swift_Montage_Batch.swift + \brief File containing Batch functions for the Swift Montage wrappers + + Each batch function allows basic Montage functions to be called on a list of + images. +*/ + + +/*! \fn Image mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) + \brief Runs the mBackground wrapper on a list of images in parallel. + + \param bg_imgs[] The list of images to run the mBackground wrapper on. + \param img_tbl Table containing information on original images. + \param corr_tbl Table containing correction coefficients to be applied to the images. + \return corr_imgs[] List of the corrected images to be added to create image mosaic. +*/ +( Image corr_imgs[] ) mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) +{ + + /* ImageStruct img_struct_tbl[] <>; */ + /* CorrectStruct correct_stuct_tbl[] <>; */ + /* Image proj_img; */ + /* Image corr_img; */ + /* int A; */ + /* int B; */ + /* int C; */ + + /* foreach img_tbl_entry, i in img_struct_tbl */ + /* { */ + /* foreach correct_tbl_entry in correct_struct_tbl */ + /* { */ + /* if( img_tbl_entry.cntr == correct_tbl_entry.id ) */ + /* { */ + /* img ; */ + + /* corr_img = mBackground( img, A, B, C ); */ + + /* corr_imgs[ i ] = corr_img; */ + /* } */ + /* } */ + /* } */ +} + + +/*! \fn Image mDiffBatch( Table diff_tbl, MosaicData hdr ) + \brief Runs the mDiff wrapper on a list of images in parallel. + + \param diff_tbl Table containing a list of images that overlap and must calculate by how much they overlap. + \param hdr File containing information on how the final mosaic will be set up. + \return diff_imgs[] List of the difference images that must have correction coefficients calculated. +*/ +( Image diff_imgs[] ) mDiffBatch( Table diff_tbl, MosaicData hdr ) +{ + DiffStruct diffs[] ; + + foreach d_entry, i in diffs + { + Image proj_1 ; + Image proj_2 ; + + Image diff_img ; + diff_img = mDiff( proj_1, proj_2, hdr ); + + diff_imgs[ i ] = diff_img; + } +} + + +/*! \fn app Table mFitBatch( Table diff_tbl, Image diff_imgs[] ) + \brief Generates a corrected image from imgs and the information from corr_tbl + + \param diff_tbl + \param diff_imgs + \return fits_tbl +*/ +( Table fits_tbl ) mFitBatch( Table diff_tbl, Image diff_imgs[] ) +{ + app + { + mFitExec @diff_tbl @fits_tbl @dirname( diff_imgs[0] ); + } +} + + +/*! \fn Image mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) + \brief Projects the raw images from raw_imgs to a new plane that was declared in hdr. + + \param raw_imgs[] List of raw images that need to be projected to new plane. + \param hdr File containing information on how the final mosaic will be set up. + \return proj_imgs[] List of newly reprojected images. +*/ +( Image proj_imgs[] ) mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) +{ + foreach img, i in raw_imgs + { + Image proj_img ; + + proj_img = mProjectPP( img, hdr ); + + proj_imgs[ i ] = proj_img; + } +} Added: SwiftApps/Montage/Swift_Montage_Types.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Types.swift (rev 0) +++ SwiftApps/Montage/Swift_Montage_Types.swift 2010-07-06 21:54:12 UTC (rev 3428) @@ -0,0 +1,59 @@ +/*! \file Swift_Montage_Types.swift + \brief File containing declared Swift types for the Swift Montage wrappers +*/ + +/*! \typdef type Image + \brief Declared Image Swift type. +*/ +type Image; + + +/*! \typdef type MosaicData + \brief Declared MosaicData Swift type. +*/ +type MosaicData; + + +/*! \typdef type Table + \brief Declared Table Swift type. +*/ +type Table; + + +/*! \typdef type JPEG + \brief Declared JPEG Swift type. +*/ +type JPEG; + + +/*! \struct CorrectStruct + \brief Struct to hold data found in the table that is generated from the mBgModel wrapper. +*/ +type CorrectStruct +{ + int id; /*!< The id referencing which image this entry belongs to */ +}; + + +/*! \struct DiffStruct + \brief Stuct to hold data found in the table that is generated from the mOverlaps wrapper. + + \var int cntr1 The id referencing the first image in the difference table. +*/ +type DiffStruct +{ + int cntr1; /*!< The id referencing the first image in the difference table. */ + int cntr2; /*!< The id referencing the second image in the difference table. */ + Image plus; /*!< The name of the image referenced by the id number cntr1. */ + Image minus; /*!< The name of the image referenced by the id number cntr2. */ + Image diff; /*!< The name of the difference image that was created by plus and minus. */ +}; + + +/*! \struct ImageStruct + \brief Struct to hold data found in the table that is generated from the mImgtbl wrapper. +*/ +type ImageStruct +{ + int cntr; /*!< The id referencing which image this entry belongs to */ +}; Modified: SwiftApps/Montage/montage.swift =================================================================== --- SwiftApps/Montage/montage.swift 2010-07-06 08:22:26 UTC (rev 3427) +++ SwiftApps/Montage/montage.swift 2010-07-06 21:54:12 UTC (rev 3428) @@ -1,137 +1,8 @@ -type Image; -type Header; -type Table; -type JPEG; -type TxtFile; -type DiffStruct -{ - int cntr1; - int cntr2; - Image plus; - Image minus; - Image diff; -} +import Swift_Montage_Types; +import Swift_Montage_Apps; +import Swift_Montage_Batch; -( Image mos ) mAdd ( Image imgs[], Table img_tbl, Header hdr ) -{ - app - { -// mAddDirec @(imgs[0]) @img_tbl @hdr @mos; - mAdd "-p" @dirname( imgs[0] ) "-n" @img_tbl @hdr @mos; - } -} - -( Image bg_img ) mBackground ( Image img, Table img_tbl, Table corr_tbl ) -{ - app - { - mBackground "-n" @img @bg_img @img_tbl @corr_tbl; - } -} - -/* ( Image corr_imgs[] ) mBgBatch ( Image bg_imgs[], Table img_tbl, Table corr_tbl ) */ -/* { */ -/* foreach img, i in bg_imgs */ -/* { */ -/* Image corr_img ; */ - -/* corr_img = mBackground( img, img_tbl, corr_tbl ); */ - -/* corr_imgs[ i ] = corr_img; */ -/* } */ -/* } */ - -/* ( Image corr_imgs[] ) mBgBatch ( Table img_tbl, Table corr_tbl ) */ -mBgBatch ( Table img_tbl, Table corr_tbl ) -{ - app - { - mBgExec "-p" "proj_dir" "-n" @img_tbl @corr_tbl "corr_dir"; - } -} - -( Table corr_tbl ) mBgModel ( Table proj_tbl, Table fits_tbl ) -{ - app - { - mBgModel @proj_tbl @fits_tbl @corr_tbl; - } -} - -( Image diff_img ) mDiff ( Image proj_img_1, Image proj_img_2, Header hdr ) -{ - app - { - mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; - } -} - -( Image diff_imgs[] ) mDiffBatch ( Table diff_tbl, Header hdr ) -{ - DiffStruct diffs[] ; - - foreach d_entry, i in diffs - { - Image proj_1 ; - Image proj_2 ; - - Image diff_img ; - diff_img = mDiff( proj_1, proj_2, hdr ); - - diff_imgs[ i ] = diff_img; - } -} - -( Table fits_tbl ) mFitBatch ( Table diff_tbl, Image diff_imgs[] ) -{ - app - { - mFitExec @diff_tbl @fits_tbl @dirname( diff_imgs[0] ); - } -} - -( Table img_tbl ) mImgtbl ( Image imgs[] ) -{ - app - { - mImgtbl @dirname( imgs[0] ) @img_tbl; - } -} - -( JPEG mos_img_jpg ) mJPEG ( Image mos_img ) -{ - app - { - mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg; - } -} - -app ( Table diff_tbl ) mOverlaps ( Table img_tbl ) -{ - mOverlaps @img_tbl @diff_tbl; -} - -( Image proj_img ) mProjectPP ( Image raw_img, Header hdr ) -{ - app - { - mProjectPP "-X" @raw_img @proj_img @hdr; - } -} - -( Image proj_imgs[] ) mProjectPPBatch ( Image raw_imgs[], Header hdr ) -{ - foreach img, i in raw_imgs - { - Image proj_img ; - - proj_img = mProjectPP( img, hdr ); - - proj_imgs[ i ] = proj_img; - } -} - -Header template <"template.hdr">; +MosaicData template <"template.hdr">; Table images_tbl <"images.tbl">; Table difference_tbl <"diffs.tbl">; Table fits_images_tbl <"fits.tbl">; @@ -156,7 +27,7 @@ // Up to here the scripts work. -/* corrected_images = mBgBatch( images_tbl, corrections_tbl ); */ +//corrected_images = mBgBatch( images_tbl, corrections_tbl ); /* mBgBatch( images_tbl, corrections_tbl ); */ /* corrected_mos = mAdd( corrected_images, images_tbl, template ); */ /* corrected_jpg_img = mJPEG( corrected_mos ); */ From noreply at svn.ci.uchicago.edu Wed Jul 7 15:20:19 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 7 Jul 2010 15:20:19 -0500 (CDT) Subject: [Swift-commit] r3429 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20100707202019.6E1539CCD0@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-07 15:20:19 -0500 (Wed, 07 Jul 2010) New Revision: 3429 Modified: trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java Log: Follow-up bug fix to CSVMapper Modified: trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2010-07-06 21:54:12 UTC (rev 3428) +++ trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2010-07-07 20:20:19 UTC (rev 3429) @@ -21,6 +21,7 @@ import org.griphyn.vdl.mapping.MappingParam; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; +import org.griphyn.vdl.type.Types; public class CSVMapper extends AbstractMapper { public static final MappingParam PARAM_FILE = new MappingParam("file"); @@ -133,10 +134,16 @@ String result = null; Object object = PARAM_FILE.getRawValue(this); DSHandle handle = (DSHandle) object; - Mapper mapper = handle.getMapper(); - PhysicalFormat format = mapper.map(Path.EMPTY_PATH); - GeneralizedFileFormat fileFormat = - (GeneralizedFileFormat) format; + GeneralizedFileFormat fileFormat; + if (handle.getType().equals(Types.STRING)) { + String path = (String) handle.getValue(); + fileFormat = new AbsFile(path); + } + else { + Mapper mapper = handle.getMapper(); + PhysicalFormat format = mapper.map(Path.EMPTY_PATH); + fileFormat = (GeneralizedFileFormat) format; + } result = fileFormat.getPath(); return result; } From noreply at svn.ci.uchicago.edu Wed Jul 7 21:19:22 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 7 Jul 2010 21:19:22 -0500 (CDT) Subject: [Swift-commit] r3430 - SwiftApps/Montage Message-ID: <20100708021922.D28C79CC90@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-07 21:19:22 -0500 (Wed, 07 Jul 2010) New Revision: 3430 Added: SwiftApps/Montage/Background_list.py Modified: SwiftApps/Montage/Swift_Montage_Apps.swift SwiftApps/Montage/Swift_Montage_Batch.swift SwiftApps/Montage/Swift_Montage_Types.swift SwiftApps/Montage/makefile SwiftApps/Montage/montage.sh SwiftApps/Montage/montage.swift Log: o SwiftApps/Montage/Background_list.py -- Script created to replace the id in the correction table with the image file name. This allows me to run mBackground insteado mBgExec. o SwiftApps/Montage/Swift_Montage_Apps.swift o SwiftApps/Montage/Swift_Montage_Batch.swift o SwiftApps/Montage/Swift_Montage_Types.swift -- Modifications to finish the mBgBatch swift script. Now all wrappers are implemented needed to run general cases of Montage. o SwiftApps/Montage/montage.swift -- Added lines to the script to run the m101 tutorial in Swift. o SwiftApps/Montage/montage.sh o SwiftApps/Montage/makefile -- Minor modifications. Added: SwiftApps/Montage/Background_list.py =================================================================== --- SwiftApps/Montage/Background_list.py (rev 0) +++ SwiftApps/Montage/Background_list.py 2010-07-08 02:19:22 UTC (rev 3430) @@ -0,0 +1,36 @@ +#!/usr/bin/python + +import sys +#import string + +args = sys.argv; + +file_tbl_1 = open( args[1], 'r' ); +file_tbl_o = open( args[3], 'w' ); +i = 0 + +for line_tbl_1 in file_tbl_1: + + l1 = line_tbl_1.split(); + + if l1[0] != "|" and l1[0] != "\\datatype": + + file_tbl_2 = open( args[2], 'r' ); + + for line_tbl_2 in file_tbl_2: + + l2 = line_tbl_2.split(); + + if l2[0] == l1[0]: + str_out = "["+str(i)+"].fname = "+l1[len(l1) - 1]+"\n" + str_out += "["+str(i)+"].a = "+l2[1]+"\n" + str_out += "["+str(i)+"].b = "+l2[2]+"\n" + str_out += "["+str(i)+"].c = "+l2[3]+"\n" + file_tbl_o.write( str_out ) + i+=1 + break + + file_tbl_2.close(); + +file_tbl_1.close(); +file_tbl_o.close(); Property changes on: SwiftApps/Montage/Background_list.py ___________________________________________________________________ Name: svn:executable + * Modified: SwiftApps/Montage/Swift_Montage_Apps.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Apps.swift 2010-07-07 20:20:19 UTC (rev 3429) +++ SwiftApps/Montage/Swift_Montage_Apps.swift 2010-07-08 02:19:22 UTC (rev 3430) @@ -1,12 +1,3 @@ -/*! \file Swift_Montage_Apps.swift - \brief File containing Swift wrappers to basic Montage functions - - Each app function is a wrapper to a scalar Montage function. - Using the word scalar becuase each function takes a single image to - work on unlike the Swift batch functions. -*/ - - /*! \fn app Image mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) \brief Adds all images in imgs[] into one image file based off the information in img_tbl and hdr. @@ -32,11 +23,11 @@ \param corr_tbl Correction table containing correction coefficients to be applied \return mos bg_img corrected image */ -( Image bg_img ) mBackground( Image img, Table img_tbl, Table corr_tbl ) +( Image bg_img ) mBackground( Image img, float a, float b, float c ) { app { - mBackground "-n" @img @bg_img @img_tbl @corr_tbl; + mBackground "-n" @img @bg_img a b c; } } @@ -133,3 +124,25 @@ mProjectPP "-X" @raw_img @proj_img @hdr; } } + + + +/*----------------------------------------------------------------------------------*/ +/* */ +/* Util Scripts */ +/* */ +/*----------------------------------------------------------------------------------*/ +/*! \fn app Table Background_list( Table imgs_tbl, Table corrs_tbl + \brief Generates a list of coefficients that need to be applied and the files to apply them to. + + \param imgs_tbl Table generated by mImgtbl containing information on the set of images + \param corrs_tbl Table containing the coefficients and the id of the image to apply them to + \return back_tbl Table containing the Image file name and the coefficients that need to be applied to the image. Data is set up so readDatat2 can read the data. +*/ +( Table back_tbl ) Background_list( Table imgs_tbl, Table corrs_tbl ) +{ + app + { + Background_list @imgs_tbl @corrs_tbl @back_tbl; + } +} Modified: SwiftApps/Montage/Swift_Montage_Batch.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Batch.swift 2010-07-07 20:20:19 UTC (rev 3429) +++ SwiftApps/Montage/Swift_Montage_Batch.swift 2010-07-08 02:19:22 UTC (rev 3430) @@ -1,54 +1,47 @@ /*! \file Swift_Montage_Batch.swift - \brief File containing Batch functions for the Swift Montage wrappers + \brief File containing Batch functions for the Swift Montage wrappers - Each batch function allows basic Montage functions to be called on a list of - images. + Each batch function allows basic Montage functions to be called on a list of + images. */ /*! \fn Image mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) - \brief Runs the mBackground wrapper on a list of images in parallel. + \brief Runs the mBackground wrapper on a list of images in parallel. - \param bg_imgs[] The list of images to run the mBackground wrapper on. - \param img_tbl Table containing information on original images. - \param corr_tbl Table containing correction coefficients to be applied to the images. - \return corr_imgs[] List of the corrected images to be added to create image mosaic. + \param bg_imgs[] The list of images to run the mBackground wrapper on. + \param img_tbl Table containing information on original images. + \param corr_tbl Table containing correction coefficients to be applied to the images. + \return corr_imgs[] List of the corrected images to be added to create image mosaic. */ ( Image corr_imgs[] ) mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) { + Table tmp <"back_tmp.tbl">; + tmp = Background_list( img_tbl, corr_tbl ); - /* ImageStruct img_struct_tbl[] <>; */ - /* CorrectStruct correct_stuct_tbl[] <>; */ - /* Image proj_img; */ - /* Image corr_img; */ - /* int A; */ - /* int B; */ - /* int C; */ + BackgroundStruct back_struct[] = readData2( tmp ); - /* foreach img_tbl_entry, i in img_struct_tbl */ - /* { */ - /* foreach correct_tbl_entry in correct_struct_tbl */ - /* { */ - /* if( img_tbl_entry.cntr == correct_tbl_entry.id ) */ - /* { */ - /* img ; */ + foreach background_entry, i in back_struct + { + Image proj_img ; + Image corr_img ; - /* corr_img = mBackground( img, A, B, C ); */ + float a = background_entry.a; + float b = background_entry.b; + float c = background_entry.c; + corr_img = mBackground( proj_img, a, b, c ); - /* corr_imgs[ i ] = corr_img; */ - /* } */ - /* } */ - /* } */ + corr_imgs[ i ] = corr_img; + } } /*! \fn Image mDiffBatch( Table diff_tbl, MosaicData hdr ) - \brief Runs the mDiff wrapper on a list of images in parallel. + \brief Runs the mDiff wrapper on a list of images in parallel. - \param diff_tbl Table containing a list of images that overlap and must calculate by how much they overlap. - \param hdr File containing information on how the final mosaic will be set up. - \return diff_imgs[] List of the difference images that must have correction coefficients calculated. + \param diff_tbl Table containing a list of images that overlap and must calculate by how much they overlap. + \param hdr File containing information on how the final mosaic will be set up. + \return diff_imgs[] List of the difference images that must have correction coefficients calculated. */ ( Image diff_imgs[] ) mDiffBatch( Table diff_tbl, MosaicData hdr ) { @@ -56,6 +49,7 @@ foreach d_entry, i in diffs { +// tracef("plus: %s \n", @d_entry.plus ); Image proj_1 ; Image proj_2 ; @@ -68,11 +62,11 @@ /*! \fn app Table mFitBatch( Table diff_tbl, Image diff_imgs[] ) - \brief Generates a corrected image from imgs and the information from corr_tbl + \brief Generates a corrected image from imgs and the information from corr_tbl - \param diff_tbl - \param diff_imgs - \return fits_tbl + \param diff_tbl + \param diff_imgs + \return fits_tbl */ ( Table fits_tbl ) mFitBatch( Table diff_tbl, Image diff_imgs[] ) { @@ -84,11 +78,11 @@ /*! \fn Image mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) - \brief Projects the raw images from raw_imgs to a new plane that was declared in hdr. + \brief Projects the raw images from raw_imgs to a new plane that was declared in hdr. - \param raw_imgs[] List of raw images that need to be projected to new plane. - \param hdr File containing information on how the final mosaic will be set up. - \return proj_imgs[] List of newly reprojected images. + \param raw_imgs[] List of raw images that need to be projected to new plane. + \param hdr File containing information on how the final mosaic will be set up. + \return proj_imgs[] List of newly reprojected images. */ ( Image proj_imgs[] ) mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) { Modified: SwiftApps/Montage/Swift_Montage_Types.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Types.swift 2010-07-07 20:20:19 UTC (rev 3429) +++ SwiftApps/Montage/Swift_Montage_Types.swift 2010-07-08 02:19:22 UTC (rev 3430) @@ -25,35 +25,26 @@ */ type JPEG; - -/*! \struct CorrectStruct - \brief Struct to hold data found in the table that is generated from the mBgModel wrapper. +/*! \struct BackgroundStruct + \brief Stuct to hold data found in the tables generated my mImgtbl and mBgModel */ -type CorrectStruct +type BackgroundStruct { - int id; /*!< The id referencing which image this entry belongs to */ + string fname; + float a; + float b; + float c; }; /*! \struct DiffStruct \brief Stuct to hold data found in the table that is generated from the mOverlaps wrapper. - - \var int cntr1 The id referencing the first image in the difference table. */ type DiffStruct { - int cntr1; /*!< The id referencing the first image in the difference table. */ - int cntr2; /*!< The id referencing the second image in the difference table. */ - Image plus; /*!< The name of the image referenced by the id number cntr1. */ - Image minus; /*!< The name of the image referenced by the id number cntr2. */ - Image diff; /*!< The name of the difference image that was created by plus and minus. */ + int cntr1; + int cntr2; + Image plus; + Image minus; + Image diff; }; - - -/*! \struct ImageStruct - \brief Struct to hold data found in the table that is generated from the mImgtbl wrapper. -*/ -type ImageStruct -{ - int cntr; /*!< The id referencing which image this entry belongs to */ -}; Modified: SwiftApps/Montage/makefile =================================================================== --- SwiftApps/Montage/makefile 2010-07-07 20:20:19 UTC (rev 3429) +++ SwiftApps/Montage/makefile 2010-07-08 02:19:22 UTC (rev 3430) @@ -7,4 +7,4 @@ swiftrun montage.swift clean: - @rm -rvf proj_dir/* diff_dir/* final/* corr_dir/* debug diffs.tbl images-rawdir.tbl images.tbl corrections.tbl fits.tbl stats.tbl + @rm -rvf proj_dir/* diff_dir/* final/* corr_dir/* debug diffs.tbl images-rawdir.tbl images.tbl corrections.tbl fits.tbl stats.tbl back_tmp.tbl *.txt Modified: SwiftApps/Montage/montage.sh =================================================================== --- SwiftApps/Montage/montage.sh 2010-07-07 20:20:19 UTC (rev 3429) +++ SwiftApps/Montage/montage.sh 2010-07-08 02:19:22 UTC (rev 3430) @@ -13,9 +13,9 @@ mDiffExec -p proj_dir -n diffs.tbl template.hdr diff_dir mFitExec diffs.tbl fits.tbl diff_dir mBgModel images.tbl fits.tbl corrections.tbl -# mBgExec -p proj_dir -n images.tbl corrections.tbl corr_dir -# mAdd -p corr_dir -n images.tbl template.hdr final/mosaic.fits -# mJPEG -gray final/mosaic.fits 20% 99.98% loglog -out final/mosaic.jpg +mBgExec -p proj_dir -n images.tbl corrections.tbl corr_dir +mAdd -p corr_dir -n images.tbl template.hdr final/mosaic.fits +mJPEG -gray final/mosaic.fits 20% 99.98% loglog -out final/mosaic.jpg END=$(date +%s) DIFF=$(( $END - $START )) Modified: SwiftApps/Montage/montage.swift =================================================================== --- SwiftApps/Montage/montage.swift 2010-07-07 20:20:19 UTC (rev 3429) +++ SwiftApps/Montage/montage.swift 2010-07-08 02:19:22 UTC (rev 3430) @@ -24,10 +24,6 @@ difference_images = mDiffBatch( difference_tbl, template ); fits_images_tbl = mFitBatch( difference_tbl, difference_images ); corrections_tbl = mBgModel( images_tbl, fits_images_tbl ); - -// Up to here the scripts work. - -//corrected_images = mBgBatch( images_tbl, corrections_tbl ); -/* mBgBatch( images_tbl, corrections_tbl ); */ -/* corrected_mos = mAdd( corrected_images, images_tbl, template ); */ -/* corrected_jpg_img = mJPEG( corrected_mos ); */ +corrected_images = mBgBatch( projected_images, images_tbl, corrections_tbl ); +corrected_mos = mAdd( corrected_images, images_tbl, template ); +corrected_jpg_img = mJPEG( corrected_mos ); From noreply at svn.ci.uchicago.edu Sat Jul 10 22:38:07 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 10 Jul 2010 22:38:07 -0500 (CDT) Subject: [Swift-commit] r3431 - in trunk: libexec resources src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100711033807.CED699CCC7@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-10 22:38:07 -0500 (Sat, 10 Jul 2010) New Revision: 3431 Modified: trunk/libexec/vdl-lib.xml trunk/resources/swiftscript.g trunk/src/org/griphyn/vdl/engine/Karajan.java trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: o src/org/griphyn/vdl/engine/Karajan.java o resources/swiftscript.g -- Modifications to allow the import directive to accept a relative or absolute path to the script being imported. o src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java o src/org/griphyn/vdl/engine/ProcedureSignature.java o libexec/vdl-lib.xml -- Added tofloat function. Takes in a string and converts the string to a floating point type. -- Added %f to tracef. This allows tracef to print out the floating point type. -- Fixed toint function. When a floating type string representation was passed to toint, the value was not truncated, an error was thrown instead. Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-07-08 02:19:22 UTC (rev 3430) +++ trunk/libexec/vdl-lib.xml 2010-07-11 03:38:07 UTC (rev 3431) @@ -12,6 +12,7 @@ + Modified: trunk/resources/swiftscript.g =================================================================== --- trunk/resources/swiftscript.g 2010-07-08 02:19:22 UTC (rev 3430) +++ trunk/resources/swiftscript.g 2010-07-11 03:38:07 UTC (rev 3431) @@ -78,7 +78,7 @@ ; importStatement [StringTemplate code] - : "import" name:ID SEMI { + : "import" name:STRING_LITERAL SEMI { StringTemplate i = template("import"); i.setAttribute("target", name.getText()); code.setAttribute("imports", i); Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-07-08 02:19:22 UTC (rev 3430) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-07-11 03:38:07 UTC (rev 3431) @@ -155,19 +155,20 @@ Imports imports = prog.getImports(); if(imports!=null) { logger.debug("Processing SwiftScript imports"); -// process imports in reverse order + // process imports in reverse order for(int i = imports.sizeOfImportArray() - 1 ; i >=0 ; i--) { String moduleToImport = imports.getImportArray(i); logger.debug("Importing module "+moduleToImport); if(!importedNames.contains(moduleToImport)) { // TODO PATH/PERL5LIB-style path handling - String swiftfilename = "./"+moduleToImport+".swift"; - String xmlfilename = "./"+moduleToImport+".xml"; + //String swiftfilename = "./"+moduleToImport+".swift"; + //String xmlfilename = "./"+moduleToImport+".xml"; + String swiftfilename = moduleToImport+".swift"; + String xmlfilename = moduleToImport+".xml"; - try { - VDLt2VDLx.compile(new FileInputStream(swiftfilename),new PrintStream(new FileOutputStream(xmlfilename))); + VDLt2VDLx.compile(new FileInputStream(swiftfilename),new PrintStream(new FileOutputStream(xmlfilename))); logger.debug("Compiled. Now reading in compiled XML for "+moduleToImport); Program importedProgram = parseProgramXML(xmlfilename).getProgram(); logger.debug("Read in compiled XML for "+moduleToImport); Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-07-08 02:19:22 UTC (rev 3430) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-07-11 03:38:07 UTC (rev 3431) @@ -188,7 +188,7 @@ filenames.addOutputArg(filenamesOut1); functionsMap.put(filenames.getName(), filenames); - ProcedureSignature dirname = new ProcedureSignature("dirname"); + ProcedureSignature dirname = new ProcedureSignature("dirname"); FormalArgumentSignature dirnameIn1 = new FormalArgumentSignature(true); /* dir can be specified as any type */ dirname.addInputArg(dirnameIn1); FormalArgumentSignature dirnameOut1 = new FormalArgumentSignature("string"); @@ -245,6 +245,13 @@ FormalArgumentSignature toOut1 = new FormalArgumentSignature("int"); toint.addOutputArg(toOut1); functionsMap.put(toint.getName(), toint); + + ProcedureSignature tofloat = new ProcedureSignature("tofloat"); + FormalArgumentSignature tofloatIn1 = new FormalArgumentSignature("string"); + tofloat.addInputArg(tofloatIn1); + FormalArgumentSignature tofloatOut1 = new FormalArgumentSignature("float"); + tofloat.addOutputArg(tofloatOut1); + functionsMap.put(tofloat.getName(), tofloat); ProcedureSignature tostring = new ProcedureSignature("tostring"); FormalArgumentSignature tostringIn1 = new FormalArgumentSignature(true); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-08 02:19:22 UTC (rev 3430) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-11 03:38:07 UTC (rev 3431) @@ -41,10 +41,11 @@ setArguments("swiftscript_tracef", new Arg[] { Arg.VARGS }); setArguments("swiftscript_strcat", new Arg[] { Arg.VARGS }); setArguments("swiftscript_strcut", new Arg[] { PA_INPUT, PA_PATTERN }); - setArguments("swiftscript_strstr", new Arg[] { PA_INPUT, PA_PATTERN }); + setArguments("swiftscript_strstr", new Arg[] { PA_INPUT, PA_PATTERN }); setArguments("swiftscript_strsplit", new Arg[] { PA_INPUT, PA_PATTERN }); setArguments("swiftscript_regexp", new Arg[] { PA_INPUT, PA_PATTERN, PA_TRANSFORM }); setArguments("swiftscript_toint", new Arg[] { PA_INPUT }); + setArguments("swiftscript_tofloat", new Arg[] { PA_INPUT }); setArguments("swiftscript_tostring", new Arg[] { PA_INPUT }); setArguments("swiftscript_dirname", new Arg[] { Arg.VARGS }); } @@ -171,6 +172,14 @@ else { throw new ExecutionException("tracef(): %i requires an int!"); } + } + else if (c == 'f') { + if (args[arg].getType() == Types.FLOAT) { + output.append(args[arg]).toString(); + } + else { + throw new ExecutionException("tracef(): %f requires a float!"); + } } else if (c == 'q') { if (args[arg] instanceof ArrayDataNode) { @@ -358,6 +367,11 @@ public DSHandle swiftscript_toint(VariableStack stack) throws ExecutionException, NoSuchTypeException, InvalidPathException { String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); + int i = inputString.indexOf("."); + if( i >= 0 ) + { + inputString = inputString.substring(0, i); + } DSHandle handle = new RootDataNode(Types.INT); handle.setValue(new Double(Integer.parseInt(inputString))); handle.closeShallow(); @@ -366,6 +380,18 @@ VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "string"); return handle; } + + public DSHandle swiftscript_tofloat(VariableStack stack) throws ExecutionException, NoSuchTypeException, + InvalidPathException { + String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); + DSHandle handle = new RootDataNode(Types.FLOAT); + handle.setValue(new Double(inputString)); + handle.closeShallow(); + int provid=VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, handle, "tofloat"); + VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "string"); + return handle; + } public DSHandle swiftscript_tostring(VariableStack stack) throws ExecutionException, NoSuchTypeException, From noreply at svn.ci.uchicago.edu Mon Jul 12 14:26:40 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 12 Jul 2010 14:26:40 -0500 (CDT) Subject: [Swift-commit] r3432 - in SwiftApps/Montage: . example scripts Message-ID: <20100712192640.E0E1F9CC6B@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-12 14:26:40 -0500 (Mon, 12 Jul 2010) New Revision: 3432 Added: SwiftApps/Montage/example/ SwiftApps/Montage/example/corr_dir/ SwiftApps/Montage/example/diff_dir/ SwiftApps/Montage/example/final/ SwiftApps/Montage/example/m101_montage.sh SwiftApps/Montage/example/m101_montage.swift SwiftApps/Montage/example/makefile SwiftApps/Montage/example/proj_dir/ SwiftApps/Montage/example/raw_dir/ SwiftApps/Montage/example/template.hdr SwiftApps/Montage/scripts/ SwiftApps/Montage/scripts/Background_list.py SwiftApps/Montage/scripts/Swift_Montage_Apps.swift SwiftApps/Montage/scripts/Swift_Montage_Batch.swift SwiftApps/Montage/scripts/Swift_Montage_Types.swift Removed: SwiftApps/Montage/Background_list.py SwiftApps/Montage/Swift_Montage_Apps.swift SwiftApps/Montage/Swift_Montage_Batch.swift SwiftApps/Montage/Swift_Montage_Types.swift SwiftApps/Montage/corr_dir/ SwiftApps/Montage/diff_dir/ SwiftApps/Montage/final/ SwiftApps/Montage/makefile SwiftApps/Montage/montage.sh SwiftApps/Montage/montage.swift SwiftApps/Montage/proj_dir/ SwiftApps/Montage/raw_dir/ SwiftApps/Montage/template.hdr Log: Organization of file. Montage wrapper moved into scripts directory. The m101 tutorial using the Swift wrappers is in the example directory. Deleted: SwiftApps/Montage/Background_list.py =================================================================== --- SwiftApps/Montage/Background_list.py 2010-07-11 03:38:07 UTC (rev 3431) +++ SwiftApps/Montage/Background_list.py 2010-07-12 19:26:40 UTC (rev 3432) @@ -1,36 +0,0 @@ -#!/usr/bin/python - -import sys -#import string - -args = sys.argv; - -file_tbl_1 = open( args[1], 'r' ); -file_tbl_o = open( args[3], 'w' ); -i = 0 - -for line_tbl_1 in file_tbl_1: - - l1 = line_tbl_1.split(); - - if l1[0] != "|" and l1[0] != "\\datatype": - - file_tbl_2 = open( args[2], 'r' ); - - for line_tbl_2 in file_tbl_2: - - l2 = line_tbl_2.split(); - - if l2[0] == l1[0]: - str_out = "["+str(i)+"].fname = "+l1[len(l1) - 1]+"\n" - str_out += "["+str(i)+"].a = "+l2[1]+"\n" - str_out += "["+str(i)+"].b = "+l2[2]+"\n" - str_out += "["+str(i)+"].c = "+l2[3]+"\n" - file_tbl_o.write( str_out ) - i+=1 - break - - file_tbl_2.close(); - -file_tbl_1.close(); -file_tbl_o.close(); Deleted: SwiftApps/Montage/Swift_Montage_Apps.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Apps.swift 2010-07-11 03:38:07 UTC (rev 3431) +++ SwiftApps/Montage/Swift_Montage_Apps.swift 2010-07-12 19:26:40 UTC (rev 3432) @@ -1,148 +0,0 @@ -/*! \fn app Image mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) - \brief Adds all images in imgs[] into one image file based off the information in img_tbl and hdr. - - \param imgs[] List of images to be added into one image. All images but reside in same directory. - \param img_tbl Table containing information about images being added - \param hdr MosaicData file containing information about how mosaic will look. - \return mos Image file containing data about mosaic. -*/ -( Image mos ) mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) -{ - app - { - mAdd "-p" @dirname( imgs[0] ) "-n" @img_tbl @hdr @mos; - } -} - - -/*! \fn app Image mBackground( Image img, Table img_tbl, Table corr_tbl ) - \brief Generates a corrected image from imgs and the information from corr_tbl - - \param img Image to be corrected - \param img_tbl Table containing information about images being added - \param corr_tbl Correction table containing correction coefficients to be applied - \return mos bg_img corrected image -*/ -( Image bg_img ) mBackground( Image img, float a, float b, float c ) -{ - app - { - mBackground "-n" @img @bg_img a b c; - } -} - - -/*! \fn app Table mBgModel( Table img_tbl, Table fits_tbl ) - \brief Generates the corrected image table from the coefficeints data from fits_tbl and data from the projected image table - - \param img_tbl Table containing information about the projected images. - \param fits_tbl Table containing information about how to correct an image - \return corr_tbl Correction table containing correction coefficients to be applied -*/ -( Table corr_tbl ) mBgModel( Table img_tbl, Table fits_tbl ) -{ - app - { - mBgModel @img_tbl @fits_tbl @corr_tbl; - } -} - - -/*! \fn app Image mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) - \brief Calculates the difference image from the two projected images and the header file. - - \param proj_img_1 The first image to calculate how much is overlapped - \param proj_img_2 The second image to calculate how much is overlapped - \param hdr MosaicData file that tells how the mosaic is set up - \return diff_img Difference image that contains overlapped information -*/ -( Image diff_img ) mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) -{ - app - { - mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; - } -} - - -/*! \fn app Table mImgtbl( Image imgs[] ) - \brief Take a list of of images and create a table out of them. All images in the list must be located in the same directory. - - \param imgs[] List of files to create an table out of. - \return img_tbl Table of information of the list of images. -*/ -( Table img_tbl ) mImgtbl( Image imgs[] ) -{ - app - { - mImgtbl @dirname( imgs[0] ) @img_tbl; - } -} - - -/*! \fn app JPEG mJPEG( Image mos_img ) - \brief Creates a viewable jpeg image from the mosaic data image - - \param mos_img Image file to create a jpeg out of. - \return mos_img_jpg Jpeg picture. -*/ -( JPEG mos_img_jpg ) mJPEG( Image mos_img ) -{ - app - { - mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg; - } -} - - -/*! \fn app Table mOverlaps( Table img_tbl ) - \brief Calculates which images overlaps each other and creates a table out of. - - \param img_tbl Table of projected images. - \return diff_tbl Table of which overlapped images -*/ -( Table diff_tbl ) mOverlaps( Table img_tbl ) -{ - app - { - mOverlaps @img_tbl @diff_tbl; - } -} - - -/*! \fn app Image mProjectPP( Image raw_img, MosaicData hdr ) - \brief Projects an image based off of the header file. - - \param raw_img Raw image to project. - \param hdr MosaicData file telling how the mosaic will be set up in the end. - \return proj_img Projected image. -*/ -( Image proj_img ) mProjectPP( Image raw_img, MosaicData hdr ) -{ - app - { - mProjectPP "-X" @raw_img @proj_img @hdr; - } -} - - - -/*----------------------------------------------------------------------------------*/ -/* */ -/* Util Scripts */ -/* */ -/*----------------------------------------------------------------------------------*/ -/*! \fn app Table Background_list( Table imgs_tbl, Table corrs_tbl - \brief Generates a list of coefficients that need to be applied and the files to apply them to. - - \param imgs_tbl Table generated by mImgtbl containing information on the set of images - \param corrs_tbl Table containing the coefficients and the id of the image to apply them to - \return back_tbl Table containing the Image file name and the coefficients that need to be applied to the image. Data is set up so readDatat2 can read the data. -*/ -( Table back_tbl ) Background_list( Table imgs_tbl, Table corrs_tbl ) -{ - app - { - Background_list @imgs_tbl @corrs_tbl @back_tbl; - } -} Deleted: SwiftApps/Montage/Swift_Montage_Batch.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Batch.swift 2010-07-11 03:38:07 UTC (rev 3431) +++ SwiftApps/Montage/Swift_Montage_Batch.swift 2010-07-12 19:26:40 UTC (rev 3432) @@ -1,97 +0,0 @@ -/*! \file Swift_Montage_Batch.swift - \brief File containing Batch functions for the Swift Montage wrappers - - Each batch function allows basic Montage functions to be called on a list of - images. -*/ - - -/*! \fn Image mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) - \brief Runs the mBackground wrapper on a list of images in parallel. - - \param bg_imgs[] The list of images to run the mBackground wrapper on. - \param img_tbl Table containing information on original images. - \param corr_tbl Table containing correction coefficients to be applied to the images. - \return corr_imgs[] List of the corrected images to be added to create image mosaic. -*/ -( Image corr_imgs[] ) mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) -{ - Table tmp <"back_tmp.tbl">; - tmp = Background_list( img_tbl, corr_tbl ); - - BackgroundStruct back_struct[] = readData2( tmp ); - - foreach background_entry, i in back_struct - { - Image proj_img ; - Image corr_img ; - - float a = background_entry.a; - float b = background_entry.b; - float c = background_entry.c; - corr_img = mBackground( proj_img, a, b, c ); - - corr_imgs[ i ] = corr_img; - } -} - - -/*! \fn Image mDiffBatch( Table diff_tbl, MosaicData hdr ) - \brief Runs the mDiff wrapper on a list of images in parallel. - - \param diff_tbl Table containing a list of images that overlap and must calculate by how much they overlap. - \param hdr File containing information on how the final mosaic will be set up. - \return diff_imgs[] List of the difference images that must have correction coefficients calculated. -*/ -( Image diff_imgs[] ) mDiffBatch( Table diff_tbl, MosaicData hdr ) -{ - DiffStruct diffs[] ; - - foreach d_entry, i in diffs - { -// tracef("plus: %s \n", @d_entry.plus ); - Image proj_1 ; - Image proj_2 ; - - Image diff_img ; - diff_img = mDiff( proj_1, proj_2, hdr ); - - diff_imgs[ i ] = diff_img; - } -} - - -/*! \fn app Table mFitBatch( Table diff_tbl, Image diff_imgs[] ) - \brief Generates a corrected image from imgs and the information from corr_tbl - - \param diff_tbl - \param diff_imgs - \return fits_tbl -*/ -( Table fits_tbl ) mFitBatch( Table diff_tbl, Image diff_imgs[] ) -{ - app - { - mFitExec @diff_tbl @fits_tbl @dirname( diff_imgs[0] ); - } -} - - -/*! \fn Image mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) - \brief Projects the raw images from raw_imgs to a new plane that was declared in hdr. - - \param raw_imgs[] List of raw images that need to be projected to new plane. - \param hdr File containing information on how the final mosaic will be set up. - \return proj_imgs[] List of newly reprojected images. -*/ -( Image proj_imgs[] ) mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) -{ - foreach img, i in raw_imgs - { - Image proj_img ; - - proj_img = mProjectPP( img, hdr ); - - proj_imgs[ i ] = proj_img; - } -} Deleted: SwiftApps/Montage/Swift_Montage_Types.swift =================================================================== --- SwiftApps/Montage/Swift_Montage_Types.swift 2010-07-11 03:38:07 UTC (rev 3431) +++ SwiftApps/Montage/Swift_Montage_Types.swift 2010-07-12 19:26:40 UTC (rev 3432) @@ -1,50 +0,0 @@ -/*! \file Swift_Montage_Types.swift - \brief File containing declared Swift types for the Swift Montage wrappers -*/ - -/*! \typdef type Image - \brief Declared Image Swift type. -*/ -type Image; - - -/*! \typdef type MosaicData - \brief Declared MosaicData Swift type. -*/ -type MosaicData; - - -/*! \typdef type Table - \brief Declared Table Swift type. -*/ -type Table; - - -/*! \typdef type JPEG - \brief Declared JPEG Swift type. -*/ -type JPEG; - -/*! \struct BackgroundStruct - \brief Stuct to hold data found in the tables generated my mImgtbl and mBgModel -*/ -type BackgroundStruct -{ - string fname; - float a; - float b; - float c; -}; - - -/*! \struct DiffStruct - \brief Stuct to hold data found in the table that is generated from the mOverlaps wrapper. -*/ -type DiffStruct -{ - int cntr1; - int cntr2; - Image plus; - Image minus; - Image diff; -}; Copied: SwiftApps/Montage/example/corr_dir (from rev 3428, SwiftApps/Montage/corr_dir) Copied: SwiftApps/Montage/example/diff_dir (from rev 3428, SwiftApps/Montage/diff_dir) Copied: SwiftApps/Montage/example/final (from rev 3428, SwiftApps/Montage/final) Copied: SwiftApps/Montage/example/m101_montage.sh (from rev 3430, SwiftApps/Montage/montage.sh) =================================================================== --- SwiftApps/Montage/example/m101_montage.sh (rev 0) +++ SwiftApps/Montage/example/m101_montage.sh 2010-07-12 19:26:40 UTC (rev 3432) @@ -0,0 +1,22 @@ +#!/bin/bash + +mImgtbl raw_dir images-rawdir.tbl + +START=$(date +%s) + +mProjExec -p raw_dir images-rawdir.tbl template.hdr proj_dir stats.tbl +rm -f proj_dir/*area* +mImgtbl proj_dir images.tbl +# mAdd -p pro_jdir -n images.tbl template.hdr final/uncorrected.fits +# mJPEG -gray final/uncorrected.fits 20% 99.98% loglog -out final/uncorrected.jpg +mOverlaps images.tbl diffs.tbl +mDiffExec -p proj_dir -n diffs.tbl template.hdr diff_dir +mFitExec diffs.tbl fits.tbl diff_dir +mBgModel images.tbl fits.tbl corrections.tbl +mBgExec -p proj_dir -n images.tbl corrections.tbl corr_dir +mAdd -p corr_dir -n images.tbl template.hdr final/mosaic.fits +mJPEG -gray final/mosaic.fits 20% 99.98% loglog -out final/mosaic.jpg + +END=$(date +%s) +DIFF=$(( $END - $START )) +echo "It took $DIFF seconds" Copied: SwiftApps/Montage/example/m101_montage.swift (from rev 3430, SwiftApps/Montage/montage.swift) =================================================================== --- SwiftApps/Montage/example/m101_montage.swift (rev 0) +++ SwiftApps/Montage/example/m101_montage.swift 2010-07-12 19:26:40 UTC (rev 3432) @@ -0,0 +1,29 @@ +import "../scripts/Swift_Montage_Types"; +import "../scripts/Swift_Montage_Apps"; +import "../scripts/Swift_Montage_Batch"; + +MosaicData template <"template.hdr">; +Table images_tbl <"images.tbl">; +Table difference_tbl <"diffs.tbl">; +Table fits_images_tbl <"fits.tbl">; +Table corrections_tbl <"corrections.tbl">; +/* Image uncorrected_mos <"final/uncorrected.fits">; */ +Image corrected_mos <"final/corrected.fits">; +/* JPEG uncorrected_jpg_img <"final/uncorrected.jpg">; */ +JPEG corrected_jpg_img <"final/corrected.jpg">; +Image raw_image_files[] ; +Image projected_images[]; +Image difference_images[]; +Image corrected_images[]; + +projected_images = mProjectPPBatch( raw_image_files, template ); +images_tbl = mImgtbl( projected_images ); +/* uncorrected_mos = mAdd( projected_images, images_tbl, template ); */ +/* uncorrected_jpg_img = mJPEG( uncorrected_mos ); */ +difference_tbl = mOverlaps( images_tbl ); +difference_images = mDiffBatch( difference_tbl, template ); +fits_images_tbl = mFitBatch( difference_tbl, difference_images ); +corrections_tbl = mBgModel( images_tbl, fits_images_tbl ); +corrected_images = mBgBatch( projected_images, images_tbl, corrections_tbl ); +corrected_mos = mAdd( corrected_images, images_tbl, template ); +corrected_jpg_img = mJPEG( corrected_mos ); Copied: SwiftApps/Montage/example/makefile (from rev 3430, SwiftApps/Montage/makefile) =================================================================== --- SwiftApps/Montage/example/makefile (rev 0) +++ SwiftApps/Montage/example/makefile 2010-07-12 19:26:40 UTC (rev 3432) @@ -0,0 +1,13 @@ +.PHONY: shell swift DAG + +shell: + ./m101_montage.sh + +swift: + swiftrun m101_montage.swift + +DAG: + swiftrun -pgraph Montage_DAT.dot montage.swift + +clean: + @rm -rvf proj_dir/* diff_dir/* final/* corr_dir/* debug scripts/*.xml diffs.tbl images-rawdir.tbl images.tbl corrections.tbl fits.tbl stats.tbl back_tmp.tbl *.txt Copied: SwiftApps/Montage/example/proj_dir (from rev 3428, SwiftApps/Montage/proj_dir) Copied: SwiftApps/Montage/example/raw_dir (from rev 3428, SwiftApps/Montage/raw_dir) Copied: SwiftApps/Montage/example/template.hdr (from rev 3428, SwiftApps/Montage/template.hdr) =================================================================== --- SwiftApps/Montage/example/template.hdr (rev 0) +++ SwiftApps/Montage/example/template.hdr 2010-07-12 19:26:40 UTC (rev 3432) @@ -0,0 +1,16 @@ +SIMPLE = T +BITPIX = -64 +NAXIS = 2 +NAXIS1 = 2259 +NAXIS2 = 2199 +CTYPE1 = 'RA---TAN' +CTYPE2 = 'DEC--TAN' +CRVAL1 = 210.835222357 +CRVAL2 = 54.367562188 +CDELT1 = -0.000277780 +CDELT2 = 0.000277780 +CRPIX1 = 1130.0000 +CRPIX2 = 1100.0000 +CROTA2 = -0.052834592 +HISTORY = MONTAGE TUTORIAL +END Deleted: SwiftApps/Montage/makefile =================================================================== --- SwiftApps/Montage/makefile 2010-07-11 03:38:07 UTC (rev 3431) +++ SwiftApps/Montage/makefile 2010-07-12 19:26:40 UTC (rev 3432) @@ -1,10 +0,0 @@ -.PHONY: shell swift - -shell: - ./montage.sh - -swift: - swiftrun montage.swift - -clean: - @rm -rvf proj_dir/* diff_dir/* final/* corr_dir/* debug diffs.tbl images-rawdir.tbl images.tbl corrections.tbl fits.tbl stats.tbl back_tmp.tbl *.txt Deleted: SwiftApps/Montage/montage.sh =================================================================== --- SwiftApps/Montage/montage.sh 2010-07-11 03:38:07 UTC (rev 3431) +++ SwiftApps/Montage/montage.sh 2010-07-12 19:26:40 UTC (rev 3432) @@ -1,22 +0,0 @@ -#!/bin/bash - -mImgtbl raw_dir images-rawdir.tbl - -START=$(date +%s) - -mProjExec -p raw_dir images-rawdir.tbl template.hdr proj_dir stats.tbl -rm -f proj_dir/*area* -mImgtbl proj_dir images.tbl -# mAdd -p pro_jdir -n images.tbl template.hdr final/uncorrected.fits -# mJPEG -gray final/uncorrected.fits 20% 99.98% loglog -out final/uncorrected.jpg -mOverlaps images.tbl diffs.tbl -mDiffExec -p proj_dir -n diffs.tbl template.hdr diff_dir -mFitExec diffs.tbl fits.tbl diff_dir -mBgModel images.tbl fits.tbl corrections.tbl -mBgExec -p proj_dir -n images.tbl corrections.tbl corr_dir -mAdd -p corr_dir -n images.tbl template.hdr final/mosaic.fits -mJPEG -gray final/mosaic.fits 20% 99.98% loglog -out final/mosaic.jpg - -END=$(date +%s) -DIFF=$(( $END - $START )) -echo "It took $DIFF seconds" Deleted: SwiftApps/Montage/montage.swift =================================================================== --- SwiftApps/Montage/montage.swift 2010-07-11 03:38:07 UTC (rev 3431) +++ SwiftApps/Montage/montage.swift 2010-07-12 19:26:40 UTC (rev 3432) @@ -1,29 +0,0 @@ -import Swift_Montage_Types; -import Swift_Montage_Apps; -import Swift_Montage_Batch; - -MosaicData template <"template.hdr">; -Table images_tbl <"images.tbl">; -Table difference_tbl <"diffs.tbl">; -Table fits_images_tbl <"fits.tbl">; -Table corrections_tbl <"corrections.tbl">; -/* Image uncorrected_mos <"final/uncorrected.fits">; */ -Image corrected_mos <"final/corrected.fits">; -/* JPEG uncorrected_jpg_img <"final/uncorrected.jpg">; */ -JPEG corrected_jpg_img <"final/corrected.jpg">; -Image raw_image_files[] ; -Image projected_images[]; -Image difference_images[]; -Image corrected_images[]; - -projected_images = mProjectPPBatch( raw_image_files, template ); -images_tbl = mImgtbl( projected_images ); -/* uncorrected_mos = mAdd( projected_images, images_tbl, template ); */ -/* uncorrected_jpg_img = mJPEG( uncorrected_mos ); */ -difference_tbl = mOverlaps( images_tbl ); -difference_images = mDiffBatch( difference_tbl, template ); -fits_images_tbl = mFitBatch( difference_tbl, difference_images ); -corrections_tbl = mBgModel( images_tbl, fits_images_tbl ); -corrected_images = mBgBatch( projected_images, images_tbl, corrections_tbl ); -corrected_mos = mAdd( corrected_images, images_tbl, template ); -corrected_jpg_img = mJPEG( corrected_mos ); Copied: SwiftApps/Montage/scripts/Background_list.py (from rev 3430, SwiftApps/Montage/Background_list.py) =================================================================== --- SwiftApps/Montage/scripts/Background_list.py (rev 0) +++ SwiftApps/Montage/scripts/Background_list.py 2010-07-12 19:26:40 UTC (rev 3432) @@ -0,0 +1,36 @@ +#!/usr/bin/python + +import sys +#import string + +args = sys.argv; + +file_tbl_1 = open( args[1], 'r' ); +file_tbl_o = open( args[3], 'w' ); +i = 0 + +for line_tbl_1 in file_tbl_1: + + l1 = line_tbl_1.split(); + + if l1[0] != "|" and l1[0] != "\\datatype": + + file_tbl_2 = open( args[2], 'r' ); + + for line_tbl_2 in file_tbl_2: + + l2 = line_tbl_2.split(); + + if l2[0] == l1[0]: + str_out = "["+str(i)+"].fname = "+l1[len(l1) - 1]+"\n" + str_out += "["+str(i)+"].a = "+l2[1]+"\n" + str_out += "["+str(i)+"].b = "+l2[2]+"\n" + str_out += "["+str(i)+"].c = "+l2[3]+"\n" + file_tbl_o.write( str_out ) + i+=1 + break + + file_tbl_2.close(); + +file_tbl_1.close(); +file_tbl_o.close(); Copied: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift (from rev 3430, SwiftApps/Montage/Swift_Montage_Apps.swift) =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift (rev 0) +++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-07-12 19:26:40 UTC (rev 3432) @@ -0,0 +1,148 @@ +/*! \fn app Image mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) + \brief Adds all images in imgs[] into one image file based off the information in img_tbl and hdr. + + \param imgs[] List of images to be added into one image. All images but reside in same directory. + \param img_tbl Table containing information about images being added + \param hdr MosaicData file containing information about how mosaic will look. + \return mos Image file containing data about mosaic. +*/ +( Image mos ) mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) +{ + app + { + mAdd "-p" @dirname( imgs[0] ) "-n" @img_tbl @hdr @mos; + } +} + + +/*! \fn app Image mBackground( Image img, Table img_tbl, Table corr_tbl ) + \brief Generates a corrected image from imgs and the information from corr_tbl + + \param img Image to be corrected + \param img_tbl Table containing information about images being added + \param corr_tbl Correction table containing correction coefficients to be applied + \return mos bg_img corrected image +*/ +( Image bg_img ) mBackground( Image img, float a, float b, float c ) +{ + app + { + mBackground "-n" @img @bg_img a b c; + } +} + + +/*! \fn app Table mBgModel( Table img_tbl, Table fits_tbl ) + \brief Generates the corrected image table from the coefficeints data from fits_tbl and data from the projected image table + + \param img_tbl Table containing information about the projected images. + \param fits_tbl Table containing information about how to correct an image + \return corr_tbl Correction table containing correction coefficients to be applied +*/ +( Table corr_tbl ) mBgModel( Table img_tbl, Table fits_tbl ) +{ + app + { + mBgModel @img_tbl @fits_tbl @corr_tbl; + } +} + + +/*! \fn app Image mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) + \brief Calculates the difference image from the two projected images and the header file. + + \param proj_img_1 The first image to calculate how much is overlapped + \param proj_img_2 The second image to calculate how much is overlapped + \param hdr MosaicData file that tells how the mosaic is set up + \return diff_img Difference image that contains overlapped information +*/ +( Image diff_img ) mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) +{ + app + { + mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; + } +} + + +/*! \fn app Table mImgtbl( Image imgs[] ) + \brief Take a list of of images and create a table out of them. All images in the list must be located in the same directory. + + \param imgs[] List of files to create an table out of. + \return img_tbl Table of information of the list of images. +*/ +( Table img_tbl ) mImgtbl( Image imgs[] ) +{ + app + { + mImgtbl @dirname( imgs[0] ) @img_tbl; + } +} + + +/*! \fn app JPEG mJPEG( Image mos_img ) + \brief Creates a viewable jpeg image from the mosaic data image + + \param mos_img Image file to create a jpeg out of. + \return mos_img_jpg Jpeg picture. +*/ +( JPEG mos_img_jpg ) mJPEG( Image mos_img ) +{ + app + { + mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg; + } +} + + +/*! \fn app Table mOverlaps( Table img_tbl ) + \brief Calculates which images overlaps each other and creates a table out of. + + \param img_tbl Table of projected images. + \return diff_tbl Table of which overlapped images +*/ +( Table diff_tbl ) mOverlaps( Table img_tbl ) +{ + app + { + mOverlaps @img_tbl @diff_tbl; + } +} + + +/*! \fn app Image mProjectPP( Image raw_img, MosaicData hdr ) + \brief Projects an image based off of the header file. + + \param raw_img Raw image to project. + \param hdr MosaicData file telling how the mosaic will be set up in the end. + \return proj_img Projected image. +*/ +( Image proj_img ) mProjectPP( Image raw_img, MosaicData hdr ) +{ + app + { + mProjectPP "-X" @raw_img @proj_img @hdr; + } +} + + + +/*----------------------------------------------------------------------------------*/ +/* */ +/* Util Scripts */ +/* */ +/*----------------------------------------------------------------------------------*/ +/*! \fn app Table Background_list( Table imgs_tbl, Table corrs_tbl + \brief Generates a list of coefficients that need to be applied and the files to apply them to. + + \param imgs_tbl Table generated by mImgtbl containing information on the set of images + \param corrs_tbl Table containing the coefficients and the id of the image to apply them to + \return back_tbl Table containing the Image file name and the coefficients that need to be applied to the image. Data is set up so readDatat2 can read the data. +*/ +( Table back_tbl ) Background_list( Table imgs_tbl, Table corrs_tbl ) +{ + app + { + Background_list @imgs_tbl @corrs_tbl @back_tbl; + } +} Copied: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift (from rev 3430, SwiftApps/Montage/Swift_Montage_Batch.swift) =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift (rev 0) +++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2010-07-12 19:26:40 UTC (rev 3432) @@ -0,0 +1,98 @@ +/*! \file Swift_Montage_Batch.swift + \brief File containing Batch functions for the Swift Montage wrappers + + Each batch function allows basic Montage functions to be called on a list of + images. +*/ + + +/*! \fn Image mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) + \brief Runs the mBackground wrapper on a list of images in parallel. + + \param bg_imgs[] The list of images to run the mBackground wrapper on. + \param img_tbl Table containing information on original images. + \param corr_tbl Table containing correction coefficients to be applied to the images. + \return corr_imgs[] List of the corrected images to be added to create image mosaic. +*/ +( Image corr_imgs[] ) mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl ) +{ + Table tmp <"back_tmp.tbl">; + tmp = Background_list( img_tbl, corr_tbl ); + + BackgroundStruct back_struct[] = readData2( tmp ); + + foreach background_entry, i in back_struct + { + Image proj_img ; + Image corr_img ; + + float a = background_entry.a; + float b = background_entry.b; + float c = background_entry.c; + corr_img = mBackground( proj_img, a, b, c ); + + corr_imgs[ i ] = corr_img; + } +} + + +/*! \fn Image mDiffBatch( Table diff_tbl, MosaicData hdr ) + \brief Runs the mDiff wrapper on a list of images in parallel. + + \param diff_tbl Table containing a list of images that overlap and must calculate by how much they overlap. + \param hdr File containing information on how the final mosaic will be set up. + \return diff_imgs[] List of the difference images that must have correction coefficients calculated. +*/ +( Image diff_imgs[] ) mDiffBatch( Table diff_tbl, MosaicData hdr ) +{ + DiffStruct diffs[] ; + + foreach d_entry, i in diffs + { + /* Not always going to be in proj_dir. Maybe read diff_tbl for directory. */ + /* Not sure if the information is there. */ + Image proj_1 ; + Image proj_2 ; + + Image diff_img ; + diff_img = mDiff( proj_1, proj_2, hdr ); + + diff_imgs[ i ] = diff_img; + } +} + + +/*! \fn app Table mFitBatch( Table diff_tbl, Image diff_imgs[] ) + \brief Generates a corrected image from imgs and the information from corr_tbl + + \param diff_tbl + \param diff_imgs + \return fits_tbl +*/ +( Table fits_tbl ) mFitBatch( Table diff_tbl, Image diff_imgs[] ) +{ + app + { + mFitExec @diff_tbl @fits_tbl @dirname( diff_imgs[0] ); + } +} + + +/*! \fn Image mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) + \brief Projects the raw images from raw_imgs to a new plane that was declared in hdr. + + \param raw_imgs[] List of raw images that need to be projected to new plane. + \param hdr File containing information on how the final mosaic will be set up. + \return proj_imgs[] List of newly reprojected images. +*/ +( Image proj_imgs[] ) mProjectPPBatch( Image raw_imgs[], MosaicData hdr ) +{ + foreach img, i in raw_imgs + { + Image proj_img ; + + proj_img = mProjectPP( img, hdr ); + + proj_imgs[ i ] = proj_img; + } +} Copied: SwiftApps/Montage/scripts/Swift_Montage_Types.swift (from rev 3430, SwiftApps/Montage/Swift_Montage_Types.swift) =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Types.swift (rev 0) +++ SwiftApps/Montage/scripts/Swift_Montage_Types.swift 2010-07-12 19:26:40 UTC (rev 3432) @@ -0,0 +1,50 @@ +/*! \file Swift_Montage_Types.swift + \brief File containing declared Swift types for the Swift Montage wrappers +*/ + +/*! \typdef type Image + \brief Declared Image Swift type. +*/ +type Image; + + +/*! \typdef type MosaicData + \brief Declared MosaicData Swift type. +*/ +type MosaicData; + + +/*! \typdef type Table + \brief Declared Table Swift type. +*/ +type Table; + + +/*! \typdef type JPEG + \brief Declared JPEG Swift type. +*/ +type JPEG; + +/*! \struct BackgroundStruct + \brief Stuct to hold data found in the tables generated my mImgtbl and mBgModel +*/ +type BackgroundStruct +{ + string fname; + float a; + float b; + float c; +}; + + +/*! \struct DiffStruct + \brief Stuct to hold data found in the table that is generated from the mOverlaps wrapper. +*/ +type DiffStruct +{ + int cntr1; + int cntr2; + Image plus; + Image minus; + Image diff; +}; Deleted: SwiftApps/Montage/template.hdr =================================================================== --- SwiftApps/Montage/template.hdr 2010-07-11 03:38:07 UTC (rev 3431) +++ SwiftApps/Montage/template.hdr 2010-07-12 19:26:40 UTC (rev 3432) @@ -1,16 +0,0 @@ -SIMPLE = T -BITPIX = -64 -NAXIS = 2 -NAXIS1 = 2259 -NAXIS2 = 2199 -CTYPE1 = 'RA---TAN' -CTYPE2 = 'DEC--TAN' -CRVAL1 = 210.835222357 -CRVAL2 = 54.367562188 -CDELT1 = -0.000277780 -CDELT2 = 0.000277780 -CRPIX1 = 1130.0000 -CRPIX2 = 1100.0000 -CROTA2 = -0.052834592 -HISTORY = MONTAGE TUTORIAL -END From noreply at svn.ci.uchicago.edu Tue Jul 13 14:43:19 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 13 Jul 2010 14:43:19 -0500 (CDT) Subject: [Swift-commit] r3433 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100713194319.337BA9CCE6@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-13 14:43:19 -0500 (Tue, 13 Jul 2010) New Revision: 3433 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData2.java Log: Fix error message Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData2.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData2.java 2010-07-12 19:26:40 UTC (rev 3432) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/ReadData2.java 2010-07-13 19:43:19 UTC (rev 3433) @@ -43,12 +43,12 @@ if (pf instanceof AbsFile) { AbsFile af = (AbsFile) pf; if (!af.getProtocol().equalsIgnoreCase("file")) { - throw new ExecutionException("readData only supports local files"); + throw new ExecutionException("readData2 only supports local files"); } readData(dest, af.getPath()); } else { - throw new ExecutionException("readData only supports reading from files"); + throw new ExecutionException("readData2 only supports reading from files"); } } return null; From noreply at svn.ci.uchicago.edu Wed Jul 14 06:03:10 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 14 Jul 2010 06:03:10 -0500 (CDT) Subject: [Swift-commit] r3434 - usertools/swift/swiftconfig/bin Message-ID: <20100714110310.6AB889CD36@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-14 06:03:10 -0500 (Wed, 14 Jul 2010) New Revision: 3434 Modified: usertools/swift/swiftconfig/bin/swiftconfig Log: interactive editor, auths.default, new split directory structure, better error handling Modified: usertools/swift/swiftconfig/bin/swiftconfig =================================================================== --- usertools/swift/swiftconfig/bin/swiftconfig 2010-07-13 19:43:19 UTC (rev 3433) +++ usertools/swift/swiftconfig/bin/swiftconfig 2010-07-14 11:03:10 UTC (rev 3434) @@ -1,132 +1,257 @@ #!/usr/bin/perl -# Find swift configuration files and libraries -if ( $ENV{'SWIFT_HOME'} ) { +use Cwd; +use Cwd qw(abs_path); +use File::Copy; +use File::Path qw(remove_tree); +use 5.010; + +my $swifthome = ''; +if ( $ENV{'SWIFT_HOME'} && -e "$ENV{'SWIFT_HOME'}/bin/swift" ) { $swifthome = $ENV{'SWIFT_HOME'}; } -else { - @path = split( '/', which("swift") ); - @path = splice( @path, 0, $#path - 1 ); - $swifthome = join( '/', @path ); + +my @path = split( '/', abs_path($0) ); +my @path = splice( @path, 0, $#path - 1 ); +my $parent_directory = join( '/', @path ); + +if ( !$swifthome ) { + if ( -e "$parent_directory/bin/swift" ) { + $swifthome = $parent_directory; + } + + else { + @path = split( '/', which("swift") ); + @path = splice( @path, 0, $#path - 1 ); + my $tmp = join( '/', @path ); + if ( -e "$tmp/bin/swift" ) { + $swifthome = $tmp; + } + } } -use FindBin; +use FindBin qw($Bin); use lib "$FindBin::Bin/../lib/perl"; # Use libraries in $swifthome/lib/perl use Getopt::Long; use File::Which qw(which where); use XML::Simple; +use Data::Dumper; # Variables used for translation catalog -my $host = 'localhost'; # Host name -my $name = ''; # Translation name -my $path = "/usr/bin/$name"; # Full path of executable location -my $status = 'INSTALLED'; # Installation status (not used) -my $platform = 'INTEL32::LINUX'; # Platform (not used) -my $profile = 'null'; # Profile entries -my $tcfile = "$swifthome/etc/tc.data"; # Location of TC file +my $host = 'localhost'; # Host name +my $name = ''; # Translation name +my $path = "/usr/bin/$name"; # Full path of executable location +my $status = 'INSTALLED'; # Installation status (not used) +my $platform = 'INTEL32::LINUX'; # Platform (not used) +my $profile = 'null'; # Profile entries # Variables used for sites.xml -my $templatefile = "$swifthome/etc/sites-template.xml"; # Template file -my $template = ''; # Use template entry -my $templates = 0; # List all templates -my $sitesfile = "$swifthome/etc/sites.xml"; # Working sites.xml file -my $entry = ''; # Name of new entry (pool handle) -my $gridftp = ''; # Specify gridftp URL -my $jobuniverse = ''; # Specify jobmanager universe -my $joburl = ''; # Specify jobmanager URL -my $jobmajor = ''; # Jobmanager major value -my $jobminor = ''; # Jobmanager minor value -my $directory = ''; # Work directory -my $exprovider = ''; # Execution provider -my $exmanager = ''; # Execution job manager -my $exurl = ''; # Execution URL -my $remove = ''; # Remove entry from sites.xml -my $modify = ''; # Modify mode +my $templates = 0; # List all templates +my $remove = ''; # Remove entry from sites.xml +my $modify = ''; # Modify mode +my @validproviders = ( "gt4", "local", "pbs", "condor", "ssh", "coaster" ); +my $keyfile = "$ENV{'HOME'}/.ssh/id_rsa"; # Assign variables from command line options GetOptions( - 'host=s' => \$host, - 'name=s' => \$name, - 'path=s' => \$path, - 'status=s' => \$status, - 'platform=s' => \$platform, - 'profile=s' => \$profile, - 'tcfile=s' => \$tcfile, - 'templatefile=s' => \$templatefile, - 'add=s' => \$add, - 'sitesfile=s' => \$sitesfile, - 'gridftp=s' => \$gridftp, - 'jobuniverse=s' => \$jobuniverse, - 'joburl=s' => \$joburl, - 'jobmajor=i' => \$jobmajor, - 'jobminor=i' => \$jobminor, - 'directory=s' => \$directory, - 'exprovider=s' => \$exprovider, - 'exmanager=s' => \$exmanager, - 'exurl=s' => \$exurl, - 'remove=s' => \$remove, - 'templates' => \$templates, - 'modify=s' => \$modify, - 'key=s' => \$key, - 'namespace=s' => \$namespace, - 'value=s' => \$value + 'add=s' => \$add, + 'remove=s' => \$remove, + 'templates' => \$templates, + 'modify=s' => \$modify, + 'swifthome=s' => \$swifthome, + 'sites' => \$sites ); +# Create a new directory if it does not exist +sub create_directory { + $directory = $_[0]; + if ( !-d "$directory" ) { + mkdir "$directory", 0700 + or die "Unable to create directory $directory\n"; + } +} + +# Process keyboard input +sub getEntry { + my ($description, $value, @valid) = @_; + + print "$description [$value]: "; + my $newvalue = ""; + chomp($newvalue = ); + if ($newvalue) { $value = $newvalue; } + my $isvalid = 0; + + if(@valid) { + foreach(@valid) { + if($_ eq $value){ $isvalid = 1; } + } + if(!$isvalid) { + my $msg = ""; + foreach(@valid) { $msg .= $_ . " "; } + print "Invalid value selected. Please select from: $msg\n"; + $value = getEntry($description, $value, @valid); + } + } + + return $value; +} + # Prepare data -$xml = new XML::Simple(); -$template_data = $xml->XMLin( - "$templatefile", - ForceArray => [qw(workdirectory pool profile)], - KeyAttr => [] -); -$sites_data = $xml->XMLin( - "$sitesfile", - ForceArray => [qw(workdirectory pool profile)], - KeyAttr => [] -); -open( TCFILESTREAM, "$tcfile" ) or die("Unable to open tc file $tcfile!"); - at tcdata = ; -close(TCFILESTREAM); +my $xml = new XML::Simple(); +# Create all required directories +my $dotswift = "$ENV{'HOME'}/.swift"; +create_directory("$ENV{'HOME'}/.ssh"); +create_directory("$dotswift"); +create_directory("$dotswift/sites"); + +# Determine the template directory +my @path = split( '/', abs_path($0) ); +my @path = splice( @path, 0, $#path - 1 ); +my $template_directory = join( '/', @path ) . "/etc/sites"; + +# Add new entry to auths.default +sub add_ssh_auth +{ + my ($entry, $site) = @_; + + # Open authfile + my $authfile = "$dotswift/sites/$entry/auth.defaults"; + create_directory("$dotswift/sites/$entry"); + if ( -e "$authfile" ) { open( AUTHFILE, "$authfile" ) || die "Unable to open auth file $authfile for reading!\n"; } + else { open( AUTHFILE, ">$authfile" ) || die "Unable to open $authfile for writing!\n"; } + @authdata = ; + close(AUTHFILE); + + # Create RSA keypair if needed + if ( !-e "$keyfile" ) { + system("ssh-keygen -t rsa -f $keyfile"); + } + + # Get existing username and passphrase values + my $username; + my $passphrase; + foreach (@authdata) { + if (/username/i) { (my $blah, $username) = split('=', $_); chomp($username); } + if (/passphrase/i) { (my $blah, $passphrase) = split('=', $_); chomp($passphrase); } + } + if(!$username){ $username = getlogin(); } + + $username = getEntry("Username", $username); + $passphrase = getEntry("Passphrase", $passphrase); + + # Remove old auth.defaults if it already exists + if(-e "$authfile") { + unlink($authfile) || die "Unable to remove $authfile\n"; + @authdata = (); + } + + # Add data to new auth.defaults + push( @authdata, "$site.type=key\n" ); + push( @authdata, "$site.username=$username\n" ); + push( @authdata, "$site.key=$keyfile\n" ); + push( @authdata, "$site.passphrase=$passphrase\n" ); + write_file( $authfile, @authdata ); +} + +# Update TC with correct hostname +sub update_tc_hostname { + my ($filename, $host) = @_; + # Store TC data + open( TCFILESTREAM, "$filename" ) or die("Unable to open tc file $tcfile!"); + @tcdata = ; + close(TCFILESTREAM); + + foreach $line (@tcdata) { + # Ignore comments + if ( substr( $line, 0, 1 ) eq '#' ) { + next; + } + + # Replace old entry with new entry + my ( $tmphost, $tmpname, $tmppath, $tmpstatus, $tmpplatform, $tmpprofile ) = split( /\s+/, $line ); + $tmphost = $host; + $line = "$tmphost\t$tmpname\t$tmppath\t$tmpstatus\t$tmpplatform\t$tmpprofile\n"; + } + + write_file( $filename, @tcdata); +} + # Update XML hash with values from command line sub update_xml_entry { - my $xml_data = @_[0]; - my $name = @_[1]; - foreach my $e ( @{ $xml_data->{pool} } ) { - if ( $name eq $e->{handle} ) { - $e->{workdirectory}[0] =~ s/\$HOME/$ENV{'HOME'}/; - if ($gridftp) { $e->{gridftp}{url} = $gridftp; } - if ($jobuniverse) { $e->{jobmanager}{universe} = $jobuniverse; } - if ($joburl) { $e->{jobmanager}{url} = $joburl; } - if ($jobmajor) { $e->{jobmanager}{major} = $jobmajor; } - if ($jobminor) { $e->{jobmanager}{minor} = $jobminor; } - if ($directory) { $e->{workdirectory} = [$directory]; } - if ($exprovider) { $e->{execution}{provider} = $exprovider; } - if ($exmanager) { $e->{execution}{jobmanager} = $exmanager; } - if ($exurl) { $e->{execution}{url} = $exurl; } - if ( $key && $namespace && $value ) { - my $found = 0; - foreach my $profile ( @{ $e->{profile} } ) { - if ( $profile->{key} eq $key ) { - $profile->{key} = $key; - $profile->{namespace} = $namespace; - $profile->{content} = $value; - $found = 1; - } - } - if ( !$found ) { - push @{ $e->{profile} }, - { - key => $key, - namespace => $namespace, - content => $value - }; - } + my ($filename) = @_; + if ( !-e $filename ) { return 0; } + my $entryname = ""; + my $tcfile = ""; + + my $xml_data = $xml->XMLin( + $filename, + ForceArray => [qw(workdirectory profile)], + KeyAttr => [] + ); + + # Renaming + if($modify) + { + $entryname = getEntry("Site Entry Name", $modify); + $tcfile = "$dotswift/sites/$modify/tc.data"; + print "tcfile: $tcfile\n"; + if($entryname ne $modify) { + if(!-d "$dotswift/sites/$entryname" && -d "$dotswift/sites/$modify") { + move("$dotswift/sites/$modify", "$dotswift/sites/$entryname") || die "Unable to rename $dotswift/sites/$modify to $dotsite/sites/$entryname"; } - return $e; + $xml_data = $xml->XMLin("$dotswift/sites/$entryname/sites.xml", ForceArray => [qw(workdirectory profile)], KeyAttr => []); + $tcfile = "$dotswift/sites/$entryname/tc.data"; } } + + # Adding + if($add){ + $entryname = getEntry("Site Entry Name", $add); + if( -e "$dotswift/sites/$entryname/sites.xml") { + die "Entry file for $entryname already exists. Use -modify to change settings\n"; + } + create_directory("$dotswift/sites/$entryname"); + if(-e "$template_directory/$add/tc.data") { $tcfile = "$template_directory/$add/tc.data"; } + elsif (-e "$dotswift/sites/$add/tc.data") { $tcfile = "$dotswift/sites/$add/tc.data"; } + copy("$tcfile", "$dotswift/sites/$entryname") || die "Unable to copy $tcfile to $dotswift/sites/$entryname\n"; + } + + $xml_data->{pool}{handle} = $entryname; + if(!-d "$dotswift/sites/$entryname") { + create_directory("$dotswift/sites/$entryname"); + } + $filename = "$dotswift/sites/$entryname/sites.xml"; + + # Replace values if requested by user + $xml_data->{pool}{workdirectory}[0] =~ s/\$HOME/$ENV{'HOME'}/; + if ( $xml_data->{pool}{gridftp} ) { $xml_data->{pool}{gridftp} = getEntry( "GridFTP URL", $xml_data->{pool}{gridftp}{url} ); } + if ( $xml_data->{pool}{workdirectory} ) { $xml_data->{pool}{workdirectory} = [ getEntry( "Work Directory", $xml_data->{pool}{workdirectory}[0] ) ]; } + if ( $xml_data->{pool}{jobmanager} ) { + if ( $xml_data->{pool}{jobmanager}{universe} ) { $xml_data->{pool}{jobmanager}{universe} = getEntry( "Job Universe", $xml_data->{pool}{jobmanager}{universe} ); } + if ( $xml_data->{pool}{jobmanager}{url} ) { $xml_data->{pool}{jobmanager}{url} = getEntry( "Job Manager URL", $xml_data->{pool}{jobmanager}{url} ); } + if ( $xml_data->{pool}{jobmanager}{major} ) { $xml_data->{pool}{jobmanager}{major} = getEntry( "Job Major Number", $xml_data->{pool}{jobmanager}{major} ); } + if ( $xml_data->{pool}{jobmanager}{minor} ) { $xml_data->{pool}{jobmanager}{minor} = getEntry( "Job Minor Number", $xml_data->{pool}{jobmanager}{minor} ); } + } + if ( $xml_data->{pool}{execution} ) { + if ( $xml_data->{pool}{execution}{provider} ) { $xml_data->{pool}{execution}{provider} = getEntry( "Execution Provider", $xml_data->{pool}{execution}{provider}, @validproviders); } + if ( $xml_data->{pool}{execution}{jobmanager} ) { $xml_data->{pool}{execution}{jobmanager} = getEntry( "Execution Job Manager", $xml_data->{pool}{execution}{jobmanager} ); } + if ( $xml_data->{pool}{execution}{url} ) { + my $previous_site = $xml_data->{pool}{execution}{url}; + $xml_data->{pool}{execution}{url} = getEntry( "Execution URL", $xml_data->{pool}{execution}{url} ); + update_tc_hostname("$tcfile", $xml_data->{pool}{execution}{url}); + } + } + if ( $xml_data->{pool}{filesystem} ) { + if ( $xml_data->{pool}{filesystem}{provider} ) { $xml_data->{pool}{filesystem}{provider} = getEntry( "Filesystem Provider", $xml_data->{pool}{filesystem}{provider}, @validproviders); } + if ( $xml_data->{pool}{filesystem}{url} ) { $xml_data->{pool}{filesystem}{url} = getEntry( "Filesystem URL", $xml_data->{pool}{filesystem}{url} ); } + } + if ( $xml_data->{pool}{execution}{provider} eq "ssh") { + add_ssh_auth($entryname, $xml_data->{pool}{execution}{url}); + } + + write_file( $filename, $xml->XMLout( $xml_data, RootName => 'config', SuppressEmpty => 1 ) ); } # Write a file given variable and filename @@ -137,92 +262,56 @@ close(TEMPFILESTREAM); } +# Print all files in a directory +sub print_directory { + my ($template_directory) = @_; + chdir($template_directory) || die "Unable to change directories to $template_directory\n"; + my @files = <*>; + foreach $file(@files) + { + (my $basename, my $ext) = split(/\./, $file); + my @path = split( '/', $basename); + print "$basename\n"; + } +} + # If a template is specified, find the correct one and modify as needed if ($add) { - my $data = update_xml_entry( $template_data, $add ); - - # Modify existing entry - my $isfound = 0; - foreach my $e ( @{ $sites_data->{pool} } ) { - if ( $add eq $e->{handle} ) { - $e = $data; - $isfound = 1; - } - } - - # Add new entry - if ( !$isfound ) { - push @{ $sites_data->{pool} }, $data; - } - - # Write to sites file - write_file( $sitesfile, $xml->XMLout( $sites_data, RootName => 'config' ) ); + my $data = 0; + $data = update_xml_entry("$template_directory/$add/sites.xml"); + if ( $data == 0 ) { $data = update_xml_entry("$dotswift/sites/$add/sites.xml"); } + if ( $data == 0 ) { die "Unable to find template for $add\n"; } } # Remove an entry if ($remove) { - # Look first for a site to remove - $isfound = 0; + if(!-d "$dotswift/sites/$remove") { + "Unable to find site entry for $remove\n"; + } - foreach my $e ( @{ $sites_data->{pool} } ) { - if ( $remove eq $e->{handle} ) { - undef $e; - $isfound = 1; - } - } - - # Write to sites file - if ($isfound) { - write_file( - $sitesfile, - $xml->XMLout( - $sites_data, - RootName => 'config', - SuppressEmpty => 1 - ) - ); - } - - # If no site found, check for a TC entry - else { - foreach $line (@tcdata) { - - # Ignore comments - if ( substr( $line, 0, 1 ) eq '#' ) { - next; - } - ( $tmphost, $tmpname, @junk ) = split( /\s+/, $line ); - - # Replace old entry with new entry - if ( $remove eq $tmpname ) { - $line = ""; - } - } - write_file( $tcfile, @tcdata ); - } + remove_tree("$dotswift/sites/$remove") || die "Unable to remove directory $dotswift/sites/$remove\n"; } # List all available templates if ($templates) { - foreach my $e ( @{ $template_data->{pool} } ) { - print $e->{handle} . "\n"; - } + print_directory($template_directory); } +# List all user-added sites +if($sites) { + print_directory("$dotswift/sites"); +} + # Modify a site entry if ($modify) { - my $data = update_xml_entry( $sites_data, $modify ); - # Modify entry if one already exists - foreach my $e ( @{ $sites_data->{pool} } ) { - if ( $modify eq $e->{handle} ) { - $e = $data; - } + if(!-e "$dotswift/sites/$modify/sites.xml") { + die "Unable to find entry for $modify\n"; } - write_file( $sitesfile, - $xml->XMLout( $sites_data, RootName => 'config', SuppressEmpty => 1 ) ); + my $data = update_xml_entry( "$dotswift/sites/$modify/sites.xml" ); + } # Add or modify a TC entry @@ -252,6 +341,7 @@ } __END__ + =head1 NAME swiftconfig - Utility for managing Swift configuration @@ -269,35 +359,9 @@ General operations: -add sitename add a site from template -remove site removes a site from sites.xml - -remove command removes a command from the catalog -templates display all available sites in template -modify site Specifies the name of a site to modify -Translation catalog settings: - -host hostname hostname of the translation catalog entry - -name name translation name - -path path full pathname to location of program - -status status installation status (deprecated) - -profile setting define the profile value for an entry - -tcfile filename explicitly specify a translation file - -Sites settings: - -templatefile file explicitly set the template file to use - -sitesfile file explicitly set the sites file to use - -gridftp GridFTPURL GridFTP URL - -jobuniverse universe job manager universe - -joburl URL job manager URL - -jobmajor major job mager number - -jobminor minor job minor number - -directory dir work directory - -exprovider name execution provider - -exmanager name execution job manager - -exurl URL execution URL - -key key profile key - -value value profile value - -namespace name profile namespace - - =head1 EXAMPLES List all templates available for adding @@ -307,20 +371,11 @@ swiftconfig -add teraport Modify the work directory of a site - swiftconfig -modify teraport -directory /var/tmp + swiftconfig -modify teraport Remove a site swiftconfig -remove teraport -Add a new command to translation catalog - swiftconfig -name convert -path /usr/local/bin/convert - -Modify an existing command in the translation catalog - swiftconfig -name convert -path /usr/bin/convert - -Remove a command from the translation catalog - swiftconfig -remove convert - =head1 CAVEATS Swiftconfig will attempt to automatically determine the location of swift configuration files. It first checks for an environment variable called $SWIFT_HOME. If that is not found, it will look for the location of "swift" in the path, and try to find the configuration files from there. This default behavior can be overwridden by manually specifying the location of files with -templatefile, -sitesfile, and -tcfile. From noreply at svn.ci.uchicago.edu Wed Jul 14 06:08:40 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 14 Jul 2010 06:08:40 -0500 (CDT) Subject: [Swift-commit] r3435 - in usertools/swift/swiftconfig/etc: . sites sites/multisite-pbs-coasters sites/multisite-ssh sites/pads-local-pbs sites/pads-local-pbs-coasters sites/pads-remote-pbs-coasters-ssh sites/pads-remote-ssh sites/teraport-local-pbs sites/teraport-local-pbs-coasters sites/teraport-remote-pbs-coasters-ssh sites/teraport-remote-ssh Message-ID: <20100714110840.6BDBA9CD36@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-14 06:08:40 -0500 (Wed, 14 Jul 2010) New Revision: 3435 Added: usertools/swift/swiftconfig/etc/sites/ usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/ usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/sites.xml usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/tc.data usertools/swift/swiftconfig/etc/sites/multisite-ssh/ usertools/swift/swiftconfig/etc/sites/multisite-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/multisite-ssh/tc.data usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/ usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/tc.data usertools/swift/swiftconfig/etc/sites/pads-local-pbs/ usertools/swift/swiftconfig/etc/sites/pads-local-pbs/sites.xml usertools/swift/swiftconfig/etc/sites/pads-local-pbs/tc.data usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/ usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/tc.data usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/ usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/tc.data usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/sites.xml usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/tc.data usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/sites.xml usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/tc.data usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/ usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/tc.data usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/ usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/tc.data Log: new templates Added: usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,15 @@ + + + + 3000 + 8 + 1 + 1 + 10 + short + 0.5 + 10000 + + $HOME/swiftwork + + Added: usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/multisite-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/multisite-ssh/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/multisite-ssh/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,8 @@ + + + + + $HOME/swiftwork + 0 + + Added: usertools/swift/swiftconfig/etc/sites/multisite-ssh/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/multisite-ssh/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/multisite-ssh/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +mct162s01.cs.ship.edu echo /bin/echo INSTALLED INTEL32::LINUX null +mct162s01.cs.ship.edu cat /bin/cat INSTALLED INTEL32::LINUX null +mct162s01.cs.ship.edu ls /bin/ls INSTALLED INTEL32::LINUX null +mct162s01.cs.ship.edu grep /bin/grep INSTALLED INTEL32::LINUX null +mct162s01.cs.ship.edu sort /bin/sort INSTALLED INTEL32::LINUX null +mct162s01.cs.ship.edu paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/pads-local-pbs/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,8 @@ + + + + + $HOME/swiftwork + 0 + + Added: usertools/swift/swiftconfig/etc/sites/pads-local-pbs/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,15 @@ + + + + + $HOME/swiftwork + 3000 + 8 + 1 + 1 + 10 + short + 0.5 + 10000 + + Added: usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,15 @@ + + + + 3000 + 8 + 1 + 1 + 10 + short + 0.5 + 10000 + + $HOME/swiftwork + + Added: usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,8 @@ + + + + + $HOME/swiftwork + 0 + + Added: usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,9 @@ + + + + + $HOME/swiftwork + 0 + + + Added: usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +none echo /bin/echo INSTALLED INTEL32::LINUX null +none cat /bin/cat INSTALLED INTEL32::LINUX null +none ls /bin/ls INSTALLED INTEL32::LINUX null +none grep /bin/grep INSTALLED INTEL32::LINUX null +none sort /bin/sort INSTALLED INTEL32::LINUX null +none paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,16 @@ + + + + + $HOME/swiftwork + 3000 + 8 + 1 + 1 + 10 + short + 0.5 + 10000 + + + Added: usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,15 @@ + + + + 3000 + 8 + 1 + 1 + 10 + short + 0.5 + 10000 + + $HOME/swiftwork + + Added: usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null Added: usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/sites.xml 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,9 @@ + + + + + $HOME/swiftwork + 0 + + + Added: usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/tc.data 2010-07-14 11:08:40 UTC (rev 3435) @@ -0,0 +1,17 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null From noreply at svn.ci.uchicago.edu Thu Jul 15 11:56:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Jul 2010 11:56:25 -0500 (CDT) Subject: [Swift-commit] r3436 - trunk/libexec Message-ID: <20100715165625.622839CC86@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-15 11:56:25 -0500 (Thu, 15 Jul 2010) New Revision: 3436 Modified: trunk/libexec/_swiftwrap Log: Remove debugging line Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-07-14 11:08:40 UTC (rev 3435) +++ trunk/libexec/_swiftwrap 2010-07-15 16:56:25 UTC (rev 3436) @@ -248,8 +248,6 @@ COMMANDLINE=$@ -echo $0 $COMMANDLINE >> /tmp/swiftwrap.out - # get the parent directory of the directory containing _swiftwrap, to use # as the run directory # this assumes that _swiftwrap is being executed from the top level of From noreply at svn.ci.uchicago.edu Thu Jul 15 13:59:33 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Jul 2010 13:59:33 -0500 (CDT) Subject: [Swift-commit] r3437 - trunk/libexec/log-processing Message-ID: <20100715185933.0E7B89CD36@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-15 13:59:32 -0500 (Thu, 15 Jul 2010) New Revision: 3437 Modified: trunk/libexec/log-processing/makefile Log: Fix for swift-plot-log generation of execstages.png Modified: trunk/libexec/log-processing/makefile =================================================================== --- trunk/libexec/log-processing/makefile 2010-07-15 16:56:25 UTC (rev 3436) +++ trunk/libexec/log-processing/makefile 2010-07-15 18:59:32 UTC (rev 3437) @@ -109,7 +109,7 @@ jobs.retrycount.summary: execute2.event cat execute2.event | cut -f 5 -d ' ' | sort | uniq -c | sort | sed 's/^ *\(.*\) .*$$/\1/' | uniq -c > jobs.retrycount.summary -execstages.png: execute2.event dostagein.event dostageout.event start-time.tmp +execstages.png: execute2.event dostagein.event dostageout.event start-time.tmp workflow.event execstages-plot # weight processing From noreply at svn.ci.uchicago.edu Thu Jul 15 15:11:56 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Jul 2010 15:11:56 -0500 (CDT) Subject: [Swift-commit] r3438 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100715201156.158B39CD36@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-15 15:11:55 -0500 (Thu, 15 Jul 2010) New Revision: 3438 Added: trunk/src/org/griphyn/vdl/karajan/lib/AppStageins.java trunk/src/org/griphyn/vdl/karajan/lib/AppStageouts.java Log: added appStage(in|out) functions Added: trunk/src/org/griphyn/vdl/karajan/lib/AppStageins.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/AppStageins.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/AppStageins.java 2010-07-15 20:11:55 UTC (rev 3438) @@ -0,0 +1,54 @@ +/* + * Created on Jan 5, 2007 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.LinkedList; +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.arguments.ArgUtil; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.griphyn.vdl.mapping.AbsFile; + +public class AppStageins extends AbstractSequentialWithArguments { + + public static final Arg JOBID = new Arg.Positional("jobid"); + public static final Arg FILES = new Arg.Positional("files"); + public static final Arg DIR = new Arg.Positional("dir"); + public static final Arg STAGING_METHOD = new Arg.Positional("stagingMethod"); + public static final Arg.Channel STAGEIN = new Arg.Channel("stagein"); + + static { + setArguments(AppStageins.class, new Arg[] { JOBID, FILES, DIR, + STAGING_METHOD }); + } + + protected void post(VariableStack stack) throws ExecutionException { + List files = TypeUtil.toList(FILES.getValue(stack)); + for (Object f : files) { + AbsFile file = new AbsFile(TypeUtil.toString(f)); + String protocol = file.getProtocol(); + if (protocol.equals("file")) { + protocol = TypeUtil.toString(STAGING_METHOD.getValue(stack)); + } + String path = file.getDir().equals("") ? file.getName() : file.getDir() + + "/" + file.getName(); + String relpath = path.startsWith("/") ? path.substring(1) : path; + ArgUtil.getChannelReturn(stack, STAGEIN).append( + makeList(protocol + "://" + file.getHost() + "/" + path, TypeUtil.toString(DIR.getValue(stack)) + + "/" + relpath)); + } + super.post(stack); + } + + private List makeList(String s1, String s2) { + List l = new LinkedList(); + l.add(s1); + l.add(s2); + return l; + } +} Added: trunk/src/org/griphyn/vdl/karajan/lib/AppStageouts.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/AppStageouts.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/AppStageouts.java 2010-07-15 20:11:55 UTC (rev 3438) @@ -0,0 +1,66 @@ +/* + * Created on Jan 5, 2007 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.LinkedList; +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.arguments.ArgUtil; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.griphyn.vdl.mapping.AbsFile; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; + +public class AppStageouts extends AbstractSequentialWithArguments { + + public static final Arg JOBID = new Arg.Positional("jobid"); + public static final Arg FILES = new Arg.Positional("files"); + public static final Arg DIR = new Arg.Positional("dir"); + public static final Arg STAGING_METHOD = new Arg.Positional("stagingMethod"); + public static final Arg VAR = new Arg.Optional("var", null); + public static final Arg.Channel STAGEOUT = new Arg.Channel("stageout"); + + static { + setArguments(AppStageouts.class, new Arg[] { JOBID, FILES, DIR, + STAGING_METHOD, VAR }); + } + + protected void post(VariableStack stack) throws ExecutionException { + try { + List files = TypeUtil.toList(FILES.getValue(stack)); + for (Object f : files) { + List pv = TypeUtil.toList(f); + Path p = Path.parse(TypeUtil.toString(pv.get(0))); + DSHandle handle = (DSHandle) pv.get(1); + ArgUtil.getNamedArguments(stack).add("var", handle.getField(p)); + AbsFile file = new AbsFile(VDLFunction.filename(stack)[0]); + String protocol = file.getProtocol(); + if (protocol.equals("file")) { + protocol = TypeUtil.toString(STAGING_METHOD.getValue(stack)); + } + String path = file.getDir().equals("") ? file.getName() : file.getDir() + + "/" + file.getName(); + String relpath = path.startsWith("/") ? path.substring(1) : path; + ArgUtil.getChannelReturn(stack, STAGEOUT).append( + makeList(TypeUtil.toString(DIR.getValue(stack)) + "/" + relpath, + protocol + "://" + file.getHost() + "/" + path)); + } + super.post(stack); + } + catch (Exception e) { + throw new ExecutionException(e); + } + } + + private List makeList(String s1, String s2) { + List l = new LinkedList(); + l.add(s1); + l.add(s2); + return l; + } +} From noreply at svn.ci.uchicago.edu Thu Jul 15 15:13:00 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Jul 2010 15:13:00 -0500 (CDT) Subject: [Swift-commit] r3439 - trunk/libexec Message-ID: <20100715201300.7A11E9CD36@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-15 15:13:00 -0500 (Thu, 15 Jul 2010) New Revision: 3439 Modified: trunk/libexec/vdl-lib.xml Log: definitions for appStage* Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-07-15 20:11:55 UTC (rev 3438) +++ trunk/libexec/vdl-lib.xml 2010-07-15 20:13:00 UTC (rev 3439) @@ -101,6 +101,9 @@ + + + From noreply at svn.ci.uchicago.edu Thu Jul 15 15:13:45 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 15 Jul 2010 15:13:45 -0500 (CDT) Subject: [Swift-commit] r3440 - trunk/libexec Message-ID: <20100715201345.3E9C19CD36@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-15 15:13:45 -0500 (Thu, 15 Jul 2010) New Revision: 3440 Modified: trunk/libexec/vdl-int-staging.k Log: removed the karajan versions of appStage* Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-07-15 20:13:00 UTC (rev 3439) +++ trunk/libexec/vdl-int-staging.k 2010-07-15 20:13:45 UTC (rev 3440) @@ -91,42 +91,6 @@ ) ) ) - - element(appStageins, [jobid, files, dir, stagingMethod] - for(file, files - protocol := vdl:provider(file) - provider := if(protocol == "file", stagingMethod, protocol) - srchost := hostname(file) - srcdir := vdl:dirname(file) - destdir := dircat(dir, reldirname(file)) - filename := basename(file) - - stageIn( - "{provider}://{srchost}/{srcdir}/{filename}", - "{destdir}/{filename}" - ) - ) - ) - - element(appStageouts, [jobid, stageouts, dir, stagingMethod] - for(pv, stageouts - [path, var] := each(pv) - file := vdl:absfilename(vdl:getfield(var, path = path)) - protocol := vdl:provider(file) - provider := if(protocol == "file", stagingMethod, protocol) - dhost := vdl:hostname(file) - rdir := dircat(dir, reldirname(file)) - bname := basename(file) - ldir := vdl:dirname(file) - fullLocal := dircat(ldir, bname) - fullRemote := dircat(rdir, bname) - - stageOut( - "{rdir}/{bname}", - "{provider}://{dhost}/{ldir}/{bname}" - ) - ) - ) element(doRestartlog, [restartouts] uParallelFor(f, restartouts, @@ -179,6 +143,7 @@ replicationGroup, replicationChannel] stagein := list(unique(each(stagein))) stageout := list(unique(each(stageout))) + allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein) ddir := initDDir() @@ -237,8 +202,8 @@ appStageins(jobid, stagein, ".", stagingMethod) stageOut("wrapper.log", "{stagingMethod}://localhost/{ddir}/{jobid}.info") - stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}") - stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}") + //stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}") + //stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}") appStageouts(jobid, stageout, ".", stagingMethod) task:cleanUp(".") //the whole job directory From noreply at svn.ci.uchicago.edu Fri Jul 16 05:50:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 16 Jul 2010 05:50:55 -0500 (CDT) Subject: [Swift-commit] r3441 - in usertools/swift/swiftconfig: bin etc etc/sites/multisite-pbs-coasters etc/sites/multisite-ssh etc/sites/pads-local-pbs etc/sites/pads-local-pbs-coasters etc/sites/pads-remote-pbs-coasters-ssh etc/sites/pads-remote-ssh etc/sites/teraport-local-pbs etc/sites/teraport-local-pbs-coasters etc/sites/teraport-remote-pbs-coasters-ssh etc/sites/teraport-remote-ssh Message-ID: <20100716105055.5470E9CC86@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-16 05:50:55 -0500 (Fri, 16 Jul 2010) New Revision: 3441 Removed: usertools/swift/swiftconfig/etc/sites-template.xml Modified: usertools/swift/swiftconfig/bin/swiftconfig usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/sites.xml usertools/swift/swiftconfig/etc/sites/multisite-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml usertools/swift/swiftconfig/etc/sites/pads-local-pbs/sites.xml usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/sites.xml usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/sites.xml usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/sites.xml Log: Initial version of swiftrun Modified: usertools/swift/swiftconfig/bin/swiftconfig =================================================================== --- usertools/swift/swiftconfig/bin/swiftconfig 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/bin/swiftconfig 2010-07-16 10:50:55 UTC (rev 3441) @@ -4,7 +4,6 @@ use Cwd qw(abs_path); use File::Copy; use File::Path qw(remove_tree); -use 5.010; my $swifthome = ''; if ( $ENV{'SWIFT_HOME'} && -e "$ENV{'SWIFT_HOME'}/bin/swift" ) { @@ -196,7 +195,6 @@ { $entryname = getEntry("Site Entry Name", $modify); $tcfile = "$dotswift/sites/$modify/tc.data"; - print "tcfile: $tcfile\n"; if($entryname ne $modify) { if(!-d "$dotswift/sites/$entryname" && -d "$dotswift/sites/$modify") { move("$dotswift/sites/$modify", "$dotswift/sites/$entryname") || die "Unable to rename $dotswift/sites/$modify to $dotsite/sites/$entryname"; @@ -216,42 +214,47 @@ if(-e "$template_directory/$add/tc.data") { $tcfile = "$template_directory/$add/tc.data"; } elsif (-e "$dotswift/sites/$add/tc.data") { $tcfile = "$dotswift/sites/$add/tc.data"; } copy("$tcfile", "$dotswift/sites/$entryname") || die "Unable to copy $tcfile to $dotswift/sites/$entryname\n"; + + if($entryname ne $add) { + update_tc_hostname("$dotswift/sites/$entryname/tc.data", $entryname); + } + } - $xml_data->{pool}{handle} = $entryname; + $xml_data->{handle} = $entryname; if(!-d "$dotswift/sites/$entryname") { create_directory("$dotswift/sites/$entryname"); } $filename = "$dotswift/sites/$entryname/sites.xml"; # Replace values if requested by user - $xml_data->{pool}{workdirectory}[0] =~ s/\$HOME/$ENV{'HOME'}/; - if ( $xml_data->{pool}{gridftp} ) { $xml_data->{pool}{gridftp} = getEntry( "GridFTP URL", $xml_data->{pool}{gridftp}{url} ); } - if ( $xml_data->{pool}{workdirectory} ) { $xml_data->{pool}{workdirectory} = [ getEntry( "Work Directory", $xml_data->{pool}{workdirectory}[0] ) ]; } - if ( $xml_data->{pool}{jobmanager} ) { - if ( $xml_data->{pool}{jobmanager}{universe} ) { $xml_data->{pool}{jobmanager}{universe} = getEntry( "Job Universe", $xml_data->{pool}{jobmanager}{universe} ); } - if ( $xml_data->{pool}{jobmanager}{url} ) { $xml_data->{pool}{jobmanager}{url} = getEntry( "Job Manager URL", $xml_data->{pool}{jobmanager}{url} ); } - if ( $xml_data->{pool}{jobmanager}{major} ) { $xml_data->{pool}{jobmanager}{major} = getEntry( "Job Major Number", $xml_data->{pool}{jobmanager}{major} ); } - if ( $xml_data->{pool}{jobmanager}{minor} ) { $xml_data->{pool}{jobmanager}{minor} = getEntry( "Job Minor Number", $xml_data->{pool}{jobmanager}{minor} ); } + $xml_data->{workdirectory}[0] =~ s/\$HOME/$ENV{'HOME'}/; + if ( $xml_data->{gridftp} ) { $xml_data->{gridftp}{url} = getEntry( "GridFTP URL", $xml_data->{gridftp}{url} ); } + if ( $xml_data->{workdirectory} ) { $xml_data->{workdirectory} = [ getEntry( "Work Directory", $xml_data->{workdirectory}[0] ) ]; } + if ( $xml_data->{jobmanager} ) { + if ( $xml_data->{jobmanager}{universe} ) { $xml_data->{jobmanager}{universe} = getEntry( "Job Universe", $xml_data->{jobmanager}{universe} ); } + if ( $xml_data->{jobmanager}{url} ) { $xml_data->{jobmanager}{url} = getEntry( "Job Manager URL", $xml_data->{jobmanager}{url} ); } + if ( $xml_data->{jobmanager}{major} ) { $xml_data->{jobmanager}{major} = getEntry( "Job Major Number", $xml_data->{jobmanager}{major} ); } + if ( $xml_data->{jobmanager}{minor} ) { $xml_data->{jobmanager}{minor} = getEntry( "Job Minor Number", $xml_data->{jobmanager}{minor} ); } } - if ( $xml_data->{pool}{execution} ) { - if ( $xml_data->{pool}{execution}{provider} ) { $xml_data->{pool}{execution}{provider} = getEntry( "Execution Provider", $xml_data->{pool}{execution}{provider}, @validproviders); } - if ( $xml_data->{pool}{execution}{jobmanager} ) { $xml_data->{pool}{execution}{jobmanager} = getEntry( "Execution Job Manager", $xml_data->{pool}{execution}{jobmanager} ); } - if ( $xml_data->{pool}{execution}{url} ) { - my $previous_site = $xml_data->{pool}{execution}{url}; - $xml_data->{pool}{execution}{url} = getEntry( "Execution URL", $xml_data->{pool}{execution}{url} ); - update_tc_hostname("$tcfile", $xml_data->{pool}{execution}{url}); + if ( $xml_data->{execution} ) { + if ( $xml_data->{execution}{provider} ) { $xml_data->{execution}{provider} = getEntry( "Execution Provider", $xml_data->{execution}{provider}, @validproviders); } + if ( $xml_data->{execution}{jobmanager} ) { $xml_data->{execution}{jobmanager} = getEntry( "Execution Job Manager", $xml_data->{execution}{jobmanager} ); } + if ( $xml_data->{execution}{url} ) { + my $previous_site = $xml_data->{execution}{url}; + $xml_data->{execution}{url} = getEntry( "Execution URL", $xml_data->{execution}{url} ); + update_tc_hostname("$tcfile", $xml_data->{execution}{url}); } } - if ( $xml_data->{pool}{filesystem} ) { - if ( $xml_data->{pool}{filesystem}{provider} ) { $xml_data->{pool}{filesystem}{provider} = getEntry( "Filesystem Provider", $xml_data->{pool}{filesystem}{provider}, @validproviders); } - if ( $xml_data->{pool}{filesystem}{url} ) { $xml_data->{pool}{filesystem}{url} = getEntry( "Filesystem URL", $xml_data->{pool}{filesystem}{url} ); } + if ( $xml_data->{filesystem} ) { + if ( $xml_data->{filesystem}{provider} ) { $xml_data->{filesystem}{provider} = getEntry( "Filesystem Provider", $xml_data->{filesystem}{provider}, @validproviders); } + if ( $xml_data->{filesystem}{url} ) { $xml_data->{filesystem}{url} = getEntry( "Filesystem URL", $xml_data->{filesystem}{url} ); } } - if ( $xml_data->{pool}{execution}{provider} eq "ssh") { - add_ssh_auth($entryname, $xml_data->{pool}{execution}{url}); + if ( $xml_data->{execution}{provider} eq "ssh") { + add_ssh_auth($entryname, $xml_data->{execution}{url}); } - write_file( $filename, $xml->XMLout( $xml_data, RootName => 'config', SuppressEmpty => 1 ) ); + write_file( $filename, $xml->XMLout( $xml_data, RootName => 'pool', SuppressEmpty => 1 ) ); } # Write a file given variable and filename Modified: usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,4 +1,3 @@ - 3000 @@ -12,4 +11,3 @@ $HOME/swiftwork - Modified: usertools/swift/swiftconfig/etc/sites/multisite-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/multisite-ssh/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/multisite-ssh/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,8 +1,6 @@ - $HOME/swiftwork 0 - Modified: usertools/swift/swiftconfig/etc/sites/pads-local-pbs/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,8 +1,6 @@ - $HOME/swiftwork 0 - Modified: usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,4 +1,3 @@ - @@ -12,4 +11,3 @@ 0.5 10000 - Modified: usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,4 +1,3 @@ - 3000 @@ -12,4 +11,3 @@ $HOME/swiftwork - Modified: usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,8 +1,6 @@ - $HOME/swiftwork 0 - Modified: usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,9 +1,7 @@ - $HOME/swiftwork 0 - Modified: usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,4 +1,3 @@ - @@ -12,5 +11,4 @@ 0.5 10000 - Modified: usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/teraport-remote-pbs-coasters-ssh/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,4 +1,3 @@ - 3000 @@ -12,4 +11,3 @@ $HOME/swiftwork - Modified: usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/sites.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/sites.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,9 +1,7 @@ - $HOME/swiftwork 0 - Deleted: usertools/swift/swiftconfig/etc/sites-template.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites-template.xml 2010-07-15 20:13:45 UTC (rev 3440) +++ usertools/swift/swiftconfig/etc/sites-template.xml 2010-07-16 10:50:55 UTC (rev 3441) @@ -1,127 +0,0 @@ - - - - - $HOME/swiftwork - 0 - - - - - - $HOME/swiftwork - 0 - - - - - 3000 - 8 - 1 - 1 - 10 - short - 0.5 - 10000 - - $HOME/swiftwork - - - - - 3000 - 8 - 1 - 1 - 10 - short - 0.5 - 10000 - - $HOME/swiftwork - - - - - - $HOME/swiftwork - 3000 - 8 - 1 - 1 - 10 - short - 0.5 - 10000 - - - - - - $HOME/swiftwork - 3000 - 8 - 1 - 1 - 10 - short - 0.5 - 10000 - - - - - - $HOME/swiftwork - 0 - - - - - - $HOME/swiftwork - 0 - - - - - - $HOME/swiftwork - 0 - - - - - - $HOME/swiftwork - 0 - - - - - 3000 - 8 - 1 - 1 - 10 - short - 0.5 - 10000 - - $HOME/swiftwork - - - - - 3000 - 8 - 1 - 1 - 10 - short - 0.5 - 10000 - - $HOME/swiftwork - - From noreply at svn.ci.uchicago.edu Mon Jul 19 13:36:58 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 19 Jul 2010 13:36:58 -0500 (CDT) Subject: [Swift-commit] r3442 - trunk/src/org/griphyn/vdl/engine Message-ID: <20100719183658.5123E9CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-19 13:36:58 -0500 (Mon, 19 Jul 2010) New Revision: 3442 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: Drop unused imports Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-07-16 10:50:55 UTC (rev 3441) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-07-19 18:36:58 UTC (rev 3442) @@ -8,7 +8,6 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.LinkedList; -import java.util.List; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -44,7 +43,6 @@ import org.griphyn.vdl.karajan.Loader; import org.griphyn.vdl.karajan.CompilationException; import org.griphyn.vdl.toolkit.VDLt2VDLx; -import org.griphyn.vdl.toolkit.VDLt2VDLx.ParsingException; import org.safehaus.uuid.UUIDGenerator; import org.w3c.dom.Node; From noreply at svn.ci.uchicago.edu Mon Jul 19 13:38:36 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 19 Jul 2010 13:38:36 -0500 (CDT) Subject: [Swift-commit] r3443 - in trunk/src/org/griphyn/vdl/karajan: functions lib Message-ID: <20100719183836.BA3079CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-19 13:38:36 -0500 (Mon, 19 Jul 2010) New Revision: 3443 Modified: trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java trunk/src/org/griphyn/vdl/karajan/lib/Execute.java trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java Log: Drop unused imports; whitespace Modified: trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2010-07-19 18:36:58 UTC (rev 3442) +++ trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2010-07-19 18:38:36 UTC (rev 3443) @@ -33,19 +33,19 @@ boolean instance = TypeUtil.toBoolean(INSTANCE.getValue(stack)); Object host = HOST.getValue(stack); if(logger.isDebugEnabled()) { - logger.debug("Getting property "+name+" with host "+host); - } - if(host!= null) { - // see if the host has this property defined, and if so - // get its value - BoundContact h = (BoundContact)host; - String prop = (String) h.getProperty(name); - if(prop != null) { - logger.debug("Found property "+name+" in BoundContact"); - return prop; - } - logger.debug("Could not find property "+name+" in BoundContact"); - } + logger.debug("Getting property "+name+" with host "+host); + } + if(host!= null) { + // see if the host has this property defined, and if so + // get its value + BoundContact h = (BoundContact)host; + String prop = (String) h.getProperty(name); + if(prop != null) { + logger.debug("Found property "+name+" in BoundContact"); + return prop; + } + logger.debug("Could not find property "+name+" in BoundContact"); + } return getProperty(name, instance, stack); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2010-07-19 18:36:58 UTC (rev 3442) +++ trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2010-07-19 18:38:36 UTC (rev 3443) @@ -46,7 +46,7 @@ Object constraints) throws ExecutionException { try { registerReplica(stack, task); - log(task, stack); + log(task, stack); scheduler.addJobStatusListener(this, task); synchronized (tasks) { tasks.put(task, stack); @@ -61,32 +61,32 @@ } } - void log(Task task, VariableStack stack) - throws ExecutionException - { - if (logger.isDebugEnabled()) { - logger.debug(task); - logger.debug("Submitting task " + task); - } - String jobid = (String)A_JOBID.getValue(stack,null); - if (logger.isDebugEnabled()) { - logger.debug("jobid="+jobid+" task=" + task); - } - else if (logger.isInfoEnabled()) { - Specification spec = task.getSpecification(); - if (spec instanceof JobSpecification) { - JobSpecification jobspec = (JobSpecification) spec; - logger.info("Submit: " + - "in: " + jobspec.getDirectory() + - " command: " + jobspec.getExecutable() + - " " + jobspec.getArguments()); - } - else { - logger.info("Submit: " + spec); - } - } - } - + void log(Task task, VariableStack stack) + throws ExecutionException + { + if (logger.isDebugEnabled()) { + logger.debug(task); + logger.debug("Submitting task " + task); + } + String jobid = (String) A_JOBID.getValue(stack,null); + if (logger.isDebugEnabled()) { + logger.debug("jobid="+jobid+" task=" + task); + } + else if (logger.isInfoEnabled()) { + Specification spec = task.getSpecification(); + if (spec instanceof JobSpecification) { + JobSpecification jobspec = (JobSpecification) spec; + logger.info("Submit: " + + "in: " + jobspec.getDirectory() + + " command: " + jobspec.getExecutable() + + " " + jobspec.getArguments()); + } + else { + logger.info("Submit: " + spec); + } + } + } + protected void registerReplica(VariableStack stack, Task task) throws CanceledReplicaException { setTaskIdentity(stack, task); try { Modified: trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2010-07-19 18:36:58 UTC (rev 3442) +++ trunk/src/org/griphyn/vdl/karajan/lib/SetFieldValue.java 2010-07-19 18:38:36 UTC (rev 3443) @@ -15,9 +15,6 @@ import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -import edu.emory.mathcs.backport.java.util.concurrent.ExecutorService; -import edu.emory.mathcs.backport.java.util.concurrent.Executors; - public class SetFieldValue extends VDLFunction { public static final Logger logger = Logger.getLogger(SetFieldValue.class); @@ -69,10 +66,10 @@ else if (logger.isInfoEnabled()) { if (leaf instanceof AbstractDataNode) { AbstractDataNode data = (AbstractDataNode) leaf; - Path path = data.getPathFromRoot(); - String p = path.toString(); - if (p.equals("$")) - p = ""; + Path path = data.getPathFromRoot(); + String p = path.toString(); + if (p.equals("$")) + p = ""; String name = data.getDisplayableName() + p; logger.info("Set: " + name + "=" + value); } From noreply at svn.ci.uchicago.edu Mon Jul 19 13:42:03 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 19 Jul 2010 13:42:03 -0500 (CDT) Subject: [Swift-commit] r3444 - branches Message-ID: <20100719184203.6B9A29CCAA@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-19 13:42:03 -0500 (Mon, 19 Jul 2010) New Revision: 3444 Added: branches/cdm-01/ Log: Add temporary branch for Thiago's CDM changes Copied: branches/cdm-01 (from rev 3443, trunk) From noreply at svn.ci.uchicago.edu Mon Jul 19 16:06:11 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 19 Jul 2010 16:06:11 -0500 (CDT) Subject: [Swift-commit] r3445 - branches Message-ID: <20100719210611.B8BC59CC98@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-19 16:06:11 -0500 (Mon, 19 Jul 2010) New Revision: 3445 Added: branches/tests-01/ Log: New branch for test improvements Copied: branches/tests-01 (from rev 3444, trunk) From noreply at svn.ci.uchicago.edu Mon Jul 19 16:10:02 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 19 Jul 2010 16:10:02 -0500 (CDT) Subject: [Swift-commit] r3446 - in branches/tests-01/tests: . language/should-not-work Message-ID: <20100719211002.25A399CC98@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-19 16:10:01 -0500 (Mon, 19 Jul 2010) New Revision: 3446 Added: branches/tests-01/tests/array_iteration.swift branches/tests-01/tests/array_wildcard.swift branches/tests-01/tests/arraymapper.swift branches/tests-01/tests/default.swift branches/tests-01/tests/diamond.swift branches/tests-01/tests/file_counter.swift branches/tests-01/tests/hello.swift branches/tests-01/tests/helloworld.swift branches/tests-01/tests/helloworld_named.swift branches/tests-01/tests/language/should-not-work/1151-bad-comment.swift branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.swift branches/tests-01/tests/language/should-not-work/117-empty-program.swift branches/tests-01/tests/language/should-not-work/case-broken2.swift branches/tests-01/tests/language/should-not-work/closes.swift branches/tests-01/tests/range.swift Removed: branches/tests-01/tests/array_iteration.dtm branches/tests-01/tests/array_wildcard.dtm branches/tests-01/tests/arraymapper.dtm branches/tests-01/tests/default.dtm branches/tests-01/tests/diamond.dtm branches/tests-01/tests/file_counter.dtm branches/tests-01/tests/hello.dtm branches/tests-01/tests/helloworld.dtm branches/tests-01/tests/helloworld_named.dtm branches/tests-01/tests/language/should-not-work/1151-bad-comment.dtm branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.dtm branches/tests-01/tests/language/should-not-work/117-empty-program.dtm branches/tests-01/tests/language/should-not-work/case-broken2.dtm branches/tests-01/tests/language/should-not-work/closes.dtm branches/tests-01/tests/range.dtm Modified: branches/tests-01/tests/nightly.sh Log: Tests: *.dtm to *.swift Deleted: branches/tests-01/tests/array_iteration.dtm =================================================================== --- branches/tests-01/tests/array_iteration.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/array_iteration.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,17 +0,0 @@ -type file {} - -(file f) echo (string s) { - app { - echo s stdout=@filename(f); - } -} - -(file fa[]) echo_batch (string sa[]) { - foreach string s, i in sa { - fa[i] = echo(s); - } -} - -string sa[] = ["hello","hi there","how are you"]; -file fa[]; -fa = echo_batch(sa); Copied: branches/tests-01/tests/array_iteration.swift (from rev 3445, branches/tests-01/tests/array_iteration.dtm) =================================================================== --- branches/tests-01/tests/array_iteration.swift (rev 0) +++ branches/tests-01/tests/array_iteration.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,17 @@ +type file {} + +(file f) echo (string s) { + app { + echo s stdout=@filename(f); + } +} + +(file fa[]) echo_batch (string sa[]) { + foreach string s, i in sa { + fa[i] = echo(s); + } +} + +string sa[] = ["hello","hi there","how are you"]; +file fa[]; +fa = echo_batch(sa); Deleted: branches/tests-01/tests/array_wildcard.dtm =================================================================== --- branches/tests-01/tests/array_wildcard.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/array_wildcard.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,13 +0,0 @@ -type file {} - - -(file t) echo_wildcard (string s[]) { - app { - echo s[*] stdout=@filename(t); - } -} - -string greetings[] = ["how","are","you"]; -file hw = echo(greetings); - - Copied: branches/tests-01/tests/array_wildcard.swift (from rev 3445, branches/tests-01/tests/array_wildcard.dtm) =================================================================== --- branches/tests-01/tests/array_wildcard.swift (rev 0) +++ branches/tests-01/tests/array_wildcard.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,13 @@ +type file {} + + +(file t) echo_wildcard (string s[]) { + app { + echo s[*] stdout=@filename(t); + } +} + +string greetings[] = ["how","are","you"]; +file hw = echo(greetings); + + Deleted: branches/tests-01/tests/arraymapper.dtm =================================================================== --- branches/tests-01/tests/arraymapper.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/arraymapper.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,8 +0,0 @@ -type file {}; - -file files[]; - -foreach f in files { - print(f); -} - Copied: branches/tests-01/tests/arraymapper.swift (from rev 3445, branches/tests-01/tests/arraymapper.dtm) =================================================================== --- branches/tests-01/tests/arraymapper.swift (rev 0) +++ branches/tests-01/tests/arraymapper.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,8 @@ +type file {}; + +file files[]; + +foreach f in files { + print(f); +} + Deleted: branches/tests-01/tests/default.dtm =================================================================== --- branches/tests-01/tests/default.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/default.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,11 +0,0 @@ -type file {} - -(file t) echo (string s="hello world") { //s has a default value - app { - echo s stdout=@filename(t); //redirect stdout to a file - } -} - -file hw1, hw2; -hw1 = echo(); // procedure call using the default value -hw2 = echo(s="hello again"); // using a different value Copied: branches/tests-01/tests/default.swift (from rev 3445, branches/tests-01/tests/default.dtm) =================================================================== --- branches/tests-01/tests/default.swift (rev 0) +++ branches/tests-01/tests/default.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,11 @@ +type file {} + +(file t) echo (string s="hello world") { //s has a default value + app { + echo s stdout=@filename(t); //redirect stdout to a file + } +} + +file hw1, hw2; +hw1 = echo(); // procedure call using the default value +hw2 = echo(s="hello again"); // using a different value Deleted: branches/tests-01/tests/diamond.dtm =================================================================== --- branches/tests-01/tests/diamond.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/diamond.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,33 +0,0 @@ -type file {}; - -(file f) generate (float p1) { - app { - generate "-aTOP -T4" "-p" p1 "-o" @f; - } -} - -(file f2) process (file f1, string name, float p2) { - app { - process "-a" name "-T4" "-p" p2 "-i" @f1 "-o" @f2; - } -} - -(file f3) combine (file f1, file f2) { - app { - combine "-aBOTTOM -T4" "-i" @f1 @f2 "-o" @f3; - } -} - -(file fd) diamond (float p1, float p2) { - file fa; - file fb; - file fc; - - fa = generate(p1); - fb = process(fa, "LEFT", p2); - fc = process(fa, "RIGHT", p2); - fd = combine(fb, fc); -} - -file final<"FINAL">; -final = diamond(1, 100); Copied: branches/tests-01/tests/diamond.swift (from rev 3445, branches/tests-01/tests/diamond.dtm) =================================================================== --- branches/tests-01/tests/diamond.swift (rev 0) +++ branches/tests-01/tests/diamond.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,33 @@ +type file {}; + +(file f) generate (float p1) { + app { + generate "-aTOP -T4" "-p" p1 "-o" @f; + } +} + +(file f2) process (file f1, string name, float p2) { + app { + process "-a" name "-T4" "-p" p2 "-i" @f1 "-o" @f2; + } +} + +(file f3) combine (file f1, file f2) { + app { + combine "-aBOTTOM -T4" "-i" @f1 @f2 "-o" @f3; + } +} + +(file fd) diamond (float p1, float p2) { + file fa; + file fb; + file fc; + + fa = generate(p1); + fb = process(fa, "LEFT", p2); + fc = process(fa, "RIGHT", p2); + fd = combine(fb, fc); +} + +file final<"FINAL">; +final = diamond(1, 100); Deleted: branches/tests-01/tests/file_counter.dtm =================================================================== --- branches/tests-01/tests/file_counter.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/file_counter.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,20 +0,0 @@ -// a two-step workflow that essentially does -// ls "*.txt" | wc - -type file {} - -(file f) ls (string s) { - app { - ls s stdout=@filename(f); - } -} - -(file c) wc (file f) { - app { - wc stdin=@filename(f) stdout=@filename(c); - } -} - -file list, count; -list = ls("/"); -count = wc(list); Copied: branches/tests-01/tests/file_counter.swift (from rev 3445, branches/tests-01/tests/file_counter.dtm) =================================================================== --- branches/tests-01/tests/file_counter.swift (rev 0) +++ branches/tests-01/tests/file_counter.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,20 @@ +// a two-step workflow that essentially does +// ls "*.txt" | wc + +type file {} + +(file f) ls (string s) { + app { + ls s stdout=@filename(f); + } +} + +(file c) wc (file f) { + app { + wc stdin=@filename(f) stdout=@filename(c); + } +} + +file list, count; +list = ls("/"); +count = wc(list); Deleted: branches/tests-01/tests/hello.dtm =================================================================== --- branches/tests-01/tests/hello.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/hello.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,12 +0,0 @@ -type file {} - - -(file t) echo (string s) { - app { - echo s stdout=@filename(t); - } -} - -file hw = echo("hello world"); // May need syntax to give a call a name. - - Copied: branches/tests-01/tests/hello.swift (from rev 3445, branches/tests-01/tests/hello.dtm) =================================================================== --- branches/tests-01/tests/hello.swift (rev 0) +++ branches/tests-01/tests/hello.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,12 @@ +type file {} + + +(file t) echo (string s) { + app { + echo s stdout=@filename(t); + } +} + +file hw = echo("hello world"); // May need syntax to give a call a name. + + Deleted: branches/tests-01/tests/helloworld.dtm =================================================================== --- branches/tests-01/tests/helloworld.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/helloworld.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,8 +0,0 @@ -type file {} //define a type for file -(file t) echo (string s) { //procedure declaration - app { - echo s stdout=@filename(t); //redirect stdout to a file - } -} - -file hw = echo("hello world"); //procedure call Copied: branches/tests-01/tests/helloworld.swift (from rev 3445, branches/tests-01/tests/helloworld.dtm) =================================================================== --- branches/tests-01/tests/helloworld.swift (rev 0) +++ branches/tests-01/tests/helloworld.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,8 @@ +type file {} //define a type for file +(file t) echo (string s) { //procedure declaration + app { + echo s stdout=@filename(t); //redirect stdout to a file + } +} + +file hw = echo("hello world"); //procedure call Deleted: branches/tests-01/tests/helloworld_named.dtm =================================================================== --- branches/tests-01/tests/helloworld_named.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/helloworld_named.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,9 +0,0 @@ -type file {} //define a type for file -(file t) echo (string s) { //procedure declaration - app { - echo s stdout=@filename(t); //redirect stdout to a file - } -} - -file hw<"helloworld.txt">; //name the output file -hw = echo("hello world"); //procedure call Copied: branches/tests-01/tests/helloworld_named.swift (from rev 3445, branches/tests-01/tests/helloworld_named.dtm) =================================================================== --- branches/tests-01/tests/helloworld_named.swift (rev 0) +++ branches/tests-01/tests/helloworld_named.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,9 @@ +type file {} //define a type for file +(file t) echo (string s) { //procedure declaration + app { + echo s stdout=@filename(t); //redirect stdout to a file + } +} + +file hw<"helloworld.txt">; //name the output file +hw = echo("hello world"); //procedure call Deleted: branches/tests-01/tests/language/should-not-work/1151-bad-comment.dtm =================================================================== --- branches/tests-01/tests/language/should-not-work/1151-bad-comment.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/language/should-not-work/1151-bad-comment.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1 +0,0 @@ -/namedOutputs = waveletTransf(waveletScript, 101, "FB"); Copied: branches/tests-01/tests/language/should-not-work/1151-bad-comment.swift (from rev 3445, branches/tests-01/tests/language/should-not-work/1151-bad-comment.dtm) =================================================================== --- branches/tests-01/tests/language/should-not-work/1151-bad-comment.swift (rev 0) +++ branches/tests-01/tests/language/should-not-work/1151-bad-comment.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1 @@ +/namedOutputs = waveletTransf(waveletScript, 101, "FB"); Deleted: branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.dtm =================================================================== --- branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1 +0,0 @@ -in a; Copied: branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.swift (from rev 3445, branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.dtm) =================================================================== --- branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.swift (rev 0) +++ branches/tests-01/tests/language/should-not-work/116-keyword-misuse-in.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1 @@ +in a; Deleted: branches/tests-01/tests/language/should-not-work/117-empty-program.dtm =================================================================== --- branches/tests-01/tests/language/should-not-work/117-empty-program.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/language/should-not-work/117-empty-program.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1 +0,0 @@ -; Copied: branches/tests-01/tests/language/should-not-work/117-empty-program.swift (from rev 3445, branches/tests-01/tests/language/should-not-work/117-empty-program.dtm) =================================================================== --- branches/tests-01/tests/language/should-not-work/117-empty-program.swift (rev 0) +++ branches/tests-01/tests/language/should-not-work/117-empty-program.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1 @@ +; Deleted: branches/tests-01/tests/language/should-not-work/case-broken2.dtm =================================================================== --- branches/tests-01/tests/language/should-not-work/case-broken2.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/language/should-not-work/case-broken2.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,14 +0,0 @@ - -switch(d) -{ - case 1: - f(g); - break; - case 5: - j=7; - uuu=fn(t+7+e+3*2); - break; - default: - s="the default"; -}} - Copied: branches/tests-01/tests/language/should-not-work/case-broken2.swift (from rev 3445, branches/tests-01/tests/language/should-not-work/case-broken2.dtm) =================================================================== --- branches/tests-01/tests/language/should-not-work/case-broken2.swift (rev 0) +++ branches/tests-01/tests/language/should-not-work/case-broken2.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,14 @@ + +switch(d) +{ + case 1: + f(g); + break; + case 5: + j=7; + uuu=fn(t+7+e+3*2); + break; + default: + s="the default"; +}} + Deleted: branches/tests-01/tests/language/should-not-work/closes.dtm =================================================================== --- branches/tests-01/tests/language/should-not-work/closes.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/language/should-not-work/closes.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,2 +0,0 @@ -}}}} - Copied: branches/tests-01/tests/language/should-not-work/closes.swift (from rev 3445, branches/tests-01/tests/language/should-not-work/closes.dtm) =================================================================== --- branches/tests-01/tests/language/should-not-work/closes.swift (rev 0) +++ branches/tests-01/tests/language/should-not-work/closes.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,2 @@ +}}}} + Modified: branches/tests-01/tests/nightly.sh =================================================================== --- branches/tests-01/tests/nightly.sh 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/nightly.sh 2010-07-19 21:10:01 UTC (rev 3446) @@ -9,28 +9,34 @@ RUNDIR=$OUTDIR/$RUNDIRBASE mkdir -p $RUNDIR -if [ "$1" == "-fco" ]; then - FCO=1 - shift -elif [ "$2" == "-fco" ]; then - FCO=1 -fi +OUTBASE=$RUNDIRBASE/tests.log -if [ "$1" == "" ]; then - OUTBASE=$RUNDIRBASE/tests.log -else - OUTBASE=$RUNDIRBASE/$1 -fi +BRANCH="branches/tests-01" + +while [ $# -gt 0 ]; do + case $1 in + -s) + SKIP_CHECKOUT=1 + shift;; + -x) + EXIT_ON_ERROR=1 + shift;; + *) + OUTBASE=$RUNDIRBASE/$1 + shift;; + esac +done + OUT=$OUTDIR/$OUTBASE -head() { +header() { HTMLBASE=tests-$DATE.html HTML=$OUTDIR/$HTMLBASE rm -f $OUTDIR/current.html - + #This doesn't work well with servers that don't follow symlinks #ln -s $HTML $OUTDIR/current.html - + cat <$OUTDIR/current.html @@ -81,10 +87,10 @@ echo $@ >>$HTML } -tail() { +footer() { MONTHS=("" "Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec") html "" - + if [ "$BINPACKAGE" != "" ]; then FBP=$OUTDIR/$BINPACKAGE SIZE=`ls -hs $FBP` @@ -95,7 +101,7 @@ $BINPACKAGE ($SIZE)
DOH fi - + LASTYR="00" LASTMO="00" html "

Older tests

" @@ -153,13 +159,13 @@ echo $@ TYPE=$1 if [ "$TYPE" == "test" ]; then - + NAME=$2 SEQ=$3 CMD=$4 RES=$5 LOG=$6 - + if [ "$FIRSTTEST" == "1" ]; then html "

Test results

" html "" @@ -171,12 +177,12 @@ html "" fi fi - + if [ "$TESTPART" != "" ]; then html "$TESTPART" TESTPART= fi - + if [ "$FLUSH" == "1" ]; then html "$NAME: " fi @@ -193,13 +199,12 @@ html "$SEQ" fi html "" - + elif [ "$TYPE" == "package" ]; then BINPACKAGE=$2 else html $@ fi - } aexec() { @@ -214,7 +219,6 @@ if [ -f $OUTDIR/x73010test.log ]; then cat $OUTDIR/x73010test.log >>$OUT fi - } tlog() { @@ -321,38 +325,36 @@ head TESTPART="Part I: Build" EXITONFAILURE=true -if [ "$FCO" != "1" ]; then +if [ "$SKIP_CHECKOUT" != "1" ]; then TEST="Checkout CoG" pexec rm -rf cog - vexec svn co https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog + COG="https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog" + vexec svn co $COG TEST="Checkout Swift" - pexec rm -rf trunk - #vexec cvs -d :pserver:anonymous at cvs.cogkit.org:/cvs/cogkit co src/vdsk - vexec svn co https://svn.ci.uchicago.edu/svn/vdl2/trunk + pexec cd cog/modules + pexec rm -rf swift + vexec svn co https://svn.ci.uchicago.edu/svn/vdl2/$BRANCH swift + pexec cd $OUTDIR fi -TEST="Directory setup" -pexec mkdir cog/modules/vdsk -vexec cp -r trunk/* cog/modules/vdsk - TEST="Compile" -pexec cd cog/modules/vdsk +pexec cd cog/modules/swift pexec rm -rf dist vexec ant -quiet dist TEST="Package" pexec cd dist -VDSK=`ls -d vdsk*.*` -pexec cd $VDSK/lib +SWIFT=$( ls -d swift-* ) +pexec cd $SWIFT/lib pexec rm -f castor*.jar *gt2ft*.jar ant.jar pexec cd ../.. -pexec rm -rf vdsk-$DATE -pexec mv $VDSK vdsk-$DATE -vexec tar -pczf $OUTDIR/vdsk-$DATE.tar.gz vdsk-$DATE -out package "vdsk-$DATE.tar.gz" +pexec rm -rf swift-$DATE +pexec mv $SWIFT swift-$DATE +vexec tar -pczf $OUTDIR/swift-$DATE.tar.gz swift-$DATE +out package "swift-$DATE.tar.gz" -PATH=$PWD/vdsk-$DATE/bin:$PATH +PATH=$PWD/swift-$DATE/bin:$PATH cd .. TESTDIR=$PWD/tests echo "Path: $PATH" >>$OUT @@ -361,21 +363,18 @@ EXITONFAILURE=false TESTPART="Part II: Local Tests" -for TEST in `ls $TESTDIR/*.dtm $TESTDIR/*.swift`; do - BN=`basename $TEST` - echo $BN - cp $TESTDIR/$BN . - - - TESTNAME=${BN%.dtm} - TESTNAME=${TESTNAME%.swift} - TEST="$TESTNAME" - - ssexec "Compile" vdlc $BN - for ((i=0; $i<9; i=$i+1)); do - pexec swift -sites.file ~/.vdl2/sites-local.xml $TESTNAME.kml - done - vexec swift -sites.file ~/.vdl2/sites-local.xml $TESTNAME.kml +for TEST in $( ls $TESTDIR/*.swift ); do # $TESTDIR/*.dtm + TESTNAME=$( basename $TEST) + echo TESTNAME: $TESTNAME + cp -uv $TESTDIR/$TESTNAME . + + TEST="$TESTNAME" + + # ssexec "Compile" vdlc $BN + for ((i=0; $i<9; i=$i+1)); do + pexec swift -sites.file ~/.vdl2/sites-local.xml $TESTNAME + done + vexec swift -sites.file ~/.vdl2/sites-local.xml $TESTNAME done TESTPART="Part III: Grid Tests" @@ -384,11 +383,11 @@ BN=`basename $TEST` echo $BN cp $TESTDIR/$BN . - + TESTNAME=${BN%.dtm} TESTNAME=${TESTNAME%.swift} TEST="$TESTNAME" - + ssexec "Compile" vdlc $BN for ((i=0; $i<9; i=$i+1)); do pexec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml @@ -397,4 +396,4 @@ done #Don't remove me: -tail +footer Deleted: branches/tests-01/tests/range.dtm =================================================================== --- branches/tests-01/tests/range.dtm 2010-07-19 21:06:11 UTC (rev 3445) +++ branches/tests-01/tests/range.dtm 2010-07-19 21:10:01 UTC (rev 3446) @@ -1,5 +0,0 @@ -int nums[] = [0:20:2]; // generate a list 0, 2, 4, 6, 8 ... - -foreach num in nums { - print(num); -} Copied: branches/tests-01/tests/range.swift (from rev 3445, branches/tests-01/tests/range.dtm) =================================================================== --- branches/tests-01/tests/range.swift (rev 0) +++ branches/tests-01/tests/range.swift 2010-07-19 21:10:01 UTC (rev 3446) @@ -0,0 +1,5 @@ +int nums[] = [0:20:2]; // generate a list 0, 2, 4, 6, 8 ... + +foreach num in nums { + print(num); +} From noreply at svn.ci.uchicago.edu Tue Jul 20 02:18:20 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 20 Jul 2010 02:18:20 -0500 (CDT) Subject: [Swift-commit] r3447 - usertools/swift/swiftconfig/bin Message-ID: <20100720071820.F41049CCAA@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-20 02:18:20 -0500 (Tue, 20 Jul 2010) New Revision: 3447 Added: usertools/swift/swiftconfig/bin/swiftrun Log: swiftrun Added: usertools/swift/swiftconfig/bin/swiftrun =================================================================== --- usertools/swift/swiftconfig/bin/swiftrun (rev 0) +++ usertools/swift/swiftconfig/bin/swiftrun 2010-07-20 07:18:20 UTC (rev 3447) @@ -0,0 +1,127 @@ +#!/usr/bin/perl + +use Cwd; +use Cwd qw(abs_path); +use File::Copy; +use File::Path qw(remove_tree); + +my $swifthome = ''; +if ( $ENV{'SWIFT_HOME'} && -e "$ENV{'SWIFT_HOME'}/bin/swift" ) { + $swifthome = $ENV{'SWIFT_HOME'}; +} + +my @path = split( '/', abs_path($0) ); +my @path = splice( @path, 0, $#path - 1 ); +my $parent_directory = join( '/', @path ); + +if ( !$swifthome ) { + if ( -e "$parent_directory/bin/swift" ) { + $swifthome = $parent_directory; + } + + else { + @path = split( '/', which("swift") ); + @path = splice( @path, 0, $#path - 1 ); + my $tmp = join( '/', @path ); + if ( -e "$tmp/bin/swift" ) { + $swifthome = $tmp; + } + } +} + +# Create a new directory if it does not exist +sub create_directory { + $directory = $_[0]; + if ( !-d "$directory" ) { + mkdir "$directory", 0700 + or die "Unable to create directory $directory\n"; + } +} + +# Append one file to another +sub cat_file { + my ($src, $dst) = @_; + if(-e $dst){ open(DSTFILE, ">>$dst") or die "Unable to open $dst for append\n"; } + else { open(DSTFILE, ">$dst") or die "Unable to open $dst for writing\n"; } + open(SRCFILE, $src) || die "Unable to open $src for reading\n"; + foreach $line() { + print DSTFILE $line; + } +} + +# Copy a file to a new location +sub copy_file { + my ($src, $dst) = @_; + if(!-e $src) { die "Unable to access $src\n"; } + if(!-e $dst) { die "Unable to access $dst\n"; } + copy($src, $dst) or die "Unable to copy $src to $dst\n"; +} + +# Strip directory name out of script +sub strip_directory { + my ($fullpath) = @_; + my @path = split( '/', $fullpath); + my $filename = @path[$#path]; + return $filename; +} + +use FindBin qw($Bin); +use lib "$FindBin::Bin/../lib/perl"; +use Getopt::Long; +use File::Which qw(which where); +use XML::Simple; +use Data::Dumper; + +GetOptions( + 'site=s' => \$site, + 'script=s' => \$script, +); + +# Verify user-created profile info exists +my ($seconds, $minutes, $hours, $days, $months, $years, @junk) = localtime(time); +$datestring = sprintf "%04s%02s%02s%02s%02s%02s%s", $years+1900, $months+1, $days, $hours, $minutes, $seconds, getpgrp(0); +my $dotswift = "$ENV{'HOME'}/.swift"; + + +# Run job + at sites = split(/,/, $site); +my $script_filename = strip_directory($script); +create_directory("$dotswift/jobs"); +create_directory("$dotswift/jobs/$script_filename"); +create_directory("$dotswift/jobs/$script_filename/$datestring"); +copy_file("$script", "$dotswift/jobs/$script_filename/$datestring"); + +# Temporarily rename existing auth.defaults +if(-e "$ENV{'HOME'}/.ssh/auth.defaults") { + move("$ENV{'HOME'}/.ssh/auth.defaults", "$ENV{'HOME'}/.ssh/auth.defaults.$datestring") || die "Unable to rename $ENV{'HOME'}/.ssh/auth.defaults"; +} + +foreach my $s(@sites) { + if(!-d "$dotswift/sites/$s"){ print "$dotswift/sites/$site\n"; die "Unable to find entry for $s\n"; } + if(!-e "$dotswift/sites/$s/sites.xml"){ die "Unable to find sites.xml for $s\n"; } + if(!-e "$dotswift/sites/$s/tc.data"){ die "Unable to find tc.data for $s\n"; } + cat_file("$dotswift/sites/$s/sites.xml", "$dotswift/jobs/$script_filename/$datestring/sites.xml"); + cat_file("$dotswift/sites/$s/tc.data", "$dotswift/jobs/$script_filename/$datestring/tc.data"); + if(-e "$dotswift/sites/$s/auth.defaults"){ + cat_file("$dotswift/sites/$s/auth.defaults", "$ENV{'HOME'}/.ssh/auth.defaults"); + } +} + +# Rename auths.default back to original +if(-e "$ENV{'HOME'}/.ssh/auth.defaults.$datestring") { + move("$ENV{'HOME'}/.ssh/auth.defaults.$datestring", "$ENV{'HOME'}/.ssh/auth.defaults") || die "Unable to rename $ENV{'HOME'}/.ssh/auth.defaults.$datestring"; +} + +# Update XML tags +open(SITEFILE, "$dotswift/jobs/$script_filename/$datestring/sites.xml") || die "Unable to open $dotswift/jobs/$script_filename/$datestring/sites.xml\n"; +my @sitedata = ; +close(SITEFILE); +open(SITEFILE, ">$dotswift/jobs/$script_filename/$datestring/sites.xml") || die "Unable to open $dotswift/jobs/$script_filename/$datestring/sites.xml\n"; +print SITEFILE "\n at sitedata\n"; +close(SITEFILE); + +# Run swift +chdir("$dotswift/jobs/$script_filename/$datestring") || die "Unable to change directories to $dotswift/job/$script_filename/$datestring\n"; +print "Submitting job. Check $dotswift/jobs/$script_filename/$datestring for details\n"; +system("swift -sites.file $dotswift/jobs/$script_filename/$datestring/sites.xml -tc.file $dotswift/jobs/$script_filename/$datestring/tc.data $script"); + Property changes on: usertools/swift/swiftconfig/bin/swiftrun ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Tue Jul 20 02:36:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 20 Jul 2010 02:36:24 -0500 (CDT) Subject: [Swift-commit] r3448 - usertools/swift/swiftconfig/lib/perl/File Message-ID: <20100720073624.DC3C69CCAA@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-20 02:36:24 -0500 (Tue, 20 Jul 2010) New Revision: 3448 Added: usertools/swift/swiftconfig/lib/perl/File/Path.pm Log: Include correct version of File::Path to avoid issues with older version which handled file deletion differently Added: usertools/swift/swiftconfig/lib/perl/File/Path.pm =================================================================== --- usertools/swift/swiftconfig/lib/perl/File/Path.pm (rev 0) +++ usertools/swift/swiftconfig/lib/perl/File/Path.pm 2010-07-20 07:36:24 UTC (rev 3448) @@ -0,0 +1,895 @@ +package File::Path; + +use 5.005_04; +use strict; + +use Cwd 'getcwd'; +use File::Basename (); +use File::Spec (); + +BEGIN { + if ($] < 5.006) { + # can't say 'opendir my $dh, $dirname' + # need to initialise $dh + eval "use Symbol"; + } +} + +use Exporter (); +use vars qw($VERSION @ISA @EXPORT @EXPORT_OK); +$VERSION = '2.07_03'; + at ISA = qw(Exporter); + at EXPORT = qw(mkpath rmtree); + at EXPORT_OK = qw(make_path remove_tree); + +my $Is_VMS = $^O eq 'VMS'; +my $Is_MacOS = $^O eq 'MacOS'; + +# These OSes complain if you want to remove a file that you have no +# write permission to: +my $Force_Writeable = grep {$^O eq $_} qw(amigaos dos epoc MSWin32 MacOS os2); + +# Unix-like systems need to stat each directory in order to detect +# race condition. MS-Windows is immune to this particular attack. +my $Need_Stat_Check = !($^O eq 'MSWin32'); + +sub _carp { + require Carp; + goto &Carp::carp; +} + +sub _croak { + require Carp; + goto &Carp::croak; +} + +sub _error { + my $arg = shift; + my $message = shift; + my $object = shift; + + if ($arg->{error}) { + $object = '' unless defined $object; + $message .= ": $!" if $!; + push @{${$arg->{error}}}, {$object => $message}; + } + else { + _carp(defined($object) ? "$message for $object: $!" : "$message: $!"); + } +} + +sub make_path { + push @_, {} unless @_ and UNIVERSAL::isa($_[-1],'HASH'); + goto &mkpath; +} + +sub mkpath { + my $old_style = !(@_ and UNIVERSAL::isa($_[-1],'HASH')); + + my $arg; + my $paths; + + if ($old_style) { + my ($verbose, $mode); + ($paths, $verbose, $mode) = @_; + $paths = [$paths] unless UNIVERSAL::isa($paths,'ARRAY'); + $arg->{verbose} = $verbose; + $arg->{mode} = defined $mode ? $mode : 0777; + } + else { + $arg = pop @_; + $arg->{mode} = delete $arg->{mask} if exists $arg->{mask}; + $arg->{mode} = 0777 unless exists $arg->{mode}; + ${$arg->{error}} = [] if exists $arg->{error}; + $paths = [@_]; + } + return _mkpath($arg, $paths); +} + +sub _mkpath { + my $arg = shift; + my $paths = shift; + + my(@created,$path); + foreach $path (@$paths) { + next unless defined($path) and length($path); + $path .= '/' if $^O eq 'os2' and $path =~ /^\w:\z/s; # feature of CRT + # Logic wants Unix paths, so go with the flow. + if ($Is_VMS) { + next if $path eq '/'; + $path = VMS::Filespec::unixify($path); + } + next if -d $path; + my $parent = File::Basename::dirname($path); + unless (-d $parent or $path eq $parent) { + push(@created,_mkpath($arg, [$parent])); + } + print "mkdir $path\n" if $arg->{verbose}; + if (mkdir($path,$arg->{mode})) { + push(@created, $path); + } + else { + my $save_bang = $!; + my ($e, $e1) = ($save_bang, $^E); + $e .= "; $e1" if $e ne $e1; + # allow for another process to have created it meanwhile + if (!-d $path) { + $! = $save_bang; + if ($arg->{error}) { + push @{${$arg->{error}}}, {$path => $e}; + } + else { + _croak("mkdir $path: $e"); + } + } + } + } + return @created; +} + +sub remove_tree { + push @_, {} unless @_ and UNIVERSAL::isa($_[-1],'HASH'); + goto &rmtree; +} + +sub _is_subdir { + my($dir, $test) = @_; + + my($dv, $dd) = File::Spec->splitpath($dir, 1); + my($tv, $td) = File::Spec->splitpath($test, 1); + + # not on same volume + return 0 if $dv ne $tv; + + my @d = File::Spec->splitdir($dd); + my @t = File::Spec->splitdir($td); + + # @t can't be a subdir if it's shorter than @d + return 0 if @t < @d; + + return join('/', @d) eq join('/', splice @t, 0, + at d); +} + +sub rmtree { + my $old_style = !(@_ and UNIVERSAL::isa($_[-1],'HASH')); + + my $arg; + my $paths; + + if ($old_style) { + my ($verbose, $safe); + ($paths, $verbose, $safe) = @_; + $arg->{verbose} = $verbose; + $arg->{safe} = defined $safe ? $safe : 0; + + if (defined($paths) and length($paths)) { + $paths = [$paths] unless UNIVERSAL::isa($paths,'ARRAY'); + } + else { + _carp ("No root path(s) specified\n"); + return 0; + } + } + else { + $arg = pop @_; + ${$arg->{error}} = [] if exists $arg->{error}; + ${$arg->{result}} = [] if exists $arg->{result}; + $paths = [@_]; + } + + $arg->{prefix} = ''; + $arg->{depth} = 0; + + my @clean_path; + $arg->{cwd} = getcwd() or do { + _error($arg, "cannot fetch initial working directory"); + return 0; + }; + for ($arg->{cwd}) { /\A(.*)\Z/; $_ = $1 } # untaint + + for my $p (@$paths) { + # need to fixup case and map \ to / on Windows + my $ortho_root = $^O eq 'MSWin32' ? _slash_lc($p) : $p; + my $ortho_cwd = $^O eq 'MSWin32' ? _slash_lc($arg->{cwd}) : $arg->{cwd}; + my $ortho_root_length = length($ortho_root); + $ortho_root_length-- if $^O eq 'VMS'; # don't compare '.' with ']' + if ($ortho_root_length && _is_subdir($ortho_root, $ortho_cwd)) { + local $! = 0; + _error($arg, "cannot remove path when cwd is $arg->{cwd}", $p); + next; + } + + if ($Is_MacOS) { + $p = ":$p" unless $p =~ /:/; + $p .= ":" unless $p =~ /:\z/; + } + elsif ($^O eq 'MSWin32') { + $p =~ s{[/\\]\z}{}; + } + else { + $p =~ s{/\z}{}; + } + push @clean_path, $p; + } + + @{$arg}{qw(device inode perm)} = (lstat $arg->{cwd})[0,1] or do { + _error($arg, "cannot stat initial working directory", $arg->{cwd}); + return 0; + }; + + return _rmtree($arg, \@clean_path); +} + +sub _rmtree { + my $arg = shift; + my $paths = shift; + + my $count = 0; + my $curdir = File::Spec->curdir(); + my $updir = File::Spec->updir(); + + my (@files, $root); + ROOT_DIR: + foreach $root (@$paths) { + # since we chdir into each directory, it may not be obvious + # to figure out where we are if we generate a message about + # a file name. We therefore construct a semi-canonical + # filename, anchored from the directory being unlinked (as + # opposed to being truly canonical, anchored from the root (/). + + my $canon = $arg->{prefix} + ? File::Spec->catfile($arg->{prefix}, $root) + : $root + ; + + my ($ldev, $lino, $perm) = (lstat $root)[0,1,2] or next ROOT_DIR; + + if ( -d _ ) { + $root = VMS::Filespec::pathify($root) if $Is_VMS; + + if (!chdir($root)) { + # see if we can escalate privileges to get in + # (e.g. funny protection mask such as -w- instead of rwx) + $perm &= 07777; + my $nperm = $perm | 0700; + if (!($arg->{safe} or $nperm == $perm or chmod($nperm, $root))) { + _error($arg, "cannot make child directory read-write-exec", $canon); + next ROOT_DIR; + } + elsif (!chdir($root)) { + _error($arg, "cannot chdir to child", $canon); + next ROOT_DIR; + } + } + + my ($cur_dev, $cur_inode, $perm) = (stat $curdir)[0,1,2] or do { + _error($arg, "cannot stat current working directory", $canon); + next ROOT_DIR; + }; + + if ($Need_Stat_Check) { + ($ldev eq $cur_dev and $lino eq $cur_inode) + or _croak("directory $canon changed before chdir, expected dev=$ldev ino=$lino, actual dev=$cur_dev ino=$cur_inode, aborting."); + } + + $perm &= 07777; # don't forget setuid, setgid, sticky bits + my $nperm = $perm | 0700; + + # notabene: 0700 is for making readable in the first place, + # it's also intended to change it to writable in case we have + # to recurse in which case we are better than rm -rf for + # subtrees with strange permissions + + if (!($arg->{safe} or $nperm == $perm or chmod($nperm, $curdir))) { + _error($arg, "cannot make directory read+writeable", $canon); + $nperm = $perm; + } + + my $d; + $d = gensym() if $] < 5.006; + if (!opendir $d, $curdir) { + _error($arg, "cannot opendir", $canon); + @files = (); + } + else { + no strict 'refs'; + if (!defined ${"\cTAINT"} or ${"\cTAINT"}) { + # Blindly untaint dir names if taint mode is + # active, or any perl < 5.006 + @files = map { /\A(.*)\z/s; $1 } readdir $d; + } + else { + @files = readdir $d; + } + closedir $d; + } + + if ($Is_VMS) { + # Deleting large numbers of files from VMS Files-11 + # filesystems is faster if done in reverse ASCIIbetical order. + # include '.' to '.;' from blead patch #31775 + @files = map {$_ eq '.' ? '.;' : $_} reverse @files; + ($root = VMS::Filespec::unixify($root)) =~ s/\.dir\z//; + } + + @files = grep {$_ ne $updir and $_ ne $curdir} @files; + + if (@files) { + # remove the contained files before the directory itself + my $narg = {%$arg}; + @{$narg}{qw(device inode cwd prefix depth)} + = ($cur_dev, $cur_inode, $updir, $canon, $arg->{depth}+1); + $count += _rmtree($narg, \@files); + } + + # restore directory permissions of required now (in case the rmdir + # below fails), while we are still in the directory and may do so + # without a race via '.' + if ($nperm != $perm and not chmod($perm, $curdir)) { + _error($arg, "cannot reset chmod", $canon); + } + + # don't leave the client code in an unexpected directory + chdir($arg->{cwd}) + or _croak("cannot chdir to $arg->{cwd} from $canon: $!, aborting."); + + # ensure that a chdir upwards didn't take us somewhere other + # than we expected (see CVE-2002-0435) + ($cur_dev, $cur_inode) = (stat $curdir)[0,1] + or _croak("cannot stat prior working directory $arg->{cwd}: $!, aborting."); + + if ($Need_Stat_Check) { + ($arg->{device} eq $cur_dev and $arg->{inode} eq $cur_inode) + or _croak("previous directory $arg->{cwd} changed before entering $canon, expected dev=$ldev ino=$lino, actual dev=$cur_dev ino=$cur_inode, aborting."); + } + + if ($arg->{depth} or !$arg->{keep_root}) { + if ($arg->{safe} && + ($Is_VMS ? !&VMS::Filespec::candelete($root) : !-w $root)) { + print "skipped $root\n" if $arg->{verbose}; + next ROOT_DIR; + } + if ($Force_Writeable and !chmod $perm | 0700, $root) { + _error($arg, "cannot make directory writeable", $canon); + } + print "rmdir $root\n" if $arg->{verbose}; + if (rmdir $root) { + push @{${$arg->{result}}}, $root if $arg->{result}; + ++$count; + } + else { + _error($arg, "cannot remove directory", $canon); + if ($Force_Writeable && !chmod($perm, ($Is_VMS ? VMS::Filespec::fileify($root) : $root)) + ) { + _error($arg, sprintf("cannot restore permissions to 0%o",$perm), $canon); + } + } + } + } + else { + # not a directory + $root = VMS::Filespec::vmsify("./$root") + if $Is_VMS + && !File::Spec->file_name_is_absolute($root) + && ($root !~ m/(?]+/); # not already in VMS syntax + + if ($arg->{safe} && + ($Is_VMS ? !&VMS::Filespec::candelete($root) + : !(-l $root || -w $root))) + { + print "skipped $root\n" if $arg->{verbose}; + next ROOT_DIR; + } + + my $nperm = $perm & 07777 | 0600; + if ($Force_Writeable and $nperm != $perm and not chmod $nperm, $root) { + _error($arg, "cannot make file writeable", $canon); + } + print "unlink $canon\n" if $arg->{verbose}; + # delete all versions under VMS + for (;;) { + if (unlink $root) { + push @{${$arg->{result}}}, $root if $arg->{result}; + } + else { + _error($arg, "cannot unlink file", $canon); + $Force_Writeable and chmod($perm, $root) or + _error($arg, sprintf("cannot restore permissions to 0%o",$perm), $canon); + last; + } + ++$count; + last unless $Is_VMS && lstat $root; + } + } + } + return $count; +} + +sub _slash_lc { + # fix up slashes and case on MSWin32 so that we can determine that + # c:\path\to\dir is underneath C:/Path/To + my $path = shift; + $path =~ tr{\\}{/}; + return lc($path); +} + +1; +__END__ + +=head1 NAME + +File::Path - Create or remove directory trees + +=head1 VERSION + +This document describes version 2.07 of File::Path, released +2008-11-09. + +=head1 SYNOPSIS + + use File::Path qw(make_path remove_tree); + + make_path('foo/bar/baz', '/zug/zwang'); + make_path('foo/bar/baz', '/zug/zwang', { + verbose => 1, + mode => 0711, + }); + + remove_tree('foo/bar/baz', '/zug/zwang'); + remove_tree('foo/bar/baz', '/zug/zwang', { + verbose => 1, + error => \my $err_list, + }); + + # legacy (interface promoted before v2.00) + mkpath('/foo/bar/baz'); + mkpath('/foo/bar/baz', 1, 0711); + mkpath(['/foo/bar/baz', 'blurfl/quux'], 1, 0711); + rmtree('foo/bar/baz', 1, 1); + rmtree(['foo/bar/baz', 'blurfl/quux'], 1, 1); + + # legacy (interface promoted before v2.06) + mkpath('foo/bar/baz', '/zug/zwang', { verbose => 1, mode => 0711 }); + rmtree('foo/bar/baz', '/zug/zwang', { verbose => 1, mode => 0711 }); + +=head1 DESCRIPTION + +This module provide a convenient way to create directories of +arbitrary depth and to delete an entire directory subtree from the +filesystem. + +The following functions are provided: + +=over + +=item make_path( $dir1, $dir2, .... ) + +=item make_path( $dir1, $dir2, ...., \%opts ) + +The C function creates the given directories if they don't +exists before, much like the Unix command C. + +The function accepts a list of directories to be created. Its +behaviour may be tuned by an optional hashref appearing as the last +parameter on the call. + +The function returns the list of directories actually created during +the call; in scalar context the number of directories created. + +The following keys are recognised in the option hash: + +=over + +=item mode => $num + +The numeric permissions mode to apply to each created directory +(defaults to 0777), to be modified by the current C. If the +directory already exists (and thus does not need to be created), +the permissions will not be modified. + +C is recognised as an alias for this parameter. + +=item verbose => $bool + +If present, will cause C to print the name of each directory +as it is created. By default nothing is printed. + +=item error => \$err + +If present, it should be a reference to a scalar. +This scalar will be made to reference an array, which will +be used to store any errors that are encountered. See the L section for more information. + +If this parameter is not used, certain error conditions may raise +a fatal error that will cause the program will halt, unless trapped +in an C block. + +=back + +=item mkpath( $dir ) + +=item mkpath( $dir, $verbose, $mode ) + +=item mkpath( [$dir1, $dir2,...], $verbose, $mode ) + +=item mkpath( $dir1, $dir2,..., \%opt ) + +The mkpath() function provide the legacy interface of make_path() with +a different interpretation of the arguments passed. The behaviour and +return value of the function is otherwise identical to make_path(). + +=item remove_tree( $dir1, $dir2, .... ) + +=item remove_tree( $dir1, $dir2, ...., \%opts ) + +The C function deletes the given directories and any +files and subdirectories they might contain, much like the Unix +command C or C on Windows. + +The function accepts a list of directories to be +removed. Its behaviour may be tuned by an optional hashref +appearing as the last parameter on the call. + +The functions returns the number of files successfully deleted. + +The following keys are recognised in the option hash: + +=over + +=item verbose => $bool + +If present, will cause C to print the name of each file as +it is unlinked. By default nothing is printed. + +=item safe => $bool + +When set to a true value, will cause C to skip the files +for which the process lacks the required privileges needed to delete +files, such as delete privileges on VMS. In other words, the code +will make no attempt to alter file permissions. Thus, if the process +is interrupted, no filesystem object will be left in a more +permissive mode. + +=item keep_root => $bool + +When set to a true value, will cause all files and subdirectories +to be removed, except the initially specified directories. This comes +in handy when cleaning out an application's scratch directory. + + remove_tree( '/tmp', {keep_root => 1} ); + +=item result => \$res + +If present, it should be a reference to a scalar. +This scalar will be made to reference an array, which will +be used to store all files and directories unlinked +during the call. If nothing is unlinked, the array will be empty. + + remove_tree( '/tmp', {result => \my $list} ); + print "unlinked $_\n" for @$list; + +This is a useful alternative to the C key. + +=item error => \$err + +If present, it should be a reference to a scalar. +This scalar will be made to reference an array, which will +be used to store any errors that are encountered. See the L section for more information. + +Removing things is a much more dangerous proposition than +creating things. As such, there are certain conditions that +C may encounter that are so dangerous that the only +sane action left is to kill the program. + +Use C to trap all that is reasonable (problems with +permissions and the like), and let it die if things get out +of hand. This is the safest course of action. + +=back + +=item rmtree( $dir ) + +=item rmtree( $dir, $verbose, $safe ) + +=item rmtree( [$dir1, $dir2,...], $verbose, $safe ) + +=item rmtree( $dir1, $dir2,..., \%opt ) + +The rmtree() function provide the legacy interface of remove_tree() +with a different interpretation of the arguments passed. The behaviour +and return value of the function is otherwise identical to +remove_tree(). + +=back + +=head2 ERROR HANDLING + +=over 4 + +=item B + +The following error handling mechanism is considered +experimental and is subject to change pending feedback from +users. + +=back + +If C or C encounter an error, a diagnostic +message will be printed to C via C (for non-fatal +errors), or via C (for fatal errors). + +If this behaviour is not desirable, the C attribute may be +used to hold a reference to a variable, which will be used to store +the diagnostics. The variable is made a reference to an array of hash +references. Each hash contain a single key/value pair where the key +is the name of the file, and the value is the error message (including +the contents of C<$!> when appropriate). If a general error is +encountered the diagnostic key will be empty. + +An example usage looks like: + + remove_tree( 'foo/bar', 'bar/rat', {error => \my $err} ); + if (@$err) { + for my $diag (@$err) { + my ($file, $message) = %$diag; + if ($file eq '') { + print "general error: $message\n"; + } + else { + print "problem unlinking $file: $message\n"; + } + } + } + else { + print "No error encountered\n"; + } + +Note that if no errors are encountered, C<$err> will reference an +empty array. This means that C<$err> will always end up TRUE; so you +need to test C<@$err> to determine if errors occured. + +=head2 NOTES + +C blindly exports C and C into the +current namespace. These days, this is considered bad style, but +to change it now would break too much code. Nonetheless, you are +invited to specify what it is you are expecting to use: + + use File::Path 'rmtree'; + +The routines C and C are B exported +by default. You must specify which ones you want to use. + + use File::Path 'remove_tree'; + +Note that a side-effect of the above is that C and C +are no longer exported at all. This is due to the way the C +module works. If you are migrating a codebase to use the new +interface, you will have to list everything explicitly. But that's +just good practice anyway. + + use File::Path qw(remove_tree rmtree); + +=head3 SECURITY CONSIDERATIONS + +There were race conditions 1.x implementations of File::Path's +C function (although sometimes patched depending on the OS +distribution or platform). The 2.0 version contains code to avoid the +problem mentioned in CVE-2002-0435. + +See the following pages for more information: + + http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=286905 + http://www.nntp.perl.org/group/perl.perl5.porters/2005/01/msg97623.html + http://www.debian.org/security/2005/dsa-696 + +Additionally, unless the C parameter is set (or the +third parameter in the traditional interface is TRUE), should a +C be interrupted, files that were originally in read-only +mode may now have their permissions set to a read-write (or "delete +OK") mode. + +=head1 DIAGNOSTICS + +FATAL errors will cause the program to halt (C), since the +problem is so severe that it would be dangerous to continue. (This +can always be trapped with C, but it's not a good idea. Under +the circumstances, dying is the best thing to do). + +SEVERE errors may be trapped using the modern interface. If the +they are not trapped, or the old interface is used, such an error +will cause the program will halt. + +All other errors may be trapped using the modern interface, otherwise +they will be Ced about. Program execution will not be halted. + +=over 4 + +=item mkdir [path]: [errmsg] (SEVERE) + +C was unable to create the path. Probably some sort of +permissions error at the point of departure, or insufficient resources +(such as free inodes on Unix). + +=item No root path(s) specified + +C was not given any paths to create. This message is only +emitted if the routine is called with the traditional interface. +The modern interface will remain silent if given nothing to do. + +=item No such file or directory + +On Windows, if C gives you this warning, it may mean that +you have exceeded your filesystem's maximum path length. + +=item cannot fetch initial working directory: [errmsg] + +C attempted to determine the initial directory by calling +C, but the call failed for some reason. No attempt +will be made to delete anything. + +=item cannot stat initial working directory: [errmsg] + +C attempted to stat the initial directory (after having +successfully obtained its name via C), however, the call +failed for some reason. No attempt will be made to delete anything. + +=item cannot chdir to [dir]: [errmsg] + +C attempted to set the working directory in order to +begin deleting the objects therein, but was unsuccessful. This is +usually a permissions issue. The routine will continue to delete +other things, but this directory will be left intact. + +=item directory [dir] changed before chdir, expected dev=[n] ino=[n], actual dev=[n] ino=[n], aborting. (FATAL) + +C recorded the device and inode of a directory, and then +moved into it. It then performed a C on the current directory +and detected that the device and inode were no longer the same. As +this is at the heart of the race condition problem, the program +will die at this point. + +=item cannot make directory [dir] read+writeable: [errmsg] + +C attempted to change the permissions on the current directory +to ensure that subsequent unlinkings would not run into problems, +but was unable to do so. The permissions remain as they were, and +the program will carry on, doing the best it can. + +=item cannot read [dir]: [errmsg] + +C tried to read the contents of the directory in order +to acquire the names of the directory entries to be unlinked, but +was unsuccessful. This is usually a permissions issue. The +program will continue, but the files in this directory will remain +after the call. + +=item cannot reset chmod [dir]: [errmsg] + +C, after having deleted everything in a directory, attempted +to restore its permissions to the original state but failed. The +directory may wind up being left behind. + +=item cannot remove [dir] when cwd is [dir] + +The current working directory of the program is F +and you are attempting to remove an ancestor, such as F. +The directory tree is left untouched. + +The solution is to C out of the child directory to a place +outside the directory tree to be removed. + +=item cannot chdir to [parent-dir] from [child-dir]: [errmsg], aborting. (FATAL) + +C, after having deleted everything and restored the permissions +of a directory, was unable to chdir back to the parent. The program +halts to avoid a race condition from occurring. + +=item cannot stat prior working directory [dir]: [errmsg], aborting. (FATAL) + +C was unable to stat the parent directory after have returned +from the child. Since there is no way of knowing if we returned to +where we think we should be (by comparing device and inode) the only +way out is to C. + +=item previous directory [parent-dir] changed before entering [child-dir], expected dev=[n] ino=[n], actual dev=[n] ino=[n], aborting. (FATAL) + +When C returned from deleting files in a child directory, a +check revealed that the parent directory it returned to wasn't the one +it started out from. This is considered a sign of malicious activity. + +=item cannot make directory [dir] writeable: [errmsg] + +Just before removing a directory (after having successfully removed +everything it contained), C attempted to set the permissions +on the directory to ensure it could be removed and failed. Program +execution continues, but the directory may possibly not be deleted. + +=item cannot remove directory [dir]: [errmsg] + +C attempted to remove a directory, but failed. This may because +some objects that were unable to be removed remain in the directory, or +a permissions issue. The directory will be left behind. + +=item cannot restore permissions of [dir] to [0nnn]: [errmsg] + +After having failed to remove a directory, C was unable to +restore its permissions from a permissive state back to a possibly +more restrictive setting. (Permissions given in octal). + +=item cannot make file [file] writeable: [errmsg] + +C attempted to force the permissions of a file to ensure it +could be deleted, but failed to do so. It will, however, still attempt +to unlink the file. + +=item cannot unlink file [file]: [errmsg] + +C failed to remove a file. Probably a permissions issue. + +=item cannot restore permissions of [file] to [0nnn]: [errmsg] + +After having failed to remove a file, C was also unable +to restore the permissions on the file to a possibly less permissive +setting. (Permissions given in octal). + +=back + +=head1 SEE ALSO + +=over 4 + +=item * + +L + +Allows files and directories to be moved to the Trashcan/Recycle +Bin (where they may later be restored if necessary) if the operating +system supports such functionality. This feature may one day be +made available directly in C. + +=item * + +L + +When removing directory trees, if you want to examine each file to +decide whether to delete it (and possibly leaving large swathes +alone), F offers a convenient and flexible approach +to examining directory trees. + +=back + +=head1 BUGS + +Please report all bugs on the RT queue: + +L + +=head1 ACKNOWLEDGEMENTS + +Paul Szabo identified the race condition originally, and Brendan +O'Dea wrote an implementation for Debian that addressed the problem. +That code was used as a basis for the current code. Their efforts +are greatly appreciated. + +Gisle Aas made a number of improvements to the documentation for +2.07 and his advice and assistance is also greatly appreciated. + +=head1 AUTHORS + +Tim Bunce and Charles Bailey. Currently maintained by David Landgren +>. + +=head1 COPYRIGHT + +This module is copyright (C) Charles Bailey, Tim Bunce and +David Landgren 1995-2008. All rights reserved. + +=head1 LICENSE + +This library is free software; you can redistribute it and/or modify +it under the same terms as Perl itself. + +=cut From noreply at svn.ci.uchicago.edu Tue Jul 20 02:51:34 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 20 Jul 2010 02:51:34 -0500 (CDT) Subject: [Swift-commit] r3449 - usertools/swift/swiftconfig/bin Message-ID: <20100720075135.013B29CCC2@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-20 02:51:34 -0500 (Tue, 20 Jul 2010) New Revision: 3449 Modified: usertools/swift/swiftconfig/bin/swiftconfig usertools/swift/swiftconfig/bin/swiftrun Log: Switch from File::Path remove_tree to legacy rmtree to ensure it runs on sites with older perl installations (eg. bridled and login) Modified: usertools/swift/swiftconfig/bin/swiftconfig =================================================================== --- usertools/swift/swiftconfig/bin/swiftconfig 2010-07-20 07:36:24 UTC (rev 3448) +++ usertools/swift/swiftconfig/bin/swiftconfig 2010-07-20 07:51:34 UTC (rev 3449) @@ -3,7 +3,7 @@ use Cwd; use Cwd qw(abs_path); use File::Copy; -use File::Path qw(remove_tree); +use File::Path; my $swifthome = ''; if ( $ENV{'SWIFT_HOME'} && -e "$ENV{'SWIFT_HOME'}/bin/swift" ) { @@ -293,7 +293,7 @@ "Unable to find site entry for $remove\n"; } - remove_tree("$dotswift/sites/$remove") || die "Unable to remove directory $dotswift/sites/$remove\n"; + rmtree("$dotswift/sites/$remove") || die "Unable to remove directory $dotswift/sites/$remove\n"; } # List all available templates Modified: usertools/swift/swiftconfig/bin/swiftrun =================================================================== --- usertools/swift/swiftconfig/bin/swiftrun 2010-07-20 07:36:24 UTC (rev 3448) +++ usertools/swift/swiftconfig/bin/swiftrun 2010-07-20 07:51:34 UTC (rev 3449) @@ -3,7 +3,7 @@ use Cwd; use Cwd qw(abs_path); use File::Copy; -use File::Path qw(remove_tree); +use File::Path; my $swifthome = ''; if ( $ENV{'SWIFT_HOME'} && -e "$ENV{'SWIFT_HOME'}/bin/swift" ) { @@ -107,11 +107,6 @@ } } -# Rename auths.default back to original -if(-e "$ENV{'HOME'}/.ssh/auth.defaults.$datestring") { - move("$ENV{'HOME'}/.ssh/auth.defaults.$datestring", "$ENV{'HOME'}/.ssh/auth.defaults") || die "Unable to rename $ENV{'HOME'}/.ssh/auth.defaults.$datestring"; -} - # Update XML tags open(SITEFILE, "$dotswift/jobs/$script_filename/$datestring/sites.xml") || die "Unable to open $dotswift/jobs/$script_filename/$datestring/sites.xml\n"; my @sitedata = ; @@ -125,3 +120,8 @@ print "Submitting job. Check $dotswift/jobs/$script_filename/$datestring for details\n"; system("swift -sites.file $dotswift/jobs/$script_filename/$datestring/sites.xml -tc.file $dotswift/jobs/$script_filename/$datestring/tc.data $script"); +# Rename auths.default back to original +if(-e "$ENV{'HOME'}/.ssh/auth.defaults.$datestring") { + move("$ENV{'HOME'}/.ssh/auth.defaults.$datestring", "$ENV{'HOME'}/.ssh/auth.defaults") || die "Unable to rename $ENV{'HOME'}/.ssh/auth.defaults.$datestring"; +} + From noreply at svn.ci.uchicago.edu Wed Jul 21 16:22:09 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 21 Jul 2010 16:22:09 -0500 (CDT) Subject: [Swift-commit] r3450 - in branches/tests-01/tests: . html Message-ID: <20100721212209.CAAA79CCCE@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-21 16:22:09 -0500 (Wed, 21 Jul 2010) New Revision: 3450 Added: branches/tests-01/tests/html/ branches/tests-01/tests/html/current.html branches/tests-01/tests/html/header.html Log: New location for HTML fragments Added: branches/tests-01/tests/html/current.html =================================================================== --- branches/tests-01/tests/html/current.html (rev 0) +++ branches/tests-01/tests/html/current.html 2010-07-21 21:22:09 UTC (rev 3450) @@ -0,0 +1,14 @@ + + + Redirecting... + + + + + You should be redirected to _HTMLBASE_ + + Added: branches/tests-01/tests/html/header.html =================================================================== --- branches/tests-01/tests/html/header.html (rev 0) +++ branches/tests-01/tests/html/header.html 2010-07-21 21:22:09 UTC (rev 3450) @@ -0,0 +1,31 @@ + + + Swift nightly integration tests and build (_DATE_ _TIME_) + + + + +

Swift nightly integration tests and build

+
    +
  • Date: _DATE_
  • +
  • Time: _TIME_
  • +
  • Test host: _HOST_
  • +
+
    +
  1. Test results +
  2. Compiled packages +
  3. Older tests +
  4. How to add new tests +
+ From noreply at svn.ci.uchicago.edu Wed Jul 21 16:22:39 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 21 Jul 2010 16:22:39 -0500 (CDT) Subject: [Swift-commit] r3451 - branches/tests-01/examples Message-ID: <20100721212239.38D3B9CCCE@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-21 16:22:39 -0500 (Wed, 21 Jul 2010) New Revision: 3451 Added: branches/tests-01/examples/array_index.swift branches/tests-01/examples/array_iteration.swift branches/tests-01/examples/array_wildcard.swift branches/tests-01/examples/arraymapper.swift branches/tests-01/examples/default.swift Removed: branches/tests-01/examples/array_index.dtm branches/tests-01/examples/array_iteration.dtm branches/tests-01/examples/array_wildcard.dtm branches/tests-01/examples/arraymapper.dtm branches/tests-01/examples/default.dtm Log: Move *.dtm -> *.swift Deleted: branches/tests-01/examples/array_index.dtm =================================================================== --- branches/tests-01/examples/array_index.dtm 2010-07-21 21:22:09 UTC (rev 3450) +++ branches/tests-01/examples/array_index.dtm 2010-07-21 21:22:39 UTC (rev 3451) @@ -1,13 +0,0 @@ -type file {} - - -(file t) echo_array (string s[]) { - app { - echo s[0] s[1] s[2] stdout=@filename(t); - } -} - -string greetings[] = ["how","are","you"]; -file hw = echo_array(greetings); - - Copied: branches/tests-01/examples/array_index.swift (from rev 3445, branches/tests-01/examples/array_index.dtm) =================================================================== --- branches/tests-01/examples/array_index.swift (rev 0) +++ branches/tests-01/examples/array_index.swift 2010-07-21 21:22:39 UTC (rev 3451) @@ -0,0 +1,13 @@ +type file {} + + +(file t) echo_array (string s[]) { + app { + echo s[0] s[1] s[2] stdout=@filename(t); + } +} + +string greetings[] = ["how","are","you"]; +file hw = echo_array(greetings); + + Deleted: branches/tests-01/examples/array_iteration.dtm =================================================================== --- branches/tests-01/examples/array_iteration.dtm 2010-07-21 21:22:09 UTC (rev 3450) +++ branches/tests-01/examples/array_iteration.dtm 2010-07-21 21:22:39 UTC (rev 3451) @@ -1,17 +0,0 @@ -type file {} - -(file f) echo (string s) { - app { - echo s stdout=@filename(f); - } -} - -(file fa[]) echo_batch (string sa[]) { - foreach string s, i in sa { - fa[i] = echo(s); - } -} - -string sa[] = ["hello","hi there","how are you"]; -file fa[]; -fa = echo_batch(sa); Copied: branches/tests-01/examples/array_iteration.swift (from rev 3445, branches/tests-01/examples/array_iteration.dtm) =================================================================== --- branches/tests-01/examples/array_iteration.swift (rev 0) +++ branches/tests-01/examples/array_iteration.swift 2010-07-21 21:22:39 UTC (rev 3451) @@ -0,0 +1,17 @@ +type file {} + +(file f) echo (string s) { + app { + echo s stdout=@filename(f); + } +} + +(file fa[]) echo_batch (string sa[]) { + foreach string s, i in sa { + fa[i] = echo(s); + } +} + +string sa[] = ["hello","hi there","how are you"]; +file fa[]; +fa = echo_batch(sa); Deleted: branches/tests-01/examples/array_wildcard.dtm =================================================================== --- branches/tests-01/examples/array_wildcard.dtm 2010-07-21 21:22:09 UTC (rev 3450) +++ branches/tests-01/examples/array_wildcard.dtm 2010-07-21 21:22:39 UTC (rev 3451) @@ -1,13 +0,0 @@ -type file {} - - -(file t) echo_wildcard (string s[]) { - app { - echo s[*] stdout=@filename(t); - } -} - -string greetings[] = ["how","are","you"]; -file hw = echo_wildcard(greetings); - - Copied: branches/tests-01/examples/array_wildcard.swift (from rev 3445, branches/tests-01/examples/array_wildcard.dtm) =================================================================== --- branches/tests-01/examples/array_wildcard.swift (rev 0) +++ branches/tests-01/examples/array_wildcard.swift 2010-07-21 21:22:39 UTC (rev 3451) @@ -0,0 +1,13 @@ +type file {} + + +(file t) echo_wildcard (string s[]) { + app { + echo s[*] stdout=@filename(t); + } +} + +string greetings[] = ["how","are","you"]; +file hw = echo_wildcard(greetings); + + Deleted: branches/tests-01/examples/arraymapper.dtm =================================================================== --- branches/tests-01/examples/arraymapper.dtm 2010-07-21 21:22:09 UTC (rev 3450) +++ branches/tests-01/examples/arraymapper.dtm 2010-07-21 21:22:39 UTC (rev 3451) @@ -1,8 +0,0 @@ -type file {}; - -file files[]; - -foreach f in files { - print(f); -} - Copied: branches/tests-01/examples/arraymapper.swift (from rev 3445, branches/tests-01/examples/arraymapper.dtm) =================================================================== --- branches/tests-01/examples/arraymapper.swift (rev 0) +++ branches/tests-01/examples/arraymapper.swift 2010-07-21 21:22:39 UTC (rev 3451) @@ -0,0 +1,8 @@ +type file {}; + +file files[]; + +foreach f in files { + print(f); +} + Deleted: branches/tests-01/examples/default.dtm =================================================================== --- branches/tests-01/examples/default.dtm 2010-07-21 21:22:09 UTC (rev 3450) +++ branches/tests-01/examples/default.dtm 2010-07-21 21:22:39 UTC (rev 3451) @@ -1,11 +0,0 @@ -type file {} - -(file t) echo (string s="hello world") { //s has a default value - app { - echo s stdout=@filename(t); //redirect stdout to a file - } -} - -file hw1, hw2; -hw1 = echo(); // procedure call using the default value -hw2 = echo(s="hello again"); // using a different value Copied: branches/tests-01/examples/default.swift (from rev 3445, branches/tests-01/examples/default.dtm) =================================================================== --- branches/tests-01/examples/default.swift (rev 0) +++ branches/tests-01/examples/default.swift 2010-07-21 21:22:39 UTC (rev 3451) @@ -0,0 +1,11 @@ +type file {} + +(file t) echo (string s="hello world") { //s has a default value + app { + echo s stdout=@filename(t); //redirect stdout to a file + } +} + +file hw1, hw2; +hw1 = echo(); // procedure call using the default value +hw2 = echo(s="hello again"); // using a different value From noreply at svn.ci.uchicago.edu Wed Jul 21 16:23:03 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 21 Jul 2010 16:23:03 -0500 (CDT) Subject: [Swift-commit] r3452 - branches/tests-01/tests Message-ID: <20100721212303.DEDB69CCCE@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-21 16:23:03 -0500 (Wed, 21 Jul 2010) New Revision: 3452 Modified: branches/tests-01/tests/array_iteration.swift Log: Small fix to test Modified: branches/tests-01/tests/array_iteration.swift =================================================================== --- branches/tests-01/tests/array_iteration.swift 2010-07-21 21:22:39 UTC (rev 3451) +++ branches/tests-01/tests/array_iteration.swift 2010-07-21 21:23:03 UTC (rev 3452) @@ -1,13 +1,11 @@ type file {} -(file f) echo (string s) { - app { +app (file f) echo (string s) { echo s stdout=@filename(f); - } } (file fa[]) echo_batch (string sa[]) { - foreach string s, i in sa { + foreach s, i in sa { fa[i] = echo(s); } } From noreply at svn.ci.uchicago.edu Wed Jul 21 16:25:39 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 21 Jul 2010 16:25:39 -0500 (CDT) Subject: [Swift-commit] r3453 - branches/tests-01/tests Message-ID: <20100721212539.24DAF9CCCE@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-21 16:25:38 -0500 (Wed, 21 Jul 2010) New Revision: 3453 Modified: branches/tests-01/tests/nightly.sh Log: Modifications to nightly.sh Modified: branches/tests-01/tests/nightly.sh =================================================================== --- branches/tests-01/tests/nightly.sh 2010-07-21 21:23:03 UTC (rev 3452) +++ branches/tests-01/tests/nightly.sh 2010-07-21 21:25:38 UTC (rev 3453) @@ -1,85 +1,95 @@ #!/bin/bash -OUTDIR=$PWD -LOGCOUNT=0 -SEQ=1 -DATE=`date +"%Y-%m-%d"` -TIME=`date +"%T %Z(%z)"` -RUNDIRBASE="run-$DATE" -RUNDIR=$OUTDIR/$RUNDIRBASE -mkdir -p $RUNDIR +# set -x -OUTBASE=$RUNDIRBASE/tests.log +# USAGE NOTES: +# Run nightly.sh -h for quick help +# When something goes wrong, find and check tests.log +# Code is checked out into TOPDIR +# Swift is installed in its source tree +# The run is executed in RUNDIR (TOPDIR/RUNDIRBASE) +# The build test is started in TOPDIR +# Everything for a Swift test is written in its RUNDIR +# The temporary output always goes to OUTPUT (TOPDIR/exec.out) -BRANCH="branches/tests-01" +printhelp() { + echo "nightly.sh " + echo "" + echo "usage:" + printf "\t -c Do not clean \n" + printf "\t -g Do not run grid tests \n" + printf "\t -h This message \n" + printf "\t -p Do not build the package \n" + printf "\t -s Do not do a fresh svn checkout \n" + printf "\t -x Do not continue after a failure \n" + printf "\t output Location for output (TOPDIR) \n" +} +# Defaults: +CLEAN=1 +BUILD_PACKAGE=1 +GRID_TESTS=1 +SKIP_CHECKOUT=0 +ALWAYS_EXITONFAILURE=0 +# The directory in which to start: +TOPDIR=$PWD + while [ $# -gt 0 ]; do case $1 in + -c) + CLEAN=0 + shift;; + -g) + GRID_TESTS=0 + shift;; + -h) + printhelp + exit 0;; + -p) + BUILD_PACKAGE=0 + shift;; -s) SKIP_CHECKOUT=1 shift;; -x) - EXIT_ON_ERROR=1 + ALWAYS_EXITONFAILURE=1 shift;; *) - OUTBASE=$RUNDIRBASE/$1 + TOPDIR=$1 shift;; esac done -OUT=$OUTDIR/$OUTBASE +LOGCOUNT=0 +SEQ=1 +DATE=$( date +"%Y-%m-%d" ) +TIME=$( date +"%T" ) +RUNDIRBASE="run-$DATE" +RUNDIR=$TOPDIR/$RUNDIRBASE +LOGBASE=$RUNDIRBASE/tests.log +LOG=$TOPDIR/$LOGBASE +OUTPUT=$TOPDIR/exec.out + +HTMLPATH=$RUNDIRBASE/tests-$DATE.html +HTML=$TOPDIR/$HTMLPATH + +BRANCH="branches/tests-01" + +SCRIPTDIR=$( dirname $0 ) + +cd $TOPDIR +mkdir -p $RUNDIR +[ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1 + header() { - HTMLBASE=tests-$DATE.html - HTML=$OUTDIR/$HTMLBASE - rm -f $OUTDIR/current.html + CURRENT=$SCRIPTDIR/html/current.html + sed "s at _HTMLBASE_@$HTMLPATH@" < $CURRENT > $TOPDIR/current.html - #This doesn't work well with servers that don't follow symlinks - #ln -s $HTML $OUTDIR/current.html - - cat <$OUTDIR/current.html - - - Redirecting... - - - - - You should be redirected to $HTMLBASE - - -DOH - cat <$HTML - - - Swift nightly integration tests and build ($DATE $TIME) - - - -

Swift nightly integration tests and build

-
    -
  • Date: $DATE
  • -
  • Time: $TIME
  • -
  • Test host: $(hostname)
  • -
-
    -
  1. Test results -
  2. Compiled packages -
  3. Older tests -
  4. How to add new tests -
-DOH + HEADER=$SCRIPTDIR/html/header.html + HOST=$( hostname ) + SEDCMD="s/_DATE_/$DATE/;s/_TIME_/$TIME/;s/_HOST_/$HOST"/ + sed $SEDCMD < $HEADER > $HTML FIRSTTEST=1 } @@ -87,12 +97,24 @@ echo $@ >>$HTML } +a_name() { + NAME=$1 + html "" +} + +a_href() { + HREF=$1 + TEXT=$2 + html "$TEXT" +} + footer() { - MONTHS=("" "Jan" "Feb" "Mar" "Apr" "May" "Jun" "Jul" "Aug" "Sep" "Oct" "Nov" "Dec") + MONTHS=("" "Jan" "Feb" "Mar" "Apr" "May" "Jun" \ + "Jul" "Aug" "Sep" "Oct" "Nov" "Dec") html "
" if [ "$BINPACKAGE" != "" ]; then - FBP=$OUTDIR/$BINPACKAGE + FBP=$RUNDIR/$BINPACKAGE SIZE=`ls -hs $FBP` SIZE=${SIZE/$FBP} cat <>$HTML @@ -148,7 +170,6 @@ DOH } - outecho() { TYPE=$1 shift @@ -156,7 +177,7 @@ } out() { - echo $@ + # echo $@ TYPE=$1 if [ "$TYPE" == "test" ]; then @@ -164,12 +185,11 @@ SEQ=$3 CMD=$4 RES=$5 - LOG=$6 if [ "$FIRSTTEST" == "1" ]; then html "

Test results

" - html "" - html "Output log from tests" + a_name "tests" + a_href "tests.log" "Output log from tests" html "" FIRSTTEST=0 else @@ -193,10 +213,12 @@ fi if [ "$RES" == "Passed" ]; then html "" @@ -208,30 +230,33 @@ } aexec() { - echo Executing "$@" >>$OUT - rm -f $OUTDIR/x73010test.log + declare -p PWD + echo "Executing: $@" >>$LOG + rm -fv $OUTPUT LASTCMD="$@" - "$@" >$OUTDIR/x73010test.log 2>&1 + "$@" > $OUTPUT 2>&1 + head $OUTPUT EXITCODE=$? if [ "$EXITCODE" == "127" ]; then - echo "Command not found: $@" >$OUTDIR/x73010test.log + echo "Command not found: $@" > $OUTPUT fi - if [ -f $OUTDIR/x73010test.log ]; then - cat $OUTDIR/x73010test.log >>$OUT + if [ -f $OUTPUT ]; then + cat $OUTPUT >>$LOG fi } +# TLOG = this (current) log tlog() { TLOG="output_$LOGCOUNT.txt" - rm -f $RUNDIR/$TLOG + rm -fv $TLOG banner "$LASTCMD" $RUNDIR/$TLOG - if [ -f $OUTDIR/x73010test.log ]; then - cat $OUTDIR/x73010test.log >>$RUNDIR/$TLOG 2>>$OUT + if [ -f $OUTPUT ]; then + cp -v $OUTPUT $RUNDIR/$TLOG 2>>$LOG fi - TLOG="$RUNDIRBASE/$TLOG" let "LOGCOUNT=$LOGCOUNT+1" } +# Fake exec fexec() { FLUSH=1 banner "$TEST (faked)" @@ -241,18 +266,29 @@ vtest } +stars() { + for i in {1..90} + do + printf "*" + done + echo +} + banner() { if [ "$2" == "" ]; then - BOUT=$OUT + BOUT=$LOG else BOUT=$2 fi - echo "">>$BOUT - echo "*****************************************************************************************">>$BOUT - echo "* $1" >>$BOUT - echo "*****************************************************************************************">>$BOUT + { + echo "" + # stars + echo "* $1" + # stars + } >>$BOUT } +# Execute as part of test set pexec() { banner "$TEST (part $SEQ)" echo "Executing $TEST (part $SEQ)" @@ -274,6 +310,7 @@ FLUSH=0 } +# Execute final test in set vexec() { if [ "$SEQ" == "1" ]; then banner "$TEST" @@ -295,10 +332,10 @@ RES="Failed" fi tlog - out test "$TEST" $SEQ "$LASTCMD" $RES $TLOG + out test "$TESTLINK" $SEQ "$LASTCMD" $RES $TLOG if [ "$EXITONFAILURE" == "true" ]; then if [ "$EXITCODE" != "0" ]; then - exit + exit $EXITCODE fi fi } @@ -311,18 +348,29 @@ RES="Failed" fi tlog - out test "$TEST" $SEQ "$LASTCMD" $RES $TLOG + out test "$TESTLINK" $SEQ "$LASTCMD" $RES $TLOG if [ "$EXITCODE" != "0" ]; then if [ "$EXITONFAILURE" == "true" ]; then - exit + exit $EXITCODE fi fi } -date > $OUT +build_package() { + TEST="Package" + pexec cd $SWIFT_HOME/lib + pexec rm -f castor*.jar *gt2ft*.jar ant.jar + pexec cd $TOPDIR + vexec tar -pczf $RUNDIR/swift-$DATE.tar.gz $SWIFT_HOME + out package "swift-$DATE.tar.gz" +} + +date > $LOG FLUSH=1 -head +header +cd $TOPDIR + TESTPART="Part I: Build" EXITONFAILURE=true if [ "$SKIP_CHECKOUT" != "1" ]; then @@ -335,48 +383,49 @@ pexec cd cog/modules pexec rm -rf swift vexec svn co https://svn.ci.uchicago.edu/svn/vdl2/$BRANCH swift - pexec cd $OUTDIR fi TEST="Compile" -pexec cd cog/modules/swift -pexec rm -rf dist +pexec cd $TOPDIR/cog/modules/swift +if [ $CLEAN == "1" ]; then + pexec rm -rf dist +fi vexec ant -quiet dist +SWIFT_HOME=$TOPDIR/cog/modules/swift/dist/swift-svn -TEST="Package" -pexec cd dist -SWIFT=$( ls -d swift-* ) -pexec cd $SWIFT/lib -pexec rm -f castor*.jar *gt2ft*.jar ant.jar -pexec cd ../.. -pexec rm -rf swift-$DATE -pexec mv $SWIFT swift-$DATE -vexec tar -pczf $OUTDIR/swift-$DATE.tar.gz swift-$DATE -out package "swift-$DATE.tar.gz" +if [ $BUILD_PACKAGE = "1" ]; then + build_package +fi -PATH=$PWD/swift-$DATE/bin:$PATH -cd .. -TESTDIR=$PWD/tests -echo "Path: $PATH" >>$OUT +PATH=$SWIFT_HOME/bin:$PATH +cd $TOPDIR +which swift +TESTDIR=$TOPDIR/cog/modules/swift/tests cd $RUNDIR -EXITONFAILURE=false +if [ $ALWAYS_EXITONFAILURE != "1" ]; then + EXITONFAILURE=false +fi TESTPART="Part II: Local Tests" for TEST in $( ls $TESTDIR/*.swift ); do # $TESTDIR/*.dtm TESTNAME=$( basename $TEST) - echo TESTNAME: $TESTNAME + echo TESTNAME: $TESTNAME $TESTDIR/$TESTNAME cp -uv $TESTDIR/$TESTNAME . + sed "s at _WORK_@$PWD/work@" < $TESTDIR/sites/localhost.xml > sites.xml - TEST="$TESTNAME" + TESTLINK="$TESTNAME" - # ssexec "Compile" vdlc $BN for ((i=0; $i<9; i=$i+1)); do - pexec swift -sites.file ~/.vdl2/sites-local.xml $TESTNAME + pexec swift -sites.file sites.xml $TESTNAME done - vexec swift -sites.file ~/.vdl2/sites-local.xml $TESTNAME + vexec swift -sites.file sites.xml $TESTNAME done +if [ $GRID_TESTS == "0" ]; then + exit +fi + TESTPART="Part III: Grid Tests" for TEST in `ls $TESTDIR/*.dtm $TESTDIR/*.swift`; do From noreply at svn.ci.uchicago.edu Wed Jul 21 17:00:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 21 Jul 2010 17:00:25 -0500 (CDT) Subject: [Swift-commit] r3454 - in branches/tests-01/src/org/griphyn/vdl: mapping type type/impl Message-ID: <20100721220025.41AA29CCC2@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-21 17:00:25 -0500 (Wed, 21 Jul 2010) New Revision: 3454 Modified: branches/tests-01/src/org/griphyn/vdl/mapping/AbstractDataNode.java branches/tests-01/src/org/griphyn/vdl/type/Type.java branches/tests-01/src/org/griphyn/vdl/type/impl/TypeImpl.java Log: Convert some data structures to Java 5 Modified: branches/tests-01/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- branches/tests-01/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2010-07-21 21:25:38 UTC (rev 3453) +++ branches/tests-01/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2010-07-21 22:00:25 UTC (rev 3454) @@ -25,579 +25,572 @@ public abstract class AbstractDataNode implements DSHandle { - static final String DATASET_URI_PREFIX = "tag:benc at ci.uchicago.edu,2008:swift:dataset:"; + static final String DATASET_URI_PREFIX = "tag:benc at ci.uchicago.edu,2008:swift:dataset:"; - public static final Logger logger = Logger - .getLogger(AbstractDataNode.class); + public static final Logger logger = Logger + .getLogger(AbstractDataNode.class); - public static final MappingParam PARAM_PREFIX = new MappingParam("prefix", - null); + public static final MappingParam PARAM_PREFIX = new MappingParam("prefix", + null); - /** - * Datasets are identified within a run by this sequence number and the - * partial ID field. The initial value is chosen to aid human recognition of - * sequence numbers in the wild. There is no requirement that it start at - * this (or any other) particular value. Note that this introduces a maximum - * on the number of datasets which can be dealt with in any run to be about - * 2^62. - */ - private static long datasetIDCounter = 720000000000l; + /** + * Datasets are identified within a run by this sequence number and the + * partial ID field. The initial value is chosen to aid human recognition of + * sequence numbers in the wild. There is no requirement that it start at + * this (or any other) particular value. Note that this introduces a maximum + * on the number of datasets which can be dealt with in any run to be about + * 2^62. + */ + private static long datasetIDCounter = 720000000000l; - /** - * This is used to provide a (hopefully) globally unique identifier for each - * time the datasetIDCounter is reset (whenever this class is loaded, which - * will usually happen once per JVM). No meaning should be inferred from - * this value - it exists purely for making unique URIs. - */ - private static final String datasetIDPartialID = Loader.getUUID(); + /** + * This is used to provide a (hopefully) globally unique identifier for each + * time the datasetIDCounter is reset (whenever this class is loaded, which + * will usually happen once per JVM). No meaning should be inferred from + * this value - it exists purely for making unique URIs. + */ + private static final String datasetIDPartialID = Loader.getUUID(); - private Field field; - private Map handles; - private Object value; - private boolean closed; - private List listeners; - final String identifierURI = makeIdentifierURIString(); - private Path pathFromRoot; + private Field field; + private Map handles; + private Object value; + private boolean closed; + private List listeners; + final String identifierURI = makeIdentifierURIString(); + private Path pathFromRoot; - protected AbstractDataNode(Field field) { - this.field = field; - handles = new HashMap(); - } + protected AbstractDataNode(Field field) { + this.field = field; + handles = new HashMap(); + } - public void init(Map params) { + public void init(Map params) { - } + } - public Type getType() { - return field.getType(); - } + public Type getType() { + return field.getType(); + } - public boolean isPrimitive() { - return field.getType().isPrimitive(); - } + public boolean isPrimitive() { + return field.getType().isPrimitive(); + } - public boolean isRestartable() { - return !isPrimitive(); - } + public boolean isRestartable() { + return !isPrimitive(); + } - protected Field getField() { - return field; - } + protected Field getField() { + return field; + } - /** - * create a String representation of this node. If the node has a value, - * then uses the String representation of that value. Otherwise, generates a - * text description. - */ - public String toString() { - if (this.value != null && !(this.value instanceof Exception)) { - // special handling for ints... - if (this.getType().equals(Types.INT)) { - try { - Number n = (Number) this.getValue(); - return String.valueOf(n.intValue()); - } - catch (ClassCastException e) { - throw new RuntimeException( - "Internal type error. Value is not a Number for " - + getDisplayableName() + getPathFromRoot()); - } - } - else { - return this.value.toString(); - } - } - return getIdentifyingString(); - } + /** + * create a String representation of this node. If the node has a value, + * then uses the String representation of that value. Otherwise, generates a + * text description. + */ + public String toString() { + if (this.value != null && !(this.value instanceof Exception)) { + // special handling for ints... + if (this.getType().equals(Types.INT)) { + try { + Number n = (Number) this.getValue(); + return String.valueOf(n.intValue()); + } + catch (ClassCastException e) { + throw new RuntimeException( + "Internal type error. Value is not a Number for " + + getDisplayableName() + getPathFromRoot()); + } + } + else { + return this.value.toString(); + } + } + return getIdentifyingString(); + } - public String getIdentifyingString() { - StringBuffer sb = new StringBuffer(); - sb.append(this.getClass().getName()); + public String getIdentifyingString() { + StringBuffer sb = new StringBuffer(); + sb.append(this.getClass().getName()); - sb.append(" identifier "); - sb.append(this.getIdentifier()); + sb.append(" identifier "); + sb.append(this.getIdentifier()); - sb.append(" type "); - sb.append(getType()); + sb.append(" type "); + sb.append(getType()); - if (value == null) { - sb.append(" with no value at dataset="); - } - else if (value instanceof Throwable) { - sb.append(" containing throwable "); - sb.append(value.getClass()); - sb.append(" dataset="); - } - else { - sb.append(" value="); - sb.append(this.value.toString()); - sb.append(" dataset="); - } + if (value == null) { + sb.append(" with no value at dataset="); + } + else if (value instanceof Throwable) { + sb.append(" containing throwable "); + sb.append(value.getClass()); + sb.append(" dataset="); + } + else { + sb.append(" value="); + sb.append(this.value.toString()); + sb.append(" dataset="); + } - sb.append(getDisplayableName()); + sb.append(getDisplayableName()); - if (!Path.EMPTY_PATH.equals(getPathFromRoot())) { - sb.append(" path="); - sb.append(getPathFromRoot().toString()); - } + if (!Path.EMPTY_PATH.equals(getPathFromRoot())) { + sb.append(" path="); + sb.append(getPathFromRoot().toString()); + } - if (closed) { - sb.append(" (closed)"); - } - else { - sb.append(" (not closed)"); - } + if (closed) { + sb.append(" (closed)"); + } + else { + sb.append(" (not closed)"); + } - return sb.toString(); - } + return sb.toString(); + } - public String getDisplayableName() { - String prefix = getRoot().getParam("dbgname"); - if (prefix == null) { - prefix = getRoot().getParam("prefix"); - } - if (prefix == null) { - prefix = "unnamed SwiftScript value"; - } - return prefix; - } + public String getDisplayableName() { + String prefix = getRoot().getParam("dbgname"); + if (prefix == null) { + prefix = getRoot().getParam("prefix"); + } + if (prefix == null) { + prefix = "unnamed SwiftScript value"; + } + return prefix; + } - public DSHandle getField(Path path) throws InvalidPathException { - if (path.isEmpty()) { - return this; - } - try { - DSHandle handle = getField(path.getFirst()); - if (path.size() > 1) { - return handle.getField(path.butFirst()); - } - else { - return handle; - } - } - catch (NoSuchFieldException e) { - throw new InvalidPathException(path, this); - } - } + public DSHandle getField(Path path) throws InvalidPathException { + if (path.isEmpty()) { + return this; + } + try { + DSHandle handle = getField(path.getFirst()); + if (path.size() > 1) { + return handle.getField(path.butFirst()); + } + else { + return handle; + } + } + catch (NoSuchFieldException e) { + throw new InvalidPathException(path, this); + } + } - public Collection getFields(Path path) throws InvalidPathException, - HandleOpenException { - List fields = new ArrayList(); - getFields(fields, path); - return fields; - } + public Collection getFields(Path path) + throws InvalidPathException, HandleOpenException { + List fields = new ArrayList(); + getFields(fields, path); + return fields; + } - private void getFields(List fields, Path path) throws InvalidPathException, - HandleOpenException { - if (path.isEmpty()) { - fields.add(this); - } - else { - Path rest = path.butFirst(); - if (path.isWildcard(0)) { - if (isArray() && !closed) { - throw new HandleOpenException(this); - } - Iterator i = this.handles.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); - ((AbstractDataNode) e.getValue()).getFields(fields, rest); - } - } - else { - try { - ((AbstractDataNode) getField(path.getFirst())).getFields( - fields, rest); - } - catch (NoSuchFieldException e) { - throw new InvalidPathException(path, this); - } - } - } - } + private void getFields(List fields, Path path) + throws InvalidPathException, HandleOpenException { + if (path.isEmpty()) { + fields.add(this); + } + else { + Path rest = path.butFirst(); + if (path.isWildcard(0)) { + if (isArray() && !closed) { + throw new HandleOpenException(this); + } + for (DSHandle handle : handles.values()) { + ((AbstractDataNode) handle).getFields(fields, rest); + } + } + else { + try { + ((AbstractDataNode) getField(path.getFirst())).getFields( + fields, rest); + } + catch (NoSuchFieldException e) { + throw new InvalidPathException(path, this); + } + } + } + } - public void set(DSHandle handle) { - // TODO check type - if (closed) { - throw new IllegalArgumentException(this.getDisplayableName() - + " is already assigned"); - } - if (getParent() == null) { - /* - * AbstractDataNode node = (AbstractDataNode)handle; field = - * node.getField(); handles = node.getHandles(); closed = - * node.isClosed(); value = node.getValue(); - */ - throw new RuntimeException("Can't set root data node!"); - } - else { - ((AbstractDataNode) getParent()).setField(field.getName(), handle); - } - } + public void set(DSHandle handle) { + // TODO check type + if (closed) { + throw new IllegalArgumentException(this.getDisplayableName() + + " is already assigned"); + } + if (getParent() == null) { + /* + * AbstractDataNode node = (AbstractDataNode)handle; field = + * node.getField(); handles = node.getHandles(); closed = + * node.isClosed(); value = node.getValue(); + */ + throw new RuntimeException("Can't set root data node!"); + } + else { + ((AbstractDataNode) getParent()).setField(field.getName(), handle); + } + } - protected void setField(String name, DSHandle handle) { - synchronized (handles) { - handles.put(name, handle); - } - } + protected void setField(String name, DSHandle handle) { + synchronized (handles) { + handles.put(name, handle); + } + } - protected synchronized DSHandle getField(String name) - throws NoSuchFieldException { - DSHandle handle = getHandle(name); - if (handle == null) { - if (closed) { - throw new NoSuchFieldException(name); - } - else { - handle = createDSHandle(name); - } + protected synchronized DSHandle getField(String name) + throws NoSuchFieldException { + DSHandle handle = getHandle(name); + if (handle == null) { + if (closed) { + throw new NoSuchFieldException(name); + } + else { + handle = createDSHandle(name); + } - } - return handle; - } + } + return handle; + } - protected DSHandle getHandle(String name) { - synchronized (handles) { - return (DSHandle) handles.get(name); - } - } + protected DSHandle getHandle(String name) { + synchronized (handles) { + return (DSHandle) handles.get(name); + } + } - protected boolean isHandlesEmpty() { - synchronized (handles) { - return handles.isEmpty(); - } - } + protected boolean isHandlesEmpty() { + synchronized (handles) { + return handles.isEmpty(); + } + } - public DSHandle createDSHandle(String fieldName) - throws NoSuchFieldException { - if (closed) { - throw new RuntimeException("Cannot write to closed handle: " + this - + " (" + fieldName + ")"); - } + public DSHandle createDSHandle(String fieldName) + throws NoSuchFieldException { + if (closed) { + throw new RuntimeException("Cannot write to closed handle: " + this + + " (" + fieldName + ")"); + } - AbstractDataNode child; - Field childField = getChildField(fieldName); - if (childField.getType().isArray()) { - child = new ArrayDataNode(getChildField(fieldName), getRoot(), this); - } - else { - child = new DataNode(getChildField(fieldName), getRoot(), this); - } + AbstractDataNode child; + Field childField = getChildField(fieldName); + if (childField.getType().isArray()) { + child = new ArrayDataNode(getChildField(fieldName), getRoot(), this); + } + else { + child = new DataNode(getChildField(fieldName), getRoot(), this); + } - synchronized (handles) { - Object o = handles.put(fieldName, child); - if (o != null) { - throw new RuntimeException( - "Trying to create a handle that already exists (" - + fieldName + ") in " + this); - } - } - return child; - } + synchronized (handles) { + Object o = handles.put(fieldName, child); + if (o != null) { + throw new RuntimeException( + "Trying to create a handle that already exists (" + + fieldName + ") in " + this); + } + } + return child; + } - protected Field getChildField(String fieldName) throws NoSuchFieldException { - return Field.Factory.createField(fieldName, field.getType().getField( - fieldName).getType()); - } + protected Field getChildField(String fieldName) throws NoSuchFieldException { + return Field.Factory.createField(fieldName, field.getType().getField( + fieldName).getType()); + } - protected void checkDataException() { - if (value instanceof DependentException) { - throw (DependentException) value; - } - } + protected void checkDataException() { + if (value instanceof DependentException) { + throw (DependentException) value; + } + } - protected void checkMappingException() { - if (value instanceof MappingDependentException) { - throw (MappingDependentException) value; - } - } + protected void checkMappingException() { + if (value instanceof MappingDependentException) { + throw (MappingDependentException) value; + } + } - public Object getValue() { - checkDataException(); - if (field.getType().isArray()) { - return handles; - } - else { - return value; - } - } + public Object getValue() { + checkDataException(); + if (field.getType().isArray()) { + return handles; + } + else { + return value; + } + } - public Map getArrayValue() { - checkDataException(); - if (!field.getType().isArray()) { - throw new RuntimeException("getArrayValue called on a struct: " - + this); - } - return handles; - } + public Map getArrayValue() { + checkDataException(); + if (!field.getType().isArray()) { + throw new RuntimeException + ("getArrayValue called on a struct: " + this); + } + return handles; + } - public boolean isArray() { - return false; - } + public boolean isArray() { + return false; + } - public void setValue(Object value) { - if (this.closed) { - throw new IllegalArgumentException(this.getDisplayableName() - + " is closed with a value of " + this.value); - } - if (this.value != null) { - throw new IllegalArgumentException(this.getDisplayableName() - + " is already assigned with a value of " + this.value); - } - this.value = value; - closeShallow(); - } + public void setValue(Object value) { + if (this.closed) { + throw new IllegalArgumentException(this.getDisplayableName() + + " is closed with a value of " + this.value); + } + if (this.value != null) { + throw new IllegalArgumentException(this.getDisplayableName() + + " is already assigned with a value of " + this.value); + } + this.value = value; + closeShallow(); + } - public Collection getFringePaths() throws HandleOpenException { - ArrayList list = new ArrayList(); - getFringePaths(list, Path.EMPTY_PATH); - return list; - } + public Collection getFringePaths() throws HandleOpenException { + List list = new ArrayList(); + getFringePaths(list, Path.EMPTY_PATH); + return list; + } - public void getFringePaths(List list, Path parentPath) - throws HandleOpenException { - checkMappingException(); - if (getField().getType().getBaseType() != null) { - list.add(Path.EMPTY_PATH); - } - else { - Iterator i = getField().getType().getFields().iterator(); - while (i.hasNext()) { - Field field = (Field) i.next(); - AbstractDataNode mapper; - try { - mapper = (AbstractDataNode) this.getField(field.getName()); - } - catch (NoSuchFieldException e) { - throw new RuntimeException( - "Inconsistency between type declaration and handle for field '" - + field.getName() + "'"); - } - // [m] Hmm. Should there be a difference between field and - // mapper.getField()? - // Path fullPath = - // parentPath.addLast(mapper.getField().getName()); - Path fullPath = parentPath.addLast(field.getName()); - if (!mapper.field.getType().isPrimitive() && !mapper.isArray() && mapper.field.getType().getFields().size() == 0) { - list.add(fullPath); - } - else { - mapper.getFringePaths(list, fullPath); - } - } - } - } + public void getFringePaths(List list, Path parentPath) + throws HandleOpenException { + checkMappingException(); + if (getField().getType().getBaseType() != null) { + list.add(Path.EMPTY_PATH); + } + else { + for (Field field : getField().getType().getFields()) { + AbstractDataNode mapper; + String name = field.getName(); + try { + mapper = (AbstractDataNode) this.getField(name); + } + catch (NoSuchFieldException e) { + throw new RuntimeException + ("Inconsistency between type declaration and " + + "handle for field '" + name + "'"); + } + // [m] Hmm. Should there be a difference between field and + // mapper.getField()? + // Path fullPath = + // parentPath.addLast(mapper.getField().getName()); + Path fullPath = parentPath.addLast(field.getName()); + Type type = mapper.field.getType(); + if (!type.isPrimitive() && + !mapper.isArray() && + type.getFields().size() == 0) { + list.add(fullPath); + } + else { + mapper.getFringePaths(list, fullPath); + } + } + } + } - public synchronized void closeShallow() { - if (this.closed) { - return; - } - this.closed = true; - notifyListeners(); - if (logger.isInfoEnabled()) { - logger.debug("closed " + this.getIdentifyingString()); - } - // so because its closed, we can dump the contents + public synchronized void closeShallow() { + if (this.closed) { + return; + } + this.closed = true; + notifyListeners(); + if (logger.isInfoEnabled()) { + logger.debug("closed " + this.getIdentifyingString()); + } + // so because its closed, we can dump the contents - try { - if(VDL2Config.getConfig().getProvenanceLog()) { - logContent(); - } - } - catch (Exception e) { - logger.warn("Exception whilst logging dataset content for " + this, - e); - } - // TODO record retrospective provenance information for this dataset - // here - // we should do it at closing time because that's the point at which we - // know the dataset has its values (all the way down the tree) assigned. + try { + if(VDL2Config.getConfig().getProvenanceLog()) { + logContent(); + } + } + catch (Exception e) { + logger.warn("Exception whilst logging dataset content for " + this, + e); + } + // TODO record retrospective provenance information for this dataset + // here + // we should do it at closing time because that's the point at which we + // know the dataset has its values (all the way down the tree) assigned. - // provenance-id for this dataset should have been assigned at creation - // time, - // though, so that we can refer to this dataset elsewhere before it is - // closed. + // provenance-id for this dataset should have been assigned at creation + // time, + // though, so that we can refer to this dataset elsewhere before it is + // closed. - // is this method the only one called to set this.closed? or do - // subclasses - // or other methods ever change it? - } + // is this method the only one called to set this.closed? or do + // subclasses + // or other methods ever change it? + } - public synchronized void logContent() { - String identifier = this.getIdentifier(); - Path pathFromRoot = this.getPathFromRoot(); - if (this.getPathFromRoot() != null) { - if (logger.isInfoEnabled()) { - logger.info("ROOTPATH dataset=" + identifier + " path=" - + pathFromRoot); - if (this.getType().isPrimitive()) { - logger.info("VALUE dataset=" + identifier + " VALUE=" - + this.toString()); - } - } + public synchronized void logContent() { + String identifier = this.getIdentifier(); + Path pathFromRoot = this.getPathFromRoot(); + if (this.getPathFromRoot() != null) { + if (logger.isInfoEnabled()) { + logger.info("ROOTPATH dataset=" + identifier + " path=" + + pathFromRoot); + if (this.getType().isPrimitive()) { + logger.info("VALUE dataset=" + identifier + " VALUE=" + + this.toString()); + } + } - Mapper m; + Mapper m; - try { - m = this.getMapper(); - } - catch (VDL2FutureException fe) { - m = null; // no mapping info if mapper isn't initialised yet - } - if (m != null) { - // TODO proper type here - // Not sure catching exception here is really the right thing to - // do here - // anyway - should perhaps only be trying to map leafnodes? - // Mapping - // non-leaf stuff is giving wierd paths anyway + try { + m = this.getMapper(); + } + catch (VDL2FutureException fe) { + m = null; // no mapping info if mapper isn't initialised yet + } + if (m != null) { + // TODO proper type here + // Not sure catching exception here is really the right thing to + // do here + // anyway - should perhaps only be trying to map leafnodes? + // Mapping + // non-leaf stuff is giving wierd paths anyway -// TODO this is perhaps an unpleasant way of finding if this is a file-backed -// leaf node or not - boolean filemapped = true; - Type type = this.getType(); - if(type.getName().equals("external")) { - filemapped = false; - } - if(type.isPrimitive()) { - filemapped = false; - } - if(type.isArray()) { - filemapped = false; - } - if(handles.size()>0) { - filemapped = false; - } + // TODO this is perhaps an unpleasant way of finding if this is a file-backed + // leaf node or not + boolean filemapped = true; + Type type = this.getType(); + if(type.getName().equals("external")) { + filemapped = false; + } + if(type.isPrimitive()) { + filemapped = false; + } + if(type.isArray()) { + filemapped = false; + } + if(handles.size()>0) { + filemapped = false; + } - try { - if(filemapped) { - Object path = m.map(pathFromRoot); - if (logger.isInfoEnabled()) { - logger.info("FILENAME dataset=" + identifier + " filename=" - + path); - } - } - } - catch (Exception e) { - if (logger.isInfoEnabled()) { - logger.info("NOFILENAME dataset=" + identifier); - } - } - } - } + try { + if(filemapped) { + Object path = m.map(pathFromRoot); + if (logger.isInfoEnabled()) { + logger.info("FILENAME dataset=" + identifier + " filename=" + + path); + } + } + } + catch (Exception e) { + if (logger.isInfoEnabled()) { + logger.info("NOFILENAME dataset=" + identifier); + } + } + } + } - synchronized (handles) { - Iterator i = handles.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); - AbstractDataNode node = (AbstractDataNode) e.getValue(); - if (logger.isInfoEnabled()) { - logger.info("CONTAINMENT parent=" + identifier + " child=" - + node.getIdentifier()); - } - node.logContent(); - } - } + synchronized (handles) { + //Iterator i = handles.entrySet().iterator(); + //while (i.hasNext()) { + // Map.Entry e = (Map.Entry) i.next(); + for (DSHandle handle : handles.values()) { + AbstractDataNode node = (AbstractDataNode) handle; + if (logger.isInfoEnabled()) { + logger.info("CONTAINMENT parent=" + identifier + + " child=" + node.getIdentifier()); + } + node.logContent(); + } + } + } - } + public boolean isClosed() { + return closed; + } - public boolean isClosed() { - return closed; - } + public void closeDeep() { + if (!this.closed) { + closeShallow(); + } + synchronized (handles) { + for (DSHandle handle : handles.values()) { + AbstractDataNode mapper = (AbstractDataNode) handle; + mapper.closeDeep(); + } + } + } - public void closeDeep() { - if (!this.closed) { - closeShallow(); - } - synchronized (handles) { - Iterator i = handles.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); - AbstractDataNode mapper = (AbstractDataNode) e.getValue(); - mapper.closeDeep(); - } - } - } - - /** Recursively closes arrays through a tree of arrays and complex + /** Recursively closes arrays through a tree of arrays and complex types. */ - public void closeDeepStructure() { - if (!this.closed && this.getType().isArray()) { - closeShallow(); - } - synchronized (handles) { - Iterator i = handles.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = (Map.Entry) i.next(); - AbstractDataNode child = (AbstractDataNode) e.getValue(); - if(child.getType().isArray() || - child.getType().getFields().size() > 0 ) { - child.closeDeepStructure(); - } - } - } - } + public void closeDeepStructure() { + if (!this.closed && this.getType().isArray()) { + closeShallow(); + } + synchronized (handles) { + for (DSHandle handle : handles.values()) { + AbstractDataNode child = (AbstractDataNode) handle; + if(child.getType().isArray() || + child.getType().getFields().size() > 0 ) { + child.closeDeepStructure(); + } + } + } + } + + public synchronized Path getPathFromRoot() { + if (pathFromRoot == null) { + AbstractDataNode parent = (AbstractDataNode) this.getParent(); + Path myPath; + if (parent != null) { + myPath = parent.getPathFromRoot(); + pathFromRoot = myPath.addLast(getField().getName(), parent + .getField().getType().isArray()); + } + else { + pathFromRoot = Path.EMPTY_PATH; + } + } + return pathFromRoot; + } - public synchronized Path getPathFromRoot() { - if (pathFromRoot == null) { - AbstractDataNode parent = (AbstractDataNode) this.getParent(); - Path myPath; - if (parent != null) { - myPath = parent.getPathFromRoot(); - pathFromRoot = myPath.addLast(getField().getName(), parent - .getField().getType().isArray()); - } - else { - pathFromRoot = Path.EMPTY_PATH; - } - } - return pathFromRoot; - } + public Mapper getMapper() { + return ((AbstractDataNode) getRoot()).getMapper(); + } - public Mapper getMapper() { - return ((AbstractDataNode) getRoot()).getMapper(); - } + protected Map getHandles() { + return handles; + } - protected Map getHandles() { - return handles; - } + public synchronized void addListener(DSHandleListener listener) { + if (logger.isDebugEnabled()) { + logger.debug("Adding handle listener \"" + listener + + "\" to \"" + getIdentifyingString() + "\""); + } + if (listeners == null) { + listeners = new ArrayList(); + } + listeners.add(listener); + if (closed) { + notifyListeners(); + } + } - public synchronized void addListener(DSHandleListener listener) { - if (logger.isDebugEnabled()) { - logger.debug("Adding handle listener \"" + listener + "\" to \"" - + getIdentifyingString() + "\""); - } - if (listeners == null) { - listeners = new LinkedList(); - } - listeners.add(listener); - if (closed) { - notifyListeners(); - } - } + protected synchronized void notifyListeners() { + if (listeners != null) { + for (DSHandleListener listener : listeners) { + if (logger.isDebugEnabled()) { + logger.debug("Notifying listener \"" + listener + + "\" about \"" + getIdentifyingString() + "\""); + } + listener.handleClosed(this); + } + listeners = null; + } + } - protected synchronized void notifyListeners() { - if (listeners != null) { - Iterator i = listeners.iterator(); - while (i.hasNext()) { - DSHandleListener listener = (DSHandleListener) i.next(); - i.remove(); - if (logger.isDebugEnabled()) { - logger.debug("Notifying listener \"" + listener - + "\" about \"" + getIdentifyingString() + "\""); - } - listener.handleClosed(this); - } - listeners = null; - } - } + public String getIdentifier() { + return identifierURI; + } - public String getIdentifier() { - return identifierURI; - } - - String makeIdentifierURIString() { - datasetIDCounter++; - return DATASET_URI_PREFIX + datasetIDPartialID + ":" + datasetIDCounter; - } + String makeIdentifierURIString() { + datasetIDCounter++; + return DATASET_URI_PREFIX + datasetIDPartialID + ":" + datasetIDCounter; + } } Modified: branches/tests-01/src/org/griphyn/vdl/type/Type.java =================================================================== --- branches/tests-01/src/org/griphyn/vdl/type/Type.java 2010-07-21 21:25:38 UTC (rev 3453) +++ branches/tests-01/src/org/griphyn/vdl/type/Type.java 2010-07-21 22:00:25 UTC (rev 3454) @@ -108,7 +108,7 @@ * * @return a list of Fields */ - public List getFields(); + public List getFields(); /** * get the base type of this type as a type can be derived from another type Modified: branches/tests-01/src/org/griphyn/vdl/type/impl/TypeImpl.java =================================================================== --- branches/tests-01/src/org/griphyn/vdl/type/impl/TypeImpl.java 2010-07-21 21:25:38 UTC (rev 3453) +++ branches/tests-01/src/org/griphyn/vdl/type/impl/TypeImpl.java 2010-07-21 22:00:25 UTC (rev 3454) @@ -80,8 +80,8 @@ return list; } - public List getFields() { - return new ArrayList(fields.values()); + public List getFields() { + return new ArrayList(fields.values()); } public Type getBaseType() { From noreply at svn.ci.uchicago.edu Wed Jul 21 17:02:03 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 21 Jul 2010 17:02:03 -0500 (CDT) Subject: [Swift-commit] r3455 - branches/tests-01/tests/sites Message-ID: <20100721220203.98C639CCC2@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-21 17:02:02 -0500 (Wed, 21 Jul 2010) New Revision: 3455 Added: branches/tests-01/tests/sites/localhost.xml Log: localhost sites file for use by nightly.sh Added: branches/tests-01/tests/sites/localhost.xml =================================================================== --- branches/tests-01/tests/sites/localhost.xml (rev 0) +++ branches/tests-01/tests/sites/localhost.xml 2010-07-21 22:02:02 UTC (rev 3455) @@ -0,0 +1,12 @@ + + + + + + + + + _WORK_ + + + From noreply at svn.ci.uchicago.edu Wed Jul 21 17:03:40 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 21 Jul 2010 17:03:40 -0500 (CDT) Subject: [Swift-commit] r3456 - branches/tests-01/libexec/log-processing Message-ID: <20100721220340.A36859CCC2@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-21 17:03:40 -0500 (Wed, 21 Jul 2010) New Revision: 3456 Added: branches/tests-01/libexec/log-processing/README.txt Log: A start on notes for log-processing Added: branches/tests-01/libexec/log-processing/README.txt =================================================================== --- branches/tests-01/libexec/log-processing/README.txt (rev 0) +++ branches/tests-01/libexec/log-processing/README.txt 2010-07-21 22:03:40 UTC (rev 3456) @@ -0,0 +1,8 @@ + +To properly generate log plots, you must: + +1) Enable VDL/Karajan logging. + +Make sure log4.properties contains: + +log4j.logger.swift=DEBUG From noreply at svn.ci.uchicago.edu Thu Jul 22 11:34:09 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 22 Jul 2010 11:34:09 -0500 (CDT) Subject: [Swift-commit] r3457 - in branches/tests-01: . libexec src/org/griphyn/vdl/karajan/lib/swiftscript tests/language-behaviour Message-ID: <20100722163409.F11F49CCAF@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-22 11:34:09 -0500 (Thu, 22 Jul 2010) New Revision: 3457 Added: branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java branches/tests-01/tests/language-behaviour/tracef.swift Modified: branches/tests-01/.classpath branches/tests-01/libexec/vdl-lib.xml branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: Move tracef() to its own class and add test Modified: branches/tests-01/.classpath =================================================================== --- branches/tests-01/.classpath 2010-07-21 22:03:40 UTC (rev 3456) +++ branches/tests-01/.classpath 2010-07-22 16:34:09 UTC (rev 3457) @@ -21,5 +21,7 @@ + + Modified: branches/tests-01/libexec/vdl-lib.xml =================================================================== --- branches/tests-01/libexec/vdl-lib.xml 2010-07-21 22:03:40 UTC (rev 3456) +++ branches/tests-01/libexec/vdl-lib.xml 2010-07-22 16:34:09 UTC (rev 3457) @@ -8,14 +8,14 @@ - + - + @@ -88,7 +88,7 @@ - + @@ -96,12 +96,12 @@ - + - + - + Modified: branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java =================================================================== --- branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java 2010-07-21 22:03:40 UTC (rev 3456) +++ branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Java.java 2010-07-22 16:34:09 UTC (rev 3457) @@ -132,8 +132,8 @@ catch (Exception e) { e.printStackTrace(); throw new RuntimeException - ("Error attempting to invoke: " + method.getDeclaringClass() + - "." + method); + ("Error attempting to invoke: " + + method.getDeclaringClass() + "." + method); } return result; } Modified: branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-21 22:03:40 UTC (rev 3456) +++ branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-22 16:34:09 UTC (rev 3457) @@ -11,14 +11,12 @@ import org.globus.cog.karajan.util.TypeUtil; import org.globus.cog.karajan.workflow.ExecutionException; import org.globus.cog.karajan.workflow.nodes.functions.FunctionsCollection; -import org.griphyn.vdl.karajan.lib.PathUtils; import org.griphyn.vdl.karajan.lib.SwiftArg; import org.griphyn.vdl.karajan.lib.VDLFunction; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Path; -import org.griphyn.vdl.mapping.ArrayDataNode; import org.griphyn.vdl.mapping.RootArrayDataNode; import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.NoSuchTypeException; @@ -38,7 +36,6 @@ static { setArguments("swiftscript_trace", new Arg[] { Arg.VARGS }); - setArguments("swiftscript_tracef", new Arg[] { Arg.VARGS }); setArguments("swiftscript_strcat", new Arg[] { Arg.VARGS }); setArguments("swiftscript_strcut", new Arg[] { PA_INPUT, PA_PATTERN }); setArguments("swiftscript_strstr", new Arg[] { PA_INPUT, PA_PATTERN }); @@ -47,7 +44,7 @@ setArguments("swiftscript_toint", new Arg[] { PA_INPUT }); setArguments("swiftscript_tofloat", new Arg[] { PA_INPUT }); setArguments("swiftscript_tostring", new Arg[] { PA_INPUT }); - setArguments("swiftscript_dirname", new Arg[] { Arg.VARGS }); + setArguments("swiftscript_dirname", new Arg[] { Arg.VARGS }); } private static final Logger traceLogger = Logger.getLogger("org.globus.swift.trace"); @@ -67,171 +64,7 @@ traceLogger.warn(buf); return null; } - - /** - Formatted trace output.
- Example: tracef("\t%s\n", "hello");
- Differences from trace(): - 1) respects \t, \n and \\; - 2) allows for typechecked format specifiers; - 3) allows for consumption of variables without display (%k); - 4) does not impose any formatting (commas, etc.).

- Format specifiers:
- %%: % sign.
- %p: Not typechecked, output as in trace().
- %i: Typechecked int output.
- %s: Typechecked string output.
- %k: Variable sKipped, no output. - */ - public DSHandle swiftscript_tracef(VariableStack stack) - throws ExecutionException, NoSuchTypeException, - InvalidPathException { - DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); - - StringBuffer buf = new StringBuffer(); - for (int i = 0; i < args.length; i++) { - DSHandle handle = args[i]; - VDLFunction.waitFor(stack, handle); - } - String msg = format(args); - buf.append(msg); - traceLogger.warn(buf); - return null; - } - - /** - Helper for {@link #swiftscript_tracef}. - */ - private String format(DSHandle[] args) throws ExecutionException { - if (! (args[0].getType() == Types.STRING)) - throw new ExecutionException("First argument to tracef() must be a string!"); - - String spec = args[0].toString(); - StringBuffer output = new StringBuffer(); - int i = 0; - int a = 1; - while (i < spec.length()) { - char c = spec.charAt(i); - if (c == '%') { - char d = spec.charAt(++i); - a = append(d, a, args, output); - } - else if (c == '\\') { - char d = spec.charAt(++i); - escape(d, output); - } - else { - output.append(c); - } - i++; - } - String result = output.toString(); - return result; - } - - /** - Helper for {@link #swiftscript_tracef}. - */ - private int append(char c, int arg, DSHandle[] args, StringBuffer output) throws ExecutionException { - if (c == '%') { - output.append('%'); - return arg; - } - if (arg >= args.length) { - throw new ExecutionException("tracef(): too many specifiers!"); - } - if (c == 'p') { - output.append(args[arg].toString()); - } - else if (c == 'M') { - try { - synchronized (args[arg].getRoot()) { - String[] names = VDLFunction.filename(args[arg]); - if (names.length > 1) - output.append(names); - else - output.append(names[0]); - } - } - catch (Exception e) { - throw new ExecutionException("tracef(%M): Could not lookup: " + args[arg]); - } - } - else if (c == 's') { - if (args[arg].getType() == Types.STRING) { - output.append(args[arg]).toString(); - } - else { - throw new ExecutionException("tracef(): %s requires a string!"); - } - } - else if (c == 'i') { - if (args[arg].getType() == Types.INT) { - output.append(args[arg]).toString(); - } - else { - throw new ExecutionException("tracef(): %i requires an int!"); - } - } - else if (c == 'f') { - if (args[arg].getType() == Types.FLOAT) { - output.append(args[arg]).toString(); - } - else { - throw new ExecutionException("tracef(): %f requires a float!"); - } - } - else if (c == 'q') { - if (args[arg] instanceof ArrayDataNode) { - ArrayDataNode node = (ArrayDataNode) args[arg]; - output.append("["); - try { - int size = node.size(); - for (int i = 0; i < size; i++) { - String entry = ""+i; - DSHandle handle = node.getField(Path.parse(entry)); - output.append(handle); - if (i < size-1) - output.append(","); - } - } - catch (Exception e) { - e.printStackTrace(); - throw new ExecutionException("trace(%q): Could not get children of: " + args[arg]); - } - output.append("]"); - } - else { - throw new ExecutionException("tracef(): %q requires an array!"); - } - } - else if (c == 'k') { - ; - } - else { - throw new ExecutionException("tracef(): Unknown format: %" + c); - } - return arg+1; - } - - /** - Helper for {@link #swiftscript_tracef}. - */ - private void escape(char c, StringBuffer output) throws ExecutionException { - if (c == '\\') { - output.append('\\'); - } - else if (c == 'n') { - output.append('\n'); - } - else if (c == 't') { - output.append('\t'); - } - else { - throw new ExecutionException("tracef(): unknown backslash escape sequence!"); - } - } - + public DSHandle swiftscript_strcat(VariableStack stack) throws ExecutionException, NoSuchTypeException, InvalidPathException { Object[] args = SwiftArg.VARGS.asArray(stack); Added: branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java =================================================================== --- branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java (rev 0) +++ branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-07-22 16:34:09 UTC (rev 3457) @@ -0,0 +1,194 @@ +package org.griphyn.vdl.karajan.lib.swiftscript; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.karajan.lib.SwiftArg; +import org.griphyn.vdl.karajan.lib.VDLFunction; +import org.griphyn.vdl.mapping.ArrayDataNode; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; +import org.griphyn.vdl.type.Types; + +/** + Formatted trace output.
+ Example: tracef("\t%s\n", "hello");
+ Differences from trace(): + 1) respects \t, \n and \\; + 2) allows for typechecked format specifiers; + 3) allows for consumption of variables without display (%k); + 4) does not impose any formatting (commas, etc.).

+ Format specifiers:
+ %%: % sign.
+ %p: Not typechecked, output as in trace().
+ %i: Typechecked int output.
+ %s: Typechecked string output.
+ %k: Variable sKipped, no output. + */ +public class Tracef extends VDLFunction { + + private static final Logger logger = + Logger.getLogger(Tracef.class); + + static { + setArguments(Tracef.class, new Arg[] { Arg.VARGS }); + } + + @Override + protected Object function(VariableStack stack) + throws ExecutionException { + DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); + + for (int i = 0; i < args.length; i++) { + DSHandle handle = args[i]; + VDLFunction.waitFor(stack, handle); + } + String msg = format(args); + logger.info(msg); + System.out.print(msg); + return null; + } + + private String format(DSHandle[] args) + throws ExecutionException { + if (! (args[0].getType() == Types.STRING)) + throw new ExecutionException + ("First argument to tracef() must be a string!"); + + String spec = args[0].toString(); + StringBuffer output = new StringBuffer(); + int i = 0; + int a = 1; + while (i < spec.length()) { + char c = spec.charAt(i); + if (c == '%') { + char d = spec.charAt(++i); + a = append(d, a, args, output); + } + else if (c == '\\') { + char d = spec.charAt(++i); + escape(d, output); + } + else { + output.append(c); + } + i++; + } + String result = output.toString(); + return result; + } + + private int append(char c, int arg, DSHandle[] args, + StringBuffer output) + throws ExecutionException { + if (c == '%') { + output.append('%'); + return arg; + } + if (arg >= args.length) { + throw new ExecutionException + ("tracef(): too many specifiers!"); + } + if (c == 'p') { + output.append(args[arg].toString()); + } + else if (c == 'M') { + try { + synchronized (args[arg].getRoot()) { + String[] names = VDLFunction.filename(args[arg]); + if (names.length > 1) + output.append(names); + else + output.append(names[0]); + } + } + catch (Exception e) { + throw new ExecutionException + ("tracef(%M): Could not lookup: " + args[arg]); + } + } + else if (c == 's') { + if (args[arg].getType() == Types.STRING) { + output.append(args[arg]).toString(); + } + else { + throw new ExecutionException + ("tracef(): %s requires a string!"); + } + } + else if (c == 'i') { + if (args[arg].getType() == Types.INT) { + output.append(args[arg]).toString(); + } + else { + throw new ExecutionException + ("tracef(): %i requires an int!"); + } + } + else if (c == 'f') { + if (args[arg].getType() == Types.FLOAT) { + output.append(args[arg]).toString(); + } + else { + throw new ExecutionException + ("tracef(): %f requires a float!"); + } + } + else if (c == 'q') { + if (args[arg] instanceof ArrayDataNode) { + ArrayDataNode node = (ArrayDataNode) args[arg]; + output.append("["); + try { + int size = node.size(); + for (int i = 0; i < size; i++) { + String entry = ""+i; + DSHandle handle = + node.getField(Path.parse(entry)); + output.append(handle); + if (i < size-1) + output.append(","); + } + } + catch (Exception e) { + e.printStackTrace(); + throw new ExecutionException + ("trace(%q): Could not get children of: " + + args[arg]); + } + output.append("]"); + } + else { + throw new ExecutionException + ("tracef(): %q requires an array!"); + } + } + else if (c == 'k') { + ; + } + else { + throw new ExecutionException + ("tracef(): Unknown format: %" + c); + } + return arg+1; + } + + private void escape(char c, StringBuffer output) + throws ExecutionException { + if (c == '\\') { + output.append('\\'); + } + else if (c == 'n') { + output.append('\n'); + } + else if (c == 't') { + output.append('\t'); + } + else { + throw new ExecutionException + ("tracef(): unknown backslash escape sequence!"); + } + } + + +} Added: branches/tests-01/tests/language-behaviour/tracef.swift =================================================================== --- branches/tests-01/tests/language-behaviour/tracef.swift (rev 0) +++ branches/tests-01/tests/language-behaviour/tracef.swift 2010-07-22 16:34:09 UTC (rev 3457) @@ -0,0 +1,4 @@ + +tracef("3:%i\n", 3); +tracef("4:%s\n", "4"); +tracef("fraction:%f\n", 22.0/7.0); From noreply at svn.ci.uchicago.edu Thu Jul 22 11:35:26 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 22 Jul 2010 11:35:26 -0500 (CDT) Subject: [Swift-commit] r3458 - branches/tests-01 Message-ID: <20100722163526.E5B189CCAF@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-22 11:35:26 -0500 (Thu, 22 Jul 2010) New Revision: 3458 Modified: branches/tests-01/ Log: Ignore dist directory Property changes on: branches/tests-01 ___________________________________________________________________ Name: svn:ignore + dist From noreply at svn.ci.uchicago.edu Thu Jul 22 11:35:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 22 Jul 2010 11:35:55 -0500 (CDT) Subject: [Swift-commit] r3459 - branches/tests-01/lib Message-ID: <20100722163555.50A749CCAF@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-22 11:35:55 -0500 (Thu, 22 Jul 2010) New Revision: 3459 Modified: branches/tests-01/lib/ Log: Ignore vdldefinitions.jar Property changes on: branches/tests-01/lib ___________________________________________________________________ Name: svn:ignore + vdldefinitions.jar From noreply at svn.ci.uchicago.edu Thu Jul 22 11:55:44 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 22 Jul 2010 11:55:44 -0500 (CDT) Subject: [Swift-commit] r3460 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100722165545.022589CCCE@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-22 11:55:44 -0500 (Thu, 22 Jul 2010) New Revision: 3460 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: o swift/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java -- Fixed @totint fucntion. This function failed if passed a string that started with a "." Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-22 16:35:55 UTC (rev 3459) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-22 16:55:44 UTC (rev 3460) @@ -368,10 +368,14 @@ InvalidPathException { String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); int i = inputString.indexOf("."); - if( i >= 0 ) + if( i > 0 ) { inputString = inputString.substring(0, i); } + if( i == 0 ) + { + inputString = "0"; + } DSHandle handle = new RootDataNode(Types.INT); handle.setValue(new Double(Integer.parseInt(inputString))); handle.closeShallow(); From noreply at svn.ci.uchicago.edu Thu Jul 22 12:14:29 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 22 Jul 2010 12:14:29 -0500 (CDT) Subject: [Swift-commit] r3461 - branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100722171429.09F019CCCE@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-22 12:14:28 -0500 (Thu, 22 Jul 2010) New Revision: 3461 Modified: branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java Log: Reorganize tracef Modified: branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java =================================================================== --- branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-07-22 16:55:44 UTC (rev 3460) +++ branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-07-22 17:14:28 UTC (rev 3461) @@ -22,9 +22,11 @@ Format specifiers:
%%: % sign.
%p: Not typechecked, output as in trace().
+ %f: Typechecked float output.
%i: Typechecked int output.
%s: Typechecked string output.
- %k: Variable sKipped, no output. + %k: Variable sKipped, no output.
+ %q: Array output */ public class Tracef extends VDLFunction { @@ -90,89 +92,113 @@ throw new ExecutionException ("tracef(): too many specifiers!"); } - if (c == 'p') { + if (c == 'M') { + append_M(args[arg], output); + } + else if (c == 'f') { + append_f(args[arg], output); + } + else if (c == 'i') { + append_i(args[arg], output); + } + else if (c == 'p') { output.append(args[arg].toString()); } - else if (c == 'M') { - try { - synchronized (args[arg].getRoot()) { - String[] names = VDLFunction.filename(args[arg]); - if (names.length > 1) - output.append(names); - else - output.append(names[0]); - } - } - catch (Exception e) { - throw new ExecutionException - ("tracef(%M): Could not lookup: " + args[arg]); - } - } else if (c == 's') { - if (args[arg].getType() == Types.STRING) { - output.append(args[arg]).toString(); - } - else { - throw new ExecutionException - ("tracef(): %s requires a string!"); - } + append_s(args[arg], output); } - else if (c == 'i') { - if (args[arg].getType() == Types.INT) { - output.append(args[arg]).toString(); - } - else { - throw new ExecutionException - ("tracef(): %i requires an int!"); - } + else if (c == 'q') { + append_q(args[arg], output); } - else if (c == 'f') { - if (args[arg].getType() == Types.FLOAT) { - output.append(args[arg]).toString(); + else if (c == 'k') { + ; + } + else { + throw new ExecutionException + ("tracef(): Unknown format: %" + c); + } + return arg+1; + } + + private void append_M(DSHandle arg, StringBuffer output) + throws ExecutionException { + try { + synchronized (arg.getRoot()) { + String[] names = VDLFunction.filename(arg); + if (names.length > 1) + output.append(names); + else + output.append(names[0]); } - else { - throw new ExecutionException - ("tracef(): %f requires a float!"); - } } - else if (c == 'q') { - if (args[arg] instanceof ArrayDataNode) { - ArrayDataNode node = (ArrayDataNode) args[arg]; - output.append("["); - try { - int size = node.size(); - for (int i = 0; i < size; i++) { - String entry = ""+i; - DSHandle handle = - node.getField(Path.parse(entry)); - output.append(handle); - if (i < size-1) - output.append(","); - } + catch (Exception e) { + throw new ExecutionException + ("tracef(%M): Could not lookup: " + arg); + } + } + + private void append_f(DSHandle arg, StringBuffer output) + throws ExecutionException { + if (arg.getType() == Types.FLOAT) { + output.append(arg).toString(); + } + else { + throw new ExecutionException + ("tracef(): %f requires a float!"); + } + } + + private void append_i(DSHandle arg, StringBuffer output) + throws ExecutionException { + if (arg.getType() == Types.INT) { + output.append(arg).toString(); + } + else { + throw new ExecutionException + ("tracef(): %i requires an int!"); + } + } + + private void append_q(DSHandle arg, StringBuffer output) + throws ExecutionException { + if (arg instanceof ArrayDataNode) { + ArrayDataNode node = (ArrayDataNode) arg; + output.append("["); + try { + int size = node.size(); + for (int i = 0; i < size; i++) { + String entry = ""+i; + DSHandle handle = + node.getField(Path.parse(entry)); + output.append(handle); + if (i < size-1) + output.append(","); } - catch (Exception e) { - e.printStackTrace(); - throw new ExecutionException - ("trace(%q): Could not get children of: " + - args[arg]); - } - output.append("]"); } - else { + catch (Exception e) { + e.printStackTrace(); throw new ExecutionException - ("tracef(): %q requires an array!"); + ("trace(%q): Could not get children of: " + arg); } + output.append("]"); } - else if (c == 'k') { - ; + else { + throw new ExecutionException + ("tracef(): %q requires an array!"); + } + } + + private void append_s(DSHandle arg, StringBuffer output) + throws ExecutionException { + if (arg.getType() == Types.STRING) { + output.append(arg).toString(); } else { throw new ExecutionException - ("tracef(): Unknown format: %" + c); + ("tracef(): %s requires a string!"); } - return arg+1; } - + private void escape(char c, StringBuffer output) throws ExecutionException { if (c == '\\') { From noreply at svn.ci.uchicago.edu Thu Jul 22 12:28:23 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 22 Jul 2010 12:28:23 -0500 (CDT) Subject: [Swift-commit] r3462 - branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100722172823.8FDC09CCCE@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-22 12:28:23 -0500 (Thu, 22 Jul 2010) New Revision: 3462 Modified: branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java Log: Minor change Modified: branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java =================================================================== --- branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-07-22 17:14:28 UTC (rev 3461) +++ branches/tests-01/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2010-07-22 17:28:23 UTC (rev 3462) @@ -61,12 +61,12 @@ String spec = args[0].toString(); StringBuffer output = new StringBuffer(); int i = 0; - int a = 1; + int arg = 1; while (i < spec.length()) { char c = spec.charAt(i); if (c == '%') { char d = spec.charAt(++i); - a = append(d, a, args, output); + arg = append(d, arg, args, output); } else if (c == '\\') { char d = spec.charAt(++i); From noreply at svn.ci.uchicago.edu Thu Jul 22 13:44:51 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 22 Jul 2010 13:44:51 -0500 (CDT) Subject: [Swift-commit] r3464 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100722184451.9447C9CCCE@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-22 13:44:51 -0500 (Thu, 22 Jul 2010) New Revision: 3464 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: o swift/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java -- Minor change in toint function. Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-22 17:35:05 UTC (rev 3463) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-22 18:44:51 UTC (rev 3464) @@ -206,7 +206,7 @@ inputString = inputString.substring(0, i); } DSHandle handle = new RootDataNode(Types.INT); - handle.setValue(new Double(Integer.parseInt(inputString))); + handle.setValue( new Integer( inputString ) ); handle.closeShallow(); int provid=VDLFunction.nextProvenanceID(); VDLFunction.logProvenanceResult(provid, handle, "toint"); From noreply at svn.ci.uchicago.edu Thu Jul 22 17:09:53 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 22 Jul 2010 17:09:53 -0500 (CDT) Subject: [Swift-commit] r3465 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100722220953.1C2249CCAF@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-22 17:09:52 -0500 (Thu, 22 Jul 2010) New Revision: 3465 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: Undo change to @toint() Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-22 18:44:51 UTC (rev 3464) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-22 22:09:52 UTC (rev 3465) @@ -206,7 +206,7 @@ inputString = inputString.substring(0, i); } DSHandle handle = new RootDataNode(Types.INT); - handle.setValue( new Integer( inputString ) ); + handle.setValue(new Double(inputString)); handle.closeShallow(); int provid=VDLFunction.nextProvenanceID(); VDLFunction.logProvenanceResult(provid, handle, "toint"); From noreply at svn.ci.uchicago.edu Fri Jul 23 23:16:02 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:16:02 -0500 (CDT) Subject: [Swift-commit] r3466 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20100724041602.510249CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:16:02 -0500 (Fri, 23 Jul 2010) New Revision: 3466 Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java trunk/src/org/griphyn/vdl/mapping/RootDataNode.java Log: renamed slightly misleading method Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2010-07-22 22:09:52 UTC (rev 3465) +++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2010-07-24 04:16:02 UTC (rev 3466) @@ -516,19 +516,20 @@ } } } - - /** Recursively closes arrays through a tree of arrays and complex - types. */ - public void closeDeepStructure() { + + /** + * Recursively closes arrays through a tree of arrays and complex types. + */ + public void closeArraySizes() { if (!this.closed && this.getType().isArray()) { closeShallow(); } synchronized (handles) { for (DSHandle handle : handles.values()) { AbstractDataNode child = (AbstractDataNode) handle; - if(child.getType().isArray() || - child.getType().getFields().size() > 0 ) { - child.closeDeepStructure(); + if (child.getType().isArray() || + child.getType().getFields().size() > 0) { + child.closeArraySizes(); } } } Modified: trunk/src/org/griphyn/vdl/mapping/RootDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2010-07-22 22:09:52 UTC (rev 3465) +++ trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2010-07-24 04:16:02 UTC (rev 3466) @@ -113,6 +113,8 @@ checkConsistency(root); } else if (mapper.isStatic()) { + // Static mappers are (array) mappers which know the size of + // an array statically. A good example is the fixed array mapper Iterator i = mapper.existing().iterator(); while (i.hasNext()) { Path p = (Path) i.next(); @@ -129,7 +131,7 @@ } } if (root.isArray()) { - root.closeDeepStructure(); + root.closeArraySizes(); } checkConsistency(root); } From noreply at svn.ci.uchicago.edu Fri Jul 23 23:22:18 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:22:18 -0500 (CDT) Subject: [Swift-commit] r3467 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20100724042218.6B13B9CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:22:18 -0500 (Fri, 23 Jul 2010) New Revision: 3467 Modified: trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java Log: user friendly error vs. NPE Modified: trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2010-07-24 04:16:02 UTC (rev 3466) +++ trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2010-07-24 04:22:18 UTC (rev 3467) @@ -21,6 +21,9 @@ public Collection existing() { List l = new ArrayList(); DSHandle dn = (DSHandle) PARAM_FILES.getRawValue(this); + if (dn == null) { + throw new RuntimeException("Missing 'files' mapper attribute"); + } Map m = dn.getArrayValue(); Set s = m.keySet(); Iterator i = s.iterator(); @@ -32,7 +35,6 @@ } public PhysicalFormat map(Path path) { - if (path.isEmpty()) { throw new IllegalArgumentException("Path cannot be empty"); } From noreply at svn.ci.uchicago.edu Fri Jul 23 23:27:28 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:27:28 -0500 (CDT) Subject: [Swift-commit] r3468 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100724042728.A359A9CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:27:28 -0500 (Fri, 23 Jul 2010) New Revision: 3468 Added: trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java trunk/src/org/griphyn/vdl/karajan/lib/Mark.java trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java Modified: trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java Log: added java implementations for some functions in vdl-int Added: trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java 2010-07-24 04:27:28 UTC (rev 3468) @@ -0,0 +1,51 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 18, 2010 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.arguments.VariableArguments; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; + +public class Flatten extends VDLFunction { + + static { + setArguments(Flatten.class, new Arg[] { Arg.VARGS }); + } + + @Override + protected Object function(VariableStack stack) throws ExecutionException { + VariableArguments v = Arg.VARGS.get(stack); + if (v.isEmpty()) { + return ""; + } + else { + StringBuilder sb = new StringBuilder(); + flatten(sb, v.getAll()); + sb.deleteCharAt(sb.length() - 1); + return sb.toString(); + } + } + + private void flatten(StringBuilder sb, List l) { + for (Object o : l) { + if (o instanceof List) { + flatten(sb, (List) o); + } + else { + sb.append(TypeUtil.toString(o)); + sb.append('|'); + } + } + } +} Added: trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2010-07-24 04:27:28 UTC (rev 3468) @@ -0,0 +1,46 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 18, 2010 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.arguments.ArgUtil; +import org.globus.cog.karajan.arguments.VariableArguments; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.griphyn.vdl.mapping.AbsFile; + +public class InFileDirs extends AbstractSequentialWithArguments { + public static final Arg STAGEINS = new Arg.Positional("stageins"); + + static { + setArguments(InFileDirs.class, new Arg[] { STAGEINS }); + } + + @Override + protected void post(VariableStack stack) throws ExecutionException { + List files = TypeUtil.toList(STAGEINS.getValue(stack)); + VariableArguments ret = ArgUtil.getVariableReturn(stack); + for (Object f : files) { + String path = (String) f; + String dir = new AbsFile(path).getDir(); + if (dir.startsWith("/")) { + ret.append(dir.substring(1)); + } + else { + ret.append(dir); + } + } + super.post(stack); + } +} Added: trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java 2010-07-24 04:27:28 UTC (rev 3468) @@ -0,0 +1,46 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 18, 2010 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.Path; + +public class IsDone extends VDLFunction { + public static final Arg STAGEOUT = new Arg.Positional("stageout"); + + static { + setArguments(IsDone.class, new Arg[] { STAGEOUT }); + } + + @Override + protected Object function(VariableStack stack) throws ExecutionException, + HandleOpenException { + List files = TypeUtil.toList(STAGEOUT.getValue(stack)); + for (Object f : files) { + List pv = TypeUtil.toList(f); + Path p = Path.parse(TypeUtil.toString(pv.get(0))); + DSHandle handle = (DSHandle) pv.get(1); + if (!IsLogged.isLogged(stack, handle, p)) { + return Boolean.FALSE; + } + } + if (files.isEmpty()) { + return Boolean.FALSE; + } + return Boolean.TRUE; + } +} Modified: trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java 2010-07-24 04:22:18 UTC (rev 3467) +++ trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java 2010-07-24 04:27:28 UTC (rev 3468) @@ -29,16 +29,20 @@ else { path = Path.parse(TypeUtil.toString(p)); } - path = var.getPathFromRoot().append(path); - LogEntry entry = LogEntry.build(var.getRoot().getParam("swift#restartid") + "." + path.stringForm()); - Map map = getLogData(stack); - boolean found = false; - synchronized (map) { - List files = (List) map.get(entry); - if (files != null && !files.isEmpty()) { - found = true; - } - } - return Boolean.valueOf(found); + return Boolean.valueOf(isLogged(stack, var, path)); } + + public static boolean isLogged(VariableStack stack, DSHandle var, Path path) throws ExecutionException { + path = var.getPathFromRoot().append(path); + LogEntry entry = LogEntry.build(var.getRoot().getParam("swift#restartid") + "." + path.stringForm()); + Map map = getLogData(stack); + boolean found = false; + synchronized (map) { + List files = (List) map.get(entry); + if (files != null && !files.isEmpty()) { + found = true; + } + } + return found; + } } Added: trunk/src/org/griphyn/vdl/karajan/lib/Mark.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Mark.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/Mark.java 2010-07-24 04:27:28 UTC (rev 3468) @@ -0,0 +1,62 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 18, 2010 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.DataDependentException; +import org.griphyn.vdl.mapping.HandleOpenException; +import org.griphyn.vdl.mapping.MappingDependentException; +import org.griphyn.vdl.mapping.Path; + +public class Mark extends VDLFunction { + public static final Arg RESTARTS = new Arg.Positional("restarts"); + public static final Arg ERR = new Arg.Positional("err"); + public static final Arg MAPPING = new Arg.Optional("mapping", Boolean.FALSE); + + static { + setArguments(Mark.class, new Arg[] { RESTARTS, ERR, MAPPING }); + } + + @Override + protected Object function(VariableStack stack) throws ExecutionException, + HandleOpenException { + try { + if (TypeUtil.toBoolean(ERR.getValue(stack))) { + boolean mapping = TypeUtil.toBoolean(MAPPING.getValue(stack)); + List files = TypeUtil.toList(RESTARTS.getValue(stack)); + for (Object f : files) { + List pv = TypeUtil.toList(f); + Path p = Path.parse(TypeUtil.toString(pv.get(0))); + DSHandle handle = (DSHandle) pv.get(1); + DSHandle leaf = handle.getField(p); + synchronized (leaf) { + if (mapping) { + leaf.setValue(new MappingDependentException(leaf, null)); + } + else { + leaf.setValue(new DataDependentException(leaf, null)); + } + closeShallow(stack, leaf); + } + } + } + } + catch (Exception e) { + throw new ExecutionException(e); + } + return null; + } +} Added: trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java 2010-07-24 04:27:28 UTC (rev 3468) @@ -0,0 +1,52 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 18, 2010 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.arguments.ArgUtil; +import org.globus.cog.karajan.arguments.VariableArguments; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.griphyn.vdl.mapping.AbsFile; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; + +public class OutFileDirs extends AbstractSequentialWithArguments { + public static final Arg STAGEOUTS = new Arg.Positional("stageouts"); + + static { + setArguments(OutFileDirs.class, new Arg[] { STAGEOUTS }); + } + + @Override + protected void post(VariableStack stack) throws ExecutionException { + List files = TypeUtil.toList(STAGEOUTS.getValue(stack)); + VariableArguments ret = ArgUtil.getVariableReturn(stack); + try { + for (Object f : files) { + List pv = TypeUtil.toList(f); + Path p = Path.parse(TypeUtil.toString(pv.get(0))); + DSHandle handle = (DSHandle) pv.get(1); + DSHandle leaf = handle.getField(p); + String fname = VDLFunction.filename(leaf)[0]; + String dir = new AbsFile(fname).getDir(); + ret.append(dir); + } + } + catch (Exception e) { + throw new ExecutionException(e); + } + super.post(stack); + } +} Added: trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java 2010-07-24 04:27:28 UTC (rev 3468) @@ -0,0 +1,51 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 18, 2010 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.arguments.ArgUtil; +import org.globus.cog.karajan.arguments.VariableArguments; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.griphyn.vdl.mapping.AbsFile; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; + +public class OutFiles extends AbstractSequentialWithArguments { + public static final Arg STAGEOUTS = new Arg.Positional("stageouts"); + + static { + setArguments(OutFiles.class, new Arg[] { STAGEOUTS }); + } + + @Override + protected void post(VariableStack stack) throws ExecutionException { + List files = TypeUtil.toList(STAGEOUTS.getValue(stack)); + VariableArguments ret = ArgUtil.getVariableReturn(stack); + try { + for (Object f : files) { + List pv = TypeUtil.toList(f); + Path p = Path.parse(TypeUtil.toString(pv.get(0))); + DSHandle handle = (DSHandle) pv.get(1); + DSHandle leaf = handle.getField(p); + String fname = VDLFunction.filename(leaf)[0]; + ret.append(fname); + } + } + catch (Exception e) { + throw new ExecutionException(e); + } + super.post(stack); + } +} Added: trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2010-07-24 04:27:28 UTC (rev 3468) @@ -0,0 +1,60 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 18, 2010 + */ +package org.griphyn.vdl.karajan.lib; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.griphyn.vdl.karajan.functions.ConfigProperty; + +public class Parameterlog extends AbstractSequentialWithArguments { + public static final Logger logger = Logger.getLogger(Parameterlog.class); + + public static final Arg DIRECTION = new Arg.Positional("direction"); + public static final Arg VAR = new Arg.Positional("variable"); + public static final Arg ID = new Arg.Positional("id"); + public static final Arg THREAD = new Arg.Positional("thread"); + + static { + setArguments(Parameterlog.class, new Arg[] { DIRECTION, VAR, ID, THREAD }); + } + + private Boolean enabled; + + @Override + public void pre(VariableStack stack) throws ExecutionException { + if (enabled == null) { + enabled = "true".equals(ConfigProperty.getProperty("provenance.log", true, stack)); + } + } + + + + @Override + protected void executeChildren(VariableStack stack) throws ExecutionException { + if (enabled) { + super.executeChildren(stack); + } + else { + complete(stack); + } + } + + @Override + protected void post(VariableStack stack) throws ExecutionException { + if (enabled) { + logger.info("PARAM thread=" + THREAD.getValue(stack) + " direction=" + + DIRECTION.getValue(stack) + " variable=" + VAR.getValue(stack) + + " provenanceid=" + ID.getValue(stack)); + } + } +} From noreply at svn.ci.uchicago.edu Fri Jul 23 23:28:33 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:28:33 -0500 (CDT) Subject: [Swift-commit] r3469 - trunk/libexec Message-ID: <20100724042833.9A2309CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:28:33 -0500 (Fri, 23 Jul 2010) New Revision: 3469 Modified: trunk/libexec/vdl-lib.xml Log: library definitions for aforementioned functions Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-07-24 04:27:28 UTC (rev 3468) +++ trunk/libexec/vdl-lib.xml 2010-07-24 04:28:33 UTC (rev 3469) @@ -104,6 +104,13 @@ + + + + + + + From noreply at svn.ci.uchicago.edu Fri Jul 23 23:31:46 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:31:46 -0500 (CDT) Subject: [Swift-commit] r3470 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100724043146.5382B9CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:31:46 -0500 (Fri, 23 Jul 2010) New Revision: 3470 Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java Log: made method static to be useable by isLogged Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-07-24 04:28:33 UTC (rev 3469) +++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2010-07-24 04:31:46 UTC (rev 3470) @@ -371,7 +371,7 @@ } } - protected Map getLogData(VariableStack stack) throws ExecutionException { + protected static Map getLogData(VariableStack stack) throws ExecutionException { try { return (Map) stack.getDeepVar(RestartLog.LOG_DATA); } From noreply at svn.ci.uchicago.edu Fri Jul 23 23:37:51 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:37:51 -0500 (CDT) Subject: [Swift-commit] r3471 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100724043751.2DA8A9CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:37:51 -0500 (Fri, 23 Jul 2010) New Revision: 3471 Added: trunk/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java Modified: trunk/src/org/griphyn/vdl/karajan/lib/LogVar.java Log: and another one... Added: trunk/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java 2010-07-24 04:37:51 UTC (rev 3471) @@ -0,0 +1,48 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 18, 2010 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.List; + +import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.arguments.ArgUtil; +import org.globus.cog.karajan.arguments.VariableArguments; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.karajan.workflow.ExecutionException; +import org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments; +import org.griphyn.vdl.mapping.DSHandle; +import org.griphyn.vdl.mapping.Path; + +public class DoRestartLog extends AbstractSequentialWithArguments { + public static final Arg RESTARTOUTS = new Arg.Positional("restartouts"); + + static { + setArguments(DoRestartLog.class, new Arg[] { RESTARTOUTS }); + } + + @Override + protected void post(VariableStack stack) throws ExecutionException { + List files = TypeUtil.toList(RESTARTOUTS.getValue(stack)); + VariableArguments ret = ArgUtil.getVariableReturn(stack); + try { + for (Object f : files) { + List pv = TypeUtil.toList(f); + Path p = Path.parse(TypeUtil.toString(pv.get(0))); + DSHandle handle = (DSHandle) pv.get(1); + LogVar.logVar(stack, handle, p); + } + } + catch (Exception e) { + throw new ExecutionException(e); + } + super.post(stack); + } +} Modified: trunk/src/org/griphyn/vdl/karajan/lib/LogVar.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/LogVar.java 2010-07-24 04:31:46 UTC (rev 3470) +++ trunk/src/org/griphyn/vdl/karajan/lib/LogVar.java 2010-07-24 04:37:51 UTC (rev 3471) @@ -4,6 +4,7 @@ package org.griphyn.vdl.karajan.lib; import org.globus.cog.karajan.arguments.Arg; +import org.globus.cog.karajan.stack.VariableNotFoundException; import org.globus.cog.karajan.stack.VariableStack; import org.globus.cog.karajan.util.TypeUtil; import org.globus.cog.karajan.workflow.ExecutionException; @@ -27,15 +28,19 @@ else { path = Path.parse(TypeUtil.toString(p)); } - path = var.getPathFromRoot().append(path); - String annotation; - if(var.getMapper() != null) { - annotation = "" + var.getMapper().map(path); - } else { - annotation = "unmapped"; - } - RestartLog.LOG_CHANNEL.ret(stack, var.getRoot().getParam("swift#restartid") - + "." + path.stringForm() + "!" + annotation); + logVar(stack, var, path); return null; } + + public static void logVar(VariableStack stack, DSHandle var, Path path) throws VariableNotFoundException { + path = var.getPathFromRoot().append(path); + String annotation; + if(var.getMapper() != null) { + annotation = "" + var.getMapper().map(path); + } else { + annotation = "unmapped"; + } + RestartLog.LOG_CHANNEL.ret(stack, var.getRoot().getParam("swift#restartid") + + "." + path.stringForm() + "!" + annotation); + } } From noreply at svn.ci.uchicago.edu Fri Jul 23 23:38:17 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:38:17 -0500 (CDT) Subject: [Swift-commit] r3472 - trunk/libexec Message-ID: <20100724043817.D96A89CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:38:17 -0500 (Fri, 23 Jul 2010) New Revision: 3472 Modified: trunk/libexec/vdl-lib.xml Log: definition for doRestartLog Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-07-24 04:37:51 UTC (rev 3471) +++ trunk/libexec/vdl-lib.xml 2010-07-24 04:38:17 UTC (rev 3472) @@ -111,6 +111,7 @@ + From noreply at svn.ci.uchicago.edu Fri Jul 23 23:42:31 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:42:31 -0500 (CDT) Subject: [Swift-commit] r3473 - trunk/libexec Message-ID: <20100724044231.8D5329CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:42:31 -0500 (Fri, 23 Jul 2010) New Revision: 3473 Modified: trunk/libexec/vdl-int-staging.k trunk/libexec/vdl-int.k Log: use the java definitions; string expansion is faster than concat() Modified: trunk/libexec/vdl-int-staging.k =================================================================== --- trunk/libexec/vdl-int-staging.k 2010-07-24 04:38:17 UTC (rev 3472) +++ trunk/libexec/vdl-int-staging.k 2010-07-24 04:42:31 UTC (rev 3473) @@ -13,35 +13,6 @@ namespace("vdl" export( - element(isDone, [stageout] - sys:and( - for(pv, stageout - [path, var] := each(pv) - vdl:isLogged(var, path) - ) - sys:not(isEmpty(stageout)) - ) - ) - - element(mark, [restarts, err, optional(mapping)] - if( - err for(pv, restarts - [path, var] := each(pv) - vdl:setFutureFault(var, path=path, mapping=mapping) - ) - ) - ) - - element(flatten, [...] - if ( - isEmpty(...) "" - - concat( - for(i, butLast(...), if(isList(i) flatten(i) i), "|") last(...) - ) - ) - ) - element(initDDir, [] ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d" once(ddir @@ -52,37 +23,10 @@ ddir ) - element(inFileDirs, [stageins] - for(file, stageins - reldirname(file) - ) - ) - - element(outFileDirs, [stageouts] - for(pv, stageouts - [path, var] := each(pv) - - file := vdl:filename(vdl:getfield(var, path = path)) - - vdl:dirname(file) - ) - ) - element(inFiles, [stageins] pathnames(stageins) ) - - element(outFiles, [stageouts] - for(pv, stageouts - [path, var] := each(pv) - - file := vdl:filename(vdl:getfield(var, path = path)) - - file - ) - ) - element(fileDirs, [stageins, stageouts] list( unique( @@ -91,13 +35,6 @@ ) ) ) - - element(doRestartlog, [restartouts] - uParallelFor(f, restartouts, - [path, var] := each(f) - vdl:logvar(var, path) - ) - ) element(graphStuff, [tr, stagein, stageout, err, optional(args)] if( @@ -150,7 +87,7 @@ uid := uid() jobdir := substring(uid, from=0, to=1) - jobid := concat(tr, "-", uid) + jobid := "{tr}-{uid}" fileDirs := fileDirs(stagein, stageout) @@ -164,7 +101,7 @@ vdl:setprogress("Stage in") wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" - tmpdir := dircat(concat(wfdir, "/jobs/", jobdir), jobid) + tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) try( sequential( Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-07-24 04:38:17 UTC (rev 3472) +++ trunk/libexec/vdl-int.k 2010-07-24 04:42:31 UTC (rev 3473) @@ -15,7 +15,7 @@ export( element(rmdir, [dir, host] parallelFor(entry, file:list(dir, host=host) - epath := concat(dir, "/", entry) + epath := "{dir}/{entry}" if( file:isDirectory(epath, host=host) rmdir(epath, host) file:remove(epath, host=host) @@ -30,35 +30,7 @@ dir:make(dc, host=host) ) - - element(isDone, [stageout] - sys:and( - for(pv, stageout - [path, var] := each(pv) - vdl:isLogged(var, path) - ) - sys:not(isEmpty(stageout)) - ) - ) - - element(mark, [restarts, err, optional(mapping)] - if( - err for(pv, restarts - [path, var] := each(pv) - vdl:setFutureFault(var, path=path, mapping=mapping) - ) - ) - ) - element(flatten, [...] - if ( - isEmpty(...) "" - - concat( - for(i, butLast(...), if(isList(i) flatten(i) i), "|") last(...) - ) - ) - ) element(checkJobStatus, [rhost, wfdir, jobid, tr, jobdir] log(LOG:DEBUG, "START jobid={jobid}") @@ -135,37 +107,11 @@ ) ddir ) - - element(inFileDirs, [stageins] - for(file, stageins - reldirname(file) - ) - ) - - element(outFileDirs, [stageouts] - for(pv, stageouts - [path, var] := each(pv) - - file := vdl:filename(vdl:getfield(var, path = path)) - vdl:dirname(file) - ) - ) - element(inFiles, [stageins] pathnames(stageins) ) - - element(outFiles, [stageouts] - for(pv, stageouts - [path, var] := each(pv) - - file := vdl:filename(vdl:getfield(var, path = path)) - file - ) - ) - element(fileDirs, [stageins, stageouts] list( unique( @@ -361,13 +307,6 @@ ) log(LOG:INFO, "END jobid={jobid} - Staging out finished") ) - - element(doRestartlog, [restartouts] - uParallelFor(f, restartouts, - [path, var] := each(f) - vdl:logvar(var, path) - ) - ) element(graphStuff, [tr, stagein, stageout, err, optional(args)] if( @@ -428,7 +367,7 @@ element(transferKickstartRec, [rhost, wfdir, jobid, jobdir] recfile := "{jobid}-kickstart.xml" - srcdir := dircat(concat(wfdir, "/kickstart/"), jobdir) + srcdir := dircat("{wfdir}/kickstart/", jobdir) try( task:transfer(srchost=rhost, srcdir=srcdir, srcfile=recfile, destdir="{VDL:SCRIPTNAME}-{VDL:RUNID}.d/") ( @@ -441,7 +380,7 @@ element(transferWrapperLog, [rhost, wfdir, jobid, jobdir] recfile := "{jobid}-info" - srcdir := dircat(concat(wfdir, "/info/"), jobdir) + srcdir := dircat("{wfdir}/info/", jobdir) try( task:transfer(srchost=rhost, srcdir=srcdir, srcfile=recfile, destdir="{VDL:SCRIPTNAME}-{VDL:RUNID}.d/") ( @@ -467,7 +406,7 @@ uid := uid() jobdir := substring(uid, from=0, to=1) - jobid := concat(tr, "-", uid) + jobid := "{tr}-{uid}" log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") @@ -503,7 +442,7 @@ ) vdl:setprogress("Stage in") - tmpdir := dircat(concat(wfdir, "/jobs/", jobdir), jobid) + tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid) From noreply at svn.ci.uchicago.edu Fri Jul 23 23:43:17 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:43:17 -0500 (CDT) Subject: [Swift-commit] r3474 - trunk/libexec Message-ID: <20100724044317.BCE979CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:43:17 -0500 (Fri, 23 Jul 2010) New Revision: 3474 Modified: trunk/libexec/vdl.k Log: use java parameterlog Modified: trunk/libexec/vdl.k =================================================================== --- trunk/libexec/vdl.k 2010-07-24 04:42:31 UTC (rev 3473) +++ trunk/libexec/vdl.k 2010-07-24 04:43:17 UTC (rev 3474) @@ -25,15 +25,6 @@ ) export( - - element(parameterlog, [direction, variable, id, thread], - - if( - vdl:configProperty("provenance.log") == "true" - log("info","PARAM thread={thread} direction={direction} variable={variable} provenanceid={id}") - ) - ) - element(split, [var], each(str:split(vdl:getFieldValue(var), " "))) element(quote, [var, optional(path)], From noreply at svn.ci.uchicago.edu Fri Jul 23 23:47:35 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 23 Jul 2010 23:47:35 -0500 (CDT) Subject: [Swift-commit] r3475 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100724044735.308689CCAF@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-23 23:47:35 -0500 (Fri, 23 Jul 2010) New Revision: 3475 Modified: trunk/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java trunk/src/org/griphyn/vdl/karajan/lib/Mark.java trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java Log: removed header with cog license Modified: trunk/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java 2010-07-24 04:43:17 UTC (rev 3474) +++ trunk/src/org/griphyn/vdl/karajan/lib/DoRestartLog.java 2010-07-24 04:47:35 UTC (rev 3475) @@ -1,9 +1,3 @@ -//---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - /* * Created on Jul 18, 2010 */ Modified: trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java 2010-07-24 04:43:17 UTC (rev 3474) +++ trunk/src/org/griphyn/vdl/karajan/lib/Flatten.java 2010-07-24 04:47:35 UTC (rev 3475) @@ -1,9 +1,3 @@ -//---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - /* * Created on Jul 18, 2010 */ Modified: trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2010-07-24 04:43:17 UTC (rev 3474) +++ trunk/src/org/griphyn/vdl/karajan/lib/InFileDirs.java 2010-07-24 04:47:35 UTC (rev 3475) @@ -1,9 +1,3 @@ -//---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - /* * Created on Jul 18, 2010 */ Modified: trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java 2010-07-24 04:43:17 UTC (rev 3474) +++ trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java 2010-07-24 04:47:35 UTC (rev 3475) @@ -1,9 +1,3 @@ -//---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - /* * Created on Jul 18, 2010 */ Modified: trunk/src/org/griphyn/vdl/karajan/lib/Mark.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Mark.java 2010-07-24 04:43:17 UTC (rev 3474) +++ trunk/src/org/griphyn/vdl/karajan/lib/Mark.java 2010-07-24 04:47:35 UTC (rev 3475) @@ -1,9 +1,3 @@ -//---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - /* * Created on Jul 18, 2010 */ Modified: trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java 2010-07-24 04:43:17 UTC (rev 3474) +++ trunk/src/org/griphyn/vdl/karajan/lib/OutFileDirs.java 2010-07-24 04:47:35 UTC (rev 3475) @@ -1,9 +1,3 @@ -//---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - /* * Created on Jul 18, 2010 */ Modified: trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java 2010-07-24 04:43:17 UTC (rev 3474) +++ trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java 2010-07-24 04:47:35 UTC (rev 3475) @@ -1,9 +1,3 @@ -//---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - /* * Created on Jul 18, 2010 */ Modified: trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2010-07-24 04:43:17 UTC (rev 3474) +++ trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2010-07-24 04:47:35 UTC (rev 3475) @@ -1,9 +1,3 @@ -//---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - /* * Created on Jul 18, 2010 */ From noreply at svn.ci.uchicago.edu Mon Jul 26 12:09:10 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 26 Jul 2010 12:09:10 -0500 (CDT) Subject: [Swift-commit] r3476 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20100726170910.9DC459CCC9@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-26 12:09:10 -0500 (Mon, 26 Jul 2010) New Revision: 3476 Modified: trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java Log: fixed bug introduced in previous commit Modified: trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java 2010-07-24 04:47:35 UTC (rev 3475) +++ trunk/src/org/griphyn/vdl/karajan/lib/OutFiles.java 2010-07-26 17:09:10 UTC (rev 3476) @@ -16,7 +16,7 @@ import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.Path; -public class OutFiles extends AbstractSequentialWithArguments { +public class OutFiles extends VDLFunction { public static final Arg STAGEOUTS = new Arg.Positional("stageouts"); static { @@ -24,7 +24,7 @@ } @Override - protected void post(VariableStack stack) throws ExecutionException { + protected Object function(VariableStack stack) throws ExecutionException { List files = TypeUtil.toList(STAGEOUTS.getValue(stack)); VariableArguments ret = ArgUtil.getVariableReturn(stack); try { @@ -33,13 +33,13 @@ Path p = Path.parse(TypeUtil.toString(pv.get(0))); DSHandle handle = (DSHandle) pv.get(1); DSHandle leaf = handle.getField(p); - String fname = VDLFunction.filename(leaf)[0]; + String fname = argList(VDLFunction.filename(leaf), true); ret.append(fname); } } catch (Exception e) { throw new ExecutionException(e); } - super.post(stack); + return null; } } From noreply at svn.ci.uchicago.edu Tue Jul 27 14:21:35 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 27 Jul 2010 14:21:35 -0500 (CDT) Subject: [Swift-commit] r3477 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20100727192135.4BCA49CCAC@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-27 14:21:35 -0500 (Tue, 27 Jul 2010) New Revision: 3477 Modified: trunk/src/org/griphyn/vdl/karajan/InHook.java Log: reenabled the inhook Modified: trunk/src/org/griphyn/vdl/karajan/InHook.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/InHook.java 2010-07-26 17:09:10 UTC (rev 3476) +++ trunk/src/org/griphyn/vdl/karajan/InHook.java 2010-07-27 19:21:35 UTC (rev 3477) @@ -14,9 +14,9 @@ public static final Logger logger = Logger.getLogger(InHook.class); public synchronized static void install(Monitor m) { - /*if (!(System.in instanceof InHook)) { + if (!(System.in instanceof InHook)) { System.setIn(new InHook(System.in, m)); - }*/ + } } private BufferedInputStream is; From noreply at svn.ci.uchicago.edu Tue Jul 27 20:11:31 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 27 Jul 2010 20:11:31 -0500 (CDT) Subject: [Swift-commit] r3478 - in trunk: libexec src/org/griphyn/vdl/engine src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20100728011131.6EE2B9CCAC@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-27 20:11:31 -0500 (Tue, 27 Jul 2010) New Revision: 3478 Modified: trunk/libexec/vdl-lib.xml trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: o trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java o trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java o trunk/libexec/vdl-lib.xml -- Added format function. Takes in a float and formats to desired precision which it returns as a string. -- Added pad function. Pads arbitrary number of zeros to the front of an int and return a string. -- Added length fucntion. Returns the number of elements of a closed list. -- toint and tofloat are now polymorphic. They will try to convert anything to an int or float respectively. If they can't they return an ExecutionException. Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2010-07-27 19:21:35 UTC (rev 3477) +++ trunk/libexec/vdl-lib.xml 2010-07-28 01:11:31 UTC (rev 3478) @@ -13,6 +13,8 @@ + + @@ -20,6 +22,7 @@ + Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-07-27 19:21:35 UTC (rev 3477) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2010-07-28 01:11:31 UTC (rev 3478) @@ -194,6 +194,13 @@ FormalArgumentSignature dirnameOut1 = new FormalArgumentSignature("string"); dirname.addOutputArg(dirnameOut1); functionsMap.put(dirname.getName(), dirname); + + ProcedureSignature length = new ProcedureSignature("length"); + FormalArgumentSignature lengthIn1 = new FormalArgumentSignature(true); + length.addInputArg(lengthIn1); + FormalArgumentSignature lengthOut1 = new FormalArgumentSignature("int"); + length.addOutputArg(lengthOut1); + functionsMap.put(length.getName(), length); ProcedureSignature regexp = new ProcedureSignature("regexp"); FormalArgumentSignature regexpIn1 = new FormalArgumentSignature("string"); @@ -240,19 +247,37 @@ functionsMap.put(strsplit.getName(), strsplit); ProcedureSignature toint = new ProcedureSignature("toint"); - FormalArgumentSignature tointIn1 = new FormalArgumentSignature("string"); + FormalArgumentSignature tointIn1 = new FormalArgumentSignature(true); toint.addInputArg(tointIn1); FormalArgumentSignature toOut1 = new FormalArgumentSignature("int"); toint.addOutputArg(toOut1); functionsMap.put(toint.getName(), toint); ProcedureSignature tofloat = new ProcedureSignature("tofloat"); - FormalArgumentSignature tofloatIn1 = new FormalArgumentSignature("string"); + FormalArgumentSignature tofloatIn1 = new FormalArgumentSignature(true); tofloat.addInputArg(tofloatIn1); FormalArgumentSignature tofloatOut1 = new FormalArgumentSignature("float"); tofloat.addOutputArg(tofloatOut1); functionsMap.put(tofloat.getName(), tofloat); + ProcedureSignature format = new ProcedureSignature("format"); + FormalArgumentSignature formatIn1 = new FormalArgumentSignature("float"); + FormalArgumentSignature formatIn2 = new FormalArgumentSignature("int"); + format.addInputArg(formatIn1); + format.addInputArg(formatIn2); + FormalArgumentSignature formatOut = new FormalArgumentSignature("string"); + format.addOutputArg(formatOut); + functionsMap.put(format.getName(), format); + + ProcedureSignature pad = new ProcedureSignature("pad"); + FormalArgumentSignature padIn1 = new FormalArgumentSignature("int"); + FormalArgumentSignature padIn2 = new FormalArgumentSignature("int"); + pad.addInputArg(padIn1); + pad.addInputArg(padIn2); + FormalArgumentSignature padOut = new FormalArgumentSignature("string"); + pad.addOutputArg(padOut); + functionsMap.put(pad.getName(), pad); + ProcedureSignature tostring = new ProcedureSignature("tostring"); FormalArgumentSignature tostringIn1 = new FormalArgumentSignature(true); tostring.addInputArg(tostringIn1); Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-27 19:21:35 UTC (rev 3477) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2010-07-28 01:11:31 UTC (rev 3478) @@ -1,6 +1,8 @@ package org.griphyn.vdl.karajan.lib.swiftscript; import java.io.IOException; +import java.util.List; +import java.util.ArrayList; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -26,6 +28,8 @@ import org.griphyn.vdl.mapping.AbsFile; import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import org.griphyn.vdl.mapping.ArrayDataNode; + public class Misc extends FunctionsCollection { private static final Logger logger = Logger.getLogger(Misc.class); @@ -43,8 +47,11 @@ setArguments("swiftscript_regexp", new Arg[] { PA_INPUT, PA_PATTERN, PA_TRANSFORM }); setArguments("swiftscript_toint", new Arg[] { PA_INPUT }); setArguments("swiftscript_tofloat", new Arg[] { PA_INPUT }); + setArguments("swiftscript_format", new Arg[] { PA_INPUT, PA_TRANSFORM }); + setArguments("swiftscript_pad", new Arg[] { PA_INPUT, PA_TRANSFORM }); setArguments("swiftscript_tostring", new Arg[] { PA_INPUT }); setArguments("swiftscript_dirname", new Arg[] { Arg.VARGS }); + setArguments("swiftscript_length", new Arg[] { Arg.VARGS }); } private static final Logger traceLogger = Logger.getLogger("org.globus.swift.trace"); @@ -206,8 +213,17 @@ inputString = inputString.substring(0, i); } DSHandle handle = new RootDataNode(Types.INT); - handle.setValue(new Double(inputString)); + + try + { + handle.setValue(new Double(inputString)); + } + catch(NumberFormatException e) + { + throw new ExecutionException(stack, "Could not convert value \""+inputString+"\" to type int"); + } handle.closeShallow(); + int provid=VDLFunction.nextProvenanceID(); VDLFunction.logProvenanceResult(provid, handle, "toint"); VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "string"); @@ -218,7 +234,15 @@ InvalidPathException { String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); DSHandle handle = new RootDataNode(Types.FLOAT); - handle.setValue(new Double(inputString)); + + try + { + handle.setValue(new Double(inputString)); + } + catch(NumberFormatException e) + { + throw new ExecutionException(stack, "Could not convert value \""+inputString+"\" to type float"); + } handle.closeShallow(); int provid=VDLFunction.nextProvenanceID(); VDLFunction.logProvenanceResult(provid, handle, "tofloat"); @@ -226,6 +250,50 @@ return handle; } + /* + * Takes in a float and formats to desired precision and returns a string + */ + public DSHandle swiftscript_format(VariableStack stack) throws ExecutionException, NoSuchTypeException, + InvalidPathException { + String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); + String inputFormat = TypeUtil.toString(PA_TRANSFORM.getValue(stack)); + DSHandle handle = new RootDataNode(Types.STRING); + + String output = String.format("%."+inputFormat+"f", Double.parseDouble(inputString)); + handle.setValue(output); + handle.closeShallow(); + + int provid=VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, handle, "format"); + VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "float"); + VDLFunction.logProvenanceParameter(provid, PA_TRANSFORM.getRawValue(stack), "float"); + return handle; + } + + /* + * Takes in an int and pads zeros to the left and returns a string + */ + public DSHandle swiftscript_pad(VariableStack stack) throws ExecutionException, NoSuchTypeException, + InvalidPathException { + String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); + String inputFormat = TypeUtil.toString(PA_TRANSFORM.getValue(stack)); + DSHandle handle = new RootDataNode(Types.STRING); + + int num_length = inputString.length(); + int zeros_to_pad = Integer.parseInt(inputFormat); + zeros_to_pad += num_length; + + String output = String.format("%0"+zeros_to_pad+"d", Integer.parseInt(inputString)); + handle.setValue(output); + handle.closeShallow(); + + int provid=VDLFunction.nextProvenanceID(); + VDLFunction.logProvenanceResult(provid, handle, "pad"); + VDLFunction.logProvenanceParameter(provid, PA_INPUT.getRawValue(stack), "int"); + VDLFunction.logProvenanceParameter(provid, PA_TRANSFORM.getRawValue(stack), "int"); + return handle; + } + public DSHandle swiftscript_tostring(VariableStack stack) throws ExecutionException, NoSuchTypeException, InvalidPathException { @@ -236,26 +304,51 @@ return handle; } - public DSHandle swiftscript_dirname(VariableStack stack) - throws ExecutionException, NoSuchTypeException, InvalidPathException { - DSHandle handle; - try - { - DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); - DSHandle arg = args[0]; - String[] input = VDLFunction.filename(arg); - String name = input[0]; - String result = new AbsFile(name).getDir(); - handle = new RootDataNode(Types.STRING); - handle.setValue(result); - handle.closeShallow(); - } - catch (HandleOpenException e) { - throw new FutureNotYetAvailable - (VDLFunction.addFutureListener(stack, e.getSource())); - } - return handle; - } + public DSHandle swiftscript_dirname(VariableStack stack) + throws ExecutionException, NoSuchTypeException, InvalidPathException { + DSHandle handle; + try + { + DSHandle[] args = SwiftArg.VARGS.asDSHandleArray(stack); + DSHandle arg = args[0]; + String[] input = VDLFunction.filename(arg); + String name = input[0]; + String result = new AbsFile(name).getDir(); + handle = new RootDataNode(Types.STRING); + handle.setValue(result); + handle.closeShallow(); + } + catch (HandleOpenException e) { + throw new FutureNotYetAvailable + (VDLFunction.addFutureListener(stack, e.getSource())); + } + return handle; + } + + /* + * This is copied from swiftscript_dirname. Both the functions could be changed to be more readable. + * Returns length of an array. + * Good for debugging because array needs to be closed before the length is determined + */ + public DSHandle swiftscript_length(VariableStack stack) + throws ExecutionException, NoSuchTypeException, InvalidPathException { + DSHandle handle; + DSHandle[] args = SwiftArg.VARGS.asDSHandleArray( stack ); + DSHandle arg = args[0]; + ArrayDataNode adn = (ArrayDataNode)arg; + + if( !( adn.isClosed() ) ) + { + throw new FutureNotYetAvailable( VDLFunction.addFutureListener( stack, adn ) ); + } + + int result = adn.size(); + handle = new RootDataNode(Types.INT); + handle.setValue(result); + handle.closeShallow(); + + return handle; + } } /* From noreply at svn.ci.uchicago.edu Tue Jul 27 20:43:34 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 27 Jul 2010 20:43:34 -0500 (CDT) Subject: [Swift-commit] r3479 - trunk/src/org/griphyn/vdl/engine Message-ID: <20100728014334.BC8D99D8E2@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-27 20:43:34 -0500 (Tue, 27 Jul 2010) New Revision: 3479 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: o trunk/src/org/griphyn/vdl/engine/Karajan/java -- Added a functionality for a SWIFT_LIB enviroment variable. If the import directive cannot find the file to import relatively it then checks the SWIFT_LIB enviroment variable. Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-07-28 01:11:31 UTC (rev 3478) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-07-28 01:43:34 UTC (rev 3479) @@ -8,12 +8,14 @@ import java.io.PrintStream; import java.util.ArrayList; import java.util.LinkedList; +import java.util.List; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.StringTokenizer; import javax.xml.namespace.QName; @@ -43,6 +45,7 @@ import org.griphyn.vdl.karajan.Loader; import org.griphyn.vdl.karajan.CompilationException; import org.griphyn.vdl.toolkit.VDLt2VDLx; +import org.griphyn.vdl.toolkit.VDLt2VDLx.ParsingException; import org.safehaus.uuid.UUIDGenerator; import org.w3c.dom.Node; @@ -162,9 +165,30 @@ // TODO PATH/PERL5LIB-style path handling //String swiftfilename = "./"+moduleToImport+".swift"; //String xmlfilename = "./"+moduleToImport+".xml"; + String lib_path = System.getenv("SWIFT_LIB"); String swiftfilename = moduleToImport+".swift"; String xmlfilename = moduleToImport+".xml"; - + + File local = new File(swiftfilename); + if( !( lib_path == null || local.exists() ) ) + { + StringTokenizer st = new StringTokenizer(lib_path, ":"); + while(st.hasMoreTokens()) + { + String path = st.nextToken(); + String lib_script_location = path + "/" + swiftfilename; + File tmp = new File(lib_script_location); + + if(tmp.exists()) + { + swiftfilename = path + "/" + swiftfilename; + xmlfilename = path + "/" + xmlfilename; + moduleToImport = path + "/" + moduleToImport; + break; + } + } + } + try { VDLt2VDLx.compile(new FileInputStream(swiftfilename),new PrintStream(new FileOutputStream(xmlfilename))); logger.debug("Compiled. Now reading in compiled XML for "+moduleToImport); From noreply at svn.ci.uchicago.edu Tue Jul 27 21:56:41 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 27 Jul 2010 21:56:41 -0500 (CDT) Subject: [Swift-commit] r3480 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20100728025641.8B69F9D8E2@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-27 21:56:41 -0500 (Tue, 27 Jul 2010) New Revision: 3480 Modified: trunk/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java Log: the copy on write set is a waste of resources since addition can never happen when iterating over listeners Modified: trunk/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java 2010-07-28 01:43:34 UTC (rev 3479) +++ trunk/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java 2010-07-28 02:56:41 UTC (rev 3480) @@ -5,7 +5,8 @@ import java.util.ArrayList; import java.util.HashSet; -import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; import java.util.Map; import java.util.Set; @@ -19,155 +20,146 @@ import org.globus.cog.karajan.workflow.futures.FutureIterator; import org.globus.cog.karajan.workflow.futures.FutureList; import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; -import org.globus.cog.util.CopyOnWriteHashSet; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DSHandleListener; public class ArrayIndexFutureList implements FutureList, DSHandleListener { - private ArrayList keys; - private Map values; - private boolean closed; - private CopyOnWriteHashSet listeners; - private FutureEvaluationException exception; + private ArrayList keys; + private Map values; + private boolean closed; + private List listeners; + private FutureEvaluationException exception; - public ArrayIndexFutureList(DSHandle handle, Map values) { - this.values = values; - keys = new ArrayList(); - handle.addListener(this); - } - - private RuntimeException notYetAvailable() { - if (exception != null) { - return exception; - } - return new FutureNotYetAvailable(this); - } + public ArrayIndexFutureList(DSHandle handle, Map values) { + this.values = values; + keys = new ArrayList(); + handle.addListener(this); + } - public Object get(int index) { - if (exception != null) { - throw exception; - } - if (!closed && index >= keys.size()) { - throw notYetAvailable(); - } - else { - Object key = keys.get(index); - return new Pair(key, values.get(key)); - } - } + private RuntimeException notYetAvailable() { + if (exception != null) { + return exception; + } + return new FutureNotYetAvailable(this); + } - public int available() { - return keys.size(); - } + public Object get(int index) { + if (exception != null) { + throw exception; + } + if (!closed && index >= keys.size()) { + throw notYetAvailable(); + } + else { + Object key = keys.get(index); + return new Pair(key, values.get(key)); + } + } - public void addKey(Object key) { - keys.add(key); - notifyListeners(); - } + public int available() { + return keys.size(); + } - public FutureIterator futureIterator() { - return new FuturePairIterator(this); - } + public void addKey(Object key) { + keys.add(key); + notifyListeners(); + } - public FutureIterator futureIterator(VariableStack stack) { - return new FuturePairIterator(this, stack); - } + public FutureIterator futureIterator() { + return new FuturePairIterator(this); + } - public synchronized void close() { - closed = true; - Set allkeys = new HashSet(values.keySet()); - Iterator i = keys.iterator(); - while (i.hasNext()) { - allkeys.remove(i.next()); - } - // remaining keys must be added - i = allkeys.iterator(); - while (i.hasNext()) { - keys.add(i.next()); - } - notifyListeners(); - } + public FutureIterator futureIterator(VariableStack stack) { + return new FuturePairIterator(this, stack); + } - public boolean isClosed() { - return closed; - } + public synchronized void close() { + closed = true; + Set allkeys = new HashSet(values.keySet()); + allkeys.removeAll(keys); + // remaining keys must be added + keys.addAll(allkeys); + notifyListeners(); + } - public Object getValue() throws VariableNotFoundException { - return this; - } + public boolean isClosed() { + return closed; + } - public synchronized void addModificationAction(EventListener target, Event event) { - if (listeners == null) { - listeners = new CopyOnWriteHashSet(); - } + public Object getValue() throws VariableNotFoundException { + return this; + } - listeners.add(new EventTargetPair(event, target)); - if (closed) { - notifyListeners(); - } - } + public synchronized void addModificationAction(EventListener target, + Event event) { + if (listeners == null) { + listeners = new LinkedList(); + } - private synchronized void notifyListeners() { - if (listeners == null) { - return; - } + listeners.add(new EventTargetPair(event, target)); + if (closed) { + notifyListeners(); + } + } - Iterator i = listeners.iterator(); - try { - while (i.hasNext()) { - try { - EventTargetPair etp = (EventTargetPair) i.next(); - etp.getTarget().event(etp.getEvent()); - } - catch (ExecutionException e) { - e.printStackTrace(); - } - } - } - finally { - listeners.release(); - } - } + private synchronized void notifyListeners() { + if (listeners == null) { + return; + } - public EventTargetPair[] getListenerEvents() { - return (EventTargetPair[]) listeners.toArray(new EventTargetPair[0]); - } + for (EventTargetPair etp : listeners) { + try { + etp.getTarget().event(etp.getEvent()); + } + catch (ExecutionException e) { + e.printStackTrace(); + } + } + listeners = null; + } - public int size() { - if (closed) { - return keys.size(); - } - else { - throw notYetAvailable(); - } - } + public EventTargetPair[] getListenerEvents() { + return listeners.toArray(new EventTargetPair[0]); + } - public String toString() { - String l; - if (listeners == null) { - l = "no listeners"; - } - else { - l = listeners.size() + " listeners"; - } - if (!closed) { - return "Open, " + keys.size() + " elements, " + l; - } - else { - return "Closed, " + keys.size() + " elements, " + l; - } - } - - public void fail(FutureEvaluationException e) { - this.exception = e; - notifyListeners(); - } - - public FutureEvaluationException getException() { - return exception; - } + public int size() { + if (closed) { + return keys.size(); + } + else { + throw notYetAvailable(); + } + } - public void handleClosed(DSHandle handle) { - close(); - } + public String toString() { + String l; + if (listeners == null) { + l = "no listeners"; + } + else { + l = listeners.size() + " listeners"; + } + if (!closed) { + return "Open, " + keys.size() + " elements, " + l; + } + else { + if (listeners != null) { + System.out.println("Badness"); + } + return "Closed, " + keys.size() + " elements, " + l; + } + } + + public void fail(FutureEvaluationException e) { + this.exception = e; + notifyListeners(); + } + + public FutureEvaluationException getException() { + return exception; + } + + public void handleClosed(DSHandle handle) { + close(); + } } From noreply at svn.ci.uchicago.edu Wed Jul 28 13:51:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 13:51:24 -0500 (CDT) Subject: [Swift-commit] r3481 - in SwiftApps/Montage: example scripts Message-ID: <20100728185124.0347D9CCA8@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-28 13:51:23 -0500 (Wed, 28 Jul 2010) New Revision: 3481 Added: SwiftApps/Montage/example/README.txt SwiftApps/Montage/example/sites.xml SwiftApps/Montage/example/stat_dir/ SwiftApps/Montage/example/tc.data SwiftApps/Montage/scripts/create_status_table.py Modified: SwiftApps/Montage/example/m101_montage.swift SwiftApps/Montage/example/makefile SwiftApps/Montage/scripts/Background_list.py SwiftApps/Montage/scripts/Swift_Montage_Apps.swift SwiftApps/Montage/scripts/Swift_Montage_Batch.swift SwiftApps/Montage/scripts/Swift_Montage_Types.swift Log: o apps/Montage/scripts/Swift_Montage_Batch.swift o apps/Montage/scripts/Swift_Montage_Apps.swift o apps/Montage/scripts/Swift_Montage_Types.swift o apps/Montage/example/stat_dir o apps/Montage/example/m101_montage.swift -- Parallelized the mFitBatch function. Now it does not use mFitExec app. -- Fix to the mBgBatch fucntion. o apps/Montage/scripts/Background_list.py o apps/Montage/scripts/create_status_table.py -- Utility files used in some functions. o apps/Montage/example/sites.xml o apps/Montage/example/tc.data o apps/Montage/example/makefile -- Updates to run the m101 Montage example using Swift. o apps/Montage/example/README.txt -- README explaining how to set up the example and how to run the example. Added: SwiftApps/Montage/example/README.txt =================================================================== --- SwiftApps/Montage/example/README.txt (rev 0) +++ SwiftApps/Montage/example/README.txt 2010-07-28 18:51:23 UTC (rev 3481) @@ -0,0 +1,14 @@ +Swift Montage Wrappers README +Runs the the m101 Montage tutorial using the Swift. + +1) Modify the tc.data entries for the Montage functions. + +2) Modify sites.xml to use your Swift work directory on localhost + +3) Run make: + make swift -> Runs the Swift Montage wrappers on localhost + make shell -> Runs the m101_montage.sh file. + make DAG -> Runs the Swift Montage wrappers and generates a DAG plot. + +4) make clean -> Removes all files generated by the Montage wrappers. + Except the log files and other files created by Swift. Modified: SwiftApps/Montage/example/m101_montage.swift =================================================================== --- SwiftApps/Montage/example/m101_montage.swift 2010-07-28 02:56:41 UTC (rev 3480) +++ SwiftApps/Montage/example/m101_montage.swift 2010-07-28 18:51:23 UTC (rev 3481) @@ -7,9 +7,10 @@ Table difference_tbl <"diffs.tbl">; Table fits_images_tbl <"fits.tbl">; Table corrections_tbl <"corrections.tbl">; -/* Image uncorrected_mos <"final/uncorrected.fits">; */ +Table stat_tbl <"stats.tbl">; +Image uncorrected_mos <"final/uncorrected.fits">; +JPEG uncorrected_jpg_img <"final/uncorrected.jpg">; Image corrected_mos <"final/corrected.fits">; -/* JPEG uncorrected_jpg_img <"final/uncorrected.jpg">; */ JPEG corrected_jpg_img <"final/corrected.jpg">; Image raw_image_files[] ; Image projected_images[]; @@ -18,11 +19,11 @@ projected_images = mProjectPPBatch( raw_image_files, template ); images_tbl = mImgtbl( projected_images ); -/* uncorrected_mos = mAdd( projected_images, images_tbl, template ); */ -/* uncorrected_jpg_img = mJPEG( uncorrected_mos ); */ +uncorrected_mos = mAdd( projected_images, images_tbl, template ); +uncorrected_jpg_img = mJPEG( uncorrected_mos ); difference_tbl = mOverlaps( images_tbl ); difference_images = mDiffBatch( difference_tbl, template ); -fits_images_tbl = mFitBatch( difference_tbl, difference_images ); +fits_images_tbl = mFitBatch( difference_images, difference_tbl ); corrections_tbl = mBgModel( images_tbl, fits_images_tbl ); corrected_images = mBgBatch( projected_images, images_tbl, corrections_tbl ); corrected_mos = mAdd( corrected_images, images_tbl, template ); Modified: SwiftApps/Montage/example/makefile =================================================================== --- SwiftApps/Montage/example/makefile 2010-07-28 02:56:41 UTC (rev 3480) +++ SwiftApps/Montage/example/makefile 2010-07-28 18:51:23 UTC (rev 3481) @@ -1,13 +1,10 @@ -.PHONY: shell swift DAG +.PHONY: swift DAG -shell: - ./m101_montage.sh - swift: - swiftrun m101_montage.swift + @swift -tc.file tc.data -sites.file sites.xml m101_montage.swift DAG: - swiftrun -pgraph Montage_DAT.dot montage.swift + @swift -tc.file tc.data -sites.file sites.xml -pgraph m101_DAG.dot m101_montage.swift clean: - @rm -rvf proj_dir/* diff_dir/* final/* corr_dir/* debug scripts/*.xml diffs.tbl images-rawdir.tbl images.tbl corrections.tbl fits.tbl stats.tbl back_tmp.tbl *.txt + @rm -rvf *~ proj_dir/* diff_dir/* final/* corr_dir/* stat_dir/* diffs.tbl images-rawdir.tbl images.tbl corrections.tbl fits.tbl stats.tbl back_tmp.tbl Added: SwiftApps/Montage/example/sites.xml =================================================================== --- SwiftApps/Montage/example/sites.xml (rev 0) +++ SwiftApps/Montage/example/sites.xml 2010-07-28 18:51:23 UTC (rev 3481) @@ -0,0 +1,10 @@ + + + + + + /Users/jonmon/Library/Swift/work/local + .05 + + + Added: SwiftApps/Montage/example/tc.data =================================================================== --- SwiftApps/Montage/example/tc.data (rev 0) +++ SwiftApps/Montage/example/tc.data 2010-07-28 18:51:23 UTC (rev 3481) @@ -0,0 +1,14 @@ +# sitename transformation path INSTALLED platform profiles +localhost mProjectPP /Users/jonmon/Library/Montage/bin/mProjectPP INSTALLED INTEL32::LINUX null +localhost mProject /Users/jonmon/Library/Montage/bin/mProject INSTALLED INTEL32::LINUX null +localhost mImgtbl /Users/jonmon/Library/Montage/bin/mImgtbl INSTALLED INTEL32::LINUX null +localhost mAdd /Users/jonmon/Library/Montage/bin/mAdd INSTALLED INTEL32::LINUX null +localhost mJPEG /Users/jonmon/Library/Montage/bin/mJPEG INSTALLED INTEL32::LINUX null +localhost mOverlaps /Users/jonmon/Library/Montage/bin/mOverlaps INSTALLED INTEL32::LINUX null +localhost mDiff /Users/jonmon/Library/Montage/bin/mDiff INSTALLED INTEL32::LINUX null +localhost mConcatFit /Users/jonmon/Library/Montage/bin/mConcatFit INSTALLED INTEL32::LINUX null +localhost mFitplane /Users/jonmon/Library/Montage/bin/mFitplane INSTALLED INTEL32::LINUX null +localhost mBgModel /Users/jonmon/Library/Montage/bin/mBgModel INSTALLED INTEL32::LINUX null +localhost mBackground /Users/jonmon/Library/Montage/bin/mBackground INSTALLED INTEL32::LINUX null +localhost Background_list /Users/jonmon/Library/Swift/apps/Montage/scripts/Background_list.py INSTALLED INTEL32::LINUX null +localhost create_status_table /Users/jonmon/Library/Swift/apps/Montage/scripts/create_status_table.py INSTALLED INTEL32::LINUX null Modified: SwiftApps/Montage/scripts/Background_list.py =================================================================== --- SwiftApps/Montage/scripts/Background_list.py 2010-07-28 02:56:41 UTC (rev 3480) +++ SwiftApps/Montage/scripts/Background_list.py 2010-07-28 18:51:23 UTC (rev 3481) @@ -1,7 +1,6 @@ #!/usr/bin/python import sys -#import string args = sys.argv; @@ -22,13 +21,13 @@ l2 = line_tbl_2.split(); if l2[0] == l1[0]: - str_out = "["+str(i)+"].fname = "+l1[len(l1) - 1]+"\n" - str_out += "["+str(i)+"].a = "+l2[1]+"\n" - str_out += "["+str(i)+"].b = "+l2[2]+"\n" - str_out += "["+str(i)+"].c = "+l2[3]+"\n" - file_tbl_o.write( str_out ) - i+=1 - break + str_out = "["+str(i)+"].fname = "+l1[len(l1) - 1]+"\n"; + str_out += "["+str(i)+"].a = "+l2[1]+"\n"; + str_out += "["+str(i)+"].b = "+l2[2]+"\n"; + str_out += "["+str(i)+"].c = "+l2[3]+"\n"; + file_tbl_o.write( str_out ); + i+=1; + break; file_tbl_2.close(); Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-07-28 02:56:41 UTC (rev 3480) +++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2010-07-28 18:51:23 UTC (rev 3481) @@ -6,12 +6,9 @@ \param hdr MosaicData file containing information about how mosaic will look. \return mos Image file containing data about mosaic. */ -( Image mos ) mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) +app ( Image mos ) mAdd( Image imgs[], Table img_tbl, MosaicData hdr ) { - app - { - mAdd "-p" @dirname( imgs[0] ) "-n" @img_tbl @hdr @mos; - } + mAdd "-p" @dirname( imgs[0] ) "-n" @img_tbl @hdr @mos; } @@ -23,12 +20,9 @@ \param corr_tbl Correction table containing correction coefficients to be applied \return mos bg_img corrected image */ -( Image bg_img ) mBackground( Image img, float a, float b, float c ) +app ( Image bg_img ) mBackground( Image img, float a, float b, float c ) { - app - { - mBackground "-n" @img @bg_img a b c; - } + mBackground "-n" @img @bg_img a b c; } @@ -39,15 +33,25 @@ \param fits_tbl Table containing information about how to correct an image \return corr_tbl Correction table containing correction coefficients to be applied */ -( Table corr_tbl ) mBgModel( Table img_tbl, Table fits_tbl ) +app ( Table corr_tbl ) mBgModel( Table img_tbl, Table fits_tbl ) { - app - { - mBgModel @img_tbl @fits_tbl @corr_tbl; - } + mBgModel @img_tbl @fits_tbl @corr_tbl; } +/*! \fn app Table mConcatFit( Table status_tbl, Status stats[] ) + \brief Takes all the status files created by mFitplane and creates the fits table. + + \param status_tbl Table containing the file names for the status files + \param stats[] Array of all the status files + \return fits_tbl Table contianing information about how to correct and image. +*/ +app ( Table fits_tbl ) mConcatFit( Table status_tbl, Status stats[] ) +{ + mConcatFit @status_tbl @fits_tbl @dirname( stats[0] ); +} + + /*! \fn app Image mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) \brief Calculates the difference image from the two projected images and the header file. @@ -56,27 +60,33 @@ \param hdr MosaicData file that tells how the mosaic is set up \return diff_img Difference image that contains overlapped information */ -( Image diff_img ) mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) +app ( Image diff_img ) mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr ) { - app - { - mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; - } + mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr; } +/*! \fn app Status mFitplane( Image diff_img ) + \brief Creates a file with all the information on how to fit the particular difference image + + \param diff_img The image to calculate best fit for. + \return stat File containing coefficents and mosaic data on how to fit diff_img. +*/ +app ( Status stat ) mFitplane( Image diff_img ) +{ + mFitplane "-s" @stat @diff_img; +} + + /*! \fn app Table mImgtbl( Image imgs[] ) \brief Take a list of of images and create a table out of them. All images in the list must be located in the same directory. \param imgs[] List of files to create an table out of. \return img_tbl Table of information of the list of images. */ -( Table img_tbl ) mImgtbl( Image imgs[] ) +app ( Table img_tbl ) mImgtbl( Image imgs[] ) { - app - { - mImgtbl @dirname( imgs[0] ) @img_tbl; - } + mImgtbl @dirname( imgs[0] ) @img_tbl; } @@ -86,12 +96,9 @@ \param mos_img Image file to create a jpeg out of. \return mos_img_jpg Jpeg picture. */ -( JPEG mos_img_jpg ) mJPEG( Image mos_img ) +app ( JPEG mos_img_jpg ) mJPEG( Image mos_img ) { - app - { - mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg; - } + mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg; } @@ -101,12 +108,9 @@ \param img_tbl Table of projected images. \return diff_tbl Table of which overlapped images */ -( Table diff_tbl ) mOverlaps( Table img_tbl ) +app ( Table diff_tbl ) mOverlaps( Table img_tbl ) { - app - { - mOverlaps @img_tbl @diff_tbl; - } + mOverlaps @img_tbl @diff_tbl; } @@ -117,12 +121,9 @@ \param hdr MosaicData file telling how the mosaic will be set up in the end. \return proj_img Projected image. */ -( Image proj_img ) mProjectPP( Image raw_img, MosaicData hdr ) +app ( Image proj_img ) mProjectPP( Image raw_img, MosaicData hdr ) { - app - { - mProjectPP "-X" @raw_img @proj_img @hdr; - } + mProjectPP "-X" @raw_img @proj_img @hdr; } @@ -139,10 +140,19 @@ \param corrs_tbl Table containing the coefficients and the id of the image to apply them to \return back_tbl Table containing the Image file name and the coefficients that need to be applied to the image. Data is set up so readDatat2 can read the data. */ -( Table back_tbl ) Background_list( Table imgs_tbl, Table corrs_tbl ) +app ( Table back_tbl ) Background_list( Table imgs_tbl, Table corrs_tbl ) { - app - { - Background_list @imgs_tbl @corrs_tbl @back_tbl; - } + Background_list @imgs_tbl @corrs_tbl @back_tbl; } + + +/*! \fn app Table create_status_table( Table diff_tbl ) + \brief Generates a Table of the status files created by mFitplane + + \param diff_tbl Difference table of the files that mFitplane will run on. + \return stat_tbl Table that contains the information of the status files that will be generated using mFitplane. +*/ +app ( Table stat_tbl ) create_status_table( Table diff_tbl ) +{ + create_status_table @diff_tbl @stat_tbl; +} Modified: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2010-07-28 02:56:41 UTC (rev 3480) +++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2010-07-28 18:51:23 UTC (rev 3481) @@ -23,7 +23,7 @@ foreach background_entry, i in back_struct { - Image proj_img ; + Image proj_img ; Image corr_img ; float a = background_entry.a; @@ -69,12 +69,19 @@ \param diff_imgs \return fits_tbl */ -( Table fits_tbl ) mFitBatch( Table diff_tbl, Image diff_imgs[] ) +( Table fits_tbl ) mFitBatch( Image diff_imgs[], Table diff_tbl ) { - app + Status stats[] ; + + Table status_tbl <"stats.tbl">; + status_tbl = create_status_table( diff_tbl ); + + foreach img, i in stats { - mFitExec @diff_tbl @fits_tbl @dirname( diff_imgs[0] ); + stats[i] = mFitplane ( diff_imgs[i] ); } + + fits_tbl = mConcatFit( status_tbl, stats ); } Modified: SwiftApps/Montage/scripts/Swift_Montage_Types.swift =================================================================== --- SwiftApps/Montage/scripts/Swift_Montage_Types.swift 2010-07-28 02:56:41 UTC (rev 3480) +++ SwiftApps/Montage/scripts/Swift_Montage_Types.swift 2010-07-28 18:51:23 UTC (rev 3481) @@ -25,6 +25,12 @@ */ type JPEG; +/*! \typdef type Status + \brief Declared Status Swift type. +*/ +type Status; + + /*! \struct BackgroundStruct \brief Stuct to hold data found in the tables generated my mImgtbl and mBgModel */ Added: SwiftApps/Montage/scripts/create_status_table.py =================================================================== --- SwiftApps/Montage/scripts/create_status_table.py (rev 0) +++ SwiftApps/Montage/scripts/create_status_table.py 2010-07-28 18:51:23 UTC (rev 3481) @@ -0,0 +1,37 @@ +#!/usr/bin/python + +# Input diff_tbl; Output Stat_tbl + +def space( num ): + spc = ""; + + for i in range( 7 - num + 1 ): + spc += " "; + + return spc + + + +import sys + +args = sys.argv; + +diff_tbl = open( args[1], 'r' ); +stat_tbl = open( args[2], 'w' ); + +stat_tbl.write( "| cntr1 | cntr2 | stat |\n" ); +stat_tbl.write( "| int | int | char |\n" ); + +for diff_line in diff_tbl: + + line = diff_line.split(); + + if line[0] != "|": + str_out = " " + line[0] + space(len(line[0])) + line[1] + space(len(line[0])) + "stat." + line[4] + "\n"; + stat_tbl.write( str_out ); + +diff_tbl.close(); +stat_tbl.close(); + + + Property changes on: SwiftApps/Montage/scripts/create_status_table.py ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Wed Jul 28 14:33:57 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 14:33:57 -0500 (CDT) Subject: [Swift-commit] r3482 - trunk/resources Message-ID: <20100728193357.7D9A29CCA8@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-28 14:33:57 -0500 (Wed, 28 Jul 2010) New Revision: 3482 Modified: trunk/resources/Karajan.stg Log: removed typecheck and modified variable template to allow non-mapped variables Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2010-07-28 18:51:23 UTC (rev 3481) +++ trunk/resources/Karajan.stg 2010-07-28 19:33:57 UTC (rev 3482) @@ -58,8 +58,6 @@ procedure(name,outputs,inputs,arguments,optargs,binding,declarations,statements,config,line) ::= << $optargs:default_arg();separator="\n"$ - $outputs:typecheck();separator="\n"$ - $inputs:typecheck();separator="\n"$ $inputs:vdl_log_input();separator="\n"$ $outputs:vdl_log_output();separator="\n"$ $if(binding)$ @@ -97,10 +95,6 @@ $endif$ >> -typecheck() ::= << - ->> - vdl_execute(outputs,inputs,application,name,line) ::= << @@ -306,7 +300,7 @@ $vdl_mapping(mapping=mapping,file=file,waitfor=waitfor)$ $else$ - $expr$ + $endif$ $endif$ $if(isGlobal)$$else$$endif$ From noreply at svn.ci.uchicago.edu Wed Jul 28 14:35:07 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 14:35:07 -0500 (CDT) Subject: [Swift-commit] r3483 - trunk/src/org/griphyn/vdl/engine Message-ID: <20100728193507.E0BAD9CCA8@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-28 14:35:07 -0500 (Wed, 28 Jul 2010) New Revision: 3483 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: do not instantiate mappers for primitive types; instantiate constant string mapper params directly Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-07-28 19:33:57 UTC (rev 3482) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2010-07-28 19:35:07 UTC (rev 3483) @@ -401,7 +401,7 @@ pparamST.setAttribute("name", "prefix"); pparamST.setAttribute("expr", parameterVariableName + "-" + UUIDGenerator.getInstance().generateRandomBasedUUID().toString()); pmappingST.setAttribute("params", pparamST); - variableDeclarationST.setAttribute("mapping", pmappingST); + //variableDeclarationST.setAttribute("mapping", pmappingST); variableDeclarationST.setAttribute("nil", Boolean.TRUE); variableDeclarationST.setAttribute("name", parameterVariableName); scope.bodyTemplate.setAttribute("declarations",variableDeclarationST); @@ -415,7 +415,14 @@ variableAssignmentST.setAttribute("var",variableReferenceST); variableAssignmentST.setAttribute("value",paramValueST); scope.appendStatement(variableAssignmentST); - paramST.setAttribute("expr",variableReferenceST); + if (param.getAbstractExpression().getDomNode().getNodeName().equals("stringConstant")) { + StringTemplate valueST = template("sConst"); + valueST.setAttribute("innervalue", param.getAbstractExpression().getDomNode().getFirstChild().getNodeValue()); + paramST.setAttribute("expr",valueST); + } + else { + paramST.setAttribute("expr",variableReferenceST); + } } mappingST.setAttribute("params", paramST); } @@ -423,15 +430,17 @@ } } else { // add temporary mapping info - StringTemplate mappingST = new StringTemplate("mapping"); - mappingST.setAttribute("descriptor", "concurrent_mapper"); - StringTemplate paramST = template("vdl_parameter"); - paramST.setAttribute("name", "prefix"); - paramST.setAttribute("expr", var.getName() + "-" - + UUIDGenerator.getInstance().generateRandomBasedUUID().toString()); - mappingST.setAttribute("params", paramST); - variableST.setAttribute("mapping", mappingST); - variableST.setAttribute("nil", Boolean.TRUE); + if (!org.griphyn.vdl.type.Types.isPrimitive(var.getType().getLocalPart())) { + StringTemplate mappingST = new StringTemplate("mapping"); + mappingST.setAttribute("descriptor", "concurrent_mapper"); + StringTemplate paramST = template("vdl_parameter"); + paramST.setAttribute("name", "prefix"); + paramST.setAttribute("expr", var.getName() + "-" + + UUIDGenerator.getInstance().generateRandomBasedUUID().toString()); + mappingST.setAttribute("params", paramST); + variableST.setAttribute("mapping", mappingST); + variableST.setAttribute("nil", Boolean.TRUE); + } } scope.bodyTemplate.setAttribute("declarations", variableST); From noreply at svn.ci.uchicago.edu Wed Jul 28 14:35:27 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 14:35:27 -0500 (CDT) Subject: [Swift-commit] r3484 - trunk/src/org/griphyn/vdl/type Message-ID: <20100728193527.BA81E9CCA8@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-28 14:35:27 -0500 (Wed, 28 Jul 2010) New Revision: 3484 Modified: trunk/src/org/griphyn/vdl/type/Types.java Log: added isPrimitive() Modified: trunk/src/org/griphyn/vdl/type/Types.java =================================================================== --- trunk/src/org/griphyn/vdl/type/Types.java 2010-07-28 19:35:07 UTC (rev 3483) +++ trunk/src/org/griphyn/vdl/type/Types.java 2010-07-28 19:35:27 UTC (rev 3484) @@ -26,6 +26,16 @@ return type; } } + + public static boolean isPrimitive(String name) { + try { + Type t = getType(name); + return t.isPrimitive(); + } + catch (NoSuchTypeException e) { + return false; + } + } //TODO: check duplicate type? public synchronized static void addType(Type type) { From noreply at svn.ci.uchicago.edu Wed Jul 28 14:36:27 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 14:36:27 -0500 (CDT) Subject: [Swift-commit] r3485 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20100728193627.948D89CCA8@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-28 14:36:27 -0500 (Wed, 28 Jul 2010) New Revision: 3485 Modified: trunk/src/org/griphyn/vdl/karajan/FuturePairIterator.java Log: no need to chain iterator future notifications in this case since they can be added directly to the array Modified: trunk/src/org/griphyn/vdl/karajan/FuturePairIterator.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/FuturePairIterator.java 2010-07-28 19:35:27 UTC (rev 3484) +++ trunk/src/org/griphyn/vdl/karajan/FuturePairIterator.java 2010-07-28 19:36:27 UTC (rev 3485) @@ -3,37 +3,25 @@ */ package org.griphyn.vdl.karajan; -import java.util.Iterator; -import java.util.LinkedList; - import org.globus.cog.karajan.stack.VariableNotFoundException; import org.globus.cog.karajan.stack.VariableStack; -import org.globus.cog.karajan.workflow.ExecutionException; -import org.globus.cog.karajan.workflow.events.ControlEvent; -import org.globus.cog.karajan.workflow.events.ControlEventType; import org.globus.cog.karajan.workflow.events.Event; -import org.globus.cog.karajan.workflow.events.EventBus; import org.globus.cog.karajan.workflow.events.EventListener; -import org.globus.cog.karajan.workflow.events.EventTargetPair; -import org.globus.cog.karajan.workflow.futures.Future; import org.globus.cog.karajan.workflow.futures.FutureEvaluationException; import org.globus.cog.karajan.workflow.futures.FutureIterator; import org.globus.cog.karajan.workflow.futures.FutureIteratorIncomplete; import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; -public class FuturePairIterator implements FutureIterator, EventListener, Mergeable { +public class FuturePairIterator implements FutureIterator { private ArrayIndexFutureList array; private int crt; - private LinkedList listeners; public FuturePairIterator(ArrayIndexFutureList array) { this.array = array; - array.addModificationAction(this, null); } public FuturePairIterator(ArrayIndexFutureList array, VariableStack stack) { this.array = array; - array.addModificationAction(this, new ControlEvent(null, ControlEventType.RESTART, stack)); } public synchronized boolean hasAvailable() { @@ -107,44 +95,12 @@ } public synchronized void addModificationAction(EventListener target, Event event) { - if (listeners == null) { - listeners = new LinkedList(); - } WaitingThreadsMonitor.addThread(event.getStack()); - listeners.add(new EventTargetPair(event, target)); + array.addModificationAction(target, event); } + + private static volatile int cnt = 0; - private void notifyListeners() { - synchronized (this) { - if (listeners == null) { - return; - } - - Iterator i = listeners.iterator(); - while (i.hasNext()) { - EventTargetPair etp = (EventTargetPair) i.next(); - WaitingThreadsMonitor.removeThread(etp.getEvent().getStack()); - EventBus.post(etp.getTarget(), etp.getEvent()); - i.remove(); - } - - listeners = null; - } - } - - public void event(Event e) throws ExecutionException { - notifyListeners(); - } - - public void mergeListeners(Future f) { - Iterator i = listeners.iterator(); - while (i.hasNext()) { - EventTargetPair etp = (EventTargetPair) i.next(); - f.addModificationAction(etp.getTarget(), etp.getEvent()); - i.remove(); - } - } - public void fail(FutureEvaluationException e) { //handled by the list } From noreply at svn.ci.uchicago.edu Wed Jul 28 14:41:08 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 14:41:08 -0500 (CDT) Subject: [Swift-commit] r3486 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20100728194108.7E6289D8E2@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-28 14:41:08 -0500 (Wed, 28 Jul 2010) New Revision: 3486 Modified: trunk/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java Log: a cow list is better here Modified: trunk/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java 2010-07-28 19:36:27 UTC (rev 3485) +++ trunk/src/org/griphyn/vdl/karajan/ArrayIndexFutureList.java 2010-07-28 19:41:08 UTC (rev 3486) @@ -5,8 +5,7 @@ import java.util.ArrayList; import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; +import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -20,6 +19,7 @@ import org.globus.cog.karajan.workflow.futures.FutureIterator; import org.globus.cog.karajan.workflow.futures.FutureList; import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable; +import org.globus.cog.util.CopyOnWriteArrayList; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DSHandleListener; @@ -27,7 +27,7 @@ private ArrayList keys; private Map values; private boolean closed; - private List listeners; + private CopyOnWriteArrayList listeners; private FutureEvaluationException exception; public ArrayIndexFutureList(DSHandle handle, Map values) { @@ -82,7 +82,7 @@ notifyListeners(); } - public boolean isClosed() { + public synchronized boolean isClosed() { return closed; } @@ -93,7 +93,7 @@ public synchronized void addModificationAction(EventListener target, Event event) { if (listeners == null) { - listeners = new LinkedList(); + listeners = new CopyOnWriteArrayList(); } listeners.add(new EventTargetPair(event, target)); @@ -107,15 +107,22 @@ return; } - for (EventTargetPair etp : listeners) { - try { - etp.getTarget().event(etp.getEvent()); + Iterator i = listeners.iterator(); + try { + while (i.hasNext()) { + try { + EventTargetPair etp = i.next(); + i.remove(); + etp.getTarget().event(etp.getEvent()); + } + catch (ExecutionException e) { + e.printStackTrace(); + } } - catch (ExecutionException e) { - e.printStackTrace(); - } } - listeners = null; + finally { + listeners.release(); + } } public EventTargetPair[] getListenerEvents() { From noreply at svn.ci.uchicago.edu Wed Jul 28 14:41:50 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 14:41:50 -0500 (CDT) Subject: [Swift-commit] r3487 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20100728194150.604C49D8E2@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-28 14:41:50 -0500 (Wed, 28 Jul 2010) New Revision: 3487 Modified: trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java Log: more friendly exception Modified: trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2010-07-28 19:41:08 UTC (rev 3486) +++ trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2010-07-28 19:41:50 UTC (rev 3487) @@ -81,6 +81,9 @@ if (header) { line = br.readLine(); + if (line == null) { + throw new RuntimeException("Invalid CSV file (" + file + "): missing header."); + } st = new StringTokenizer(line, hdelim); int ix = 0; while (st.hasMoreTokens()) { From noreply at svn.ci.uchicago.edu Wed Jul 28 14:42:39 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 14:42:39 -0500 (CDT) Subject: [Swift-commit] r3488 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20100728194239.7989E9D8E2@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-28 14:42:39 -0500 (Wed, 28 Jul 2010) New Revision: 3488 Modified: trunk/src/org/griphyn/vdl/mapping/MappingParam.java Log: cleaned up a bit Modified: trunk/src/org/griphyn/vdl/mapping/MappingParam.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/MappingParam.java 2010-07-28 19:41:50 UTC (rev 3487) +++ trunk/src/org/griphyn/vdl/mapping/MappingParam.java 2010-07-28 19:42:39 UTC (rev 3488) @@ -86,8 +86,7 @@ /** return the raw value of this parameter. Defaulting and type * conversion will not occur. */ public Object getRawValue(Mapper mapper) { - Object value = mapper.getParam(name); - return value; + return mapper.getParam(name); } From noreply at svn.ci.uchicago.edu Wed Jul 28 14:44:36 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 14:44:36 -0500 (CDT) Subject: [Swift-commit] r3489 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20100728194436.76D299D8E2@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2010-07-28 14:44:36 -0500 (Wed, 28 Jul 2010) New Revision: 3489 Modified: trunk/src/org/griphyn/vdl/mapping/RootDataNode.java Log: fixed mapper dependency bug for single values too, not just for arrays Modified: trunk/src/org/griphyn/vdl/mapping/RootDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2010-07-28 19:42:39 UTC (rev 3488) +++ trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2010-07-28 19:44:36 UTC (rev 3489) @@ -15,6 +15,7 @@ private boolean initialized=false; private Mapper mapper; private Map params; + private DSHandle waitingMapperParam; public static DSHandle newNode(Type type, Object value) { DSHandle handle = new RootDataNode(type); @@ -40,14 +41,13 @@ /** must have this.params set to the appropriate parameters before being called. */ private synchronized void innerInit() { - Iterator i = params.entrySet().iterator(); while(i.hasNext()) { Map.Entry entry = (Map.Entry) i.next(); Object v = entry.getValue(); if(v instanceof DSHandle && !( (DSHandle)v).isClosed()) { - DSHandle dh = (DSHandle)v; - dh.addListener(this); + waitingMapperParam = (DSHandle) v; + waitingMapperParam.addListener(this); return; } } @@ -205,7 +205,8 @@ if(initialized) { return mapper; } else { - throw new VDL2FutureException(this); + assert(waitingMapperParam != null); + throw new VDL2FutureException(waitingMapperParam); } } From noreply at svn.ci.uchicago.edu Wed Jul 28 17:44:31 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 28 Jul 2010 17:44:31 -0500 (CDT) Subject: [Swift-commit] r3490 - in trunk: bin libexec/log-processing Message-ID: <20100728224431.6AB179CCA8@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-28 17:44:31 -0500 (Wed, 28 Jul 2010) New Revision: 3490 Added: trunk/libexec/log-processing/simple-start-time.sh Modified: trunk/bin/swift-plot-log trunk/libexec/log-processing/extract-start-time trunk/libexec/log-processing/makefile trunk/libexec/log-processing/weights.sh Log: Fixes for weights.png Modified: trunk/bin/swift-plot-log =================================================================== --- trunk/bin/swift-plot-log 2010-07-28 19:44:36 UTC (rev 3489) +++ trunk/bin/swift-plot-log 2010-07-28 22:44:31 UTC (rev 3490) @@ -1,11 +1,12 @@ #!/bin/bash +# $1 should be the pathname of the log file to plot +# The rest of the line is passed to make +# They must be figures + ORIGDIR=$(pwd) WORKINGDIR=$(mktemp -d /tmp/swift-plot-log-XXXXXXXXXXXXXXXX) - -# $1 should be the pathname of the log file to plot - LOG_CODE_HOME="`dirname $0`/../libexec/log-processing/" export SWIFT_PLOT_HOME=$LOG_CODE_HOME @@ -41,13 +42,13 @@ fi if [ "X$2" = "X" ]; then -make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean webpage.kara webpage.weights webpage.coasters karatasks.JOB_SUBMISSION.Queue.transitions karatasks.JOB_SUBMISSION.Queue.event karatasks.JOB_SUBMISSION.eip $MAKETARGETS webpage distribute +make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH webpage.kara webpage.weights webpage.coasters karatasks.JOB_SUBMISSION.Queue.transitions karatasks.JOB_SUBMISSION.Queue.event karatasks.JOB_SUBMISSION.eip $MAKETARGETS webpage distribute else shift -make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean $@ +make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH $@ cp $@ $ORIGDIR/ fi # mv report-$LOG_FILE_BASE $LOG_DIRECTORY/ -rm -r $WORKINGDIR +# rm -r $WORKINGDIR Modified: trunk/libexec/log-processing/extract-start-time =================================================================== --- trunk/libexec/log-processing/extract-start-time 2010-07-28 19:44:36 UTC (rev 3489) +++ trunk/libexec/log-processing/extract-start-time 2010-07-28 22:44:31 UTC (rev 3490) @@ -1,3 +1,9 @@ +#!/bin/sh + +set -x + +/bin/pwd 1>&2 + rm -f tmp-extract-starttime2 if [ "X$LOG" != "X" ] ; then Modified: trunk/libexec/log-processing/makefile =================================================================== --- trunk/libexec/log-processing/makefile 2010-07-28 19:44:36 UTC (rev 3489) +++ trunk/libexec/log-processing/makefile 2010-07-28 22:44:31 UTC (rev 3490) @@ -112,9 +112,12 @@ execstages.png: execute2.event dostagein.event dostageout.event start-time.tmp workflow.event execstages-plot +simple-start-time.tmp: $(LOG) + simple-start-time.sh $(LOG) $(@) + # weight processing -weights.png: $(LOG) - weights.sh $(LOG) +weights.png: simple-start-time.tmp $(LOG) + weights.sh $(LOG) $(shell cat simple-start-time.tmp ) info.transitions: info-to-transitions $(IDIR) > $@ Added: trunk/libexec/log-processing/simple-start-time.sh =================================================================== --- trunk/libexec/log-processing/simple-start-time.sh (rev 0) +++ trunk/libexec/log-processing/simple-start-time.sh 2010-07-28 22:44:31 UTC (rev 3490) @@ -0,0 +1,6 @@ +#!/bin/sh + +LOG=$1 +OUTPUT=$2 + +iso-to-secs < ${LOG} | grep RUNID | awk '{ print $1 }' > ${OUTPUT} Property changes on: trunk/libexec/log-processing/simple-start-time.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/libexec/log-processing/weights.sh =================================================================== --- trunk/libexec/log-processing/weights.sh 2010-07-28 19:44:36 UTC (rev 3489) +++ trunk/libexec/log-processing/weights.sh 2010-07-28 22:44:31 UTC (rev 3490) @@ -1,8 +1,10 @@ #!/bin/bash -grep 'WeightedHostScoreScheduler CONTACT_SELECTED' $1 | iso-to-secs | normalise-event-start-time | sed 's/^\([^ ]*\) .* score=\(.*\)$/\1 \2/' | sed 's/,//g' > weights.tmp +LOG=$1 +TIME=$2 +grep 'WeightedHostScoreScheduler CONTACT_SELECTED' $1 | iso-to-secs | normalise-event-start-time-to-any ${TIME} | sed 's/^\([^ ]*\) .* score=\(.*\)$/\1 \2/' | sed 's/,//g' > weights.tmp + if [ -s weights.tmp ]; then gnuplot ${SWIFT_PLOT_HOME}/weights.plot fi - From noreply at svn.ci.uchicago.edu Fri Jul 30 04:43:43 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 30 Jul 2010 04:43:43 -0500 (CDT) Subject: [Swift-commit] r3491 - in usertools/swift/swiftconfig: bin etc/sites etc/sites/localhost etc/sites/pads-local-pbs etc/sites/pads-local-pbs-coasters etc/sites/pads-remote-pbs-coasters-ssh etc/sites/pads-remote-ssh etc/sites/teraport-local-pbs etc/sites/teraport-local-pbs-coasters etc/sites/teraport-remote-ssh Message-ID: <20100730094343.EEBEB9CCC8@vm-125-59.ci.uchicago.edu> Author: davidk Date: 2010-07-30 04:43:43 -0500 (Fri, 30 Jul 2010) New Revision: 3491 Added: usertools/swift/swiftconfig/etc/sites/localhost/ usertools/swift/swiftconfig/etc/sites/localhost/sites.xml usertools/swift/swiftconfig/etc/sites/localhost/tc.data Removed: usertools/swift/swiftconfig/etc/sites/multisite-pbs-coasters/ usertools/swift/swiftconfig/etc/sites/multisite-ssh/ Modified: usertools/swift/swiftconfig/bin/swiftconfig usertools/swift/swiftconfig/bin/swiftrun usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/tc.data usertools/swift/swiftconfig/etc/sites/pads-local-pbs/tc.data usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/tc.data usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/tc.data usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/tc.data usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/tc.data usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/tc.data Log: Modified: usertools/swift/swiftconfig/bin/swiftconfig =================================================================== --- usertools/swift/swiftconfig/bin/swiftconfig 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/bin/swiftconfig 2010-07-30 09:43:43 UTC (rev 3491) @@ -5,60 +5,64 @@ use File::Copy; use File::Path; -my $swifthome = ''; +my $swift_home = q{}; if ( $ENV{'SWIFT_HOME'} && -e "$ENV{'SWIFT_HOME'}/bin/swift" ) { - $swifthome = $ENV{'SWIFT_HOME'}; + $swift_home = $ENV{'SWIFT_HOME'}; } -my @path = split( '/', abs_path($0) ); -my @path = splice( @path, 0, $#path - 1 ); -my $parent_directory = join( '/', @path ); +my @execution_path = split( '/', abs_path($0) ); +my @execution_path = splice( @execution_path, 0, $#execution_path - 1 ); +my $parent_directory = join( '/', @execution_path ); -if ( !$swifthome ) { +if ( !$swift_home ) { if ( -e "$parent_directory/bin/swift" ) { - $swifthome = $parent_directory; + $swift_home = $parent_directory; } else { - @path = split( '/', which("swift") ); - @path = splice( @path, 0, $#path - 1 ); - my $tmp = join( '/', @path ); + @execution_path = split( '/', which("swift") ); + @execution_path = splice( @execution_path, 0, $#execution_path - 1 ); + my $tmp = join( '/', @execution_path ); if ( -e "$tmp/bin/swift" ) { - $swifthome = $tmp; + $swift_home = $tmp; } } } use FindBin qw($Bin); -use lib "$FindBin::Bin/../lib/perl"; # Use libraries in $swifthome/lib/perl +use lib "$FindBin::Bin/../lib/perl"; # Use libraries in $swift_home/lib/perl use Getopt::Long; use File::Which qw(which where); use XML::Simple; use Data::Dumper; # Variables used for translation catalog -my $host = 'localhost'; # Host name -my $name = ''; # Translation name -my $path = "/usr/bin/$name"; # Full path of executable location -my $status = 'INSTALLED'; # Installation status (not used) -my $platform = 'INTEL32::LINUX'; # Platform (not used) -my $profile = 'null'; # Profile entries +my $tc_host = 'localhost'; # Host name +my $tc_name = q{}; # Translation name +my $tc_path = "/usr/bin/$name"; # Full path of executable location +my $tc_status = 'INSTALLED'; # Installation status (not used) +my $tc_platform = 'INTEL32::LINUX'; # Platform (not used) +my $tc_profile = 'null'; # Profile entries # Variables used for sites.xml -my $templates = 0; # List all templates -my $remove = ''; # Remove entry from sites.xml -my $modify = ''; # Modify mode -my @validproviders = ( "gt4", "local", "pbs", "condor", "ssh", "coaster" ); -my $keyfile = "$ENV{'HOME'}/.ssh/id_rsa"; +my $option_templates = q{}; # List all templates +my $option_remove = q{}; # Remove entry from sites.xml +my $option_modify = q{}; # Modify mode +my $option_add = q{}; # Add entry +my $option_sites = q{}; # List all usable sites +my @execution_providers = + ( "gt4", "local", "pbs", "condor", "ssh", "coaster", ); +my $key_file = "$ENV{'HOME'}/.ssh/id_rsa"; # Assign variables from command line options GetOptions( - 'add=s' => \$add, - 'remove=s' => \$remove, - 'templates' => \$templates, - 'modify=s' => \$modify, - 'swifthome=s' => \$swifthome, - 'sites' => \$sites + 'add=s' => \$option_add, + 'remove=s' => \$option_remove, + 'templates' => \$option_templates, + 'modify=s' => \$option_modify, + 'swift_home=s' => \$swift_home, + 'sites' => \$option_sites, + 'describe=s' => \$option_describe, ); # Create a new directory if it does not exist @@ -66,281 +70,386 @@ $directory = $_[0]; if ( !-d "$directory" ) { mkdir "$directory", 0700 - or die "Unable to create directory $directory\n"; + or die "Unable to create directory $directory\n"; } } # Process keyboard input -sub getEntry { - my ($description, $value, @valid) = @_; +sub get_entry { + my ( $entry_description, $entry_value, @allowable_values, ) = @_; - print "$description [$value]: "; - my $newvalue = ""; - chomp($newvalue = ); - if ($newvalue) { $value = $newvalue; } - my $isvalid = 0; + print "$entry_description [$entry_value]: "; + my $new_value = ; + chomp($new_value); + if ($new_value) { + $entry_value = $new_value; + } - if(@valid) { - foreach(@valid) { - if($_ eq $value){ $isvalid = 1; } + # Check if value entered is valid (if valid values were passed) + my $is_valid = 0; + if (@allowable_values) { + foreach my $allowable (@allowable_values) { + if ( $allowable eq $entry_value ) { + $is_valid = 1; + } } - if(!$isvalid) { - my $msg = ""; - foreach(@valid) { $msg .= $_ . " "; } - print "Invalid value selected. Please select from: $msg\n"; - $value = getEntry($description, $value, @valid); + if ( !$is_valid ) { + my $error_message = q{}; + foreach my $allowable (@allowable_values) { + $error_message = $error_message . $allowable . ' '; + } + print 'Invalid value selected. Please select from: ' + . "$error_message\n"; + $entry_value = get_entry( $entry_description, $entry_value, + @allowable_entry_values, ); } } - return $value; + return $entry_value; } # Prepare data my $xml = new XML::Simple(); # Create all required directories -my $dotswift = "$ENV{'HOME'}/.swift"; +my $dotswift_directory = "$ENV{'HOME'}/.swift"; create_directory("$ENV{'HOME'}/.ssh"); -create_directory("$dotswift"); -create_directory("$dotswift/sites"); +create_directory("$dotswift_directory"); +create_directory("$dotswift_directory/sites"); # Determine the template directory -my @path = split( '/', abs_path($0) ); -my @path = splice( @path, 0, $#path - 1 ); -my $template_directory = join( '/', @path ) . "/etc/sites"; +my @execution_path = split( '/', abs_path($0) ); + at execution_path = splice( @execution_path, 0, $#execution_path - 1 ); +my $template_directory = join( '/', @execution_path ) . '/etc/sites'; # Add new entry to auths.default -sub add_ssh_auth -{ - my ($entry, $site) = @_; - +sub add_ssh { + my ( $ssh_site, $ssh_url) = @_; + # Open authfile - my $authfile = "$dotswift/sites/$entry/auth.defaults"; - create_directory("$dotswift/sites/$entry"); - if ( -e "$authfile" ) { open( AUTHFILE, "$authfile" ) || die "Unable to open auth file $authfile for reading!\n"; } - else { open( AUTHFILE, ">$authfile" ) || die "Unable to open $authfile for writing!\n"; } - @authdata = ; - close(AUTHFILE); + my $auth_file = "$dotswift_directory/sites/$ssh_site/auth.defaults"; + create_directory("$dotswift_directory/sites/$ssh_site"); + if ( -e "$auth_file" ) { + open( AUTH_FILE, "$auth_file" ) + || die "Unable to open auth file $auth_file for reading!\n"; + } + else { + open( AUTH_FILE, ">$auth_file" ) + || die "Unable to open $auth_file for writing!\n"; + } + my @auth_data = ; + close(AUTH_FILE); # Create RSA keypair if needed - if ( !-e "$keyfile" ) { - system("ssh-keygen -t rsa -f $keyfile"); - } + #my $keyfile = "$ENV{'HOME'}/.ssh/auth.defaults"; + #if ( !-e "$keyfile" ) { + # system("ssh-keygen -t rsa -f $keyfile"); + #} # Get existing username and passphrase values - my $username; - my $passphrase; - foreach (@authdata) { - if (/username/i) { (my $blah, $username) = split('=', $_); chomp($username); } - if (/passphrase/i) { (my $blah, $passphrase) = split('=', $_); chomp($passphrase); } + my $ssh_username; + my $ssh_passphrase; + foreach (@auth_data) { + if (/username/i) { + ( my $not_used, $ssh_username, ) = split( '=', $_ ); + chomp($ssh_username); + } + if (/passphrase/i) { + ( my $not_used, $ssh_passphrase, ) = split( '=', $_ ); + chomp($ssh_passphrase); + } } - if(!$username){ $username = getlogin(); } + if ( !$ssh_username ) { + $ssh_username = getlogin(); + } + $ssh_username = get_entry( 'Username', $ssh_username ); + $ssh_passphrase = get_entry( 'Passphrase', $ssh_passphrase ); - $username = getEntry("Username", $username); - $passphrase = getEntry("Passphrase", $passphrase); - # Remove old auth.defaults if it already exists - if(-e "$authfile") { - unlink($authfile) || die "Unable to remove $authfile\n"; - @authdata = (); + if ( -e "$auth_file" ) { + unlink($auth_file) || die "Unable to remove $auth_file\n"; + @auth_data = (); } - # Add data to new auth.defaults - push( @authdata, "$site.type=key\n" ); - push( @authdata, "$site.username=$username\n" ); - push( @authdata, "$site.key=$keyfile\n" ); - push( @authdata, "$site.passphrase=$passphrase\n" ); - write_file( $authfile, @authdata ); + # Key based authentication + #push( @authdata, "$site.type=key\n" ); + #push( @authdata, "$site.username=$username\n" ); + #push( @authdata, "$site.key=$keyfile\n" ); + #push( @authdata, "$site.passphrase=$passphrase\n" ); + + # Password based authentication + push( @auth_data, "$ssh_url.type=password\n" ); + push( @auth_data, "$ssh_url.username=$ssh_username\n" ); + push( @auth_data, "$ssh_url.password=$ssh_passphrase\n" ); + write_file( $auth_file, @auth_data ); } # Update TC with correct hostname sub update_tc_hostname { - my ($filename, $host) = @_; + my ( $tc_filename, $tc_host ) = @_; + # Store TC data - open( TCFILESTREAM, "$filename" ) or die("Unable to open tc file $tcfile!"); - @tcdata = ; - close(TCFILESTREAM); + open( TC_FILESTREAM, "$tc_filename" ) + or die("Unable to open tc file $tc_filename!"); + @tc_data = ; + close(TC_FILESTREAM); - foreach $line (@tcdata) { + foreach $line (@tc_data) { + # Ignore comments - if ( substr( $line, 0, 1 ) eq '#' ) { + my $first_character = substr( $line, 0, 1 ); + if ( $first_character eq '#' ) { next; } - # Replace old entry with new entry - my ( $tmphost, $tmpname, $tmppath, $tmpstatus, $tmpplatform, $tmpprofile ) = split( /\s+/, $line ); - $tmphost = $host; - $line = "$tmphost\t$tmpname\t$tmppath\t$tmpstatus\t$tmpplatform\t$tmpprofile\n"; - } + # Replace old entry with new entry + my ($line_tc_host, $line_tc_name, $line_tc_path, + $line_tc_status, $line_tc_platform, $line_tc_profile, + ) = split( /\s+/, $line ); + $line = + "$tc_host\t" + . "$line_tc_name\t" + . "$line_tc_path\t" + . "$line_tc_status\t" + . "$line_tc_platform\t" + . "$line_tc_profile\n"; + } - write_file( $filename, @tcdata); + write_file( $tc_filename, @tc_data ); } # Update XML hash with values from command line -sub update_xml_entry { +sub update_xml { - my ($filename) = @_; - if ( !-e $filename ) { return 0; } - my $entryname = ""; - my $tcfile = ""; + # Set up data + my ($xml_filename) = @_; + if ( !-e $xml_filename ) { + die "Unable to update xml file $xml_filename\n"; + } - my $xml_data = $xml->XMLin( - $filename, + my $xml_ref = $xml->XMLin( + $xml_filename, ForceArray => [qw(workdirectory profile)], - KeyAttr => [] + KeyAttr => [], ); - # Renaming - if($modify) - { - $entryname = getEntry("Site Entry Name", $modify); - $tcfile = "$dotswift/sites/$modify/tc.data"; - if($entryname ne $modify) { - if(!-d "$dotswift/sites/$entryname" && -d "$dotswift/sites/$modify") { - move("$dotswift/sites/$modify", "$dotswift/sites/$entryname") || die "Unable to rename $dotswift/sites/$modify to $dotsite/sites/$entryname"; - } - $xml_data = $xml->XMLin("$dotswift/sites/$entryname/sites.xml", ForceArray => [qw(workdirectory profile)], KeyAttr => []); - $tcfile = "$dotswift/sites/$entryname/tc.data"; + # Handle + $xml_ref->{handle} = get_entry( 'Site Entry Name', $xml_ref->{handle} ); + + # Grid FTP + if ( $xml_ref->{gridftp} ) { + $xml_ref->{gridftp}{url} = + get_entry( 'GridFTP URL', $xml_ref->{gridftp}{url} ); + } + + # Work directory + if ( $xml_ref->{workdirectory} ) { + $xml_ref->{workdirectory}[0] =~ s/\$HOME/$ENV{'HOME'}/; + $xml_ref->{workdirectory} = + [ get_entry( 'Work Directory', $xml_ref->{workdirectory}[0] ) ]; + } + + # Job manager + if ( $xml_ref->{jobmanager} ) { + if ( $xml_ref->{jobmanager}{universe} ) { + $xml_ref->{jobmanager}{universe} = + get_entry( 'Job Universe', $xml_ref->{jobmanager}{universe} ); } + if ( $xml_ref->{jobmanager}{url} ) { + $xml_ref->{jobmanager}{url} = + get_entry( 'Job Manager URL', $xml_ref->{jobmanager}{url} ); + } + if ( $xml_ref->{jobmanager}{major} ) { + $xml_ref->{jobmanager}{major} = + get_entry( 'Job Major Number', + $xml_ref->{jobmanager}{major} ); + } + if ( $xml_ref->{jobmanager}{minor} ) { + $xml_ref->{jobmanager}{minor} = + get_entry( 'Job Minor Number', + $xml_ref->{jobmanager}{minor} ); + } } - - # Adding - if($add){ - $entryname = getEntry("Site Entry Name", $add); - if( -e "$dotswift/sites/$entryname/sites.xml") { - die "Entry file for $entryname already exists. Use -modify to change settings\n"; - } - create_directory("$dotswift/sites/$entryname"); - if(-e "$template_directory/$add/tc.data") { $tcfile = "$template_directory/$add/tc.data"; } - elsif (-e "$dotswift/sites/$add/tc.data") { $tcfile = "$dotswift/sites/$add/tc.data"; } - copy("$tcfile", "$dotswift/sites/$entryname") || die "Unable to copy $tcfile to $dotswift/sites/$entryname\n"; - if($entryname ne $add) { - update_tc_hostname("$dotswift/sites/$entryname/tc.data", $entryname); + # Execution + if ( $xml_ref->{execution} ) { + if ( $xml_ref->{execution}{provider} ) { + $xml_ref->{execution}{provider} = get_entry( + 'Execution Provider', + $xml_ref->{execution}{provider}, + @validproviders + ); } + if ( $xml_ref->{execution}{jobmanager} ) { + $xml_ref->{execution}{jobmanager} = + get_entry( 'Execution Job Manager', + $xml_ref->{execution}{jobmanager} ); + } + if ( $xml_ref->{execution}{url} ) { + $xml_ref->{execution}{url} = + get_entry( 'Execution URL', $xml_ref->{execution}{url} ); + } } - $xml_data->{handle} = $entryname; - if(!-d "$dotswift/sites/$entryname") { - create_directory("$dotswift/sites/$entryname"); + # Filesystem + if ( $xml_ref->{filesystem} ) { + if ( $xml_ref->{filesystem}{provider} ) { + $xml_ref->{filesystem}{provider} = get_entry( + 'Filesystem Provider', + $xml_ref->{filesystem}{provider}, + @validproviders + ); + } + if ( $xml_ref->{filesystem}{url} ) { + $xml_ref->{filesystem}{url} = + get_entry( 'Filesystem URL', $xml_ref->{filesystem}{url} ); + } } - $filename = "$dotswift/sites/$entryname/sites.xml"; - # Replace values if requested by user - $xml_data->{workdirectory}[0] =~ s/\$HOME/$ENV{'HOME'}/; - if ( $xml_data->{gridftp} ) { $xml_data->{gridftp}{url} = getEntry( "GridFTP URL", $xml_data->{gridftp}{url} ); } - if ( $xml_data->{workdirectory} ) { $xml_data->{workdirectory} = [ getEntry( "Work Directory", $xml_data->{workdirectory}[0] ) ]; } - if ( $xml_data->{jobmanager} ) { - if ( $xml_data->{jobmanager}{universe} ) { $xml_data->{jobmanager}{universe} = getEntry( "Job Universe", $xml_data->{jobmanager}{universe} ); } - if ( $xml_data->{jobmanager}{url} ) { $xml_data->{jobmanager}{url} = getEntry( "Job Manager URL", $xml_data->{jobmanager}{url} ); } - if ( $xml_data->{jobmanager}{major} ) { $xml_data->{jobmanager}{major} = getEntry( "Job Major Number", $xml_data->{jobmanager}{major} ); } - if ( $xml_data->{jobmanager}{minor} ) { $xml_data->{jobmanager}{minor} = getEntry( "Job Minor Number", $xml_data->{jobmanager}{minor} ); } + # Profiles + foreach my $profile ( @{ $xml_ref->{profile} } ) { + $profile->{content} = + get_entry( $profile->{key}, $profile->{content} ); } - if ( $xml_data->{execution} ) { - if ( $xml_data->{execution}{provider} ) { $xml_data->{execution}{provider} = getEntry( "Execution Provider", $xml_data->{execution}{provider}, @validproviders); } - if ( $xml_data->{execution}{jobmanager} ) { $xml_data->{execution}{jobmanager} = getEntry( "Execution Job Manager", $xml_data->{execution}{jobmanager} ); } - if ( $xml_data->{execution}{url} ) { - my $previous_site = $xml_data->{execution}{url}; - $xml_data->{execution}{url} = getEntry( "Execution URL", $xml_data->{execution}{url} ); - update_tc_hostname("$tcfile", $xml_data->{execution}{url}); - } - } - if ( $xml_data->{filesystem} ) { - if ( $xml_data->{filesystem}{provider} ) { $xml_data->{filesystem}{provider} = getEntry( "Filesystem Provider", $xml_data->{filesystem}{provider}, @validproviders); } - if ( $xml_data->{filesystem}{url} ) { $xml_data->{filesystem}{url} = getEntry( "Filesystem URL", $xml_data->{filesystem}{url} ); } - } - if ( $xml_data->{execution}{provider} eq "ssh") { - add_ssh_auth($entryname, $xml_data->{execution}{url}); - } - write_file( $filename, $xml->XMLout( $xml_data, RootName => 'pool', SuppressEmpty => 1 ) ); + return $xml_ref; } # Write a file given variable and filename sub write_file { my ( $filename, @data ) = @_; - open( TEMPFILESTREAM, ">$filename" ) or die("Unable to open $filename!\n"); + open( TEMPFILESTREAM, ">$filename" ) + or die("Unable to open $filename!\n"); print TEMPFILESTREAM @data; close(TEMPFILESTREAM); } # Print all files in a directory sub print_directory { - my ($template_directory) = @_; - chdir($template_directory) || die "Unable to change directories to $template_directory\n"; + my ($directory) = @_; + chdir($directory) + || die "Unable to change directories to $directory\n"; my @files = <*>; - foreach $file(@files) - { - (my $basename, my $ext) = split(/\./, $file); - my @path = split( '/', $basename); - print "$basename\n"; - } + foreach $file (@files) { + ( my $basename, my $ext ) = split( /\./, $file ); + my @path = split( '/', $basename ); + print "$basename\n"; + } } # If a template is specified, find the correct one and modify as needed -if ($add) { - my $data = 0; - $data = update_xml_entry("$template_directory/$add/sites.xml"); - if ( $data == 0 ) { $data = update_xml_entry("$dotswift/sites/$add/sites.xml"); } - if ( $data == 0 ) { die "Unable to find template for $add\n"; } +if ($option_add) { + + # Update XML info + my $from_directory = "$template_directory/$option_add"; + my $sites_input_file = "$from_directory/sites.xml"; + if ( !-e "$sites_input_file" ) { + $from_directory = "$dotswift_directory/sites/$option_add"; + $sites_input_file = "$from_directory/sites.xml"; + if ( !-e "$sites_input_file" ) { + die "Unable to find $sites_input_file\n"; + } + } + my $xml_ref = update_xml("$sites_input_file"); + my $new_sitename = $xml_ref->{handle}; + my $output_directory = "$dotswift_directory/sites/$new_sitename"; + if ( -d "$output_directory" ) { + die "Entry for $new_sitename already exists\n"; + } + + # Update and copy files to reflect input + my @files_to_copy = <$from_directory/*>; + create_directory("$output_directory"); + foreach $file_to_copy (@files_to_copy) { + copy( $file_to_copy, "$output_directory" ) + || die "Unable to copy to $file to $output_directory\n"; + } + my $xml_out_ref = $xml->XMLout( + $xml_ref, + RootName => 'pool', + SuppressEmpty => 1, + ); + write_file( "$output_directory/sites.xml", $xml_out_ref ); + + if ( $option_add ne $new_sitename ) { + my $tc_filename = "$output_directory/tc.data"; + update_tc_hostname( "$tc_filename", $new_sitename ); + } + + # Handle SSH + if ( $xml_ref->{execution}{provider} eq 'ssh' ) { + add_ssh( $new_sitename, $xml_ref->{execution}{url} ); + } } +# Describe a site +if ($option_describe) { + my $site_file = "$dotswift_directory/sites/$option_describe/sites.xml"; + if ( !-e $site_file ) { + die "Unable to find $site_file\n"; + } + system("cat $site_file"); +} + # Remove an entry -if ($remove) { +if ($option_remove) { - if(!-d "$dotswift/sites/$remove") { - "Unable to find site entry for $remove\n"; - } + if ( !-d "$dotswift_directory/sites/$option_remove" ) { + "Unable to find site entry for $option_remove\n"; + } - rmtree("$dotswift/sites/$remove") || die "Unable to remove directory $dotswift/sites/$remove\n"; + rmtree("$dotswift_directory/sites/$option_remove") + || die 'Unable to remove directory ' + . "$dotswift_directory/sites/$option_remove\n"; } # List all available templates -if ($templates) { +if ($option_templates) { print_directory($template_directory); } # List all user-added sites -if($sites) { - print_directory("$dotswift/sites"); +if ($option_sites) { + print_directory("$dotswift_directory/sites"); } # Modify a site entry -if ($modify) { - - if(!-e "$dotswift/sites/$modify/sites.xml") { - die "Unable to find entry for $modify\n"; +if ($option_modify) { + my $output_directory = "$dotswift_directory/sites/$option_modify"; + my $sites_file = "$output_directory/sites.xml"; + if ( !-e "$sites_file" ) { + die "Unable to find configuration for $option_modify\n"; } - my $data = update_xml_entry( "$dotswift/sites/$modify/sites.xml" ); + my $xml_ref = update_xml("$sites_file"); + my $new_sitename = $xml_ref->{handle}; -} + create_directory($output_directory); + my $xml_out_ref = $xml->XMLout( + $xml_ref, + RootName => 'pool', + SuppressEmpty => 1, + ); + write_file( "$output_directory/sites.xml", $xml_out_ref ); -# Add or modify a TC entry -if ($name) { - foreach $line (@tcdata) { - - # Ignore comments - if ( substr( $line, 0, 1 ) eq '#' ) { - next; + # If the sitename has changed, move files + if ( $new_sitename ne $option_modify ) { + $output_directory = "$dotswift_directory/sites/$new_sitename"; + if ( -d "$output_directory" ) { + die "Directory for $new_sitename already exists\n"; } - ( $tmphost, $tmpname, @junk ) = split( /\s+/, $line ); - - # Replace old entry with new entry - if ( $name eq $tmpname && $host eq $tmphost ) { - $isfound = 1; - $line = "$host\t$name\t$path\t$status\t$platform\t$profile\n"; - } + move( "$dotswift_directory/sites/$option_modify", + "$output_directory" ) + || die "Unable to rename $dotswift_directory/sites/$option_modify" + . " to $output_directory"; + my $tc_filename = "$output_directory/tc.data"; + update_tc_hostname( "$tc_filename", $new_sitename ); } - # Add new entry - if ( !$isfound ) { - push( @tcdata, "$host\t$name\t$path\t$status\t$platform\t$profile\n" ); + if ( $xml_ref->{execution}{provider} eq 'ssh' ) { + add_ssh( $xml_ref->{handle}, $xml_ref->{execution}{url} ); } - - # Write to tc.data - write_file( $tcfile, @tcdata ); } __END__ Modified: usertools/swift/swiftconfig/bin/swiftrun =================================================================== --- usertools/swift/swiftconfig/bin/swiftrun 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/bin/swiftrun 2010-07-30 09:43:43 UTC (rev 3491) @@ -74,7 +74,7 @@ GetOptions( 'site=s' => \$site, - 'script=s' => \$script, + 'script=s' => \$script, ); # Verify user-created profile info exists Added: usertools/swift/swiftconfig/etc/sites/localhost/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/localhost/sites.xml (rev 0) +++ usertools/swift/swiftconfig/etc/sites/localhost/sites.xml 2010-07-30 09:43:43 UTC (rev 3491) @@ -0,0 +1,6 @@ + + + + 0 + $HOME/swiftwork + Added: usertools/swift/swiftconfig/etc/sites/localhost/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/localhost/tc.data (rev 0) +++ usertools/swift/swiftconfig/etc/sites/localhost/tc.data 2010-07-30 09:43:43 UTC (rev 3491) @@ -0,0 +1,8 @@ +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null +localhost tr /usr/bin/tr INSTALLED INTEL32::LINUX null Modified: usertools/swift/swiftconfig/etc/sites/pads-local-pbs/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs/tc.data 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs/tc.data 2010-07-30 09:43:43 UTC (rev 3491) @@ -9,9 +9,9 @@ #there must be no trailing whitespace at the end of each line. # # sitename transformation path INSTALLED platform profiles -localhost echo /bin/echo INSTALLED INTEL32::LINUX null -localhost cat /bin/cat INSTALLED INTEL32::LINUX null -localhost ls /bin/ls INSTALLED INTEL32::LINUX null -localhost grep /bin/grep INSTALLED INTEL32::LINUX null -localhost sort /bin/sort INSTALLED INTEL32::LINUX null -localhost paste /bin/paste INSTALLED INTEL32::LINUX null +pads-local-pbs echo /bin/echo INSTALLED INTEL32::LINUX null +pads-local-pbs cat /bin/cat INSTALLED INTEL32::LINUX null +pads-local-pbs ls /bin/ls INSTALLED INTEL32::LINUX null +pads-local-pbs grep /bin/grep INSTALLED INTEL32::LINUX null +pads-local-pbs sort /bin/sort INSTALLED INTEL32::LINUX null +pads-local-pbs paste /bin/paste INSTALLED INTEL32::LINUX null Modified: usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/sites.xml 2010-07-30 09:43:43 UTC (rev 3491) @@ -1,13 +1,13 @@ - - - - $HOME/swiftwork - 3000 - 8 - 1 - 1 - 10 - short - 0.5 - 10000 - + + + + $HOME/swiftwork + 3000 + 8 + 1 + 1 + 10 + short + 0.5 + 10000 + Modified: usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/tc.data 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/pads-local-pbs-coasters/tc.data 2010-07-30 09:43:43 UTC (rev 3491) @@ -9,9 +9,9 @@ #there must be no trailing whitespace at the end of each line. # # sitename transformation path INSTALLED platform profiles -localhost echo /bin/echo INSTALLED INTEL32::LINUX null -localhost cat /bin/cat INSTALLED INTEL32::LINUX null -localhost ls /bin/ls INSTALLED INTEL32::LINUX null -localhost grep /bin/grep INSTALLED INTEL32::LINUX null -localhost sort /bin/sort INSTALLED INTEL32::LINUX null -localhost paste /bin/paste INSTALLED INTEL32::LINUX null +pads-local-pbs-coasters echo /bin/echo INSTALLED INTEL32::LINUX null +pads-local-pbs-coasters cat /bin/cat INSTALLED INTEL32::LINUX null +pads-local-pbs-coasters ls /bin/ls INSTALLED INTEL32::LINUX null +pads-local-pbs-coasters grep /bin/grep INSTALLED INTEL32::LINUX null +pads-local-pbs-coasters sort /bin/sort INSTALLED INTEL32::LINUX null +pads-local-pbs-coasters paste /bin/paste INSTALLED INTEL32::LINUX null Modified: usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/sites.xml 2010-07-30 09:43:43 UTC (rev 3491) @@ -1,4 +1,4 @@ - + 3000 8 Modified: usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/tc.data 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-pbs-coasters-ssh/tc.data 2010-07-30 09:43:43 UTC (rev 3491) @@ -9,9 +9,9 @@ #there must be no trailing whitespace at the end of each line. # # sitename transformation path INSTALLED platform profiles -localhost echo /bin/echo INSTALLED INTEL32::LINUX null -localhost cat /bin/cat INSTALLED INTEL32::LINUX null -localhost ls /bin/ls INSTALLED INTEL32::LINUX null -localhost grep /bin/grep INSTALLED INTEL32::LINUX null -localhost sort /bin/sort INSTALLED INTEL32::LINUX null -localhost paste /bin/paste INSTALLED INTEL32::LINUX null +pads-remote-pbs-coasters echo /bin/echo INSTALLED INTEL32::LINUX null +pads-remote-pbs-coasters cat /bin/cat INSTALLED INTEL32::LINUX null +pads-remote-pbs-coasters ls /bin/ls INSTALLED INTEL32::LINUX null +pads-remote-pbs-coasters grep /bin/grep INSTALLED INTEL32::LINUX null +pads-remote-pbs-coasters sort /bin/sort INSTALLED INTEL32::LINUX null +pads-remote-pbs-coasters paste /bin/paste INSTALLED INTEL32::LINUX null Modified: usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/tc.data 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/pads-remote-ssh/tc.data 2010-07-30 09:43:43 UTC (rev 3491) @@ -9,9 +9,9 @@ #there must be no trailing whitespace at the end of each line. # # sitename transformation path INSTALLED platform profiles -localhost echo /bin/echo INSTALLED INTEL32::LINUX null -localhost cat /bin/cat INSTALLED INTEL32::LINUX null -localhost ls /bin/ls INSTALLED INTEL32::LINUX null -localhost grep /bin/grep INSTALLED INTEL32::LINUX null -localhost sort /bin/sort INSTALLED INTEL32::LINUX null -localhost paste /bin/paste INSTALLED INTEL32::LINUX null +pads-remote-ssh echo /bin/echo INSTALLED INTEL32::LINUX null +pads-remote-ssh cat /bin/cat INSTALLED INTEL32::LINUX null +pads-remote-ssh ls /bin/ls INSTALLED INTEL32::LINUX null +pads-remote-ssh grep /bin/grep INSTALLED INTEL32::LINUX null +pads-remote-ssh sort /bin/sort INSTALLED INTEL32::LINUX null +pads-remote-ssh paste /bin/paste INSTALLED INTEL32::LINUX null Modified: usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/tc.data 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs/tc.data 2010-07-30 09:43:43 UTC (rev 3491) @@ -9,9 +9,9 @@ #there must be no trailing whitespace at the end of each line. # # sitename transformation path INSTALLED platform profiles -none echo /bin/echo INSTALLED INTEL32::LINUX null -none cat /bin/cat INSTALLED INTEL32::LINUX null -none ls /bin/ls INSTALLED INTEL32::LINUX null -none grep /bin/grep INSTALLED INTEL32::LINUX null -none sort /bin/sort INSTALLED INTEL32::LINUX null -none paste /bin/paste INSTALLED INTEL32::LINUX null +teraport-local-pbs echo /bin/echo INSTALLED INTEL32::LINUX null +teraport-local-pbs cat /bin/cat INSTALLED INTEL32::LINUX null +teraport-local-pbs ls /bin/ls INSTALLED INTEL32::LINUX null +teraport-local-pbs grep /bin/grep INSTALLED INTEL32::LINUX null +teraport-local-pbs sort /bin/sort INSTALLED INTEL32::LINUX null +teraport-local-pbs paste /bin/paste INSTALLED INTEL32::LINUX null Modified: usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/tc.data 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/teraport-local-pbs-coasters/tc.data 2010-07-30 09:43:43 UTC (rev 3491) @@ -9,9 +9,9 @@ #there must be no trailing whitespace at the end of each line. # # sitename transformation path INSTALLED platform profiles -localhost echo /bin/echo INSTALLED INTEL32::LINUX null -localhost cat /bin/cat INSTALLED INTEL32::LINUX null -localhost ls /bin/ls INSTALLED INTEL32::LINUX null -localhost grep /bin/grep INSTALLED INTEL32::LINUX null -localhost sort /bin/sort INSTALLED INTEL32::LINUX null -localhost paste /bin/paste INSTALLED INTEL32::LINUX null +teraport-local-pbs-coasters echo /bin/echo INSTALLED INTEL32::LINUX null +teraport-local-pbs-coasters cat /bin/cat INSTALLED INTEL32::LINUX null +teraport-local-pbs-coasters ls /bin/ls INSTALLED INTEL32::LINUX null +teraport-local-pbs-coasters grep /bin/grep INSTALLED INTEL32::LINUX null +teraport-local-pbs-coasters sort /bin/sort INSTALLED INTEL32::LINUX null +teraport-local-pbs-coasters paste /bin/paste INSTALLED INTEL32::LINUX null Modified: usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/tc.data =================================================================== --- usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/tc.data 2010-07-28 22:44:31 UTC (rev 3490) +++ usertools/swift/swiftconfig/etc/sites/teraport-remote-ssh/tc.data 2010-07-30 09:43:43 UTC (rev 3491) @@ -9,9 +9,9 @@ #there must be no trailing whitespace at the end of each line. # # sitename transformation path INSTALLED platform profiles -localhost echo /bin/echo INSTALLED INTEL32::LINUX null -localhost cat /bin/cat INSTALLED INTEL32::LINUX null -localhost ls /bin/ls INSTALLED INTEL32::LINUX null -localhost grep /bin/grep INSTALLED INTEL32::LINUX null -localhost sort /bin/sort INSTALLED INTEL32::LINUX null -localhost paste /bin/paste INSTALLED INTEL32::LINUX null +teraport-remote-ssh echo /bin/echo INSTALLED INTEL32::LINUX null +teraport-remote-ssh cat /bin/cat INSTALLED INTEL32::LINUX null +teraport-remote-ssh ls /bin/ls INSTALLED INTEL32::LINUX null +teraport-remote-ssh grep /bin/grep INSTALLED INTEL32::LINUX null +teraport-remote-ssh sort /bin/sort INSTALLED INTEL32::LINUX null +teraport-remote-ssh paste /bin/paste INSTALLED INTEL32::LINUX null From noreply at svn.ci.uchicago.edu Fri Jul 30 10:47:01 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 30 Jul 2010 10:47:01 -0500 (CDT) Subject: [Swift-commit] r3492 - trunk/tests Message-ID: <20100730154701.AE8E29CC87@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-30 10:47:01 -0500 (Fri, 30 Jul 2010) New Revision: 3492 Modified: trunk/tests/arraymapper.swift trunk/tests/nightly.sh trunk/tests/range.swift Log: Minor fixes to tests Includes indentation overhaul Modified: trunk/tests/arraymapper.swift =================================================================== --- trunk/tests/arraymapper.swift 2010-07-30 09:43:43 UTC (rev 3491) +++ trunk/tests/arraymapper.swift 2010-07-30 15:47:01 UTC (rev 3492) @@ -3,6 +3,5 @@ file files[]; foreach f in files { - print(f); + tracef("file: %s\n", @filename(f)); } - Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-07-30 09:43:43 UTC (rev 3491) +++ trunk/tests/nightly.sh 2010-07-30 15:47:01 UTC (rev 3492) @@ -1,7 +1,5 @@ #!/bin/bash -# set -x - # USAGE NOTES: # Run nightly.sh -h for quick help # When something goes wrong, find and check tests.log @@ -22,6 +20,7 @@ printf "\t -p Do not build the package \n" printf "\t -s Do not do a fresh svn checkout \n" printf "\t -x Do not continue after a failure \n" + printf "\t -v Verbose (set -x) \n" printf "\t output Location for output (TOPDIR) \n" } @@ -31,6 +30,7 @@ GRID_TESTS=1 SKIP_CHECKOUT=0 ALWAYS_EXITONFAILURE=0 +VERBOSE=0 # The directory in which to start: TOPDIR=$PWD @@ -54,12 +54,20 @@ -x) ALWAYS_EXITONFAILURE=1 shift;; + -v) + VERBOSE=1 + shift;; *) TOPDIR=$1 shift;; esac done +(( VERBOSE )) && set -x + +# Iterations +ITERS_LOCAL=1 + LOGCOUNT=0 SEQ=1 DATE=$( date +"%Y-%m-%d" ) @@ -83,18 +91,18 @@ [ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1 header() { - CURRENT=$SCRIPTDIR/html/current.html - sed "s at _HTMLBASE_@$HTMLPATH@" < $CURRENT > $TOPDIR/current.html + CURRENT=$SCRIPTDIR/html/current.html + sed "s at _HTMLBASE_@$HTMLPATH@" < $CURRENT > $TOPDIR/current.html - HEADER=$SCRIPTDIR/html/header.html - HOST=$( hostname ) - SEDCMD="s/_DATE_/$DATE/;s/_TIME_/$TIME/;s/_HOST_/$HOST"/ - sed $SEDCMD < $HEADER > $HTML - FIRSTTEST=1 + HEADER=$SCRIPTDIR/html/header.html + HOST=$( hostname ) + SEDCMD="s/_DATE_/$DATE/;s/_TIME_/$TIME/;s/_HOST_/$HOST"/ + sed $SEDCMD < $HEADER > $HTML + FIRSTTEST=1 } html() { - echo $@ >>$HTML + echo $@ >>$HTML } a_name() { @@ -109,163 +117,137 @@ } footer() { - MONTHS=("" "Jan" "Feb" "Mar" "Apr" "May" "Jun" \ - "Jul" "Aug" "Sep" "Oct" "Nov" "Dec") - html "
" - html "$SEQ" + html "$SEQ" else + echo "FAILED" + cat $TLOG < /dev/null html "" - html "$SEQ" + html "$SEQ" fi html "
" + MONTHS=("" "Jan" "Feb" "Mar" "Apr" "May" "Jun" \ + "Jul" "Aug" "Sep" "Oct" "Nov" "Dec") + html "" - if [ "$BINPACKAGE" != "" ]; then - FBP=$RUNDIR/$BINPACKAGE - SIZE=`ls -hs $FBP` - SIZE=${SIZE/$FBP} - cat <>$HTML + if [ "$BINPACKAGE" != "" ]; then + FBP=$RUNDIR/$BINPACKAGE + SIZE=`ls -hs $FBP` + SIZE=${SIZE/$FBP} + cat <>$HTML

Binary packages

$BINPACKAGE ($SIZE)
DOH - fi + fi - LASTYR="00" - LASTMO="00" - html "

Older tests

" - html '' - html "" - for OLDER in `ls $OUTDIR/tests-*.html|sort`; do - O=`basename $OLDER` - YR=${O:6:2} - MO=${O:8:2} - DY=${O:10:2} - if echo "$DY$MO$YR"|egrep -v "[0-9]{6}"; then - YR=${O#tests-} - YR=${YR%.html} - MO=0 - DY=$YR - else - YR="20$YR" - fi - if [ $LASTYR != $YR ]; then - html "
" - html "

$YR

" - LASTYR=$YR - fi - if [ $LASTMO != $MO ]; then - html "" - html "

${MONTHS[$MO]}

" - html "" - LASTMO=$MO - fi - SUCCESS=`grep 'class="success"' $OLDER|wc -l` - FAILURE=`grep 'class="failure"' $OLDER|wc -l` - if [ "$SUCCESS$FAILURE" == "00" ]; then - COLOR="#e0e0e0" - else - COLOR=`perl -e "printf \"#%02x%02x%02x\", $FAILURE/($SUCCESS+$FAILURE)*220+35, $SUCCESS/($SUCCESS+$FAILURE)*220+35, 40;"` - fi - html "" - done - html "
$DY


" - cat <>$HTML + LASTYR="00" + LASTMO="00" + html "

Older tests

" + html '
' + html "" + for OLDER in `ls $OUTDIR/tests-*.html|sort`; do + O=`basename $OLDER` + YR=${O:6:2} + MO=${O:8:2} + DY=${O:10:2} + if echo "$DY$MO$YR"|egrep -v "[0-9]{6}"; then + YR=${O#tests-} + YR=${YR%.html} + MO=0 + DY=$YR + else + YR="20$YR" + fi + if [ $LASTYR != $YR ]; then + html "
" + html "

$YR

" + LASTYR=$YR + fi + if [ $LASTMO != $MO ]; then + html "" + html "

${MONTHS[$MO]}

" + html "" + LASTMO=$MO + fi + SUCCESS=`grep 'class="success"' $OLDER|wc -l` + FAILURE=`grep 'class="failure"' $OLDER|wc -l` + if [ "$SUCCESS$FAILURE" == "00" ]; then + COLOR="#e0e0e0" + else + COLOR=`perl -e "printf \"#%02x%02x%02x\", $FAILURE/($SUCCESS+$FAILURE)*220+35, $SUCCESS/($SUCCESS+$FAILURE)*220+35, 40;"` + fi + html "" + done + html "
$DY


" + cat <>$HTML
How to add new tests - + DOH } outecho() { - TYPE=$1 - shift - echo "<$TYPE>$1|$2|$3|$4|$5|$6|$7|$8|$9|" + TYPE=$1 + shift + echo "<$TYPE>$1|$2|$3|$4|$5|$6|$7|$8|$9|" } out() { # echo $@ - TYPE=$1 - if [ "$TYPE" == "test" ]; then + TYPE=$1 + if [ "$TYPE" == "test" ]; then - NAME=$2 - SEQ=$3 - CMD=$4 - RES=$5 + NAME=$2 # + LABEL=$3 # Text on link to output + CMD=$4 + RES=$5 - if [ "$FIRSTTEST" == "1" ]; then - html "

Test results

" - a_name "tests" - a_href "tests.log" "Output log from tests" - html "" - FIRSTTEST=0 - else - if [ "$FLUSH" == "1" ]; then - html "
" - fi - fi + if [ "$FIRSTTEST" == "1" ]; then + html "

Test results

" + a_name "tests" + a_href "tests.log" "Output log from tests" + html "" + FIRSTTEST=0 + else + if [ "$FLUSH" == "1" ]; then + html "
" + fi + fi - if [ "$TESTPART" != "" ]; then - html "$TESTPART" - TESTPART= - fi + if [ "$TESTPART" != "" ]; then + html "$TESTPART" + TESTPART= + fi - if [ "$FLUSH" == "1" ]; then - html "$NAME: " - fi - if [ ${#SEQ} -gt 2 ]; then - WIDTH="" - else - WIDTH="width=\"20\"" - fi - if [ "$RES" == "Passed" ]; then - html "" + if [ "$FLUSH" == "1" ]; then + html "
" - html "$SEQ" - else - echo "FAILED" - cat $TLOG < /dev/null - html "" - html "$SEQ" - fi - html "
$NAME: " + fi + if [ ${#LABEL} -gt 2 ]; then + WIDTH="" + else + WIDTH="width=\"20\"" + fi + if [ "$RES" == "Passed" ]; then + html "" - elif [ "$TYPE" == "package" ]; then - BINPACKAGE=$2 - else - html $@ - fi + elif [ "$TYPE" == "package" ]; then + BINPACKAGE=$2 + else + html $@ + fi } -aexec() { - declare -p PWD - echo "Executing: $@" >>$LOG - rm -fv $OUTPUT - LASTCMD="$@" - "$@" > $OUTPUT 2>&1 - head $OUTPUT - EXITCODE=$? - if [ "$EXITCODE" == "127" ]; then - echo "Command not found: $@" > $OUTPUT - fi - if [ -f $OUTPUT ]; then - cat $OUTPUT >>$LOG - fi -} - # TLOG = this (current) log tlog() { - TLOG="output_$LOGCOUNT.txt" - rm -fv $TLOG - banner "$LASTCMD" $RUNDIR/$TLOG - if [ -f $OUTPUT ]; then - cp -v $OUTPUT $RUNDIR/$TLOG 2>>$LOG - fi - let "LOGCOUNT=$LOGCOUNT+1" + TLOG="output_$LOGCOUNT.txt" + rm -fv $TLOG + banner "$LASTCMD" $RUNDIR/$TLOG + if [ -f $OUTPUT ]; then + cp -v $OUTPUT $RUNDIR/$TLOG 2>>$LOG + fi + let "LOGCOUNT=$LOGCOUNT+1" } -# Fake exec -fexec() { - FLUSH=1 - banner "$TEST (faked)" - echo "Faking $TEST" - EXITCODE=0 - LASTCMD="" - vtest -} - stars() { for i in {1..90} do @@ -275,85 +257,110 @@ } banner() { - if [ "$2" == "" ]; then - BOUT=$LOG - else - BOUT=$2 - fi - { - echo "" + if [ "$2" == "" ]; then + BOUT=$LOG + else + BOUT=$2 + fi + { + echo "" # stars - echo "* $1" + echo "* $1" # stars - } >>$BOUT + } >>$BOUT } +aexec() { + declare -p PWD + printf "\nExecuting: $@" >>$LOG + rm -fv $OUTPUT + LASTCMD="$@" + "$@" > $OUTPUT 2>&1 + EXITCODE=$? + if [ "$EXITCODE" == "127" ]; then + echo "Command not found: $@" > $OUTPUT + fi + if [ -f $OUTPUT ]; then + cat $OUTPUT >>$LOG + fi +} + # Execute as part of test set pexec() { - banner "$TEST (part $SEQ)" - echo "Executing $TEST (part $SEQ)" - aexec "$@" - ptest - let "SEQ=$SEQ+1" - FLUSH=0 + banner "$TEST (part $SEQ)" + echo "Executing $TEST (part $SEQ)" + aexec "$@" + ptest + let "SEQ=$SEQ+1" + FLUSH=0 } ssexec() { - SEQSAVE=$SEQ - SEQ=$1 - shift - banner "$TEST (part $SEQ)" - echo "Executing $TEST (part $SEQ)" - aexec "$@" - ptest - SEQ=$SEQSAVE - FLUSH=0 + SEQSAVE=$SEQ + SEQ=$1 + shift + banner "$TEST (part $SEQ)" + echo "Executing $TEST (part $SEQ)" + aexec "$@" + ptest + SEQ=$SEQSAVE + FLUSH=0 } # Execute final test in set vexec() { - if [ "$SEQ" == "1" ]; then - banner "$TEST" - echo "Executing $TEST" - else - banner "$TEST (part $SEQ)" - echo "Executing $TEST (part $SEQ)" - fi - aexec "$@" - vtest - SEQ=1 - FLUSH=1 + if [ "$SEQ" == "1" ]; then + banner "$TEST" + echo "Executing $TEST" + else + banner "$TEST (part $SEQ)" + echo "Executing $TEST (part $SEQ)" + fi + aexec "$@" + vtest + SEQ=1 + FLUSH=1 } +# Fake exec +fexec() { + FLUSH=1 + banner "$TEST (faked)" + echo "Faking $TEST" + EXITCODE=0 + LASTCMD="" + vtest +} + ptest() { - if [ "$EXITCODE" == "0" ]; then - RES="Passed" - else - RES="Failed" - fi - tlog - out test "$TESTLINK" $SEQ "$LASTCMD" $RES $TLOG - if [ "$EXITONFAILURE" == "true" ]; then - if [ "$EXITCODE" != "0" ]; then - exit $EXITCODE - fi - fi + if [ "$EXITCODE" == "0" ]; then + RES="Passed" + else + RES="Failed" + fi + tlog + out test "$TESTLINK" $SEQ "$LASTCMD" $RES $TLOG + if [ "$EXITONFAILURE" == "true" ]; then + if [ "$EXITCODE" != "0" ]; then + exit $EXITCODE + fi + fi } vtest() { - EC=$? - if [ "$EXITCODE" == "0" ]; then - RES="Passed" - else - RES="Failed" - fi - tlog - out test "$TESTLINK" $SEQ "$LASTCMD" $RES $TLOG - if [ "$EXITCODE" != "0" ]; then - if [ "$EXITONFAILURE" == "true" ]; then - exit $EXITCODE - fi - fi + EC=$? + if [ "$EXITCODE" == "0" ]; then + RES="Passed" + else + RES="Failed" + fi + tlog + out test "$TESTLINK" $SEQ "$LASTCMD" $RES $TLOG + if [ "$EXITCODE" != "0" ]; then + if [ "$EXITONFAILURE" == "true" ]; then + exit $EXITCODE + fi + fi } build_package() { @@ -374,15 +381,15 @@ TESTPART="Part I: Build" EXITONFAILURE=true if [ "$SKIP_CHECKOUT" != "1" ]; then - TEST="Checkout CoG" - pexec rm -rf cog - COG="https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog" - vexec svn co $COG + TEST="Checkout CoG" + pexec rm -rf cog + COG="https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog" + vexec svn co $COG - TEST="Checkout Swift" - pexec cd cog/modules - pexec rm -rf swift - vexec svn co https://svn.ci.uchicago.edu/svn/vdl2/$BRANCH swift + TEST="Checkout Swift" + pexec cd cog/modules + pexec rm -rf swift + vexec svn co https://svn.ci.uchicago.edu/svn/vdl2/$BRANCH swift fi TEST="Compile" @@ -404,7 +411,7 @@ cd $RUNDIR if [ $ALWAYS_EXITONFAILURE != "1" ]; then - EXITONFAILURE=false + EXITONFAILURE=false fi TESTPART="Part II: Local Tests" @@ -416,7 +423,7 @@ TESTLINK="$TESTNAME" - for ((i=0; $i<9; i=$i+1)); do + for ((i=1; $i<$ITERS_LOCAL; i=$i+1)); do pexec swift -sites.file sites.xml $TESTNAME done vexec swift -sites.file sites.xml $TESTNAME @@ -429,20 +436,24 @@ TESTPART="Part III: Grid Tests" for TEST in `ls $TESTDIR/*.dtm $TESTDIR/*.swift`; do - BN=`basename $TEST` - echo $BN - cp $TESTDIR/$BN . + BN=`basename $TEST` + echo $BN + cp $TESTDIR/$BN . - TESTNAME=${BN%.dtm} - TESTNAME=${TESTNAME%.swift} - TEST="$TESTNAME" + TESTNAME=${BN%.dtm} + TESTNAME=${TESTNAME%.swift} + TEST="$TESTNAME" - ssexec "Compile" vdlc $BN - for ((i=0; $i<9; i=$i+1)); do - pexec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml - done - vexec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml + ssexec "Compile" vdlc $BN + for ((i=0; $i<9; i=$i+1)); do + pexec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml + done + vexec swift -sites.file ~/.vdl2/sites-grid.xml $TESTNAME.kml done #Don't remove me: footer + +# Local Variables: +# sh-basic-offset: 2 +# End: Modified: trunk/tests/range.swift =================================================================== --- trunk/tests/range.swift 2010-07-30 09:43:43 UTC (rev 3491) +++ trunk/tests/range.swift 2010-07-30 15:47:01 UTC (rev 3492) @@ -1,5 +1,5 @@ int nums[] = [0:20:2]; // generate a list 0, 2, 4, 6, 8 ... foreach num in nums { - print(num); + tracef("num: %i\n", num); } From noreply at svn.ci.uchicago.edu Fri Jul 30 11:17:58 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 30 Jul 2010 11:17:58 -0500 (CDT) Subject: [Swift-commit] r3493 - trunk/tests Message-ID: <20100730161758.1BAA29CC87@vm-125-59.ci.uchicago.edu> Author: wozniak Date: 2010-07-30 11:17:57 -0500 (Fri, 30 Jul 2010) New Revision: 3493 Modified: trunk/tests/arraymapper.swift trunk/tests/nightly.sh Log: Minor test fixes Modified: trunk/tests/arraymapper.swift =================================================================== --- trunk/tests/arraymapper.swift 2010-07-30 15:47:01 UTC (rev 3492) +++ trunk/tests/arraymapper.swift 2010-07-30 16:17:57 UTC (rev 3493) @@ -3,5 +3,6 @@ file files[]; foreach f in files { - tracef("file: %s\n", @filename(f)); + trace(2); + // tracef("file: %s\n", @filename(f)); } Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-07-30 15:47:01 UTC (rev 3492) +++ trunk/tests/nightly.sh 2010-07-30 16:17:57 UTC (rev 3493) @@ -82,7 +82,8 @@ HTMLPATH=$RUNDIRBASE/tests-$DATE.html HTML=$TOPDIR/$HTMLPATH -BRANCH="branches/tests-01" +BRANCH=trunk +#BRANCH="branches/tests-01" SCRIPTDIR=$( dirname $0 ) From noreply at svn.ci.uchicago.edu Fri Jul 30 13:56:12 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 30 Jul 2010 13:56:12 -0500 (CDT) Subject: [Swift-commit] r3494 - trunk/tests Message-ID: <20100730185612.172A69CCC8@vm-125-59.ci.uchicago.edu> Author: jonmon Date: 2010-07-30 13:56:11 -0500 (Fri, 30 Jul 2010) New Revision: 3494 Added: trunk/tests/filesysmapper.swift Removed: trunk/tests/arraymapper.swift Log: o trunk/tests/filesysmapper.swift -- Renamed the old arraymapper.swift to filesysmapper.swift. filesysmapper.swift uses the filesys_mapper since that is what the old arraymapper.swift file tried emulating. Deleted: trunk/tests/arraymapper.swift =================================================================== --- trunk/tests/arraymapper.swift 2010-07-30 16:17:57 UTC (rev 3493) +++ trunk/tests/arraymapper.swift 2010-07-30 18:56:11 UTC (rev 3494) @@ -1,8 +0,0 @@ -type file {}; - -file files[]; - -foreach f in files { - trace(2); - // tracef("file: %s\n", @filename(f)); -} Copied: trunk/tests/filesysmapper.swift (from rev 3493, trunk/tests/arraymapper.swift) =================================================================== --- trunk/tests/filesysmapper.swift (rev 0) +++ trunk/tests/filesysmapper.swift 2010-07-30 18:56:11 UTC (rev 3494) @@ -0,0 +1,7 @@ +type file {}; + +file files[]; + +foreach f in files { + tracef("file: %s\n", @filename(f)); +}
" + a_href $TLOG $LABEL + else + echo "FAILED" + cat $TLOG < /dev/null + html "" + a_href $TLOG $LABEL + fi + html "