From noreply at svn.ci.uchicago.edu Mon May 3 14:56:05 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:05 -0500 (CDT)
Subject: [Swift-commit] r3300 - SwiftApps/adem-osg
Message-ID: <20100503195605.B95389CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:05 -0500 (Mon, 03 May 2010)
New Revision: 3300
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Full sites support
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:02 UTC (rev 3299)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:05 UTC (rev 3300)
@@ -4,16 +4,27 @@
require 'adem'
require 'yaml'
-class AdemTest < Test::Unit::TestCase
-
- def test_sites_from_file
- conf = @conf
- site_list = YAML.load @site_list
- assert_equal(site_list, sites(nil, conf, "sites"))
+module Adem
+ module TestSetup
+ def setup
+ @conf = {
+ :pacman_cache => "http://www.ci.uchicago.edu/~aespinosa/pacman",
+ :ress_server => "osg-ress-1.fnal.gov",
+ :virtual_organization => "engage"
+ }
+ @site_list = File.open("sites").read
+ #@site_list.gsub! /^\s+/, ''
+ @ress = File.open("dummy_ress").read
+ end
end
+end
+class OnlineTest < Test::Unit::TestCase
+ include Adem::TestSetup
+
def test_sites_exception
conf = @conf
+ conf[:ress_server] = nil
site_list = YAML.load @site_list
begin
sites(nil, conf, "non_existent_file")
@@ -22,28 +33,59 @@
end
end
+ def test_sites_live
+ conf = @conf
+ assert_equal(YAML.load(@site_list), sites(nil, conf, "non_existent_file"))
+ end
+
+ def test_query_ress
+ conf = @conf
+ ress = query_ress(conf).split "\n\n"
+ assumption = true
+ class_ads = [
+ "GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
+ "GlueCEInfoGatekeeperPort", "GlueSEAccessProtocolEndpoint",
+ "GlueSEAccessProtocolType", "GlueCEInfoApplicationDir", "GlueCEInfoDataDir"
+ ]
+ missing = []
+ ress.each do |entry|
+ class_ads.each do |ad|
+ assumption = assumption && entry.include?(ad)
+ missing << ad if not entry.include?(ad)
+ end
+ end
+ # False: there exist an entry without the expected classad.
+ # Must report to osg support then
+ assert assumption, "Missing attributes: \n\t#{missing.uniq.join(", ")}"
+ end
+end
+
+class OfflineTest < Test::Unit::TestCase
+ include Adem::TestSetup
+
+ def test_sites_from_file
+ conf = @conf
+ site_list = YAML.load @site_list
+ assert_equal(site_list, sites(nil, conf, "sites"))
+ end
+
+
def test_parse_classads
site_list = YAML.load @site_list
- assert_equal site_list, parse_classads(@ress)
+ assert_equal site_list, parse_classads(@ress.split("\n\n"))
end
+
def test_app
conf = @conf
assert_equal("app", app(nil, conf))
end
def test_config
- assert_equal(@conf, config(nil))
+ assert_equal(@conf, config(nil, "config"))
end
+end
- def setup
- @conf = {
- "pacman_cache" => "http://www.ci.uchicago.edu/~aespinosa/pacman",
- "ress_server" => "osg-ress-1.fnal.gov",
- "virtual_organization" => "engage"
- }
- @site_list = File.open("sites").read
- #@site_list.gsub! /^\s+/, ''
- @ress = File.open("dummy_ress").read.split "\n\n"
- end
+class RunTest < Test::Unit::TestCase
+ include Adem::TestSetup
end
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:02 UTC (rev 3299)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:05 UTC (rev 3300)
@@ -10,11 +10,18 @@
SITES_FILE = "#{ENV['HOME']}/.adem/sites"
def load_config(yaml_config)
- YAML.load yaml_config
+ conf = YAML.load yaml_config
+ conf.each do |key, val|
+ conf.delete(key)
+ conf[key.to_sym] = val
+ end
+ conf
end
def query_ress(conf)
- File.open("dummy_ress").read.split '\n\n'
+ `condor_status -pool #{conf[:ress_server]} -const \
+ 'stringListIMember(\"VO:#{conf[:virtual_organization]}\", \
+ GlueCEAccessControlBaseRule)' -long`
end
def parse_classads(ress)
@@ -78,7 +85,7 @@
begin
YAML.load File.open sites_file
rescue Errno::ENOENT
- #parse_classads query_ress conf
+ parse_classads query_ress(conf).split("\n\n")
end
end
@@ -86,20 +93,20 @@
"app"
end
-def config(args)
- load_config File.open("config")
+def config(args, config_file)
+ load_config File.open(config_file)
end
def run_command(args)
command = args.shift
if command != "config"
- conf = load_configuration File.open(CONFIGURATION_FILE)
- sites args, conf, File.open(SITES_FILE).read
+ conf = load_config File.open(CONFIGURATION_FILE)
+ puts sites(args, conf, SITES_FILE).to_yaml if command == "sites"
else
- config args
+ config args, CONFIGURATION_FILE
end
end
if $0 == __FILE__
- run_command ARGV, conf
+ run_command ARGV
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:07 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:07 -0500 (CDT)
Subject: [Swift-commit] r3301 - SwiftApps/adem-osg
Message-ID: <20100503195607.EDC899CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:07 -0500 (Mon, 03 May 2010)
New Revision: 3301
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Added site exceptions and tests
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:05 UTC (rev 3300)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:07 UTC (rev 3301)
@@ -13,7 +13,6 @@
:virtual_organization => "engage"
}
@site_list = File.open("sites").read
- #@site_list.gsub! /^\s+/, ''
@ress = File.open("dummy_ress").read
end
end
@@ -22,22 +21,15 @@
class OnlineTest < Test::Unit::TestCase
include Adem::TestSetup
- def test_sites_exception
+ def test_sites_live
conf = @conf
- conf[:ress_server] = nil
- site_list = YAML.load @site_list
begin
sites(nil, conf, "non_existent_file")
- rescue Errno::ENOENT
- assert true, "Received exception"
+ rescue SiteError => e
+ assert_equal YAML.load(@site_list), e.output
end
end
- def test_sites_live
- conf = @conf
- assert_equal(YAML.load(@site_list), sites(nil, conf, "non_existent_file"))
- end
-
def test_query_ress
conf = @conf
ress = query_ress(conf).split "\n\n"
@@ -63,19 +55,26 @@
class OfflineTest < Test::Unit::TestCase
include Adem::TestSetup
+ def test_sites_exception
+ conf = @conf
+ conf[:ress_server] = nil
+ site_list = YAML.load @site_list
+ assert_raise SiteError do
+ sites(nil, conf, "non_existent_file")
+ end
+ end
+
def test_sites_from_file
conf = @conf
site_list = YAML.load @site_list
assert_equal(site_list, sites(nil, conf, "sites"))
end
-
def test_parse_classads
site_list = YAML.load @site_list
assert_equal site_list, parse_classads(@ress.split("\n\n"))
end
-
def test_app
conf = @conf
assert_equal("app", app(nil, conf))
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:05 UTC (rev 3300)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:07 UTC (rev 3301)
@@ -1,14 +1,26 @@
#!/usr/bin/env ruby
# = ADEM: Application software DEployment and Management
-# authors:: Zhengxiong Hou (original sh prototype)
-# Allan Espinosa (rewrite to ruby)
+# Author:: Zhengxiong Hou (original sh prototype)
+# Author:: Allan Espinosa (rewrite to ruby)
+#
require 'yaml'
CONFIGURATION_FILE = "#{ENV['HOME']}/.adem/config"
-SITES_FILE = "#{ENV['HOME']}/.adem/sites"
+SITES_FILE = "#{ENV['HOME']}/.adem/sites"
+APPS_FILE = "#{ENV['HOME']}/.adem/apps"
+class SiteError < Exception
+ attr_reader :output
+ def initialize(output)
+ @output = output
+ end
+end
+
+class ConfigError < RuntimeError
+end
+
def load_config(yaml_config)
conf = YAML.load yaml_config
conf.each do |key, val|
@@ -65,6 +77,7 @@
site[site_name]["data_directory"] = tmp['GlueCEInfoDataDir']
site[site_name]["app_directory"] = tmp['GlueCEInfoApplicationDir']
+ # Disabled because some storage endpoints are broken
#storage_element = tmp['GlueSEAccessProtocolEndpoint'] if tmp['GlueSEAccessProtocolType'] == 'gsiftp'
#next if storage_element == nil
#if storage_element.include? ','
@@ -85,7 +98,8 @@
begin
YAML.load File.open sites_file
rescue Errno::ENOENT
- parse_classads query_ress(conf).split("\n\n")
+ result = parse_classads query_ress(conf).split("\n\n")
+ raise SiteError.new(result)
end
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:10 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:10 -0500 (CDT)
Subject: [Swift-commit] r3302 - SwiftApps/adem-osg
Message-ID: <20100503195610.4199C9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:10 -0500 (Mon, 03 May 2010)
New Revision: 3302
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Refactored configurations as method parameters
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:07 UTC (rev 3301)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:10 UTC (rev 3302)
@@ -87,4 +87,14 @@
class RunTest < Test::Unit::TestCase
include Adem::TestSetup
+
+ def test_config
+ assert_equal(@conf, run_command(["config"], "config", nil))
+ end
+
+ def test_sites_from_file
+ conf = @conf
+ site_list = YAML.load @site_list
+ assert_equal(site_list, run_command(["sites"], "config", "sites"))
+ end
end
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:07 UTC (rev 3301)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:10 UTC (rev 3302)
@@ -111,16 +111,27 @@
load_config File.open(config_file)
end
-def run_command(args)
+def run_command(args, config_file, sites_file)
command = args.shift
- if command != "config"
- conf = load_config File.open(CONFIGURATION_FILE)
- puts sites(args, conf, SITES_FILE).to_yaml if command == "sites"
+ output = nil
+ if command == "sites"
+ conf = load_config File.open(config_file)
+ begin
+ output = sites(args, conf, sites_file) if command == "sites"
+ rescue SiteError => exception
+ output = exception.output
+ File.open(sites_file, "w") do |sites_file|
+ sites_file << output.to_yaml
+ end
+ end
+ elsif command == "app"
+ conf[:sites] = sites(nil, conf, sites_file)
+ app(args, conf)
else
- config args, CONFIGURATION_FILE
+ output = config args, config_file
end
end
if $0 == __FILE__
- run_command ARGV
+ run_command ARGV, CONFIGURATION_FILE, SITES_FILE
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:12 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:12 -0500 (CDT)
Subject: [Swift-commit] r3303 - SwiftApps/adem-osg
Message-ID: <20100503195612.88E619CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:12 -0500 (Mon, 03 May 2010)
New Revision: 3303
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
List available pacman packages
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:10 UTC (rev 3302)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:12 UTC (rev 3303)
@@ -8,7 +8,7 @@
module TestSetup
def setup
@conf = {
- :pacman_cache => "http://www.ci.uchicago.edu/~aespinosa/pacman",
+ :pacman_cache => "http://www.ci.uchicago.edu/~aespinosa/Cybershake",
:ress_server => "osg-ress-1.fnal.gov",
:virtual_organization => "engage"
}
@@ -50,6 +50,15 @@
# Must report to osg support then
assert assumption, "Missing attributes: \n\t#{missing.uniq.join(", ")}"
end
+
+ def test_app_avail
+ pacman_cache = @conf[:pacman_cache]
+ response = <<-eos
+http://www.ci.uchicago.edu/~aespinosa/Cybershake
+ [ ] jbsim3d
+ eos
+ assert_equal(response, app_avail(pacman_cache))
+ end
end
class OfflineTest < Test::Unit::TestCase
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:10 UTC (rev 3302)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:12 UTC (rev 3303)
@@ -107,6 +107,9 @@
"app"
end
+def app_avail(pacman_cache)
+ `pacman -trust-all-caches -lc #{pacman_cache}`
+end
def config(args, config_file)
load_config File.open(config_file)
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:14 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:14 -0500 (CDT)
Subject: [Swift-commit] r3304 - SwiftApps/adem-osg
Message-ID: <20100503195614.D264A9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:14 -0500 (Mon, 03 May 2010)
New Revision: 3304
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Pacman finding utilities
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:12 UTC (rev 3303)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:14 UTC (rev 3304)
@@ -59,6 +59,15 @@
eos
assert_equal(response, app_avail(pacman_cache))
end
+
+ def test_pacman_find_firefly
+ # Firefly
+ site = {
+ :compute_element => "ff-grid.unl.edu:2119/jobmanager-pbs",
+ :app_directory => "/panfs/panasas/CMS/app"
+ }
+ assert_equal("/opt/pacman/pacman-3.28", pacman_find(site, @conf))
+ end
end
class OfflineTest < Test::Unit::TestCase
@@ -79,6 +88,10 @@
assert_equal(site_list, sites(nil, conf, "sites"))
end
+ def test_site_fork
+ assert_equal "ff-grid.unl.edu:2119/jobmanager-fork", site_fork("ff-grid.unl.edu:2119/jobmanager-pbs")
+ end
+
def test_parse_classads
site_list = YAML.load @site_list
assert_equal site_list, parse_classads(@ress.split("\n\n"))
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:12 UTC (rev 3303)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:14 UTC (rev 3304)
@@ -6,6 +6,7 @@
#
require 'yaml'
+require 'ftools'
CONFIGURATION_FILE = "#{ENV['HOME']}/.adem/config"
SITES_FILE = "#{ENV['HOME']}/.adem/sites"
@@ -110,6 +111,35 @@
def app_avail(pacman_cache)
`pacman -trust-all-caches -lc #{pacman_cache}`
end
+
+def app_deploy(app, site, conf)
+ sites.each do |site|
+ root = site[:pacman] || pacman_find(site, conf)
+ site[:pacman] = root
+ pacman_install site, root, conf
+ end
+end
+
+def site_fork(compute_element)
+ compute_element.gsub /jobmanager-.*$/, "jobmanager-fork"
+end
+
+def pacman_find(site, conf)
+ contact = site_fork site[:compute_element]
+ rootdir = site[:app_directory] + "/" + conf[:virtual_organization]
+ script = <<-eos
+#!/bin/bash
+which pacman
+ eos
+ File.open("/tmp/find_pacman.sh", "w") do |dump|
+ dump << script
+ end
+ `globus-job-run #{contact} /bin/mkdir -p #{rootdir}`
+ `globus-job-run #{contact} -d #{rootdir} -stdin -s /tmp/find_pacman.sh /bin/bash -c 'cat > find_pacman.sh'`
+ `globus-job-run #{contact} -d #{rootdir} /bin/chmod 755 find_pacman.sh`
+ File.dirname(File.dirname(`globus-job-run #{contact} -d #{rootdir} find_pacman.sh`))
+end
+
def config(args, config_file)
load_config File.open(config_file)
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:17 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:17 -0500 (CDT)
Subject: [Swift-commit] r3305 - SwiftApps/adem-osg
Message-ID: <20100503195617.1189C9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:16 -0500 (Mon, 03 May 2010)
New Revision: 3305
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Changed script upload into a gridftp session
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:14 UTC (rev 3304)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:16 UTC (rev 3305)
@@ -64,7 +64,8 @@
# Firefly
site = {
:compute_element => "ff-grid.unl.edu:2119/jobmanager-pbs",
- :app_directory => "/panfs/panasas/CMS/app"
+ :app_directory => "/panfs/panasas/CMS/app",
+ :storage_element => "gsiftp://ff-gridftp.unl.edu:2811"
}
assert_equal("/opt/pacman/pacman-3.28", pacman_find(site, @conf))
end
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:14 UTC (rev 3304)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:16 UTC (rev 3305)
@@ -126,6 +126,7 @@
def pacman_find(site, conf)
contact = site_fork site[:compute_element]
+ storage = site[:storage_element].first
rootdir = site[:app_directory] + "/" + conf[:virtual_organization]
script = <<-eos
#!/bin/bash
@@ -134,8 +135,7 @@
File.open("/tmp/find_pacman.sh", "w") do |dump|
dump << script
end
- `globus-job-run #{contact} /bin/mkdir -p #{rootdir}`
- `globus-job-run #{contact} -d #{rootdir} -stdin -s /tmp/find_pacman.sh /bin/bash -c 'cat > find_pacman.sh'`
+ `globus-url-copy -cd file:///tmp/find_pacman.sh #{storage}#{rootdir}/find_pacman.sh`
`globus-job-run #{contact} -d #{rootdir} /bin/chmod 755 find_pacman.sh`
File.dirname(File.dirname(`globus-job-run #{contact} -d #{rootdir} find_pacman.sh`))
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:19 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:19 -0500 (CDT)
Subject: [Swift-commit] r3306 - SwiftApps/adem-osg
Message-ID: <20100503195619.2878E9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:19 -0500 (Mon, 03 May 2010)
New Revision: 3306
Modified:
SwiftApps/adem-osg/adem.rb
Log:
Reduced external globus calls via staging executables
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:16 UTC (rev 3305)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:19 UTC (rev 3306)
@@ -128,16 +128,14 @@
contact = site_fork site[:compute_element]
storage = site[:storage_element].first
rootdir = site[:app_directory] + "/" + conf[:virtual_organization]
- script = <<-eos
-#!/bin/bash
-which pacman
- eos
- File.open("/tmp/find_pacman.sh", "w") do |dump|
- dump << script
+ File.open("find_pacman.sh", "w") do |dump|
+ dump.puts "#!/bin/bash"
+ dump.puts "which pacman"
end
- `globus-url-copy -cd file:///tmp/find_pacman.sh #{storage}#{rootdir}/find_pacman.sh`
- `globus-job-run #{contact} -d #{rootdir} /bin/chmod 755 find_pacman.sh`
- File.dirname(File.dirname(`globus-job-run #{contact} -d #{rootdir} find_pacman.sh`))
+ File.chmod 0755, "find_pacman.sh"
+ resp = `globus-job-run #{contact} -d #{rootdir} -stage find_pacman.sh`
+ File.delete "find_pacman.sh"
+ File.dirname(File.dirname(resp))
end
def config(args, config_file)
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:21 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:21 -0500 (CDT)
Subject: [Swift-commit] r3307 - SwiftApps/adem-osg
Message-ID: <20100503195621.7B5289CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:21 -0500 (Mon, 03 May 2010)
New Revision: 3307
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Refactored rootdir
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:19 UTC (rev 3306)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:21 UTC (rev 3307)
@@ -67,7 +67,10 @@
:app_directory => "/panfs/panasas/CMS/app",
:storage_element => "gsiftp://ff-gridftp.unl.edu:2811"
}
- assert_equal("/opt/pacman/pacman-3.28", pacman_find(site, @conf))
+ assert_equal(
+ "/opt/pacman/pacman-3.28",
+ pacman_find(site, "/panfs/panasas/CMS/app/engage")
+ )
end
end
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:19 UTC (rev 3306)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:21 UTC (rev 3307)
@@ -112,11 +112,11 @@
`pacman -trust-all-caches -lc #{pacman_cache}`
end
-def app_deploy(app, site, conf)
- sites.each do |site|
- root = site[:pacman] || pacman_find(site, conf)
- site[:pacman] = root
- pacman_install site, root, conf
+def app_deploy(app, conf)
+ conf[:sites].each do |site|
+ path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
+ site[:pacman] = pacman_find(site, path) if not site[:pacman]
+ pacman_install site, path
end
end
@@ -124,10 +124,8 @@
compute_element.gsub /jobmanager-.*$/, "jobmanager-fork"
end
-def pacman_find(site, conf)
+def pacman_find(site, rootdir)
contact = site_fork site[:compute_element]
- storage = site[:storage_element].first
- rootdir = site[:app_directory] + "/" + conf[:virtual_organization]
File.open("find_pacman.sh", "w") do |dump|
dump.puts "#!/bin/bash"
dump.puts "which pacman"
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:23 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:23 -0500 (CDT)
Subject: [Swift-commit] r3308 - SwiftApps/adem-osg
Message-ID: <20100503195623.C264D9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:23 -0500 (Mon, 03 May 2010)
New Revision: 3308
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
changed to literal contact string
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:21 UTC (rev 3307)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:23 UTC (rev 3308)
@@ -69,7 +69,10 @@
}
assert_equal(
"/opt/pacman/pacman-3.28",
- pacman_find(site, "/panfs/panasas/CMS/app/engage")
+ pacman_find(
+ "ff-grid.unl.edu/jobmanager-fork",
+ "/panfs/panasas/CMS/app/engage"
+ )
)
end
end
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:21 UTC (rev 3307)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:23 UTC (rev 3308)
@@ -115,8 +115,9 @@
def app_deploy(app, conf)
conf[:sites].each do |site|
path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
- site[:pacman] = pacman_find(site, path) if not site[:pacman]
- pacman_install site, path
+ contact = site_fork site[:compute_element]
+ site[:pacman] = pacman_find(contact, path) if not site[:pacman]
+ pacman_install contact, path
end
end
@@ -124,8 +125,7 @@
compute_element.gsub /jobmanager-.*$/, "jobmanager-fork"
end
-def pacman_find(site, rootdir)
- contact = site_fork site[:compute_element]
+def pacman_find(contact, rootdir)
File.open("find_pacman.sh", "w") do |dump|
dump.puts "#!/bin/bash"
dump.puts "which pacman"
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:25 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:25 -0500 (CDT)
Subject: [Swift-commit] r3309 - SwiftApps/adem-osg
Message-ID: <20100503195625.E0FF99CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:25 -0500 (Mon, 03 May 2010)
New Revision: 3309
Modified:
SwiftApps/adem-osg/config
Log:
Updated pacman repository
Modified: SwiftApps/adem-osg/config
===================================================================
--- SwiftApps/adem-osg/config 2010-05-03 19:56:23 UTC (rev 3308)
+++ SwiftApps/adem-osg/config 2010-05-03 19:56:25 UTC (rev 3309)
@@ -1,4 +1,4 @@
---
ress_server: osg-ress-1.fnal.gov
-pacman_cache: http://www.ci.uchicago.edu/~aespinosa/pacman
+pacman_cache: http://www.ci.uchicago.edu/~aespinosa/Cybershake
virtual_organization: engage
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:28 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:28 -0500 (CDT)
Subject: [Swift-commit] r3310 - SwiftApps/adem-osg
Message-ID: <20100503195628.447799CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:28 -0500 (Mon, 03 May 2010)
New Revision: 3310
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Renamed ress functions for namespace consistency
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:25 UTC (rev 3309)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:28 UTC (rev 3310)
@@ -30,9 +30,9 @@
end
end
- def test_query_ress
+ def test_ress_query
conf = @conf
- ress = query_ress(conf).split "\n\n"
+ ress = ress_query(conf).split "\n\n"
assumption = true
class_ads = [
"GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
@@ -99,9 +99,9 @@
assert_equal "ff-grid.unl.edu:2119/jobmanager-fork", site_fork("ff-grid.unl.edu:2119/jobmanager-pbs")
end
- def test_parse_classads
+ def test_ress_parse
site_list = YAML.load @site_list
- assert_equal site_list, parse_classads(@ress.split("\n\n"))
+ assert_equal site_list, ress_parse(@ress.split("\n\n"))
end
def test_app
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:25 UTC (rev 3309)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:28 UTC (rev 3310)
@@ -31,13 +31,13 @@
conf
end
-def query_ress(conf)
+def ress_query(conf)
`condor_status -pool #{conf[:ress_server]} -const \
'stringListIMember(\"VO:#{conf[:virtual_organization]}\", \
GlueCEAccessControlBaseRule)' -long`
end
-def parse_classads(ress)
+def ress_parse(ress)
class_ads = [
"GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
"GlueCEInfoGatekeeperPort", "GlueSEAccessProtocolEndpoint",
@@ -99,7 +99,7 @@
begin
YAML.load File.open sites_file
rescue Errno::ENOENT
- result = parse_classads query_ress(conf).split("\n\n")
+ result = ress_parse ress_query(conf).split("\n\n")
raise SiteError.new(result)
end
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:30 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:30 -0500 (CDT)
Subject: [Swift-commit] r3311 - SwiftApps/adem-osg
Message-ID: <20100503195630.8A1AA9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:30 -0500 (Mon, 03 May 2010)
New Revision: 3311
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Refactored application runner
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:28 UTC (rev 3310)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:30 UTC (rev 3311)
@@ -124,6 +124,7 @@
def test_sites_from_file
conf = @conf
site_list = YAML.load @site_list
- assert_equal(site_list, run_command(["sites"], "config", "sites"))
+ conf[:sites] = site_list
+ assert_equal(conf, run_command(["sites"], "config", "sites"))
end
end
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:28 UTC (rev 3310)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:30 UTC (rev 3311)
@@ -143,22 +143,20 @@
def run_command(args, config_file, sites_file)
command = args.shift
output = nil
- if command == "sites"
- conf = load_config File.open(config_file)
- begin
- output = sites(args, conf, sites_file) if command == "sites"
- rescue SiteError => exception
- output = exception.output
- File.open(sites_file, "w") do |sites_file|
- sites_file << output.to_yaml
- end
+ conf = load_config File.open(config_file)
+ return conf if command == "config"
+ begin
+ site_args = nil
+ site_args = args if command == "sites"
+ conf[:sites] = sites(site_args, conf, sites_file)
+ rescue SiteError => exception
+ conf[:sites] = exception.output
+ File.open(sites_file, "w") do |sites_file|
+ sites_file << output.to_yaml
end
- elsif command == "app"
- conf[:sites] = sites(nil, conf, sites_file)
- app(args, conf)
- else
- output = config args, config_file
end
+ return conf if command == "sites"
+ app(args, conf)
end
if $0 == __FILE__
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:32 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:32 -0500 (CDT)
Subject: [Swift-commit] r3312 - SwiftApps/adem-osg
Message-ID: <20100503195632.DF19E9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:32 -0500 (Mon, 03 May 2010)
New Revision: 3312
Modified:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Pacman installer and corresponding unit test
Modified: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:30 UTC (rev 3311)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:32 UTC (rev 3312)
@@ -75,6 +75,27 @@
)
)
end
+
+ def test_pacman_install_firefly_jbsim3d
+ # Cleanup target first
+ `globus-job-run ff-grid.unl.edu -env PACMAN_LOCATION=/opt/pacman/pacman-3.28 -d /panfs/panasas/CMS/app/engage /opt/pacman/pacman-3.28/bin/pacman -remove jbsim3d`
+ expected = <<-eos
+ jbsim3d found in http://www.ci.uchicago.edu/~aespinosa/Cybershake...
+ Installing jbsim3d...
+ Downloading jbsim3d_r794~RHEL5_amd64.tar.gz...
+ Untarring jbsim3d_r794~RHEL5_amd64.tar.gz...
+ jbsim3d has been installed.
+ eos
+ assert_equal(
+ expected,
+ pacman_install(
+ "http://www.ci.uchicago.edu/~aespinosa/Cybershake:jbsim3d",
+ { :contact => "ff-grid.unl.edu/jobmanager-fork",
+ :pacman => "/opt/pacman/pacman-3.28",
+ :path => "/panfs/panasas/CMS/app/engage" }
+ )
+ )
+ end
end
class OfflineTest < Test::Unit::TestCase
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:30 UTC (rev 3311)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:32 UTC (rev 3312)
@@ -117,7 +117,13 @@
path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
contact = site_fork site[:compute_element]
site[:pacman] = pacman_find(contact, path) if not site[:pacman]
- pacman_install contact, path
+ target = {
+ :contact => contact,
+ :pacman => site[:pacman],
+ :path => path
+ }
+ package = "#{conf[:pacman_cache]}:#{app}"
+ pacman_install package, target
end
end
@@ -136,6 +142,19 @@
File.dirname(File.dirname(resp))
end
+def pacman_install(package, target)
+ File.open("pacman_install.sh", "w") do |dump|
+ dump.puts "#!/bin/bash"
+ dump.puts "source #{target[:pacman]}/setup.sh"
+ dump.puts "pacman -trust-all-caches -install #{package}"
+ end
+ File.chmod 0755, "pacman_install.sh"
+ `globus-job-run #{target[:contact]} /bin/mkdir -p #{target[:path]}`
+ resp = `globus-job-run #{target[:contact]} -d #{target[:path]} -stage pacman_install.sh`
+ File.delete "pacman_install.sh"
+ resp
+end
+
def config(args, config_file)
load_config File.open(config_file)
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:38 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:38 -0500 (CDT)
Subject: [Swift-commit] r3314 - in SwiftApps/adem-osg: . lib test
Message-ID: <20100503195638.E5AC89CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:38 -0500 (Mon, 03 May 2010)
New Revision: 3314
Added:
SwiftApps/adem-osg/lib/
SwiftApps/adem-osg/lib/adem.rb
SwiftApps/adem-osg/test/
SwiftApps/adem-osg/test/setup_test.rb
SwiftApps/adem-osg/test/test_offline.rb
SwiftApps/adem-osg/test/test_online.rb
SwiftApps/adem-osg/test/test_run.rb
Removed:
SwiftApps/adem-osg/adem-test.rb
SwiftApps/adem-osg/adem.rb
Log:
Moved to standard package directories
Deleted: SwiftApps/adem-osg/adem-test.rb
===================================================================
--- SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:34 UTC (rev 3313)
+++ SwiftApps/adem-osg/adem-test.rb 2010-05-03 19:56:38 UTC (rev 3314)
@@ -1,151 +0,0 @@
-#!/usb/bin/env ruby
-
-require 'test/unit'
-require 'adem'
-require 'yaml'
-
-module Adem
- module TestSetup
- def setup
- @conf = {
- :pacman_cache => "http://www.ci.uchicago.edu/~aespinosa/Cybershake",
- :ress_server => "osg-ress-1.fnal.gov",
- :virtual_organization => "engage"
- }
- @site_list = File.open("sites").read
- @ress = File.open("dummy_ress").read
- end
- end
-end
-
-class OnlineTest < Test::Unit::TestCase
- include Adem::TestSetup
-
- def test_sites_live
- conf = @conf
- begin
- sites(nil, conf, "non_existent_file")
- rescue SiteError => e
- assert_equal YAML.load(@site_list), e.output
- end
- end
-
- def test_ress_query
- conf = @conf
- ress = ress_query(conf).split "\n\n"
- assumption = true
- class_ads = [
- "GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
- "GlueCEInfoGatekeeperPort", "GlueSEAccessProtocolEndpoint",
- "GlueSEAccessProtocolType", "GlueCEInfoApplicationDir", "GlueCEInfoDataDir"
- ]
- missing = []
- ress.each do |entry|
- class_ads.each do |ad|
- assumption = assumption && entry.include?(ad)
- missing << ad if not entry.include?(ad)
- end
- end
- # False: there exist an entry without the expected classad.
- # Must report to osg support then
- assert assumption, "Missing attributes: \n\t#{missing.uniq.join(", ")}"
- end
-
- def test_app_avail
- pacman_cache = @conf[:pacman_cache]
- response = <<-eos
-http://www.ci.uchicago.edu/~aespinosa/Cybershake
- [ ] jbsim3d
- eos
- assert_equal(response, app_avail(pacman_cache))
- end
-
- def test_pacman_find_firefly
- # Firefly
- site = {
- :compute_element => "ff-grid.unl.edu:2119/jobmanager-pbs",
- :app_directory => "/panfs/panasas/CMS/app",
- :storage_element => "gsiftp://ff-gridftp.unl.edu:2811"
- }
- assert_equal(
- "/opt/pacman/pacman-3.28",
- pacman_find(
- "ff-grid.unl.edu/jobmanager-fork",
- "/panfs/panasas/CMS/app/engage"
- )
- )
- end
-
- def test_pacman_install_firefly_jbsim3d
- # Cleanup target first
- `globus-job-run ff-grid.unl.edu -env PACMAN_LOCATION=/opt/pacman/pacman-3.28 -d /panfs/panasas/CMS/app/engage /opt/pacman/pacman-3.28/bin/pacman -remove jbsim3d`
- expected = <<-eos
- jbsim3d found in http://www.ci.uchicago.edu/~aespinosa/Cybershake...
- Installing jbsim3d...
- Downloading jbsim3d_r794~RHEL5_amd64.tar.gz...
- Untarring jbsim3d_r794~RHEL5_amd64.tar.gz...
- jbsim3d has been installed.
- eos
- assert_equal(
- expected,
- pacman_install(
- "http://www.ci.uchicago.edu/~aespinosa/Cybershake:jbsim3d",
- { :contact => "ff-grid.unl.edu/jobmanager-fork",
- :pacman => "/opt/pacman/pacman-3.28",
- :path => "/panfs/panasas/CMS/app/engage" }
- )
- )
- end
-end
-
-class OfflineTest < Test::Unit::TestCase
- include Adem::TestSetup
-
- def test_sites_exception
- conf = @conf
- conf[:ress_server] = nil
- site_list = YAML.load @site_list
- assert_raise SiteError do
- sites(nil, conf, "non_existent_file")
- end
- end
-
- def test_sites_from_file
- conf = @conf
- site_list = YAML.load @site_list
- assert_equal(site_list, sites(nil, conf, "sites"))
- end
-
- def test_site_fork
- assert_equal "ff-grid.unl.edu:2119/jobmanager-fork", site_fork("ff-grid.unl.edu:2119/jobmanager-pbs")
- end
-
- def test_ress_parse
- site_list = YAML.load @site_list
- assert_equal site_list, ress_parse(@ress.split("\n\n"))
- end
-
- def test_app
- conf = @conf
- assert_equal("app", app(nil, conf))
- end
-
- def test_config
- assert_equal(@conf, config(nil, "config"))
- end
-end
-
-class RunTest < Test::Unit::TestCase
- include Adem::TestSetup
-
- def test_config
- assert_equal(@conf, run_command(["config"], "config", nil))
- end
-
- def test_sites_from_file
- conf = @conf
- site_list = YAML.load @site_list
- conf[:sites] = site_list
- assert_equal(conf, run_command(["sites"], "config", "sites"))
- end
-end
Deleted: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:34 UTC (rev 3313)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:38 UTC (rev 3314)
@@ -1,183 +0,0 @@
-#!/usr/bin/env ruby
-
-# = ADEM: Application software DEployment and Management
-# Author:: Zhengxiong Hou (original sh prototype)
-# Author:: Allan Espinosa (rewrite to ruby)
-#
-
-require 'yaml'
-require 'ftools'
-
-CONFIGURATION_FILE = "#{ENV['HOME']}/.adem/config"
-SITES_FILE = "#{ENV['HOME']}/.adem/sites"
-APPS_FILE = "#{ENV['HOME']}/.adem/apps"
-
-class SiteError < Exception
- attr_reader :output
- def initialize(output)
- @output = output
- end
-end
-
-class ConfigError < RuntimeError
-end
-
-def load_config(yaml_config)
- conf = YAML.load yaml_config
- conf.each do |key, val|
- conf.delete(key)
- conf[key.to_sym] = val
- end
- conf
-end
-
-def sites(args, conf, sites_file)
- begin
- YAML.load File.open sites_file
- rescue Errno::ENOENT
- result = ress_parse ress_query(conf).split("\n\n")
- raise SiteError.new(result)
- end
-end
-
-def ress_query(conf)
- `condor_status -pool #{conf[:ress_server]} -const \
- 'stringListIMember(\"VO:#{conf[:virtual_organization]}\", \
- GlueCEAccessControlBaseRule)' -long`
-end
-
-def ress_parse(ress)
- class_ads = [
- "GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
- "GlueCEInfoGatekeeperPort", "GlueSEAccessProtocolEndpoint",
- "GlueSEAccessProtocolType", "GlueCEInfoApplicationDir", "GlueCEInfoDataDir"
- ]
- site = {}
- ress.each do |entry|
- tmp = {}
- entry.each do |attr|
- attr.chomp!
- class_ads.each do |class_ad|
- if attr.include? class_ad
- tmp[class_ad] = attr.gsub!(/.*\=\ (.*)$/, '\1').gsub /"/, ''
- end
- end
- end
-
- next if tmp["GlueSiteUniqueID"] == nil
- site_name = tmp["GlueSiteUniqueID"]
- site[site_name] = {} if site[site_name] == nil
-
- compute_element = "#{tmp['GlueCEInfoHostName']}:#{tmp['GlueCEInfoGatekeeperPort']}/jobmanager-#{tmp['GlueCEInfoJobManager']}"
- if site[site_name]["compute_element"] == nil
- site[site_name]["compute_element"] = compute_element
- elsif site[site_name]["compute_element"] != compute_element
- old = site[site_name]["compute_element"].to_a
- old << compute_element
- site[site_name]["compute_element"] = old.uniq
- end
- storage_element = "gsiftp://#{tmp['GlueCEInfoHostName']}:2811"
- if site[site_name]["storage_element"] == nil
- site[site_name]["storage_element"] = storage_element
- elsif site[site_name]["storage_element"] != storage_element
- old = site[site_name]["storage_element"].to_a
- old << storage_element
- site[site_name]["storage_element"] = old.flatten.uniq
- end
- site[site_name]["data_directory"] = tmp['GlueCEInfoDataDir']
- site[site_name]["app_directory"] = tmp['GlueCEInfoApplicationDir']
-
- # Disabled because some storage endpoints are broken
- #storage_element = tmp['GlueSEAccessProtocolEndpoint'] if tmp['GlueSEAccessProtocolType'] == 'gsiftp'
- #next if storage_element == nil
- #if storage_element.include? ','
- #storage_element = storage_element.split(',').grep /gsiftp/
- #end
- #if site[site_name]["storage_element"] == nil
- #site[site_name]["storage_element"] = storage_element
- #elsif site[site_name]["storage_element"] != storage_element
- #old = site[site_name]["storage_element"].to_a
- #old << storage_element
- #site[site_name]["storage_element"] = old.flatten.uniq
- #end
- end
- site
-end
-
-def app(args, conf)
- "app"
-end
-
-def app_avail(pacman_cache)
- `pacman -trust-all-caches -lc #{pacman_cache}`
-end
-
-def app_deploy(app, conf)
- conf[:sites].each do |site|
- path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
- contact = site_fork site[:compute_element]
- site[:pacman] = pacman_find(contact, path) if not site[:pacman]
- target = {
- :contact => contact,
- :pacman => site[:pacman],
- :path => path
- }
- package = "#{conf[:pacman_cache]}:#{app}"
- pacman_install package, target
- end
-end
-
-def site_fork(compute_element)
- compute_element.gsub /jobmanager-.*$/, "jobmanager-fork"
-end
-
-def pacman_find(contact, rootdir)
- File.open("find_pacman.sh", "w") do |dump|
- dump.puts "#!/bin/bash"
- dump.puts "which pacman"
- end
- File.chmod 0755, "find_pacman.sh"
- resp = `globus-job-run #{contact} -d #{rootdir} -stage find_pacman.sh`
- File.delete "find_pacman.sh"
- File.dirname(File.dirname(resp))
-end
-
-def pacman_install(package, target)
- File.open("pacman_install.sh", "w") do |dump|
- dump.puts "#!/bin/bash"
- dump.puts "source #{target[:pacman]}/setup.sh"
- dump.puts "pacman -trust-all-caches -install #{package}"
- end
- File.chmod 0755, "pacman_install.sh"
- `globus-job-run #{target[:contact]} /bin/mkdir -p #{target[:path]}`
- resp = `globus-job-run #{target[:contact]} -d #{target[:path]} -stage pacman_install.sh`
- File.delete "pacman_install.sh"
- resp
-end
-
-def config(args, config_file)
- load_config File.open(config_file)
-end
-
-def run_command(args, config_file, sites_file)
- command = args.shift
- output = nil
- conf = load_config File.open(config_file)
- return conf if command == "config"
- begin
- site_args = nil
- site_args = args if command == "sites"
- conf[:sites] = sites(site_args, conf, sites_file)
- rescue SiteError => exception
- conf[:sites] = exception.output
- File.open(sites_file, "w") do |sites_file|
- sites_file << output.to_yaml
- end
- end
- return conf if command == "sites"
- app(args, conf)
-end
-
-if $0 == __FILE__
- run_command ARGV, CONFIGURATION_FILE, SITES_FILE
-end
Copied: SwiftApps/adem-osg/lib/adem.rb (from rev 3313, SwiftApps/adem-osg/adem.rb)
===================================================================
--- SwiftApps/adem-osg/lib/adem.rb (rev 0)
+++ SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:56:38 UTC (rev 3314)
@@ -0,0 +1,183 @@
+#!/usr/bin/env ruby
+
+# = ADEM: Application software DEployment and Management
+# Author:: Zhengxiong Hou (original sh prototype)
+# Author:: Allan Espinosa (rewrite to ruby)
+#
+
+require 'yaml'
+require 'ftools'
+
+CONFIGURATION_FILE = "#{ENV['HOME']}/.adem/config"
+SITES_FILE = "#{ENV['HOME']}/.adem/sites"
+APPS_FILE = "#{ENV['HOME']}/.adem/apps"
+
+class SiteError < Exception
+ attr_reader :output
+ def initialize(output)
+ @output = output
+ end
+end
+
+class ConfigError < RuntimeError
+end
+
+def load_config(yaml_config)
+ conf = YAML.load yaml_config
+ conf.each do |key, val|
+ conf.delete(key)
+ conf[key.to_sym] = val
+ end
+ conf
+end
+
+def sites(args, conf, sites_file)
+ begin
+ YAML.load File.open sites_file
+ rescue Errno::ENOENT
+ result = ress_parse ress_query(conf).split("\n\n")
+ raise SiteError.new(result)
+ end
+end
+
+def ress_query(conf)
+ `condor_status -pool #{conf[:ress_server]} -const \
+ 'stringListIMember(\"VO:#{conf[:virtual_organization]}\", \
+ GlueCEAccessControlBaseRule)' -long`
+end
+
+def ress_parse(ress)
+ class_ads = [
+ "GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
+ "GlueCEInfoGatekeeperPort", "GlueSEAccessProtocolEndpoint",
+ "GlueSEAccessProtocolType", "GlueCEInfoApplicationDir", "GlueCEInfoDataDir"
+ ]
+ site = {}
+ ress.each do |entry|
+ tmp = {}
+ entry.each do |attr|
+ attr.chomp!
+ class_ads.each do |class_ad|
+ if attr.include? class_ad
+ tmp[class_ad] = attr.gsub!(/.*\=\ (.*)$/, '\1').gsub /"/, ''
+ end
+ end
+ end
+
+ next if tmp["GlueSiteUniqueID"] == nil
+ site_name = tmp["GlueSiteUniqueID"]
+ site[site_name] = {} if site[site_name] == nil
+
+ compute_element = "#{tmp['GlueCEInfoHostName']}:#{tmp['GlueCEInfoGatekeeperPort']}/jobmanager-#{tmp['GlueCEInfoJobManager']}"
+ if site[site_name]["compute_element"] == nil
+ site[site_name]["compute_element"] = compute_element
+ elsif site[site_name]["compute_element"] != compute_element
+ old = site[site_name]["compute_element"].to_a
+ old << compute_element
+ site[site_name]["compute_element"] = old.uniq
+ end
+ storage_element = "gsiftp://#{tmp['GlueCEInfoHostName']}:2811"
+ if site[site_name]["storage_element"] == nil
+ site[site_name]["storage_element"] = storage_element
+ elsif site[site_name]["storage_element"] != storage_element
+ old = site[site_name]["storage_element"].to_a
+ old << storage_element
+ site[site_name]["storage_element"] = old.flatten.uniq
+ end
+ site[site_name]["data_directory"] = tmp['GlueCEInfoDataDir']
+ site[site_name]["app_directory"] = tmp['GlueCEInfoApplicationDir']
+
+ # Disabled because some storage endpoints are broken
+ #storage_element = tmp['GlueSEAccessProtocolEndpoint'] if tmp['GlueSEAccessProtocolType'] == 'gsiftp'
+ #next if storage_element == nil
+ #if storage_element.include? ','
+ #storage_element = storage_element.split(',').grep /gsiftp/
+ #end
+ #if site[site_name]["storage_element"] == nil
+ #site[site_name]["storage_element"] = storage_element
+ #elsif site[site_name]["storage_element"] != storage_element
+ #old = site[site_name]["storage_element"].to_a
+ #old << storage_element
+ #site[site_name]["storage_element"] = old.flatten.uniq
+ #end
+ end
+ site
+end
+
+def app(args, conf)
+ "app"
+end
+
+def app_avail(pacman_cache)
+ `pacman -trust-all-caches -lc #{pacman_cache}`
+end
+
+def app_deploy(app, conf)
+ conf[:sites].each do |site|
+ path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
+ contact = site_fork site[:compute_element]
+ site[:pacman] = pacman_find(contact, path) if not site[:pacman]
+ target = {
+ :contact => contact,
+ :pacman => site[:pacman],
+ :path => path
+ }
+ package = "#{conf[:pacman_cache]}:#{app}"
+ pacman_install package, target
+ end
+end
+
+def site_fork(compute_element)
+ compute_element.gsub /jobmanager-.*$/, "jobmanager-fork"
+end
+
+def pacman_find(contact, rootdir)
+ File.open("find_pacman.sh", "w") do |dump|
+ dump.puts "#!/bin/bash"
+ dump.puts "which pacman"
+ end
+ File.chmod 0755, "find_pacman.sh"
+ resp = `globus-job-run #{contact} -d #{rootdir} -stage find_pacman.sh`
+ File.delete "find_pacman.sh"
+ File.dirname(File.dirname(resp))
+end
+
+def pacman_install(package, target)
+ File.open("pacman_install.sh", "w") do |dump|
+ dump.puts "#!/bin/bash"
+ dump.puts "source #{target[:pacman]}/setup.sh"
+ dump.puts "pacman -trust-all-caches -install #{package}"
+ end
+ File.chmod 0755, "pacman_install.sh"
+ `globus-job-run #{target[:contact]} /bin/mkdir -p #{target[:path]}`
+ resp = `globus-job-run #{target[:contact]} -d #{target[:path]} -stage pacman_install.sh`
+ File.delete "pacman_install.sh"
+ resp
+end
+
+def config(args, config_file)
+ load_config File.open(config_file)
+end
+
+def run_command(args, config_file, sites_file)
+ command = args.shift
+ output = nil
+ conf = load_config File.open(config_file)
+ return conf if command == "config"
+ begin
+ site_args = nil
+ site_args = args if command == "sites"
+ conf[:sites] = sites(site_args, conf, sites_file)
+ rescue SiteError => exception
+ conf[:sites] = exception.output
+ File.open(sites_file, "w") do |sites_file|
+ sites_file << output.to_yaml
+ end
+ end
+ return conf if command == "sites"
+ app(args, conf) if command == "app"
+end
+
+if $0 == __FILE__
+ run_command ARGV, CONFIGURATION_FILE, SITES_FILE
+end
Added: SwiftApps/adem-osg/test/setup_test.rb
===================================================================
--- SwiftApps/adem-osg/test/setup_test.rb (rev 0)
+++ SwiftApps/adem-osg/test/setup_test.rb 2010-05-03 19:56:38 UTC (rev 3314)
@@ -0,0 +1,19 @@
+#!/usb/bin/env ruby
+
+require 'test/unit'
+require 'adem'
+require 'yaml'
+
+module Adem
+ module TestSetup
+ def setup
+ @conf = {
+ :pacman_cache => "http://www.ci.uchicago.edu/~aespinosa/Cybershake",
+ :ress_server => "osg-ress-1.fnal.gov",
+ :virtual_organization => "engage"
+ }
+ @site_list = File.open("sites").read
+ @ress = File.open("dummy_ress").read
+ end
+ end
+end
Added: SwiftApps/adem-osg/test/test_offline.rb
===================================================================
--- SwiftApps/adem-osg/test/test_offline.rb (rev 0)
+++ SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:56:38 UTC (rev 3314)
@@ -0,0 +1,38 @@
+require 'setup_test'
+
+class OfflineTest < Test::Unit::TestCase
+ include Adem::TestSetup
+
+ def test_sites_exception
+ conf = @conf
+ conf[:ress_server] = nil
+ site_list = YAML.load @site_list
+ assert_raise SiteError do
+ sites(nil, conf, "non_existent_file")
+ end
+ end
+
+ def test_sites_from_file
+ conf = @conf
+ site_list = YAML.load @site_list
+ assert_equal(site_list, sites(nil, conf, "sites"))
+ end
+
+ def test_site_fork
+ assert_equal "ff-grid.unl.edu:2119/jobmanager-fork", site_fork("ff-grid.unl.edu:2119/jobmanager-pbs")
+ end
+
+ def test_ress_parse
+ site_list = YAML.load @site_list
+ assert_equal site_list, ress_parse(@ress.split("\n\n"))
+ end
+
+ def test_app
+ conf = @conf
+ assert_equal("app", app(nil, conf))
+ end
+
+ def test_config
+ assert_equal(@conf, config(nil, "config"))
+ end
+end
Added: SwiftApps/adem-osg/test/test_online.rb
===================================================================
--- SwiftApps/adem-osg/test/test_online.rb (rev 0)
+++ SwiftApps/adem-osg/test/test_online.rb 2010-05-03 19:56:38 UTC (rev 3314)
@@ -0,0 +1,81 @@
+require 'setup_test'
+
+class OnlineTest < Test::Unit::TestCase
+ include Adem::TestSetup
+
+ def test_sites_live
+ conf = @conf
+ begin
+ sites(nil, conf, "non_existent_file")
+ rescue SiteError => e
+ assert_equal YAML.load(@site_list), e.output
+ end
+ end
+
+ def test_ress_query
+ conf = @conf
+ ress = ress_query(conf).split "\n\n"
+ assumption = true
+ class_ads = [
+ "GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
+ "GlueCEInfoGatekeeperPort", "GlueSEAccessProtocolEndpoint",
+ "GlueSEAccessProtocolType", "GlueCEInfoApplicationDir", "GlueCEInfoDataDir"
+ ]
+ missing = []
+ ress.each do |entry|
+ class_ads.each do |ad|
+ assumption = assumption && entry.include?(ad)
+ missing << ad if not entry.include?(ad)
+ end
+ end
+ # False: there exist an entry without the expected classad.
+ # Must report to osg support then
+ assert assumption, "Missing attributes: \n\t#{missing.uniq.join(", ")}"
+ end
+
+ def test_app_avail
+ pacman_cache = @conf[:pacman_cache]
+ response = <<-eos
+http://www.ci.uchicago.edu/~aespinosa/Cybershake
+ [ ] jbsim3d
+ eos
+ assert_equal(response, app_avail(pacman_cache))
+ end
+
+ def test_pacman_find_firefly
+ # Firefly
+ site = {
+ :compute_element => "ff-grid.unl.edu:2119/jobmanager-pbs",
+ :app_directory => "/panfs/panasas/CMS/app",
+ :storage_element => "gsiftp://ff-gridftp.unl.edu:2811"
+ }
+ assert_equal(
+ "/opt/pacman/pacman-3.28",
+ pacman_find(
+ "ff-grid.unl.edu/jobmanager-fork",
+ "/panfs/panasas/CMS/app/engage"
+ )
+ )
+ end
+
+ def test_pacman_install_firefly_jbsim3d
+ # Cleanup target first
+ `globus-job-run ff-grid.unl.edu -env PACMAN_LOCATION=/opt/pacman/pacman-3.28 -d /panfs/panasas/CMS/app/engage /opt/pacman/pacman-3.28/bin/pacman -remove jbsim3d`
+ expected = <<-eos
+ jbsim3d found in http://www.ci.uchicago.edu/~aespinosa/Cybershake...
+ Installing jbsim3d...
+ Downloading jbsim3d_r794~RHEL5_amd64.tar.gz...
+ Untarring jbsim3d_r794~RHEL5_amd64.tar.gz...
+ jbsim3d has been installed.
+ eos
+ assert_equal(
+ expected,
+ pacman_install(
+ "http://www.ci.uchicago.edu/~aespinosa/Cybershake:jbsim3d",
+ { :contact => "ff-grid.unl.edu/jobmanager-fork",
+ :pacman => "/opt/pacman/pacman-3.28",
+ :path => "/panfs/panasas/CMS/app/engage" }
+ )
+ )
+ end
+end
Added: SwiftApps/adem-osg/test/test_run.rb
===================================================================
--- SwiftApps/adem-osg/test/test_run.rb (rev 0)
+++ SwiftApps/adem-osg/test/test_run.rb 2010-05-03 19:56:38 UTC (rev 3314)
@@ -0,0 +1,16 @@
+require 'setup_test'
+
+class RunTest < Test::Unit::TestCase
+ include Adem::TestSetup
+
+ def test_config
+ assert_equal(@conf, run_command(["config"], "config", nil))
+ end
+
+ def test_sites_from_file
+ conf = @conf
+ site_list = YAML.load @site_list
+ conf[:sites] = site_list
+ assert_equal(conf, run_command(["sites"], "config", "sites"))
+ end
+end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:35 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:35 -0500 (CDT)
Subject: [Swift-commit] r3313 - SwiftApps/adem-osg
Message-ID: <20100503195635.054039CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:34 -0500 (Mon, 03 May 2010)
New Revision: 3313
Modified:
SwiftApps/adem-osg/adem.rb
Log:
Moved sites() method
Modified: SwiftApps/adem-osg/adem.rb
===================================================================
--- SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:32 UTC (rev 3312)
+++ SwiftApps/adem-osg/adem.rb 2010-05-03 19:56:34 UTC (rev 3313)
@@ -31,6 +31,15 @@
conf
end
+def sites(args, conf, sites_file)
+ begin
+ YAML.load File.open sites_file
+ rescue Errno::ENOENT
+ result = ress_parse ress_query(conf).split("\n\n")
+ raise SiteError.new(result)
+ end
+end
+
def ress_query(conf)
`condor_status -pool #{conf[:ress_server]} -const \
'stringListIMember(\"VO:#{conf[:virtual_organization]}\", \
@@ -95,15 +104,6 @@
site
end
-def sites(args, conf, sites_file)
- begin
- YAML.load File.open sites_file
- rescue Errno::ENOENT
- result = ress_parse ress_query(conf).split("\n\n")
- raise SiteError.new(result)
- end
-end
-
def app(args, conf)
"app"
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:46 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:46 -0500 (CDT)
Subject: [Swift-commit] r3316 - SwiftApps/adem-osg
Message-ID: <20100503195646.86E429CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:46 -0500 (Mon, 03 May 2010)
New Revision: 3316
Added:
SwiftApps/adem-osg/Rakefile
Log:
Rake test tasks
Added: SwiftApps/adem-osg/Rakefile
===================================================================
--- SwiftApps/adem-osg/Rakefile (rev 0)
+++ SwiftApps/adem-osg/Rakefile 2010-05-03 19:56:46 UTC (rev 3316)
@@ -0,0 +1,9 @@
+require 'rake/testtask'
+
+task :default => [:test]
+
+Rake::TestTask.new do |test|
+ test.libs << "test"
+ test.test_files = Dir[ "test/test_*.rb" ]
+ test.verbose = true
+end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:49 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:49 -0500 (CDT)
Subject: [Swift-commit] r3317 - in SwiftApps/adem-osg: bin lib
Message-ID: <20100503195649.C32A19CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:49 -0500 (Mon, 03 May 2010)
New Revision: 3317
Added:
SwiftApps/adem-osg/bin/adem
Modified:
SwiftApps/adem-osg/lib/adem.rb
Log:
Moved executable script part to bin/
Added: SwiftApps/adem-osg/bin/adem
===================================================================
--- SwiftApps/adem-osg/bin/adem (rev 0)
+++ SwiftApps/adem-osg/bin/adem 2010-05-03 19:56:49 UTC (rev 3317)
@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'adem'
+
+if $0 == __FILE__
+ run_command ARGV, CONFIGURATION_FILE, SITES_FILE
+end
Property changes on: SwiftApps/adem-osg/bin/adem
___________________________________________________________________
Name: svn:executable
+ *
Modified: SwiftApps/adem-osg/lib/adem.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:56:46 UTC (rev 3316)
+++ SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:56:49 UTC (rev 3317)
@@ -1,10 +1,3 @@
-#!/usr/bin/env ruby
-
-# = ADEM: Application software DEployment and Management
-# Author:: Zhengxiong Hou (original sh prototype)
-# Author:: Allan Espinosa (rewrite to ruby)
-#
-
require 'yaml'
require 'ftools'
@@ -177,7 +170,3 @@
return conf if command == "sites"
app(args, conf) if command == "app"
end
-
-if $0 == __FILE__
- run_command ARGV, CONFIGURATION_FILE, SITES_FILE
-end
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:52 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:52 -0500 (CDT)
Subject: [Swift-commit] r3318 - SwiftApps/adem-osg/test
Message-ID: <20100503195652.334419CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:52 -0500 (Mon, 03 May 2010)
New Revision: 3318
Modified:
SwiftApps/adem-osg/test/test_offline.rb
Log:
Fixed dummy paths
Modified: SwiftApps/adem-osg/test/test_offline.rb
===================================================================
--- SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:56:49 UTC (rev 3317)
+++ SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:56:52 UTC (rev 3318)
@@ -15,7 +15,7 @@
def test_sites_from_file
conf = @conf
site_list = YAML.load @site_list
- assert_equal(site_list, sites(nil, conf, "sites"))
+ assert_equal(site_list, sites(nil, conf, "test/dummy_sites"))
end
def test_site_fork
From noreply at svn.ci.uchicago.edu Mon May 3 14:56:55 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:56:55 -0500 (CDT)
Subject: [Swift-commit] r3319 - in SwiftApps/adem-osg: . test
Message-ID: <20100503195655.63F549CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:56:55 -0500 (Mon, 03 May 2010)
New Revision: 3319
Added:
SwiftApps/adem-osg/test/dummy_config
Removed:
SwiftApps/adem-osg/config
Modified:
SwiftApps/adem-osg/test/test_run.rb
Log:
Updated RunTest dummy paths
Deleted: SwiftApps/adem-osg/config
===================================================================
--- SwiftApps/adem-osg/config 2010-05-03 19:56:52 UTC (rev 3318)
+++ SwiftApps/adem-osg/config 2010-05-03 19:56:55 UTC (rev 3319)
@@ -1,4 +0,0 @@
----
-ress_server: osg-ress-1.fnal.gov
-pacman_cache: http://www.ci.uchicago.edu/~aespinosa/Cybershake
-virtual_organization: engage
Copied: SwiftApps/adem-osg/test/dummy_config (from rev 3318, SwiftApps/adem-osg/config)
===================================================================
--- SwiftApps/adem-osg/test/dummy_config (rev 0)
+++ SwiftApps/adem-osg/test/dummy_config 2010-05-03 19:56:55 UTC (rev 3319)
@@ -0,0 +1,4 @@
+---
+ress_server: osg-ress-1.fnal.gov
+pacman_cache: http://www.ci.uchicago.edu/~aespinosa/Cybershake
+virtual_organization: engage
Modified: SwiftApps/adem-osg/test/test_run.rb
===================================================================
--- SwiftApps/adem-osg/test/test_run.rb 2010-05-03 19:56:52 UTC (rev 3318)
+++ SwiftApps/adem-osg/test/test_run.rb 2010-05-03 19:56:55 UTC (rev 3319)
@@ -4,13 +4,16 @@
include Adem::TestSetup
def test_config
- assert_equal(@conf, run_command(["config"], "config", nil))
+ assert_equal(@conf, run_command(["config"], "test/dummy_config", nil))
end
def test_sites_from_file
conf = @conf
site_list = YAML.load @site_list
conf[:sites] = site_list
- assert_equal(conf, run_command(["sites"], "config", "sites"))
+ assert_equal(
+ conf,
+ run_command(["sites"],"test/dummy_config", "test/dummy_sites")
+ )
end
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:01 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:01 -0500 (CDT)
Subject: [Swift-commit] r3320 - in SwiftApps/adem-osg/lib: . adem
Message-ID: <20100503195701.A83EB9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:01 -0500 (Mon, 03 May 2010)
New Revision: 3320
Added:
SwiftApps/adem-osg/lib/adem/
SwiftApps/adem-osg/lib/adem/app.rb
SwiftApps/adem-osg/lib/adem/config.rb
SwiftApps/adem-osg/lib/adem/sites.rb
Modified:
SwiftApps/adem-osg/lib/adem.rb
Log:
Grouped behavior by file
Added: SwiftApps/adem-osg/lib/adem/app.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/app.rb (rev 0)
+++ SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:01 UTC (rev 3320)
@@ -0,0 +1,52 @@
+def app(args, conf)
+ "app"
+end
+
+def app_avail(pacman_cache)
+ `pacman -trust-all-caches -lc #{pacman_cache}`
+end
+
+def app_deploy(app, conf)
+ conf[:sites].each do |site|
+ path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
+ contact = site_fork site[:compute_element]
+ site[:pacman] = pacman_find(contact, path) if not site[:pacman]
+ target = {
+ :contact => contact,
+ :pacman => site[:pacman],
+ :path => path
+ }
+ package = "#{conf[:pacman_cache]}:#{app}"
+ pacman_install package, target
+ end
+end
+
+def site_fork(compute_element)
+ compute_element.gsub /jobmanager-.*$/, "jobmanager-fork"
+end
+
+def pacman_find(contact, rootdir)
+ File.open("find_pacman.sh", "w") do |dump|
+ dump.puts "#!/bin/bash"
+ dump.puts "which pacman"
+ end
+ File.chmod 0755, "find_pacman.sh"
+ resp = `globus-job-run #{contact} -d #{rootdir} -stage find_pacman.sh`
+ File.delete "find_pacman.sh"
+ File.dirname(File.dirname(resp))
+end
+
+def pacman_install(package, target)
+ File.open("pacman_install.sh", "w") do |dump|
+ dump.puts "#!/bin/bash"
+ dump.puts "source #{target[:pacman]}/setup.sh"
+ dump.puts "pacman -trust-all-caches -install #{package}"
+ end
+ File.chmod 0755, "pacman_install.sh"
+ `globus-job-run #{target[:contact]} /bin/mkdir -p #{target[:path]}`
+ resp = `globus-job-run #{target[:contact]} -d #{target[:path]} -stage pacman_install.sh`
+ File.delete "pacman_install.sh"
+ resp
+end
+
+
Added: SwiftApps/adem-osg/lib/adem/config.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/config.rb (rev 0)
+++ SwiftApps/adem-osg/lib/adem/config.rb 2010-05-03 19:57:01 UTC (rev 3320)
@@ -0,0 +1,15 @@
+class ConfigError < RuntimeError
+end
+
+def load_config(yaml_config)
+ conf = YAML.load yaml_config
+ conf.each do |key, val|
+ conf.delete(key)
+ conf[key.to_sym] = val
+ end
+ conf
+end
+
+def config(args, config_file)
+ load_config File.open(config_file)
+end
Added: SwiftApps/adem-osg/lib/adem/sites.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/sites.rb (rev 0)
+++ SwiftApps/adem-osg/lib/adem/sites.rb 2010-05-03 19:57:01 UTC (rev 3320)
@@ -0,0 +1,81 @@
+class SiteError < Exception
+ attr_reader :output
+ def initialize(output)
+ @output = output
+ end
+end
+
+def sites(args, conf, sites_file)
+ begin
+ YAML.load File.open sites_file
+ rescue Errno::ENOENT
+ result = ress_parse ress_query(conf).split("\n\n")
+ raise SiteError.new(result)
+ end
+end
+
+def ress_query(conf)
+ `condor_status -pool #{conf[:ress_server]} -const \
+ 'stringListIMember(\"VO:#{conf[:virtual_organization]}\", \
+ GlueCEAccessControlBaseRule)' -long`
+end
+
+def ress_parse(ress)
+ class_ads = [
+ "GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
+ "GlueCEInfoGatekeeperPort", "GlueSEAccessProtocolEndpoint",
+ "GlueSEAccessProtocolType", "GlueCEInfoApplicationDir", "GlueCEInfoDataDir"
+ ]
+ site = {}
+ ress.each do |entry|
+ tmp = {}
+ entry.each do |attr|
+ attr.chomp!
+ class_ads.each do |class_ad|
+ if attr.include? class_ad
+ tmp[class_ad] = attr.gsub!(/.*\=\ (.*)$/, '\1').gsub /"/, ''
+ end
+ end
+ end
+
+ next if tmp["GlueSiteUniqueID"] == nil
+ site_name = tmp["GlueSiteUniqueID"]
+ site[site_name] = {} if site[site_name] == nil
+
+ compute_element = "#{tmp['GlueCEInfoHostName']}:#{tmp['GlueCEInfoGatekeeperPort']}/jobmanager-#{tmp['GlueCEInfoJobManager']}"
+ if site[site_name]["compute_element"] == nil
+ site[site_name]["compute_element"] = compute_element
+ elsif site[site_name]["compute_element"] != compute_element
+ old = site[site_name]["compute_element"].to_a
+ old << compute_element
+ site[site_name]["compute_element"] = old.uniq
+ end
+ storage_element = "gsiftp://#{tmp['GlueCEInfoHostName']}:2811"
+ if site[site_name]["storage_element"] == nil
+ site[site_name]["storage_element"] = storage_element
+ elsif site[site_name]["storage_element"] != storage_element
+ old = site[site_name]["storage_element"].to_a
+ old << storage_element
+ site[site_name]["storage_element"] = old.flatten.uniq
+ end
+ site[site_name]["data_directory"] = tmp['GlueCEInfoDataDir']
+ site[site_name]["app_directory"] = tmp['GlueCEInfoApplicationDir']
+
+ # Disabled because some storage endpoints are broken
+ #storage_element = tmp['GlueSEAccessProtocolEndpoint'] if tmp['GlueSEAccessProtocolType'] == 'gsiftp'
+ #next if storage_element == nil
+ #if storage_element.include? ','
+ #storage_element = storage_element.split(',').grep /gsiftp/
+ #end
+ #if site[site_name]["storage_element"] == nil
+ #site[site_name]["storage_element"] = storage_element
+ #elsif site[site_name]["storage_element"] != storage_element
+ #old = site[site_name]["storage_element"].to_a
+ #old << storage_element
+ #site[site_name]["storage_element"] = old.flatten.uniq
+ #end
+ end
+ site
+end
+
+
Modified: SwiftApps/adem-osg/lib/adem.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:56:55 UTC (rev 3319)
+++ SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:57:01 UTC (rev 3320)
@@ -1,157 +1,14 @@
require 'yaml'
require 'ftools'
+require 'adem/config'
+require 'adem/sites'
+require 'adem/app'
+
CONFIGURATION_FILE = "#{ENV['HOME']}/.adem/config"
SITES_FILE = "#{ENV['HOME']}/.adem/sites"
APPS_FILE = "#{ENV['HOME']}/.adem/apps"
-class SiteError < Exception
- attr_reader :output
- def initialize(output)
- @output = output
- end
-end
-
-class ConfigError < RuntimeError
-end
-
-def load_config(yaml_config)
- conf = YAML.load yaml_config
- conf.each do |key, val|
- conf.delete(key)
- conf[key.to_sym] = val
- end
- conf
-end
-
-def sites(args, conf, sites_file)
- begin
- YAML.load File.open sites_file
- rescue Errno::ENOENT
- result = ress_parse ress_query(conf).split("\n\n")
- raise SiteError.new(result)
- end
-end
-
-def ress_query(conf)
- `condor_status -pool #{conf[:ress_server]} -const \
- 'stringListIMember(\"VO:#{conf[:virtual_organization]}\", \
- GlueCEAccessControlBaseRule)' -long`
-end
-
-def ress_parse(ress)
- class_ads = [
- "GlueSiteUniqueID", "GlueCEInfoHostName", "GlueCEInfoJobManager",
- "GlueCEInfoGatekeeperPort", "GlueSEAccessProtocolEndpoint",
- "GlueSEAccessProtocolType", "GlueCEInfoApplicationDir", "GlueCEInfoDataDir"
- ]
- site = {}
- ress.each do |entry|
- tmp = {}
- entry.each do |attr|
- attr.chomp!
- class_ads.each do |class_ad|
- if attr.include? class_ad
- tmp[class_ad] = attr.gsub!(/.*\=\ (.*)$/, '\1').gsub /"/, ''
- end
- end
- end
-
- next if tmp["GlueSiteUniqueID"] == nil
- site_name = tmp["GlueSiteUniqueID"]
- site[site_name] = {} if site[site_name] == nil
-
- compute_element = "#{tmp['GlueCEInfoHostName']}:#{tmp['GlueCEInfoGatekeeperPort']}/jobmanager-#{tmp['GlueCEInfoJobManager']}"
- if site[site_name]["compute_element"] == nil
- site[site_name]["compute_element"] = compute_element
- elsif site[site_name]["compute_element"] != compute_element
- old = site[site_name]["compute_element"].to_a
- old << compute_element
- site[site_name]["compute_element"] = old.uniq
- end
- storage_element = "gsiftp://#{tmp['GlueCEInfoHostName']}:2811"
- if site[site_name]["storage_element"] == nil
- site[site_name]["storage_element"] = storage_element
- elsif site[site_name]["storage_element"] != storage_element
- old = site[site_name]["storage_element"].to_a
- old << storage_element
- site[site_name]["storage_element"] = old.flatten.uniq
- end
- site[site_name]["data_directory"] = tmp['GlueCEInfoDataDir']
- site[site_name]["app_directory"] = tmp['GlueCEInfoApplicationDir']
-
- # Disabled because some storage endpoints are broken
- #storage_element = tmp['GlueSEAccessProtocolEndpoint'] if tmp['GlueSEAccessProtocolType'] == 'gsiftp'
- #next if storage_element == nil
- #if storage_element.include? ','
- #storage_element = storage_element.split(',').grep /gsiftp/
- #end
- #if site[site_name]["storage_element"] == nil
- #site[site_name]["storage_element"] = storage_element
- #elsif site[site_name]["storage_element"] != storage_element
- #old = site[site_name]["storage_element"].to_a
- #old << storage_element
- #site[site_name]["storage_element"] = old.flatten.uniq
- #end
- end
- site
-end
-
-def app(args, conf)
- "app"
-end
-
-def app_avail(pacman_cache)
- `pacman -trust-all-caches -lc #{pacman_cache}`
-end
-
-def app_deploy(app, conf)
- conf[:sites].each do |site|
- path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
- contact = site_fork site[:compute_element]
- site[:pacman] = pacman_find(contact, path) if not site[:pacman]
- target = {
- :contact => contact,
- :pacman => site[:pacman],
- :path => path
- }
- package = "#{conf[:pacman_cache]}:#{app}"
- pacman_install package, target
- end
-end
-
-def site_fork(compute_element)
- compute_element.gsub /jobmanager-.*$/, "jobmanager-fork"
-end
-
-def pacman_find(contact, rootdir)
- File.open("find_pacman.sh", "w") do |dump|
- dump.puts "#!/bin/bash"
- dump.puts "which pacman"
- end
- File.chmod 0755, "find_pacman.sh"
- resp = `globus-job-run #{contact} -d #{rootdir} -stage find_pacman.sh`
- File.delete "find_pacman.sh"
- File.dirname(File.dirname(resp))
-end
-
-def pacman_install(package, target)
- File.open("pacman_install.sh", "w") do |dump|
- dump.puts "#!/bin/bash"
- dump.puts "source #{target[:pacman]}/setup.sh"
- dump.puts "pacman -trust-all-caches -install #{package}"
- end
- File.chmod 0755, "pacman_install.sh"
- `globus-job-run #{target[:contact]} /bin/mkdir -p #{target[:path]}`
- resp = `globus-job-run #{target[:contact]} -d #{target[:path]} -stage pacman_install.sh`
- File.delete "pacman_install.sh"
- resp
-end
-
-def config(args, config_file)
- load_config File.open(config_file)
-end
-
def run_command(args, config_file, sites_file)
command = args.shift
output = nil
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:05 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:05 -0500 (CDT)
Subject: [Swift-commit] r3321 - in SwiftApps/adem-osg: lib/adem test
test/dummy_cache
Message-ID: <20100503195705.5BC829CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:05 -0500 (Mon, 03 May 2010)
New Revision: 3321
Added:
SwiftApps/adem-osg/test/dummy_cache/
SwiftApps/adem-osg/test/dummy_cache/jbsim3d.pacman
Modified:
SwiftApps/adem-osg/lib/adem/app.rb
SwiftApps/adem-osg/test/test_online.rb
Log:
Mocked a dummy pacman cache
Modified: SwiftApps/adem-osg/lib/adem/app.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:01 UTC (rev 3320)
+++ SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:05 UTC (rev 3321)
@@ -1,5 +1,26 @@
+require 'optparse'
+
def app(args, conf)
- "app"
+ options = {}
+ optparse = OptionParser.new do |opts|
+ opts.banner = "Usage: adem app [options]"
+
+ opts.on('-l', '--avail', 'Available packages from cache') do
+ puts "Packages available from #{conf[:pacman_cache]}"
+ puts app_avail(conf[:pacman_cache]).grep /[(\ |\*)]/
+ end
+
+ opts.on('-i', '--install PACKAGE', 'Package to install') do |package|
+ puts "Installing #{package}"
+ app_install package, conf
+ end
+
+ opts.on_tail('-h', '--help', 'Show this help message') do
+ puts opts
+ exit
+ end
+ end
+ optparse.parse!
end
def app_avail(pacman_cache)
@@ -48,5 +69,3 @@
File.delete "pacman_install.sh"
resp
end
-
-
Added: SwiftApps/adem-osg/test/dummy_cache/jbsim3d.pacman
===================================================================
--- SwiftApps/adem-osg/test/dummy_cache/jbsim3d.pacman (rev 0)
+++ SwiftApps/adem-osg/test/dummy_cache/jbsim3d.pacman 2010-05-03 19:57:05 UTC (rev 3321)
@@ -0,0 +1,8 @@
+description = 'jbsim3d package'
+version('r794')
+platform('linux-rhel-5')
+processor('x86_64');
+{
+ downloadUntarzip('http://www.ci.uchicago.edu/~aespinosa/Cybershake/tarballs/jbsim3d_r794~RHEL5_amd64.tar.gz',
+ 'POSTPROC')
+}
Modified: SwiftApps/adem-osg/test/test_online.rb
===================================================================
--- SwiftApps/adem-osg/test/test_online.rb 2010-05-03 19:57:01 UTC (rev 3320)
+++ SwiftApps/adem-osg/test/test_online.rb 2010-05-03 19:57:05 UTC (rev 3321)
@@ -34,9 +34,9 @@
end
def test_app_avail
- pacman_cache = @conf[:pacman_cache]
+ pacman_cache = "test/dummy_cache"
response = <<-eos
-http://www.ci.uchicago.edu/~aespinosa/Cybershake
+test/dummy_cache
[ ] jbsim3d
eos
assert_equal(response, app_avail(pacman_cache))
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:07 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:07 -0500 (CDT)
Subject: [Swift-commit] r3322 - SwiftApps/adem-osg/test
Message-ID: <20100503195707.E8B559CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:07 -0500 (Mon, 03 May 2010)
New Revision: 3322
Modified:
SwiftApps/adem-osg/test/test_offline.rb
SwiftApps/adem-osg/test/test_online.rb
Log:
Changed test_app_avail to offline
Modified: SwiftApps/adem-osg/test/test_offline.rb
===================================================================
--- SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:57:05 UTC (rev 3321)
+++ SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:57:07 UTC (rev 3322)
@@ -35,4 +35,15 @@
def test_config
assert_equal(@conf, config(nil, "config"))
end
+
+ def test_app_avail
+ pacman_cache = "test/dummy_cache"
+ response = <<-eos
+test/dummy_cache
+ [ ] jbsim3d
+ eos
+ assert_equal(response, app_avail(pacman_cache))
+ end
+
+
end
Modified: SwiftApps/adem-osg/test/test_online.rb
===================================================================
--- SwiftApps/adem-osg/test/test_online.rb 2010-05-03 19:57:05 UTC (rev 3321)
+++ SwiftApps/adem-osg/test/test_online.rb 2010-05-03 19:57:07 UTC (rev 3322)
@@ -78,4 +78,21 @@
)
)
end
+
+ def test_pacman_remove_firefly_jbsim3d
+ expected = <<-eos
+ Uninstalling jbsim3d...
+ Removing /panfs/panasas/CMS/app/engage/jbsim3d_r794~RHEL5_amd64.tar.gz contents...
+ jbsim3d has been uninstalled.
+ eos
+ assert_equal(
+ expected,
+ pacman_remove(
+ "jbsim3d",
+ { :contact => "ff-grid.unl.edu/jobmanager-fork",
+ :pacman => "/opt/pacman/pacman-3.28",
+ :path => "/panfs/panasas/CMS/app/engage" }
+ )
+ )
+ end
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:13 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:13 -0500 (CDT)
Subject: [Swift-commit] r3324 - in SwiftApps/adem-osg: lib/adem test
Message-ID: <20100503195713.EAF7F9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:13 -0500 (Mon, 03 May 2010)
New Revision: 3324
Modified:
SwiftApps/adem-osg/lib/adem/app.rb
SwiftApps/adem-osg/test/test_online.rb
Log:
Package remove functionality and tests
Modified: SwiftApps/adem-osg/lib/adem/app.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:10 UTC (rev 3323)
+++ SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:13 UTC (rev 3324)
@@ -50,6 +50,24 @@
conf[:sites]
end
+def app_remove(app, conf)
+ conf[:sites].each do |site|
+ puts "Site #{site.key}"
+ path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
+ contact = site_fork site[:compute_element]
+ site[:pacman] = pacman_find(contact, path) if not site[:pacman]
+ target = {
+ :contact => contact,
+ :pacman => site[:pacman],
+ :path => path
+ }
+ package = "#{conf[:pacman_cache]}:#{app}"
+ pacman_remove package, target
+ end
+ conf[:sites]
+end
+
+
def site_fork(compute_element)
compute_element.gsub /jobmanager-.*$/, "jobmanager-fork"
end
@@ -77,3 +95,16 @@
File.delete "pacman_install.sh"
resp
end
+
+def pacman_remove(package, target)
+ File.open("pacman_install.sh", "w") do |dump|
+ dump.puts "#!/bin/bash"
+ dump.puts "source #{target[:pacman]}/setup.sh"
+ dump.puts "pacman -trust-all-caches -remove #{package}"
+ end
+ File.chmod 0755, "pacman_remove.sh"
+ `globus-job-run #{target[:contact]} /bin/mkdir -p #{target[:path]}`
+ resp = `globus-job-run #{target[:contact]} -d #{target[:path]} -stage pacman_remove.sh`
+ File.delete "pacman_remove.sh"
+ resp
+end
Modified: SwiftApps/adem-osg/test/test_online.rb
===================================================================
--- SwiftApps/adem-osg/test/test_online.rb 2010-05-03 19:57:10 UTC (rev 3323)
+++ SwiftApps/adem-osg/test/test_online.rb 2010-05-03 19:57:13 UTC (rev 3324)
@@ -33,15 +33,6 @@
assert assumption, "Missing attributes: \n\t#{missing.uniq.join(", ")}"
end
- def test_app_avail
- pacman_cache = "test/dummy_cache"
- response = <<-eos
-test/dummy_cache
- [ ] jbsim3d
- eos
- assert_equal(response, app_avail(pacman_cache))
- end
-
def test_pacman_find_firefly
# Firefly
site = {
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:10 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:10 -0500 (CDT)
Subject: [Swift-commit] r3323 - SwiftApps/adem-osg/lib/adem
Message-ID: <20100503195710.B72749CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:10 -0500 (Mon, 03 May 2010)
New Revision: 3323
Modified:
SwiftApps/adem-osg/lib/adem/app.rb
Log:
changed app usage messages and added verbosity
Modified: SwiftApps/adem-osg/lib/adem/app.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:07 UTC (rev 3322)
+++ SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:10 UTC (rev 3323)
@@ -2,6 +2,7 @@
def app(args, conf)
options = {}
+ site_conf = nil
optparse = OptionParser.new do |opts|
opts.banner = "Usage: adem app [options]"
@@ -10,11 +11,16 @@
puts app_avail(conf[:pacman_cache]).grep /[(\ |\*)]/
end
- opts.on('-i', '--install PACKAGE', 'Package to install') do |package|
- puts "Installing #{package}"
- app_install package, conf
+ opts.on('-i', '--install PACKAGE', 'Install package') do |package|
+ puts "Installing #{package} on sites supporting the virtual organization #{conf[:virtual_organization]}"
+ site_conf = app_install package, conf
end
+ opts.on('-r', '--remove PACKAGE', 'Remove package') do |package|
+ puts "Removing #{package} on sites supporting the virtual organization #{conf[:virtual_organization]}"
+ site_conf = app_remove package, conf
+ end
+
opts.on_tail('-h', '--help', 'Show this help message') do
puts opts
exit
@@ -29,6 +35,7 @@
def app_deploy(app, conf)
conf[:sites].each do |site|
+ puts "Site #{site.key}"
path = "#{site[:app_directory]}/#{conf[:virtual_organization]}"
contact = site_fork site[:compute_element]
site[:pacman] = pacman_find(contact, path) if not site[:pacman]
@@ -40,6 +47,7 @@
package = "#{conf[:pacman_cache]}:#{app}"
pacman_install package, target
end
+ conf[:sites]
end
def site_fork(compute_element)
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:17 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:17 -0500 (CDT)
Subject: [Swift-commit] r3325 - in SwiftApps/adem-osg: lib lib/adem test
Message-ID: <20100503195717.6C69E9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:17 -0500 (Mon, 03 May 2010)
New Revision: 3325
Modified:
SwiftApps/adem-osg/lib/adem.rb
SwiftApps/adem-osg/lib/adem/config.rb
SwiftApps/adem-osg/test/test_offline.rb
Log:
Added configuration options and tests
Modified: SwiftApps/adem-osg/lib/adem/config.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/config.rb 2010-05-03 19:57:13 UTC (rev 3324)
+++ SwiftApps/adem-osg/lib/adem/config.rb 2010-05-03 19:57:17 UTC (rev 3325)
@@ -1,7 +1,56 @@
+require 'fileutils'
+
class ConfigError < RuntimeError
end
-def load_config(yaml_config)
+def config(args, config_file)
+ options = {}
+ conf_new = {}
+ optparse = OptionParser.new do |opts|
+ opts.banner = 'Usage: adem config [options]'
+
+ opts.on(nil, '--ress-server SERVER', 'RESS server name') do |server|
+ conf_new[:ress_server] = server
+ end
+
+ opts.on(nil, '--pacman-cache CACHE', 'Location of pacman cache') do |cache|
+ conf_new[:pacman_cache] = cache
+ end
+
+ opts.on(nil, '--virtual-organization VO', 'Name of virtual organization') do |vo|
+ conf_new[:virtual_organization] = vo
+ end
+
+ opts.on(nil, '--display', 'Display current configuration') do
+ options[:display] = true
+ end
+
+ opts.on_tail('-h', '--help', 'Show this message') do
+ puts opts
+ exit
+ end
+ end
+ optparse.parse!
+
+ conf_yaml = nil
+ conf = nil
+ begin
+ conf_yaml = File.open(config_file).read
+ conf = config_load conf_yaml
+ if options[:display]
+ puts conf_yaml
+ exit
+ end
+ rescue Errno::ENOENT
+ # must be complete if we are creating a new file
+ config_check conf_new
+ end
+ conf.merge!(conf_new)
+ config_update config_file if not conf_new.empty?
+ conf
+end
+
+def config_load(yaml_config)
conf = YAML.load yaml_config
conf.each do |key, val|
conf.delete(key)
@@ -10,6 +59,15 @@
conf
end
-def config(args, config_file)
- load_config File.open(config_file)
+def config_check conf
+ raise ConfigError, "RESS server missing" if not conf[:ress_server]
+ raise ConfigError, "Pacman cache missing" if not conf[:pacman_cache]
+ raise ConfigError, "Virtual organization missing" if not conf[:virtual_organization]
end
+
+def config_update config_file
+ FileUtils.mkdir_p File.dirname(config_file)
+ File.open(config_file, "w") do |dump|
+ dump << conf.to_yaml
+ end
+end
Modified: SwiftApps/adem-osg/lib/adem.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:57:13 UTC (rev 3324)
+++ SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:57:17 UTC (rev 3325)
@@ -1,3 +1,4 @@
+require 'optparse'
require 'yaml'
require 'ftools'
@@ -12,7 +13,7 @@
def run_command(args, config_file, sites_file)
command = args.shift
output = nil
- conf = load_config File.open(config_file)
+ conf = config args, config_file
return conf if command == "config"
begin
site_args = nil
Modified: SwiftApps/adem-osg/test/test_offline.rb
===================================================================
--- SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:57:13 UTC (rev 3324)
+++ SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:57:17 UTC (rev 3325)
@@ -32,10 +32,6 @@
assert_equal("app", app(nil, conf))
end
- def test_config
- assert_equal(@conf, config(nil, "config"))
- end
-
def test_app_avail
pacman_cache = "test/dummy_cache"
response = <<-eos
@@ -45,5 +41,26 @@
assert_equal(response, app_avail(pacman_cache))
end
+ def test_config
+ assert_equal(@conf, config(nil, "test/dummy_config"))
+ end
+ def test_config_load
+ assert_equal(@conf, config_load(File.open("test/dummy_config").read))
+ end
+
+ def test_config_check
+ exception = assert_raise ConfigError do
+ config_check({})
+ end
+ assert_match /RESS server/, exception.message
+ exception = assert_raise ConfigError do
+ config_check({:ress_server => "foo"})
+ end
+ assert_match /Pacman cache/, exception.message
+ exception = assert_raise ConfigError do
+ config_check({:ress_server => "foo", :pacman_cache => "bar"})
+ end
+ assert_match /Virtual organization/, exception.message
+ end
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:19 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:19 -0500 (CDT)
Subject: [Swift-commit] r3326 - SwiftApps/adem-osg/test
Message-ID: <20100503195719.C347F9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:19 -0500 (Mon, 03 May 2010)
New Revision: 3326
Modified:
SwiftApps/adem-osg/test/test_offline.rb
Log:
Mocked ress_query for offline test
Modified: SwiftApps/adem-osg/test/test_offline.rb
===================================================================
--- SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:57:17 UTC (rev 3325)
+++ SwiftApps/adem-osg/test/test_offline.rb 2010-05-03 19:57:19 UTC (rev 3326)
@@ -7,6 +7,9 @@
conf = @conf
conf[:ress_server] = nil
site_list = YAML.load @site_list
+ def ress_query(x)
+ ""
+ end
assert_raise SiteError do
sites(nil, conf, "non_existent_file")
end
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:22 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:22 -0500 (CDT)
Subject: [Swift-commit] r3327 - SwiftApps/adem-osg/lib/adem
Message-ID: <20100503195722.64CAC9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:22 -0500 (Mon, 03 May 2010)
New Revision: 3327
Modified:
SwiftApps/adem-osg/lib/adem/sites.rb
Log:
Cleaned whitespace
Modified: SwiftApps/adem-osg/lib/adem/sites.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/sites.rb 2010-05-03 19:57:19 UTC (rev 3326)
+++ SwiftApps/adem-osg/lib/adem/sites.rb 2010-05-03 19:57:22 UTC (rev 3327)
@@ -77,5 +77,3 @@
end
site
end
-
-
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:25 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:25 -0500 (CDT)
Subject: [Swift-commit] r3328 - in SwiftApps/adem-osg/lib: . adem
Message-ID: <20100503195725.B41239CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:25 -0500 (Mon, 03 May 2010)
New Revision: 3328
Added:
SwiftApps/adem-osg/lib/adem/help.rb
Modified:
SwiftApps/adem-osg/lib/adem.rb
SwiftApps/adem-osg/lib/adem/app.rb
Log:
Added general help command
Modified: SwiftApps/adem-osg/lib/adem/app.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:22 UTC (rev 3327)
+++ SwiftApps/adem-osg/lib/adem/app.rb 2010-05-03 19:57:25 UTC (rev 3328)
@@ -1,8 +1,6 @@
-require 'optparse'
-
def app(args, conf)
options = {}
- site_conf = nil
+ site_conf = conf[:sites]
optparse = OptionParser.new do |opts|
opts.banner = "Usage: adem app [options]"
@@ -27,6 +25,7 @@
end
end
optparse.parse!
+ site_conf
end
def app_avail(pacman_cache)
Added: SwiftApps/adem-osg/lib/adem/help.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem/help.rb (rev 0)
+++ SwiftApps/adem-osg/lib/adem/help.rb 2010-05-03 19:57:25 UTC (rev 3328)
@@ -0,0 +1,22 @@
+def help
+ text = <<-eos
+
+ADEM is a tool for deploying and managing software on the Open Science Grid.
+
+ Usage:
+ adem command [options]
+
+ Examples:
+ adem config --display
+ adem sites --update
+ adem app --avail
+
+ Further help:
+ adem config -h/--help Configure ADEM
+ adem sites -h/--help Manipulate the site list
+ adem app -h/--help Application installation
+ adem help This help message
+ eos
+ puts text
+ text
+end
Modified: SwiftApps/adem-osg/lib/adem.rb
===================================================================
--- SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:57:22 UTC (rev 3327)
+++ SwiftApps/adem-osg/lib/adem.rb 2010-05-03 19:57:25 UTC (rev 3328)
@@ -5,6 +5,7 @@
require 'adem/config'
require 'adem/sites'
require 'adem/app'
+require 'adem/help'
CONFIGURATION_FILE = "#{ENV['HOME']}/.adem/config"
SITES_FILE = "#{ENV['HOME']}/.adem/sites"
@@ -13,6 +14,7 @@
def run_command(args, config_file, sites_file)
command = args.shift
output = nil
+ return help if command == "help"
conf = config args, config_file
return conf if command == "config"
begin
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:27 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:27 -0500 (CDT)
Subject: [Swift-commit] r3329 - SwiftApps/adem-osg
Message-ID: <20100503195727.C5DDA9CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:27 -0500 (Mon, 03 May 2010)
New Revision: 3329
Modified:
SwiftApps/adem-osg/README
Log:
Updated README
Modified: SwiftApps/adem-osg/README
===================================================================
--- SwiftApps/adem-osg/README 2010-05-03 19:57:25 UTC (rev 3328)
+++ SwiftApps/adem-osg/README 2010-05-03 19:57:27 UTC (rev 3329)
@@ -1,30 +1,25 @@
-adem-osg
+= ADEM: Application software DEployment and Management
-Version 1.0
+Author:: Zhengxiong Hou, Allan Espinosa
+Maintainer:: Allan Espinosa (aespinosa AT cs DOT uchicago DOT edu)
-This ADEM tool is for Application software automatic DEployment and Management based on an OSG software binary or source code repository. You can also build and maintain your own application software repository. There are also examples of Application Execution by Swift.
+== Description
-1. scripts
-==========
-setup.sh source this script to set-up the user env for Bash
-setup.csh source this script to set-up the user env for Csh
+This ADEM tool is for Application software automatic DEployment and Management
+based on an OSG software binary or source code repository. You can also build
+and maintain your own application software repository. There are also examples
+of Application Execution by Swift.
-2. directories
-==============
-bin/
- There are some scripts commands for automatically getting available grid sites, automatically deploy given application software to the set of grid sites, automatically check the results; automatically remove or update a given application software; and get the sites.xml, tc.data files for Swift.
+== Usage
+ adem command [options]
-doc/
- The documents about how to use this tool, and pacman-3.21.tar.gz for possible usage
+ Examples:
+ adem config --display
+ adem sites --update
+ adem app --avail
-log/
- The records for the grid sites, applications, deployment and management results, and all of the commands.
-
-swift-execution-example/
- Some examples for application execution by Swift
-
-test-sites
- Some grid sites for test
-
-tmp/
- Temperory directory for the output files
+ Further help:
+ adem config -h/--help Configure ADEM
+ adem sites -h/--help Manipulate the site list
+ adem app -h/--help Application installation
+ adem help This help message
From noreply at svn.ci.uchicago.edu Mon May 3 14:57:30 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 3 May 2010 14:57:30 -0500 (CDT)
Subject: [Swift-commit] r3330 - SwiftApps/adem-osg
Message-ID: <20100503195730.368719CC92@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-03 14:57:30 -0500 (Mon, 03 May 2010)
New Revision: 3330
Added:
SwiftApps/adem-osg/ChangeLog
Removed:
SwiftApps/adem-osg/CHANGES.txt
Log:
renamed ChangeLog and summarized the reimplementation
Deleted: SwiftApps/adem-osg/CHANGES.txt
===================================================================
Added: SwiftApps/adem-osg/ChangeLog
===================================================================
--- SwiftApps/adem-osg/ChangeLog (rev 0)
+++ SwiftApps/adem-osg/ChangeLog 2010-05-03 19:57:30 UTC (rev 3330)
@@ -0,0 +1,7 @@
+
+2010-05-03
+==========
+
+by Allan Espinosa:
+ * Reimplemented functionality into a single script interface
+ * Modularized features
From noreply at svn.ci.uchicago.edu Thu May 13 14:51:28 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 13 May 2010 14:51:28 -0500 (CDT)
Subject: [Swift-commit] r3332 - text/internals/trunk
Message-ID: <20100513195128.485489CC86@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-13 14:51:27 -0500 (Thu, 13 May 2010)
New Revision: 3332
Modified:
text/internals/trunk/internals.tex
Log:
Raw notes last last week
Modified: text/internals/trunk/internals.tex
===================================================================
--- text/internals/trunk/internals.tex 2010-05-05 16:06:08 UTC (rev 3331)
+++ text/internals/trunk/internals.tex 2010-05-13 19:51:27 UTC (rev 3332)
@@ -182,5 +182,183 @@
)
\end{verbatim}
+\subsection{Elements}
+Example: recursive list
+
+\begin{verbatim}
+element(f, [n]
+ if(n == 0, 0,
+ n,
+ f(n-1) )
+)
+l := list(f(10))
+\end{verbatim}
+
+equivalent to
+\begin{verbatim}
+l := list(10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+\end{verbatim}
+
+Expansion :
+
+f(10) -> 10, f(9) -> 10, 9, f(8) -> ...
+
+At some point there will be 10 stackframes active.
+
+Positional arguments are always required, Optional arguments cannot be specified
+positionally.
+
+\section{Unreturned arguments}
+... = vargs. should always be last in an element definition. Stuck in the
+default channel
+
+element(f, [...]
+ each(...)
+)
+
+f(1,2,3) -> 1,2,3
+
+element(f, [...]
+ for(i, ...
+ i
+ )
+)
+
+Optional arguments cannot be expressed in positional form
+
+\section{Named channels}
+
+Multiple varargs
+
+channel(a) - an implicit channel
+
+to(a, 1,2,3)
+
+example
+
+element(fc, [b, channel(a)]
+ for(i, a,
+ echo(b, '-', i)
+ )
+)
+
+Used in generating a graphviz DAG
+
+parallelElement(con, [channel(c)]
+ for(i, c
+ echo(i)
+ )
+)
+
+element(gen, []
+ for(i, range(0,10),
+ wait(delay=1000), i
+ )
+)
+
+con(gen())
+
+Print as a channel:
+print("a") -> to(stdout, "a")
+
+You can intercept a channel. Other purely functional languages like Haskell
+messes up this side effect.
+
+element(x, [channel(stdout)]
+ a := first(stdout)
+ echo(reverse(a))
+)
+
+x(print("abc"))
+
+dumps "cba"
+
+\sectio{Swift compilation}
+
+swift source -> parser -> vdlx -> dump parser / compiled /translated -> .kml
+
+down tree. static type checking, limited data flow analysis
+
+.kml uses some libraries vdl-int.k, vdl.k, vdl-sc.k, vdl-lib.xml - java defined
+elements
+
+vdl-sc.k -> site catalog parsing
+
+execute-*.k
+
+vdl.k, vdl-lib.k -> publicly accessible by the kml file. encapsulation issues
+
+vdl-lib.xml -> mapping functions (extractint, etc, @java),
+ -> data manipulation
+ -> two namespaces vdl and swiftscript
+
+"swiftscript" : actual mapping functions @name
+ @name = filename(foo) equivalent to C's '&'
+
+originally @no data flow with mapping functions. limited processing on a string
+. supposed to assist in the mapping functions
+
+vdl : - dataset functions
+ - directory manipulation (basename, dirname). used internally
+
+cdm: - cdm functions
+
+\section{first.swift}
+
+In normal karajan (not compiled xml)
+
+element(greeting, [t]
+ vdl:execute(
+ vdl:tr("echo")
+ vdl:stageout(t)
+ vdl:arguments("Hello world")
+ vdl:stdout(swifscript:filename(t))
+ )
+ vdl:closedataset(t)
+)
+
+With input parameter
+
+type file;
+app (file m) greeting(string msg) {
+ echo msg stdout=@filename(t);
+}
+
+file outf<"a.txt">;
+outf = greeting("Hello world");
+
+compiled
+
+element(greeting, [t, msg]
+ vdl:execute(
+ vdl:tr("echo")
+ vdl:stageout(t)
+ vdl:arguments(vdl:getfieldvalue(msg))
+ vdl:stdout(swifscript:filename(t))
+ )
+ vdl:closedataset(t)
+)
+
+\section{DSHandle}
+
+Most primitive representation of a swift piece of data. A pointer. The hash
+table is filled with DSHandle values. Pointer descriptor of a swift object
+
+A tree
+
+Scalar = just a struct
+
+Implements -> scalar, array, structs, etc.
+
+More or relation to nodes than the names of the nodes in itself
+
+All data variables in swift (dshandles) are futures.
+
+
+string s = "x"; ->
+
+s=vdl:new(...)
+vdl:setfieldvalue(s, "x")
+
\end{document}
From noreply at svn.ci.uchicago.edu Fri May 21 15:15:34 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 21 May 2010 15:15:34 -0500 (CDT)
Subject: [Swift-commit] r3333 - text/internals/trunk
Message-ID: <20100521201534.9935A9CC97@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2010-05-21 15:15:34 -0500 (Fri, 21 May 2010)
New Revision: 3333
Modified:
text/internals/trunk/internals.tex
Log:
Last week's notes
Modified: text/internals/trunk/internals.tex
===================================================================
--- text/internals/trunk/internals.tex 2010-05-13 19:51:27 UTC (rev 3332)
+++ text/internals/trunk/internals.tex 2010-05-21 20:15:34 UTC (rev 3333)
@@ -361,4 +361,66 @@
s=vdl:new(...)
vdl:setfieldvalue(s, "x")
+\section{vdl:execute}
+
+vdl:tr - looks at the tc.data for program translation (not really). somehow
+passes this deep down. a wrapper that gets delegated down the line.
+
+vdl.k line 49 - decides which runmode to do.
+
+swift (runmode) - dryrun, normal, etc.
+
+different implementations delegated by vdl:execute execute-*.k files
+
+\section{Lazy errors}
+
+Poisoning vdl:closedataset
+
+\section{vdl:execute2}
+
+1. staging setup/ retrieving
+2. initialize job directories
+2.5 allocate a sie via allocateHost()
+3. initialize shared dirs (if not already initialized)
+4. create job directories. job input file directories -> a.out a/x.txt b/y.txt
+ jobdir:
+ a/x.txt
+ b/b.txt
+
+5. stage-in (parallel via uParallelFor)
+
+vdl:execute - used to be a plain karajan task:execute . different from the
+outer vdl:execute
+
+gt2 option = status.mode = provider | files
+
+provider staging = not implemented
+
+a. status: files/ providers
+b. parameters: files/ providers
+
+6. actual execute
+7. check status
+7.5 transfer stdout/stderr/info/kickstart
+8. stageout
+
+\section{uParallelFor vs parallel vs foreach.max.threads}
+
+\subsection{parallel}
+
+parallel(1,2) == (1,2) -> has some efficient barrier thing inside
+ - takes more memory - storing ordering information
+
+\subsetion{uparallel}
+uParallel(1,2) -> (1,2) or (2,1)
+ - saves on memory usage
+
+\section{Replication groups}
+
+\section{Restarting}
+
+\section{Scheduler}
+
+\section{Coasters}
+
\end{document}
From noreply at svn.ci.uchicago.edu Mon May 24 16:28:39 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 24 May 2010 16:28:39 -0500 (CDT)
Subject: [Swift-commit] r3334 - provenancedb
Message-ID: <20100524212839.27A649CC95@vm-125-59.ci.uchicago.edu>
Author: lgadelha
Date: 2010-05-24 16:28:38 -0500 (Mon, 24 May 2010)
New Revision: 3334
Modified:
provenancedb/prov-init.sql
Log:
Creation of annotation table.
Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql 2010-05-21 20:15:34 UTC (rev 3333)
+++ provenancedb/prov-init.sql 2010-05-24 21:28:38 UTC (rev 3334)
@@ -16,7 +16,8 @@
DROP TABLE extrainfo;
DROP TABLE createarray;
DROP TABLE createarray_member;
-DROP TABLE arrayrange;
+DROP TABLE array_range;
+DROP TABLE annotations;
-- associates each process with its containing workflow
@@ -24,14 +25,14 @@
-- in which case this looks very much like a compound/app
-- containment?
CREATE TABLE processes_in_workflows
- (workflow_id char(128),
- process_id char(128)
+ (workflow_id char(256),
+ process_id char(256)
);
-- processes gives information about each process (in the OPM sense)
-- it is augmented by information in other tables
CREATE TABLE processes
- (id char(128) PRIMARY KEY, -- a uri
+ (id char(256) PRIMARY KEY, -- a uri
type char(16) -- specifies the type of process. for any type, it
-- must be the case that the specific type table
-- has an entry for this process.
@@ -43,12 +44,12 @@
-- each execute is identified by a unique URI. other information from
-- swift logs is also stored here. an execute is an OPM process.
CREATE TABLE executes
- (id char(128) PRIMARY KEY, -- actually foreign key to processes
+ (id char(256) PRIMARY KEY, -- actually foreign key to processes
starttime numeric,
duration numeric,
- finalstate char(128),
- app char(128),
- scratch char(128)
+ finalstate char(256),
+ app char(256),
+ scratch char(256)
);
-- this gives information about each execute2, which is an attempt to
@@ -56,12 +57,12 @@
-- information such as wrapper logs
CREATE TABLE execute2s
- (id char(128) PRIMARY KEY,
- execute_id, -- secondary key to executes and processes tables
+ (id char(256) PRIMARY KEY,
+ execute_id char(256), -- secondary key to executes and processes tables
starttime numeric,
duration numeric,
- finalstate char(128),
- site char(128)
+ finalstate char(256),
+ site char(256)
);
-- dataset_usage records usage relationships between processes and datasets;
@@ -73,11 +74,11 @@
-- dataset_id for common queries? maybe add arbitrary ID for sake of it?
CREATE TABLE dataset_usage
- (process_id char(128), -- foreign key but not enforced because maybe process
+ (process_id char(256), -- foreign key but not enforced because maybe process
-- doesn't exist at time. same type as processes.id
direction char(1), -- I or O for input or output
- dataset_id char(128), -- this will perhaps key against dataset table
- param_name char(128) -- the name of the parameter in this execute that
+ dataset_id char(256), -- this will perhaps key against dataset table
+ param_name char(256) -- the name of the parameter in this execute that
-- this dataset was bound to. sometimes this must
-- be contrived (for example, in positional varargs)
);
@@ -91,8 +92,8 @@
-- TODO primary key should be execute_id
CREATE TABLE invocation_procedure_names
- (execute_id char(128),
- procedure_name char(128)
+ (execute_id char(256),
+ procedure_name char(256)
);
@@ -106,8 +107,8 @@
-- a containment hierarchy. The relationship (such as array index or
-- structure member name) should also be stored in this table.
CREATE TABLE dataset_containment
- ( outer_dataset_id char(128),
- inner_dataset_id char(128)
+ ( outer_dataset_id char(256),
+ inner_dataset_id char(256)
);
@@ -117,8 +118,8 @@
-- TODO dataset_id should be primary key
CREATE TABLE dataset_filenames
- ( dataset_id char(128),
- filename char(128)
+ ( dataset_id char(256),
+ filename char(256)
);
-- dataset_values stores the value for each dataset which is known to have
@@ -127,8 +128,8 @@
-- example) SQL numerical operations should not be expected to work, even
-- though the user knows that a particular dataset stores a numeric value.
CREATE TABLE dataset_values
- ( dataset_id char(128), -- should be primary key
- value char(128)
+ ( dataset_id char(256), -- should be primary key
+ value char(256)
);
-- The above dataset_* tables are the original containment representation
@@ -138,21 +139,21 @@
-- It is unclear which is the better representation.
CREATE TABLE createarray
- ( array_id char(128)
+ ( array_id char(256)
);
CREATE TABLE createarray_member
- ( array_id char(128),
- ix char(128),
- member_id char(128)
+ ( array_id char(256),
+ ix char(256),
+ member_id char(256)
);
-- TODO step
CREATE TABLE array_range
- ( array_id char(128),
- from_id char(128),
- to_id char(128),
- step_id char(128) -- nullable, if step is unspecified
+ ( array_id char(256),
+ from_id char(256),
+ to_id char(256),
+ step_id char(256) -- nullable, if step is unspecified
);
-- known_workflows stores some information about each workflow log that has
@@ -160,27 +161,33 @@
-- status.
CREATE TABLE known_workflows
(
- workflow_id char(128),
- workflow_log_filename char(128),
- version char(128),
- importstatus char(128)
+ workflow_id char(256),
+ workflow_log_filename char(256),
+ version char(256),
+ importstatus char(256)
);
-- workflow_events stores the start time and duration for each workflow
-- that has been successfully imported.
CREATE TABLE workflow_events
- ( workflow_id char(128),
+ ( workflow_id char(256),
starttime numeric,
duration numeric
);
-- extrainfo stores lines generated by the SWIFT_EXTRA_INFO feature
CREATE TABLE extrainfo
- ( execute2id char(128),
+ ( execute2id char(256),
extrainfo char(1024)
);
+-- annotations
+CREATE TABLE annotations
+ ( id char(256), -- either dataset_id, process_id, or workflow_id
+ name char(256),
+ value char(256)
+ );
-- this GRANT does not work for sqlite; you'll get a syntax error but
-- ignore it, as it is not needed in sqlite
From noreply at svn.ci.uchicago.edu Thu May 27 16:33:34 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 27 May 2010 16:33:34 -0500 (CDT)
Subject: [Swift-commit] r3335 - in provenancedb: . apps apps/oops
Message-ID: <20100527213334.B869F9CCBC@vm-125-59.ci.uchicago.edu>
Author: lgadelha
Date: 2010-05-27 16:33:34 -0500 (Thu, 27 May 2010)
New Revision: 3335
Added:
provenancedb/apps/
provenancedb/apps/oops/
provenancedb/apps/oops/oops_extractor.sh
Modified:
provenancedb/swift-prov-import-all-logs
Log:
Started scientific parameter extractor for the OOPS application
Added: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh (rev 0)
+++ provenancedb/apps/oops/oops_extractor.sh 2010-05-27 21:33:34 UTC (rev 3335)
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Annotation extractor for the OOPS application
+# Author: Luiz Gadelha
+# Date: 2010-05-25
+#
+# The directory $PROTESTS/swift-logs contains symbolic links to
+# OOPS' Swift logs.
+
+PROVDB_HOME=~/provenancedb
+PROTESTS_HOME=~/protests
+workflow_id=$1
+
+source $PROVDB_HOME/etc/provenance.config
+
+
+# provdb_imported records runs already imported to the provenance database
+cd $PROTESTS_HOME
+if [ ! -a provdb_imported ]; then
+ touch provdb_imported
+fi
+
+
+for i in `ls | grep run.loops`;
+do
+ if ! grep $i provdb_imported; then
+ if grep "Swift finished with no errors" $i/psim.loops-*.log; then
+ cd swift-logs
+ for j in `ls ../$i | grep psim.loops-`; do
+ ln -s ../$i/$j
+ done
+ cd import
+ # swift-prov-import-all-logs also controls what already has been
+ # imported, so it does not repeat work
+ $PROVDB_HOME/swift-prov-import-all-logs
+ cd $PROTESTS_HOME
+ echo $i >> provdb_imported
+
+ # annotate workflows with their oops runid
+ oops_run_id=`echo $i | awk -F . '{print $3}'`
+ log_filename=`ls $i | grep psim.loops- | grep "\."log$`
+ workflow_id=`echo "select workflow_id from known_workflows where workflow_log_filename like '%$log_filename%'" | $SQLCMD -t | awk '{print $1}'`
+ echo "insert into annotations values ('$workflow_id','oops_run_id','$oops_run_id');" | $SQLCMD
+
+ # annotate dataset with scientific parameters passed to doLoopRound
+ #echo "\pset border 0;" > query.sql
+ #echo "select dataset_filenames.dataset_id,dataset_filenames.filename from dataset_usage,invocation_procedure_names,dataset_containment,dataset_filenames where dataset_usage.process_id=invocation_procedure_names.execute_id and dataset_containment.inner_dataset_id=dataset_filenames.dataset_id and procedure_name='doLoopRound' and param_name='modelData' and dataset_containment.outer_dataset_id=dataset_usage.dataset_id and dataset_filenames.filename like '%.params%';" >> query.sql
+ #$SQLCMD -t -A -F " " -f query.sql -o result.txt
+ #dataset_id=`awk '{print $1}' result.txt`
+ #filename=`awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g'`
+ #TODO extract name-value pairs
+ fi
+ fi
+done
+
+
+
Property changes on: provenancedb/apps/oops/oops_extractor.sh
___________________________________________________________________
Name: svn:executable
+ *
Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs 2010-05-24 21:28:38 UTC (rev 3334)
+++ provenancedb/swift-prov-import-all-logs 2010-05-27 21:33:34 UTC (rev 3335)
@@ -35,41 +35,40 @@
if [ $version -ge 1538 ]; then
echo -n "Log: $filename ... "
-# TODO: this only works for psql, not for other DBs, so for now force it
-# to be non-existing
-#EXISTING=$(psql -p 5435 -d $PROVDB -U benc --tuples-only -c "select count(*) from known_workflows where workflow_log_filename='$filename';")
-EXISTING=0
+# TODO: make it work in general (sqlite, mysql, ...)
+# Works only with PostgreSQL
+EXISTING=$($SQLCMD --tuples-only -c "select count(*) from known_workflows where workflow_log_filename='$filename';")
if [ "$EXISTING" -eq "0" ]; then
-echo IMPORTING
-
- if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then
- wfstatus="SUCCESS"
- else
- wfstatus="FAIL"
- fi
-
- echo version $version in log file $filename
- echo ============= will import =============
- prepare-for-import $filename
- if [ "$?" != "0" ]; then
- echo prepare-for-import failed
- exit 2
- fi
- import-run-to-sql $filename
- if [ "$?" != "0" ]; then
- echo import-run-to-sql failed
- exit 3
- fi
-
+ echo IMPORTING
+
+ if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then
+ wfstatus="SUCCESS"
+ else
+ wfstatus="FAIL"
+ fi
+
+ echo version $version in log file $filename
+ echo ============= will import =============
+ prepare-for-import $filename
+ if [ "$?" != "0" ]; then
+ echo prepare-for-import failed
+ exit 2
+ fi
+ import-run-to-sql $filename
+ if [ "$?" != "0" ]; then
+ echo import-run-to-sql failed
+ exit 3
+ fi
+
# import-run-to-xml $filename
-
- export RUNID=$(basename $filename .log)
- export WF="tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:${RUNID}:run"
-
- echo "INSERT INTO known_workflows (workflow_id, workflow_log_filename, version, importstatus) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD
+
+ export RUNID=$(basename $filename .log)
+ export WF="tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:${RUNID}:run"
+
+ echo "INSERT INTO known_workflows (workflow_id, workflow_log_filename, version, importstatus) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD
else
-echo SKIP: Already known in workflow
+ echo SKIP: Already known in workflow
fi
fi
done < /tmp/everylog-vs-versions.data
From noreply at svn.ci.uchicago.edu Mon May 31 04:57:16 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 31 May 2010 04:57:16 -0500 (CDT)
Subject: [Swift-commit] r3336 - trunk/docs
Message-ID: <20100531095716.C586D9CC80@vm-125-59.ci.uchicago.edu>
Author: davidk
Date: 2010-05-31 04:57:16 -0500 (Mon, 31 May 2010)
New Revision: 3336
Modified:
trunk/docs/tutorial.xml
Log:
Multiple corrections and updates to tutorial
Modified: trunk/docs/tutorial.xml
===================================================================
--- trunk/docs/tutorial.xml 2010-05-27 21:33:34 UTC (rev 3335)
+++ trunk/docs/tutorial.xml 2010-05-31 09:57:16 UTC (rev 3336)
@@ -37,14 +37,16 @@
There is also a
Swift User's Guide
which contains more detailed reference
-material on topics covered in this manual.
+material on topics covered in this manual.
+
+All of the programs included in this tutorial can be found in your Swift distribution in the examples/swift directory.
Hello World
-The first example program (found in the file
-examples/swift/first.swift)
+The first example program,
+first.swift,
outputs a hello world message into
a file called hello.txt.
@@ -52,10 +54,8 @@
type messagefile;
-(messagefile t) greeting () {
- app {
+app (messagefile t) greeting () {
echo "Hello, world!" stdout=@filename(t);
- }
}
messagefile outfile <"hello.txt">;
@@ -68,12 +68,11 @@
$ cd examples/swift/
$ swift first.swift
-Swift v0.2
+Swift svn swift-r3334 (swift modified locally) cog-r2752
-RunID: e1bupgygrzn12
-echo started
-echo completed
-
+RunID: 20100526-1925-8zjupq1b
+Progress:
+Final status: Finished successfully:1
$ cat hello.txt
Hello, world!
@@ -103,15 +102,13 @@
-(messagefile t) greeting () {
- app {
- echo "Hello, world!" stdout=@filename(t);
- }
+app (messagefile t) greeting() {
+ echo "Hello, world!" stdout=@filename(t);
}
-Next we define a procedure called write. This procedure will write out
+Next we define a procedure called greeting. This procedure will write out
the "hello world" message to a file.
@@ -212,7 +209,6 @@
messagefile english <"english.txt">;
messagefile french <"francais.txt">;
-
english = greeting("hello");
french = greeting("bonjour");
@@ -259,7 +255,7 @@
transformation catalog to define
a logical transformation for the tc utility. The transformation
catalog can be found in etc/tc.data.
-There is already one entry specifying where echo can
+There are already several entries specifying where programs can
be found. Add a new line to the file, specifying where
tr can be found
(usually in /usr/bin/tr
@@ -267,11 +263,11 @@
-local translate /usr/bin/tr INSTALLED INTEL32::LINUX null
+localhost tr /usr/bin/tr INSTALLED INTEL32::LINUX null
For now, ignore all of the fields except the second and the third.
-The second field 'translate' specifies a logical application name and the
+The second field 'tr' specifies a logical application name and the
third specifies the location of the application executable.
@@ -281,13 +277,13 @@
We can define a new procedure, capitalise which calls
-transform.
+tr.
-(messagefile t) capitalise (messagefile f) {
- app {
- translate "[a-z]" "[A-Z]" stdin=@filename(f) stdout=@filename(t);
- }
-}
+(messagefile o) capitalise(messagefile i) {
+ app {
+ tr "[a-z]" "[A-Z]" stdin=@filename(i) stdout=@filename(o);
+ }
+}
@@ -295,7 +291,8 @@
We can call capitalise like this:
-cfile = capitalise(outfile);
+messagefile final <"capitals.txt">;
+final = capitalise(hellofile);
@@ -303,34 +300,34 @@
So a full program based on the first exercise might look like this:
-type messagefile;
-
-(messagefile t) greeting (string s) {
- app {
- echo s stdout=@filename(t);
- }
-}
-
-(messagefile t) capitalise (messagefile f) {
- app {
- translate "[a-z]" "[A-Z]" stdin=@filename(f) stdout=@filename(t);
- }
-}
-
-messagefile outfile <"greeting.txt">;
-messagefile cfile <"capitalised.txt">;
-
-outfile = greeting("hello from Swift");
-cfile = capitalise(outfile);
+type messagefile {}
+
+(messagefile t) greeting (string s) {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+(messagefile o) capitalise(messagefile i) {
+ app {
+ tr "[a-z]" "[A-Z]" stdin=@filename(i) stdout=@filename(o);
+ }
+}
+
+messagefile hellofile <"hello.txt">;
+messagefile final <"capitals.txt">;
+
+hellofile = greeting("hello from Swift");
+final = capitalise(hellofile);
We can use the swift command to run this:
-$ swift t.swift
+$ swift second_procedure.swift
[...]
-$ cat capitalised.txt
+$ cat capitals.txt
HELLO FROM SWIFT
@@ -383,7 +380,7 @@
type details {
string name;
- string place;
+ int pies;
}
Each element of the structured type can be accessed using a . like this:
@@ -393,36 +390,36 @@
-The following complete program outputs a greeting using a user-defined
+The following complete program, types.swift, outputs a greeting using a user-defined
structure type to hold parameters for the message:
-type messagefile;
+type messagefile {}
type details {
string name;
- string place;
+ int pies;
}
-(messagefile t) greeting (details d) {
+(messagefile t) greeting (details d) {
app {
- echo "Hello" d.name "You live in" d.place stdout=@filename(t);
+ echo "Hello. Your name is" d.name "and you have eaten" d.pies "pies." stdout=@filename(t);
}
}
details person;
-person.name = "John";
-person.place = "Namibia";
+person.name = "John";
+person.pies = 3;
-messagefile outfile <"types.txt">;
+messagefile outfile <"q15.txt>";
outfile = greeting(person);
-Structured types can comprised marker types for files. See the later
+Structured types can be comprised of marker types for files. See the later
section on mappers for more information about this.
@@ -433,7 +430,7 @@
messagefile m[];
-This will declare an array of message files.
+This program, q5.swift, will declare an array of message files.
@@ -527,9 +524,9 @@
is based on the name of the input file: our input file is mapped
to the inputfile variable using the simple named mapper, and then
we use the regular expression mapper to map the output file. Then we
-use the countwords() procedure that we defined in exercise ANOTHERPROCEDURE
-to count the works in the input file and store the result in the
-output file.
+use the countwords() procedure to count the works in the input file and
+store the result in the output file. In order for the countwords() procedure
+to work correctly, add the wc utility (usually found in /usr/bin/wc) to tc.data.
@@ -682,24 +679,18 @@
-$ swift fold9.swift
-Swift v0.2-dev r1230
+$ swift iterate.swift
+Swift svn swift-r3334 cog-r2752
-RunID: 20070918-1434-l6bt8x4a
-echo started
-echo completed
-wcl started
-extract int value 16.0
-wcl completed
-wcl started
-extract int value 2.0
-wcl completed
-wcl started
-extract int value 1.0
-wcl completed
+RunID: 20100526-2259-gtlz8zf4
+Progress:
+SwiftScript trace: extract int value , 16.0
+SwiftScript trace: extract int value , 2.0
+SwiftScript trace: extract int value , 1.0
+Final status: Finished successfully:4
-$ ls foldout.*
-foldout.0 foldout.1 foldout.2 foldout.3
+$ ls foldout*
+foldout0000 foldout0001 foldout0002 foldout0003
@@ -714,9 +705,9 @@
same time:
$ swift -pgraph graph.dot first.swift
-$ dot -ograph.png -Tpng graph1.dot
+$ dot -ograph.png -Tpng graph.dot
-which can then be viewed using your favourite image viewer.
+graph.png can then be viewed using your favourite image viewer.
@@ -925,7 +916,7 @@
type messagefile;
-(messagefile t) greeting() {.
+(messagefile t) greeting() {
app {
echo "hello" stdout=@filename(t);
}
@@ -1162,8 +1153,8 @@
-localhost touch /usr/bin/touch INSTALLED INTEL32::LINUX null
-localhost broken /bin/true INSTALLED INTEL32::LINUX null
+localhost touch /usr/bin/touch INSTALLED INTEL32::LINUX null
+localhost broken /bin/true INSTALLED INTEL32::LINUX null
@@ -1172,18 +1163,11 @@
$ swift restart.swift
+Swift 0.9 swift-r2860 cog-r2388
-Swift v0.1-dev
-
-RunID: php8y7ydim8x0
-touch started
-echo started
-broken started
-touch completed
-broken completed
-echo completed
-echo started
-echo completed
+RunID: 20100526-1119-3kgzzi15
+Progress:
+Final status: Finished successfully:4
@@ -1204,23 +1188,19 @@
$ swift restart.swift
-Swift v0.1-dev
+Swift 0.9 swift-r2860 cog-r2388
-RunID: 6y3urvnm5kch1
-touch started
-broken started
-touch completed
-echo started
-echo completed
-broken failed
-The following errors have occurred:
-1. Application echo not executed due to errors in dependencies
-2. Application "broken" failed (Job failed with an exit code of 1)
- Arguments: "process"
- Host: localhost
- Directory: restart-6y3urvnm5kch1/broken-4empvyci
- STDERR:
- STDOUT:
+RunID: 20100526-1121-tssdcljg
+Progress:
+Progress: Stage in:1 Finished successfully:2
+Execution failed:
+ Exception in broken:
+Arguments: [process]
+Host: localhost
+Directory: restart-20100526-1121-tssdcljg/jobs/1/broken-1i6ufisj
+stderr.txt:
+stdout.txt:
+
From the output we can see that touch and the first echo completed,
@@ -1231,8 +1211,8 @@
-$ ls *6y3urvnm5kch1*rlog
-restart-6y3urvnm5kch1.0.rlog
+$ ls *20100526-1121-tssdcljg*rlog
+restart-20100526-1121-tssdcljg.0.rlog
This restart log contains enough information for swift to know
@@ -1241,21 +1221,25 @@
We can try to rerun it immediately, like this:
-$ swift -resume restart-6y3urvnm5kch1.0.rlog restart.swift
+$ swift -resume restart-20100526-1121-tssdcljg.0.rlog restart.swift
-Swift v0.1-dev
+Swift 0.9 swift-r2860 cog-r2388
-RunID: nyrg0sqeudnu1
-broken started
-broken failed
-The following errors have occurred:
-1. Application echo not executed due to errors in dependencies
-2. Application "broken" failed (Job failed with an exit code of 1)
- Arguments: "process"
- Host: localhost
- Directory: restart-nyrg0sqeudnu1/broken-i2guvyci
- STDERR:
- STDOUT:
+RunID: 20100526-1125-7yx0zi6d
+Progress:
+Execution failed:
+ Exception in broken:
+Arguments: [process]
+Host: localhost
+Directory: restart-20100526-1125-7yx0zi6d/jobs/m/broken-msn1gisj
+stderr.txt:
+stdout.txt:
+
+----
+
+Caused by:
+ Exit code 1
+
@@ -1282,16 +1266,13 @@
-$ swift -resume restart-6y3urvnm5kch1.0.rlog restart.swift
+$ swift -resume restart-20100526-1121-tssdcljg.0.rlog restart.swift
-Swift v0.1-dev
+wift 0.9 swift-r2860 cog-r2388
-RunID: x03l8zci03il1
-broken started
-echo started
-broken completed
-echo completed
-
+RunID: 20100526-1128-a2gfuxhg
+Progress:
+Final status: Initializing:2 Finished successfully:2
Swift tries to run 'broken' again. This time it works, and so
@@ -1316,7 +1297,7 @@
When we invoke the procedure, we can specify values for the parameters
-by name:
+by name. The following code can be found in q21.swift.
french = greeting(s="bonjour");
From noreply at svn.ci.uchicago.edu Mon May 31 05:20:20 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 31 May 2010 05:20:20 -0500 (CDT)
Subject: [Swift-commit] r3337 - trunk/examples
Message-ID: <20100531102020.E3B669CC80@vm-125-59.ci.uchicago.edu>
Author: davidk
Date: 2010-05-31 05:20:20 -0500 (Mon, 31 May 2010)
New Revision: 3337
Modified:
trunk/examples/foreach.swift
trunk/examples/manyparam.swift
trunk/examples/parameter.swift
trunk/examples/regexp.swift
trunk/examples/types.swift
Log:
Corrections and updates to existing tutorial scripts
Modified: trunk/examples/foreach.swift
===================================================================
--- trunk/examples/foreach.swift 2010-05-31 09:57:16 UTC (rev 3336)
+++ trunk/examples/foreach.swift 2010-05-31 10:20:20 UTC (rev 3337)
@@ -13,7 +13,10 @@
foreach f in inputfiles {
- countfile c;
+ countfile c;
c = countwords(f);
}
Modified: trunk/examples/manyparam.swift
===================================================================
--- trunk/examples/manyparam.swift 2010-05-31 09:57:16 UTC (rev 3336)
+++ trunk/examples/manyparam.swift 2010-05-31 10:20:20 UTC (rev 3337)
@@ -1,4 +1,4 @@
-type messagefile {}
+type messagefile;
(messagefile t) greeting (string s) {
app {
Modified: trunk/examples/parameter.swift
===================================================================
--- trunk/examples/parameter.swift 2010-05-31 09:57:16 UTC (rev 3336)
+++ trunk/examples/parameter.swift 2010-05-31 10:20:20 UTC (rev 3337)
@@ -1,4 +1,4 @@
-type messagefile {}
+type messagefile;
(messagefile t) greeting (string s) {
app {
Modified: trunk/examples/regexp.swift
===================================================================
--- trunk/examples/regexp.swift 2010-05-31 09:57:16 UTC (rev 3336)
+++ trunk/examples/regexp.swift 2010-05-31 10:20:20 UTC (rev 3337)
@@ -9,7 +9,10 @@
messagefile inputfile <"q16.txt">;
-countfile c;
+countfile c ;
c = countwords(inputfile);
Modified: trunk/examples/types.swift
===================================================================
--- trunk/examples/types.swift 2010-05-31 09:57:16 UTC (rev 3336)
+++ trunk/examples/types.swift 2010-05-31 10:20:20 UTC (rev 3337)
@@ -7,7 +7,7 @@
(messagefile t) greeting (details d) {
app {
- echo "Hello. Your names is" d.name "and you have eaten" d.pies "pies." stdout=@filename(t);
+ echo "Hello. Your name is" d.name "and you have eaten" d.pies "pies." stdout=@filename(t);
}
}
From noreply at svn.ci.uchicago.edu Mon May 31 05:57:08 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 31 May 2010 05:57:08 -0500 (CDT)
Subject: [Swift-commit] r3338 - trunk/examples
Message-ID: <20100531105708.9B0C49CCBD@vm-125-59.ci.uchicago.edu>
Author: davidk
Date: 2010-05-31 05:57:08 -0500 (Mon, 31 May 2010)
New Revision: 3338
Added:
trunk/examples/if.swift
trunk/examples/iterate.swift
trunk/examples/one.txt
trunk/examples/q21.swift
trunk/examples/q3.swift
trunk/examples/q5.swift
trunk/examples/q6.swift
trunk/examples/q7.swift
trunk/examples/q8.swift
trunk/examples/restart.swift
trunk/examples/second_procedure.swift
trunk/examples/three.txt
trunk/examples/two.txt
trunk/examples/wcl
Removed:
trunk/examples/tutorial/
Log:
Further modifications, updates and changes to tutorial scripts
Added: trunk/examples/if.swift
===================================================================
--- trunk/examples/if.swift (rev 0)
+++ trunk/examples/if.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,18 @@
+type messagefile {}
+
+(messagefile t) greeting (string s) {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+messagefile outfile <"hello20.txt">;
+
+boolean morning = true;
+
+if(morning) {
+ outfile = greeting("good morning");
+} else {
+ outfile = greeting("good afternoon");
+}
+
Added: trunk/examples/iterate.swift
===================================================================
--- trunk/examples/iterate.swift (rev 0)
+++ trunk/examples/iterate.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,22 @@
+type counterfile;
+
+(counterfile t) echo(string m) {
+ app {
+ echo m stdout=@filename(t);
+ }
+}
+
+(counterfile t) countstep(counterfile i) {
+ app {
+ wcl @filename(i) @filename(t);
+ }
+}
+
+counterfile a[] ;
+
+a[0] = echo("793578934574893");
+
+iterate v {
+ a[v+1] = countstep(a[v]);
+ trace("extract int value ", at extractint(a[v+1]));
+} until (@extractint(a[v+1]) <= 1);
Added: trunk/examples/one.txt
===================================================================
--- trunk/examples/one.txt (rev 0)
+++ trunk/examples/one.txt 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1 @@
+this is one.txt
Added: trunk/examples/q21.swift
===================================================================
--- trunk/examples/q21.swift (rev 0)
+++ trunk/examples/q21.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,14 @@
+type messagefile {}
+
+(messagefile t) greeting (string s="hello") {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+messagefile english <"english2.txt">;
+messagefile french <"francais2.txt">;
+
+english = greeting();
+french = greeting(s="bonjour");
+
Added: trunk/examples/q3.swift
===================================================================
--- trunk/examples/q3.swift (rev 0)
+++ trunk/examples/q3.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,10 @@
+type file {}
+
+(file t) echo (string s = "default greeting") {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+file hw = echo();
+
Added: trunk/examples/q5.swift
===================================================================
--- trunk/examples/q5.swift (rev 0)
+++ trunk/examples/q5.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,14 @@
+type messagefile;
+
+(messagefile t) greeting (string s[]) {
+ app {
+ echo s[0] s[1] s[2] stdout=@filename(t);
+ }
+}
+
+messagefile outfile <"q5out.txt">;
+
+string words[] = ["how","are","you"];
+
+outfile = greeting(words);
+
Added: trunk/examples/q6.swift
===================================================================
--- trunk/examples/q6.swift (rev 0)
+++ trunk/examples/q6.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,14 @@
+type file {}
+
+(file t) echo (string s) {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+string greetings[] = ["how","are","you"];
+
+foreach g in greetings {
+ file hw = echo(g);
+}
+
Added: trunk/examples/q7.swift
===================================================================
--- trunk/examples/q7.swift (rev 0)
+++ trunk/examples/q7.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,16 @@
+type file {}
+
+(file t) echo (string s) {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+string outputNames = "one two three";
+
+file outputFiles[] ;
+
+foreach f in outputFiles {
+ f = echo("hello");
+}
+
Added: trunk/examples/q8.swift
===================================================================
--- trunk/examples/q8.swift (rev 0)
+++ trunk/examples/q8.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,14 @@
+
+type file {}
+
+(file t) echo (string s) {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+file inputFiles[] ;
+
+file o <"foo.out">;
+o = echo(@filenames(inputFiles));
+
Added: trunk/examples/restart.swift
===================================================================
--- trunk/examples/restart.swift (rev 0)
+++ trunk/examples/restart.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,32 @@
+type file;
+
+(file f) touch() {
+ app {
+ touch @f;
+ }
+}
+
+(file f) processL(file inp) {
+ app {
+ echo "processL" stdout=@f;
+ }
+}
+
+(file f) processR(file inp) {
+ app {
+ broken "process" stdout=@f;
+ }
+}
+
+(file f) join(file left, file right) {
+ app {
+ echo "join" @left @right stdout=@f;
+ }
+}
+
+file f = touch();
+
+file g = processL(f);
+file h = processR(f);
+
+file i = join(g,h);
Added: trunk/examples/second_procedure.swift
===================================================================
--- trunk/examples/second_procedure.swift (rev 0)
+++ trunk/examples/second_procedure.swift 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,20 @@
+type messagefile {}
+
+(messagefile t) greeting (string s) {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+(messagefile o) capitalise(messagefile i) {
+ app {
+ tr "[a-z]" "[A-Z]" stdin=@filename(i) stdout=@filename(o);
+ }
+}
+
+messagefile hellofile <"hello.txt">;
+messagefile final <"capitals.txt">;
+
+hellofile = greeting("hello from Swift");
+final = capitalise(hellofile);
+
Added: trunk/examples/three.txt
===================================================================
--- trunk/examples/three.txt (rev 0)
+++ trunk/examples/three.txt 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1 @@
+three
Added: trunk/examples/two.txt
===================================================================
--- trunk/examples/two.txt (rev 0)
+++ trunk/examples/two.txt 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1 @@
+a second file
Added: trunk/examples/wcl
===================================================================
--- trunk/examples/wcl (rev 0)
+++ trunk/examples/wcl 2010-05-31 10:57:08 UTC (rev 3338)
@@ -0,0 +1,3 @@
+#!/bin/bash
+echo -n $(wc -c < $1) > $2
+
Property changes on: trunk/examples/wcl
___________________________________________________________________
Name: svn:executable
+ *