[Swift-commit] r4932 - in SwiftApps/GOSwift: bin pykoa/tools
jonmon at ci.uchicago.edu
jonmon at ci.uchicago.edu
Tue Aug 2 16:35:31 CDT 2011
Author: jonmon
Date: 2011-08-02 16:35:30 -0500 (Tue, 02 Aug 2011)
New Revision: 4932
Added:
SwiftApps/GOSwift/bin/koa-goswift-run
SwiftApps/GOSwift/pykoa/tools/koa_goswift_run.py
Removed:
SwiftApps/GOSwift/bin/koa-goswift
SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
Log:
o rename koa_goswift to koa_goswift_run
o cleaned up the getting of the credentials business
o Fixed issues with writing information to the log. Wrong error code was being written to the log
o remove proxy file when execution is finished
Deleted: SwiftApps/GOSwift/bin/koa-goswift
===================================================================
--- SwiftApps/GOSwift/bin/koa-goswift 2011-08-02 16:28:12 UTC (rev 4931)
+++ SwiftApps/GOSwift/bin/koa-goswift 2011-08-02 21:35:30 UTC (rev 4932)
@@ -1,15 +0,0 @@
-#!/bin/sh
-
-if [ -z "$KOA_HOME" ]; then
- echo "Error: KOA_HOME must be set"
- exit 1
-fi
-
-if [ -s "$KOA_USER" ]; then
- echo "Error: KOA_USER must be set"
- exit 1
-fi
-
-export KOA_API_11=TRUE
-
-exec $KOA_HOME/pykoa/tools/koa_goswift.py "$@"
Copied: SwiftApps/GOSwift/bin/koa-goswift-run (from rev 4924, SwiftApps/GOSwift/bin/koa-goswift)
===================================================================
--- SwiftApps/GOSwift/bin/koa-goswift-run (rev 0)
+++ SwiftApps/GOSwift/bin/koa-goswift-run 2011-08-02 21:35:30 UTC (rev 4932)
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+if [ -z "$KOA_HOME" ]; then
+ echo "Error: KOA_HOME must be set"
+ exit 1
+fi
+
+if [ -s "$KOA_USER" ]; then
+ echo "Error: KOA_USER must be set"
+ exit 1
+fi
+
+export KOA_API_11=TRUE
+
+exec $KOA_HOME/pykoa/tools/koa_goswift_run.py "$@"
Deleted: SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
===================================================================
--- SwiftApps/GOSwift/pykoa/tools/koa_goswift.py 2011-08-02 16:28:12 UTC (rev 4931)
+++ SwiftApps/GOSwift/pykoa/tools/koa_goswift.py 2011-08-02 21:35:30 UTC (rev 4932)
@@ -1,478 +0,0 @@
-#!/usr/bin/python
-import os
-import re
-import sys
-import subprocess
-import commands
-import datetime
-import stat
-
-from sqlalchemy.sql import text
-
-import pykoa
-import pykoa.tools
-from pykoa.process_observer import ProcessObserver
-from pykoa.signal_util import BlockInterrupt, enable_sigint
-from pykoa.tools import url_parse, myproxy
-from pykoa.tools import koa_transfer
-from pykoa.tools.koa_ep_activate import check_explicit_activate
-from pykoa.consumers import guc_util
-from pykoa.constants import get_scheme
-from pykoa.data import endpoint_dao
-from pykoa.data import creds_dao
-from pykoa.koaexception import ToolsException, cli_exception_handler
-
-class SwiftObserver(ProcessObserver):
- def __init__(self):
- self.lines = []
-
- def stdout_event(self, line):
- sys.stdout.write(line)
- self.lines.append(line)
- return
-
- def stderr_event(self, line):
- sys.stderr.write(line)
- self.lines.append(line)
- return
-
-def setup_opts(argv):
- # Set up help message
- help_screen = """ \
-goswift [options] <credentials> < \"goswift.package\"
-
-Executes Swift under Globus Online. The <sites> must be logical sites. This will not work for physical sites.
-Add site using endpoint-add and configure site using endpoint-modify.
-
-Type 'man goswift' for details. \
-"""
-
- # parse out options
- parser = pykoa.tools.get_option_parser(help_screen)
- parser.add_option("--dry-run", dest="do_not_execute",
- default=False, action="store_true",
- help="Output the Swift command line that will execute the workflow and the configuration files")
-
- parser.add_option("--watch", dest="watch_swift_stdoutput",
- default=False, action="store_true",
- help="Run Swift as an immediate command. Watch the output that Swift produces")
-
- parser.add_option("--trunk", dest="trunk",
- default=False, action="store_true",
- help="Run the development trunk version of Swift")
-
- parser.add_option("--stable", dest="stable",
- default=True, action="store_true",
- help="Run the most stable release version")
-
- parser.add_option("-d", dest="deadline",
- default="30m",
- help="Set the time limit for this workflow. " + \
- "Default: 30m (30 minutes)")
-
- parser.add_option("-g", dest="gsi_activation",
- default=False, action="store_true",
- help="Use gsi-ssh delegated credential")
-
-
- myproxy.add_myproxy_options(parser)
- (options, args) = pykoa.tools.parse_args(parser, argv)
-
- if options.gsi_activation and not os.getenv("X509_USER_PROXY"):
- raise ToolsException("ENO_GSI_SSH")
-
- return (parser, options, args)
-
-def parse_stdin():
- lines = []
- index = 0
-
- lines = sys.stdin.readlines()
-
- tc = extract_tc( lines )
- sites_file = extract_sites( lines )
- config = extract_config( lines )
- script = extract_script( lines )
-
- return ( tc, sites_file, config, script )
-
-def extract_tc( lines ):
- begin_tc = False
- tc = ""
-
- for line in lines:
-
- line=line.rstrip()
-
- if line == "# begin tc":
- begin_tc = True
-
- elif line == "# end tc":
- begin_tc = False
- break
-
- elif begin_tc:
- tc += line + "\n"
-
- return tc
-
-def extract_sites( lines ):
- begin_sites = False
- sites = ""
-
- for line in lines:
-
- line=line.rstrip()
-
- if line == "# begin sites":
- begin_sites = True
-
- elif line == "# end sites":
- begin_sites = False
- break
-
- elif begin_sites:
- sites += line + "\n"
-
- return sites
-
-def extract_config( lines ):
- begin_config = False
- config = ""
-
- for line in lines:
-
- line=line.rstrip()
-
- if line == "# begin config":
- begin_config = True
-
- elif line == "# end config":
- begin_config = False
- break
-
- elif begin_config:
- config += line + "\n"
-
- return config
-
-def extract_script( lines ):
- begin_script = False
- script = ""
-
- for line in lines:
-
- line=line.rstrip()
-
- if line == "# begin script":
- begin_script = True
-
- elif line == "# end script":
- begin_script = False
- break
-
- elif begin_script:
- script += line + "\n"
-
- return script
-
-def get_creds(args, conn, user_id, options):
- # A logical endpoint (no dots) can be a scp like path, if it has a :
- # This for loop is wrong. If I am understanding this correctly, this should only parsethe last site I pass the script.
- # A current issue is that the site must also be accompanied by a path, which is not necessary for Swift execution.
- # Unless I leave it and use the path given as the work directory on the site in the sites.xml file?
- for arg in args:
- if re.match("^[^.]+:", arg):
- url = url_parse.SCPUrl(arg, allow_empty_path=True)
- url.update_relative_path()
- else:
- url = url_parse.TransferURL(arg)
-
- src_url = None
- src_cred = None
- default_myproxy = None
- ep = None
-
- if not url.scheme:
- # See if it's a logical endpoint
- ep = endpoint_dao.get_logical_by_name(conn, user_id, url.name)
- if not ep:
- ep = endpoint_dao.get_public_by_name(conn, url.name)
-
- if ep:
- default_myproxy = ep.myproxy_server
- # Get first physical ep, if exists
- phys = endpoint_dao.get_physicals_for_logical(conn, ep.id)
- if not phys:
- raise ToolsException("No physical server exists for '%s'" % (
- url.name))
- phys = phys[0]
- scheme = get_scheme(phys.xfer_service_id)
- src_url = "%s://%s:%d%s" % (scheme, phys.hostname, phys.port, url.path)
- src_subject = phys.x509_dn
-
- # See if LTA endpoint is connected
- if phys.is_lta and phys.alive == 0:
- raise ToolsException("The requested Globus Connect endpoint is not currently connected to Globus Online")
-
- # Get creds
- row = creds_dao.get_cred_for_logical(conn, user_id, ep.id)
- if row and row.exp_time > datetime.datetime.utcnow():
- src_cred = row.sec_info
-
- if not src_url:
- # Not a logical endpoint
- if "." not in url.name:
- raise ToolsException("EEXIST_ENDPOINT", url.name)
- if not url.scheme:
- url.scheme = "gsiftp"
- if not url.port:
- url.port = "2811"
- src_url = url.full_name + url.path
- src_subject = ""
- src_cred = None
-
- # Get creds
- if options.gsi_activation:
- if ep:
- check_explicit_activate([ep])
- cred_file = os.environ["X509_USER_PROXY"]
- src_cred = open(cred_file).read()
-
- elif options.myproxy_host or ((not src_cred) and default_myproxy):
- if not options.myproxy_host:
- options.myproxy_host = default_myproxy
- if ep:
- check_explicit_activate([ep], options.myproxy_host)
-
- cred_file = myproxy.contact_myproxy_server(options.myproxy_host,
- options.myproxy_user, options.myproxy_dn,
- lifetime=options.myproxy_lifetime)
-
- if not cred_file:
- return 1
- try:
- src_cred = open(cred_file).read()
- if ep:
- # Save to the database
- ep_cache = {"foo": ep.id}
- koa_transfer.do_activate(conn, cred_file, ep_cache, user_id,
- quiet=True)
- finally:
- os.remove(cred_file)
-
- if not src_cred:
- raise ToolsException("A credential is needed. Try -g or -m.")
-
- return (src_url, src_cred, src_subject)
-
-def execute_swift(user_id, options, src_url, src_cred, src_subject, work_directory, run_id):
-
- if src_subject:
- os.environ["GLOBUS_FTP_CLIENT_TEST_SUBJECT"] = src_subject
- if src_cred:
- proxyfile = guc_util.sec_text_to_proxy_file(user_id, src_cred)
- os.environ["X509_USER_CERT"] = proxyfile
- os.environ["X509_USER_KEY"] = proxyfile
- os.environ["X509_USER_PROXY"] = proxyfile
-
- # Not sure this will work all the time, but this is what the internet said i could do.
- # Need to check for errors, maybe should use the commands module for this part so I can get the status and output at the same time
- cmd = "curl http://169.254.169.254/2009-04-04/meta-data/public-ipv4"
- (status, host ) = commands.getstatusoutput( cmd )
- if status != 0:
- sys.stderr.write( "Could not determine hostname\n" )
- return status
-
- # There has to be an easier way to get the last line of a string
- # Maybe limit the curl output from above
- lines = []
- for line in host.split("\n"):
- lines.append(line)
-
- host = lines[len(lines)-1]
- os.environ["GLOBUS_HOSTNAME"] = host
- os.environ["GLOBUS_TCP_PORT_RANGE"] = "50000,51000"
- os.environ["GLOBUS_SOURCE_PORT_RANGE"] = "50000,51000"
-
- # Execute the script
- cmd = ""
- if options.trunk:
- cmd = os.path.join(os.environ["HOME"], "Swift_lib/trunk", "swift") # trunk release
- elif options.stable:
- cmd = os.path.join(os.environ["HOME"], "Swift_lib/0.92", "swift") # stable 0.92.1 release
-
- os.chdir(work_directory)
- status = -1
- pid = -1
-
- if options.watch_swift_stdoutput:
- try:
- with BlockInterrupt():
- execute = cmd + " -runid "+str(run_id)+" -sites.file sites.xml -tc.file tc -config cf script.swift"
- pykoa.debug( "running %s in watch mode" % (execute))
-
- with open(work_directory+"/run.sh", "w") as run_sh:
- run_sh.write( "#!/bin/bash\n" )
- run_sh.write( "[ -f /opt/osg/setup.sh ] && source /opt/osg/setup.sh\n")
- run_sh.write( execute + "\n" )
- run_sh.write( "\n" )
- run_sh.write( "echo \"\nExit Code: $?\" >> swift.info\n" )
- run_sh.write( "exit $?" )
-
- os.chmod(work_directory+"/run.sh", stat.S_IRUSR | stat.S_IXUSR )
-
- p = subprocess.Popen("exec "+work_directory+"/run.sh",
- close_fds=True, shell=True,
- preexec_fn=enable_sigint,
- cwd=os.getcwd(),
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
-
- pid = p.pid
-
- with open("swift.info", "w") as swift_info:
- swift_info.write("StartTime: "+str(datetime.datetime.utcnow())+"\n")
- swift_info.write("Swift pid: "+str(pid)+"\n")
- swift_info.write("Run id: "+str(run_id)+"\n")
- swift_info.write("\n")
- swift_info.write("Command Line:\n "+execute+"\n")
-
- c = SwiftObserver()
- status = c.run(p)
- finally:
- if src_cred:
- os.remove( proxyfile )
-
- with open("swift.out", "w") as swift_out:
- for line in c.lines:
- swift_out.write(line)
-
- sys.stdout.write( "Swift exit status: " + str(status) + "\n" )
-
- else:
- swift_out = open("swift.out", "w")
- execute = cmd + " -runid "+str(run_id)+" -sites.file sites.xml -tc.file tc -config cf script.swift"
- pykoa.debug( "running %s in background mode" % (execute))
-
- with open(work_directory+"/run.sh", "w") as run_sh:
- run_sh.write( "#!/bin/bash\n" )
- run_sh.write( "[ -f /opt/osg/setup.sh ] && source /opt/osg/setup.sh\n")
- run_sh.write( execute + "\n" )
- run_sh.write( "echo \"\nExit Code: $? >> swift.info\"\n" )
- run_sh.write( "exit $?" )
-
- os.chmod(work_directory+"/run.sh", stat.S_IRUSR | stat.S_IXUSR )
-
- p = subprocess.Popen("exec "+work_directory+"/run.sh",
- close_fds=True, shell=True,
- preexec_fn=enable_sigint,
- cwd=os.getcwd(),
- stderr=subprocess.STDOUT,
- stdout=swift_out)
- pid = p.pid
-
- swift_out.close()
-
- return status
-
-def create_work_directory():
- # check to see is the swift directory has been created
- home = os.getenv( "HOME" )
- swift_dir = home + "/Swift"
- if not os.access( swift_dir, os.F_OK ):
- os.mkdir( swift_dir )
-
- # default is run.0, if there are other directories then adjust the run directory count
- run_directory = "run.0"
- last_run = 0
- directories = os.listdir( swift_dir )
-
- # This might cause a performance problem(to even begin to start Swift)
- # Need to figure out how maybe purge these run directories
- # For now I can manually purge
- runs = []
- for direc in directories:
- ( name, sep, run_num ) = direc.partition( "." )
- try:
- runs.append( int( run_num) )
- except ValueError:
- pykoa.debug( "Directory "+direc+" is not a run directory" )
-
- runs.sort()
-
- # if the directories list is not empty, create the name of the run directory
- if runs:
- last_run = runs[ len(runs)-1 ] + 1
- run_directory = "run."+str(last_run)
-
- work_directory = swift_dir+"/"+run_directory
- return (work_directory, last_run)
-
-# Do not know what this syntax is but pops up in other pykoa_* files, so I kept it
- at cli_exception_handler
-def main( argv=sys.argv[1:] ):
- (parser, options, args) = setup_opts(argv)
-
- # If I did not specify at least one site, print help message
- if len(args) != 1:
- parser.print_help()
- return 1
-
- # Not sure what these lines do yet, just copied from koa-ly.py
- guc_util.init_guc_env()
-
- conn = pykoa.connect()
- user_row = pykoa.tools.cli_get_user( conn )
- user_id = user_row.id
-
- # get the swift input package
- ( tc, sites, config, script ) = parse_stdin()
-
- (work_directory, run_id) = create_work_directory()
-
- # get a space seperated String of the sites to execute on
- execution_sites=" ".join( map ( str, args ) )
- sys.stdout.write( "\nExecuting Swift with " + execution_sites + " credentials\n" )
- sys.stdout.write( "Execution directory: " + work_directory + "\n" )
-
- # If I do not execute the script, just print out all important information
- # useful to verify that the inputs to the Swift command line have been parsed correctly
- if options.do_not_execute:
- sys.stdout.write( "\nTC file: \n" + tc + "\n" )
- sys.stdout.write( "SITES file: \n" + sites + "\n" )
- sys.stdout.write( "CONFIG file: \n" + config + "\n" )
- sys.stdout.write( "SWIFTSCRIPT file: \n" + script + "\n" )
- sys.stdout.write( "swift -sites.file sites.xml -tc.file tc -config cf script.swift\n\n" )
- return 0
-
- # Set up for the acutal execution
- try:
- os.mkdir( work_directory )
- pykoa.debug( "Created work_directory: " + work_directory )
- except os.error:
- sys.stderr.write( work_directory + " already exists, clean up " + os.getenv("HOME") + "/Swift" )
- pykoa.debug( work_directory + " already exists" )
- return 1
-
- with open( work_directory+"/tc", "w" ) as tc_file:
- tc_file.write( tc )
-
- with open( work_directory+"/sites.xml", "w" ) as sites_file:
- sites_file.write( sites )
-
- with open( work_directory+"/cf", "w" ) as config_file:
- config_file.write( config )
-
- with open( work_directory+"/script.swift", "w" ) as script_file:
- script_file.write( script )
-
- (src_url, src_cred, src_subject ) = get_creds(args, conn, user_id, options)
-
- ret = execute_swift(user_id, options, src_url, src_cred, src_subject, work_directory, run_id)
-
- return ret
-
-if __name__ == "__main__":
- rc = main()
- sys.exit(rc)
Copied: SwiftApps/GOSwift/pykoa/tools/koa_goswift_run.py (from rev 4931, SwiftApps/GOSwift/pykoa/tools/koa_goswift.py)
===================================================================
--- SwiftApps/GOSwift/pykoa/tools/koa_goswift_run.py (rev 0)
+++ SwiftApps/GOSwift/pykoa/tools/koa_goswift_run.py 2011-08-02 21:35:30 UTC (rev 4932)
@@ -0,0 +1,432 @@
+#!/usr/bin/python
+import os
+import re
+import sys
+import subprocess
+import commands
+import datetime
+import stat
+
+from sqlalchemy.sql import text
+
+import pykoa
+import pykoa.tools
+from pykoa.process_observer import ProcessObserver
+from pykoa.signal_util import BlockInterrupt, enable_sigint
+from pykoa.tools import url_parse, myproxy
+from pykoa.tools import koa_transfer
+from pykoa.tools.koa_ep_activate import check_explicit_activate
+from pykoa.consumers import guc_util
+from pykoa.constants import get_scheme
+from pykoa.data import endpoint_dao
+from pykoa.data import creds_dao
+from pykoa.koaexception import ToolsException, cli_exception_handler
+
+class SwiftObserver(ProcessObserver):
+ def __init__(self):
+ self.lines = []
+
+ def stdout_event(self, line):
+ sys.stdout.write(line)
+ self.lines.append(line)
+ return
+
+ def stderr_event(self, line):
+ sys.stderr.write(line)
+ self.lines.append(line)
+ return
+
+def setup_opts(argv):
+ # Set up help message
+ help_screen = """ \
+goswift [options] <credentials> < \"goswift.package\"
+
+Executes Swift under Globus Online. The <sites> must be logical sites. This will not work for physical sites.
+Add site using endpoint-add and configure site using endpoint-modify.
+
+Type 'man goswift' for details. \
+"""
+
+ # parse out options
+ parser = pykoa.tools.get_option_parser(help_screen)
+ parser.add_option("--dry-run", dest="do_not_execute",
+ default=False, action="store_true",
+ help="Output the Swift command line that will execute the workflow and the configuration files")
+
+ parser.add_option("--now", dest="watch_swift_stdoutput",
+ default=False, action="store_true",
+ help="Run Swift as an immediate command. Watch the output that Swift produces")
+
+ parser.add_option("--trunk", dest="trunk",
+ default=False, action="store_true",
+ help="Run the development trunk version of Swift")
+
+ parser.add_option("--stable", dest="stable",
+ default=True, action="store_true",
+ help="Run the most stable release version")
+
+ parser.add_option("-d", dest="deadline",
+ default="30m",
+ help="Set the time limit for this workflow. " + \
+ "Default: 30m (30 minutes)")
+
+ parser.add_option("-g", dest="gsi_activation",
+ default=False, action="store_true",
+ help="Use gsi-ssh delegated credential")
+
+
+ myproxy.add_myproxy_options(parser)
+ (options, args) = pykoa.tools.parse_args(parser, argv)
+
+ if options.gsi_activation and not os.getenv("X509_USER_PROXY"):
+ raise ToolsException("ENO_GSI_SSH")
+
+ return (parser, options, args)
+
+def parse_stdin():
+ lines = []
+ index = 0
+
+ lines = sys.stdin.readlines()
+
+ tc = extract_tc( lines )
+ sites_file = extract_sites( lines )
+ config = extract_config( lines )
+ script = extract_script( lines )
+
+ return ( tc, sites_file, config, script )
+
+def extract_tc( lines ):
+ begin_tc = False
+ tc = ""
+
+ for line in lines:
+
+ line=line.rstrip()
+
+ if line == "# begin tc":
+ begin_tc = True
+
+ elif line == "# end tc":
+ begin_tc = False
+ break
+
+ elif begin_tc:
+ tc += line + "\n"
+
+ return tc
+
+def extract_sites( lines ):
+ begin_sites = False
+ sites = ""
+
+ for line in lines:
+
+ line=line.rstrip()
+
+ if line == "# begin sites":
+ begin_sites = True
+
+ elif line == "# end sites":
+ begin_sites = False
+ break
+
+ elif begin_sites:
+ sites += line + "\n"
+
+ return sites
+
+def extract_config( lines ):
+ begin_config = False
+ config = ""
+
+ for line in lines:
+
+ line=line.rstrip()
+
+ if line == "# begin config":
+ begin_config = True
+
+ elif line == "# end config":
+ begin_config = False
+ break
+
+ elif begin_config:
+ config += line + "\n"
+
+ return config
+
+def extract_script( lines ):
+ begin_script = False
+ script = ""
+
+ for line in lines:
+
+ line=line.rstrip()
+
+ if line == "# begin script":
+ begin_script = True
+
+ elif line == "# end script":
+ begin_script = False
+ break
+
+ elif begin_script:
+ script += line + "\n"
+
+ return script
+
+def get_creds(args, conn, user_id, options):
+ # get activate credentials for a logical endpoint
+ # endpoint must be logical
+ # if endpoint is not activated(or credential is expired), raise ToolsException
+ src_cred = None
+
+ ep = endpoint_dao.get_logical_by_name( conn, user_id, args[0] )
+ if not ep:
+ ep = endpoint_dao.get_public_by_name( conn, args[0] )
+
+ if ep:
+ default_myproxy = ep.myproxy_server
+ # Get first physical ep, if exists
+ phys = endpoint_dao.get_physicals_for_logical(conn, ep.id)
+ if not phys:
+ raise ToolsException("No physical server exists for '%s'" % (
+ args[0]))
+ phys = phys[0]
+ scheme = get_scheme(phys.xfer_service_id)
+ src_subject = phys.x509_dn
+
+ # See if LTA endpoint is connected
+ if phys.is_lta and phys.alive == 0:
+ raise ToolsException("The requested Globus Connect endpoint is not currently connected to Globus Online")
+
+ # Get creds
+ row = creds_dao.get_cred_for_logical(conn, user_id, ep.id)
+ if row and row.exp_time > datetime.datetime.utcnow():
+ src_cred = row.sec_info
+
+ if not src_cred:
+ raise ToolsException("A credential is needed. Activate the endpoint with endpoint-activate")
+
+ return ( src_cred, src_subject)
+
+def execute_swift(user_id, options, src_cred, src_subject, work_directory, run_id):
+
+ if src_subject:
+ os.environ["GLOBUS_FTP_CLIENT_TEST_SUBJECT"] = src_subject
+ if src_cred:
+ proxyfile = guc_util.sec_text_to_proxy_file(user_id, src_cred)
+ os.environ["X509_USER_CERT"] = proxyfile
+ os.environ["X509_USER_KEY"] = proxyfile
+ os.environ["X509_USER_PROXY"] = proxyfile
+
+ # Not sure this will work all the time, but this is what the internet said i could do.
+ cmd = "curl http://169.254.169.254/2009-04-04/meta-data/public-ipv4"
+ (status, host ) = commands.getstatusoutput( cmd )
+ if status != 0:
+ sys.stderr.write( "Could not determine hostname\n" )
+ return status
+
+ # There has to be an easier way to get the last line of a string
+ # Maybe limit the curl output from above?
+ lines = []
+ for line in host.split("\n"):
+ lines.append(line)
+
+ host = lines[len(lines)-1]
+ os.environ["GLOBUS_HOSTNAME"] = host
+ os.environ["GLOBUS_TCP_PORT_RANGE"] = "50000,51000"
+ os.environ["GLOBUS_SOURCE_PORT_RANGE"] = "50000,51000"
+
+ # Execute the script
+ cmd = ""
+ if options.trunk:
+ cmd = os.path.join(os.environ["HOME"], "Swift_lib/trunk", "swift") # trunk release
+ elif options.stable:
+ cmd = os.path.join(os.environ["HOME"], "Swift_lib/0.92", "swift") # stable 0.92.1 release
+
+ os.chdir(work_directory)
+ status = -1
+ pid = -1
+
+ if options.watch_swift_stdoutput:
+ try:
+ with BlockInterrupt():
+ execute = cmd + " -runid "+str(run_id)+" -sites.file sites.xml -tc.file tc -config cf script.swift"
+ pykoa.debug( "running %s in watch mode" % (execute))
+
+ with open(work_directory+"/run.sh", "w") as run_sh:
+ run_sh.write( "#!/bin/bash\n" )
+ run_sh.write( "[ -f /opt/osg/setup.sh ] && source /opt/osg/setup.sh\n")
+ run_sh.write( execute + "\n" )
+ run_sh.write( "ret=$?\n" )
+ run_sh.write( "\n" )
+ run_sh.write( "echo \"\nExit Code: $ret\" >> swift.info\n" )
+ run_sh.write( "exit $ret" )
+
+ os.chmod(work_directory+"/run.sh", stat.S_IRUSR | stat.S_IXUSR )
+
+ p = subprocess.Popen("exec "+work_directory+"/run.sh",
+ close_fds=True, shell=True,
+ preexec_fn=enable_sigint,
+ cwd=os.getcwd(),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
+ pid = p.pid
+
+ with open("swift.info", "w") as swift_info:
+ swift_info.write("StartTime: "+str(datetime.datetime.utcnow())+"\n")
+ swift_info.write("Swift pid: "+str(pid)+"\n")
+ swift_info.write("Run id: "+str(run_id)+"\n")
+ swift_info.write("\n")
+ swift_info.write("Command Line:\n "+execute+"\n")
+
+ c = SwiftObserver()
+ status = c.run(p)
+ finally:
+ if src_cred:
+ os.remove( proxyfile )
+
+ with open("swift.out", "a") as swift_out:
+ for line in c.lines:
+ swift_out.write(line)
+
+ sys.stdout.write( "Swift exit status: " + str(status) + "\n" )
+
+ else:
+ swift_out = open("swift.out", "w")
+ execute = cmd + " -runid "+str(run_id)+" -sites.file sites.xml -tc.file tc -config cf script.swift"
+ pykoa.debug( "running %s in background mode" % (execute))
+
+ with open(work_directory+"/run.sh", "w") as run_sh:
+ run_sh.write( "#!/bin/bash\n" )
+ run_sh.write( "[ -f /opt/osg/setup.sh ] && source /opt/osg/setup.sh\n")
+ run_sh.write( execute + "\n" )
+ run_sh.write( "ret=$?\n" )
+ run_sh.write( "echo \"\nExit Code: $ret\" >> swift.info\n" )
+ run_sh.write( "rm -f $X509_USER_PROXY\n" )
+ run_sh.write( "exit $ret" )
+
+ os.chmod(work_directory+"/run.sh", stat.S_IRUSR | stat.S_IXUSR )
+
+ p = subprocess.Popen("exec "+work_directory+"/run.sh",
+ close_fds=True, shell=True,
+ preexec_fn=enable_sigint,
+ cwd=os.getcwd(),
+ stderr=subprocess.STDOUT,
+ stdout=swift_out)
+ pid = p.pid
+
+ with open("swift.info", "w") as swift_info:
+ swift_info.write("StartTime: "+str(datetime.datetime.utcnow())+"\n")
+ swift_info.write("Swift pid: "+str(pid)+"\n")
+ swift_info.write("Run id: "+str(run_id)+"\n")
+ swift_info.write("\n")
+ swift_info.write("Command Line:\n "+execute+"\n")
+
+ status = 0
+
+ return status
+
+def create_work_directory():
+ # check to see is the swift directory has been created
+ home = os.getenv( "HOME" )
+ swift_dir = home + "/Swift"
+ if not os.access( swift_dir, os.F_OK ):
+ os.mkdir( swift_dir )
+
+ # default is run.0, if there are other directories then adjust the run directory count
+ run_directory = "run.0"
+ last_run = 0
+ directories = os.listdir( swift_dir )
+
+ # This might cause a performance problem(to even begin to start Swift)
+ # Need to figure out how maybe purge these run directories
+ # For now I can manually purge
+ runs = []
+ for direc in directories:
+ ( name, sep, run_num ) = direc.partition( "." )
+ try:
+ runs.append( int( run_num) )
+ except ValueError:
+ pykoa.debug( "Directory "+direc+" is not a run directory" )
+
+ runs.sort()
+
+ # if the directories list is not empty, create the name of the run directory
+ if runs:
+ last_run = runs[ len(runs)-1 ] + 1
+ run_directory = "run."+str(last_run)
+
+ work_directory = swift_dir+"/"+run_directory
+ return (work_directory, last_run)
+
+# Do not know what this syntax is but pops up in other pykoa_* files, so I kept it
+ at cli_exception_handler
+def main( argv=sys.argv[1:] ):
+ (parser, options, args) = setup_opts(argv)
+
+ # If I did not specify at least one site, print help message
+ if len(args) != 1:
+ parser.print_help()
+ return 1
+
+ # Not sure what these lines do yet, just copied from koa-ly.py
+ guc_util.init_guc_env()
+
+ conn = pykoa.connect()
+ user_row = pykoa.tools.cli_get_user( conn )
+ user_id = user_row.id
+
+ # get the swift input package
+ ( tc, sites, config, script ) = parse_stdin()
+
+ (work_directory, run_id) = create_work_directory()
+
+ # get a space seperated String of the sites to execute on
+ execution_sites=" ".join( map ( str, args ) )
+ sys.stdout.write( "\nExecuting Swift with " + execution_sites + " credentials\n" )
+ sys.stdout.write( "Execution directory: " + work_directory + "\n" )
+
+ # If I do not execute the script, just print out all important information
+ # useful to verify that the inputs to the Swift command line have been parsed correctly
+ if options.do_not_execute:
+ sys.stdout.write( "\nTC file: \n" + tc + "\n" )
+ sys.stdout.write( "SITES file: \n" + sites + "\n" )
+ sys.stdout.write( "CONFIG file: \n" + config + "\n" )
+ sys.stdout.write( "SWIFTSCRIPT file: \n" + script + "\n" )
+ sys.stdout.write( "swift -sites.file sites.xml -tc.file tc -config cf script.swift\n\n" )
+ return 0
+
+ # Set up for the acutal execution
+ try:
+ os.mkdir( work_directory )
+ pykoa.debug( "Created work_directory: " + work_directory )
+ except os.error:
+ sys.stderr.write( work_directory + " already exists, clean up " + os.getenv("HOME") + "/Swift" )
+ pykoa.debug( work_directory + " already exists" )
+ return 1
+
+ with open( work_directory+"/tc", "w" ) as tc_file:
+ tc_file.write( tc )
+
+ with open( work_directory+"/sites.xml", "w" ) as sites_file:
+ sites_file.write( sites )
+
+ with open( work_directory+"/cf", "w" ) as config_file:
+ config_file.write( config )
+
+ with open( work_directory+"/script.swift", "w" ) as script_file:
+ script_file.write( script )
+
+ ( src_cred, src_subject ) = get_creds(args, conn, user_id, options)
+
+ ret = execute_swift(user_id, options, src_cred, src_subject, work_directory, run_id)
+
+ return ret
+
+if __name__ == "__main__":
+ rc = main()
+ sys.exit(rc)
More information about the Swift-commit
mailing list