[Swift-commit] r4829 - SwiftApps/GOSwift/pykoa/tools
jonmon at ci.uchicago.edu
jonmon at ci.uchicago.edu
Mon Jul 18 09:25:42 CDT 2011
Author: jonmon
Date: 2011-07-18 09:25:41 -0500 (Mon, 18 Jul 2011)
New Revision: 4829
Modified:
SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
Log:
o Code cleanup - moved code into there own definitions and moved definitions around
o Handled a possible mkdir error
Modified: SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
===================================================================
--- SwiftApps/GOSwift/pykoa/tools/koa_goswift.py 2011-07-18 01:51:26 UTC (rev 4828)
+++ SwiftApps/GOSwift/pykoa/tools/koa_goswift.py 2011-07-18 14:25:41 UTC (rev 4829)
@@ -20,6 +20,20 @@
from pykoa.data import creds_dao
from pykoa.koaexception import ToolsException, cli_exception_handler
+class SwiftObserver(ProcessObserver):
+ def __init__(self):
+ self.lines = []
+
+ def stdout_event(self, line):
+ sys.stdout.write(line)
+ self.lines.append(line)
+ return
+
+ def stderr_event(self, line):
+ sys.stderr.write(line)
+ self.lines.append(line)
+ return
+
def setup_opts(argv):
# Set up help message
help_screen = """ \
@@ -149,84 +163,7 @@
return script
-# Do not know what this syntax is but pops up in other pykoa_* files, so I kept it
- at cli_exception_handler
-def main( argv=sys.argv[1:] ):
- (parser, options, args) = setup_opts(argv)
-
- # If I did not specify at least one site, print help message
- if len(args) < 1:
- parser.print_help()
- return 1
-
- # Not sure what these lines do yet, just copied from koa-ly.py
- guc_util.init_guc_env()
-
- conn = pykoa.connect()
- user_row = pykoa.tools.cli_get_user( conn )
- user_id = user_row.id
-
- # get the swift input package
- ( tc, sites, config, script ) = parse_stdin()
-
- # check to see is the swift directory has been created
- home = os.getenv( "HOME" )
- swift_dir = home + "/Swift"
- if not os.access( swift_dir, os.F_OK ):
- os.mkdir( swift_dir )
-
- # default is run.0, if there are other directories then adjust the run directory count
- run_directory = "run.0"
- directories = os.listdir( swift_dir )
-
- # This might cause a performance problem(to even begin to start Swift)
- # Need to figure out how maybe purge these run directories
- # For now I can manually purge
- runs = []
- for direc in directories:
- ( name, sep, run_num ) = direc.partition( "." )
- try:
- runs.append( int( run_num) )
- except ValueError:
- pykoa.debug( "Directory "+direc+" is not a run directory" )
-
- runs.sort()
-
- # if the directories list is not empty, create the name of the run directory
- if runs:
- last_run = runs[ len(runs)-1 ] + 1
- run_directory = "run."+str(last_run)
-
- # get a space seperated String of the sites to execute on
- execution_sites=" ".join( map ( str, args ) )
- sys.stdout.write( "\nExecuting Swift on: " + execution_sites + "\n" )
- sys.stdout.write( "Execution directory: "+swift_dir+"/" + run_directory + "\n" )
-
- # If I do not execute the script, just print out all important information
- # useful to verify that the inputs to the Swift command line have been parsed correctly
- if options.do_not_execute:
- sys.stdout.write( "\nTC file: \n" + tc + "\n" )
- sys.stdout.write( "SITES file: \n" + sites + "\n" )
- sys.stdout.write( "CONFIG file: \n" + config + "\n" )
- sys.stdout.write( "SWIFTSCRIPT file: \n" + script + "\n" )
- sys.stdout.write( "swift -sites.file sites.xml -tc.file tc -config cf script.swift\n\n" )
- return 0
-
- # Set up for the acutal execution
- os.mkdir( swift_dir+"/"+run_directory )
-
- with open( swift_dir+"/"+run_directory+"/tc", "w" ) as tc_file:
- tc_file.write( tc )
-
- with open( swift_dir+"/"+run_directory+"/sites.xml", "w" ) as sites_file:
- sites_file.write( sites )
-
- with open( swift_dir+"/"+run_directory+"/cf", "w" ) as config_file:
- config_file.write( config )
-
- with open( swift_dir+"/"+run_directory+"/script.swift", "w" ) as script_file:
- script_file.write( script )
-
+def get_creds(args, conn, user_id, options):
# A logical endpoint (no dots) can be a scp like path, if it has a :
# This for loop is wrong. If I am understanding this correctly, this should only parsethe last site I pass the script.
# A current issue is that the site must also be accompanied by a path, which is not necessary for Swift execution.
@@ -314,26 +251,10 @@
if not src_cred:
raise ToolsException("A credential is needed. Try -g or -m.")
- ret = execute_swift(user_id, options, src_url, src_cred, src_subject, swift_dir, run_directory, home)
+ return (src_url, src_cred, src_subject)
- return ret
+def execute_swift(user_id, options, src_url, src_cred, src_subject, work_directory):
-class SwiftObserver(ProcessObserver):
- def __init__(self):
- self.lines = []
-
- def stdout_event(self, line):
- sys.stdout.write(line)
- self.lines.append(line)
- return
-
- def stderr_event(self, line):
- sys.stderr.write(line)
- self.lines.append(line)
- return
-
-def execute_swift(user_id, options, src_url, src_cred, src_subject, swift_dir, run_directory, home):
-
if src_subject:
os.environ["GLOBUS_FTP_CLIENT_TEST_SUBJECT"] = src_subject
if src_cred:
@@ -349,7 +270,7 @@
# Execute the script
cmd = os.path.join(os.environ["HOME"], "swift-0.92/bin", "swift")
- os.chdir( swift_dir+"/"+run_directory )
+ os.chdir(work_directory)
execute = cmd + " -sites.file sites.xml -tc.file tc -config cf script.swift"
pykoa.debug( "running %s" % (execute))
@@ -367,13 +288,108 @@
if src_cred:
os.remove( proxyfile )
- with open( swift_dir+"/"+run_directory+"/swift.out", "w" ) as swift_out:
+ with open( work_directory+"/swift.out", "w" ) as swift_out:
for line in c.lines:
swift_out.write( line )
sys.stdout.write( "Swift exit status: " + str(status) + "\n" )
return status
+def create_work_directory():
+ # check to see is the swift directory has been created
+ home = os.getenv( "HOME" )
+ swift_dir = home + "/Swift"
+ if not os.access( swift_dir, os.F_OK ):
+ os.mkdir( swift_dir )
+
+ # default is run.0, if there are other directories then adjust the run directory count
+ run_directory = "run.0"
+ directories = os.listdir( swift_dir )
+
+ # This might cause a performance problem(to even begin to start Swift)
+ # Need to figure out how maybe purge these run directories
+ # For now I can manually purge
+ runs = []
+ for direc in directories:
+ ( name, sep, run_num ) = direc.partition( "." )
+ try:
+ runs.append( int( run_num) )
+ except ValueError:
+ pykoa.debug( "Directory "+direc+" is not a run directory" )
+
+ runs.sort()
+
+ # if the directories list is not empty, create the name of the run directory
+ if runs:
+ last_run = runs[ len(runs)-1 ] + 1
+ run_directory = "run."+str(last_run)
+
+ work_directory = swift_dir+"/"+run_directory
+ return work_directory
+
+# Do not know what this syntax is but pops up in other pykoa_* files, so I kept it
+ at cli_exception_handler
+def main( argv=sys.argv[1:] ):
+ (parser, options, args) = setup_opts(argv)
+
+ # If I did not specify at least one site, print help message
+ if len(args) < 1:
+ parser.print_help()
+ return 1
+
+ # Not sure what these lines do yet, just copied from koa-ly.py
+ guc_util.init_guc_env()
+
+ conn = pykoa.connect()
+ user_row = pykoa.tools.cli_get_user( conn )
+ user_id = user_row.id
+
+ # get the swift input package
+ ( tc, sites, config, script ) = parse_stdin()
+
+ work_directory = create_work_directory()
+
+ # get a space seperated String of the sites to execute on
+ execution_sites=" ".join( map ( str, args ) )
+ sys.stdout.write( "\nExecuting Swift on: " + execution_sites + "\n" )
+ sys.stdout.write( "Execution directory: " + work_directory + "\n" )
+
+ # If I do not execute the script, just print out all important information
+ # useful to verify that the inputs to the Swift command line have been parsed correctly
+ if options.do_not_execute:
+ sys.stdout.write( "\nTC file: \n" + tc + "\n" )
+ sys.stdout.write( "SITES file: \n" + sites + "\n" )
+ sys.stdout.write( "CONFIG file: \n" + config + "\n" )
+ sys.stdout.write( "SWIFTSCRIPT file: \n" + script + "\n" )
+ sys.stdout.write( "swift -sites.file sites.xml -tc.file tc -config cf script.swift\n\n" )
+ return 0
+
+ # Set up for the acutal execution
+ try:
+ os.mkdir( work_directory )
+ pykoa.debug( "Created work_directory: " + work_directory )
+ except os.error:
+ sys.stderr.write( work_directory + " already exists, clean up " + os.getenv("HOME") + "/Swift" )
+ return 1
+
+ with open( work_directory+"/tc", "w" ) as tc_file:
+ tc_file.write( tc )
+
+ with open( work_directory+"/sites.xml", "w" ) as sites_file:
+ sites_file.write( sites )
+
+ with open( work_directory+"/cf", "w" ) as config_file:
+ config_file.write( config )
+
+ with open( work_directory+"/script.swift", "w" ) as script_file:
+ script_file.write( script )
+
+ (src_url, src_cred, src_subject ) = get_creds(args, conn, user_id, options)
+
+ ret = execute_swift(user_id, options, src_url, src_cred, src_subject, work_directory)
+
+ return ret
+
if __name__ == "__main__":
rc = main()
sys.exit(rc)
More information about the Swift-commit
mailing list