[Swift-commit] r4839 - in SwiftApps/GOSwift: bin pykoa/tools
jonmon at ci.uchicago.edu
jonmon at ci.uchicago.edu
Mon Jul 25 13:55:16 CDT 2011
Author: jonmon
Date: 2011-07-25 13:55:16 -0500 (Mon, 25 Jul 2011)
New Revision: 4839
Added:
SwiftApps/GOSwift/bin/koa-goswift-stat
SwiftApps/GOSwift/pykoa/tools/koa_goswift_stat.py
Modified:
SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
Log:
o modified koa_goswift to run Swift in the background as well as give it an option to run in the foreground
o added another goswift function to get the status about a certain run
Added: SwiftApps/GOSwift/bin/koa-goswift-stat
===================================================================
--- SwiftApps/GOSwift/bin/koa-goswift-stat (rev 0)
+++ SwiftApps/GOSwift/bin/koa-goswift-stat 2011-07-25 18:55:16 UTC (rev 4839)
@@ -0,0 +1,15 @@
+#!/bin/sh
+
+if [ -z "$KOA_HOME" ]; then
+ echo "Error: KOA_HOME must be set"
+ exit 1
+fi
+
+if [ -s "$KOA_USER" ]; then
+ echo "Error: KOA_USER must be set"
+ exit 1
+fi
+
+export KOA_API_11=TRUE
+
+exec $KOA_HOME/pykoa/tools/koa_goswift_stat.py "$@"
Modified: SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
===================================================================
--- SwiftApps/GOSwift/pykoa/tools/koa_goswift.py 2011-07-25 18:05:08 UTC (rev 4838)
+++ SwiftApps/GOSwift/pykoa/tools/koa_goswift.py 2011-07-25 18:55:16 UTC (rev 4839)
@@ -26,24 +26,22 @@
def stdout_event(self, line):
sys.stdout.write(line)
- self.lines.append(line)
+# self.lines.append(line)
return
def stderr_event(self, line):
sys.stderr.write(line)
- self.lines.append(line)
+# self.lines.append(line)
return
def setup_opts(argv):
# Set up help message
help_screen = """ \
-goswift [options] <sites> < \"goswift.package\"
+goswift [options] <credentials> < \"goswift.package\"
Executes Swift under Globus Online. The <sites> must be logical sites. This will not work for physical sites.
Add site using endpoint-add and configure site using endpoint-modify.
-Note: localhost runs on the VM.\
-
Type 'man goswift' for details. \
"""
@@ -51,8 +49,12 @@
parser = pykoa.tools.get_option_parser(help_screen)
parser.add_option("--dry-run", dest="do_not_execute",
default=False, action="store_true",
- help="Output the Swift command line that will execute the workflow")
+ help="Output the Swift command line that will execute the workflow and the configuration files")
+ parser.add_option("--watch", dest="watch_swift_stdoutput",
+ default=False, action="store_true",
+ help="Run Swift as an immediate command. Watch the output that Swift produces")
+
parser.add_option("-d", dest="deadline",
default="30m",
help="Set the time limit for this workflow. " + \
@@ -62,6 +64,7 @@
default=False, action="store_true",
help="Use gsi-ssh delegated credential")
+
myproxy.add_myproxy_options(parser)
(options, args) = pykoa.tools.parse_args(parser, argv)
@@ -253,7 +256,7 @@
return (src_url, src_cred, src_subject)
-def execute_swift(user_id, options, src_url, src_cred, src_subject, work_directory):
+def execute_swift(user_id, options, src_url, src_cred, src_subject, work_directory, run_id):
if src_subject:
os.environ["GLOBUS_FTP_CLIENT_TEST_SUBJECT"] = src_subject
@@ -269,30 +272,37 @@
os.environ["GLOBUS_SOURCE_PORT_RANGE"] = "50000,51000"
# Execute the script
- cmd = os.path.join(os.environ["HOME"], "swift-0.92/bin", "swift")
+ cmd = os.path.join(os.environ["HOME"], "Swift_lib/swift-0.92/bin", "swift")
os.chdir(work_directory)
- execute = cmd + " -sites.file sites.xml -tc.file tc -config cf script.swift"
+ execute = cmd + " -runid "+str(run_id)+" -sites.file sites.xml -tc.file tc -config cf script.swift 2>&1 | tee -a swift.out"
+ status = 0
- pykoa.debug( "running %s" % (execute))
- try:
- with BlockInterrupt():
- p = subprocess.Popen(execute, close_fds=True,
- shell=True, preexec_fn=enable_sigint,
- cwd=os.getcwd(),
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+ if options.watch_swift_stdoutput:
+ pykoa.debug( "running %s in watch mode" % (execute))
+ try:
+ with BlockInterrupt():
+ p = subprocess.Popen(execute, close_fds=True,
+ shell=True, preexec_fn=enable_sigint,
+ cwd=os.getcwd(),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
- c = SwiftObserver()
- status = c.run(p)
- finally:
- if src_cred:
- os.remove( proxyfile )
+ c = SwiftObserver()
+ status = c.run(p)
+ finally:
+ if src_cred:
+ os.remove( proxyfile )
- with open( work_directory+"/swift.out", "w" ) as swift_out:
- for line in c.lines:
- swift_out.write( line )
-
- sys.stdout.write( "Swift exit status: " + str(status) + "\n" )
+ sys.stdout.write( "Swift exit status: " + str(status) + "\n" )
+
+ else:
+ pykoa.debug( "running %s in background mode" % (execute))
+ p = subprocess.Popen(execute, close_fds=True,
+ shell=True, preexec_fn=enable_sigint,
+ cwd=os.getcwd(),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
return status
def create_work_directory():
@@ -304,6 +314,7 @@
# default is run.0, if there are other directories then adjust the run directory count
run_directory = "run.0"
+ last_run = 0
directories = os.listdir( swift_dir )
# This might cause a performance problem(to even begin to start Swift)
@@ -325,7 +336,7 @@
run_directory = "run."+str(last_run)
work_directory = swift_dir+"/"+run_directory
- return work_directory
+ return (work_directory, last_run)
# Do not know what this syntax is but pops up in other pykoa_* files, so I kept it
@cli_exception_handler
@@ -347,7 +358,7 @@
# get the swift input package
( tc, sites, config, script ) = parse_stdin()
- work_directory = create_work_directory()
+ (work_directory, run_id) = create_work_directory()
# get a space seperated String of the sites to execute on
execution_sites=" ".join( map ( str, args ) )
@@ -386,7 +397,7 @@
(src_url, src_cred, src_subject ) = get_creds(args, conn, user_id, options)
- ret = execute_swift(user_id, options, src_url, src_cred, src_subject, work_directory)
+ ret = execute_swift(user_id, options, src_url, src_cred, src_subject, work_directory, run_id)
return ret
Added: SwiftApps/GOSwift/pykoa/tools/koa_goswift_stat.py
===================================================================
--- SwiftApps/GOSwift/pykoa/tools/koa_goswift_stat.py (rev 0)
+++ SwiftApps/GOSwift/pykoa/tools/koa_goswift_stat.py 2011-07-25 18:55:16 UTC (rev 4839)
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+import commands
+
+import os
+import re
+import sys
+import subprocess
+import datetime
+
+from sqlalchemy.sql import text
+
+import pykoa
+import pykoa.tools
+from pykoa.process_observer import ProcessObserver
+from pykoa.signal_util import BlockInterrupt, enable_sigint
+from pykoa.tools import url_parse, myproxy
+from pykoa.tools import koa_transfer
+from pykoa.tools.koa_ep_activate import check_explicit_activate
+from pykoa.consumers import guc_util
+from pykoa.constants import get_scheme
+from pykoa.data import endpoint_dao
+from pykoa.data import creds_dao
+from pykoa.koaexception import ToolsException, cli_exception_handler
+
+def setup_opts(argv):
+ help_screen = """
+goswift_stat [options] <run-number>
+
+Checks the status of a run started in the background using goswift by reading the last n lines of the stdout/stderr file. Only 1 run is allowed to get the status of at a time.
+
+Default number of lines is 10. This can be controlled by the -n option.
+
+Type 'man goswift_stat' for details. \
+"""
+
+ parser = pykoa.tools.get_option_parser(help_screen)
+ parser.add_option("--debug", dest="debugging",
+ default=False, action="store_true",
+ help="Get status from the last n lines of the Swift log file. See ")
+
+ parser.add_option("-n", dest="lines",
+ default=10, action="store_true",
+ help="Controls the number of lines to print from the swift log file or the stdout/stderr file.")
+
+ (options, args) = pykoa.tools.parse_args(parser, argv)
+
+ return (parser, options, args)
+
+ at cli_exception_handler
+def main( argv=sys.argv[1:] ):
+ (parser, options, args) = setup_opts(argv)
+
+ if len(args) != 1:
+ parser.print_help()
+ return 1
+
+ status = 0
+ home = os.environ["HOME"]
+ swift_dir = home +"/Swift"
+ work_dir = ""
+
+ try:
+ work_dir = "run."+str(int(args[0]))
+ except ValueError:
+ sys.stderr.write( "Please specify a valid run id: "+args[0]+"\n" )
+ return 1
+
+ run_dir = swift_dir + "/" + work_dir
+ if not os.path.isdir( run_dir ):
+ sys.stderr.write( run_dir+" does not exist!\n" )
+ return 1
+
+ if options.debugging:
+ if not os.path.isfile( run_dir+"/script-"+args[0]+".log"):
+ sys.stderr.write( run_dir+"/script-"+args[0]+".log does not exitst!\n" )
+ return 1
+
+ try:
+ int(options.lines)
+ except ValueError:
+ sys.stderr.write( "The option passed to -n is not a number: "+option.lines )
+ return 1
+
+ (status, output) = commands.getstatusoutput( "tail -"+str(options.lines)+" "+run_dir+"/script-"+args[0]+".log" )
+ sys.stdout.write(output+"\n")
+ else:
+ if not os.path.isfile( run_dir+"/swift.out"):
+ sys.stderr.write( run_dir+"/swift.out does not exitst!\n" )
+ return 1
+
+ try:
+ int(options.lines)
+ except ValueError:
+ sys.stderr.write( "The option passed to -n is not a number: "+option.lines )
+ return 1
+
+ (status, output) = commands.getstatusoutput( "tail -"+str(options.lines)+" "+run_dir+"/swift.out" )
+ sys.stdout.write(output+"\n")
+
+ return status
+
+if __name__ == "__main__":
+ rc = main()
+ sys.exit(rc)
More information about the Swift-commit
mailing list