[Swift-commit] r4711 - in SwiftApps/GOSwift: . pykoa pykoa/tools sbin
jonmon at ci.uchicago.edu
jonmon at ci.uchicago.edu
Wed Jun 29 13:20:06 CDT 2011
Author: jonmon
Date: 2011-06-29 13:20:06 -0500 (Wed, 29 Jun 2011)
New Revision: 4711
Added:
SwiftApps/GOSwift/pykoa/
SwiftApps/GOSwift/pykoa/tools/
SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
SwiftApps/GOSwift/sbin/
SwiftApps/GOSwift/sbin/remote-cli.py
Log:
o backing up files for goswift
Added: SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
===================================================================
--- SwiftApps/GOSwift/pykoa/tools/koa_goswift.py (rev 0)
+++ SwiftApps/GOSwift/pykoa/tools/koa_goswift.py 2011-06-29 18:20:06 UTC (rev 4711)
@@ -0,0 +1,43 @@
+#!/usr/bin/python
+import os
+import sys
+import uuid
+import datetime
+import readline
+import time
+import signal
+from optparse import OptionGroup
+
+from sqlalchemy.sql import text
+
+import pykoa
+import pykoa.tools
+from pykoa.constants import GroupStatusCodes as grpc
+from pykoa.consumers import guc_util
+from pykoa.koaexception import ToolsException, cli_exception_handler
+from pykoa.tools.url_parse import SCPUrl
+from pykoa.tools.koa_ep_activate import check_explicit_activate
+from pykoa.tools import koa_cancel
+from pykoa.tools import koa_transfer
+from pykoa.tools import myproxy
+from pykoa.tools import koa_wait
+from pykoa.data import groups_dao
+from pykoa.data import endpoint_dao
+from pykoa.data import creds_dao
+from pykoa.data import preflight_dao
+
+
+ at cli_exception_handler
+def main( argv=sys.argv[1:] ):
+ guc_util.init_guc_env()
+
+ conn = pykoa.connect()
+ user_row = pykoa.tools.cli_get_user( conn )
+ user_id = user_row.id
+
+ sys.stdout.write( "Executing Swift on: " + argv[0] + "\n" )
+ return 0
+
+if __name__ == "__main__":
+ rc = main()
+ sys.exit(rc)
Property changes on: SwiftApps/GOSwift/pykoa/tools/koa_goswift.py
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/GOSwift/sbin/remote-cli.py
===================================================================
--- SwiftApps/GOSwift/sbin/remote-cli.py (rev 0)
+++ SwiftApps/GOSwift/sbin/remote-cli.py 2011-06-29 18:20:06 UTC (rev 4711)
@@ -0,0 +1,542 @@
+#!/usr/bin/python -u
+import readline
+import shlex
+import os
+import sys
+import subprocess
+import signal
+import setproctitle
+
+"""
+[gsi]ssh commands all run this program. We get the original command from
+SSH_ORIGINAL_COMMAND
+XXX: We are losing quotes, or at least token separators using
+SSH_ORIGINAL_COMMAND. We should patch sshd to url encode the arguments so
+spaces are included.
+
+Note: Do not import any pykoa modules. Those will stay cached and
+could cause problems if you change them on a running instance.
+"""
+
+
+# XXX: Use /usr/local/koa symlink
+g_base_dir = '/home/koa/live_version'
+
+shell_aliases = {}
+shell_aliases['exit'] = "exit"
+shell_aliases['quit'] = "exit"
+shell_aliases['?'] = "help"
+shell_aliases['h'] = "help"
+
+HELP_BANNER = """\
+Type '<command> -h' for basic help on a command.
+Type 'man <command>' for detailed help.
+
+Task Management: cancel modify
+ details status
+ events wait
+
+Task Creation: scp transfer
+
+Endpoint Management: endpoint-add endpoint-modify
+ endpoint-activate endpoint-remove
+ endpoint-deactivate endpoint-rename
+ endpoint-list diagnose (beta)
+
+Other: help profile (deprecated)
+ history reset
+ ls versions
+ man
+"""
+
+
+prog_map = {}
+
+# Builtins and other scripts
+prog_map['help'] = ""
+prog_map['versions'] = ""
+prog_map['history'] = ""
+prog_map['reset'] = "%s/bin/koa-reset" % g_base_dir
+
+# Note: This block generated by ../pykoa/tools/script_map.py
+prog_map['cancel'] = 'pykoa.tools.koa_cancel'
+prog_map['details'] = 'pykoa.tools.koa_details'
+prog_map['diagnose'] = 'pykoa.tools.koa_diagnose'
+prog_map['endpoint-activate'] = 'pykoa.tools.koa_ep_activate'
+prog_map['endpoint-add'] = 'pykoa.tools.koa_ep_add'
+prog_map['endpoint-deactivate'] = 'pykoa.tools.koa_ep_deactivate'
+prog_map['endpoint-list'] = 'pykoa.tools.koa_ep_list'
+prog_map['endpoint-modify'] = 'pykoa.tools.koa_ep_modify'
+prog_map['endpoint-remove'] = 'pykoa.tools.koa_ep_remove'
+prog_map['endpoint-rename'] = 'pykoa.tools.koa_ep_rename'
+prog_map['events'] = 'pykoa.tools.koa_events'
+prog_map['goswift'] = 'pykoa.tools.koa_goswift'
+prog_map['ls'] = 'pykoa.tools.koa_ls'
+prog_map['man'] = 'pykoa.tools.koa_man'
+prog_map['modify'] = 'pykoa.tools.koa_modify'
+prog_map['profile'] = 'pykoa.tools.koa_profile'
+prog_map['report'] = 'pykoa.tools.koa_report'
+prog_map['scp'] = 'pykoa.tools.koa_scp'
+prog_map['status'] = 'pykoa.tools.koa_status'
+prog_map['transfer'] = 'pykoa.tools.koa_transfer'
+prog_map['wait'] = 'pykoa.tools.koa_wait'
+
+g_version_set = False
+g_manpages = None
+g_sig_save = None
+g_preloaded = False
+g_user = ""
+
+HELP_COMMANDS = [x for x in prog_map.keys() if x != "report"]
+
+def get_man_pages():
+ # Lazy loading
+ global g_manpages
+ if g_manpages:
+ return g_manpages
+ g_manpages = os.listdir(g_base_dir + "/man")
+ g_manpages = [x.replace(".1", "") for x in g_manpages if x.endswith(".1")]
+ return g_manpages
+
+
+def preload_modules():
+ # Note: For extra safety, we do not preload pykoa modules.
+ # We don't want to change these on production and have to
+ # boot people off the CLI for changes to be applied.
+ global g_preloaded
+ if g_preloaded:
+ return
+ import uuid, re, ctypes, logging
+ import gettext, locale, optparse
+ import sqlalchemy.sql
+ import sqlalchemy.exc
+ import sqlalchemy.dialects.postgresql
+ import psycopg2
+ g_preloaded = True
+
+
+def setup_child(short_name, args):
+ # Use separate logs rather than the remote cli log.
+ #import pykoa
+ #pykoa.config.configure_logging(short_name)
+
+ # Set proper title for ps instead of 'remote-cli.py'
+ if args:
+ title = "[%s %s]" % (short_name, " ".join(args))
+ else:
+ title = "[%s]" % short_name
+ setproctitle.setproctitle(title)
+
+
+def completer(text, state):
+ cmds = HELP_COMMANDS[:]
+ # Auto complete man command
+ if text.startswith("man"):
+ cmds += ["man " + x for x in get_man_pages()]
+ if text.startswith("help"):
+ cmds += ["help " + x for x in HELP_COMMANDS]
+ options = [x for x in cmds if x.startswith(text)]
+ try:
+ return options[state]
+ except IndexError:
+ return None
+
+
+# Note: only called for shell mode
+def line_translate(line):
+ line = line.strip()
+ cmd = shell_aliases.get(line)
+ if not cmd:
+ cmd = line
+ return cmd
+
+
+def do_version(args):
+ if "-v" in args:
+ # verbose
+ print "SVN Version:", \
+ os.popen("svnversion " + g_base_dir).readline().strip()
+ return 0
+
+ if args:
+ print """\
+Usage: versions [-v]
+
+Display versions of services supported by the server.
+
+Type 'man versions' for details. \
+"""
+ if "-h" in args or "--help" in args:
+ return 0
+ else:
+ return 2
+
+ print "transfer/1.2"
+ print "transfer/1.1"
+ print "transfer/1.0"
+ return 0
+
+
+def shell():
+ """
+ Read commands from user and execute them
+ """
+
+ # Read history
+ readline.set_history_length(1000)
+ load_history()
+
+ readline.set_completer(completer)
+ readline.parse_and_bind("tab: complete")
+ # For some reason, - is in the default delims.
+ # Reset it to default bash delimiters.
+ readline.set_completer_delims("\t\n\"\\'`@$><=;|&{(")
+
+ print "Welcome to globusonline.org, %s. Type 'help' for help." % (
+ g_user)
+ while True:
+ try:
+ line = raw_input("$ ")
+ # readline automatically adds lines to the history buffer
+ line = line_translate(line)
+ if line == "exit":
+ break
+ args = do_split(line)
+ if args is not None:
+ handle_cmd(args)
+ except KeyboardInterrupt:
+ sys.stdout.write("\n")
+ except EOFError:
+ break
+
+ save_history()
+
+
+def connect_to_db():
+ import psycopg2
+ from ConfigParser import SafeConfigParser
+
+ s = SafeConfigParser()
+ s.read("/etc/koa/koa.ini")
+ host = s.get("db", "dbhost")
+ conn = psycopg2.connect(database='koa', host=host, user='cliuser',
+ password='cliuser')
+ conn.set_isolation_level(0) # Autocommit
+ return conn
+
+
+def load_history():
+ """
+ Load history from db.
+ If a user row doesn't exist, add a row so the save function can just assume
+ an UPDATE will work. (Postgres doesn't support MERGE yet)
+ """
+ conn = connect_to_db()
+ cur = conn.cursor()
+ cur.execute("select history from user_history where name = %s",
+ (g_user,))
+ ret = cur.fetchone()
+ if ret:
+ # Could have a null history, if they logged in and died
+ if ret[0]:
+ for cmd in ret[0].rstrip().split('\n'):
+ readline.add_history(cmd)
+ else:
+ cur.execute("insert into user_history (name) values (%s)",
+ (g_user,))
+ cur.close()
+ conn.close()
+
+
+def save_history():
+ """
+ Save history to database.
+ We just make a text blob, lines with '\n' endings
+ """
+ n = readline.get_current_history_length()
+ buf = ""
+ for i in range(0, n):
+ cmd = readline.get_history_item(i + 1)
+ if cmd.strip():
+ buf += cmd + "\n"
+
+ conn = connect_to_db()
+ cur = conn.cursor()
+ cur.execute("update user_history set history = %s where name = %s",
+ (buf, g_user))
+ cur.close()
+ conn.close()
+
+
+def restore_signals():
+ # We don't want to inherit an ignored SIGINT mask, at least
+ # for xfer and scp commands that the user may want to cancel.
+ signal.signal(signal.SIGINT, g_sig_save)
+
+
+def do_variable_set(v):
+ key, value = v.split("=", 1)
+ if key == "v":
+ return do_set_version(value)
+ else:
+ sys.stderr.write("Unknown variable '%s'\n" % key)
+ return False
+
+
+def do_set_version(value):
+ global g_version_set
+ if ":" not in value:
+ sys.stderr.write("Please specify your user agent string when " +
+ "requesting a version.\n")
+ sys.stderr.write("Example: v=1.0:MyProgram/3.1 <command>\n")
+ return False
+ try:
+ version, user_agent = value.split(":")
+ except:
+ sys.stderr.write("Invalid user agent string\n")
+ return False
+
+ if len(user_agent) < 6:
+ sys.stderr.write("User agent string too short\n")
+ return False
+
+ service = 'transfer'
+ if '/' in version:
+ try:
+ service, version = version.split("/")
+ except:
+ sys.stderr.write("Invalid version string\n")
+ return False
+
+ if service != 'transfer':
+ sys.stderr.write("Unknown service '%s'\n" % service)
+ return False
+
+ if version == "1.0":
+ g_version_set = True
+ return True
+
+ if version in ("1.1", "1.2"):
+ # The existence of this variable is checked, value can be anything
+ os.environ["KOA_API_11"] = "TRUE"
+ g_version_set = True
+ return True
+
+ sys.stderr.write("This service does not support '%s/%s'\n" % (
+ service, version))
+ return False
+
+
+def do_help(args):
+ if args:
+ print """\
+Usage: help [command]
+
+With no arguments, display available commands. Help 'command' is equivalent
+to calling 'command -h'. \
+"""
+ if "-h" in args or "--help" in args:
+ return 0
+ else:
+ return 2
+ print HELP_BANNER
+ return 0
+
+
+def do_split(s):
+ """
+ Split @s into parts, using 'quotes' and \ escaping.
+ On error, display error to stderr and return None.
+ Save original command (with quotes) to KOA_COMMAND.
+ """
+ try:
+ os.environ['KOA_COMMAND'] = s
+ ret = shlex.split(s)
+ return ret
+ except ValueError, e:
+ sys.stderr.write("Error: %s\n" % e)
+ return None
+
+
+def do_history(args):
+ end = readline.get_current_history_length()
+ start = 0
+
+ usage = """\
+Usage: history [N]
+
+Display commands used in current and previous sessions.
+If N is given, display only the most recent N commands.\
+"""
+
+ if args:
+ if "-h" in args or "--help" in args:
+ print usage
+ return 0
+ elif not args[0].startswith("-"):
+ try:
+ limit = int(args[0])
+ start = end - limit
+ except ValueError:
+ sys.stderr.write("Invalid number '%s'\n" % args[0])
+ print usage
+ return 2
+ else:
+ print usage
+ return 2
+
+ for i in range(start, end):
+ cmd = readline.get_history_item(i + 1)
+ print " %-6d %s" % (i + 1, cmd)
+
+
+def log_timing(cmd):
+ if False:
+ import time
+ print "%.03f %s" % (time.time(), cmd)
+
+
+def _run_command(args):
+ """
+ Fork and exec/import module
+ """
+ cmd = args[0]
+ args = args[1:]
+ is_module = False
+ if cmd.startswith("pykoa.tools"):
+ # Make sure interactive and noninteractive have the
+ # same code paths
+ preload_modules()
+ is_module = True
+
+ pid = os.fork()
+ if pid == 0:
+ restore_signals()
+ if not is_module:
+ # Exec
+ args = [os.path.basename(cmd)] + args
+ os.execv(cmd, args)
+ else:
+ short_name = cmd.split(".")[-1]
+ log_timing('setup-child')
+ setup_child(short_name, args)
+ log_timing('import')
+ #old_modules = sys.modules.keys()[:]
+ mod = __import__(cmd, globals(), locals(), ['main'])
+ #for m in sys.modules.keys():
+ #if m not in old_modules:
+ #print "new: %s" % m
+ log_timing('run')
+ rc = mod.main(args)
+ log_timing('done')
+ sys.exit(rc)
+ else:
+ rc = os.waitpid(pid, 0)[1]
+ rc = os.WEXITSTATUS(rc)
+ return rc
+
+
+def run_command(args):
+ """
+ Run command and return rc code
+ """
+ # Disable sigint to ourself while cmd is running
+ # Readline puts in a signal handler. Save it.
+ global g_sig_save
+ g_sig_save = signal.getsignal(signal.SIGINT)
+ signal.signal(signal.SIGINT, signal.SIG_IGN)
+ rc = _run_command(args)
+ signal.signal(signal.SIGINT, g_sig_save)
+ return rc
+
+
+def handle_cmd(args):
+ """
+ Run command and return exit code
+ """
+ if not args:
+ return 1
+
+ if args and args[0].startswith("v="):
+ sys.stderr.write("Error: Version may only be set " + \
+ "when starting a shell\n")
+ return 1
+
+ # help cmd = alias for cmd -h
+ progname = args[0]
+ if progname == "help" and len(args) == 2:
+ progname = args[1]
+ args[1:] = ["-h"]
+
+ if progname == "help":
+ return do_help(args[1:])
+ elif progname == "versions":
+ return do_version(args[1:])
+ elif progname == "history":
+ return do_history(args[1:])
+ else:
+ prog = prog_map.get(progname)
+ if not prog:
+ msg = "Command '%s' not found. Type 'help' for help.\n"
+ sys.stderr.write(msg % progname)
+ return 1
+ fire = [prog] + args[1:]
+ return run_command(fire)
+
+
+def main():
+ global g_user
+
+ # Down for maintenance?
+ FLAG_FILE = "/tmp/koa-down"
+ if os.path.exists(FLAG_FILE):
+ for l in open(FLAG_FILE):
+ sys.stderr.write(l)
+ return 1
+
+ # We are running as a user account via ssh/gsissh. Set our user name.
+ # We could do os.getuid() and pwd.getpwuid(), but ssh sets USER for us
+ # so that's more efficient.
+ g_user = os.environ['KOA_USER'] = os.environ['USER']
+ os.environ['KOA_HOME'] = g_base_dir
+ os.environ['PYTHONPATH'] = g_base_dir
+ sys.path.append(g_base_dir)
+ os.environ['PATH'] = '/usr/local/globus/bin:/usr/local/globus/sbin:' + \
+ os.environ['PATH']
+ os.environ['GLOBUS_LOCATION'] = '/usr/local/globus'
+ os.environ['LD_LIBRARY_PATH'] = '/usr/local/globus/lib'
+
+ cmd = os.environ.get('SSH_ORIGINAL_COMMAND', '')
+ cmd = cmd.strip()
+
+ # Attempt parse
+ args = do_split(cmd)
+ if args is None:
+ return 1
+
+ # Check variable setting
+ if args and "=" in args[0]:
+ if not do_variable_set(args[0]):
+ return 1
+ args = args[1:]
+
+ # Set default version level
+ if not g_version_set:
+ os.environ["KOA_API_11"] = "TRUE"
+
+ if not args:
+ setproctitle.setproctitle("[cli shell]")
+ preload_modules()
+ shell()
+ rc = 0
+ else:
+ setproctitle.setproctitle("[cli run]")
+ rc = handle_cmd(args)
+
+ return rc
+
+
+if __name__ == "__main__":
+ rc = main()
+ sys.exit(rc)
+
Property changes on: SwiftApps/GOSwift/sbin/remote-cli.py
___________________________________________________________________
Added: svn:executable
+ *
More information about the Swift-commit
mailing list