[Swift-commit] cog r4027
swift at ci.uchicago.edu
swift at ci.uchicago.edu
Tue Jul 22 15:45:03 CDT 2014
------------------------------------------------------------------------
r4027 | yadunand | 2014-07-22 15:41:15 -0500 (Tue, 22 Jul 2014) | 17 lines
Adding first cut of the ec2 cloud provider.
Must test from a machine with a publically reachable ip.
Tested using conf :
execution {
type:"coaster"
URL: "127.0.0.1"
jobManager: "local:ec2-cloud"
options {
maxJobs: 5
tasksPerNode: 2
workerLoggingLevel: TRACE
workerLoggingDirectory: /tmp
}
}
------------------------------------------------------------------------
Index: modules/provider-localscheduler/resources/cog-provider.properties
===================================================================
--- modules/provider-localscheduler/resources/cog-provider.properties (revision 4026)
+++ modules/provider-localscheduler/resources/cog-provider.properties (working copy)
@@ -33,3 +33,8 @@
sandbox=false
executionTaskHandler=org.globus.cog.abstraction.impl.scheduler.lsf.execution.TaskHandlerImpl
securityContext=org.globus.cog.abstraction.impl.common.task.SecurityContextImpl
+
+provider=ec2-cloud
+sandbox=false
+executionTaskHandler=org.globus.cog.abstraction.impl.scheduler.shell.execution.TaskHandlerImpl
+securityContext=org.globus.cog.abstraction.impl.common.task.SecurityContextImpl
Index: modules/provider-localscheduler/etc/provider-ec2-cloud.properties
===================================================================
--- modules/provider-localscheduler/etc/provider-ec2-cloud.properties (revision 0)
+++ modules/provider-localscheduler/etc/provider-ec2-cloud.properties (revision 4027)
@@ -0,0 +1,3 @@
+submit.command=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/stsubmit
+cancel.command=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/stcancel
+poll.command=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/ststat
\ No newline at end of file
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/provider-ec2-cloud.properties
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/provider-ec2-cloud.properties (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/provider-ec2-cloud.properties (revision 4027)
@@ -0,0 +1,3 @@
+submit.command=/path/to/stsubmit
+cancel.command=/path/to/stcancel
+poll.command=/path/to/ststat
\ No newline at end of file
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/log
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/log (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/log (revision 4027)
@@ -0,0 +1,136 @@
+Start
+CMD : /usr/bin/perl /usr/local/bin/swift-trunk/bin/worker.pl http://128.135.250.235:45185 0722-3703570-000000 /tmp 1> /tmp/tmp.dXjNyXhGn8 2> /tmp/tmp.gLOB9vWNvL
+AWS_CREDENTIALS_FILE=~/.ssh/boto-test-credentials.csv
+
+# Update the following only if you want to use a specific keypair
+AWS_KEYPAIR_NAME=swift-test-pair
+AWS_KEYPAIR_FILE=~/.ssh/swift-test-pair.pem
+
+AWS_REGION=us-west-2
+SECURITY_GROUP=swift_security_group1
+
+WORKER_IMAGE=ami-23700813
+WORKER_MACHINE_TYPE=t1.micro
+
+CMD_STRING="mkdir /tmp/; cd /tmp/; /usr/bin/perl /usr/local/bin/swift-trunk/bin/worker.pl http://128.135.250.235:45185 0722-3703570-000000 /tmp 1> /tmp/tmp.dXjNyXhGn8 2> /tmp/tmp.gLOB9vWNvL"
+/home/yadu/src/swift-trunk/cog/modules/swift/tests/trunk_config/wrapper
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --submit /tmp/submit_script_9016
+jobid=549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+ENV
+XDG_VTNR=7
+SSH_AGENT_PID=2641
+XDG_SESSION_ID=c2
+CLUTTER_IM_MODULE=xim
+SELINUX_INIT=YES
+XDG_GREETER_DATA_DIR=/var/lib/lightdm-data/yadu
+GPG_AGENT_INFO=/run/user/1000/keyring-EqXXD5/gpg:0:1
+TERM=xterm
+SHELL=/bin/bash
+VTE_VERSION=3409
+SSH_AGENT_LAUNCHER=upstart
+WINDOWID=79691788
+UPSTART_SESSION=unix:abstract=/com/ubuntu/upstart-session/1000/2585
+GNOME_KEYRING_CONTROL=/run/user/1000/keyring-EqXXD5
+GTK_MODULES=overlay-scrollbar:unity-gtk-module
+USER=yadu
+LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arj=01;31:*.taz=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.lz=01;31:*.xz=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:
*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.axv=01;35:*.anx=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.axa=00;36:*.oga=00;36:*.spx=00;36:*.xspf=00;36:
+DEBEMAIL=yadudoc1729 at gmail.com
+XDG_SESSION_PATH=/org/freedesktop/DisplayManager/Session0
+XDG_SEAT_PATH=/org/freedesktop/DisplayManager/Seat0
+SSH_AUTH_SOCK=/run/user/1000/keyring-EqXXD5/ssh
+DEFAULTS_PATH=/usr/share/gconf/ubuntu.default.path
+XDG_CONFIG_DIRS=/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg
+NLSPATH=/usr/dt/lib/nls/msg/%L/%N.cat
+GPGKEY=01E58D97
+PATH=/home/yadu/src/swift-trunk/cog/modules/swift/dist/swift-svn/bin:/home/yadu/src/google-cloud-sdk/bin:/home/yadu/src/swift-0.95/cog/modules/swift/dist/swift-svn/bin:/home/yadu/bin/jdk1.7.0_25/bin:/home/yadu/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/yadu/cctools/bin:/sbin
+DESKTOP_SESSION=ubuntu
+QT_QPA_PLATFORMTHEME=appmenu-qt5
+JOB=gnome-session
+PWD=/home/yadu/src/swift-trunk/cog/modules/swift/tests/trunk_config/wrapper
+XMODIFIERS=@im=none
+GNOME_KEYRING_PID=2582
+LANG=en_US.UTF-8
+GDM_LANG=en_US
+MANDATORY_PATH=/usr/share/gconf/ubuntu.mandatory.path
+NODE_PATH=/usr/lib/nodejs:/usr/lib/node_modules:/usr/share/javascript
+XFILESEARCHPATH=/usr/dt/app-defaults/%L/Dt
+UBUNTU_MENUPROXY=1
+IM_CONFIG_PHASE=1
+COMPIZ_CONFIG_PROFILE=ubuntu
+GDMSESSION=ubuntu
+SESSIONTYPE=gnome-session
+SHLVL=3
+XDG_SEAT=seat0
+HOME=/home/yadu
+LANGUAGE=en_US
+GNOME_DESKTOP_SESSION_ID=this-is-deprecated
+UPSTART_INSTANCE=
+UPSTART_EVENTS=started starting
+LOGNAME=yadu
+QT4_IM_MODULE=xim
+XDG_DATA_DIRS=/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/
+DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-nBcGU7DKx6
+WORKER_LOGGING_LEVELTRACE=
+DEBFULLNAME=Yadu Nand
+LESSOPEN=| /usr/bin/lesspipe %s
+INSTANCE=Unity
+UPSTART_JOB=unity-settings-daemon
+TEXTDOMAIN=im-config
+XDG_RUNTIME_DIR=/run/user/1000
+DISPLAY=:0
+XDG_CURRENT_DESKTOP=Unity
+GTK_IM_MODULE=xim
+LESSCLOSE=/usr/bin/lesspipe %s %s
+TEXTDOMAINDIR=/usr/share/locale/
+COLORTERM=gnome-terminal
+XAUTHORITY=/home/yadu/.Xauthority
+_=/usr/bin/env
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 Q
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 Q
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 Q
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 Q
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
+549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
+Cancelling 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/README.asc
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/README.asc (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/README.asc (revision 4027)
@@ -0,0 +1,35 @@
+Swift interface to the Clouds
+-----------------------------
+
+There are 3 core functions that the Cloud interface does:
+
+submit : Submits a string for execution on a cloud instance.
+ The string for execution will be the in the config file in the format :
+ CMD_STRING="<string>"
+ Once the job is submitted, cloud_submit will emit the string jobid=<uniq_jobid>
+ Returns 0 on successful submission.
+
+ If specification is incorrect, or an error occurred, the error message will go to stderr
+ Error: codes, not defined.
+
+status : Takes the unique ids of runs, and returns the status of job submitted to the cloud.
+ Prints out following:
+ <jobid> Q|R|C|F [ exitcode [<error message>] ]
+ Q - Queued - Pending run
+ R - Running
+ C - Completed
+ F - Failed optionally
+
+cancel : Takes one jobid as an argument. Cancels/Terminates the job.
+
+
+Examples:
+
+./cloud.py cloud_submit configs <string>
+./cloud.py cloud_status configs jobid1 jobid2
+./cloud.py cloud_cancel configs jobid1
+
+
+
+
+
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/ststat
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/ststat (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/ststat (revision 4027)
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+LOG=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/log
+CONF=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs
+
+if [[ "$1" != "" ]]
+then
+ echo "Stat'ing for $1 " >> $LOG
+ echo "python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status $CONF --jobid $1" >> $LOG
+ python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status $CONF --jobid $1 | tee -a $LOG
+fi
Property changes on: modules/provider-localscheduler/examples/ec2-cloud-provider/ststat
___________________________________________________________________
Added: svn:executable
+ *
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/configs
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/configs (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/configs (revision 4027)
@@ -0,0 +1,12 @@
+AWS_CREDENTIALS_FILE=~/.ssh/boto-test-credentials.csv
+
+# Update the following only if you want to use a specific keypair
+AWS_KEYPAIR_NAME=swift-test-pair
+AWS_KEYPAIR_FILE=~/.ssh/swift-test-pair.pem
+
+AWS_REGION=us-west-2
+SECURITY_GROUP=swift_security_group1
+
+WORKER_IMAGE=ami-23700813
+WORKER_MACHINE_TYPE=t1.micro
+
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/stsubmit
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/stsubmit (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/stsubmit (revision 4027)
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+
+rm -f /tmp/stsubmit
+EXECUTABLE=
+DIR=
+ARGS=
+STDOUTLOC=
+STDOUTPATH=
+STDERRLOC=
+STDERRPATH=
+STDINLOC=
+STDINPATH=
+STDIN=
+STDOUT=
+STDERR=
+
+while read LINE; do
+ echo $LINE >>/tmp/stsubmit
+ case $LINE in
+ executable=*)
+ EXECUTABLE=${LINE#executable=}
+ ;;
+ directory=*)
+ DIR=${LINE#directory=}
+ ;;
+ arg=*)
+ if [[ ${LINE#arg=} == *cscript*pl ]]
+ then
+ # We are replacing with a specific temporary worker script
+ ARGS="$ARGS /usr/local/bin/swift-trunk/bin/worker.pl"
+ else
+ ARGS="$ARGS ${LINE#arg=}"
+ fi
+ ;;
+ attr.*)
+ # ignore attributes
+ ;;
+ stdin.location=*)
+ STDINLOC=${LINE#stdin.location=}
+ ;;
+ stdin.path=*)
+ STDINPATH=${LINE#stdin.path=}
+ ;;
+ stdout.location=*)
+ STDOUTLOC=${LINE#stdout.location=}
+ ;;
+ stdout.path=*)
+ STDOUTPATH=${LINE#stdout.path=}
+ ;;
+ stderr.location=*)
+ STDERRLOC=${LINE#stderr.location=}
+ ;;
+ stderr.path=*)
+ STDERRPATH=${LINE#stderr.path=}
+ ;;
+ env.*)
+ LINE2=${LINE#env.}
+ # split on '='
+ ELS=(${LINE2//=/})
+ NAME=${ELS[0]}
+ VALUE=${ELS[1]}
+ export $NAME=$VALUE
+ ;;
+ *)
+ echo "Don't know how to interpret line: $LINE" >&2
+ exit 2
+ esac
+done < /dev/stdin
+
+if [ "$STDOUTLOC" == "tmp" ]; then
+ STDOUTPATH=$(mktemp)
+ echo "stdout.path=$STDOUTPATH"
+fi
+if [ "$STDOUTPATH" != "" ]; then
+ STDOUT="1> $STDOUTPATH"
+fi
+
+if [ "$STDERRLOC" == "tmp" ]; then
+ STDERRPATH=$(mktemp)
+ echo "stderr.path=$STDERRPATH"
+fi
+if [ "$STDERRPATH" != "" ]; then
+ STDERR="2> $STDERRPATH"
+fi
+
+if [ "$STDINLOC" != "" ]; then
+ STDIN="< $STDINLOC"
+fi
+
+LOG=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/log
+echo "Start" > $LOG
+CMD="$EXECUTABLE $ARGS $STDIN $STDOUT $STDERR"
+echo "CMD : $CMD" >> $LOG
+
+#SUBMIT_SCRIPT=$(mktemp)
+SUBMIT_SCRIPT=/tmp/submit_script_$RANDOM
+
+# TODO : Cleanup
+cat /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs > $SUBMIT_SCRIPT
+DIR=/tmp/
+# TODO : Fix me
+
+cat<<EOF >> $SUBMIT_SCRIPT
+CMD_STRING="mkdir $DIR; cd $DIR; $CMD"
+EOF
+
+cat $SUBMIT_SCRIPT >> $LOG
+
+echo "$PWD" >> $LOG
+echo "python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --submit $SUBMIT_SCRIPT" >> $LOG
+python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --submit $SUBMIT_SCRIPT | tee -a $LOG
+
+echo "ENV" >> $LOG
+env >> $LOG
+exit 0
Property changes on: modules/provider-localscheduler/examples/ec2-cloud-provider/stsubmit
___________________________________________________________________
Added: svn:executable
+ *
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/cog-provider.properties
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/cog-provider.properties (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/cog-provider.properties (revision 4027)
@@ -0,0 +1,4 @@
+provider=ec2-cloud
+sandbox=false
+executionTaskHandler=org.globus.cog.abstraction.impl.scheduler.shell.execution.TaskHandlerImpl
+securityContext=org.globus.cog.abstraction.impl.common.task.SecurityContextImpl
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/stcancel
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/stcancel (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/stcancel (revision 4027)
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+LOG=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/log
+CONF=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs
+
+if [[ "$1" != "" ]]
+then
+ echo "Cancelling $1" >> $LOG
+ python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --cancel $CONF --jobid $1 | tee -a $LOG
+fi
Property changes on: modules/provider-localscheduler/examples/ec2-cloud-provider/stcancel
___________________________________________________________________
Added: svn:executable
+ *
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py (revision 0)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py (revision 4027)
@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import random
+import logging
+import pprint
+import argparse
+import datetime
+import time
+#from __future__ import print_function
+
+from libcloud.compute.types import Provider
+from libcloud.compute.providers import get_driver
+from libcloud.compute.base import NodeSize, NodeImage
+from libcloud.compute.types import NodeState
+import libcloud.compute.types
+
+NODESTATES = { NodeState.RUNNING : "RUNNING",
+ NodeState.REBOOTING : "REBOOTING",
+ NodeState.TERMINATED : "TERMINATED",
+ NodeState.STOPPED : "STOPPED",
+ NodeState.PENDING : "PENDING",
+ NodeState.UNKNOWN : "UNKNOWN" }
+
+timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S');
+logging.basicConfig(filename='cloud_ec2'+timestamp+'.log', level=logging.INFO)
+
+WORKER_USERDATA='''#!/bin/bash
+export JAVA=/usr/local/bin/jdk1.7.0_51/bin
+export SWIFT=/usr/local/bin/swift-trunk/bin
+export PATH=$JAVA:$SWIFT:$PATH
+export WORKER_LOGGING_LEVEL=TRACE
+mkdir -p /home/yadu/.globus/coasters
+'''
+
+def aws_create_security_group(driver, configs):
+ group_name = configs["SECURITY_GROUP"]
+ current = driver.ex_list_security_groups()
+ if group_name in current:
+ logging.info("Security group: %s is already present", group_name)
+ else:
+ logging.info("Creating new security group: %s", group_name)
+ res = driver.ex_create_security_group(name=group_name,description="Open all ports")
+ if not driver.ex_authorize_security_group(group_name, 0, 65000, '0.0.0.0/0'):
+ logging.info("Authorizing ports for security group failed")
+ if not driver.ex_authorize_security_group(group_name, 0, 65000, '0.0.0.0/0', protocol='udp'):
+ logging.info("Authorizing ports for security group failed")
+ logging.debug("Security group: %s", str(res))
+
+def check_keypair(driver, configs):
+ if "AWS_KEYPAIR_NAME" in configs and "AWS_KEYPAIR_FILE" in configs:
+ logging.debug("AWS_KEYPAIR_NAME : %s", configs['AWS_KEYPAIR_NAME'])
+ logging.debug("AWS_KEYPAIR_FILE : %s", configs['AWS_KEYPAIR_FILE'])
+ all_pairs = driver.list_key_pairs()
+ for pair in all_pairs:
+ if pair.name == configs['AWS_KEYPAIR_NAME']:
+ logging.info("KEYPAIR exists, registered")
+ return 0
+
+ logging.info("KEYPAIR does not exist. Creating keypair")
+ key_pair = driver.create_key_pair(name=configs['AWS_KEYPAIR_NAME'])
+ f = open(configs['AWS_KEYPAIR_FILE'], 'w')
+ f.write(str(key_pair.private_key))
+ f.close()
+ os.chmod(configs['AWS_KEYPAIR_FILE'], 0600)
+ logging.info("KEYPAIR created")
+ else:
+ logging.error("AWS_KEYPAIR_NAME and/or AWS_KEYPAIR_FILE missing")
+ logging.error("Cannot proceed without AWS_KEYPAIR_NAME and AWS_KEYPAIR_FILE")
+ exit(-1)
+
+def _read_conf(config_file):
+ cfile = open(config_file, 'r').read()
+ config = {}
+ for line in cfile.split('\n'):
+
+ # Checking if empty line or comment
+ if line.startswith('#') or not line :
+ continue
+
+ temp = line.split('=')
+ config[temp[0]] = temp[1].strip('\r')
+ return config
+
+def pretty_configs(configs):
+ printer = pprint.PrettyPrinter(indent=4)
+ printer.pprint(configs)
+
+def read_configs(config_file):
+ config = _read_conf(config_file)
+
+ if 'AWS_CREDENTIALS_FILE' in config :
+ config['AWS_CREDENTIALS_FILE'] = os.path.expanduser(config['AWS_CREDENTIALS_FILE'])
+ config['AWS_CREDENTIALS_FILE'] = os.path.expandvars(config['AWS_CREDENTIALS_FILE'])
+
+ cred_lines = open(config['AWS_CREDENTIALS_FILE']).readlines()
+ cred_details = cred_lines[1].split(',')
+ credentials = { 'AWS_Username' : cred_details[0],
+ 'AWSAccessKeyId' : cred_details[1],
+ 'AWSSecretKey' : cred_details[2] }
+ config.update(credentials)
+ else:
+ print "AWS_CREDENTIALS_FILE , Missing"
+ print "ERROR: Cannot proceed without access to AWS_CREDENTIALS_FILE"
+ exit(-1)
+
+ if 'AWS_KEYPAIR_FILE' in config:
+ config['AWS_KEYPAIR_FILE'] = os.path.expanduser(config['AWS_KEYPAIR_FILE'])
+ config['AWS_KEYPAIR_FILE'] = os.path.expandvars(config['AWS_KEYPAIR_FILE'])
+ return config
+
+def node_status(driver, node_uuids):
+ logging.info("Checking status of : %s", str(node_uuids))
+ nodes = driver.list_nodes()
+ for node in nodes:
+ logging.info("INFO: Node status : %s",str(node))
+ if node.uuid in node_uuids :
+ if node.state == NodeState.RUNNING:
+ print node.uuid, "R"
+ elif node.state == NodeState.PENDING:
+ print node.uuid, "Q"
+ elif node.state == NodeState.TERMINATED:
+ print node.uuid, "C"
+ elif node.state == NodeState.STOPPED:
+ print node.uuid, "C"
+ elif node.state == NodeState.UNKNOWN:
+ print node.uuid, "Q" # This state could be wrong
+ else:
+ logging.warn("Node state unknown/invalid %s", NODESTATE[node.state])
+ return -1
+ return 0
+
+def node_start(driver, configs, WORKER_STRING):
+ # Setup userdata
+ userdata = WORKER_USERDATA + WORKER_STRING.lstrip('"').rstrip('"')
+ logging.info("Worker userdata : %s", userdata)
+
+ size = NodeSize(id=configs['WORKER_MACHINE_TYPE'], name="swift_worker",
+ ram=None, disk=None, bandwidth=None, price=None, driver=driver)
+ image = NodeImage(id=configs['WORKER_IMAGE'], name=None, driver=driver)
+ node = driver.create_node(name="swift_worker",
+ image=image,
+ size=size,
+ ex_keyname=configs['AWS_KEYPAIR_NAME'],
+ ex_securitygroup=configs['SECURITY_GROUP'],
+ ex_userdata=userdata )
+ logging.info("Worker node started : %s", str(node))
+ print 'jobid={0}'.format(node.uuid)
+
+# node_names is a list
+def node_terminate(driver, node_uuids):
+ nodes = driver.list_nodes()
+ deleted_flag = False
+ for node in nodes:
+ if node.uuid in node_uuids and node.state == NodeState.RUNNING :
+ logging.info("Terminating node : %s", str(node))
+ code = driver.destroy_node(node)
+ deleted_flag = True
+ return deleted_flag
+
+
+def init_checks(driver, configs):
+ aws_create_security_group(driver, configs)
+ check_keypair(driver, configs)
+
+def init(conf_file):
+ configs = read_configs(conf_file)
+ #pretty_configs(configs)
+ driver = get_driver(Provider.EC2_US_WEST_OREGON) # was EC2
+ ec2_driver = driver(configs['AWSAccessKeyId'], configs['AWSSecretKey'])
+ return configs,ec2_driver
+
+# Main driver section
+#configs, driver = init()
+#args = sys.argv[1:]
+#print "All args : ",str(args)
+
+
+if __name__ == '__main__' :
+ parser = argparse.ArgumentParser()
+ mu_group = parser.add_mutually_exclusive_group(required=True)
+ mu_group.add_argument("-s", "--submit", default=None , help='Takes a config file. Submits the CMD_STRING in the configs for execution on a cloud resource')
+ mu_group.add_argument("-t", "--status", default=None , help='gets the status of the CMD_STRING in the configs for execution on a cloud resource')
+ mu_group.add_argument("-c", "--cancel", default=None , help='cancels the jobs with jobids')
+ parser.add_argument("-v", "--verbose", help="set level of verbosity, DEBUG, INFO, WARN")
+
+ parser.add_argument("-j", "--jobid", type=str, action='append')
+ args = parser.parse_args()
+
+ config_file = ( args.status or args.submit or args.cancel )
+ configs, driver = init(config_file)
+
+ if args.submit :
+
+ # Init checks confirm keypairs and security groups to allow for access to ports
+ init_checks(driver, configs)
+ node_start(driver, configs, configs['CMD_STRING'])
+
+ elif args.status :
+
+ node_status(driver, args.jobid )
+
+ elif args.cancel :
+
+ node_terminate(driver, args.jobid)
+
+ else:
+
+ sys.stderr.write("ERROR: Undefined args, cannot be handled")
+ sys.stderr.write("ERROR: Exiting...")
+ exit(-1)
+
+ exit(0)
Property changes on: modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py
___________________________________________________________________
Added: svn:executable
+ *
More information about the Swift-commit
mailing list