[Swift-commit] cog r4028
swift at ci.uchicago.edu
swift at ci.uchicago.edu
Wed Jul 23 17:00:06 CDT 2014
------------------------------------------------------------------------
r4028 | yadunand | 2014-07-23 16:59:25 -0500 (Wed, 23 Jul 2014) | 6 lines
Updates for ec2-provider. Build issues remain, and submit scripts require absolute paths.
Added support for passing ec2 attributes as coaster jobOptions from swift.conf
Cleaned up handling of configs in the job handler scripts
Tested.
------------------------------------------------------------------------
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/log
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/log (revision 4027)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/log (working copy)
@@ -1,136 +0,0 @@
-Start
-CMD : /usr/bin/perl /usr/local/bin/swift-trunk/bin/worker.pl http://128.135.250.235:45185 0722-3703570-000000 /tmp 1> /tmp/tmp.dXjNyXhGn8 2> /tmp/tmp.gLOB9vWNvL
-AWS_CREDENTIALS_FILE=~/.ssh/boto-test-credentials.csv
-
-# Update the following only if you want to use a specific keypair
-AWS_KEYPAIR_NAME=swift-test-pair
-AWS_KEYPAIR_FILE=~/.ssh/swift-test-pair.pem
-
-AWS_REGION=us-west-2
-SECURITY_GROUP=swift_security_group1
-
-WORKER_IMAGE=ami-23700813
-WORKER_MACHINE_TYPE=t1.micro
-
-CMD_STRING="mkdir /tmp/; cd /tmp/; /usr/bin/perl /usr/local/bin/swift-trunk/bin/worker.pl http://128.135.250.235:45185 0722-3703570-000000 /tmp 1> /tmp/tmp.dXjNyXhGn8 2> /tmp/tmp.gLOB9vWNvL"
-/home/yadu/src/swift-trunk/cog/modules/swift/tests/trunk_config/wrapper
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --submit /tmp/submit_script_9016
-jobid=549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-ENV
-XDG_VTNR=7
-SSH_AGENT_PID=2641
-XDG_SESSION_ID=c2
-CLUTTER_IM_MODULE=xim
-SELINUX_INIT=YES
-XDG_GREETER_DATA_DIR=/var/lib/lightdm-data/yadu
-GPG_AGENT_INFO=/run/user/1000/keyring-EqXXD5/gpg:0:1
-TERM=xterm
-SHELL=/bin/bash
-VTE_VERSION=3409
-SSH_AGENT_LAUNCHER=upstart
-WINDOWID=79691788
-UPSTART_SESSION=unix:abstract=/com/ubuntu/upstart-session/1000/2585
-GNOME_KEYRING_CONTROL=/run/user/1000/keyring-EqXXD5
-GTK_MODULES=overlay-scrollbar:unity-gtk-module
-USER=yadu
-LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arj=01;31:*.taz=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.dz=01;31:*.gz=01;31:*.lz=01;31:*.xz=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:
*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.axv=01;35:*.anx=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.axa=00;36:*.oga=00;36:*.spx=00;36:*.xspf=00;36:
-DEBEMAIL=yadudoc1729 at gmail.com
-XDG_SESSION_PATH=/org/freedesktop/DisplayManager/Session0
-XDG_SEAT_PATH=/org/freedesktop/DisplayManager/Seat0
-SSH_AUTH_SOCK=/run/user/1000/keyring-EqXXD5/ssh
-DEFAULTS_PATH=/usr/share/gconf/ubuntu.default.path
-XDG_CONFIG_DIRS=/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg
-NLSPATH=/usr/dt/lib/nls/msg/%L/%N.cat
-GPGKEY=01E58D97
-PATH=/home/yadu/src/swift-trunk/cog/modules/swift/dist/swift-svn/bin:/home/yadu/src/google-cloud-sdk/bin:/home/yadu/src/swift-0.95/cog/modules/swift/dist/swift-svn/bin:/home/yadu/bin/jdk1.7.0_25/bin:/home/yadu/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/yadu/cctools/bin:/sbin
-DESKTOP_SESSION=ubuntu
-QT_QPA_PLATFORMTHEME=appmenu-qt5
-JOB=gnome-session
-PWD=/home/yadu/src/swift-trunk/cog/modules/swift/tests/trunk_config/wrapper
-XMODIFIERS=@im=none
-GNOME_KEYRING_PID=2582
-LANG=en_US.UTF-8
-GDM_LANG=en_US
-MANDATORY_PATH=/usr/share/gconf/ubuntu.mandatory.path
-NODE_PATH=/usr/lib/nodejs:/usr/lib/node_modules:/usr/share/javascript
-XFILESEARCHPATH=/usr/dt/app-defaults/%L/Dt
-UBUNTU_MENUPROXY=1
-IM_CONFIG_PHASE=1
-COMPIZ_CONFIG_PROFILE=ubuntu
-GDMSESSION=ubuntu
-SESSIONTYPE=gnome-session
-SHLVL=3
-XDG_SEAT=seat0
-HOME=/home/yadu
-LANGUAGE=en_US
-GNOME_DESKTOP_SESSION_ID=this-is-deprecated
-UPSTART_INSTANCE=
-UPSTART_EVENTS=started starting
-LOGNAME=yadu
-QT4_IM_MODULE=xim
-XDG_DATA_DIRS=/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/
-DBUS_SESSION_BUS_ADDRESS=unix:abstract=/tmp/dbus-nBcGU7DKx6
-WORKER_LOGGING_LEVELTRACE=
-DEBFULLNAME=Yadu Nand
-LESSOPEN=| /usr/bin/lesspipe %s
-INSTANCE=Unity
-UPSTART_JOB=unity-settings-daemon
-TEXTDOMAIN=im-config
-XDG_RUNTIME_DIR=/run/user/1000
-DISPLAY=:0
-XDG_CURRENT_DESKTOP=Unity
-GTK_IM_MODULE=xim
-LESSCLOSE=/usr/bin/lesspipe %s %s
-TEXTDOMAINDIR=/usr/share/locale/
-COLORTERM=gnome-terminal
-XAUTHORITY=/home/yadu/.Xauthority
-_=/usr/bin/env
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 Q
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 Q
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 Q
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 Q
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Stat'ing for 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs --jobid 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
-549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7 R
-Cancelling 549ec5ddf5bcd4fd985f86ce48a83d0b0116fbd7
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/README.asc
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/README.asc (revision 4027)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/README.asc (working copy)
@@ -23,6 +23,38 @@
cancel : Takes one jobid as an argument. Cancels/Terminates the job.
+Example config:
+site.cloud {
+ execution {
+ type:"coaster"
+ URL: "127.0.0.1"
+ jobManager: "local:ec2-cloud"
+ options {
+ maxJobs: 10
+ tasksPerNode: 4
+ workerLoggingLevel: TRACE
+ workerLoggingDirectory: /tmp
+ jobOptions {
+ ec2CredentialsFile: ${env.HOME}/.ssh/boto-test-credentials.csv
+ ec2SecurityGroup: swift_security_group1
+ ec2KeypairName: swift-test-pair
+ ec2KeypairFile: ${env.HOME}/swift-test-pair.pem
+ ec2WorkerImage: ami-23700813
+ ec2WorkerType: t1.micro
+ }
+ }
+ }
+ initialParallelTasks: 10
+ maxParallelTasks: 20
+ filesystem.type: swift
+ workDirectory: /tmp/swift-trunk-test
+ staging: "local"
+ app.ALL {executable: "*"}
+}
+
+sites: cloud
+
+
Examples:
./cloud.py cloud_submit configs <string>
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/ststat
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/ststat (revision 4027)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/ststat (working copy)
@@ -1,11 +1,24 @@
#!/bin/bash
LOG=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/log
-CONF=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs
+LOGGING=1 # Either 1 or 0
+log()
+{
+ [[ "$LOGGING" == "1" ]] && echo $* >> $LOG
+}
+JOBID=$1
+CONF=/tmp/$JOBID
+
+# Check for the conf file for the job at /tmp/<jobid>
+if [ ! -f "$CONF" ]
+then
+ echo "ERROR: Could not find conf script at /tmp/$1" 1>&2
+ exit -1
+fi
+
if [[ "$1" != "" ]]
then
- echo "Stat'ing for $1 " >> $LOG
- echo "python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status $CONF --jobid $1" >> $LOG
- python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status $CONF --jobid $1 | tee -a $LOG
+ log "Stat'ing for $1 PWD: $PWD"
+ python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --status $CONF --jobid $JOBID | tee -a $LOG
fi
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/configs
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/configs (revision 4027)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/configs (working copy)
@@ -1,12 +0,0 @@
-AWS_CREDENTIALS_FILE=~/.ssh/boto-test-credentials.csv
-
-# Update the following only if you want to use a specific keypair
-AWS_KEYPAIR_NAME=swift-test-pair
-AWS_KEYPAIR_FILE=~/.ssh/swift-test-pair.pem
-
-AWS_REGION=us-west-2
-SECURITY_GROUP=swift_security_group1
-
-WORKER_IMAGE=ami-23700813
-WORKER_MACHINE_TYPE=t1.micro
-
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/stsubmit
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/stsubmit (revision 4027)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/stsubmit (working copy)
@@ -1,6 +1,5 @@
#!/bin/bash
-
rm -f /tmp/stsubmit
EXECUTABLE=
DIR=
@@ -15,6 +14,13 @@
STDOUT=
STDERR=
+LOG=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/log
+
+SUBMIT_SCRIPT=$(mktemp)
+touch $SUBMIT_SCRIPT
+
+#SUBMIT_SCRIPT=$(mktemp)
+
while read LINE; do
echo $LINE >>/tmp/stsubmit
case $LINE in
@@ -27,14 +33,18 @@
arg=*)
if [[ ${LINE#arg=} == *cscript*pl ]]
then
- # We are replacing with a specific temporary worker script
+ # Replacing temporary worker script with a specific remote worker script
ARGS="$ARGS /usr/local/bin/swift-trunk/bin/worker.pl"
else
ARGS="$ARGS ${LINE#arg=}"
fi
;;
attr.*)
- # ignore attributes
+ LINE2=${LINE#attr.}
+ if [[ $LINE2 == ec2* ]]
+ then
+ echo $LINE2 >> $SUBMIT_SCRIPT
+ fi
;;
stdin.location=*)
STDINLOC=${LINE#stdin.location=}
@@ -88,18 +98,10 @@
STDIN="< $STDINLOC"
fi
-LOG=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/log
-echo "Start" > $LOG
CMD="$EXECUTABLE $ARGS $STDIN $STDOUT $STDERR"
-echo "CMD : $CMD" >> $LOG
+echo "CMD : $CMD" >> $LOG
-#SUBMIT_SCRIPT=$(mktemp)
-SUBMIT_SCRIPT=/tmp/submit_script_$RANDOM
-
-# TODO : Cleanup
-cat /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs > $SUBMIT_SCRIPT
DIR=/tmp/
-# TODO : Fix me
cat<<EOF >> $SUBMIT_SCRIPT
CMD_STRING="mkdir $DIR; cd $DIR; $CMD"
@@ -109,8 +111,14 @@
echo "$PWD" >> $LOG
echo "python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --submit $SUBMIT_SCRIPT" >> $LOG
-python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --submit $SUBMIT_SCRIPT | tee -a $LOG
+JOBINFO=$(python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --submit $SUBMIT_SCRIPT)
+retcode="$?"
+echo $JOBINFO
+[[ "$retcode" != "0" ]] && exit retcode
-echo "ENV" >> $LOG
-env >> $LOG
+if [[ $JOBINFO == jobid\=* ]]
+then
+ cat $SUBMIT_SCRIPT > /tmp/${JOBINFO#jobid=}
+fi
+
exit 0
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/stcancel
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/stcancel (revision 4027)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/stcancel (working copy)
@@ -1,10 +1,33 @@
#!/bin/bash
LOG=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/log
-CONF=/home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/configs
+LOGGING=1 # Either 1 or 0
+log()
+{
+ [[ "$LOGGING" == "1" ]] && echo $* >> $LOG
+}
+JOBID=$1
+CONF=/tmp/$JOBID
+EXITCODE=0
+
+# Check for the conf file for the job at /tmp/<jobid>
+if [ ! -f "$CONF" ]
+then
+ echo "ERROR: Could not find conf script at /tmp/$1" 1>&2
+ exit -1
+fi
+
if [[ "$1" != "" ]]
then
- echo "Cancelling $1" >> $LOG
- python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --cancel $CONF --jobid $1 | tee -a $LOG
+ log "Cancelling $JOBID"
+ python /home/yadu/src/swift-trunk/cog/modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py --cancel $CONF --jobid $JOBID
+ EXITCODE=$?
+ if [[ "$EXITCODE" == "0" ]]
+ then
+ log "Done cancelling $JOBID"
+ rm $CONF
+ fi
fi
+
+exit $EXITCODE
Index: modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py
===================================================================
--- modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py (revision 4027)
+++ modules/provider-localscheduler/examples/ec2-cloud-provider/cloud.py (working copy)
@@ -23,8 +23,8 @@
NodeState.PENDING : "PENDING",
NodeState.UNKNOWN : "UNKNOWN" }
-timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H:%M:%S');
-logging.basicConfig(filename='cloud_ec2'+timestamp+'.log', level=logging.INFO)
+#timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d_%H');
+#logging.basicConfig(filename='cloud_ec2'+timestamp+'.log', level=logging.WARN)
WORKER_USERDATA='''#!/bin/bash
export JAVA=/usr/local/bin/jdk1.7.0_51/bin
@@ -35,50 +35,39 @@
'''
def aws_create_security_group(driver, configs):
- group_name = configs["SECURITY_GROUP"]
+ group_name = configs["ec2securitygroup"]
current = driver.ex_list_security_groups()
- if group_name in current:
- logging.info("Security group: %s is already present", group_name)
- else:
- logging.info("Creating new security group: %s", group_name)
+ if group_name not in current:
res = driver.ex_create_security_group(name=group_name,description="Open all ports")
if not driver.ex_authorize_security_group(group_name, 0, 65000, '0.0.0.0/0'):
- logging.info("Authorizing ports for security group failed")
+ sys.stderr.write("Authorizing ports for security group failed \n")
if not driver.ex_authorize_security_group(group_name, 0, 65000, '0.0.0.0/0', protocol='udp'):
- logging.info("Authorizing ports for security group failed")
- logging.debug("Security group: %s", str(res))
+ sys.stderr.write("Authorizing ports for security group failed \n")
def check_keypair(driver, configs):
- if "AWS_KEYPAIR_NAME" in configs and "AWS_KEYPAIR_FILE" in configs:
- logging.debug("AWS_KEYPAIR_NAME : %s", configs['AWS_KEYPAIR_NAME'])
- logging.debug("AWS_KEYPAIR_FILE : %s", configs['AWS_KEYPAIR_FILE'])
+ if "ec2keypairname" in configs and "ec2keypairfile" in configs:
all_pairs = driver.list_key_pairs()
for pair in all_pairs:
- if pair.name == configs['AWS_KEYPAIR_NAME']:
- logging.info("KEYPAIR exists, registered")
+ if pair.name == configs['ec2keypairname']:
return 0
- logging.info("KEYPAIR does not exist. Creating keypair")
- key_pair = driver.create_key_pair(name=configs['AWS_KEYPAIR_NAME'])
- f = open(configs['AWS_KEYPAIR_FILE'], 'w')
+ key_pair = driver.create_key_pair(name=configs['ec2keypairname'])
+ f = open(configs['ec2keypairfile'], 'w')
f.write(str(key_pair.private_key))
f.close()
- os.chmod(configs['AWS_KEYPAIR_FILE'], 0600)
- logging.info("KEYPAIR created")
+ os.chmod(configs['ec2keypairfile'], 0600)
else:
- logging.error("AWS_KEYPAIR_NAME and/or AWS_KEYPAIR_FILE missing")
- logging.error("Cannot proceed without AWS_KEYPAIR_NAME and AWS_KEYPAIR_FILE")
+ sys.stderr.write("ec2keypairname and/or ec2keypairfile missing\n")
+ sys.stderr.write("Cannot proceed without ec2keypairname and ec2keypairfile\n")
exit(-1)
def _read_conf(config_file):
cfile = open(config_file, 'r').read()
config = {}
for line in cfile.split('\n'):
-
# Checking if empty line or comment
if line.startswith('#') or not line :
continue
-
temp = line.split('=')
config[temp[0]] = temp[1].strip('\r')
return config
@@ -90,31 +79,26 @@
def read_configs(config_file):
config = _read_conf(config_file)
- if 'AWS_CREDENTIALS_FILE' in config :
- config['AWS_CREDENTIALS_FILE'] = os.path.expanduser(config['AWS_CREDENTIALS_FILE'])
- config['AWS_CREDENTIALS_FILE'] = os.path.expandvars(config['AWS_CREDENTIALS_FILE'])
+ if 'ec2credentialsfile' in config :
+ config['ec2credentialsfile'] = os.path.expanduser(config['ec2credentialsfile'])
+ config['ec2credentialsfile'] = os.path.expandvars(config['ec2credentialsfile'])
- cred_lines = open(config['AWS_CREDENTIALS_FILE']).readlines()
+ cred_lines = open(config['ec2credentialsfile']).readlines()
cred_details = cred_lines[1].split(',')
credentials = { 'AWS_Username' : cred_details[0],
'AWSAccessKeyId' : cred_details[1],
'AWSSecretKey' : cred_details[2] }
config.update(credentials)
else:
- print "AWS_CREDENTIALS_FILE , Missing"
- print "ERROR: Cannot proceed without access to AWS_CREDENTIALS_FILE"
+ print "ec2credentialsfile , Missing"
+ print "ERROR: Cannot proceed without access to ec2credentialsfile"
exit(-1)
- if 'AWS_KEYPAIR_FILE' in config:
- config['AWS_KEYPAIR_FILE'] = os.path.expanduser(config['AWS_KEYPAIR_FILE'])
- config['AWS_KEYPAIR_FILE'] = os.path.expandvars(config['AWS_KEYPAIR_FILE'])
return config
def node_status(driver, node_uuids):
- logging.info("Checking status of : %s", str(node_uuids))
nodes = driver.list_nodes()
for node in nodes:
- logging.info("INFO: Node status : %s",str(node))
if node.uuid in node_uuids :
if node.state == NodeState.RUNNING:
print node.uuid, "R"
@@ -127,25 +111,23 @@
elif node.state == NodeState.UNKNOWN:
print node.uuid, "Q" # This state could be wrong
else:
- logging.warn("Node state unknown/invalid %s", NODESTATE[node.state])
+ sys.stderr.write("Node state unknown/invalid " + str(NODESTATE[node.state]))
return -1
return 0
def node_start(driver, configs, WORKER_STRING):
- # Setup userdata
- userdata = WORKER_USERDATA + WORKER_STRING.lstrip('"').rstrip('"')
- logging.info("Worker userdata : %s", userdata)
- size = NodeSize(id=configs['WORKER_MACHINE_TYPE'], name="swift_worker",
+ userdata = WORKER_USERDATA + WORKER_STRING.lstrip('"').rstrip('"')
+ size = NodeSize(id=configs['ec2workertype'], name="swift_worker",
ram=None, disk=None, bandwidth=None, price=None, driver=driver)
- image = NodeImage(id=configs['WORKER_IMAGE'], name=None, driver=driver)
+ image = NodeImage(id=configs['ec2workerimage'], name=None, driver=driver)
+
node = driver.create_node(name="swift_worker",
image=image,
size=size,
- ex_keyname=configs['AWS_KEYPAIR_NAME'],
- ex_securitygroup=configs['SECURITY_GROUP'],
+ ex_keyname=configs['ec2keypairname'],
+ ex_securitygroup=configs['ec2securitygroup'],
ex_userdata=userdata )
- logging.info("Worker node started : %s", str(node))
print 'jobid={0}'.format(node.uuid)
# node_names is a list
@@ -154,7 +136,7 @@
deleted_flag = False
for node in nodes:
if node.uuid in node_uuids and node.state == NodeState.RUNNING :
- logging.info("Terminating node : %s", str(node))
+ #logging.info("Terminating node : %s", str(node))
code = driver.destroy_node(node)
deleted_flag = True
return deleted_flag
@@ -166,7 +148,6 @@
def init(conf_file):
configs = read_configs(conf_file)
- #pretty_configs(configs)
driver = get_driver(Provider.EC2_US_WEST_OREGON) # was EC2
ec2_driver = driver(configs['AWSAccessKeyId'], configs['AWSSecretKey'])
return configs,ec2_driver
More information about the Swift-commit
mailing list