[Swift-commit] r2771 - in usertools/cio: falkon tools tools/zoid
noreply at svn.ci.uchicago.edu
noreply at svn.ci.uchicago.edu
Sun Mar 29 23:52:05 CDT 2009
Author: aespinosa
Date: 2009-03-29 23:52:04 -0500 (Sun, 29 Mar 2009)
New Revision: 2771
Added:
usertools/cio/falkon/cobaltrun.job
usertools/cio/falkon/falkon-start-bgp-logging.sh
usertools/cio/tools/zoid/
usertools/cio/tools/zoid/logging-script.sh
Log:
Added tools for logging GPFS traffic on ZeptoOS ION.
also included a sample script to run
Added: usertools/cio/falkon/cobaltrun.job
===================================================================
--- usertools/cio/falkon/cobaltrun.job (rev 0)
+++ usertools/cio/falkon/cobaltrun.job 2009-03-30 04:52:04 UTC (rev 2771)
@@ -0,0 +1 @@
+./falkon-start-bgp-logging.sh default 64 60 4
Added: usertools/cio/falkon/falkon-start-bgp-logging.sh
===================================================================
--- usertools/cio/falkon/falkon-start-bgp-logging.sh (rev 0)
+++ usertools/cio/falkon/falkon-start-bgp-logging.sh 2009-03-30 04:52:04 UTC (rev 2771)
@@ -0,0 +1,148 @@
+#!/bin/bash
+
+# Patched falkon-start-bgp-ram.sh which logs GPFS IO activity in ZeptoOS
+CIOROOT=/home/espinosa/cio
+
+
+ if [ -z "$3" ]; then
+ echo "usage: $0 <QueueName> <NumNodes> <MaxTimeMin>"
+ echo "usage: $0 prod 1024 60"
+ echo "-- or --"
+ echo "usage: $0 <QueueName> <NumNodes> <MaxTimeMin> <WorkersPerNode>"
+ echo "usage: $0 prod 1024 60 4"
+ exit 1
+ fi
+
+
+if [ -z "${FALKON_HOME}" ]; then
+ echo "ERROR: environment variable FALKON_HOME not defined"
+ exit 1
+fi
+
+
+QUEUE_NAME=$1
+PROFILE_NAME="zeptocn-log"
+NUM_NODES=$2
+let NUM_ION=NUM_NODES/64
+MAX_TIME_MIN=$3
+SERVICE_IP="192.168.1.254"
+SERVICE_PORT1=55000
+SERVICE_PORT2=55001
+WORKERS_PER_NODE=4
+if [ ! -z $4 ];then
+ WORKERS_PER_NODE=$4
+fi
+
+
+
+cp $CIOROOT/tools/zoid/logging-script.sh ${HOME}/zoid-user-script.sh
+chmod +x ${HOME}/zoid-user-script.sh
+
+FALKON_JOB_ID=`falkon-id-get.sh N/A`
+EXIT_CODE=$?
+
+
+ if [ "${EXIT_CODE}" -ne "0" ]; then
+ echo "Error in geting a unique falkon ID.. 'falkon-get-id.sh N/A'"
+ cqdel ${ZOID_JOB_ID}
+ exit ${EXIT_CODE}
+ fi
+
+
+
+ZOID_JOB_ID=`cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib /bgsys/linux/1.2.020080512/bin/bash /fuse/${FALKON_WORKER_HOME}/run.worker-c-bgp.sh ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT}`
+
+echo $ZOID_JOB_ID $FALKON_JOB_ID $QUEUE_NAME $NUM_NODES $MAX_TIME_MIN $WORKERS_PER_NODE >>$HOME/.falkonjobs
+
+EXIT_CODE=$?
+
+ if [ "${EXIT_CODE}" -ne "0" ]; then
+ echo "Error in submitting job to Cobalt.. 'cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib /bgsys/linux/1.2.020080512/bin/bash /fuse/${FALKON_WORKER_HOME}/run.worker-c-bgp.sh ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT}' "
+ exit ${EXIT_CODE}
+ fi
+
+falkon-id-update.sh ${FALKON_JOB_ID} ${ZOID_JOB_ID}
+
+EXIT_CODE=$?
+
+ if [ "${EXIT_CODE}" -ne "0" ]; then
+ echo "Error in updating cobalt job info for falkon.. 'falkon-update-id.sh ${ZOID_JOB_ID}'"
+ cqdel ${ZOID_JOB_ID}
+ exit ${EXIT_CODE}
+ fi
+
+
+FALKON_JOBID_HOME=${FALKON_ROOT}/users/${USER}/${FALKON_JOB_ID}
+
+echo "Submitted job ${ZOID_JOB_ID} to Cobalt, creating the job specific Falkon tree for logs and configuration in ${FALKON_JOBID_HOME}..."
+
+mkdir -p ${FALKON_JOBID_HOME}
+
+cp ${FALKON_HOME}/falkon.env.bgp* ${FALKON_JOBID_HOME}/
+cp -r ${FALKON_HOME}/config ${FALKON_JOBID_HOME}/
+cp ${FALKON_JOBID_HOME}/config/Client-service-URIs.config2 ${FALKON_JOBID_HOME}/config/Client-service-URIs.config
+mkdir -p ${FALKON_JOBID_HOME}/logs/client ${FALKON_JOBID_HOME}/logs/service ${FALKON_JOBID_HOME}/logs/provisioner ${FALKON_JOBID_HOME}/logs/worker
+
+
+DATE=`date +%s`
+echo "$DATE: pre-creating log dirs for Falkon service..."
+RACK_START=0
+RACK_END=48
+SEQUENCE_DIR=`seq -w ${RACK_START} ${RACK_END}`
+PSET_START=1
+PSET_END=16
+for a in ${SEQUENCE_DIR}
+do
+ for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$".
+ do
+ DIR_NAME="ion-R${a}-${b}"
+ mkdir -p ${FALKON_JOBID_HOME}/logs/service/$DIR_NAME
+ done
+done
+
+for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$".
+do
+ DIR_NAME="ion-${b}"
+ mkdir -p ${FALKON_JOBID_HOME}/logs/service/$DIR_NAME
+done
+
+
+DATE=`date +%s`
+echo "$DATE: done creating log dirs for Falkon service!"
+
+
+
+FALKON_HOME_RAM=/tmp/${USER}/falkon
+
+ln -s ${FALKON_HOME}/apps ${FALKON_JOBID_HOME}/apps
+ln -s ${FALKON_HOME_RAM}/container ${FALKON_JOBID_HOME}/container
+ln -s ${FALKON_HOME}/service ${FALKON_JOBID_HOME}/service
+ln -s ${FALKON_HOME}/worker ${FALKON_JOBID_HOME}/worker
+ln -s ${FALKON_HOME}/AstroPortal ${FALKON_JOBID_HOME}/AstroPortal
+ln -s ${FALKON_HOME}/client ${FALKON_JOBID_HOME}/client
+ln -s ${FALKON_HOME}/monitor ${FALKON_JOBID_HOME}/monitor
+ln -s ${FALKON_HOME}/bin ${FALKON_JOBID_HOME}/bin
+ln -s ${FALKON_HOME}/config ${FALKON_JOBID_HOME}/config
+ln -s ${FALKON_HOME}/ploticus ${FALKON_JOBID_HOME}/ploticus
+ln -s ${FALKON_HOME}/webserver ${FALKON_JOBID_HOME}/webserver
+ln -s ${FALKON_HOME}/workloads ${FALKON_JOBID_HOME}/workloads
+ln -s ${FALKON_HOME}/id ${FALKON_JOBID_HOME}/id
+ln -s ${FALKON_HOME}/apache-ant-1.7.0 ${FALKON_JOBID_HOME}/apache-ant-1.7.0
+ln -s ${FALKON_HOME}/ibm-java2-ppc64-50 ${FALKON_JOBID_HOME}/ibm-java2-ppc64-50
+ln -s ${FALKON_HOME_RAM}/ibm-java2-ppc-50 ${FALKON_JOBID_HOME}/ibm-java2-ppc-50
+ln -s ${FALKON_HOME}/falkon.tgz ${FALKON_JOBID_HOME}/falkon.tgz
+
+
+if [ ! -d "${FALKON_JOBID_HOME}" ]; then
+ echo "ERROR: invalid path ${FALKON_JOBID_HOME}... exiting"
+ cqdel ${ZOID_JOB_ID}
+ exit 1
+fi
+
+echo "Succesfully submitted the job to Cobalt, and setup job specific Falkon tree!"
+echo "To monitor the job status, type 'cqstat | grep ${USER}'; once it is in running state, you can use the Falkon specific command ...."
+echo "To submit your Falkon-based workload, type: ....; you can do this any time, the falkon workload will wait for the resources to come online, and will only be submitted when everything is ready; the script is run in the background, so the workload will run even if the ssh session gets disconnected."
+echo ""
+echo "Remember, your job id is ${ZOID_JOB_ID}, and if you need to look through the logs manually for anything, remember that you can find them at ${HOME}/${ZOID_JOB_ID}.output, ${HOME}/${ZOID_JOB_ID}.error, and ${FALKON_JOBID_HOME}/logs/..."
+
+
Property changes on: usertools/cio/falkon/falkon-start-bgp-logging.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: usertools/cio/tools/zoid/logging-script.sh
===================================================================
--- usertools/cio/tools/zoid/logging-script.sh (rev 0)
+++ usertools/cio/tools/zoid/logging-script.sh 2009-03-30 04:52:04 UTC (rev 2771)
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Added GPFS activity logging of IO calls -ame
+COBALT_JOBID=`echo $ZOID_JOB_ENV | sed 's/^.*COBALT_JOBID=\([^:]*\)/\1/'`
+
+TARGET_DIR=$HOME/performance/$COBALT_JOBID
+
+mkdir -p $TARGET_DIR || exit 1
+
+echo -n $TARGET_DIR/gpfs.`hostname` >/tmp/gpfs_log_name.new
+touch $TARGET_DIR/gpfs.`hostname` || exit 1
+mv /tmp/gpfs_log_name.new /tmp/gpfs_log_name
+echo -n $TARGET_DIR/zoid.`hostname` >/tmp/zoid_log_name.new
+mv /tmp/zoid_log_name.new /tmp/zoid_log_name
+# end addition
+
+#startup
+ if [ "${1}" -eq "1" ]; then
+
+#ERROR_LOG=/home/falkon/users/error.log
+ERROR_LOG=/dev/null
+
+ echo "starting zoid-user-script.sh..." >> ${ERROR_LOG}
+ #echo "reviewing the command and arguements: $0 $1 $2 $3 $4 $5 $6 $7..." >> ${ERROR_LOG}
+ #tENV=`env`
+ #echo "reviewing the envirnoment: $tENV..." >> ${ERROR_LOG}
+
+original=$ZOID_JOB_ARGS
+a0=${original%%:*}; rest=${original#*:}
+a1=${rest%%:*}; rest=${rest#*:}
+a2=${rest%%:*}; rest=${rest#*:}
+a3=${rest%%:*}; rest=${rest#*:}
+a4=${rest%%:*}; rest=${rest#*:}
+a5=${rest%%:*}; rest=${rest#*:}
+a6=${rest%%:*}; rest=${rest#*:}
+a7=${rest%%:*};
+
+
+ echo "reviewing the command and arguements: $a0 $a1 $a2 $a3 $a4 $a5 $a6 $a7..." >> ${ERROR_LOG}
+
+
+#FALKON_JOBID_HOME=/home/falkon/users/${USER}/${ZOID_JOB_ID}
+FALKON_JOBID_HOME=${a7}/users/${a5}/${a6}
+FALKON_HOME=${a7}/users/${a5}/${a6}
+ echo "FALKON_JOBID_HOME: $FALKON_JOBID_HOME" >> ${ERROR_LOG}
+
+
+if [ ! -d "${FALKON_JOBID_HOME}" ]; then
+ echo "ERROR: invalid path ${FALKON_JOBID_HOME}... exiting" >> ${ERROR_LOG}
+ exit 1
+fi
+
+
+
+ MACH_ID=`uname -n`
+ #this should be done via some configuration parameter
+
+ cd ${FALKON_JOBID_HOME}
+ #source falkon.env.bgp-io
+
+ echo "ION at ${MACH_ID} ==> ${FALKON_HOME}..." >> ${ERROR_LOG}
+
+ LOG_FILE_DEBUG="${FALKON_HOME}/logs/provisioner/STATUS_ION_${MACH_ID}"
+ DATE=`date`
+ echo "${DATE} : falkon-ion-start : starting the I/O node Falkon startup in the background..." >> ${LOG_FILE_DEBUG}
+
+
+if [ ! -e "${FALKON_HOME}/bin/falkon-ion-start-blocking-zeptocn.sh" ]; then
+ echo "ERROR: invalid script ${FALKON_HOME}/bin/falkon-ion-start-blocking-zeptocn.sh" >> ${ERROR_LOG}
+ exit 1
+fi
+
+ #used to run from GPFS
+ #${FALKON_JOBID_HOME}/bin/falkon-ion-start-blocking-zeptocn.sh ${FALKON_JOBID_HOME} &
+ #used to run from RAM
+ ${FALKON_JOBID_HOME}/bin/falkon-ion-start-blocking-zeptocn-ram.sh ${FALKON_JOBID_HOME} &
+
+
+ EXIT_CODE=$?
+ DATE=`date`
+
+ echo "${DATE} : falkon-ion-start : completed the I/O node Falkon startup in the background..." >> ${LOG_FILE_DEBUG}
+
+ echo "ION at ${MACH_ID} finished OK!" >> ${ERROR_LOG}
+
+
+exit ${EXIT_CODE}
+
+
+ fi
+
+
+
+#cleanup
+ if [ "${1}" -eq "0" ]; then
+
+ killall -9 java
+ exit 0
+
+ fi
+
Property changes on: usertools/cio/tools/zoid/logging-script.sh
___________________________________________________________________
Name: svn:executable
+ *
More information about the Swift-commit
mailing list