From noreply at svn.ci.uchicago.edu Wed Apr 1 16:24:57 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:24:57 -0500 (CDT) Subject: [Swift-commit] r2796 - usertools/cio/bin Message-ID: <20090401212457.3E8D0228223@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:24:55 -0500 (Wed, 01 Apr 2009) New Revision: 2796 Added: usertools/cio/bin/ciologic-bgp.sh usertools/cio/bin/falkon-start.sh Removed: usertools/cio/bin/cnip-work.sh usertools/cio/bin/torus-falkon-start-bgp-ram.sh usertools/cio/bin/zoid-user-script.sh Log: Cleanup of MTIO logic startup scripts Added: usertools/cio/bin/ciologic-bgp.sh =================================================================== --- usertools/cio/bin/ciologic-bgp.sh (rev 0) +++ usertools/cio/bin/ciologic-bgp.sh 2009-04-01 21:24:55 UTC (rev 2796) @@ -0,0 +1,130 @@ +#!/fuse/bin/bash + +# Script: ciologic-bgp.sh +# Description: starts ciologic for the BlueGene + +# Check sanity of environment + +if [ -z $CIOROOT ]; then + echo "CIOROOT not defined" + exit 1 +fi +if [ -z $CIOARCH ]; then + echo "CIOARCH note defined" + exit 1 +fi + +# BGP specific initialization + +LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib +PATH=/fuse/bin:/fuse/usr/bin:$PATH + +/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd.2409 & +while [ ! -f /tmp/ifconfig.cmd ]; do + sleep 1 +done +. /tmp/ifconfig.cmd + +# function: get_rank +# return the rank of the node this is running on + +get_rank(){ + echo $CONTROL_INIT | awk -F, '{print $4}' +} + +# function: get_ip [rank] +# given a rank of a node. return its ip address + +get_ip(){ + rank=$1 + echo "10.128.$(( rank / 256)).$((rank % 256))" +} + +ifs_head(){ + stripe_size=$1 + rank=`get_rank` + + mkdir -p /dev/shm/share/stripe/root + cat > /dev/shm/share/.__acl << EOF +address:192.168.1.* rwlda +address:10.* rwlda +address:127.0.0.1 rwlda +EOF + + $CHIRPROOT/bin/chirp_server -r /dev/shm/share + + # Enable striping + cat /dev/shm/share/.__acl > /dev/shm/share/stripe/.__acl + cat /dev/shm/share/.__acl > /dev/shm/share/stripe/root/.__acl + echo bigvolfiles > /dev/shm/share/stripe/key + for (( i = 0; i < stripe_size; i++ )); do + slave_rank=$((rank + i)) + echo `get_ip $slave_rank` >> /dev/shm/share/stripe/hosts + done + + mkdir -p /chirp + $CHIRPROOT/bin/chirp_fuse -a address /chirp + + ln -sf /chirp/`get_ip $rank` /dataifs +} + +# function: ifs_slave [head] +# Starts the supporting data nodes for the stripe to the +# [head] node. + +ifs_slave(){ + ifs_rank=$1 + # For Chirp, just start the server + mkdir -p /dev/shm/share + cat > /dev/shm/share/.__acl << EOF +address:192.168.1.* rwlda +address:10.* rwlda +address:127.0.0.1 rwlda +EOF + + $CHIRPROOT/bin/chirp_server -r /dev/shm/share +} + +ifs_mount() { + ifs_rank=$1 + # For Chirp, just start the server + mkdir -p /dev/shm/share + cat > /dev/shm/share/.__acl << EOF +address:192.168.1.* rwlda +address:10.* rwlda +address:127.0.0.1 rwlda +EOF + + $CHIRPROOT/bin/chirp_server -r /dev/shm/share & + mkdir /chirp + $CHIRPROOT/bin/chirp_fuse -a address /chirp +} + +PART_SIZE=$8 +IFS_NUM=$9 +STRIPE_SIZE=${10} +echo $PART_SIZE $IFS_NUM $STRIPE_SIZE + +RANK=`get_rank` +IFS_GRP=$(( PART_SIZE / IFS_NUM )) +IFS_RANK=$(( RANK / IFS_GRP + 1 )) +IFS_SLAVE=$(( IFS_RANK + STRIPE_SIZE )) + +export IFS_RANK +export CHIRP_ADD=`get_ip $IFS_RANK` + +echo "RANK: $RANK" "IFS_RANK: $IFS_RANK" + +#Core MTIO logic +if [ $RANK -eq 0 ]; then + $CIOROOT/libexec/hashserver.rb +elif [ $RANK -eq $IFS_RANK ]; then + ifs_head $STRIPE_SIZE + /home/zzhang/cio/bin/collector.sh +elif [ $RANK -lt $IFS_SLAVE ]; then + ifs_slave $IFS_RANK + sleep 3600 +else + ifs_mount $IFS_RANK + $CIOROOT/libexec/falkon/runworker-bgp.sh $1 $2 $3 $4 $5 $6 $7 +fi Property changes on: usertools/cio/bin/ciologic-bgp.sh ___________________________________________________________________ Name: svn:executable + * Deleted: usertools/cio/bin/cnip-work.sh =================================================================== --- usertools/cio/bin/cnip-work.sh 2009-04-01 05:20:45 UTC (rev 2795) +++ usertools/cio/bin/cnip-work.sh 2009-04-01 21:24:55 UTC (rev 2796) @@ -1,69 +0,0 @@ -#!/fuse/bin/bash -LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib -PATH=/fuse/bin:/fuse/usr/bin:$PATH - -#Torus START -/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd.2409 & - -while [ ! -f /tmp/ifconfig.cmd ]; do - sleep 1 -done - -. /tmp/ifconfig.cmd - -#IP Calculation -RANK=`echo $CONTROL_INIT | awk -F, '{print $4}'` -FIRST=`expr $RANK % 256` -SECOND=`expr $RANK / 256` - -BASE=`expr $FIRST / 64` -MODULO=`expr $FIRST % 64` -CHIRP_BASE=`expr $BASE \* 64` -CHIRP_1=`expr $CHIRP_BASE + 1` -CHIRP_2=`expr $CHIRP_BASE + 2` -CHIRP_3=`expr $CHIRP_BASE + 3` -CHIRP_4=`expr $CHIRP_BASE + 4` -export CHIRP_ADD=10.128.$SECOND.$CHIRP_1 -#Chirp START -if [ -d /dev/shm/share ]; - then - rm -rf /dev/shm/share -fi -mkdir /dev/shm/share -echo "address:192.168.1.* rwlda" >>/dev/shm/share/.__acl -echo "address:10.* rwlda" >> /dev/shm/share/.__acl -echo "address:127.0.0.1 rwlda" >> /dev/shm/share/.__acl - -/home/zzhang/chirp/bin/chirp_server -r /dev/shm/share & - -#MAIN -if [ "$RANK" = "0" ] -then - /home/zzhang/cio/bin/hashserver.rb & -elif [ "$MODULO" = "1" ] -then - cd /dev/shm/share - mkdir stripe - cd stripe - cp ../.__acl . - mkdir root - cp .__acl root/ - echo 10.128.$SECOND.$CHIRP_1 >> hosts - echo 10.128.$SECOND.$CHIRP_2 >> hosts - echo 10.128.$SECOND.$CHIRP_3 >> hosts - echo 10.128.$SECOND.$CHIRP_4 >> hosts - echo bigvolfiles > key - - mkdir /chirp - /home/zzhang/chirp/bin/chirp_fuse -a address /chirp - /home/zzhang/cio/bin/collector.sh & -elif [ "$MODULO" = "2" ] || [ "$MODULO" = "3" ] || [ "$MODULO" = "4" ] -then - sleep 3600 -else - mkdir /chirp - /home/zzhang/chirp/bin/chirp_fuse -a address /chirp - /home/zzhang/falkon/worker/run.worker-c-bgp.sh $1 $2 $3 $4 $5 $6 $7 -fi - -sleep 3600 Added: usertools/cio/bin/falkon-start.sh =================================================================== --- usertools/cio/bin/falkon-start.sh (rev 0) +++ usertools/cio/bin/falkon-start.sh 2009-04-01 21:24:55 UTC (rev 2796) @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ -z $CIOROOT ]; then + echo "ERROR: CIOROOT env not defined" + exit 1 +fi +if [ -z $CIOARCH ]; then + echo "ERROR: CIOARCH not defined" + exit 1 +fi +if [ -z $CHIRPROOT ]; then + echo "ERROR: CHIRPROOT not defined" + exit 1 +fi +$CIOROOT/libexec/falkon/falkon-start-$CIOARCH.sh Property changes on: usertools/cio/bin/falkon-start.sh ___________________________________________________________________ Name: svn:executable + * Deleted: usertools/cio/bin/torus-falkon-start-bgp-ram.sh =================================================================== --- usertools/cio/bin/torus-falkon-start-bgp-ram.sh 2009-04-01 05:20:45 UTC (rev 2795) +++ usertools/cio/bin/torus-falkon-start-bgp-ram.sh 2009-04-01 21:24:55 UTC (rev 2796) @@ -1,174 +0,0 @@ -#!/bin/bash - - - if [ -z "$3" ]; then - echo "usage: $0 " - echo "usage: $0 prod 1024 60" - echo "-- or --" - echo "usage: $0 " - echo "usage: $0 prod 1024 60 4" - exit 1 - fi - - -if [ -z "${FALKON_HOME}" ]; then - echo "ERROR: environment variable FALKON_HOME not defined" - exit 1 -fi - - -QUEUE_NAME=$1 -PROFILE_NAME="zeptocn" -NUM_NODES=$2 -let NUM_ION=NUM_NODES/64 -MAX_TIME_MIN=$3 -SERVICE_IP="192.168.1.254" -SERVICE_PORT1=55000 -SERVICE_PORT2=55001 -WORKERS_PER_NODE=4 -if [ ! -z $4 ];then - WORKERS_PER_NODE=$4 -fi - - -cp /home/zzhang/cio/bin/zoid-user-script.sh ${HOME}/zoid-user-script.sh -chmod +x ${HOME}/zoid-user-script.sh - -FALKON_JOB_ID=`falkon-id-get.sh N/A` -EXIT_CODE=$? - - - if [ "${EXIT_CODE}" -ne "0" ]; then - echo "Error in geting a unique falkon ID.. 'falkon-get-id.sh N/A'" - cqdel ${ZOID_JOB_ID} - exit ${EXIT_CODE} - fi - - - -#ZOID_JOB_ID=`cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib /bgsys/linux/1.2.020080512/bin/bash /home/zzhang/falkon/worker/run.worker-c-bgp.sh ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT}` - -ZOID_JOB_ID=`cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib /bgsys/linux/1.2.020080512/bin/bash /home/zzhang/cio/bin/cnip-work.sh ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT}` - -#ZOID_JOB_ID=`cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib /bin/sleep 1800` - -EXIT_CODE=$? - - if [ "${EXIT_CODE}" -ne "0" ]; then - echo "Error in submitting job to Cobalt.. 'cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib /bgsys/linux/1.2.020080512/bin/bash /fuse/${FALKON_WORKER_HOME}/run.worker-c-bgp.sh ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT}' " - exit ${EXIT_CODE} - fi - -falkon-id-update.sh ${FALKON_JOB_ID} ${ZOID_JOB_ID} - -EXIT_CODE=$? - - if [ "${EXIT_CODE}" -ne "0" ]; then - echo "Error in updating cobalt job info for falkon.. 'falkon-update-id.sh ${ZOID_JOB_ID}'" - cqdel ${ZOID_JOB_ID} - exit ${EXIT_CODE} - fi - - -FALKON_JOBID_HOME=${FALKON_ROOT}/users/${USER}/${FALKON_JOB_ID} - -echo "Submitted job ${ZOID_JOB_ID} to Cobalt, creating the job specific Falkon tree for logs and configuration in ${FALKON_JOBID_HOME}..." - -mkdir -p ${FALKON_JOBID_HOME} - -cp ${FALKON_HOME}/falkon.env.bgp* ${FALKON_JOBID_HOME}/ -cp -r ${FALKON_HOME}/config ${FALKON_JOBID_HOME}/ -cp ${FALKON_JOBID_HOME}/config/Client-service-URIs.config2 ${FALKON_JOBID_HOME}/config/Client-service-URIs.config -mkdir -p ${FALKON_JOBID_HOME}/logs/client ${FALKON_JOBID_HOME}/logs/service ${FALKON_JOBID_HOME}/logs/provisioner ${FALKON_JOBID_HOME}/logs/worker - - -DATE=`date +%s` -echo "$DATE: pre-creating log dirs for Falkon service..." -RACK_START=0 -RACK_END=48 -SEQUENCE_DIR=`seq -w ${RACK_START} ${RACK_END}` -PSET_START=1 -PSET_END=16 -for a in ${SEQUENCE_DIR} -do - for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$". - do - DIR_NAME="ion-R${a}-${b}" - mkdir -p ${FALKON_JOBID_HOME}/logs/service/$DIR_NAME - done -done - -for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$". -do - DIR_NAME="ion-${b}" - mkdir -p ${FALKON_JOBID_HOME}/logs/service/$DIR_NAME -done - - -DATE=`date +%s` -echo "$DATE: done creating log dirs for Falkon service!" - - - -DATE=`date +%s` -echo "$DATE: pre-creating data dirs for Falkon service..." -RACK_START=0 -RACK_END=48 -SEQUENCE_DIR=`seq -w ${RACK_START} ${RACK_END}` -PSET_START=1 -PSET_END=16 -for a in ${SEQUENCE_DIR} -do - for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$". - do - DIR_NAME="ion-R${a}-${b}" - mkdir -p ${FALKON_JOBID_HOME}/data/$DIR_NAME - done -done - -for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$". -do - DIR_NAME="ion-${b}" - mkdir -p ${FALKON_JOBID_HOME}/data/$DIR_NAME -done - - -DATE=`date +%s` -echo "$DATE: done creating data dirs for Falkon service!" - - - -FALKON_HOME_RAM=/tmp/${USER}/falkon - -ln -s ${FALKON_HOME}/apps ${FALKON_JOBID_HOME}/apps -ln -s ${FALKON_HOME_RAM}/container ${FALKON_JOBID_HOME}/container -ln -s ${FALKON_HOME}/service ${FALKON_JOBID_HOME}/service -ln -s ${FALKON_HOME}/worker ${FALKON_JOBID_HOME}/worker -ln -s ${FALKON_HOME}/AstroPortal ${FALKON_JOBID_HOME}/AstroPortal -ln -s ${FALKON_HOME}/client ${FALKON_JOBID_HOME}/client -ln -s ${FALKON_HOME}/monitor ${FALKON_JOBID_HOME}/monitor -ln -s ${FALKON_HOME}/bin ${FALKON_JOBID_HOME}/bin -ln -s ${FALKON_HOME}/config ${FALKON_JOBID_HOME}/config -ln -s ${FALKON_HOME}/ploticus ${FALKON_JOBID_HOME}/ploticus -ln -s ${FALKON_HOME}/webserver ${FALKON_JOBID_HOME}/webserver -ln -s ${FALKON_HOME}/workloads ${FALKON_JOBID_HOME}/workloads -ln -s ${FALKON_HOME}/id ${FALKON_JOBID_HOME}/id -ln -s ${FALKON_HOME}/apache-ant-1.7.0 ${FALKON_JOBID_HOME}/apache-ant-1.7.0 -ln -s ${FALKON_HOME}/ibm-java2-ppc64-50 ${FALKON_JOBID_HOME}/ibm-java2-ppc64-50 -ln -s ${FALKON_HOME_RAM}/ibm-java2-ppc-50 ${FALKON_JOBID_HOME}/ibm-java2-ppc-50 -ln -s ${FALKON_HOME}/falkon.tgz ${FALKON_JOBID_HOME}/falkon.tgz - - -if [ ! -d "${FALKON_JOBID_HOME}" ]; then - echo "ERROR: invalid path ${FALKON_JOBID_HOME}... exiting" - cqdel ${ZOID_JOB_ID} - exit 1 -fi - -echo "Succesfully submitted the job to Cobalt, and setup job specific Falkon tree!" -echo "To monitor the job status, type 'cqstat | grep ${USER}'; once it is in running state, you can use the Falkon specific command ...." -echo "To submit your Falkon-based workload, type: ....; you can do this any time, the falkon workload will wait for the resources to come online, and will only be submitted when everything is ready; the script is run in the background, so the workload will run even if the ssh session gets disconnected." -echo "" -echo "Remember, your job id is ${ZOID_JOB_ID}, and if you need to look through the logs manually for anything, remember that you can find them at ${HOME}/${ZOID_JOB_ID}.output, ${HOME}/${ZOID_JOB_ID}.error, and ${FALKON_JOBID_HOME}/logs/..." - - Deleted: usertools/cio/bin/zoid-user-script.sh =================================================================== --- usertools/cio/bin/zoid-user-script.sh 2009-04-01 05:20:45 UTC (rev 2795) +++ usertools/cio/bin/zoid-user-script.sh 2009-04-01 21:24:55 UTC (rev 2796) @@ -1,90 +0,0 @@ -#!/bin/bash - -#IP=`hostname -i` -#echo ${IP} >> /home/falkon/falkon/logs/provisioner/zoid-log.txt - -#startup - if [ "${1}" -eq "1" ]; then - -#ERROR_LOG=/home/falkon/users/error.log -ERROR_LOG=/dev/null - - echo "starting zoid-user-script.sh..." >> ${ERROR_LOG} - #echo "reviewing the command and arguements: $0 $1 $2 $3 $4 $5 $6 $7..." >> ${ERROR_LOG} - #tENV=`env` - #echo "reviewing the envirnoment: $tENV..." >> ${ERROR_LOG} - -original=$ZOID_JOB_ARGS -a0=${original%%:*}; rest=${original#*:} -a1=${rest%%:*}; rest=${rest#*:} -a2=${rest%%:*}; rest=${rest#*:} -a3=${rest%%:*}; rest=${rest#*:} -a4=${rest%%:*}; rest=${rest#*:} -a5=${rest%%:*}; rest=${rest#*:} -a6=${rest%%:*}; rest=${rest#*:} -a7=${rest%%:*}; - - - echo "reviewing the command and arguements: $a0 $a1 $a2 $a3 $a4 $a5 $a6 $a7..." >> ${ERROR_LOG} - - -#FALKON_JOBID_HOME=/home/falkon/users/${USER}/${ZOID_JOB_ID} -FALKON_JOBID_HOME=${a7}/users/${a5}/${a6} -FALKON_HOME=${a7}/users/${a5}/${a6} - echo "FALKON_JOBID_HOME: $FALKON_JOBID_HOME" >> ${ERROR_LOG} -MACH_ID=`uname -n` - #this should be done via some configuration parameter - -echo ${FALKON_JOBID_HOME}/data/${MACH_ID} >> /tmp/iohost -if [ ! -d "${FALKON_JOBID_HOME}" ]; then - echo "ERROR: invalid path ${FALKON_JOBID_HOME}... exiting" >> ${ERROR_LOG} - exit 1 -fi - - - - - cd ${FALKON_JOBID_HOME} - #source falkon.env.bgp-io - - echo "ION at ${MACH_ID} ==> ${FALKON_HOME}..." >> ${ERROR_LOG} - - LOG_FILE_DEBUG="${FALKON_HOME}/logs/provisioner/STATUS_ION_${MACH_ID}" - DATE=`date` - echo "${DATE} : falkon-ion-start : starting the I/O node Falkon startup in the background..." >> ${LOG_FILE_DEBUG} - - -if [ ! -e "${FALKON_HOME}/bin/falkon-ion-start-blocking-zeptocn.sh" ]; then - echo "ERROR: invalid script ${FALKON_HOME}/bin/falkon-ion-start-blocking-zeptocn.sh" >> ${ERROR_LOG} - exit 1 -fi - - #used to run from GPFS - #${FALKON_JOBID_HOME}/bin/falkon-ion-start-blocking-zeptocn.sh ${FALKON_JOBID_HOME} & - #used to run from RAM - ${FALKON_JOBID_HOME}/bin/falkon-ion-start-blocking-zeptocn-ram.sh ${FALKON_JOBID_HOME} & - - - EXIT_CODE=$? - DATE=`date` - - echo "${DATE} : falkon-ion-start : completed the I/O node Falkon startup in the background..." >> ${LOG_FILE_DEBUG} - - echo "ION at ${MACH_ID} finished OK!" >> ${ERROR_LOG} - - -exit ${EXIT_CODE} - - - fi - - - -#cleanup - if [ "${1}" -eq "0" ]; then - - killall -9 java - exit 0 - - fi - From noreply at svn.ci.uchicago.edu Wed Apr 1 16:26:00 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:26:00 -0500 (CDT) Subject: [Swift-commit] r2797 - in usertools/cio: . libexec libexec/falkon Message-ID: <20090401212600.D34BE228223@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:26:00 -0500 (Wed, 01 Apr 2009) New Revision: 2797 Added: usertools/cio/libexec/ usertools/cio/libexec/falkon/ usertools/cio/libexec/falkon/runworker-bgp.sh usertools/cio/libexec/hashserver.rb Removed: usertools/cio/libexec/falkon/ciologic-bgp.sh Modified: usertools/cio/libexec/falkon/cobaltrun.job usertools/cio/libexec/falkon/falkon-start-bgp-cio.sh Log: Inserted libexec directories. Scripts that are indirectly called are placed in here Copied: usertools/cio/libexec/falkon (from rev 2793, usertools/cio/falkon) Deleted: usertools/cio/libexec/falkon/ciologic-bgp.sh =================================================================== --- usertools/cio/falkon/ciologic-bgp.sh 2009-04-01 01:26:56 UTC (rev 2793) +++ usertools/cio/libexec/falkon/ciologic-bgp.sh 2009-04-01 21:26:00 UTC (rev 2797) @@ -1,136 +0,0 @@ -#!/fuse/bin/bash - -CIOROOT=/home/espinosa/cio -ERROR_LOG=/dev/null - -if [ $# -lt 7 ]; then - cat << EOF -Usage: $0 \ - -Example: $0 192.168.1.254 55000 55001 4 iraicu 0001 [FALKON_ROOT] \ - 1024 1 1" -EOF - exit 2 -fi - -# Initiate the IP stack of the Torus network -# -aespinosa - -/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd.2409 & - -while [ ! -f /tmp/ifconfig.cmd ]; do - sleep 1 -done - -. /tmp/ifconfig.cmd - -RANK=`/home/espinosa/bin/rank` - -# size of the allocated partition. also known as the total number of CNs -PART_SIZE=$8 -# number of data CNs in partition -DATA_NUM=$9 -STR=${10} - -ROLE=`/home/espinosa/bin/isdata $PART_SIZE $DATA_NUM $STR` -ISDATA=$ROLE -tmp=`/home/espinosa/bin/getdata $PART_SIZE $DATA_NUM $STR` -DATACN=`/fuse/gpfs/home/espinosa/bin/ipcal 10.128.0.0 $tmp` - -if [ $RANK -eq 0 ]; then - # Run Central Hash Service - $CIOROOT/tools/hashserver.rb -elif [ "$ISDATA" != 'app' ]; then - if [ "$ROLE" = 'head' ]; then - # Run head IFS service - $CIOROOT/tools/ifshead.sh $DATACN $STR >&2 - else - # Contribute stripes to the IFS service - $CIOROOT/tools/ifsslave.sh $DATACN >&2 - fi -else - /bin/mkdir /dataifs - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/lib:/fuse/lib:/usr/lib - sleep 30 - # Contact and mount assigned IFS head - $CIOROOT/tools/ifsmount.sh $DATACN >&2 - - echo "worker started..." >> ${ERROR_LOG} - - # Regular Falkon worker startup script - - FALKON_SERVICE_IP=$1 - TCPCORE_PORT1=$2 - TCPCORE_PORT2=$3 - NUM_WORKERS=$4 - USER_ID=$5 - FALKON_JOB_ID=$6 - FALKON_ROOT_PATH=$7 - - - FALKON_HOME=${FALKON_ROOT_PATH}/users/${USER_ID}/${FALKON_JOB_ID} - - echo "FALKON_HOME = ${FALKON_HOME}" >> ${ERROR_LOG} - - #FALKON_HOME=/home/falkon/falkon - - HOME=${FALKON_ROOT_PATH} - echo "HOME = ${HOME}" >> ${ERROR_LOG} - #HOME=${FALKON_HOME} - - if [ -z "${FALKON_HOME}" ]; then - echo "ERROR: environment variable FALKON_HOME not defined" - echo "ERROR: environment variable FALKON_HOME not defined" >> ${ERROR_LOG} - exit 1 - fi - - if [ ! -d "${FALKON_HOME}" ]; then - echo "ERROR: invalid FALKON_HOME set: $FALKON_HOME" - echo "ERROR: invalid FALKON_HOME set: $FALKON_HOME" >> ${ERROR_LOG} - exit 1 - fi - - cd ${FALKON_HOME} - source falkon.env.bgp-cn - - - - HOME_PATH=`pwd` - export MACH_ID=`uname -i` - RAND_ID2=`/home/espinosa/bin/rank` - - a0=${RAND_ID2%%=*}; rest=${RAND_ID2#*=} - RAND_ID=${rest%%=*}; - - #PLATFORM=`uname -m` - - - - EXP_START=`date +%Y.%m.%d_%H.%M.%S` - - - for ((b=1; b < ${NUM_WORKERS} ; b++)) - do - #LOG_FILE=${FALKON_LOGS}/worker/${RAND_ID}.${b}.${EXP_START}.txt - LOG_FILE=/dev/null - - #${FALKON_WORKER_HOME}/src-c/BGexec_${PLATFORM} ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf >> "${LOG_FILE}" 2>&1 & - #echo "starting .... ${FALKON_WORKER_HOME}/src-c/BGexec ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf" - echo "starting worker ${RAND_ID} CPU ${b}..." >> ${ERROR_LOG} - ${FALKON_WORKER_HOME}/src-c/BGexec ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf >> ${LOG_FILE} 2>&1 & - #${HOME_PATH}/src-c/BGexec_${PLATFORM} ${FALKON_SERVICE_IP} 55000 55001 -debug - /bin/sleep 1 - done - - #LOG_FILE=${FALKON_LOGS}/worker/${RAND_ID}.${NUM_WORKERS}.${EXP_START}.txt - LOG_FILE=/dev/null - echo "starting worker ${RAND_ID} CPU ${NUM_WORKERS}..." >> ${ERROR_LOG} - - #${FALKON_WORKER_HOME}/src-c/BGexec_${PLATFORM} ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf >> "${LOG_FILE}" 2>&1 - #echo "starting .... ${FALKON_WORKER_HOME}/src-c/BGexec ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf" - - ${FALKON_WORKER_HOME}/src-c/BGexec ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf >> ${LOG_FILE} 2>&1 - - - echo "finished!" >> ${ERROR_LOG} -fi Modified: usertools/cio/libexec/falkon/cobaltrun.job =================================================================== --- usertools/cio/falkon/cobaltrun.job 2009-04-01 01:26:56 UTC (rev 2793) +++ usertools/cio/libexec/falkon/cobaltrun.job 2009-04-01 21:26:00 UTC (rev 2797) @@ -1 +1 @@ -./falkon-start-bgp-logging.sh default 64 60 4 +./falkon-start-bgp-cio.sh default 64 15 4 1 3 Modified: usertools/cio/libexec/falkon/falkon-start-bgp-cio.sh =================================================================== --- usertools/cio/falkon/falkon-start-bgp-cio.sh 2009-04-01 01:26:56 UTC (rev 2793) +++ usertools/cio/libexec/falkon/falkon-start-bgp-cio.sh 2009-04-01 21:26:00 UTC (rev 2797) @@ -1,7 +1,5 @@ #!/bin/bash -CIOROOT=/home/espinosa/cio - if [ $# -lt 3 ]; then cat << EOF Usage: $0 @@ -17,6 +15,14 @@ echo "ERROR: environment variable FALKON_HOME not defined" exit 1 fi +if [ -z $CIOROOT ]; then + echo "ERROR: CIOROOT env not defined" + exit 1 +fi +if [[ $CIOARCH != "bgp" ]]; then + echo "ERROR: Wrong architecture. Must be bgp" + exit 1 +fi PROFILE_NAME="zeptocn" QUEUE_NAME=$1 @@ -30,17 +36,17 @@ WORKERS_PER_NODE=4 DATA_NUM=1 STR=1 -if [ -n $4 ];then +if [ -n "$4" ];then WORKERS_PER_NODE=$4 fi -if [ -n $5 ];then +if [ -n "$5" ];then DATA_NUM=$5 fi -if [ -n $6 ];then +if [ -n "$6" ];then STR=$6 + echo true fi - cp ${FALKON_HOME}/bin/zoid-user-script.sh ${HOME}/zoid-user-script.sh chmod +x ${HOME}/zoid-user-script.sh @@ -55,13 +61,14 @@ -WORKER_SCRIPT=$CIOROOT/falkon/ciologic-bgp.sh +WORKER_SCRIPT=$CIOROOT/bin/ciologic-$CIOARCH.sh ZOID_JOB_ID=`cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t \ ${MAX_TIME_MIN} -n ${NUM_NODES} \ - -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib \ + -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib:CIOROOT=$CIOROOT:CIOARCH=$CIOARCH:CHIRPROOT=$CHIRPROOT \ /bgsys/linux/1.2.020080512/bin/bash $WORKER_SCRIPT \ ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} \ - ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT} ${NUM_NODES} ${DATA_NUM} ${STR}` + ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT} \ + $NUM_NODES $DATA_NUM $STR` # MTIO parameters EXIT_CODE=$? Added: usertools/cio/libexec/falkon/runworker-bgp.sh =================================================================== --- usertools/cio/libexec/falkon/runworker-bgp.sh (rev 0) +++ usertools/cio/libexec/falkon/runworker-bgp.sh 2009-04-01 21:26:00 UTC (rev 2797) @@ -0,0 +1,100 @@ +#!/fuse/bin/bash + +#source /home/iraicu/.bashrc + +#ERROR_LOG=${FALKON_ROOT_PATH}/users/error.log +#ERROR_LOG=/home/falkon/users/error-workers.log +ERROR_LOG=/dev/null + echo "worker started..." >> ${ERROR_LOG} + + + if [ -z "$7" ]; then + echo "usage: $0 " + echo "usage: $0 " >> ${ERROR_LOG} + echo "$0 192.168.1.254 55000 55001 4 iraicu 0001" + echo "$0 192.168.1.254 55000 55001 4 iraicu 0001" >> ${ERROR_LOG} + exit 2 + fi + +FALKON_SERVICE_IP=$1 +TCPCORE_PORT1=$2 +TCPCORE_PORT2=$3 +NUM_WORKERS=$4 +USER_ID=$5 +FALKON_JOB_ID=$6 +FALKON_ROOT_PATH=$7 + + + ln -s /fuse/gpfs1 /gpfs1 + #FALKON_HOME=/gpfs1/falkon/falkon + #HOME=/gpfs1/falkon + + + + #echo "Worker for job id ${ZOID_JOB_ID} starting..." >> ${ERROR_LOG} + + FALKON_HOME=${FALKON_ROOT_PATH}/users/${USER_ID}/${FALKON_JOB_ID} + + echo "FALKON_HOME = ${FALKON_HOME}" >> ${ERROR_LOG} + + #FALKON_HOME=/home/falkon/falkon + + HOME=${FALKON_ROOT_PATH} + echo "HOME = ${HOME}" >> ${ERROR_LOG} + #HOME=${FALKON_HOME} + + if [ -z "${FALKON_HOME}" ]; then + echo "ERROR: environment variable FALKON_HOME not defined" + echo "ERROR: environment variable FALKON_HOME not defined" >> ${ERROR_LOG} + exit 1 + fi + + if [ ! -d "${FALKON_HOME}" ]; then + echo "ERROR: invalid FALKON_HOME set: $FALKON_HOME" + echo "ERROR: invalid FALKON_HOME set: $FALKON_HOME" >> ${ERROR_LOG} + exit 1 + fi + +cd ${FALKON_HOME} +source falkon.env.bgp-cn + + + + +HOME_PATH=`pwd` +#export MACH_ID=`uname -i` +RAND_ID2=`grep BG_RANK /proc/personality.sh` + +a0=${RAND_ID2%%=*}; rest=${RAND_ID2#*=} +RAND_ID=${rest%%=*}; + +#PLATFORM=`uname -m` + + + +EXP_START=`date +%Y.%m.%d_%H.%M.%S` + +for ((b=1; b < ${NUM_WORKERS} ; b++)) +do + #LOG_FILE=${FALKON_LOGS}/worker/${RAND_ID}.${b}.${EXP_START}.txt + LOG_FILE=/dev/null + + #${FALKON_WORKER_HOME}/src-c/BGexec_${PLATFORM} ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf >> "${LOG_FILE}" 2>&1 & + #echo "starting .... ${FALKON_WORKER_HOME}/src-c/BGexec ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf" + echo "starting worker ${RAND_ID} CPU ${b}..." >> ${ERROR_LOG} + /home/zzhang/falkon/worker/src-c/BGexec_new ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf >> ${LOG_FILE} 2>&1 & + #${HOME_PATH}/src-c/BGexec_${PLATFORM} ${FALKON_SERVICE_IP} 55000 55001 -debug +/bin/sleep 1 +done + + #LOG_FILE=${FALKON_LOGS}/worker/${RAND_ID}.${NUM_WORKERS}.${EXP_START}.txt + LOG_FILE=/dev/null + echo "starting worker ${RAND_ID} CPU ${NUM_WORKERS}..." >> ${ERROR_LOG} + + #${FALKON_WORKER_HOME}/src-c/BGexec_${PLATFORM} ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf >> "${LOG_FILE}" 2>&1 + #echo "starting .... ${FALKON_WORKER_HOME}/src-c/BGexec ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf" + /home/zzhang/falkon/worker/src-c/BGexec_new ${FALKON_SERVICE_IP} ${TCPCORE_PORT1} ${TCPCORE_PORT2} -perf >> ${LOG_FILE} 2>&1 + + + echo "finished!" >> ${ERROR_LOG} + Property changes on: usertools/cio/libexec/falkon/runworker-bgp.sh ___________________________________________________________________ Name: svn:executable + * Copied: usertools/cio/libexec/hashserver.rb (from rev 2793, usertools/cio/tools/hashserver.rb) =================================================================== --- usertools/cio/libexec/hashserver.rb (rev 0) +++ usertools/cio/libexec/hashserver.rb 2009-04-01 21:26:00 UTC (rev 2797) @@ -0,0 +1,26 @@ +#!/home/espinosa/local/bin/ruby + +require 'drb' + +class HashServer + attr_reader :file + + def initialize + @file = Hash.new { |hash, key| hash[key] = Array.new } + end + + def put(fname, rank) + @file[fname] << rank + end + + def get(fname) + @file[fname][ rand( @file[fname].size ) ] + end + +end + +server = HashServer.new + +DRb.start_service('druby://*:9000', server) +DRb.thread.join + From noreply at svn.ci.uchicago.edu Wed Apr 1 16:26:27 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:26:27 -0500 (CDT) Subject: [Swift-commit] r2798 - usertools/cio/tools Message-ID: <20090401212627.C2CC7228223@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:26:27 -0500 (Wed, 01 Apr 2009) New Revision: 2798 Removed: usertools/cio/tools/hashserver.rb Log: Removed "tools" directory and moved to libexec Deleted: usertools/cio/tools/hashserver.rb =================================================================== --- usertools/cio/tools/hashserver.rb 2009-04-01 21:26:00 UTC (rev 2797) +++ usertools/cio/tools/hashserver.rb 2009-04-01 21:26:27 UTC (rev 2798) @@ -1,26 +0,0 @@ -#!/home/espinosa/local/bin/ruby - -require 'drb' - -class HashServer - attr_reader :file - - def initialize - @file = {} - end - - def put(fname, rank) - @file[fname] = rank - end - - def get(fname) - @file[fname] - end - -end - -server = HashServer.new - -DRb.start_service('druby://*:9000', server) -DRb.thread.join - From noreply at svn.ci.uchicago.edu Wed Apr 1 16:27:09 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:27:09 -0500 (CDT) Subject: [Swift-commit] r2799 - in usertools/cio: . etc Message-ID: <20090401212709.5CFB9228223@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:27:09 -0500 (Wed, 01 Apr 2009) New Revision: 2799 Added: usertools/cio/etc/ usertools/cio/etc/cio-env.sh Log: Inserted a CIO configuration file which should be sourced everytime we use the utilities in this tree Added: usertools/cio/etc/cio-env.sh =================================================================== --- usertools/cio/etc/cio-env.sh (rev 0) +++ usertools/cio/etc/cio-env.sh 2009-04-01 21:27:09 UTC (rev 2799) @@ -0,0 +1,4 @@ +#!/bin/bash +export CIOROOT="/home/espinosa/cio" +export CIOARCH="bgp" +export CHIRPROOT=/home/zzhang/chirp Property changes on: usertools/cio/etc/cio-env.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Wed Apr 1 16:27:35 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:27:35 -0500 (CDT) Subject: [Swift-commit] r2800 - usertools/cio Message-ID: <20090401212735.EE08D228223@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:27:35 -0500 (Wed, 01 Apr 2009) New Revision: 2800 Removed: usertools/cio/falkon/ Log: Deleted falkon and moved to libexec From noreply at svn.ci.uchicago.edu Wed Apr 1 16:29:16 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:29:16 -0500 (CDT) Subject: [Swift-commit] r2801 - usertools/cio/science/blast Message-ID: <20090401212916.D8DF8228220@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:29:16 -0500 (Wed, 01 Apr 2009) New Revision: 2801 Modified: usertools/cio/science/blast/blast.swift Log: BLAST workflow now working with mockblast too Modified: usertools/cio/science/blast/blast.swift =================================================================== --- usertools/cio/science/blast/blast.swift 2009-04-01 21:27:35 UTC (rev 2800) +++ usertools/cio/science/blast/blast.swift 2009-04-01 21:29:16 UTC (rev 2801) @@ -27,8 +27,10 @@ BlastQuery input[] ; BlastQuery in_test <"test.in">; +BlastResult out_test ; -out[0] = blastall(in_test, pir); + +out_test = blastall(in_test, pir); /* foreach data ,i in input { (out[i],err[i]) = blastall(data, pir); From noreply at svn.ci.uchicago.edu Wed Apr 1 16:30:02 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:30:02 -0500 (CDT) Subject: [Swift-commit] r2802 - usertools/cio/libexec/falkon Message-ID: <20090401213002.0B342228220@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:30:01 -0500 (Wed, 01 Apr 2009) New Revision: 2802 Added: usertools/cio/libexec/falkon/falkon-start-bgp.sh Removed: usertools/cio/libexec/falkon/falkon-start-bgp-cio.sh Log: Removed '-cio' suffix to falkon-start scripts Deleted: usertools/cio/libexec/falkon/falkon-start-bgp-cio.sh =================================================================== --- usertools/cio/libexec/falkon/falkon-start-bgp-cio.sh 2009-04-01 21:29:16 UTC (rev 2801) +++ usertools/cio/libexec/falkon/falkon-start-bgp-cio.sh 2009-04-01 21:30:01 UTC (rev 2802) @@ -1,167 +0,0 @@ -#!/bin/bash - -if [ $# -lt 3 ]; then - cat << EOF -Usage: $0 -Example: $0 prod 1024 60 ---or-- -Usage: $0 \ - -EOF - exit 1 -fi - -if [ -z $FALKON_HOME ]; then - echo "ERROR: environment variable FALKON_HOME not defined" - exit 1 -fi -if [ -z $CIOROOT ]; then - echo "ERROR: CIOROOT env not defined" - exit 1 -fi -if [[ $CIOARCH != "bgp" ]]; then - echo "ERROR: Wrong architecture. Must be bgp" - exit 1 -fi - -PROFILE_NAME="zeptocn" -QUEUE_NAME=$1 -NUM_NODES=$2 -let NUM_ION=NUM_NODES/64 -MAX_TIME_MIN=$3 -SERVICE_IP="192.168.1.254" -SERVICE_PORT1=55000 -SERVICE_PORT2=55001 - -WORKERS_PER_NODE=4 -DATA_NUM=1 -STR=1 -if [ -n "$4" ];then - WORKERS_PER_NODE=$4 -fi -if [ -n "$5" ];then - DATA_NUM=$5 -fi -if [ -n "$6" ];then - STR=$6 - echo true -fi - -cp ${FALKON_HOME}/bin/zoid-user-script.sh ${HOME}/zoid-user-script.sh -chmod +x ${HOME}/zoid-user-script.sh - -FALKON_JOB_ID=`falkon-id-get.sh N/A` -EXIT_CODE=$? - -if [ "${EXIT_CODE}" -ne "0" ]; then - echo "Error in geting a unique falkon ID.. 'falkon-get-id.sh N/A'" - cqdel ${ZOID_JOB_ID} - exit ${EXIT_CODE} -fi - - - -WORKER_SCRIPT=$CIOROOT/bin/ciologic-$CIOARCH.sh -ZOID_JOB_ID=`cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t \ - ${MAX_TIME_MIN} -n ${NUM_NODES} \ - -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib:CIOROOT=$CIOROOT:CIOARCH=$CIOARCH:CHIRPROOT=$CHIRPROOT \ - /bgsys/linux/1.2.020080512/bin/bash $WORKER_SCRIPT \ - ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} \ - ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT} \ - $NUM_NODES $DATA_NUM $STR` # MTIO parameters - -EXIT_CODE=$? - -if [ "${EXIT_CODE}" -ne "0" ]; then - echo "Error in submitting job to Cobalt.. 'cqsub -q ${QUEUE_NAME} \ - -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} \ - -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib \ - /bgsys/linux/1.2.020080512/bin/bash $WORKER_SCRIPT \ - ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} \ - ${WORKERS_PER_NODE} ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT}' " - exit ${EXIT_CODE} -fi - -falkon-id-update.sh ${FALKON_JOB_ID} ${ZOID_JOB_ID} - -EXIT_CODE=$? - -if [ "${EXIT_CODE}" -ne "0" ]; then - echo "Error in updating cobalt job info for falkon.. 'falkon-update-id.sh ${ZOID_JOB_ID}'" - cqdel ${ZOID_JOB_ID} - exit ${EXIT_CODE} -fi - - -FALKON_JOBID_HOME=${FALKON_ROOT}/users/${USER}/${FALKON_JOB_ID} - -echo "Submitted job ${ZOID_JOB_ID} to Cobalt, creating the job specific Falkon tree for logs and configuration in ${FALKON_JOBID_HOME}..." - -mkdir -p ${FALKON_JOBID_HOME} - -cp ${FALKON_HOME}/falkon.env.bgp* ${FALKON_JOBID_HOME}/ -cp -r ${FALKON_HOME}/config ${FALKON_JOBID_HOME}/ -cp ${FALKON_JOBID_HOME}/config/Client-service-URIs.config2 ${FALKON_JOBID_HOME}/config/Client-service-URIs.config -mkdir -p ${FALKON_JOBID_HOME}/logs/client ${FALKON_JOBID_HOME}/logs/service ${FALKON_JOBID_HOME}/logs/provisioner ${FALKON_JOBID_HOME}/logs/worker - - -DATE=`date +%s` -echo "$DATE: pre-creating log dirs for Falkon service..." -RACK_START=0 -RACK_END=48 -SEQUENCE_DIR=`seq -w ${RACK_START} ${RACK_END}` -PSET_START=1 -PSET_END=16 -for a in ${SEQUENCE_DIR} -do - for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$". - do - DIR_NAME="ion-R${a}-${b}" - mkdir -p ${FALKON_JOBID_HOME}/logs/service/$DIR_NAME - done -done - -for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$". -do - DIR_NAME="ion-${b}" - mkdir -p ${FALKON_JOBID_HOME}/logs/service/$DIR_NAME -done - - -DATE=`date +%s` -echo "$DATE: done creating log dirs for Falkon service!" - -FALKON_HOME_RAM=/tmp/${USER}/falkon - -ln -s ${FALKON_HOME}/apps ${FALKON_JOBID_HOME}/apps -ln -s ${FALKON_HOME_RAM}/container ${FALKON_JOBID_HOME}/container -ln -s ${FALKON_HOME}/service ${FALKON_JOBID_HOME}/service -ln -s ${FALKON_HOME}/worker ${FALKON_JOBID_HOME}/worker -ln -s ${FALKON_HOME}/AstroPortal ${FALKON_JOBID_HOME}/AstroPortal -ln -s ${FALKON_HOME}/client ${FALKON_JOBID_HOME}/client -ln -s ${FALKON_HOME}/monitor ${FALKON_JOBID_HOME}/monitor -ln -s ${FALKON_HOME}/bin ${FALKON_JOBID_HOME}/bin -ln -s ${FALKON_HOME}/config ${FALKON_JOBID_HOME}/config -ln -s ${FALKON_HOME}/ploticus ${FALKON_JOBID_HOME}/ploticus -ln -s ${FALKON_HOME}/webserver ${FALKON_JOBID_HOME}/webserver -ln -s ${FALKON_HOME}/workloads ${FALKON_JOBID_HOME}/workloads -ln -s ${FALKON_HOME}/id ${FALKON_JOBID_HOME}/id -ln -s ${FALKON_HOME}/apache-ant-1.7.0 ${FALKON_JOBID_HOME}/apache-ant-1.7.0 -ln -s ${FALKON_HOME}/ibm-java2-ppc64-50 ${FALKON_JOBID_HOME}/ibm-java2-ppc64-50 -ln -s ${FALKON_HOME_RAM}/ibm-java2-ppc-50 ${FALKON_JOBID_HOME}/ibm-java2-ppc-50 -ln -s ${FALKON_HOME}/falkon.tgz ${FALKON_JOBID_HOME}/falkon.tgz - - -if [ ! -d "${FALKON_JOBID_HOME}" ]; then - echo "ERROR: invalid path ${FALKON_JOBID_HOME}... exiting" - cqdel ${ZOID_JOB_ID} - exit 1 -fi - -echo "Succesfully submitted the job to Cobalt, and setup job specific Falkon tree!" -echo "To monitor the job status, type 'cqstat | grep ${USER}'; once it is in running state, you can use the Falkon specific command ...." -echo "To submit your Falkon-based workload, type: ....; you can do this any time, the falkon workload will wait for the resources to come online, and will only be submitted when everything is ready; the script is run in the background, so the workload will run even if the ssh session gets disconnected." -echo "" -echo "Remember, your job id is ${ZOID_JOB_ID}, and if you need to look through the logs manually for anything, remember that you can find them at ${HOME}/${ZOID_JOB_ID}.output, ${HOME}/${ZOID_JOB_ID}.error, and ${FALKON_JOBID_HOME}/logs/..." - - Copied: usertools/cio/libexec/falkon/falkon-start-bgp.sh (from rev 2801, usertools/cio/libexec/falkon/falkon-start-bgp-cio.sh) =================================================================== --- usertools/cio/libexec/falkon/falkon-start-bgp.sh (rev 0) +++ usertools/cio/libexec/falkon/falkon-start-bgp.sh 2009-04-01 21:30:01 UTC (rev 2802) @@ -0,0 +1,167 @@ +#!/bin/bash + +if [ $# -lt 3 ]; then + cat << EOF +Usage: $0 +Example: $0 prod 1024 60 +--or-- +Usage: $0 \ + +EOF + exit 1 +fi + +if [ -z $FALKON_HOME ]; then + echo "ERROR: environment variable FALKON_HOME not defined" + exit 1 +fi +if [ -z $CIOROOT ]; then + echo "ERROR: CIOROOT env not defined" + exit 1 +fi +if [[ $CIOARCH != "bgp" ]]; then + echo "ERROR: Wrong architecture. Must be bgp" + exit 1 +fi + +PROFILE_NAME="zeptocn" +QUEUE_NAME=$1 +NUM_NODES=$2 +let NUM_ION=NUM_NODES/64 +MAX_TIME_MIN=$3 +SERVICE_IP="192.168.1.254" +SERVICE_PORT1=55000 +SERVICE_PORT2=55001 + +WORKERS_PER_NODE=4 +DATA_NUM=1 +STR=1 +if [ -n "$4" ];then + WORKERS_PER_NODE=$4 +fi +if [ -n "$5" ];then + DATA_NUM=$5 +fi +if [ -n "$6" ];then + STR=$6 + echo true +fi + +cp ${FALKON_HOME}/bin/zoid-user-script.sh ${HOME}/zoid-user-script.sh +chmod +x ${HOME}/zoid-user-script.sh + +FALKON_JOB_ID=`falkon-id-get.sh N/A` +EXIT_CODE=$? + +if [ "${EXIT_CODE}" -ne "0" ]; then + echo "Error in geting a unique falkon ID.. 'falkon-get-id.sh N/A'" + cqdel ${ZOID_JOB_ID} + exit ${EXIT_CODE} +fi + + + +WORKER_SCRIPT=$CIOROOT/bin/ciologic-$CIOARCH.sh +ZOID_JOB_ID=`cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t \ + ${MAX_TIME_MIN} -n ${NUM_NODES} \ + -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib:CIOROOT=$CIOROOT:CIOARCH=$CIOARCH:CHIRPROOT=$CHIRPROOT \ + /bgsys/linux/1.2.020080512/bin/bash $WORKER_SCRIPT \ + ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} \ + ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT} \ + $NUM_NODES $DATA_NUM $STR` # MTIO parameters + +EXIT_CODE=$? + +if [ "${EXIT_CODE}" -ne "0" ]; then + echo "Error in submitting job to Cobalt.. 'cqsub -q ${QUEUE_NAME} \ + -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} \ + -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib \ + /bgsys/linux/1.2.020080512/bin/bash $WORKER_SCRIPT \ + ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} \ + ${WORKERS_PER_NODE} ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT}' " + exit ${EXIT_CODE} +fi + +falkon-id-update.sh ${FALKON_JOB_ID} ${ZOID_JOB_ID} + +EXIT_CODE=$? + +if [ "${EXIT_CODE}" -ne "0" ]; then + echo "Error in updating cobalt job info for falkon.. 'falkon-update-id.sh ${ZOID_JOB_ID}'" + cqdel ${ZOID_JOB_ID} + exit ${EXIT_CODE} +fi + + +FALKON_JOBID_HOME=${FALKON_ROOT}/users/${USER}/${FALKON_JOB_ID} + +echo "Submitted job ${ZOID_JOB_ID} to Cobalt, creating the job specific Falkon tree for logs and configuration in ${FALKON_JOBID_HOME}..." + +mkdir -p ${FALKON_JOBID_HOME} + +cp ${FALKON_HOME}/falkon.env.bgp* ${FALKON_JOBID_HOME}/ +cp -r ${FALKON_HOME}/config ${FALKON_JOBID_HOME}/ +cp ${FALKON_JOBID_HOME}/config/Client-service-URIs.config2 ${FALKON_JOBID_HOME}/config/Client-service-URIs.config +mkdir -p ${FALKON_JOBID_HOME}/logs/client ${FALKON_JOBID_HOME}/logs/service ${FALKON_JOBID_HOME}/logs/provisioner ${FALKON_JOBID_HOME}/logs/worker + + +DATE=`date +%s` +echo "$DATE: pre-creating log dirs for Falkon service..." +RACK_START=0 +RACK_END=48 +SEQUENCE_DIR=`seq -w ${RACK_START} ${RACK_END}` +PSET_START=1 +PSET_END=16 +for a in ${SEQUENCE_DIR} +do + for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$". + do + DIR_NAME="ion-R${a}-${b}" + mkdir -p ${FALKON_JOBID_HOME}/logs/service/$DIR_NAME + done +done + +for ((b=${PSET_START}; b <= ${PSET_END} ; b++)) # Double parentheses, and "LIMIT" with no "$". +do + DIR_NAME="ion-${b}" + mkdir -p ${FALKON_JOBID_HOME}/logs/service/$DIR_NAME +done + + +DATE=`date +%s` +echo "$DATE: done creating log dirs for Falkon service!" + +FALKON_HOME_RAM=/tmp/${USER}/falkon + +ln -s ${FALKON_HOME}/apps ${FALKON_JOBID_HOME}/apps +ln -s ${FALKON_HOME_RAM}/container ${FALKON_JOBID_HOME}/container +ln -s ${FALKON_HOME}/service ${FALKON_JOBID_HOME}/service +ln -s ${FALKON_HOME}/worker ${FALKON_JOBID_HOME}/worker +ln -s ${FALKON_HOME}/AstroPortal ${FALKON_JOBID_HOME}/AstroPortal +ln -s ${FALKON_HOME}/client ${FALKON_JOBID_HOME}/client +ln -s ${FALKON_HOME}/monitor ${FALKON_JOBID_HOME}/monitor +ln -s ${FALKON_HOME}/bin ${FALKON_JOBID_HOME}/bin +ln -s ${FALKON_HOME}/config ${FALKON_JOBID_HOME}/config +ln -s ${FALKON_HOME}/ploticus ${FALKON_JOBID_HOME}/ploticus +ln -s ${FALKON_HOME}/webserver ${FALKON_JOBID_HOME}/webserver +ln -s ${FALKON_HOME}/workloads ${FALKON_JOBID_HOME}/workloads +ln -s ${FALKON_HOME}/id ${FALKON_JOBID_HOME}/id +ln -s ${FALKON_HOME}/apache-ant-1.7.0 ${FALKON_JOBID_HOME}/apache-ant-1.7.0 +ln -s ${FALKON_HOME}/ibm-java2-ppc64-50 ${FALKON_JOBID_HOME}/ibm-java2-ppc64-50 +ln -s ${FALKON_HOME_RAM}/ibm-java2-ppc-50 ${FALKON_JOBID_HOME}/ibm-java2-ppc-50 +ln -s ${FALKON_HOME}/falkon.tgz ${FALKON_JOBID_HOME}/falkon.tgz + + +if [ ! -d "${FALKON_JOBID_HOME}" ]; then + echo "ERROR: invalid path ${FALKON_JOBID_HOME}... exiting" + cqdel ${ZOID_JOB_ID} + exit 1 +fi + +echo "Succesfully submitted the job to Cobalt, and setup job specific Falkon tree!" +echo "To monitor the job status, type 'cqstat | grep ${USER}'; once it is in running state, you can use the Falkon specific command ...." +echo "To submit your Falkon-based workload, type: ....; you can do this any time, the falkon workload will wait for the resources to come online, and will only be submitted when everything is ready; the script is run in the background, so the workload will run even if the ssh session gets disconnected." +echo "" +echo "Remember, your job id is ${ZOID_JOB_ID}, and if you need to look through the logs manually for anything, remember that you can find them at ${HOME}/${ZOID_JOB_ID}.output, ${HOME}/${ZOID_JOB_ID}.error, and ${FALKON_JOBID_HOME}/logs/..." + + From noreply at svn.ci.uchicago.edu Wed Apr 1 16:46:47 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:46:47 -0500 (CDT) Subject: [Swift-commit] r2803 - in usertools/cio: bin libexec/falkon Message-ID: <20090401214647.A1695228223@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:46:46 -0500 (Wed, 01 Apr 2009) New Revision: 2803 Removed: usertools/cio/bin/hashserver.rb Modified: usertools/cio/bin/ciologic-bgp.sh usertools/cio/libexec/falkon/cobaltrun.job usertools/cio/libexec/falkon/falkon-start-bgp.sh Log: Bug fixes on ciologi startup regarding spawning background processes Modified: usertools/cio/bin/ciologic-bgp.sh =================================================================== --- usertools/cio/bin/ciologic-bgp.sh 2009-04-01 21:30:01 UTC (rev 2802) +++ usertools/cio/bin/ciologic-bgp.sh 2009-04-01 21:46:46 UTC (rev 2803) @@ -51,7 +51,6 @@ address:127.0.0.1 rwlda EOF - $CHIRPROOT/bin/chirp_server -r /dev/shm/share # Enable striping cat /dev/shm/share/.__acl > /dev/shm/share/stripe/.__acl @@ -64,8 +63,9 @@ mkdir -p /chirp $CHIRPROOT/bin/chirp_fuse -a address /chirp + ln -sf /chirp/`get_ip $rank` /dataifs - ln -sf /chirp/`get_ip $rank` /dataifs + $CHIRPROOT/bin/chirp_server -r /dev/shm/share } # function: ifs_slave [head] Deleted: usertools/cio/bin/hashserver.rb =================================================================== --- usertools/cio/bin/hashserver.rb 2009-04-01 21:30:01 UTC (rev 2802) +++ usertools/cio/bin/hashserver.rb 2009-04-01 21:46:46 UTC (rev 2803) @@ -1,26 +0,0 @@ -#!/home/espinosa/local/bin/ruby - -require 'drb' - -class HashServer - attr_reader :file - - def initialize - @file = Hash.new([]) - end - - def put(fname, rank) - @file[fname].push(rank) - end - - def get(fname) - @file[fname][ rand( @file[fname].size ) ] - end - -end - -server = HashServer.new - -DRb.start_service('druby://*:9000', server) -DRb.thread.join - Modified: usertools/cio/libexec/falkon/cobaltrun.job =================================================================== --- usertools/cio/libexec/falkon/cobaltrun.job 2009-04-01 21:30:01 UTC (rev 2802) +++ usertools/cio/libexec/falkon/cobaltrun.job 2009-04-01 21:46:46 UTC (rev 2803) @@ -1 +1 @@ -./falkon-start-bgp-cio.sh default 64 15 4 1 3 +./falkon-start-bgp.sh default 64 15 4 1 3 Modified: usertools/cio/libexec/falkon/falkon-start-bgp.sh =================================================================== --- usertools/cio/libexec/falkon/falkon-start-bgp.sh 2009-04-01 21:30:01 UTC (rev 2802) +++ usertools/cio/libexec/falkon/falkon-start-bgp.sh 2009-04-01 21:46:46 UTC (rev 2803) @@ -44,7 +44,6 @@ fi if [ -n "$6" ];then STR=$6 - echo true fi cp ${FALKON_HOME}/bin/zoid-user-script.sh ${HOME}/zoid-user-script.sh From noreply at svn.ci.uchicago.edu Wed Apr 1 16:47:30 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:47:30 -0500 (CDT) Subject: [Swift-commit] r2804 - usertools/cio/bin Message-ID: <20090401214730.C50F9228223@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:47:30 -0500 (Wed, 01 Apr 2009) New Revision: 2804 Modified: usertools/cio/bin/falkon-start.sh Log: general falkon-start script to pass arguments to architecture specific one Modified: usertools/cio/bin/falkon-start.sh =================================================================== --- usertools/cio/bin/falkon-start.sh 2009-04-01 21:46:46 UTC (rev 2803) +++ usertools/cio/bin/falkon-start.sh 2009-04-01 21:47:30 UTC (rev 2804) @@ -12,4 +12,4 @@ echo "ERROR: CHIRPROOT not defined" exit 1 fi -$CIOROOT/libexec/falkon/falkon-start-$CIOARCH.sh +$CIOROOT/libexec/falkon/falkon-start-$CIOARCH.sh $@ From noreply at svn.ci.uchicago.edu Wed Apr 1 16:51:46 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 16:51:46 -0500 (CDT) Subject: [Swift-commit] r2805 - in usertools/cio: . libexec/falkon Message-ID: <20090401215146.98157228238@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 16:51:45 -0500 (Wed, 01 Apr 2009) New Revision: 2805 Added: usertools/cio/examples/ Modified: usertools/cio/libexec/falkon/cobaltrun.job Log: modified sample cobaltrun on surveyor to the generalized script Modified: usertools/cio/libexec/falkon/cobaltrun.job =================================================================== --- usertools/cio/libexec/falkon/cobaltrun.job 2009-04-01 21:47:30 UTC (rev 2804) +++ usertools/cio/libexec/falkon/cobaltrun.job 2009-04-01 21:51:45 UTC (rev 2805) @@ -1 +1 @@ -./falkon-start-bgp.sh default 64 15 4 1 3 +$CIOROOT/bin/falkon-start.sh default 64 15 4 1 3 From noreply at svn.ci.uchicago.edu Wed Apr 1 17:09:30 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 17:09:30 -0500 (CDT) Subject: [Swift-commit] r2806 - in usertools/cio: examples libexec/falkon Message-ID: <20090401220930.41C7D22817C@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 17:09:29 -0500 (Wed, 01 Apr 2009) New Revision: 2806 Added: usertools/cio/examples/cobaltrun.job Removed: usertools/cio/libexec/falkon/cobaltrun.job Log: moved cobalrun.job to examples dir Copied: usertools/cio/examples/cobaltrun.job (from rev 2805, usertools/cio/libexec/falkon/cobaltrun.job) =================================================================== --- usertools/cio/examples/cobaltrun.job (rev 0) +++ usertools/cio/examples/cobaltrun.job 2009-04-01 22:09:29 UTC (rev 2806) @@ -0,0 +1 @@ +$CIOROOT/bin/falkon-start.sh default 64 15 4 1 3 Deleted: usertools/cio/libexec/falkon/cobaltrun.job =================================================================== --- usertools/cio/libexec/falkon/cobaltrun.job 2009-04-01 21:51:45 UTC (rev 2805) +++ usertools/cio/libexec/falkon/cobaltrun.job 2009-04-01 22:09:29 UTC (rev 2806) @@ -1 +0,0 @@ -$CIOROOT/bin/falkon-start.sh default 64 15 4 1 3 From noreply at svn.ci.uchicago.edu Wed Apr 1 22:27:20 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Apr 2009 22:27:20 -0500 (CDT) Subject: [Swift-commit] r2807 - usertools/cio/science/blast Message-ID: <20090402032720.223E8228238@www.ci.uchicago.edu> Author: aespinosa Date: 2009-04-01 22:27:18 -0500 (Wed, 01 Apr 2009) New Revision: 2807 Modified: usertools/cio/science/blast/blast.swift usertools/cio/science/blast/mapper-in.sh usertools/cio/science/blast/mapper-out.sh Log: Added the summarizer workflow. Also swift seemed to change sorting sequences of file to what it should be. thus the index of *.seq file is back to normal. Modified: usertools/cio/science/blast/blast.swift =================================================================== --- usertools/cio/science/blast/blast.swift 2009-04-01 22:09:29 UTC (rev 2806) +++ usertools/cio/science/blast/blast.swift 2009-04-02 03:27:18 UTC (rev 2807) @@ -9,30 +9,28 @@ error err; } -(BlastResult out) blastall(BlastQuery i, BlastDatabase db[]) { - app { - /* - blastall "-p" "blastp" "-F" "F" "-d" @filename(db[10]) - "-i" @filename(i) "-v" "300" "-b" "300" "-m8" - "-o" @filename(out.out) stderr=@filename(out.err); - */ - mockblast "-p" "blastp" "-F" "F" "-d" @filename(db[10]) - "-i" @filename(i) "-v" "300" "-b" "300" "-m8" - "-o" @filename(out.out) stderr=@filename(out.err); - } +type BlastSummary; + +app (BlastResult out) blastall(BlastQuery i, BlastDatabase db[]) { + /* + blastall "-p" "blastp" "-F" "F" "-d" @filename(db[0]) + */ + mockblast "-p" "blastp" "-F" "F" "-d" @filename(db[0]) + "-i" @filename(i) "-v" "300" "-b" "300" "-m8" + "-o" @filename(out.out) stderr=@filename(out.err); } -BlastDatabase pir[] ; -BlastResult out[] ; -BlastQuery input[] ; +app (BlastSummary sum) summarize(BlastResult out[]){ + cat @filenames(out.out) stdout=@sum; +} -BlastQuery in_test <"test.in">; -BlastResult out_test ; +BlastDatabase pir[] ; +BlastResult out[] ; +BlastQuery input[] ; - -out_test = blastall(in_test, pir); -/* -foreach data ,i in input { - (out[i],err[i]) = blastall(data, pir); +foreach data,i in input { + (out[i]) = blastall(data, pir); } -*/ + +BlastSummary sum <"summary.out">; +sum = summarize(out); Modified: usertools/cio/science/blast/mapper-in.sh =================================================================== --- usertools/cio/science/blast/mapper-in.sh 2009-04-01 22:09:29 UTC (rev 2806) +++ usertools/cio/science/blast/mapper-in.sh 2009-04-02 03:27:18 UTC (rev 2807) @@ -1,14 +1,18 @@ #!/bin/bash -while getopts ":a:b:d:" options; do - case $options in - a) export a=$OPTARG ;; - b) export b=$OPTARG ;; - d) export d=$OPTARG ;; - *) exit 1;; - esac -done +#while getopts ":a:b:d:" options; do +# case $options in +# a) export a=$OPTARG ;; +# b) export b=$OPTARG ;; +# d) export d=$OPTARG ;; +# *) exit 1;; +# esac +#done +export a=1 +export b=2 +export d=0000 + ls /disks/ci-gpfs/swift/blast/pir/input/$d/*.qry | \ perl -wnl -e '$. >= $ENV{"a"} and $. <= $ENV{"b"} and print "[", $.-$ENV{"a"}, "] $_";' Modified: usertools/cio/science/blast/mapper-out.sh =================================================================== --- usertools/cio/science/blast/mapper-out.sh 2009-04-01 22:09:29 UTC (rev 2806) +++ usertools/cio/science/blast/mapper-out.sh 2009-04-02 03:27:18 UTC (rev 2807) @@ -1,13 +1,16 @@ #!/bin/bash -while getopts ":a:b:d:" options; do - case $options in - a) export a=$OPTARG ;; - b) export b=$OPTARG ;; - d) export d=$OPTARG ;; - *) exit 1;; - esac -done +#while getopts ":a:b:d:" options; do +# case $options in +# a) export a=$OPTARG ;; +# b) export b=$OPTARG ;; +# d) export d=$OPTARG ;; +# *) exit 1;; +# esac +#done +export a=1 +export b=2 +export d=0000 ls /disks/ci-gpfs/swift/blast/pir/input/$d/*.qry | \ perl -wnl -e '$. >= $ENV{"a"} and $. <= $ENV{"b"} From noreply at svn.ci.uchicago.edu Thu Apr 2 08:53:23 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 2 Apr 2009 08:53:23 -0500 (CDT) Subject: [Swift-commit] r2808 - trunk Message-ID: <20090402135324.0E3B02281A1@www.ci.uchicago.edu> Author: benc Date: 2009-04-02 08:53:22 -0500 (Thu, 02 Apr 2009) New Revision: 2808 Modified: trunk/CHANGES.txt Log: fixup manifestation of my ongoing inability to deal with mid-endian data formats Modified: trunk/CHANGES.txt =================================================================== --- trunk/CHANGES.txt 2009-04-02 03:27:18 UTC (rev 2807) +++ trunk/CHANGES.txt 2009-04-02 13:53:22 UTC (rev 2808) @@ -27,26 +27,26 @@ concurrent iterations allowed, which should allow swift to scale better in certain cases. -(02/09/24) +(02/24/09) *** New configuration option wrapper.invocation.mode, specifiable either globally in the configuration file or per-site as a profile key, that configures whether wrapper script invocations are made with an absolute path (as was the behaviour in Swift 0.8) or with a relative path (as was behaviour in previous versions of Swift). -(02/09/22) +(02/22/09) *** Recompilation will happen if a .kml file was compiled with a different version of Swift to the version being invoked. This is in addition to the existing behaviour where a .swift file will be recompiled if it is newer than the corresponding .kml file. -(02/09/17) +(02/17/09) *** There is a new command swift-osg-ress-site-catalog which will generate a site catalog based on data gathered from OSG's ReSS information system. This can be used by OSG users to easily generate a large number of site catalog entries for sites that are likely to work. -(02/09/14) +(02/14/09) *** Handling of [*] and . has changed. [*] has been broken for several Swift releases, but now has some approximation of its former behaviour, in a cleaner implementation. From noreply at svn.ci.uchicago.edu Thu Apr 2 08:54:17 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 2 Apr 2009 08:54:17 -0500 (CDT) Subject: [Swift-commit] r2809 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20090402135417.D6CAC2281A1@www.ci.uchicago.edu> Author: benc Date: 2009-04-02 08:54:17 -0500 (Thu, 02 Apr 2009) New Revision: 2809 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: change initial maxheap logging so that it does not appear on the console and does appear in the log file Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2009-04-02 13:53:22 UTC (rev 2808) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2009-04-02 13:54:17 UTC (rev 2809) @@ -73,7 +73,6 @@ public static void main(String[] argv) { logger.debug("Loader started"); - logger.info("Max heap: " + Runtime.getRuntime().maxMemory()); ArgumentParser ap = buildArgumentParser(); String project = null; try { @@ -118,6 +117,7 @@ try { setupLogging(ap, projectName, runID); + logger.debug("Max heap: " + Runtime.getRuntime().maxMemory()); if (!(new File(project).exists())) { logger.error("Input file " + project + " does not exist."); From noreply at svn.ci.uchicago.edu Thu Apr 2 17:34:14 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 2 Apr 2009 17:34:14 -0500 (CDT) Subject: [Swift-commit] r2810 - trunk/docs Message-ID: <20090402223414.93336228220@www.ci.uchicago.edu> Author: benc Date: 2009-04-02 17:34:13 -0500 (Thu, 02 Apr 2009) New Revision: 2810 Modified: trunk/docs/quickstartguide.xml Log: fix a couple of things in the quickstart guide Modified: trunk/docs/quickstartguide.xml =================================================================== --- trunk/docs/quickstartguide.xml 2009-04-02 13:54:17 UTC (rev 2809) +++ trunk/docs/quickstartguide.xml 2009-04-02 22:34:13 UTC (rev 2810) @@ -125,10 +125,6 @@ class="directory">/path/to/swift/bin:$PATH - If you are installing Swift on a machine with access to a graphical - display, you can run the cog-setup tool to - configure various aspects of the environment required by Swift. - Grid Security For local execution of jobs, no grid security configuration @@ -258,8 +254,9 @@ workflow file name as an argument: +> cd examples/swift > swift +class="file">first.swift The Author: aespinosa Date: 2009-04-03 15:45:49 -0500 (Fri, 03 Apr 2009) New Revision: 2811 Added: usertools/cio/libexec/zoid/ Removed: usertools/cio/tools/zoid/ Modified: usertools/cio/bin/ciologic-bgp.sh usertools/cio/libexec/falkon/falkon-start-bgp.sh usertools/cio/libexec/zoid/logging-script.sh usertools/cio/science/blast/tc.data Log: Inserted ZeptOS logging code in core ciologic. Moved zoid scripts to libexec Modified: usertools/cio/bin/ciologic-bgp.sh =================================================================== --- usertools/cio/bin/ciologic-bgp.sh 2009-04-02 22:34:13 UTC (rev 2810) +++ usertools/cio/bin/ciologic-bgp.sh 2009-04-03 20:45:49 UTC (rev 2811) @@ -19,7 +19,7 @@ LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib PATH=/fuse/bin:/fuse/usr/bin:$PATH -/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd.2409 & +/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd & while [ ! -f /tmp/ifconfig.cmd ]; do sleep 1 done Modified: usertools/cio/libexec/falkon/falkon-start-bgp.sh =================================================================== --- usertools/cio/libexec/falkon/falkon-start-bgp.sh 2009-04-02 22:34:13 UTC (rev 2810) +++ usertools/cio/libexec/falkon/falkon-start-bgp.sh 2009-04-03 20:45:49 UTC (rev 2811) @@ -24,7 +24,7 @@ exit 1 fi -PROFILE_NAME="zeptocn" +PROFILE_NAME="zeptocn-swift" QUEUE_NAME=$1 NUM_NODES=$2 let NUM_ION=NUM_NODES/64 @@ -46,7 +46,7 @@ STR=$6 fi -cp ${FALKON_HOME}/bin/zoid-user-script.sh ${HOME}/zoid-user-script.sh +cp $CIOROOT/libexec/zoid/logging-script.sh ${HOME}/zoid-user-script.sh chmod +x ${HOME}/zoid-user-script.sh FALKON_JOB_ID=`falkon-id-get.sh N/A` @@ -61,23 +61,19 @@ WORKER_SCRIPT=$CIOROOT/bin/ciologic-$CIOARCH.sh -ZOID_JOB_ID=`cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t \ +SUBMIT_CMD="cqsub -q ${QUEUE_NAME} -k ${PROFILE_NAME} -C ${HOME} -t \ ${MAX_TIME_MIN} -n ${NUM_NODES} \ -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib:CIOROOT=$CIOROOT:CIOARCH=$CIOARCH:CHIRPROOT=$CHIRPROOT \ /bgsys/linux/1.2.020080512/bin/bash $WORKER_SCRIPT \ ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} ${WORKERS_PER_NODE} \ ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT} \ - $NUM_NODES $DATA_NUM $STR` # MTIO parameters + $NUM_NODES $DATA_NUM $STR" # MTIO parameters +ZOID_JOB_ID=`$SUBMIT_CMD` EXIT_CODE=$? if [ "${EXIT_CODE}" -ne "0" ]; then - echo "Error in submitting job to Cobalt.. 'cqsub -q ${QUEUE_NAME} \ - -k ${PROFILE_NAME} -C ${HOME} -t ${MAX_TIME_MIN} -n ${NUM_NODES} \ - -e LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib \ - /bgsys/linux/1.2.020080512/bin/bash $WORKER_SCRIPT \ - ${SERVICE_IP} ${SERVICE_PORT1} ${SERVICE_PORT2} \ - ${WORKERS_PER_NODE} ${USER} ${FALKON_JOB_ID} ${FALKON_ROOT}' " + echo "Error in submitting job to Cobalt.. $SUBMIT_CMD" exit ${EXIT_CODE} fi @@ -157,6 +153,9 @@ exit 1 fi +echo $ZOID_JOB_ID $FALKON_JOB_ID $QUEUE_NAME \ + $NUM_NODES $MAX_TIME_MIN $WORKERS_PER_NODE >>$HOME/.falkonjobs + echo "Succesfully submitted the job to Cobalt, and setup job specific Falkon tree!" echo "To monitor the job status, type 'cqstat | grep ${USER}'; once it is in running state, you can use the Falkon specific command ...." echo "To submit your Falkon-based workload, type: ....; you can do this any time, the falkon workload will wait for the resources to come online, and will only be submitted when everything is ready; the script is run in the background, so the workload will run even if the ssh session gets disconnected." Copied: usertools/cio/libexec/zoid (from rev 2810, usertools/cio/tools/zoid) Modified: usertools/cio/libexec/zoid/logging-script.sh =================================================================== --- usertools/cio/tools/zoid/logging-script.sh 2009-04-02 22:34:13 UTC (rev 2810) +++ usertools/cio/libexec/zoid/logging-script.sh 2009-04-03 20:45:49 UTC (rev 2811) @@ -1,7 +1,7 @@ #!/bin/bash # Added GPFS activity logging of IO calls -ame -COBALT_JOBID=`echo $ZOID_JOB_ENV | sed 's/^.*COBALT_JOBID=\([^:]*\)/\1/'` +COBALT_JOBID=`echo $ZOID_JOB_ENV | sed 's/^.*COBALT_JOBID=\([^:]*\)*/\1/' | cut -f 1 -d ':'` TARGET_DIR=$HOME/performance/$COBALT_JOBID @@ -10,8 +10,7 @@ echo -n $TARGET_DIR/gpfs.`hostname` >/tmp/gpfs_log_name.new touch $TARGET_DIR/gpfs.`hostname` || exit 1 mv /tmp/gpfs_log_name.new /tmp/gpfs_log_name -echo -n $TARGET_DIR/zoid.`hostname` >/tmp/zoid_log_name.new -mv /tmp/zoid_log_name.new /tmp/zoid_log_name +/bin.rd/statquery 60 >$TARGET_DIR/zoid.`hostname` & # end addition #startup Modified: usertools/cio/science/blast/tc.data =================================================================== --- usertools/cio/science/blast/tc.data 2009-04-02 22:34:13 UTC (rev 2810) +++ usertools/cio/science/blast/tc.data 2009-04-03 20:45:49 UTC (rev 2811) @@ -17,9 +17,7 @@ localhost paste /bin/paste INSTALLED INTEL32::LINUX null # BLAST binaries -localhost blastall /home/aespinosa/science/blast/ncbi.communicado/bin/blastall INSTALLED INTEL32::LINUX null -localhost mockblast /home/aespinosa/cio/science/blast/mockblast.sh INSTALLED INTEL32::LINUX null -# TERAPORT -TERAPORT blastall /home/aespinosa/science/blast/ncbi.teraport/bin/blastall INSTALLED INTEL32::LINUX null -# RANGER -RANGER blastall /share/home/01035/tg802895/ncbi/bin/blastall INSTALLED INTEL32::LINUX null +communicado blastall /home/aespinosa/science/blast/ncbi.communicado/bin/blastall INSTALLED INTEL32::LINUX null +communicado mockblast /home/aespinosa/cio/science/blast/mockblast.sh INSTALLED INTEL32::LINUX null +surveyor blastall /home/espinosa/science/blast/ncbi.communicado/bin/blastall INSTALLED INTEL32::LINUX null +surveyor mockblast /home/espinosa/cio/science/blast/mockblast.sh INSTALLED INTEL32::LINUX null From noreply at svn.ci.uchicago.edu Sat Apr 4 11:06:24 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 11:06:24 -0500 (CDT) Subject: [Swift-commit] r2812 - in usertools/cio/tools: . iomonitor_zoid Message-ID: <20090404160624.DB78C9D793@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-04 11:06:23 -0500 (Sat, 04 Apr 2009) New Revision: 2812 Added: usertools/cio/tools/iomonitor_zoid/ usertools/cio/tools/iomonitor_zoid/columnize.rb usertools/cio/tools/iomonitor_zoid/gpfs.sample usertools/cio/tools/iomonitor_zoid/zoid.sample Log: Zoid IOstats filtering mechanism Added: usertools/cio/tools/iomonitor_zoid/columnize.rb =================================================================== --- usertools/cio/tools/iomonitor_zoid/columnize.rb (rev 0) +++ usertools/cio/tools/iomonitor_zoid/columnize.rb 2009-04-04 16:06:23 UTC (rev 2812) @@ -0,0 +1,42 @@ +#!/usr/bin/env ruby -n + +BEGIN { + puts "Timestamp\t\ +Total messages sent\t\ +Total bytes sent\t\ +Total messages received\t\ +Total bytes received\t\ +IP fwd messages sent\t\ +IP fwd bytes sent\t\ +IP fwd messages received\t\ +IP fwd bytes received\t\ +Stream messages sent\t\ +Stream bytes sent\t\ +Stream messages received\t\ +Stream bytes received\t\ +Broadcast messages sent\t\ +Broadcast bytes sent\t\ +Internal messages sent\t\ +Internal bytes sent\t\ +Internal messages received\t\ +Internal bytes received\t\ +Plugin 1 messages sent\t\ +Plugin 1 bytes sent\t\ +Plugin 1 messages received\t\ +Plugin 1 bytes received\t\ +Plugin 5 messages sent\t\ +Plugin 5 bytes sent\t\ +Plugin 5 messages received\t\ +Plugin 5 bytes received\t\ +Plugin 2 messages sent\t\ +Plugin 2 bytes sent\t\ +Plugin 2 messages received\t\ +Plugin 2 bytes received" + +} + +print "#{$_.split(':')[1].chomp.gsub(/\t/,'')}\t" + +if $_ =~ /Plugin 2 bytes received/ + print "\n" +end Added: usertools/cio/tools/iomonitor_zoid/gpfs.sample =================================================================== --- usertools/cio/tools/iomonitor_zoid/gpfs.sample (rev 0) +++ usertools/cio/tools/iomonitor_zoid/gpfs.sample 2009-04-04 16:06:23 UTC (rev 2812) @@ -0,0 +1,9 @@ +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238791887 _tu_ 44995 _br_ 276177149 _bw_ 1041 _oc_ 558 _cc_ 281 _rdc_ 2715 _wc_ 33 _dir_ 1 _iu_ 0 +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238791947 _tu_ 56835 _br_ 277251904 _bw_ 21884 _oc_ 816 _cc_ 313 _rdc_ 2879 _wc_ 593 _dir_ 1 _iu_ 14 +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238792007 _tu_ 66849 _br_ 277251904 _bw_ 28660 _oc_ 816 _cc_ 313 _rdc_ 2879 _wc_ 596 _dir_ 1 _iu_ 21 +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238792067 _tu_ 76815 _br_ 277251904 _bw_ 34985 _oc_ 816 _cc_ 313 _rdc_ 2879 _wc_ 599 _dir_ 1 _iu_ 22 +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238792127 _tu_ 86841 _br_ 8917594176 _bw_ 65846 _oc_ 1074 _cc_ 569 _rdc_ 68898 _wc_ 605 _dir_ 1 _iu_ 27 +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238792187 _tu_ 96838 _br_ 8917594176 _bw_ 74422 _oc_ 1074 _cc_ 569 _rdc_ 68898 _wc_ 609 _dir_ 1 _iu_ 30 +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238792247 _tu_ 106833 _br_ 8917594176 _bw_ 80925 _oc_ 1074 _cc_ 569 _rdc_ 68898 _wc_ 612 _dir_ 1 _iu_ 34 +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238792307 _tu_ 116835 _br_ 8917594176 _bw_ 87337 _oc_ 1074 _cc_ 569 _rdc_ 68898 _wc_ 615 _dir_ 1 _iu_ 37 +_io_s_ _n_ 172.16.3.13 _nn_ ion-13 _rc_ 0 _t_ 1238792367 _tu_ 126835 _br_ 8917594176 _bw_ 93751 _oc_ 1074 _cc_ 569 _rdc_ 68898 _wc_ 618 _dir_ 1 _iu_ 40 Added: usertools/cio/tools/iomonitor_zoid/zoid.sample =================================================================== --- usertools/cio/tools/iomonitor_zoid/zoid.sample (rev 0) +++ usertools/cio/tools/iomonitor_zoid/zoid.sample 2009-04-04 16:06:23 UTC (rev 2812) @@ -0,0 +1,279 @@ +Timestamp: 1238791882.750431 +Total messages sent: 1 +Total bytes sent: 164 +Total messages received: 64 +Total bytes received: 256 +IP fwd messages sent: 0 +IP fwd bytes sent: 0 +IP fwd messages received: 0 +IP fwd bytes received: 0 +Stream messages sent: 0 +Stream bytes sent: 0 +Stream messages received: 0 +Stream bytes received: 0 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 1 +Internal bytes sent: 164 +Internal messages received: 64 +Internal bytes received: 256 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 0 +Plugin 5 bytes sent: 0 +Plugin 5 messages received: 0 +Plugin 5 bytes received: 0 +Plugin 2 messages sent: 0 +Plugin 2 bytes sent: 0 +Plugin 2 messages received: 0 +Plugin 2 bytes received: 0 +Timestamp: 1238791942.754140 +Total messages sent: 51604 +Total bytes sent: 419971992 +Total messages received: 49790 +Total bytes received: 1311245 +IP fwd messages sent: 4234 +IP fwd bytes sent: 84680 +IP fwd messages received: 2357 +IP fwd bytes received: 137580 +Stream messages sent: 275 +Stream bytes sent: 2200 +Stream messages received: 275 +Stream bytes received: 17302 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 257 +Internal bytes sent: 54372 +Internal messages received: 320 +Internal bytes received: 2560 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 4096 +Plugin 5 bytes sent: 81920 +Plugin 5 messages received: 4096 +Plugin 5 bytes received: 16384 +Plugin 2 messages sent: 42742 +Plugin 2 bytes sent: 419752992 +Plugin 2 messages received: 42742 +Plugin 2 bytes received: 1137419 +Timestamp: 1238792002.764136 +Total messages sent: 52564 +Total bytes sent: 419996952 +Total messages received: 50270 +Total bytes received: 1336205 +IP fwd messages sent: 5194 +IP fwd bytes sent: 109640 +IP fwd messages received: 2837 +IP fwd bytes received: 162540 +Stream messages sent: 275 +Stream bytes sent: 2200 +Stream messages received: 275 +Stream bytes received: 17302 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 257 +Internal bytes sent: 54372 +Internal messages received: 320 +Internal bytes received: 2560 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 4096 +Plugin 5 bytes sent: 81920 +Plugin 5 messages received: 4096 +Plugin 5 bytes received: 16384 +Plugin 2 messages sent: 42742 +Plugin 2 bytes sent: 419752992 +Plugin 2 messages received: 42742 +Plugin 2 bytes received: 1137419 +Timestamp: 1238792062.774137 +Total messages sent: 54484 +Total bytes sent: 420046872 +Total messages received: 51230 +Total bytes received: 1386125 +IP fwd messages sent: 7114 +IP fwd bytes sent: 159560 +IP fwd messages received: 3797 +IP fwd bytes received: 212460 +Stream messages sent: 275 +Stream bytes sent: 2200 +Stream messages received: 275 +Stream bytes received: 17302 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 257 +Internal bytes sent: 54372 +Internal messages received: 320 +Internal bytes received: 2560 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 4096 +Plugin 5 bytes sent: 81920 +Plugin 5 messages received: 4096 +Plugin 5 bytes received: 16384 +Plugin 2 messages sent: 42742 +Plugin 2 bytes sent: 419752992 +Plugin 2 messages received: 42742 +Plugin 2 bytes received: 1137419 +Timestamp: 1238792122.784134 +Total messages sent: 147012 +Total bytes sent: 9123521197 +Total messages received: 141998 +Total bytes received: 3247824 +IP fwd messages sent: 10550 +IP fwd bytes sent: 286425 +IP fwd messages received: 5473 +IP fwd bytes received: 308845 +Stream messages sent: 275 +Stream bytes sent: 2200 +Stream messages received: 275 +Stream bytes received: 17302 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 257 +Internal bytes sent: 54372 +Internal messages received: 320 +Internal bytes received: 2560 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 4096 +Plugin 5 bytes sent: 81920 +Plugin 5 messages received: 4096 +Plugin 5 bytes received: 16384 +Plugin 2 messages sent: 131834 +Plugin 2 bytes sent: 9123100452 +Plugin 2 messages received: 131834 +Plugin 2 bytes received: 2902733 +Timestamp: 1238792182.794139 +Total messages sent: 147972 +Total bytes sent: 9123546157 +Total messages received: 142478 +Total bytes received: 3272784 +IP fwd messages sent: 11510 +IP fwd bytes sent: 311385 +IP fwd messages received: 5953 +IP fwd bytes received: 333805 +Stream messages sent: 275 +Stream bytes sent: 2200 +Stream messages received: 275 +Stream bytes received: 17302 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 257 +Internal bytes sent: 54372 +Internal messages received: 320 +Internal bytes received: 2560 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 4096 +Plugin 5 bytes sent: 81920 +Plugin 5 messages received: 4096 +Plugin 5 bytes received: 16384 +Plugin 2 messages sent: 131834 +Plugin 2 bytes sent: 9123100452 +Plugin 2 messages received: 131834 +Plugin 2 bytes received: 2902733 +Timestamp: 1238792242.804133 +Total messages sent: 149036 +Total bytes sent: 9123573821 +Total messages received: 143010 +Total bytes received: 3300448 +IP fwd messages sent: 12574 +IP fwd bytes sent: 339049 +IP fwd messages received: 6485 +IP fwd bytes received: 361469 +Stream messages sent: 275 +Stream bytes sent: 2200 +Stream messages received: 275 +Stream bytes received: 17302 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 257 +Internal bytes sent: 54372 +Internal messages received: 320 +Internal bytes received: 2560 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 4096 +Plugin 5 bytes sent: 81920 +Plugin 5 messages received: 4096 +Plugin 5 bytes received: 16384 +Plugin 2 messages sent: 131834 +Plugin 2 bytes sent: 9123100452 +Plugin 2 messages received: 131834 +Plugin 2 bytes received: 2902733 +Timestamp: 1238792302.814129 +Total messages sent: 150852 +Total bytes sent: 9123621037 +Total messages received: 143918 +Total bytes received: 3347664 +IP fwd messages sent: 14390 +IP fwd bytes sent: 386265 +IP fwd messages received: 7393 +IP fwd bytes received: 408685 +Stream messages sent: 275 +Stream bytes sent: 2200 +Stream messages received: 275 +Stream bytes received: 17302 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 257 +Internal bytes sent: 54372 +Internal messages received: 320 +Internal bytes received: 2560 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 4096 +Plugin 5 bytes sent: 81920 +Plugin 5 messages received: 4096 +Plugin 5 bytes received: 16384 +Plugin 2 messages sent: 131834 +Plugin 2 bytes sent: 9123100452 +Plugin 2 messages received: 131834 +Plugin 2 bytes received: 2902733 +Timestamp: 1238792362.824136 +Total messages sent: 151812 +Total bytes sent: 9123645997 +Total messages received: 144398 +Total bytes received: 3372624 +IP fwd messages sent: 15350 +IP fwd bytes sent: 411225 +IP fwd messages received: 7873 +IP fwd bytes received: 433645 +Stream messages sent: 275 +Stream bytes sent: 2200 +Stream messages received: 275 +Stream bytes received: 17302 +Broadcast messages sent: 0 +Broadcast bytes sent: 0 +Internal messages sent: 257 +Internal bytes sent: 54372 +Internal messages received: 320 +Internal bytes received: 2560 +Plugin 1 messages sent: 0 +Plugin 1 bytes sent: 0 +Plugin 1 messages received: 0 +Plugin 1 bytes received: 0 +Plugin 5 messages sent: 4096 +Plugin 5 bytes sent: 81920 +Plugin 5 messages received: 4096 +Plugin 5 bytes received: 16384 +Plugin 2 messages sent: 131834 +Plugin 2 bytes sent: 9123100452 +Plugin 2 messages received: 131834 +Plugin 2 bytes received: 2902733 From noreply at svn.ci.uchicago.edu Sat Apr 4 11:35:26 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 11:35:26 -0500 (CDT) Subject: [Swift-commit] r2813 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui Message-ID: <20090404163527.054BB9D792@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-04 11:35:25 -0500 (Sat, 04 Apr 2009) New Revision: 2813 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Key.java Log: better cleanup and proper handling of tab key Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java 2009-04-04 16:06:23 UTC (rev 2812) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java 2009-04-04 16:35:25 UTC (rev 2813) @@ -128,6 +128,11 @@ terminal.initializeTerminal(); os.write(ANSI.cursorVisible(false)); os.flush(); + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + exit(); + } + }); } catch (Exception e) { e.printStackTrace(); @@ -309,7 +314,12 @@ } } else if (c < 32 && c != 0x0a && c != 0x0d) { - key = new Key(Key.MOD_CTRL, c + 96); + if (c == 9) { + key = new Key(Key.TAB); + } + else { + key = new Key(Key.MOD_CTRL, c + 96); + } } else if (c > 128) { // XTerm @@ -334,7 +344,7 @@ e.printStackTrace(); } } - //screen.status(key.toString()); + screen.status(key.toString()); } } catch (Exception e) { @@ -360,6 +370,9 @@ screen.redraw(); } else { + if (done) { + return 0; + } try { checkSize(); Thread.sleep(10); @@ -534,6 +547,7 @@ } public void exit() { + done = true; try { os.write(ANSI.cursorVisible(true)); os.flush(); @@ -541,7 +555,6 @@ catch (Exception e) { e.printStackTrace(); } - done = true; screen = null; if (terminal instanceof UnixTerminal) { try { Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Key.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Key.java 2009-04-04 16:06:23 UTC (rev 2812) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Key.java 2009-04-04 16:35:25 UTC (rev 2813) @@ -45,6 +45,7 @@ public static final int CR = 0x0d; public static final int BACKSPACE = 0x7f; public static final int ESC = 0x1b; + public static final int TAB = 0x09; private static Map names; @@ -80,6 +81,7 @@ putName(END, "END"); putName(INS, "INS"); putName(DEL, "DEL"); + putName(TAB, "TAB"); } private final int type; From noreply at svn.ci.uchicago.edu Sat Apr 4 16:12:45 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 16:12:45 -0500 (CDT) Subject: [Swift-commit] r2815 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20090404211245.5292A9D793@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-04 16:12:40 -0500 (Sat, 04 Apr 2009) New Revision: 2815 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: the console appender gets its stdout before the monitor appender gets to replace it; make sure only really bad stuff makes it through Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2009-04-04 18:11:52 UTC (rev 2814) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2009-04-04 21:12:40 UTC (rev 2815) @@ -444,6 +444,7 @@ Logger.getLogger(VDL2ExecutionContext.class).setLevel(Level.INFO); Logger.getLogger(WeightedHostScoreScheduler.class).setLevel( Level.INFO); + ca.setThreshold(Level.FATAL); } } From noreply at svn.ci.uchicago.edu Sat Apr 4 16:14:02 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 16:14:02 -0500 (CDT) Subject: [Swift-commit] r2816 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui Message-ID: <20090404211402.7F4429D793@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-04 16:13:59 -0500 (Sat, 04 Apr 2009) New Revision: 2816 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java Log: HOME and END were not handled properly Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java 2009-04-04 21:12:40 UTC (rev 2815) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java 2009-04-04 21:13:59 UTC (rev 2816) @@ -284,7 +284,7 @@ } else if (c == '[') { int c0 = read(); - if (c0 <= 54) { + if (c0 <= 56) { int c1 = read(); if (c1 == '~') { key = new Key(0, Key.KEYPAD2 + c0); @@ -578,4 +578,9 @@ public void setDoubleBuffered(boolean doubleBuffered) { this.doubleBuffered = doubleBuffered; } + + public void setCursorVisible(boolean b) throws IOException { + os.write(ANSI.cursorVisible(b)); + os.flush(); + } } From noreply at svn.ci.uchicago.edu Sat Apr 4 16:15:03 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 16:15:03 -0500 (CDT) Subject: [Swift-commit] r2817 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui Message-ID: <20090404211503.A174C9D793@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-04 16:15:02 -0500 (Sat, 04 Apr 2009) New Revision: 2817 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Key.java Log: I'm beginning to dislike automatic formatting Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Key.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Key.java 2009-04-04 21:13:59 UTC (rev 2816) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Key.java 2009-04-04 21:15:02 UTC (rev 2817) @@ -7,184 +7,188 @@ import java.util.Map; public class Key { - public static final int KEY_PRESSED = 0; - public static final int MOD_ALT = 0x01; - public static final int MOD_CTRL = 0x02; - public static final int MOD_SHIFT = 0x03; - - public static final int KEYPAD = 0x1000; - public static final int KEYPAD2 = 0x2000; - public static final int FN1 = 0x3100; - public static final int FN10 = 0x3200; - - public static final int F1 = FN1 + 0; - public static final int F2 = FN1 + 1; - public static final int F3 = FN1 + 2; - public static final int F4 = FN1 + 3; - public static final int F5 = FN1 + 4; - public static final int F6 = FN1 + 6; - public static final int F7 = FN1 + 7; - public static final int F8 = FN1 + 8; - public static final int F9 = FN1 + 9; - public static final int F10 = FN10 + 0; - public static final int F11 = FN10 + 1; - public static final int F12 = FN10 + 2; - - public static final int UP = KEYPAD + 65; - public static final int DOWN = KEYPAD + 66; - public static final int RIGHT = KEYPAD + 67; - public static final int LEFT = KEYPAD + 68; - public static final int PGDN = KEYPAD2 + 54; - public static final int PGUP = KEYPAD2 + 53; - public static final int HOME = KEYPAD2 + 55; - public static final int END = KEYPAD2 + 56; - public static final int INS = KEYPAD2 + 51; - public static final int DEL = KEYPAD2 + 52; - - public static final int LF = 0x0a; - public static final int CR = 0x0d; - public static final int BACKSPACE = 0x7f; - public static final int ESC = 0x1b; - public static final int TAB = 0x09; - - private static Map names; - - private static void putName(int key, String name) { - names.put(new Integer(key), name); - } - - static { - names = new HashMap(); - putName(CR, "CR"); - putName(LF, "LF"); - putName(F1, "F1"); - putName(F2, "F2"); - putName(F3, "F3"); - putName(F4, "F4"); - putName(F5, "F5"); - putName(F6, "F6"); - putName(F7, "F7"); - putName(F8, "F8"); - putName(F9, "F9"); - putName(F10, "F10"); - putName(F11, "F11"); - putName(F12, "F12"); - putName(DOWN, "DOWN"); - putName(UP, "UP"); - putName(LEFT, "LEFT"); - putName(RIGHT, "RIGHT"); - putName(BACKSPACE, "BACKSPACE"); - putName(ESC, "ESC"); - putName(PGDN, "PGDN"); - putName(PGUP, "PGUP"); - putName(HOME, "HOME"); - putName(END, "END"); - putName(INS, "INS"); - putName(DEL, "DEL"); - putName(TAB, "TAB"); - } - - private final int type; - private final int modifiers; - private final int key; - - public Key(int key) { - this(0, key); - } - - public Key(int modifiers, int key) { - this.type = KEY_PRESSED; - this.modifiers = modifiers; - this.key = key; - } + public static final int KEY_PRESSED = 0; + public static final int MOD_ALT = 0x01; + public static final int MOD_CTRL = 0x02; + public static final int MOD_SHIFT = 0x03; - public Key(int code0, int code1, int code2) { - this.type = KEY_PRESSED; - this.modifiers = code2; - this.key = code0 * 0xff + code1; - } + public static final int KEYPAD = 0x1000; + public static final int KEYPAD2 = 0x2000; + public static final int FN1 = 0x3100; + public static final int FN10 = 0x3200; - public int getKey() { - return key; - } + public static final int F1 = FN1 + 0; + public static final int F2 = FN1 + 1; + public static final int F3 = FN1 + 2; + public static final int F4 = FN1 + 3; + public static final int F5 = FN1 + 4; + public static final int F6 = FN1 + 6; + public static final int F7 = FN1 + 7; + public static final int F8 = FN1 + 8; + public static final int F9 = FN1 + 9; + public static final int F10 = FN10 + 0; + public static final int F11 = FN10 + 1; + public static final int F12 = FN10 + 2; - public int getModifiers() { - return modifiers; - } + public static final int UP = KEYPAD + 65; + public static final int DOWN = KEYPAD + 66; + public static final int RIGHT = KEYPAD + 67; + public static final int LEFT = KEYPAD + 68; + public static final int PGDN = KEYPAD2 + 54; + public static final int PGUP = KEYPAD2 + 53; + public static final int HOME = KEYPAD2 + 55; + public static final int END = KEYPAD2 + 56; + public static final int INS = KEYPAD2 + 51; + public static final int DEL = KEYPAD2 + 52; - public int getType() { - return type; - } - - public boolean modALT() { - return (modifiers & MOD_ALT) != 0; - } - - public boolean modCTRL() { - return (modifiers & MOD_CTRL) != 0; - } - - public boolean modSHIFT() { - return (modifiers & MOD_SHIFT) != 0; - } - - public boolean isEnter() { - return modifiers == 0 && (key == CR || key == LF); - } - - public String toString() { - StringBuffer sb = new StringBuffer(); - if (modCTRL()) { - sb.append("CTRL+"); - } - if (modALT()) { - sb.append("ALT+"); - } - if (key < 128 && key >= 32) { - sb.append((char) key); - } - else { - if (modSHIFT()) { - sb.append("SHIFT+"); - } - String name = (String) names.get(new Integer(key)); - if (name != null) { - sb.append(name); - } - else { - sb.append('#'); - sb.append(key); - } - } - sb.append("-"+modifiers); - sb.append("-"+key); - return sb.toString(); - } + public static final int LF = 0x0a; + public static final int CR = 0x0d; + public static final int BACKSPACE = 0x7f; + public static final int ESC = 0x1b; + public static final int TAB = 0x09; - public boolean equals(Object obj) { - if (obj instanceof Key) { - Key other = (Key) obj; - return modifiers == other.modifiers && key == other.key; - } - else { - return false; - } - } - - public boolean equals(int key) { - if (this.key == key && modifiers == 0) { - return true; - } - else { - return false; - } - } + private static Map names; - public int hashCode() { - return modifiers + key; - } + private static void putName(int key, String name) { + names.put(new Integer(key), name); + } - public boolean isFunctionKey() { - return key >= F1 && key <= F9; - } + static { + names = new HashMap(); + putName(CR, "CR"); + putName(LF, "LF"); + putName(F1, "F1"); + putName(F2, "F2"); + putName(F3, "F3"); + putName(F4, "F4"); + putName(F5, "F5"); + putName(F6, "F6"); + putName(F7, "F7"); + putName(F8, "F8"); + putName(F9, "F9"); + putName(F10, "F10"); + putName(F11, "F11"); + putName(F12, "F12"); + putName(DOWN, "DOWN"); + putName(UP, "UP"); + putName(LEFT, "LEFT"); + putName(RIGHT, "RIGHT"); + putName(BACKSPACE, "BACKSPACE"); + putName(ESC, "ESC"); + putName(PGDN, "PGDN"); + putName(PGUP, "PGUP"); + putName(HOME, "HOME"); + putName(END, "END"); + putName(INS, "INS"); + putName(DEL, "DEL"); + putName(TAB, "TAB"); + } + + private final int type; + private final int modifiers; + private final int key; + + public Key(int key) { + this(0, key); + } + + public Key(int modifiers, int key) { + this.type = KEY_PRESSED; + this.modifiers = modifiers; + this.key = key; + } + + public Key(int code0, int code1, int code2) { + this.type = KEY_PRESSED; + this.modifiers = code2; + this.key = code0 * 0xff + code1; + } + + public int getKey() { + return key; + } + + public int getModifiers() { + return modifiers; + } + + public int getType() { + return type; + } + + public boolean modALT() { + return (modifiers & MOD_ALT) != 0; + } + + public boolean modCTRL() { + return (modifiers & MOD_CTRL) != 0; + } + + public boolean modSHIFT() { + return (modifiers & MOD_SHIFT) != 0; + } + + public boolean isEnter() { + return modifiers == 0 && (key == CR || key == LF); + } + + public String toString() { + StringBuffer sb = new StringBuffer(); + if (modCTRL()) { + sb.append("CTRL+"); + } + if (modALT()) { + sb.append("ALT+"); + } + if (key < 128 && key >= 32) { + sb.append((char) key); + } + else { + if (modSHIFT()) { + sb.append("SHIFT+"); + } + String name = (String) names.get(new Integer(key)); + if (name != null) { + sb.append(name); + } + else { + sb.append('#'); + sb.append(key); + } + } + sb.append("-" + modifiers); + sb.append("-" + key); + return sb.toString(); + } + + public char getChar() { + return (char) key; + } + + public boolean equals(Object obj) { + if (obj instanceof Key) { + Key other = (Key) obj; + return modifiers == other.modifiers && key == other.key; + } + else { + return false; + } + } + + public boolean equals(int key) { + if (this.key == key && modifiers == 0) { + return true; + } + else { + return false; + } + } + + public int hashCode() { + return modifiers + key; + } + + public boolean isFunctionKey() { + return key >= F1 && key <= F9; + } } From noreply at svn.ci.uchicago.edu Sat Apr 4 16:15:59 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 16:15:59 -0500 (CDT) Subject: [Swift-commit] r2818 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui Message-ID: <20090404211559.AE22F9D793@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-04 16:15:56 -0500 (Sat, 04 Apr 2009) New Revision: 2818 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Button.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Component.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Container.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/DefaultTableCellRenderer.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Dialog.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Frame.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/LayeredContainer.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Screen.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Table.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/TableColumn.java Log: better handling of focus/focus colors Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Button.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Button.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Button.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -15,8 +15,8 @@ public Button(String label) { setLabel(label); - bgColor = ANSI.BLACK; - fgColor = ANSI.WHITE; + bgColor = ANSI.WHITE; + fgColor = ANSI.BLACK; } public Button() { @@ -43,21 +43,22 @@ context.lock(); try { context.moveTo(sx, sy); + context.fgColor(getFgColor()); + context.bgColor(getBgColor()); + context.putChar('['); if (this.hasFocus()) { - context.putChar('['); + context.bgColor(ANSI.YELLOW); } - else { - context.putChar(' '); - } context.spaces(pad); twlabel.draw(context); + if (this.hasFocus()) { + context.bgColor(ANSI.YELLOW); + } context.spaces(width - 2 - pad - len); - if (this.hasFocus()) { - context.putChar(']'); - } - else { - context.putChar(' '); - } + + context.bgColor(getBgColor()); + context.fgColor(getFgColor()); + context.putChar(']'); } finally { context.unlock(); Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Component.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Component.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Component.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -5,12 +5,14 @@ import java.io.IOException; +import org.apache.log4j.Logger; + public class Component { public static final int BOTTOM_LAYER = 0; public static final int NORMAL_LAYER = 1; public static final int TOP_LAYER = 2; - protected int x, y, width, height, bgColor, fgColor; + protected int x, y, width, height, bgColor, fgColor, focusedBgColor; protected int sx, sy; private boolean visible, valid, focus, focusable; private Screen screen; @@ -19,6 +21,8 @@ public Component() { visible = true; + focusable = true; + focusedBgColor = ANSI.YELLOW; } protected void redraw() { @@ -117,6 +121,14 @@ this.fgColor = fgColor; } + public int getFocusedBgColor() { + return focusedBgColor; + } + + public void setFocusedBgColor(int focusedBgColor) { + this.focusedBgColor = focusedBgColor; + } + protected void draw(ANSIContext context) throws IOException { } @@ -191,15 +203,19 @@ if (parent.focus()) { focus = parent.childFocused(this); } + redraw(); } return focus; } public boolean unfocus() { - focus = false; - Container parent = getParent(); - if (parent != null) { - parent.childUnfocused(this); + if (focus) { + focus = false; + Container parent = getParent(); + if (parent != null) { + parent.childUnfocused(this); + } + redraw(); } return true; } @@ -234,4 +250,17 @@ public void setLayer(int layer) { this.layer = layer; } + + public boolean focusNext() { + return false; + } + + public boolean focusFirst() { + if (isFocusable()) { + return focus(); + } + else { + return false; + } + } } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Container.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Container.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Container.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -9,108 +9,173 @@ import java.util.List; public class Container extends Component { - protected List components; - protected Component focused, oldFocused; - + protected List components; + protected Component focused, oldFocused; + private Boolean focusable; + public Container() { components = new ArrayList(); } - + public void add(Component comp) { - components.add(comp); + components.add(comp); comp.setParent(this); invalidate(); } - + public void remove(Component comp) { - components.remove(comp); - if (focused == comp) { - focused = null; - } - invalidate(); + components.remove(comp); + if (focused == comp) { + focused = null; + } + invalidate(); } - + + protected void invalidate() { + super.invalidate(); + focusable = null; + } + public List getComponents() { - return components; + return components; } - + public void removeAll() { - components.clear(); - invalidate(); + components.clear(); + invalidate(); } - + protected void drawTree(ANSIContext context) throws IOException { - super.drawTree(context); + super.drawTree(context); Iterator i = components.iterator(); while (i.hasNext()) { - Component c = (Component) i.next(); - if (c.isVisible()) { - drawChild(c, context); - } + Component c = (Component) i.next(); + if (c.isVisible()) { + drawChild(c, context); + } } } - - protected void drawChild(Component c, ANSIContext context) throws IOException { - c.drawTree(context); + + protected void drawChild(Component c, ANSIContext context) + throws IOException { + c.drawTree(context); } - protected void validate() { - if (isValid()) { - return; - } - Iterator i = components.iterator(); - boolean focus = false; + protected void validate() { + if (isValid()) { + return; + } + Iterator i = components.iterator(); + boolean focus = false; while (i.hasNext()) { - Component c = (Component) i.next(); - if (c.hasFocus() && !hasFocus()) { - focus(); - } + Component c = (Component) i.next(); + if (c.hasFocus() && !hasFocus()) { + focus(); + } c.validate(); } super.validate(); - } + } - public boolean childFocused(Component component) { - oldFocused = focused; - boolean f = true; - if (focused != null) { - f = focused.unfocus(); - if (f) { - focused.focusLost(); - } - } - if (f) { - focused = component; - focused.focusGained(); - } - return f; - } - - public void childUnfocused(Component component) { + public boolean childFocused(Component component) { + oldFocused = focused; + boolean f = true; + if (focused != null) { + f = focused.unfocus(); + if (f) { + focused.focusLost(); + } + } + if (f) { + focused = component; + focused.focusGained(); + } + return f; + } + + public void childUnfocused(Component component) { if (oldFocused != null && oldFocused != component) { oldFocused.focus(); } } - - public boolean keyboardEvent(Key key) { - if (key.modALT() || key.isFunctionKey()) { - Iterator i = components.iterator(); - while (i.hasNext()) { - if (((Component) i.next()).keyboardEvent(key)) { - return true; - } - } - return false; - } - else if (focused != null) { - return focused.keyboardEvent(key); - } - else { - return false; - } - } - - public Component getFocusedComponent() { - return focused; - } + + public boolean keyboardEvent(Key key) { + if (key.modALT() || key.isFunctionKey()) { + Iterator i = components.iterator(); + while (i.hasNext()) { + if (((Component) i.next()).keyboardEvent(key)) { + return true; + } + } + return false; + } + else if (focused != null) { + return focused.keyboardEvent(key); + } + else { + return false; + } + } + + public Component getFocusedComponent() { + return focused; + } + + public boolean unfocus() { + if (focused != null && !focused.unfocus()) { + return false; + } + return super.unfocus(); + } + + public boolean focusFirst() { + Iterator j = components.iterator(); + while (j.hasNext()) { + Component comp = (Component) j.next(); + if (comp.focusFirst()) { + return true; + } + } + return false; + } + + public boolean focusNext() { + if (focused == null) { + return focusFirst(); + } + else if (focused.focusNext()) { + return true; + } + Iterator i = components.iterator(); + while (i.hasNext()) { + if (i.next() == focused) { + while (i.hasNext()) { + Component comp = (Component) i.next(); + if (comp.isFocusable()) { + comp.focus(); + return true; + } + } + return false; + } + } + return false; + } + + public boolean isFocusable() { + Boolean f = focusable; + if (f != null) { + return f.booleanValue(); + } + Iterator i = components.iterator(); + while (i.hasNext()) { + if (((Component) i.next()).isFocusable()) { + focusable = Boolean.TRUE; + return true; + } + } + //focusable = Boolean.FALSE; + return false; + } + } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/DefaultTableCellRenderer.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/DefaultTableCellRenderer.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/DefaultTableCellRenderer.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -18,19 +18,25 @@ public Component getComponent(Table table, Object value, boolean isSelected, boolean hasFocus, int row, int column) { - setColors(table, isSelected); + setColors(table, isSelected, hasFocus); label.setText(String.valueOf(value)); return label; } - private void setColors(Table table, boolean selected) { + private void setColors(Table table, boolean selected, boolean hasFocus) { if (!selected) { label.setBgColor(table.getBgColor()); label.setFgColor(table.getFgColor()); } else { - label.setBgColor(table.getHighlightBgColor()); - label.setFgColor(table.getHighlightFgColor()); + if (hasFocus) { + label.setBgColor(table.getFocusedBgColor()); + label.setFgColor(table.getFgColor()); + } + else { + label.setBgColor(table.getHighlightBgColor()); + label.setFgColor(table.getHighlightFgColor()); + } } } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Dialog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Dialog.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Dialog.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -3,47 +3,48 @@ */ package org.griphyn.vdl.karajan.monitor.monitors.ansi.tui; +import java.util.Iterator; public class Dialog extends Frame { - private Component sfocus; + private Component sfocus; - public Dialog() { - bgColor = ANSI.WHITE; - fgColor = ANSI.BLACK; - setFilled(true); - } + public Dialog() { + bgColor = ANSI.WHITE; + fgColor = ANSI.BLACK; + setFilled(true); + } - public void display(Screen screen) { - sfocus = screen.getFocusedComponent(); - screen.add(this); - focus(); - } + public void display(Screen screen) { + sfocus = screen.getFocusedComponent(); + screen.add(this); + focusFirst(); + } - public void close() { - getScreen().remove(this); - setVisible(false); - if (sfocus != null) { - sfocus.focus(); - } - } + public void close() { + getScreen().remove(this); + setVisible(false); + if (sfocus != null) { + sfocus.focus(); + } + } - public void center(Container c) { - x = (c.getWidth() - width) / 2; - y = (c.getHeight() - height) / 2; - } + public void center(Container c) { + x = (c.getWidth() - width) / 2; + y = (c.getHeight() - height) / 2; + } - public boolean keyboardEvent(Key key) { - if (key.getKey() == Key.ESC) { - close(); - return true; - } - else { - return super.keyboardEvent(key); - } - } + public boolean keyboardEvent(Key key) { + if (key.getKey() == Key.ESC) { + close(); + return true; + } + else { + return super.keyboardEvent(key); + } + } - public static int displaySimpleDialog(Screen screen, String title, String msg, - String[] buttons) { + public static int displaySimpleDialog(Screen screen, String title, + String msg, String[] buttons) { final Dialog d = new Dialog(); d.setTitle(title); Label l = new Label(); @@ -68,7 +69,8 @@ } } d.close(); - }}); + } + }); d.add(bs[i]); } d.center(screen); @@ -84,4 +86,25 @@ } return r[0]; } + + public boolean focusNext() { + if (focused == null) { + return focusFirst(); + } + else if (focused.focusNext()) { + return true; + } + Iterator i = components.iterator(); + while (i.hasNext()) { + if (i.next() == focused) { + while (i.hasNext()) { + Component comp = (Component) i.next(); + if (comp.focusFirst()) { + return true; + } + } + } + } + return focusFirst(); + } } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Frame.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Frame.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Frame.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -21,11 +21,11 @@ context.bgColor(bgColor); context.fgColor(fgColor); if (filled) { - context.filledFrame(sx, sy, width, height); + context.filledRect(sx + 1, sy + 1, width - 2, height - 2); } - else { - context.frame(sx, sy, width, height); - } + + context.frame(sx, sy, width, height); + context.bgColor(bgColor); if (title != null) { int tl = title.length() + 2; int space = width - tl; Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/LayeredContainer.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/LayeredContainer.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/LayeredContainer.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -114,6 +114,20 @@ return keyboardEvent(key, TOP_LAYER) || keyboardEvent(key, NORMAL_LAYER) || keyboardEvent(key, BOTTOM_LAYER); } + else if (key.equals(Key.TAB)) { + if (focused == null) { + focusFirst(); + } + else { + if (focused.keyboardEvent(key)) { + return true; + } + else { + focused.focusNext(); + } + } + return true; + } else if (focused != null) { return focused.keyboardEvent(key); } @@ -122,7 +136,50 @@ } } - protected boolean keyboardEvent(Key key, int layer) { + public boolean focusFirst() { + for (int i = 0; i < layers.length; i++) { + if (layers[i] != null) { + Iterator j = layers[i].iterator(); + if (j.hasNext()) { + Component comp = (Component) j.next(); + if (((Component) j.next()).focusFirst()) { + return true; + } + } + } + } + return false; + } + + + public boolean focusNext() { + boolean found = false; + if (focused == null) { + return focusFirst(); + } + else if (focused.focusNext()) { + return true; + } + + for (int i = 0; i < layers.length; i++) { + if (layers[i] != null) { + Iterator j = layers[i].iterator(); + while (j.hasNext()) { + if (found) { + if (((Component) j.next()).focusFirst()) { + return true; + } + } + if (j.next() == focused) { + found = true; + } + } + } + } + return false; + } + + protected boolean keyboardEvent(Key key, int layer) { if (layers[layer] == null) { return false; } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Screen.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Screen.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Screen.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -42,7 +42,7 @@ context.fgColor(fgColor); drawTree(context); if (status != null) { - context.moveTo(0, height); + context.moveTo(1, height); context.bgColor(ANSI.RED); context.fgColor(ANSI.YELLOW); context.text(status); Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Table.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Table.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Table.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -210,4 +210,18 @@ public int getHighlightBgColor() { return fgColor; } + + public boolean isFocusable() { + return true; + } + + + public boolean focusFirst() { + focus(); + return true; + } + + public boolean focusNext() { + return false; + } } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/TableColumn.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/TableColumn.java 2009-04-04 21:15:02 UTC (rev 2817) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/TableColumn.java 2009-04-04 21:15:56 UTC (rev 2818) @@ -28,7 +28,7 @@ int fr = table.getFirstRow(); for (int i = 0; i < Math.min(model.getRowCount(), height - 2); i++) { Component comp = table.getCellRenderer().getComponent(table, - model.getValueAt(i + fr, index), i + fr == selectedRow, false, + model.getValueAt(i + fr, index), i + fr == selectedRow, getParent().hasFocus(), i + fr, index); comp.setAbsoluteLocation(sx, sy + i + 2); comp.setSize(getWidth(), 1); From noreply at svn.ci.uchicago.edu Sat Apr 4 16:16:37 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 16:16:37 -0500 (CDT) Subject: [Swift-commit] r2819 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui Message-ID: <20090404211637.9B09E9D793@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-04 16:16:35 -0500 (Sat, 04 Apr 2009) New Revision: 2819 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java Log: removed debugging stuff Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java 2009-04-04 21:15:56 UTC (rev 2818) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/ANSIContext.java 2009-04-04 21:16:35 UTC (rev 2819) @@ -344,7 +344,7 @@ e.printStackTrace(); } } - screen.status(key.toString()); + //screen.status(key.toString()); } } catch (Exception e) { From noreply at svn.ci.uchicago.edu Sat Apr 4 16:17:05 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 16:17:05 -0500 (CDT) Subject: [Swift-commit] r2820 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui Message-ID: <20090404211705.15AD99D793@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-04 16:17:03 -0500 (Sat, 04 Apr 2009) New Revision: 2820 Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Terminal.java Log: added a terminal like thing Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Terminal.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Terminal.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/tui/Terminal.java 2009-04-04 21:17:03 UTC (rev 2820) @@ -0,0 +1,160 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Apr 4, 2009 + */ +package org.griphyn.vdl.karajan.monitor.monitors.ansi.tui; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.ListIterator; + +public class Terminal extends Component { + public static final int SCROLLBACK_BUFFER_SIZE = 512; + + private LinkedList lines; + private String prompt; + private StringBuffer input; + private InputHandler inputHandler; + private int end = -1; + + public Terminal() { + lines = new LinkedList(); + bgColor = ANSI.BLACK; + fgColor = ANSI.WHITE; + prompt = "> "; + input = new StringBuffer(); + } + + protected void draw(ANSIContext context) throws IOException { + synchronized (lines) { + context.bgColor(bgColor); + context.filledRect(sx, sy, width, height); + context.fgColor(fgColor); + + int crt = Math.min(height - 2, lines.size()) + 1; + if (end < 0) { + context.moveTo(sx, sy + crt); + context.text(prompt); + context.text(input.toString()); + context.bgColor(fgColor); + context.putChar(' '); + context.bgColor(bgColor); + context.fgColor(fgColor); + crt--; + } + else { + System.out.println(); + } + ListIterator li = lines.listIterator(lines.size()); + int skip = end; + while (li.hasPrevious() && skip > 0) { + li.previous(); + skip--; + } + while (li.hasPrevious() && crt >= 0) { + String line = (String) li.previous(); + context.moveTo(sx, sy + crt); + context.text(line); + crt--; + } + } + } + + public String getPrompt() { + return prompt; + } + + public void setPrompt(String prompt) { + this.prompt = prompt; + } + + public boolean keyboardEvent(Key key) { + if (key.isEnter()) { + end = -1; + processInput(); + } + else if (key.equals(Key.BACKSPACE)) { + end = -1; + input.deleteCharAt(input.length() - 1); + } + else if (key.equals(Key.TAB)) { + end = -1; + if (inputHandler != null) { + int si = input.lastIndexOf(" "); + String ac = inputHandler.autoComplete(input.substring(si + 1)); + if (ac != null) { + input.delete(si + 1, input.length()); + input.append(ac); + } + redraw(); + } + } + else if (key.equals(Key.HOME)) { + end = lines.size() - height; + } + else if (key.equals(Key.END)) { + end = -1; + } + else if (key.equals(Key.PGUP)) { + end = Math.min(lines.size() - height, end + height); + } + else if (key.equals(Key.PGDN)) { + end -= height; + } + else { + end = -1; + input.append(key.getChar()); + } + return true; + } + + private void processInput() { + String in = input.toString().trim(); + append(prompt + in); + input = new StringBuffer(); + if (inputHandler != null) { + inputHandler.handleInput(in); + } + else { + append("Invalid command: " + in); + } + } + + public void append(String str) { + synchronized (lines) { + str = str.replaceAll("\\t", " "); + String[] l = str.split("\\n"); + for (int i = 0; i < l.length; i++) { + while (l[i].length() > width) { + lines.add(l[i].substring(0, width)); + l[i] = l[i].substring(width); + } + lines.add(l[i]); + } + while (lines.size() > SCROLLBACK_BUFFER_SIZE) { + lines.removeFirst(); + } + redraw(); + } + } + + public InputHandler getInputHandler() { + return inputHandler; + } + + public void setInputHandler(InputHandler inputHandler) { + this.inputHandler = inputHandler; + } + + public static interface InputHandler { + void handleInput(String in); + + String autoComplete(String in); + } + +} From noreply at svn.ci.uchicago.edu Sat Apr 4 16:17:52 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 16:17:52 -0500 (CDT) Subject: [Swift-commit] r2821 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi Message-ID: <20090404211752.CDCED9D793@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-04 16:17:51 -0500 (Sat, 04 Apr 2009) New Revision: 2821 Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/STable.java Log: added worker terminals Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/STable.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/STable.java 2009-04-04 21:17:03 UTC (rev 2820) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/STable.java 2009-04-04 21:17:51 UTC (rev 2821) @@ -7,6 +7,7 @@ import org.globus.cog.abstraction.interfaces.FileTransferSpecification; import org.globus.cog.abstraction.interfaces.JobSpecification; +import org.globus.cog.abstraction.interfaces.Status; import org.globus.cog.abstraction.interfaces.Task; import org.griphyn.vdl.karajan.monitor.common.StatefulItemModel; import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; @@ -18,24 +19,29 @@ import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Dialog; import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Key; import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Table; +import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Terminal; import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.TextArea; import org.griphyn.vdl.karajan.monitor.monitors.swing.SimpleTableClassRenderer.Model; public class STable extends Table implements ActionListener { - private Dialog d; - private Button close; - public boolean keyboardEvent(Key key) { - if (key.isEnter()) { - createDialog(); - return true; - } - else { - return super.keyboardEvent(key); - } - } - - protected void createDialog() { + private Dialog d, term; + private Button close, wterm; + private Task termTask; + private String termWorkerId; + + public boolean keyboardEvent(Key key) { + if (key.isEnter()) { + createDialog(); + return true; + } + else { + return super.keyboardEvent(key); + } + } + + protected void createDialog() { d = new Dialog(); + d.setTitle("Details"); d.setSize(getScreen().getWidth() * 2 / 3, getScreen().getHeight() * 1 / 3); d.center(getScreen()); TextArea ta = new TextArea(); @@ -43,89 +49,164 @@ d.add(ta); ta.setSize(d.getWidth() - 2, d.getHeight() - 4); ta.setLocation(1, 1); + boolean term = hasCoasterWorker(); + int btncount = term ? 1 : 0; close = new Button("&Close"); - close.setLocation(d.getWidth() / 2 - 5, d.getHeight() - 2); - close.setSize(9, 1); + close.setLocation(d.getWidth() / 2 - 7 - (btncount) * 9, d.getHeight() - 2); + close.setSize(15, 1); close.addActionListener(this); d.add(close); + + if (term) { + wterm = new Button("Worker &Terminal"); + wterm.setLocation(d.getWidth() / 2 - 7 + (btncount) * 9, d.getHeight() - 2); + wterm.setSize(15, 1); + wterm.addActionListener(this); + d.add(wterm); + } + d.display(getScreen()); close.focus(); } - protected String getText() { - StatefulItemModel model = (StatefulItemModel) getModel(); - return format(model.getItem(getSelectedRow())); - } - - protected StatefulItem getItem() { - Model model = (Model) getModel(); - return model.getItem(getSelectedRow()); - } + protected String getText() { + StatefulItemModel model = (StatefulItemModel) getModel(); + return format(model.getItem(getSelectedRow())); + } - public void actionPerformed(Component source) { - if (source == close) { - d.close(); - } - } - - protected static String format(Object o) { - if (o instanceof TaskItem) { - Task t = ((TaskItem) o).getTask(); - if (t == null) { - return "?"; - } - if (t.getType() == Task.FILE_TRANSFER) { - FileTransferSpecification spec = (FileTransferSpecification) t.getSpecification(); - StringBuffer sb = new StringBuffer(); - sb.append("Source: "); - sb.append(spec.getSource()); - sb.append('\n'); - sb.append("Destination: "); - sb.append(spec.getDestination()); - sb.append('\n'); - return sb.toString(); - } - else if (t.getType() == Task.JOB_SUBMISSION) { - JobSpecification spec = (JobSpecification) t.getSpecification(); - StringBuffer sb = new StringBuffer(); - sb.append("Executable: "); - sb.append(spec.getExecutable()); - sb.append("\nArguments: "); - sb.append(spec.getArguments()); - sb.append("\nDirectory: "); - sb.append(spec.getDirectory()); - sb.append("\nAttributes: "); - Iterator i = spec.getAttributeNames().iterator(); - while (i.hasNext()) { - String name = (String) i.next(); - sb.append(name); - sb.append("="); - sb.append(spec.getAttribute(name)); - if (i.hasNext()) { - sb.append(", "); - } - } - return sb.toString(); - } - else { - return t.toString(); - } - } - else if (o instanceof ApplicationItem) { - ApplicationItem app = (ApplicationItem) o; - StringBuffer sb = new StringBuffer(); - sb.append("Name: "); - sb.append(app.getName()); - sb.append("\nArguments: "); - sb.append(app.getArguments()); - sb.append("\nHost: "); - sb.append(app.getHost()); - sb.append("\nStart time: "); - sb.append(app.getStartTime()); - return sb.toString(); - } - else { - return String.valueOf(o); - } - } + protected boolean hasCoasterWorker() { + StatefulItemModel model = (StatefulItemModel) getModel(); + StatefulItem si = model.getItem(getSelectedRow()); + if (si instanceof TaskItem) { + Task t = ((TaskItem) si).getTask(); + if (t == null) { + return false; + } + else { + Status s = t.getStatus(); + if (s == null) { + return false; + } + else { + if (s.getMessage() != null) { + String msg = s.getMessage(); + int index = msg.indexOf("workerid="); + if (index == -1) { + return false; + } + else { + termTask = t; + termWorkerId = msg.substring(index + "workerid=".length()); + return true; + } + } + else { + return false; + } + } + } + } + else { + return false; + } + } + + protected StatefulItem getItem() { + Model model = (Model) getModel(); + return model.getItem(getSelectedRow()); + } + + public void actionPerformed(Component source) { + if (source == close) { + d.close(); + } + else if (source == wterm) { + openWorkerTerminal(); + } + } + + private void openWorkerTerminal() { + term = new Dialog(); + term.setTitle("Worker terminal"); + term.setSize(getScreen().getWidth() * 3 / 4, getScreen().getHeight() * 3 / 4); + Terminal t = new Terminal(); + t.setSize(term.getWidth() - 2, term.getHeight() - 2); + t.setLocation(1, 1); + t.append("Type exit to close terminal"); + t.append(""); + t.setInputHandler(new WorkerTerminalInputHandler(term, t, termTask, termWorkerId)); + t.setPrompt(termTask.getService(0).getServiceContact().getContact() + "/" + termWorkerId + "$ "); + term.add(t); + term.center(getScreen()); + term.display(getScreen()); + } + + protected static String format(Object o) { + if (o instanceof TaskItem) { + Task t = ((TaskItem) o).getTask(); + if (t == null) { + return "?"; + } + if (t.getType() == Task.FILE_TRANSFER) { + FileTransferSpecification spec = (FileTransferSpecification) t + .getSpecification(); + StringBuffer sb = new StringBuffer(); + sb.append("Source: "); + sb.append(spec.getSource()); + sb.append('\n'); + sb.append("Destination: "); + sb.append(spec.getDestination()); + sb.append('\n'); + return sb.toString(); + } + else if (t.getType() == Task.JOB_SUBMISSION) { + JobSpecification spec = (JobSpecification) t.getSpecification(); + StringBuffer sb = new StringBuffer(); + sb.append("Executable: "); + sb.append(spec.getExecutable()); + sb.append("\nArguments: "); + sb.append(spec.getArguments()); + sb.append("\nDirectory: "); + sb.append(spec.getDirectory()); + sb.append("\nStatus: "); + sb.append(t.getStatus().getStatusString()); + String msg = t.getStatus().getMessage(); + if (msg != null) { + sb.append(" "); + sb.append(t.getStatus().getMessage()); + } + sb.append("\nAttributes: "); + Iterator i = spec.getAttributeNames().iterator(); + while (i.hasNext()) { + String name = (String) i.next(); + sb.append(name); + sb.append("="); + sb.append(spec.getAttribute(name)); + if (i.hasNext()) { + sb.append(", "); + } + } + return sb.toString(); + } + else { + return t.toString(); + } + } + else if (o instanceof ApplicationItem) { + ApplicationItem app = (ApplicationItem) o; + StringBuffer sb = new StringBuffer(); + sb.append("Name: "); + sb.append(app.getName()); + sb.append("\nArguments: "); + sb.append(app.getArguments()); + sb.append("\nHost: "); + sb.append(app.getHost()); + sb.append("\nStart time: "); + sb.append(app.getStartTime()); + return sb.toString(); + } + else { + return String.valueOf(o); + } + } } Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java 2009-04-04 21:17:51 UTC (rev 2821) @@ -0,0 +1,108 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Apr 4, 2009 + */ +package org.griphyn.vdl.karajan.monitor.monitors.ansi; + +import java.io.CharArrayWriter; +import java.io.PrintWriter; + +import org.apache.log4j.Logger; +import org.globus.cog.abstraction.coaster.service.local.LocalRequestManager; +import org.globus.cog.abstraction.impl.execution.coaster.WorkerShellCommand; +import org.globus.cog.abstraction.interfaces.Task; +import org.globus.cog.karajan.workflow.service.ProtocolException; +import org.globus.cog.karajan.workflow.service.channels.ChannelManager; +import org.globus.cog.karajan.workflow.service.channels.KarajanChannel; +import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Dialog; +import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Terminal; +import org.griphyn.vdl.karajan.monitor.monitors.ansi.tui.Terminal.InputHandler; +import org.ietf.jgss.GSSCredential; + +public class WorkerTerminalInputHandler implements InputHandler { + public static final Logger logger = Logger + .getLogger(WorkerTerminalInputHandler.class); + + private Dialog dialog; + private Terminal term; + private Task task; + private String workerId, contact; + private GSSCredential cred; + + public WorkerTerminalInputHandler(Dialog dialog, Terminal term, Task task, + String workerId) { + this.dialog = dialog; + this.term = term; + this.task = task; + this.workerId = workerId; + this.contact = task.getService(0).getServiceContact().getContact(); + this.cred = (GSSCredential) task.getService(0).getSecurityContext() + .getCredentials(); + } + + public void handleInput(String in) { + if (in.equals("exit")) { + dialog.close(); + } + else { + String result = runcmd(in); + if (result != null && !result.equals("")) { + term.append(runcmd(in)); + } + } + } + + private String runcmd(String cmd) { + try { + KarajanChannel channel = ChannelManager.getManager() + .reserveChannel(contact, cred, LocalRequestManager.INSTANCE); + WorkerShellCommand wsc = new WorkerShellCommand(workerId, cmd); + wsc.execute(channel); + return wsc.getInDataAsString(0); + } + catch (ProtocolException e) { + term.append(e.getMessage()); + return null; + } + catch (Exception e) { + logger.warn("Cannot execute worker command", e); + CharArrayWriter caw = new CharArrayWriter(); + e.printStackTrace(new PrintWriter(caw)); + term.append(caw.toString()); + return null; + } + } + + public String autoComplete(String in) { + String result = runcmd("mls " + in + "*"); + if (result == null) { + return null; + } + String[] r = result.split("\\s+"); + if (r.length == 0) { + return null; + } + else if (r.length == 1) { + return r[0]; + } + else { + term.append(join(r)); + return null; + } + } + + private String join(String[] s) { + StringBuffer sb = new StringBuffer(); + for (int i = 0; i < s.length - 1; i++) { + sb.append(s[i]); + sb.append(' '); + } + sb.append(s[s.length - 1]); + return sb.toString(); + } +} From noreply at svn.ci.uchicago.edu Sat Apr 4 17:53:40 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 17:53:40 -0500 (CDT) Subject: [Swift-commit] r2822 - trunk/docs Message-ID: <20090404225340.991C39D792@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-04 17:53:35 -0500 (Sat, 04 Apr 2009) New Revision: 2822 Modified: trunk/docs/userguide.xml Log: make 'since version ...' style more consistent Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-04 21:17:51 UTC (rev 2821) +++ trunk/docs/userguide.xml 2009-04-04 22:53:35 UTC (rev 2822) @@ -2041,12 +2041,9 @@ For arrays of structs, the file should contain a heading row listing structure member names separated by whitespace. There should be one row for each element of the array, with structure member elements listed in -the same order as the header row and separated by whitespace. +the same order as the header row and separated by whitespace. (since Swift 0.4) - -This procedure is new in 0.4. -
readdata2 @@ -2082,7 +2079,7 @@ -This procedure is new in 0.7. +(since Swift 0.7)
print From noreply at svn.ci.uchicago.edu Sat Apr 4 17:54:28 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 17:54:28 -0500 (CDT) Subject: [Swift-commit] r2823 - trunk/docs Message-ID: <20090404225428.EB57F9D792@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-04 17:54:25 -0500 (Sat, 04 Apr 2009) New Revision: 2823 Modified: trunk/docs/userguide.xml Log: more description of floats Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-04 22:53:35 UTC (rev 2822) +++ trunk/docs/userguide.xml 2009-04-04 22:54:25 UTC (rev 2823) @@ -384,7 +384,7 @@ intintegers stringstrings of text - floatfloating point numbers + floatfloating point numbers, that behave the same as Java doubles booleantrue/false From noreply at svn.ci.uchicago.edu Sat Apr 4 21:10:01 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 4 Apr 2009 21:10:01 -0500 (CDT) Subject: [Swift-commit] r2824 - usertools/cio/bin Message-ID: <20090405021001.435699D793@vm-125-59.ci.uchicago.edu> Author: zzhang Date: 2009-04-04 21:09:57 -0500 (Sat, 04 Apr 2009) New Revision: 2824 Modified: usertools/cio/bin/broadcast.sh usertools/cio/bin/cnip.sh usertools/cio/bin/wrapper.sh Log: DHT integrated Modified: usertools/cio/bin/broadcast.sh =================================================================== --- usertools/cio/bin/broadcast.sh 2009-04-04 22:54:25 UTC (rev 2823) +++ usertools/cio/bin/broadcast.sh 2009-04-05 02:09:57 UTC (rev 2824) @@ -1,5 +1,6 @@ #!/fuse/bin/bash +LD_LIBRARY_PATH=/lib:/fuse/lib:/usr/lib export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/zzhang/cio/lib PATH=/fuse/bin:/fuse/usr/bin:$PATH Modified: usertools/cio/bin/cnip.sh =================================================================== --- usertools/cio/bin/cnip.sh 2009-04-04 22:54:25 UTC (rev 2823) +++ usertools/cio/bin/cnip.sh 2009-04-05 02:09:57 UTC (rev 2824) @@ -2,16 +2,29 @@ LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib PATH=/fuse/bin:/fuse/usr/bin:$PATH -/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd.2409 & +/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd & while [ ! -f /tmp/ifconfig.cmd ]; do sleep 1 done . /tmp/ifconfig.cmd - +ALLOCATION=$BG_SIZE RANK=`echo $CONTROL_INIT | awk -F, '{print $4}'` +ITER=`expr $ALLOCATION / 128` +if [ $ITER = "0" ] +then + echo 10.128.0.0 >> /dev/shm/DHTlist +fi +for (( i=0; i<$ITER; i++)) +do + J=`expr $i / 2` + X=`expr $i \* 128` + K=`expr $X \% 256` + echo 10.128.$J.$K >> /dev/shm/DHTlist +done + if [ -d /dev/shm/share ]; then rm -rf /dev/shm/share @@ -23,10 +36,10 @@ /home/zzhang/chirp/bin/chirp_server -r /dev/shm/share & -if [ "$RANK" = "0" ]; +if [ "$RANK" = "0" ] || [ "$RANK" = "64" ]; then /home/zzhang/cio/bin/hashserver.rb & -else +#else /home/zzhang/falkon/worker/run.worker-c-bgp.sh $1 $2 $3 $4 $5 $6 $7 fi Modified: usertools/cio/bin/wrapper.sh =================================================================== --- usertools/cio/bin/wrapper.sh 2009-04-04 22:54:25 UTC (rev 2823) +++ usertools/cio/bin/wrapper.sh 2009-04-05 02:09:57 UTC (rev 2824) @@ -1,3 +1,4 @@ +#!/fuse/bin/bash # this script must be invoked inside of bash, not plain sh PATH=/fuse/bin:/fuse/usr/bin:$PATH infosection() { @@ -94,11 +95,12 @@ ln -s "/dev/shm/share/$FILENAME" "$DIR/$L" elif [ "$TYPE" == "_concurrent" ]; then echo "cioinput(): toruscp for intermediate data $INPUT" >> /dev/shm/cio - mkdir -p $FILEPATH - echo "cioinput(): $DIR/$FILEPATH" >> /dev/shm/cio - /home/zzhang/cio/bin/toruscp.sh $FILENAME $DIR/$FILEPATH + echo DIR: `pwd` >> /dev/shm/torus + mkdir -p $DIR/$FILEPATH + echo "cioinput(): $INPUT" >> /dev/shm/cio + /home/zzhang/DHT/bin/DHTcp.sh $INPUT $DIR else - echo "cioinput(): copy from GPFS $INPUT" >> /dev/shm/cio + echo "cioinput(): copy from GPFS $INPUT pwd:`pwd` " >> /dev/shm/cio cp "$PWD/shared/$L" "$DIR/$L" fi } @@ -111,18 +113,79 @@ if [ "$TYPE" == "_concurrent" ]; then echo "ciooutput(): write intermediate data $OUTPUT" >> /dev/shm/cio - /home/zzhang/cio/bin/chtput.rb $FILENAME $RANK - cp $OUTPUT /dev/shm/share/ + echo `pwd` >> /dev/shm/cio + /home/zzhang/DHT/bin/DHTregister.sh $OUTPUT $RANK else echo "ciooutput(): write regular data $OUTPUT" >> /dev/shm/cio #dd if="$OUTPUT" of="$WFDIR/shared/$OUTPUT" bs=128k - echo "$OUTPUT /chirp/multi/${CHIRP_ADD}@stripe/" >> /dev/shm/chirp_add + #echo "$OUTPUT /chirp/multi/${CHIRP_ADD}@stripe/" >> /dev/shm/chirp_add cp "$OUTPUT" /chirp/multi/${CHIRP_ADD}@stripe/ fi } + +extractfile() +{ + ARCHIVE="$1" + START_LOCK=$tmp/mtio-lock-started + END_LOCK=$tmp/mtio-lock-finished + current=`pwd` + cd $tmp + mkdir ${START_LOCK} + + EXIT_CODE=$? + # EXIT_CODE=0 ###### FOR TESTING - forces creation of new dir + + if [ "${EXIT_CODE}" -ne "0" ]; then + echo "waiting for data to be extracted" + ((i = 0)) + while (( i == 0 )) + do + if [ -d "${END_LOCK}" ]; then + ((i = 1)) + fi + + if [ ! -d "${END_LOCK}" ]; then + /bin/sleep 1 + fi + done + else + if [ ! -f "${ARCHIVE}" ]; then + echo "archive doesn't exist... exiting" + rmdir $START_LOCK + exit -2 + fi + + echo "extract the archive" + cd $tmp + echo "extracted" >> /dev/shm/extract + tar xf $ARCHIVE + + EXIT_CODE=$? + + if [ "${EXIT_CODE}" -ne "0" ]; then + echo "Error in untar of ${ARCHIVE} /... exit code ${EXIT_CODE}" + exit ${EXIT_CODE} + rmdir $START_LOCK + fi + + mkdir -p ${END_LOCK} + + EXIT_CODE=$? + if [ "${EXIT_CODE}" -ne "0" ]; then + echo "Error in mkdir ${END_LOCK}... exit code ${EXIT_CODE}" + rmdir $START_LOCK + exit ${EXIT_CODE} + fi + fi + cd $current +} +#/home/zzhang/bashtest/tar.sh +tmp=/dev/shm/share +extractfile common.tar +cd $PWD + +RANK=`echo $CONTROL_INIT | awk -F, '{print $4}'` echo $@ >> /dev/shm/log -RANK=`echo $CONTROL_INIT | awk -F, '{print $4}'` -echo "RANK: $RANK" >> /dev/shm/log COMMANDLINE=$@ WFDIR=$PWD ID=$1 @@ -216,7 +279,7 @@ else cioinput $L #cp "$PWD/shared/$L" "$DIR/$L" - checkError 254 "Failed to link input file $L" + checkError 254 "Failed to link input file $L `ls -l $DIR/$L`" fi done From noreply at svn.ci.uchicago.edu Sun Apr 5 00:10:38 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 00:10:38 -0500 (CDT) Subject: [Swift-commit] r2826 - usertools/cio/science/blast Message-ID: <20090405051038.31DA59D793@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-05 00:10:36 -0500 (Sun, 05 Apr 2009) New Revision: 2826 Added: usertools/cio/science/blast/traceblast.sh Removed: usertools/cio/science/blast/runblast.sh Log: Rename runblast to traceblast Deleted: usertools/cio/science/blast/runblast.sh =================================================================== --- usertools/cio/science/blast/runblast.sh 2009-04-05 05:08:54 UTC (rev 2825) +++ usertools/cio/science/blast/runblast.sh 2009-04-05 05:10:36 UTC (rev 2826) @@ -1,6 +0,0 @@ - -DB=/disks/ci-gpfs/swift/blast/pir/UNIPROT_for_blast_14.0.seq -strace -tt -ff -o traceblast \ - /home/aespinosa/science/blast/ncbi.communicado/bin/blastall -p blastp -F F \ - -d $DB -i test.in -v 300 -b 300 -m8 -o test.out - Copied: usertools/cio/science/blast/traceblast.sh (from rev 2824, usertools/cio/science/blast/runblast.sh) =================================================================== --- usertools/cio/science/blast/traceblast.sh (rev 0) +++ usertools/cio/science/blast/traceblast.sh 2009-04-05 05:10:36 UTC (rev 2826) @@ -0,0 +1,6 @@ + +DB=/disks/ci-gpfs/swift/blast/pir/UNIPROT_for_blast_14.0.seq +strace -tt -ff -o traceblast \ + /home/aespinosa/science/blast/ncbi.communicado/bin/blastall -p blastp -F F \ + -d $DB -i test.in -v 300 -b 300 -m8 -o test.out + From noreply at svn.ci.uchicago.edu Sun Apr 5 00:12:44 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 00:12:44 -0500 (CDT) Subject: [Swift-commit] r2827 - usertools/cio/science/blast Message-ID: <20090405051245.0F5329D793@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-05 00:12:43 -0500 (Sun, 05 Apr 2009) New Revision: 2827 Added: usertools/cio/science/blast/measure-runblast.sh usertools/cio/science/blast/readseq.rb usertools/cio/science/blast/runblast.sh Log: Initial commit of test framework Added: usertools/cio/science/blast/measure-runblast.sh =================================================================== --- usertools/cio/science/blast/measure-runblast.sh (rev 0) +++ usertools/cio/science/blast/measure-runblast.sh 2009-04-05 05:12:43 UTC (rev 2827) @@ -0,0 +1 @@ +./swiftblast.sh - UNIPROT_for_blast_14.0.seq 100001 100010 Property changes on: usertools/cio/science/blast/measure-runblast.sh ___________________________________________________________________ Name: svn:executable + * Added: usertools/cio/science/blast/readseq.rb =================================================================== --- usertools/cio/science/blast/readseq.rb (rev 0) +++ usertools/cio/science/blast/readseq.rb 2009-04-05 05:12:43 UTC (rev 2827) @@ -0,0 +1,33 @@ +#!/usr/bin/env ruby +# +# Script: readseq.rb +# Description: Parses the a BLAST Fasta file and dumps each sequence to a +# file. + +require 'fileutils' + + +fasta_db = File.new(ARGV[0]) +seq_start = ARGV[1].to_i +seq_end = ARGV[2].to_i +prefix = ARGV[3] # Output dir prefix +pad = ARGV[4] # Limit files per directory? "l" + +while true + x = fasta_db.readline("\n>").sub(/>$/, "") + if fasta_db.eof or fasta_db.lineno > seq_end + break + end + x =~ />(.*)\n/ + seqname = $1 + if seq_start <= fasta_db.lineno + dir = pad == "l" ? prefix + + sprintf("/%04d", fasta_db.lineno / 1000) : prefix + fname = sprintf "SEQ%07d_%s.qry", fasta_db.lineno, seqname + FileUtils.mkdir_p dir + file = File.new("#{dir}/#{fname}","w") + file << x + file.close + end + fasta_db.ungetc ?> +end Added: usertools/cio/science/blast/runblast.sh =================================================================== --- usertools/cio/science/blast/runblast.sh (rev 0) +++ usertools/cio/science/blast/runblast.sh 2009-04-05 05:12:43 UTC (rev 2827) @@ -0,0 +1,37 @@ +#!/bin/bash + +# Script: swiftblast.sh [blast_db] [start_seq] [end_seq] [outdir] +# Invokes the swift workflow of reciprocal blast + +if [ $# -lt 3 ]; then + cat << EOF +ERROR : too few arguments +Usage : $0 [blast_db] [start_seq] [end_seq] [outdir] +Example : Reciprocal blast on the 100th to 200th sequences +of UNIPRT.seq and dump results to run_100: +$0 UNIPROT.seq 100 200 run_100 +EOF +fi + +FALKON_ID=$1 +BLAST_DB=$2 +START_SEQ=$3 +END_SEQ=$4 + +# Limit to 1k sequences per dir "l" +LIMIT=$5 + +BLASTROOT=$CIOROOT/science/blast + +# Extract sequences +ruby readseq.rb $BLAST_DB $START_SEQ $END_SEQ seqs + +if [ $FALKON_ID != "-" ]; then # Real Falkon job + sleep 0 + # Regenerate tc.data and sites.xml @BGP@ +else + sleep 0 +fi + +# Run swift: +#swift -sites.file sites.xml -tc.file tc.data $BLASTROOT/blast.swift Property changes on: usertools/cio/science/blast/runblast.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Sun Apr 5 07:59:09 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 07:59:09 -0500 (CDT) Subject: [Swift-commit] r2828 - SwiftApps/pc3 Message-ID: <20090405125909.3E70E9D793@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-05 07:59:08 -0500 (Sun, 05 Apr 2009) New Revision: 2828 Modified: SwiftApps/pc3/CompactDatabase.sh SwiftApps/pc3/CountEntries.sh SwiftApps/pc3/ExtractEntry.sh SwiftApps/pc3/PSLoadExecutable.sh SwiftApps/pc3/PSLoadExecutableDB.sh SwiftApps/pc3/ParseXMLBooleanValue.sh SwiftApps/pc3/README Log: provenance challenge fiddles to luiz's code for me Modified: SwiftApps/pc3/CompactDatabase.sh =================================================================== --- SwiftApps/pc3/CompactDatabase.sh 2009-04-05 05:12:43 UTC (rev 2827) +++ SwiftApps/pc3/CompactDatabase.sh 2009-04-05 12:59:08 UTC (rev 2828) @@ -1,7 +1,7 @@ #!/bin/bash INPUT=$1 -$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute CompactDatabase -f $INPUT +java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute CompactDatabase -f $INPUT - \ No newline at end of file + Modified: SwiftApps/pc3/CountEntries.sh =================================================================== --- SwiftApps/pc3/CountEntries.sh 2009-04-05 05:12:43 UTC (rev 2827) +++ SwiftApps/pc3/CountEntries.sh 2009-04-05 12:59:08 UTC (rev 2828) @@ -2,4 +2,5 @@ INPUT=$1 OUTPUT=$2 -xpath -q -e "count(/java/object/void/object)" $INPUT > $OUTPUT +# xpath -q -e "count(/java/object/void/object)" $INPUT > $OUTPUT +xpath $INPUT "count(/java/object/void/object)" > $OUTPUT Modified: SwiftApps/pc3/ExtractEntry.sh =================================================================== --- SwiftApps/pc3/ExtractEntry.sh 2009-04-05 05:12:43 UTC (rev 2827) +++ SwiftApps/pc3/ExtractEntry.sh 2009-04-05 12:59:08 UTC (rev 2828) @@ -5,6 +5,7 @@ echo "" > $OUTPUT echo "" >> $OUTPUT -xpath -q -e "/java/object/void[$INTEGER]/object" $INPUT >> $OUTPUT +# xpath -q -e "/java/object/void[$INTEGER]/object" $INPUT >> $OUTPUT +xpath $INPUT "/java/object/void[$INTEGER]/object" >> $OUTPUT -echo "" >> $OUTPUT \ No newline at end of file +echo "" >> $OUTPUT Modified: SwiftApps/pc3/PSLoadExecutable.sh =================================================================== --- SwiftApps/pc3/PSLoadExecutable.sh 2009-04-05 05:12:43 UTC (rev 2827) +++ SwiftApps/pc3/PSLoadExecutable.sh 2009-04-05 12:59:08 UTC (rev 2828) @@ -3,5 +3,5 @@ INPUT2=$2 OUTPUT=$3 -$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $INPUT2 -o $OUTPUT -f $INPUT1 +java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $INPUT2 -o $OUTPUT -f $INPUT1 Modified: SwiftApps/pc3/PSLoadExecutableDB.sh =================================================================== --- SwiftApps/pc3/PSLoadExecutableDB.sh 2009-04-05 05:12:43 UTC (rev 2827) +++ SwiftApps/pc3/PSLoadExecutableDB.sh 2009-04-05 12:59:08 UTC (rev 2828) @@ -4,4 +4,4 @@ SINPUT=$3 OUTPUT=$4 -$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $SINPUT -o $OUTPUT -f $DB -f $INPUT +java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $SINPUT -o $OUTPUT -f $DB -f $INPUT Modified: SwiftApps/pc3/ParseXMLBooleanValue.sh =================================================================== --- SwiftApps/pc3/ParseXMLBooleanValue.sh 2009-04-05 05:12:43 UTC (rev 2827) +++ SwiftApps/pc3/ParseXMLBooleanValue.sh 2009-04-05 12:59:08 UTC (rev 2828) @@ -1,4 +1,5 @@ #!/bin/bash INPUT=$1 OUTPUT=$2 -xpath -q -e '/java/boolean/text()' $INPUT > $OUTPUT +# xpath -q -e '/java/boolean/text()' $INPUT > $OUTPUT +xpath $INPUT '/java/boolean/text()' > $OUTPUT Modified: SwiftApps/pc3/README =================================================================== --- SwiftApps/pc3/README 2009-04-05 05:12:43 UTC (rev 2827) +++ SwiftApps/pc3/README 2009-04-05 12:59:08 UTC (rev 2828) @@ -1,10 +1,10 @@ -* This is a work-space for the Swift entry to the third provenance challenge +* Download and extract PC3.tar.gz from the PC3 home page at + http://twiki.ipaw.info/bin/view/Challenge/ThirdProvenanceChallenge -* Download and extract PC3.tar.gz from the PC3 home page. - * Append tc.data.append to your tc.data. -* Requires xpath program (in Debian: $ apt-get install libxml-xpath-perl). +* Requires xpath program (in Debian: $ apt-get install libxml-xpath-perl) +* Requires Java 1.6 * Example: @@ -12,8 +12,21 @@ $ cd PC3 $ export JAVA_PATH= $ export PC3LIB=`pwd` -$ mkdir Java/tmp -$ cd Java/tmp +$ cd Java/ +$ ant jar +$ mkdir tmp +$ cd tmp $ cp /pc3/* . $ export PATH=$PATH:`pwd` $ swift pc3.swift -csvpath=CSVRootPathInput.xml -jobid=JobIDInput.xml + + +Extra notes for running on Ben's OS X 10.5 laptop with multiple Java versions +installed, the default of which does not work: + +Set these: +$ echo $JAVACMD +/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Commands//java +$ echo $JAVA_HOME +/System/Library/Frameworks/JavaVM.framework/Versions/1.6/ + From noreply at svn.ci.uchicago.edu Sun Apr 5 08:00:08 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 08:00:08 -0500 (CDT) Subject: [Swift-commit] r2829 - nmi-build-test/submit-machine Message-ID: <20090405130008.60CC79D793@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-05 08:00:07 -0500 (Sun, 05 Apr 2009) New Revision: 2829 Modified: nmi-build-test/submit-machine/build-hourly nmi-build-test/submit-machine/provider-deef.svn nmi-build-test/submit-machine/swifthead.svn nmi-build-test/submit-machine/swiftnmi.svn Log: change nmi build test scripts to use svn.ci.uchicago.edu as this is now a separate machine from www.ci.uchicago.edu and so the tests were broken Modified: nmi-build-test/submit-machine/build-hourly =================================================================== --- nmi-build-test/submit-machine/build-hourly 2009-04-05 12:59:08 UTC (rev 2828) +++ nmi-build-test/submit-machine/build-hourly 2009-04-05 13:00:07 UTC (rev 2829) @@ -1,7 +1,7 @@ #!/bin/bash source /home/benc/.bash_profile cd /home/benc/swift -svn info https://www.ci.uchicago.edu/svn/vdl2/trunk | grep -E '^Last Changed Rev: ' > swift.repo.latest +svn info https://svn.ci.uchicago.edu/svn/vdl2/trunk | grep -E '^Last Changed Rev: ' > swift.repo.latest svn info https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog | grep -E '^Last Changed Rev: ' > cog.repo.latest REBUILD="" if (! diff swift.repo.latest swift.repo.last ) ; then Modified: nmi-build-test/submit-machine/provider-deef.svn =================================================================== --- nmi-build-test/submit-machine/provider-deef.svn 2009-04-05 12:59:08 UTC (rev 2828) +++ nmi-build-test/submit-machine/provider-deef.svn 2009-04-05 13:00:07 UTC (rev 2829) @@ -1,2 +1,2 @@ method = svn -url = https://www.ci.uchicago.edu/svn/vdl2/provider-deef +url = https://svn.ci.uchicago.edu/svn/vdl2/provider-deef Modified: nmi-build-test/submit-machine/swifthead.svn =================================================================== --- nmi-build-test/submit-machine/swifthead.svn 2009-04-05 12:59:08 UTC (rev 2828) +++ nmi-build-test/submit-machine/swifthead.svn 2009-04-05 13:00:07 UTC (rev 2829) @@ -1,2 +1,2 @@ method = svn -url = https://www.ci.uchicago.edu/svn/vdl2/trunk +url = https://svn.ci.uchicago.edu/svn/vdl2/trunk Modified: nmi-build-test/submit-machine/swiftnmi.svn =================================================================== --- nmi-build-test/submit-machine/swiftnmi.svn 2009-04-05 12:59:08 UTC (rev 2828) +++ nmi-build-test/submit-machine/swiftnmi.svn 2009-04-05 13:00:07 UTC (rev 2829) @@ -1,2 +1,2 @@ method = svn -url = https://www.ci.uchicago.edu/svn/vdl2/nmi-build-test +url = https://svn.ci.uchicago.edu/svn/vdl2/nmi-build-test From noreply at svn.ci.uchicago.edu Sun Apr 5 08:07:17 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 08:07:17 -0500 (CDT) Subject: [Swift-commit] r2830 - nmi-build-test/submit-machine Message-ID: <20090405130717.901049D793@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-05 08:07:17 -0500 (Sun, 05 Apr 2009) New Revision: 2830 Modified: nmi-build-test/submit-machine/build-hourly nmi-build-test/submit-machine/kickstart-hourly.tmp nmi-build-test/submit-machine/swift-hourly nmi-build-test/submit-machine/swift-hourly.tmp Log: incorrect variable name fixed on nmi-s005 long ago but not committed until now Modified: nmi-build-test/submit-machine/build-hourly =================================================================== --- nmi-build-test/submit-machine/build-hourly 2009-04-05 13:00:07 UTC (rev 2829) +++ nmi-build-test/submit-machine/build-hourly 2009-04-05 13:07:17 UTC (rev 2830) @@ -28,7 +28,7 @@ if [ "X$KSREBUILD" != "X" ]; then cp kickstart.repo.latest kickstart.repo.last cp kickstart-hourly kickstart-hourly.tmp - echo "description = $REBUILD" >> kickstart-hourly.tmp + echo "description = $K5REBUILD" >> kickstart-hourly.tmp nmi_submit kickstart-hourly.tmp fi Modified: nmi-build-test/submit-machine/kickstart-hourly.tmp =================================================================== --- nmi-build-test/submit-machine/kickstart-hourly.tmp 2009-04-05 13:00:07 UTC (rev 2829) +++ nmi-build-test/submit-machine/kickstart-hourly.tmp 2009-04-05 13:07:17 UTC (rev 2830) @@ -16,4 +16,4 @@ notify = benc at hawaga.org.uk remote_task_timeout = 12h -description = Kickstart hourly: +description = Modified: nmi-build-test/submit-machine/swift-hourly =================================================================== --- nmi-build-test/submit-machine/swift-hourly 2009-04-05 13:00:07 UTC (rev 2829) +++ nmi-build-test/submit-machine/swift-hourly 2009-04-05 13:07:17 UTC (rev 2830) @@ -1,7 +1,7 @@ project = swift component = swift component_version = HEAD -prereqs = apache-ant-1.7.0, java-1.4.2_05 +prereqs = apache-ant-1.6.2, java-1.5.0_08 run_type = build inputs = coghead.svn, swifthead.svn, swiftnmi.svn Modified: nmi-build-test/submit-machine/swift-hourly.tmp =================================================================== --- nmi-build-test/submit-machine/swift-hourly.tmp 2009-04-05 13:00:07 UTC (rev 2829) +++ nmi-build-test/submit-machine/swift-hourly.tmp 2009-04-05 13:07:17 UTC (rev 2830) @@ -1,7 +1,7 @@ project = swift component = swift component_version = HEAD -prereqs = apache-ant-1.7.0, java-1.4.2_05 +prereqs = apache-ant-1.6.2, java-1.5.0_08 run_type = build inputs = coghead.svn, swifthead.svn, swiftnmi.svn @@ -10,9 +10,9 @@ #platforms = x86_rhas_3, x86_cent_4.2, x86_rhas_4, x86_suse_10.0, x86_64_sles_9, x86_sl_4.4, ia64_sles_9, ia64_rhas_4, ppc_macos_10.3, x86_sles_8, x86_64_fc_4, x86_fc_2, x86_fc_3, ia64_sles_8, x86_rh_8.0, ia64_rhas_3, x86_deb_3.1, x86_rh_9, x86_64_rhas_3, x86_slf_3, x86_deb_4.0, x86_64_rhas_4, x86_64_deb_4.0, x86_fc_4, x86_slc_3, x86_rh_7.2, x86_ubuntu_5.10, x86_sles_9 -platforms = x86_deb_4.0 +platforms = x86_fc_5 notify = benc at hawaga.org.uk -remote_task_timeout = 12h -description = Swift >=r2011. +remote_task_timeout = 720m +description = Swift >=r2823. From noreply at svn.ci.uchicago.edu Sun Apr 5 08:08:24 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 08:08:24 -0500 (CDT) Subject: [Swift-commit] r2831 - nmi-build-test/submit-machine Message-ID: <20090405130824.255C89D793@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-05 08:08:22 -0500 (Sun, 05 Apr 2009) New Revision: 2831 Removed: nmi-build-test/submit-machine/kickstart-hourly.tmp nmi-build-test/submit-machine/swift-hourly.tmp Log: these should not be in svn Deleted: nmi-build-test/submit-machine/kickstart-hourly.tmp =================================================================== --- nmi-build-test/submit-machine/kickstart-hourly.tmp 2009-04-05 13:07:17 UTC (rev 2830) +++ nmi-build-test/submit-machine/kickstart-hourly.tmp 2009-04-05 13:08:22 UTC (rev 2831) @@ -1,19 +0,0 @@ -project = swift -component = kickstart -component_version = HEAD - -run_type = build - -inputs = kickstart.svn, swiftnmi.svn - -remote_task = nmi-build-test/kickstart-build - -platforms = x86_fc_3, x86_fc_4, x86_fc_5, x86_cent_4.2, x86_64_fc_4, x86_64_fc_5, x86_macos_10.4, ppc_macos_10.4, x86_deb_3.1 - -# these two don't build kickstart: x86_64_deb_4.0 x86_deb_4.0 - - -notify = benc at hawaga.org.uk - -remote_task_timeout = 12h -description = Deleted: nmi-build-test/submit-machine/swift-hourly.tmp =================================================================== --- nmi-build-test/submit-machine/swift-hourly.tmp 2009-04-05 13:07:17 UTC (rev 2830) +++ nmi-build-test/submit-machine/swift-hourly.tmp 2009-04-05 13:08:22 UTC (rev 2831) @@ -1,18 +0,0 @@ -project = swift -component = swift -component_version = HEAD -prereqs = apache-ant-1.6.2, java-1.5.0_08 -run_type = build - -inputs = coghead.svn, swifthead.svn, swiftnmi.svn - -remote_task = nmi-build-test/main - -#platforms = x86_rhas_3, x86_cent_4.2, x86_rhas_4, x86_suse_10.0, x86_64_sles_9, x86_sl_4.4, ia64_sles_9, ia64_rhas_4, ppc_macos_10.3, x86_sles_8, x86_64_fc_4, x86_fc_2, x86_fc_3, ia64_sles_8, x86_rh_8.0, ia64_rhas_3, x86_deb_3.1, x86_rh_9, x86_64_rhas_3, x86_slf_3, x86_deb_4.0, x86_64_rhas_4, x86_64_deb_4.0, x86_fc_4, x86_slc_3, x86_rh_7.2, x86_ubuntu_5.10, x86_sles_9 - -platforms = x86_fc_5 - -notify = benc at hawaga.org.uk - -remote_task_timeout = 720m -description = Swift >=r2823. From noreply at svn.ci.uchicago.edu Sun Apr 5 08:58:08 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 08:58:08 -0500 (CDT) Subject: [Swift-commit] r2832 - SwiftApps/pc3 Message-ID: <20090405135808.D8D879D7A0@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-05 08:58:08 -0500 (Sun, 05 Apr 2009) New Revision: 2832 Modified: SwiftApps/pc3/CompactDatabase.sh SwiftApps/pc3/CountEntries.sh SwiftApps/pc3/ExtractEntry.sh SwiftApps/pc3/PSLoadExecutable.sh SwiftApps/pc3/PSLoadExecutableDB.sh SwiftApps/pc3/ParseXMLBooleanValue.sh SwiftApps/pc3/README Log: revert r2828 which was accidentally committed Modified: SwiftApps/pc3/CompactDatabase.sh =================================================================== --- SwiftApps/pc3/CompactDatabase.sh 2009-04-05 13:08:22 UTC (rev 2831) +++ SwiftApps/pc3/CompactDatabase.sh 2009-04-05 13:58:08 UTC (rev 2832) @@ -1,7 +1,7 @@ #!/bin/bash INPUT=$1 -java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute CompactDatabase -f $INPUT +$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute CompactDatabase -f $INPUT - + \ No newline at end of file Modified: SwiftApps/pc3/CountEntries.sh =================================================================== --- SwiftApps/pc3/CountEntries.sh 2009-04-05 13:08:22 UTC (rev 2831) +++ SwiftApps/pc3/CountEntries.sh 2009-04-05 13:58:08 UTC (rev 2832) @@ -2,5 +2,4 @@ INPUT=$1 OUTPUT=$2 -# xpath -q -e "count(/java/object/void/object)" $INPUT > $OUTPUT -xpath $INPUT "count(/java/object/void/object)" > $OUTPUT +xpath -q -e "count(/java/object/void/object)" $INPUT > $OUTPUT Modified: SwiftApps/pc3/ExtractEntry.sh =================================================================== --- SwiftApps/pc3/ExtractEntry.sh 2009-04-05 13:08:22 UTC (rev 2831) +++ SwiftApps/pc3/ExtractEntry.sh 2009-04-05 13:58:08 UTC (rev 2832) @@ -5,7 +5,6 @@ echo "" > $OUTPUT echo "" >> $OUTPUT -# xpath -q -e "/java/object/void[$INTEGER]/object" $INPUT >> $OUTPUT -xpath $INPUT "/java/object/void[$INTEGER]/object" >> $OUTPUT +xpath -q -e "/java/object/void[$INTEGER]/object" $INPUT >> $OUTPUT -echo "" >> $OUTPUT +echo "" >> $OUTPUT \ No newline at end of file Modified: SwiftApps/pc3/PSLoadExecutable.sh =================================================================== --- SwiftApps/pc3/PSLoadExecutable.sh 2009-04-05 13:08:22 UTC (rev 2831) +++ SwiftApps/pc3/PSLoadExecutable.sh 2009-04-05 13:58:08 UTC (rev 2832) @@ -3,5 +3,5 @@ INPUT2=$2 OUTPUT=$3 -java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $INPUT2 -o $OUTPUT -f $INPUT1 +$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $INPUT2 -o $OUTPUT -f $INPUT1 Modified: SwiftApps/pc3/PSLoadExecutableDB.sh =================================================================== --- SwiftApps/pc3/PSLoadExecutableDB.sh 2009-04-05 13:08:22 UTC (rev 2831) +++ SwiftApps/pc3/PSLoadExecutableDB.sh 2009-04-05 13:58:08 UTC (rev 2832) @@ -4,4 +4,4 @@ SINPUT=$3 OUTPUT=$4 -java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $SINPUT -o $OUTPUT -f $DB -f $INPUT +$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $SINPUT -o $OUTPUT -f $DB -f $INPUT Modified: SwiftApps/pc3/ParseXMLBooleanValue.sh =================================================================== --- SwiftApps/pc3/ParseXMLBooleanValue.sh 2009-04-05 13:08:22 UTC (rev 2831) +++ SwiftApps/pc3/ParseXMLBooleanValue.sh 2009-04-05 13:58:08 UTC (rev 2832) @@ -1,5 +1,4 @@ #!/bin/bash INPUT=$1 OUTPUT=$2 -# xpath -q -e '/java/boolean/text()' $INPUT > $OUTPUT -xpath $INPUT '/java/boolean/text()' > $OUTPUT +xpath -q -e '/java/boolean/text()' $INPUT > $OUTPUT Modified: SwiftApps/pc3/README =================================================================== --- SwiftApps/pc3/README 2009-04-05 13:08:22 UTC (rev 2831) +++ SwiftApps/pc3/README 2009-04-05 13:58:08 UTC (rev 2832) @@ -1,10 +1,10 @@ -* Download and extract PC3.tar.gz from the PC3 home page at - http://twiki.ipaw.info/bin/view/Challenge/ThirdProvenanceChallenge +* This is a work-space for the Swift entry to the third provenance challenge +* Download and extract PC3.tar.gz from the PC3 home page. + * Append tc.data.append to your tc.data. -* Requires xpath program (in Debian: $ apt-get install libxml-xpath-perl) -* Requires Java 1.6 +* Requires xpath program (in Debian: $ apt-get install libxml-xpath-perl). * Example: @@ -12,21 +12,8 @@ $ cd PC3 $ export JAVA_PATH= $ export PC3LIB=`pwd` -$ cd Java/ -$ ant jar -$ mkdir tmp -$ cd tmp +$ mkdir Java/tmp +$ cd Java/tmp $ cp /pc3/* . $ export PATH=$PATH:`pwd` $ swift pc3.swift -csvpath=CSVRootPathInput.xml -jobid=JobIDInput.xml - - -Extra notes for running on Ben's OS X 10.5 laptop with multiple Java versions -installed, the default of which does not work: - -Set these: -$ echo $JAVACMD -/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Commands//java -$ echo $JAVA_HOME -/System/Library/Frameworks/JavaVM.framework/Versions/1.6/ - From noreply at svn.ci.uchicago.edu Sun Apr 5 16:46:12 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 16:46:12 -0500 (CDT) Subject: [Swift-commit] r2833 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20090405214612.BD3129D7A0@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-05 16:46:11 -0500 (Sun, 05 Apr 2009) New Revision: 2833 Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java Log: add "Initializing site shared directory" into the runtime ticker sequence so that it appears at a more appropriate place in the output Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2009-04-05 13:58:08 UTC (rev 2832) +++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2009-04-05 21:46:11 UTC (rev 2833) @@ -31,6 +31,7 @@ "uninitialized", "Initializing", "Selecting site", + "Initializing site shared directory", "Stage in", "Submitting", "Submitted", From noreply at svn.ci.uchicago.edu Sun Apr 5 18:06:28 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 18:06:28 -0500 (CDT) Subject: [Swift-commit] r2834 - usertools/cio/bin Message-ID: <20090405230628.B9EE79D793@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-05 18:06:26 -0500 (Sun, 05 Apr 2009) New Revision: 2834 Added: usertools/cio/bin/falkonps Log: Added script to query list of falkon jobs Added: usertools/cio/bin/falkonps =================================================================== --- usertools/cio/bin/falkonps (rev 0) +++ usertools/cio/bin/falkonps 2009-04-05 23:06:26 UTC (rev 2834) @@ -0,0 +1,10 @@ +export falkonjob +echo 3 latest Falkon jobs are: +tail -3 ~/.falkonjobs +echo +tail -3 ~/.falkonjobs | while read cobjob falkonjob rest; do + echo + echo For job $falkonjob: + tail +2 /home/falkon/users/$USER/$falkonjob/config/Client-service-URIs.config + qstat -f $cobjob +done \ No newline at end of file Property changes on: usertools/cio/bin/falkonps ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Sun Apr 5 18:07:02 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 5 Apr 2009 18:07:02 -0500 (CDT) Subject: [Swift-commit] r2835 - usertools/cio/bin Message-ID: <20090405230702.E798E9D793@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-05 18:07:02 -0500 (Sun, 05 Apr 2009) New Revision: 2835 Modified: usertools/cio/bin/ciologic-bgp.sh Log: Merged DHT service from Zhao's codebase Modified: usertools/cio/bin/ciologic-bgp.sh =================================================================== --- usertools/cio/bin/ciologic-bgp.sh 2009-04-05 23:06:26 UTC (rev 2834) +++ usertools/cio/bin/ciologic-bgp.sh 2009-04-05 23:07:02 UTC (rev 2835) @@ -88,6 +88,9 @@ ifs_mount() { ifs_rank=$1 # For Chirp, just start the server + if [ -d /dev/shm/share ]; then + rm -rf /dev/shm/share + fi mkdir -p /dev/shm/share cat > /dev/shm/share/.__acl << EOF address:192.168.1.* rwlda @@ -98,12 +101,15 @@ $CHIRPROOT/bin/chirp_server -r /dev/shm/share & mkdir /chirp $CHIRPROOT/bin/chirp_fuse -a address /chirp + # TODO: make a symlink to a slave's proper IFS to + # prevent always recalculating it in higher-level + # scripts + # ln -sf /chirp/`get_ip $ifs_rank`@stripe /ifsmount } -PART_SIZE=$8 +PART_SIZE=$8 # also BG_SIZE IFS_NUM=$9 STRIPE_SIZE=${10} -echo $PART_SIZE $IFS_NUM $STRIPE_SIZE RANK=`get_rank` IFS_GRP=$(( PART_SIZE / IFS_NUM )) @@ -113,18 +119,33 @@ export IFS_RANK export CHIRP_ADD=`get_ip $IFS_RANK` -echo "RANK: $RANK" "IFS_RANK: $IFS_RANK" +# Save information +echo $RANK > /dev/shm/RANK +echo $IFS_RANK > /dev/shm/IFS_RANK +# Generate Hash services +DHT=$(( RANK % 128 )) +if [ -f /dev/shm/DHTlist ]; then + rm -f /dev/shm/DHTlist +fi +for (( i = 0; i < $PART_SIZE; i = i + 128 )); do + echo `get_ip $i` >> /dev/shm/DHTlist +done + #Core MTIO logic -if [ $RANK -eq 0 ]; then - $CIOROOT/libexec/hashserver.rb -elif [ $RANK -eq $IFS_RANK ]; then +if [ $DHT -eq 0 ]; then + $CIOROOT/libexec/hashserver.rb +else [ $RANK -eq $IFS_RANK ]; then ifs_head $STRIPE_SIZE - /home/zzhang/cio/bin/collector.sh + /home/zzhang/cio/bin/collector.sh elif [ $RANK -lt $IFS_SLAVE ]; then ifs_slave $IFS_RANK - sleep 3600 else ifs_mount $IFS_RANK $CIOROOT/libexec/falkon/runworker-bgp.sh $1 $2 $3 $4 $5 $6 $7 fi + +# Quick hack +# Make sure this job does not finish +# Guarantee to not miss any background processes +sleep 30d From noreply at svn.ci.uchicago.edu Mon Apr 6 16:58:22 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Apr 2009 16:58:22 -0500 (CDT) Subject: [Swift-commit] r2836 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi Message-ID: <20090406215822.EFD239D7A5@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-04-06 16:58:20 -0500 (Mon, 06 Apr 2009) New Revision: 2836 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java Log: use coaster-url attribute if set Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java 2009-04-05 23:07:02 UTC (rev 2835) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/WorkerTerminalInputHandler.java 2009-04-06 21:58:20 UTC (rev 2836) @@ -15,6 +15,7 @@ import org.apache.log4j.Logger; import org.globus.cog.abstraction.coaster.service.local.LocalRequestManager; import org.globus.cog.abstraction.impl.execution.coaster.WorkerShellCommand; +import org.globus.cog.abstraction.interfaces.Service; import org.globus.cog.abstraction.interfaces.Task; import org.globus.cog.karajan.workflow.service.ProtocolException; import org.globus.cog.karajan.workflow.service.channels.ChannelManager; @@ -40,7 +41,11 @@ this.term = term; this.task = task; this.workerId = workerId; - this.contact = task.getService(0).getServiceContact().getContact(); + Service s = task.getService(0); + this.contact = (String) s.getAttribute("coaster-url"); + if (this.contact == null) { + this.contact = task.getService(0).getServiceContact().getContact(); + } this.cred = (GSSCredential) task.getService(0).getSecurityContext() .getCredentials(); } From noreply at svn.ci.uchicago.edu Tue Apr 7 08:03:02 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Apr 2009 08:03:02 -0500 (CDT) Subject: [Swift-commit] r2837 - trunk/docs Message-ID: <20090407130302.B51429D7A7@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2009-04-07 08:03:00 -0500 (Tue, 07 Apr 2009) New Revision: 2837 Modified: trunk/docs/userguide.xml Log: Added link to coaster design description page. Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-06 21:58:20 UTC (rev 2836) +++ trunk/docs/userguide.xml 2009-04-07 13:03:00 UTC (rev 2837) @@ -3473,7 +3473,10 @@ The coaster mechanism submits a head job using some other execution mechanism such as GRAM, and for each worker node that will be used in a remote cluster, it submits a worker job, again using some other -execution mechanism such as GRAM. +execution mechanism such as GRAM. Details on the design of the coaster +mechanism can be found + +here The head job manages file transfers and the dispatch of execution jobs From noreply at svn.ci.uchicago.edu Tue Apr 7 08:14:20 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Apr 2009 08:14:20 -0500 (CDT) Subject: [Swift-commit] r2838 - trunk/docs Message-ID: <20090407131420.902D59D7A7@vm-125-59.ci.uchicago.edu> Author: wilde Date: 2009-04-07 08:14:17 -0500 (Tue, 07 Apr 2009) New Revision: 2838 Modified: trunk/docs/userguide.xml Log: Fixed link to coaster design description page. Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-07 13:03:00 UTC (rev 2837) +++ trunk/docs/userguide.xml 2009-04-07 13:14:17 UTC (rev 2838) @@ -3475,8 +3475,8 @@ a remote cluster, it submits a worker job, again using some other execution mechanism such as GRAM. Details on the design of the coaster mechanism can be found - -here + +here. The head job manages file transfers and the dispatch of execution jobs From noreply at svn.ci.uchicago.edu Tue Apr 7 21:29:45 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Apr 2009 21:29:45 -0500 (CDT) Subject: [Swift-commit] r2839 - in SwiftApps/SIDGrid/swift/swift_scripts: . sem Message-ID: <20090408022945.A9C409D7A7@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-04-07 21:29:39 -0500 (Tue, 07 Apr 2009) New Revision: 2839 Added: SwiftApps/SIDGrid/swift/swift_scripts/sem/ SwiftApps/SIDGrid/swift/swift_scripts/sem/sem_demo.swift Log: prototype for frontiers paper Added: SwiftApps/SIDGrid/swift/swift_scripts/sem/sem_demo.swift =================================================================== --- SwiftApps/SIDGrid/swift/swift_scripts/sem/sem_demo.swift (rev 0) +++ SwiftApps/SIDGrid/swift/swift_scripts/sem/sem_demo.swift 2009-04-08 02:29:39 UTC (rev 2839) @@ -0,0 +1,87 @@ +type file; +type mxMin; +type Rscript; +type dbConnect; +type mxModel{ +int modnum; +int dof; +string best; +} +# -------------- atomic procedures -------------- # + +(file matrix) runQuery (dbConnect dbconn, string query, Rscript calcCov){ +app{ +mysqlPythonAPI query @calcCov @dbconn; +} +} + +(external inserted) insertMxResult (dbConnect dbconn, string query, file datafile){ +app{ +mysqlPythonAPI query @dbconn stdout=@filename(inserted) @datafile; +} +} + +(file min) mxModelProcessor ( file cov, Rscript mxModProc, int modnum, float weight, string cond, int net){ +app{ +RInvoke @mxModProc @filename(cov) modnum weight cond net; +} +} + +# ------------ SEM user-defined procedures --------------- # + +multiNetworkSEM(string condition,dbConnect emblemdb, dbConnect semdb, int n, string net, int totalperms[]){ +float initweight = .75; +file covariance; +covariance = getCovariance(condition, n, net, emblemdb); +Rscript mxModProc; +foreach perm in totalperms{ +file modmin; +modmin = mxModelProcessor(covariance,mxModProc,perm,initweight,condition,n); +external doneflag = insertOptMod(n, semdb, condition, modmin); + +} + +(external ins) insertOptMod(int net, dbConnect dbconn, string cond, file modfile){ +string mysqlstr = @strcat("INSERT INTO optimized_models (network,deg_of_freedom,mx_minimum, modnum, cond) VALUES (",net,",DOF,BEST,MODNUM,",cond,");"); +string argList = @strcat( +" --query ", mysqlstr, +" --data ", @filename(modfile), +" --conf ", @filename(dbconn)); +ins = insertMxResult(dbconn, argList, modfile); +} + +(file covariance) getCovariance (string cond, int net, string rois, dbConnect dbconn){ +string mysqlstr = @strcat("SELECT avg(",cond,"0B), avg(",cond,"1B), avg(",cond,"2B),", +"avg(",cond,"3B), avg(",cond,"4B), avg(",cond,"5B),", +"avg(",cond,"6B), avg(",cond,"7B), avg(",cond,"8B) ", +"FROM emblemfemlh where roi in (",rois,") group by roi "); +string argList = @strcat( +" --conf ", "user.config", +" --query ", mysqlstr, +" --r_script ", "scripts/cov.R", +" --r_swift_args ", "matrices/net",net, "/", cond); +Rscript calcCov; +covariance = runQuery(dbconn, argList, calcCov); +} + +(file plotfile) plotLogLik(int net, string cond, dbConnect dbconn){ +Rscript rplot; +string mysqlstr = @strcat("SELECT deg_of_freedom,mx_minimum FROM optimized_models", +" where network = ",net,";"); +string argList = @strcat( +" --conf ", @filename(dbconn), +" --query ", mysqlstr, +" --r_script ", "scripts/plotloglik.R", +" --r_swift_args ", @filename(plotfile)); +plotfile = runQuery(dbconn, argList, rplot); +}# ----------------------- Main ------------------------ # + + +string condition = "gestspeech"; +string networks[] = ["42, 34, 33, 60", "42, 15, 60, 80", "33, 34, 23, 80"]; +dbConnect emblemdb ; +dbConnect semdb ; +int totalperms[] = [1:65536]; +foreach net,n in networks{ +multiNetworkSEM(condition,emblemdb,semdb,n,net); +} From noreply at svn.ci.uchicago.edu Wed Apr 8 07:24:16 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 07:24:16 -0500 (CDT) Subject: [Swift-commit] r2840 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20090408122416.981069D7A8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-08 07:24:13 -0500 (Wed, 08 Apr 2009) New Revision: 2840 Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java Log: Change spacing in runtime ticker for better readability Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2009-04-08 02:29:39 UTC (rev 2839) +++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2009-04-08 12:24:13 UTC (rev 2840) @@ -150,14 +150,14 @@ long now = System.currentTimeMillis(); if(lastDumpTime + MIN_PERIOD_MS > now) return; lastDumpTime = now; - printStates("Progress: "); + printStates("Progress:"); } void finalDumpState() { if (disabled) { return; } - printStates("Final status: "); + printStates("Final status:"); } public Map getSummary() { @@ -192,7 +192,7 @@ String key = preferredOutputOrder[pos]; Object value = summary.get(key); if(value != null) - System.err.print(" "+key+":"+value); + System.err.print(" "+key+":"+value); summary.remove(key); } From noreply at svn.ci.uchicago.edu Wed Apr 8 07:26:22 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 07:26:22 -0500 (CDT) Subject: [Swift-commit] r2841 - trunk/docs Message-ID: <20090408122622.78AF89D7A8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-08 07:26:19 -0500 (Wed, 08 Apr 2009) New Revision: 2841 Modified: trunk/docs/userguide.xml Log: sort swift.properties documentation by property name Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-08 12:24:13 UTC (rev 2840) +++ trunk/docs/userguide.xml 2009-04-08 12:26:19 UTC (rev 2841) @@ -2189,58 +2189,144 @@ Swift Properties - - + + - sites.file + caching.algorithm - + - Valid values: <file> + Valid values: LRU - Default value: ${swift.home}/etc/sites.xml + Default value: LRU - Points to the location of the site - catalog, which contains a list of all sites that - Swift should use. + Swift caches files that are staged in on remote + resources, and files that are produced remotely + by applications, such that they can be re-used + if needed without being transfered again. + However, the amount of remote file system space + to be used for caching can be limited using the + swift:storagesize profile + entry in the sites.xml file. Example: + + +<pool handle="example" sysinfo="INTEL32::LINUX"> + <gridftp url="gsiftp://example.org" storage="/scratch/swift" major="2" minor="4" patch="3"/> + <jobmanager universe="vanilla" url="example.org/jobmanager-pbs" major="2" minor="4" patch="3"/> + <workdirectory>/scratch/swift</workdirectory> + <profile namespace="SWIFT" key="storagesize">20000000</profile> +</pool> + + + + The decision of which files to keep in the cache + and which files to remove is made considering + the value of the + caching.algorithm property. + Currently, the only available value for this + property is LRU, which would + cause the least recently used files to be + deleted first. + - - + + - tc.file + clustering.enabled + - Valid values: <file> + Valid values: true, false - Default value: ${swift.home}/etc/tc.data + Default value: false + + + Enables clustering. + + + + + + + clustering.min.time + + + + Valid values: <int> + - Points to the location of the transformation - catalog file which contains information about - installed applications. Details about the format - of the transformation catalog can be found - here. + + Default value: 60 + + + + + Indicates the threshold wall time for + clustering, in seconds. Jobs that have a + wall time smaller than the value of this + property will be considered for clustering. + + + + clustering.queue.delay + + + + + Valid values: <int> + + + + Default value: 4 + + + + + This property indicates the interval, in + seconds, at which the clustering queue is + processed. + + + + + + execution.retries + + + Valid values: positive integers + + + Default value: 2 + + + The number of time a job will be retried if it + fails (giving a maximum of 1 + + execution.retries attempts at execution) + + + + ip.address @@ -2274,11 +2360,10 @@ - - + - lazy.errors + kickstart.always.transfer @@ -2289,97 +2374,93 @@ Default value: false - + - Swift can report application errors in two - modes, depending on the value of this property. - If set to false, Swift will - report the first error encountered and - immediately stop execution. If set to - true, Swift will attempt to - run as much as possible from a SwiftScript program before - stopping execution and reporting all errors - encountered. + This property controls when output from + Kickstart is transfered back to the submit site, + if Kickstart is enabled. When set to + false, Kickstart output is + only transfered for jobs that fail. If set to + true, Kickstart output is + transfered after every job is completed or + failed. + - When developing SwiftScript programs, using the - default value of false can - make the program easier to debug. However - in production runs, using true - will allow more of a SwiftScript program to be run before - Swift aborts execution. - - - + + - caching.algorithm + kickstart.enabled - + - Valid values: LRU + Valid values: true, false, maybe - Default value: LRU + Default value: maybe - + - - Swift caches files that are staged in on remote - resources, and files that are produced remotely - by applications, such that they can be re-used - if needed without being transfered again. - However, the amount of remote file system space - to be used for caching can be limited using the - swift:storagesize profile - entry in the sites.xml file. Example: - + This option allows controlling of + when Swift uses Kickstart. A value of + false disables the use of + Kickstart, while a value of + true enables the use of + Kickstart, in which case sites specified in the + sites.xml file + must have valid + gridlaunch attributes. + The maybe value will + enable the use of Kickstart only + on sites that have the + gridlaunch attribute + specified. -<pool handle="example" sysinfo="INTEL32::LINUX"> - <gridftp url="gsiftp://example.org" storage="/scratch/swift" major="2" minor="4" patch="3"/> - <jobmanager universe="vanilla" url="example.org/jobmanager-pbs" major="2" minor="4" patch="3"/> - <workdirectory>/scratch/swift</workdirectory> - <profile namespace="SWIFT" key="storagesize">20000000</profile> -</pool> - - - - - The decision of which files to keep in the cache - and which files to remove is made considering - the value of the - caching.algorithm property. - Currently, the only available value for this - property is LRU, which would - cause the least recently used files to be - deleted first. - - - - execution.retries + + + + lazy.errors + + - Valid values: positive integers + Valid values: true, false + - Default value: 2 + Default value: false + - The number of time a job will be retried if it - fails (giving a maximum of 1 + - execution.retries attempts at execution) + + Swift can report application errors in two + modes, depending on the value of this property. + If set to false, Swift will + report the first error encountered and + immediately stop execution. If set to + true, Swift will attempt to + run as much as possible from a SwiftScript program before + stopping execution and reporting all errors + encountered. + When developing SwiftScript programs, using the + default value of false can + make the program easier to debug. However + in production runs, using true + will allow more of a SwiftScript program to be run before + Swift aborts execution. + - - + pgraph @@ -2472,173 +2553,113 @@ - - + + - clustering.enabled + sitedir.keep - - Valid values: true, false + Valid values: true, false - + Default value: false - + - Enables clustering. +Indicates whether the working directory on the remote site should be +left intact even when a run completes successfully. This can be +used to inspect the site working directory for debugging purposes. - - + - clustering.queue.delay + sites.file - - - Valid values: <int> - - Default value: 4 + Valid values: <file> - - This property indicates the interval, in - seconds, at which the clustering queue is - processed. - + Default value: ${swift.home}/etc/sites.xml - - - - - - clustering.min.time - - - Valid values: <int> - - - Default value: 60 - - - - - Indicates the threshold wall time for - clustering, in seconds. Jobs that have a - wall time smaller than the value of this - property will be considered for clustering. + Points to the location of the site + catalog, which contains a list of all sites that + Swift should use. + - + - kickstart.enabled + status.mode - - Valid values: true, false, maybe + Valid values: files, provider - + - Default value: maybe + Default value: files - - - - This option allows controlling of - when Swift uses Kickstart. A value of - false disables the use of - Kickstart, while a value of - true enables the use of - Kickstart, in which case sites specified in the - sites.xml file - must have valid - gridlaunch attributes. - The maybe value will - enable the use of Kickstart only - on sites that have the - gridlaunch attribute - specified. - - - - - - - kickstart.always.transfer - - - - Valid values: true, false +Controls how Swift will communicate the result code of running user programs +from workers to the submit side. In files mode, a file +indicating success or failure will be created on the site shared filesystem. +In provider mode, the execution provider job status will +be used. - - Default value: false +provider mode requires the underlying job execution system +to correctly return exit codes. In at least the cases of GRAM2, and clusters +used with any provider, exit codes are not returned, and so +files mode must be used in those cases. Otherwise, +provider mode can be used to reduce the amount of +filesystem access. (since Swift 0.8) - - - - This property controls when output from - Kickstart is transfered back to the submit site, - if Kickstart is enabled. When set to - false, Kickstart output is - only transfered for jobs that fail. If set to - true, Kickstart output is - transfered after every job is completed or - failed. - - - wrapperlog.always.transfer + tc.file - - Valid values: true, false + Valid values: <file> - Default value: false + Default value: ${swift.home}/etc/tc.data - + - This property controls when output from - the Swift remote wrapper is transfered - back to the submit site. When set to - false, wrapper logs are - only transfered for jobs that fail. If set to - true, wrapper logs are - transfered after every job is completed or - failed. + Points to the location of the transformation + catalog file which contains information about + installed applications. Details about the format + of the transformation catalog can be found + here. - + - throttle.submit + throttle.file.operations @@ -2647,28 +2668,28 @@ - Default value: 4 + Default value: 8 - + - Limits the number of concurrent submissions for - a run. This throttle only limits - the number of concurrent tasks (jobs) that are - being sent to sites, not the total number of - concurrent jobs that can be run. The submission - stage in GRAM is one of the most CPU expensive - stages (due mostly to the mutual authentication - and delegation). Having too many concurrent - submissions can overload either or both the - submit host CPU and the remote host/head node - causing degraded performance. + Limits the total number of concurrent file + operations that can happen at any given time. + File operations (like transfers) require an + exclusive connection to a site. These + connections can be expensive to establish. A + large number of concurrent file operations may + cause Swift to attempt to establish many such + expensive connections to various sites. Limiting + the number of concurrent file operations causes + Swift to use a small number of cached + connections and achieve better overall + performance. - - + throttle.host.submit @@ -2695,68 +2716,7 @@ - - - - throttle.transfers - - - - - Valid values: <int>, off - - - - Default value: 4 - - - - - Limits the total number of concurrent file - transfers that can happen at any given time. - File transfers consume bandwidth. Too many - concurrent transfers can cause the network to be - overloaded preventing various other signaling - traffic from flowing properly. - - - - - - - throttle.file.operations - - - - - Valid values: <int>, off - - - - Default value: 8 - - - - - Limits the total number of concurrent file - operations that can happen at any given time. - File operations (like transfers) require an - exclusive connection to a site. These - connections can be expensive to establish. A - large number of concurrent file operations may - cause Swift to attempt to establish many such - expensive connections to various sites. Limiting - the number of concurrent file operations causes - Swift to use a small number of cached - connections and achieve better overall - performance. - - - - - - throttle.score.job.factor @@ -2803,54 +2763,64 @@ - sitedir.keep + throttle.submit + - Valid values: true, false + Valid values: <int>, off - + - Default value: false + Default value: 4 - + -Indicates whether the working directory on the remote site should be -left intact even when a run completes successfully. This can be -used to inspect the site working directory for debugging purposes. + + Limits the number of concurrent submissions for + a run. This throttle only limits + the number of concurrent tasks (jobs) that are + being sent to sites, not the total number of + concurrent jobs that can be run. The submission + stage in GRAM is one of the most CPU expensive + stages (due mostly to the mutual authentication + and delegation). Having too many concurrent + submissions can overload either or both the + submit host CPU and the remote host/head node + causing degraded performance. + + + - status.mode + throttle.transfers + - Valid values: files, provider + Valid values: <int>, off - + - Default value: files + Default value: 4 - + -Controls how Swift will communicate the result code of running user programs -from workers to the submit side. In files mode, a file -indicating success or failure will be created on the site shared filesystem. -In provider mode, the execution provider job status will -be used. + + Limits the total number of concurrent file + transfers that can happen at any given time. + File transfers consume bandwidth. Too many + concurrent transfers can cause the network to be + overloaded preventing various other signaling + traffic from flowing properly. + - -provider mode requires the underlying job execution system -to correctly return exit codes. In at least the cases of GRAM2, and clusters -used with any provider, exit codes are not returned, and so -files mode must be used in those cases. Otherwise, -provider mode can be used to reduce the amount of -filesystem access. (since Swift 0.8) - + ticker.disable @@ -2897,7 +2867,34 @@ - + + + wrapperlog.always.transfer + + + + + Valid values: true, false + + + + Default value: false + + + + + This property controls when output from + the Swift remote wrapper is transfered + back to the submit site. When set to + false, wrapper logs are + only transfered for jobs that fail. If set to + true, wrapper logs are + transfered after every job is completed or + failed. + + + + From noreply at svn.ci.uchicago.edu Wed Apr 8 07:27:10 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 07:27:10 -0500 (CDT) Subject: [Swift-commit] r2842 - trunk/docs Message-ID: <20090408122710.63D169D7A8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-08 07:27:07 -0500 (Wed, 08 Apr 2009) New Revision: 2842 Modified: trunk/docs/userguide.xml Log: duplicate property listing Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-08 12:26:19 UTC (rev 2841) +++ trunk/docs/userguide.xml 2009-04-08 12:27:07 UTC (rev 2842) @@ -1457,7 +1457,6 @@ caching.algorithm clustering.enabled -clustering.enabled clustering.min.time clustering.queue.delay ip.address From noreply at svn.ci.uchicago.edu Wed Apr 8 07:28:00 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 07:28:00 -0500 (CDT) Subject: [Swift-commit] r2843 - trunk/docs Message-ID: <20090408122800.F1EAC9D7A8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-08 07:27:56 -0500 (Wed, 08 Apr 2009) New Revision: 2843 Modified: trunk/docs/userguide.xml Log: userguide doc for foreach.max.threads Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-08 12:27:07 UTC (rev 2842) +++ trunk/docs/userguide.xml 2009-04-08 12:27:56 UTC (rev 2843) @@ -2326,6 +2326,26 @@ + + + foreach.max.threads + + + Valid values: positive integers + + + Default value: 1024 + + +Limits the number of concurrent iterations that each foreach statement +can have at one time. This conserves memory for swift programs that +have large numbers of iterations (which would otherwise all be executed +in parallel). (since Swift 0.9) + + + + + ip.address From noreply at svn.ci.uchicago.edu Wed Apr 8 07:28:50 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 07:28:50 -0500 (CDT) Subject: [Swift-commit] r2844 - trunk/docs Message-ID: <20090408122850.89F739D7A8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-08 07:28:46 -0500 (Wed, 08 Apr 2009) New Revision: 2844 Modified: trunk/docs/userguide.xml Log: userguide doc for replication properties Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-08 12:27:56 UTC (rev 2843) +++ trunk/docs/userguide.xml 2009-04-08 12:28:46 UTC (rev 2844) @@ -2572,9 +2572,52 @@ - + + replication.enabled + + + + + Valid values: true, false + + + + Default value: false + + + +Enables/disables replication. Replication is used to deal with jobs sitting +in batch queues for abnormally large amounts of time. If replication is enabled +and certain conditions are met, Swift creates and submits replicas of jobs, and +allows multiple instances of a job to compete. + + + + + + + replication.limit + + + + + Valid values: positive integers + + + + Default value: 3 + + + +The maximum number of replicas that Swift should attempt. + + + + + + sitedir.keep From noreply at svn.ci.uchicago.edu Wed Apr 8 07:29:40 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 07:29:40 -0500 (CDT) Subject: [Swift-commit] r2845 - trunk/docs Message-ID: <20090408122940.861EC9D7A8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-08 07:29:35 -0500 (Wed, 08 Apr 2009) New Revision: 2845 Modified: trunk/docs/userguide.xml Log: doc that ip.address is deprecated (as indicated in swift.properties) Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-08 12:28:46 UTC (rev 2844) +++ trunk/docs/userguide.xml 2009-04-08 12:29:35 UTC (rev 2845) @@ -2361,7 +2361,6 @@ - The Globus GRAM service uses a callback mechanism to send notifications about the status of submitted jobs. The callback mechanism @@ -2375,8 +2374,11 @@ address of the Swift client machine can be specified using this property. The value of this property must be a numeric address without quotes. - + + This option is deprecated and the hostname + property should be used instead. + From noreply at svn.ci.uchicago.edu Wed Apr 8 07:30:27 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 07:30:27 -0500 (CDT) Subject: [Swift-commit] r2846 - trunk/docs Message-ID: <20090408123027.B50309D7A8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-08 07:30:24 -0500 (Wed, 08 Apr 2009) New Revision: 2846 Modified: trunk/docs/userguide.xml Log: move tcp.port.range doc to swift.properties section Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-04-08 12:29:35 UTC (rev 2845) +++ trunk/docs/userguide.xml 2009-04-08 12:30:24 UTC (rev 2846) @@ -1433,16 +1433,6 @@ - -tcp.port.range start,end - - A TCP port range can be specified to restrict the ports on which - GRAM callback services are started. This is likely needed if your - submit host is behind a firewall, in which case the firewall - should be configured to allow incoming connections on ports in - the range. - - - -tui Displays an interactive text mode monitor during a run. (since Swift 0.9) @@ -1469,6 +1459,7 @@ sitedir.keep sites.file tc.file +tcp.port.range @@ -2721,6 +2712,24 @@ + + + tcp.port.range + + + Valid values: <start>,<end> where start and end are integers + Default value: none + +A TCP port range can be specified to restrict the ports on which +GRAM callback services are started. This is likely needed if your + submit host is behind a firewall, in which case the firewall +should be configured to allow incoming connections on ports in +the range. + + + + + throttle.file.operations From noreply at svn.ci.uchicago.edu Wed Apr 8 09:12:27 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 09:12:27 -0500 (CDT) Subject: [Swift-commit] r2847 - nmi-build-test/submit-machine Message-ID: <20090408141227.DAAF39D7A8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-08 09:12:19 -0500 (Wed, 08 Apr 2009) New Revision: 2847 Modified: nmi-build-test/submit-machine/kickstart.svn Log: change kickstart SVN location Modified: nmi-build-test/submit-machine/kickstart.svn =================================================================== --- nmi-build-test/submit-machine/kickstart.svn 2009-04-08 12:30:24 UTC (rev 2846) +++ nmi-build-test/submit-machine/kickstart.svn 2009-04-08 14:12:19 UTC (rev 2847) @@ -1,2 +1,2 @@ method = svn -url = https://smarty.isi.edu/svn/repo1/pegasus/trunk/src/tools/kickstart +url = http://pegasus.isi.edu/svn/pegasus/trunk/src/tools/kickstart From noreply at svn.ci.uchicago.edu Wed Apr 8 17:38:46 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Apr 2009 17:38:46 -0500 (CDT) Subject: [Swift-commit] r2848 - in usertools/cio: bin libexec Message-ID: <20090408223846.781E09D7A0@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-08 17:38:45 -0500 (Wed, 08 Apr 2009) New Revision: 2848 Modified: usertools/cio/bin/chtget.rb usertools/cio/bin/chtput.rb usertools/cio/libexec/hashserver.rb Log: Reimplemented hash service using raw TCP sockets. DRb support was removed. Typical torus requests now takes 10-20 milliseconds Modified: usertools/cio/bin/chtget.rb =================================================================== --- usertools/cio/bin/chtget.rb 2009-04-08 14:12:19 UTC (rev 2847) +++ usertools/cio/bin/chtget.rb 2009-04-08 22:38:45 UTC (rev 2848) @@ -1,16 +1,13 @@ -#!/home/espinosa/local/bin/ruby +#!/usr/bin/env ruby -require 'drb' +# Script: chtget.rb [file] [hashserver_ip] +# Description: Registers a file [file] with rank rank to the hash service +# [hashserver_ip] -DRb.start_service -cht = DRbObject.new(nil, 'druby://10.128.0.0:9000') +require 'socket' -#cht.put("/x/y", 0) -#cht.put("/x/z", 0) -#cht.put("/x/a", 3) -#cht.put("/x/b", 2) -#cht.put("/x/c", 2) -#cht.put("/x/d", 2) +client = TCPSocket.new(ARGV[1], 9000) - puts cht.get(ARGV[0]) - +client.print("get #{ARGV[0]}\n") +puts client.read +client.close Modified: usertools/cio/bin/chtput.rb =================================================================== --- usertools/cio/bin/chtput.rb 2009-04-08 14:12:19 UTC (rev 2847) +++ usertools/cio/bin/chtput.rb 2009-04-08 22:38:45 UTC (rev 2848) @@ -1,16 +1,12 @@ -#!/home/espinosa/local/bin/ruby +#!/usr/bin/env ruby -require 'drb' +# Script: chtput.rb [file] [rank] [hashserver_ip] +# Description: Registers a file [file] with rank rank to the hash service +# [hashserver_ip] -DRb.start_service -cht = DRbObject.new(nil, 'druby://10.128.0.0:9000') +require 'socket' -#cht.put("/x/y", 0) -#cht.put("/x/z", 0) -#cht.put("/x/a", 3) -#cht.put("/x/b", 2) -#cht.put("/x/c", 2) -puts ARGV[0] -puts ARGV[1] -cht.put(ARGV[0], ARGV[1]) +client = TCPSocket.new(ARGV[2], 9000) +client.print "put #{ARGV[0]} #{ARGV[1]}" +client.close Modified: usertools/cio/libexec/hashserver.rb =================================================================== --- usertools/cio/libexec/hashserver.rb 2009-04-08 14:12:19 UTC (rev 2847) +++ usertools/cio/libexec/hashserver.rb 2009-04-08 22:38:45 UTC (rev 2848) @@ -1,26 +1,19 @@ -#!/home/espinosa/local/bin/ruby +#!/usr/bin/env ruby -require 'drb' +require 'socket' +require 'scanf' -class HashServer - attr_reader :file +server = TCPServer.new( nil, 9000 ) +file = Hash.new { |hash, key| hash[key] = Array.new } - def initialize - @file = Hash.new { |hash, key| hash[key] = Array.new } +while true + Thread.start server.accept do |session| + (method, fname, rank) = session.gets.scanf "%s %s %d\n" + if method == "put" + file[fname] << rank + elsif method == "get" + session.print "#{file[fname][ rand( file[fname].size ) ]}\n" + end + session.close end - - def put(fname, rank) - @file[fname] << rank - end - - def get(fname) - @file[fname][ rand( @file[fname].size ) ] - end - end - -server = HashServer.new - -DRb.start_service('druby://*:9000', server) -DRb.thread.join - From noreply at svn.ci.uchicago.edu Thu Apr 9 14:25:10 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 9 Apr 2009 14:25:10 -0500 (CDT) Subject: [Swift-commit] r2849 - usertools/cio/science/blast Message-ID: <20090409192510.122C89D7AA@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-09 14:25:09 -0500 (Thu, 09 Apr 2009) New Revision: 2849 Added: usertools/cio/science/blast/tc.bgp Removed: usertools/cio/science/blast/tc.data Modified: usertools/cio/science/blast/blast.swift usertools/cio/science/blast/mapper-in.sh usertools/cio/science/blast/mapper-out.sh usertools/cio/science/blast/measure-runblast.sh usertools/cio/science/blast/mockblast.sh usertools/cio/science/blast/readseq.rb usertools/cio/science/blast/runblast.sh Log: Test framework for MTDM and vanilla+bgp_enhancement swift Modified: usertools/cio/science/blast/blast.swift =================================================================== --- usertools/cio/science/blast/blast.swift 2009-04-08 22:38:45 UTC (rev 2848) +++ usertools/cio/science/blast/blast.swift 2009-04-09 19:25:09 UTC (rev 2849) @@ -24,13 +24,23 @@ cat @filenames(out.out) stdout=@sum; } -BlastDatabase pir[] ; -BlastResult out[] ; -BlastQuery input[] ; +string db=@arg("db"); +string db_prefix=@arg("db_pref"); +string start=@arg("start"); +string end=@arg("end"); +string limit=@arg("limit"); +string runid=@arg("run"); +BlastDatabase pir[] ; +BlastResult out[] ; +BlastQuery input[] ; + foreach data,i in input { (out[i]) = blastall(data, pir); } -BlastSummary sum <"summary.out">; +BlastSummary sum ; sum = summarize(out); Modified: usertools/cio/science/blast/mapper-in.sh =================================================================== --- usertools/cio/science/blast/mapper-in.sh 2009-04-08 22:38:45 UTC (rev 2848) +++ usertools/cio/science/blast/mapper-in.sh 2009-04-09 19:25:09 UTC (rev 2849) @@ -1,18 +1,21 @@ #!/bin/bash -#while getopts ":a:b:d:" options; do -# case $options in -# a) export a=$OPTARG ;; -# b) export b=$OPTARG ;; -# d) export d=$OPTARG ;; -# *) exit 1;; -# esac -#done +while getopts ":a:b:d:l:" options; do + case $options in + a) export a=$OPTARG ;; + b) export b=$OPTARG ;; + d) export d=$OPTARG ;; + l) export l=$OPTARG ;; + *) exit 1;; + esac +done -export a=1 -export b=2 -export d=0000 - -ls /disks/ci-gpfs/swift/blast/pir/input/$d/*.qry | \ - perl -wnl -e '$. >= $ENV{"a"} and $. <= $ENV{"b"} - and print "[", $.-$ENV{"a"}, "] $_";' +for (( i = a; i <= b; i++ )); do + seqid=`printf %07d $i` + if [[ $l == "l" ]]; then + prefix=$d/`printf %04d $(( i / 1000 ))` + else + prefix=$d + fi + echo [$(($i-a))] $prefix/SEQ$seqid\_*.qry +done Modified: usertools/cio/science/blast/mapper-out.sh =================================================================== --- usertools/cio/science/blast/mapper-out.sh 2009-04-08 22:38:45 UTC (rev 2848) +++ usertools/cio/science/blast/mapper-out.sh 2009-04-09 19:25:09 UTC (rev 2849) @@ -1,22 +1,26 @@ #!/bin/bash -#while getopts ":a:b:d:" options; do -# case $options in -# a) export a=$OPTARG ;; -# b) export b=$OPTARG ;; -# d) export d=$OPTARG ;; -# *) exit 1;; -# esac -#done -export a=1 -export b=2 -export d=0000 +while getopts ":a:b:d:l:" options; do + case $options in + a) export a=$OPTARG ;; + b) export b=$OPTARG ;; + d) export d=$OPTARG ;; + l) export l=$OPTARG ;; + *) exit 1;; + esac +done -ls /disks/ci-gpfs/swift/blast/pir/input/$d/*.qry | \ - perl -wnl -e '$. >= $ENV{"a"} and $. <= $ENV{"b"} - and s/qry/out/ and s/input/output/ - and print "[", $.-$ENV{"a"}, "].out $_";' -ls /disks/ci-gpfs/swift/blast/pir/input/$d/*.qry | \ - perl -wnl -e '$. >= $ENV{"a"} and $. <= $ENV{"b"} - and s/qry/err/ and s/input/error/ - and print "[", $.-$ENV{"a"}, "].err $_";' + +for (( i = a; i <= b; i++ )); do + seqid=`printf %07d $i` + if [[ $l == "l" ]]; then + prefix=$d/`printf %04d $(( i / 1000 ))` + else + prefix=$d + fi + if [ ! -d $prefix ]; then + mkdir -p $prefix + fi + echo [$(($i-a))].out $prefix/SEQ$seqid.out + echo [$(($i-a))].err $prefix/SEQ$seqid.err +done Modified: usertools/cio/science/blast/measure-runblast.sh =================================================================== --- usertools/cio/science/blast/measure-runblast.sh 2009-04-08 22:38:45 UTC (rev 2848) +++ usertools/cio/science/blast/measure-runblast.sh 2009-04-09 19:25:09 UTC (rev 2849) @@ -1 +1,34 @@ -./swiftblast.sh - UNIPROT_for_blast_14.0.seq 100001 100010 +#!/bin/bash + +PATH=$PATH:$CIOROOT/bin + +set_vanilla() +{ + cat > ~/bin/swift << EOF +#!/bin/bash +/home/falkon/swift_vanilla/cog/modules/swift/dist/swift-svn/bin/swift $@ +EOF +} + +set_mtdm() +{ + cat > ~/bin/swift << EOF +#!/bin/bash +/home/falkon/swift_working/cog/modules/swift/dist/swift-svn/bin/swift $@ +EOF +} + +NSIZE="64 128 256 512 1024" +for i in $NSIZE; do + $CIOROOT/bin/falkon-start.sh default $i 60 4 1 4 + set_vanilla + cobalt_id=`tail -1 $HOME/.falkonjobs | awk '{print $1}'` + ./runblast.sh . common/UNIPROT_for_blast_14.0.seq \ + 100001 104000 l out.run_`printf %03d $i` + set_mtdm + falkon_id=`tail -1 $HOME/.falkonjobs | awk '{print $2}'` + /home/zzhang/DHT/bin/treebroadcast.sh $falkon_id common/common.tar + ./runblast.sh . common/UNIPROT_for_blast_14.0.seq \ + 100001 104000 l out.run_`printf %03d $i` + qdel $cobalt_id +done Modified: usertools/cio/science/blast/mockblast.sh =================================================================== --- usertools/cio/science/blast/mockblast.sh 2009-04-08 22:38:45 UTC (rev 2848) +++ usertools/cio/science/blast/mockblast.sh 2009-04-09 19:25:09 UTC (rev 2849) @@ -21,21 +21,24 @@ BLOCKS=1310272 -dd if=$query of=/dev/null bs=$BLOCKS #2> /dev/null +dd=/fuse/bin/dd +ls=/fuse/bin/ls +$dd if=$query of=/dev/null bs=$BLOCKS #2> /dev/null + for i in $db; do if [ -h $i ]; then truefile=`readlink $i` else truefile=$i fi - size=`ls --block-size=$BLOCKS -s $truefile | cut -f 1 -d ' '` + size=`$ls --block-size=$BLOCKS -s $truefile | cut -f 1 -d ' '` count=$(( size / 4)) if [ $count -eq 0 ]; then count=$size fi echo $count - dd if=$i of=/dev/null bs=$BLOCKS count=$count #2> /dev/null + $dd if=$i of=/dev/null bs=$BLOCKS count=$count #2> /dev/null done sleep 41 -dd if=/dev/zero of=$output bs=15812 count=1 #2> /dev/null +$dd if=/dev/zero of=$output bs=15812 count=1 #2> /dev/null Modified: usertools/cio/science/blast/readseq.rb =================================================================== --- usertools/cio/science/blast/readseq.rb 2009-04-08 22:38:45 UTC (rev 2848) +++ usertools/cio/science/blast/readseq.rb 2009-04-09 19:25:09 UTC (rev 2849) @@ -21,7 +21,7 @@ x =~ />(.*)\n/ seqname = $1 if seq_start <= fasta_db.lineno - dir = pad == "l" ? prefix + + dir = pad == "l" ? prefix + sprintf("/%04d", fasta_db.lineno / 1000) : prefix fname = sprintf "SEQ%07d_%s.qry", fasta_db.lineno, seqname FileUtils.mkdir_p dir Modified: usertools/cio/science/blast/runblast.sh =================================================================== --- usertools/cio/science/blast/runblast.sh 2009-04-08 22:38:45 UTC (rev 2848) +++ usertools/cio/science/blast/runblast.sh 2009-04-09 19:25:09 UTC (rev 2849) @@ -1,18 +1,25 @@ #!/bin/bash -# Script: swiftblast.sh [blast_db] [start_seq] [end_seq] [outdir] +# Script: swiftblast.sh [falkon_id] [blast_db] [start_seq] [end_seq] [outdir] # Invokes the swift workflow of reciprocal blast if [ $# -lt 3 ]; then cat << EOF ERROR : too few arguments -Usage : $0 [blast_db] [start_seq] [end_seq] [outdir] +Usage : $0 [falkon_id][blast_db] [start_seq] [end_seq] [outdir] Example : Reciprocal blast on the 100th to 200th sequences of UNIPRT.seq and dump results to run_100: $0 UNIPROT.seq 100 200 run_100 EOF fi +if [ -z $CIOROOT ]; then + echo "ERROR: CIOROOT not defined" + exit 1 +fi + +PATH=$PATH:$CIOROOT/bin + FALKON_ID=$1 BLAST_DB=$2 START_SEQ=$3 @@ -20,18 +27,49 @@ # Limit to 1k sequences per dir "l" LIMIT=$5 +RUNID=$6 # Default output directory of everything! BLASTROOT=$CIOROOT/science/blast +if [ ! -d $RUNID ]; then + mkdir -p $RUNID +fi + # Extract sequences -ruby readseq.rb $BLAST_DB $START_SEQ $END_SEQ seqs -if [ $FALKON_ID != "-" ]; then # Real Falkon job - sleep 0 - # Regenerate tc.data and sites.xml @BGP@ +ruby readseq.rb `dirname $BLAST_DB`/fasta/`basename $BLAST_DB` \ + $START_SEQ $END_SEQ $RUNID $LIMIT + +if [ $FALKON_ID == "." ]; then + FALKON_ID=`tail -1 $HOME/.falkonjobs | awk '{print $2}'` +fi + +if [ $FALKON_ID == "-" ]; then # Real Falkon job + tcfile=tc.local + cp $tcfile tc.data else - sleep 0 + tcfile=tc.bgp + gentcdata 32 < $tcfile > tc.data fi -# Run swift: -#swift -sites.file sites.xml -tc.file tc.data $BLASTROOT/blast.swift + +if [ $FALKON_ID == "-" ]; then + echo "local" + swift -sites.file ./sites.xml -tc.file tc.data blast.swift \ + -db=`basename $BLAST_DB` -db_pref=`dirname $BLAST_DB`\ + -limit=$LIMIT -start=$START_SEQ -end=$END_SEQ \ + -run=$RUNID +else # Real Falkon job + fjob=`echo $FALKON_ID|awk '{printf "%.4d",$1}'` + nodes=`cat $HOME/.falkonjobs | awk "\\$2==\"$fjob\" {print \\$4}"` + fdir=/home/falkon/users/$USER + if [ ! -d $fdir/$fjob ]; then + echo $0: Falkon job id $fjob does not exist in $fdir. Last 5 jobs there are: + (cd $fdir; ls -1 | tail -5) + exit 1 + fi + /usr/bin/time -p swift_bgp.sh $fjob $nodes $BLASTROOT/blast.swift \ + -db=`basename $BLAST_DB` -db_pref=`dirname $BLAST_DB`\ + -limit=$LIMIT -start=$START_SEQ -end=$END_SEQ \ + -run=$RUNID >&swift_out.$RUNID +fi Copied: usertools/cio/science/blast/tc.bgp (from rev 2835, usertools/cio/science/blast/tc.data) =================================================================== --- usertools/cio/science/blast/tc.bgp (rev 0) +++ usertools/cio/science/blast/tc.bgp 2009-04-09 19:25:09 UTC (rev 2849) @@ -0,0 +1,24 @@ +#This is the transformation catalog. +# +#It comes pre-configured with a number of simple transformations with +#paths that are likely to work on a linux box. However, on some systems, +#the paths to these executables will be different (for example, sometimes +#some of these programs are found in /usr/bin rather than in /bin) +# +#NOTE WELL: fields in this file must be separated by tabs, not spaces; and +#there must be no trailing whitespace at the end of each line. +# +# sitename transformation path INSTALLED platform profiles +localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost cat /bin/cat INSTALLED INTEL32::LINUX null +localhost ls /bin/ls INSTALLED INTEL32::LINUX null +localhost grep /bin/grep INSTALLED INTEL32::LINUX null +localhost sort /bin/sort INSTALLED INTEL32::LINUX null +localhost paste /bin/paste INSTALLED INTEL32::LINUX null + +# BLAST binaries +communicado blastall /home/aespinosa/science/blast/ncbi.communicado/bin/blastall INSTALLED INTEL32::LINUX null +communicado mockblast /home/aespinosa/cio/science/blast/mockblast.sh INSTALLED INTEL32::LINUX null + at BGP@ blastall /home/espinosa/blast/ncbi/bin/blastall INSTALLED INTEL32::LINUX null + at BGP@ mockblast /home/espinosa/cio/science/blast/mockblast.sh INSTALLED INTEL32::LINUX null + at BGP@ cat /bin/cat INSTALLED INTEL32::LINUX null Deleted: usertools/cio/science/blast/tc.data =================================================================== --- usertools/cio/science/blast/tc.data 2009-04-08 22:38:45 UTC (rev 2848) +++ usertools/cio/science/blast/tc.data 2009-04-09 19:25:09 UTC (rev 2849) @@ -1,23 +0,0 @@ -#This is the transformation catalog. -# -#It comes pre-configured with a number of simple transformations with -#paths that are likely to work on a linux box. However, on some systems, -#the paths to these executables will be different (for example, sometimes -#some of these programs are found in /usr/bin rather than in /bin) -# -#NOTE WELL: fields in this file must be separated by tabs, not spaces; and -#there must be no trailing whitespace at the end of each line. -# -# sitename transformation path INSTALLED platform profiles -localhost echo /bin/echo INSTALLED INTEL32::LINUX null -localhost cat /bin/cat INSTALLED INTEL32::LINUX null -localhost ls /bin/ls INSTALLED INTEL32::LINUX null -localhost grep /bin/grep INSTALLED INTEL32::LINUX null -localhost sort /bin/sort INSTALLED INTEL32::LINUX null -localhost paste /bin/paste INSTALLED INTEL32::LINUX null - -# BLAST binaries -communicado blastall /home/aespinosa/science/blast/ncbi.communicado/bin/blastall INSTALLED INTEL32::LINUX null -communicado mockblast /home/aespinosa/cio/science/blast/mockblast.sh INSTALLED INTEL32::LINUX null -surveyor blastall /home/espinosa/science/blast/ncbi.communicado/bin/blastall INSTALLED INTEL32::LINUX null -surveyor mockblast /home/espinosa/cio/science/blast/mockblast.sh INSTALLED INTEL32::LINUX null From noreply at svn.ci.uchicago.edu Thu Apr 9 14:29:16 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 9 Apr 2009 14:29:16 -0500 (CDT) Subject: [Swift-commit] r2850 - usertools/cio/bin Message-ID: <20090409192916.E73EF9D7AA@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-09 14:29:16 -0500 (Thu, 09 Apr 2009) New Revision: 2850 Modified: usertools/cio/bin/chtget.rb usertools/cio/bin/chtput.rb Log: CHT to use adhoc TCP protocol instead of DRb Modified: usertools/cio/bin/chtget.rb =================================================================== --- usertools/cio/bin/chtget.rb 2009-04-09 19:25:09 UTC (rev 2849) +++ usertools/cio/bin/chtget.rb 2009-04-09 19:29:16 UTC (rev 2850) @@ -1,3 +1,4 @@ +#!/home/espinosa/local/bin/ruby #!/usr/bin/env ruby # Script: chtget.rb [file] [hashserver_ip] Modified: usertools/cio/bin/chtput.rb =================================================================== --- usertools/cio/bin/chtput.rb 2009-04-09 19:25:09 UTC (rev 2849) +++ usertools/cio/bin/chtput.rb 2009-04-09 19:29:16 UTC (rev 2850) @@ -1,3 +1,4 @@ +#!/home/espinosa/local/bin/ruby #!/usr/bin/env ruby # Script: chtput.rb [file] [rank] [hashserver_ip] From noreply at svn.ci.uchicago.edu Thu Apr 9 14:29:45 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 9 Apr 2009 14:29:45 -0500 (CDT) Subject: [Swift-commit] r2851 - usertools/cio/bin Message-ID: <20090409192945.1F2829D7AA@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-09 14:29:44 -0500 (Thu, 09 Apr 2009) New Revision: 2851 Added: usertools/cio/bin/falkon_vanilla.sh usertools/cio/bin/gentcdata usertools/cio/bin/swift_bgp.sh Modified: usertools/cio/bin/ciologic-bgp.sh Log: Swift wrapper for the bluegene Modified: usertools/cio/bin/ciologic-bgp.sh =================================================================== --- usertools/cio/bin/ciologic-bgp.sh 2009-04-09 19:29:16 UTC (rev 2850) +++ usertools/cio/bin/ciologic-bgp.sh 2009-04-09 19:29:44 UTC (rev 2851) @@ -134,8 +134,9 @@ #Core MTIO logic if [ $DHT -eq 0 ]; then - $CIOROOT/libexec/hashserver.rb -else [ $RANK -eq $IFS_RANK ]; then + RUBY=/home/espinosa/local/bin/ruby + $RUBY $CIOROOT/libexec/hashserver.rb +elif [ $RANK -eq $IFS_RANK ]; then ifs_head $STRIPE_SIZE /home/zzhang/cio/bin/collector.sh elif [ $RANK -lt $IFS_SLAVE ]; then Copied: usertools/cio/bin/falkon_vanilla.sh (from rev 2835, usertools/cio/bin/falkon-start.sh) =================================================================== --- usertools/cio/bin/falkon_vanilla.sh (rev 0) +++ usertools/cio/bin/falkon_vanilla.sh 2009-04-09 19:29:44 UTC (rev 2851) @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ -z $CIOROOT ]; then + echo "ERROR: CIOROOT env not defined" + exit 1 +fi +if [ -z $CIOARCH ]; then + echo "ERROR: CIOARCH not defined" + exit 1 +fi +if [ -z $CHIRPROOT ]; then + echo "ERROR: CHIRPROOT not defined" + exit 1 +fi +$CIOROOT/libexec/falkon/falkon_vanilla-$CIOARCH.sh $@ Added: usertools/cio/bin/gentcdata =================================================================== --- usertools/cio/bin/gentcdata (rev 0) +++ usertools/cio/bin/gentcdata 2009-04-09 19:29:44 UTC (rev 2851) @@ -0,0 +1,13 @@ +#!/bin/sh + +awk -v count=$1 ' + +/@BGP@/ { + for (i=0;i " + exit 1 +fi + +CUR_DIR=`pwd` + +FALKON_JOB_ID=$1 +NUM_NODES=$2 +SWIFT_SCRIPT=$3 + +SERVICE_LIST_FILE=/home/falkon/users/${USER}/${FALKON_JOB_ID}/config/Client-service-URIs.config +let NUM_ION=NUM_NODES/64 + +echo "waiting for at least ${NUM_NODES} nodes to register before submitting workload..." +falkon-wait-for-allocation.sh ${SERVICE_LIST_FILE} ${NUM_ION} + +echo "found at least ${NUM_NODES} registered, submitting workload..." + +workdir=`pwd`/work + +# Base entry for localhost + +cat >sites.xml < +END + +# Create one entry per falkon server (i.e. one per pset) + +let s=0 +tail +2 $SERVICE_LIST_FILE | +while read ip port; do + server=`echo $s | awk '{printf "%.3d", $1}'` + cat >>sites.xml < + + + $workdir + 8 + 1000 + + +END + let s=$s+1 +done + +echo "" >> sites.xml + +shift 2 # leave $3 and following - swift script and args + +if [ ! -f ./tc.data ]; then + echo "ERROR: tc.data not found" + exit 1 +fi + +swift Author: andric Date: 2009-04-09 16:35:01 -0500 (Thu, 09 Apr 2009) New Revision: 2852 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/PK2.swift Log: for doing peak analysis Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/PK2.swift =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/PK2.swift (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/PK2.swift 2009-04-09 21:35:01 UTC (rev 2852) @@ -0,0 +1,36 @@ +#---- doing peakfit on PK2 pilot subject +type file{} + +type PeakResult{ + file pdf; + file a; + file b; +} + +app (PeakResult outObj) peakScript (file scriptFile, file tsfile, file preprocR, file peakfitR){ + RInvoke @filename(scriptFile) @filename(tsfile) @filename(outObj); +} + + +file script<"peak_scripts/Shellpeak_PK2runbyrun.R">; +file preprocR<"peak_scripts/preprocess.Enewsmooth.R">; +file peakfitR<"peak_scripts/peakfit_2.1.R">; + +string declarelist[] = ["PK2"]; +foreach subject in declarelist{ + int runs[] = [1:8]; + foreach rn in runs{ + string hemilist[] = ["lh","rh"]; + foreach h in hemilist{ + int inputRegionNo[] = [15,33,34,60]; + foreach rr in inputRegionNo{ + string project_dir=@strcat("/disks/ci-gpfs/fmri/cnari/swift/projects/andric/peakfit_pilots/",subject); + ## map inputs + file inFile; + ## map results + PeakResult outObj; + (outObj)=peakScript(script,inFile, preprocR, peakfitR); + } + } + } +} From noreply at svn.ci.uchicago.edu Thu Apr 9 16:37:19 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 9 Apr 2009 16:37:19 -0500 (CDT) Subject: [Swift-commit] r2853 - in SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2: . peak_scripts Message-ID: <20090409213719.81E2F9D7AA@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-04-09 16:37:19 -0500 (Thu, 09 Apr 2009) New Revision: 2853 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak.R SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak_PK2runbyrun.R SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/peakfit_2.1.R SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/preprocess.Enewsmooth.R Log: scripts dir for doing peak analysis Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak.R =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak.R (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak.R 2009-04-09 21:37:19 UTC (rev 2853) @@ -0,0 +1,59 @@ +library(stats) +source("scripts/preprocess.Enewsmooth.R") +source("scripts/peakfit_2.1.R") + +# get quick and dirty estimate for where baseline should be for the assessment. +# because the cycle is 17on,10off, 63% of points should be above whatever baseline really is +# meaning that if we sort the y-=values of the 108TR series, it should be rank #33 + +allinputs <- Sys.getenv("R_SWIFT_ARGS") +inputfilename <- noquote(strsplit(allinputs," ")[[1]][1]) +outputPDFfilename <- noquote(strsplit(allinputs," ")[[1]][2]) +outputAfilename <- noquote(strsplit(allinputs," ")[[1]][3]) +outputBfilename <- noquote(strsplit(allinputs," ")[[1]][4]) + + +dd <- read.table(inputfilename) +## make sure to change line below to fit the number of timepoints in ts +quickbase <- (sort(dd[,1])[66]) +adjusted.input = dd[,1]-quickbase +res_preprocess <- preprocess_1(input = adjusted.input); + +test2_res_D2_nlminb <- nlminb( +start = res_preprocess$guess, +objective = fcn_L2, +gradient = test_D_fcn_L2, +#hessian = test_D2_fcn_L2, +control = list(eval.max = 50000, iter.max = 5000), +# the lower line with fewer iterations is just for testing purposes. the upper line is the real one +# control = list(eval.max = 100, iter.max = 50), +lower = rep(c(0),length(res_preprocess$guess)), +fcall = f, +n = res_preprocess$n, +x = res_preprocess$interpolated_x, +y_and_noise = res_preprocess$interpolated_y) + +pdf(file=outputPDFfilename) +op <- par(mfrow=c(1,1)) +plot(res_preprocess$interpolated_x,res_preprocess$interpolated_y,xlab = "",ylab = "BOLD (1E-3)",type = "b"); +plot.est(par = test2_res_D2_nlminb$par, fcall = f, n = res_preprocess$n, res_preprocess$interpolated_x,col = "blue"); +plot.est.individual(par = test2_res_D2_nlminb$par, fcall = f, n = res_preprocess$n, res_preprocess$interpolated_x,col = "blue"); +dev.off() +par(op) + +result1<- est.stats.summary( +par = test2_res_D2_nlminb$par, +sse = test2_res_D2_nlminb$objective, +x = res_preprocess$interpolated_x, +y = res_preprocess$interpolated_y) + +result2 <- est.summary( +par = test2_res_D2_nlminb$par, +sse = test2_res_D2_nlminb$objective, +n = res_preprocess$n, +x = res_preprocess$interpolated_x, +y = res_preprocess$interpolated_y) + + +write.table(result1,outputAfilename,row.name = F,col.name = F) +write.table(result2,outputBfilename,row.name = F,col.name = F) Property changes on: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak.R ___________________________________________________________________ Name: svn:executable + * Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak_PK2runbyrun.R =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak_PK2runbyrun.R (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/Shellpeak_PK2runbyrun.R 2009-04-09 21:37:19 UTC (rev 2853) @@ -0,0 +1,64 @@ +library(stats) +source("peak_scripts/preprocess.Enewsmooth.R") +source("peak_scripts/peakfit_2.1.R") + +# get quick and dirty estimate for where baseline should be for the assessment. +# because the cycle is 17on,10off, 63% of points should be above whatever baseline really is +# meaning that if we sort the y-=values of the 108TR series, it should be rank #33 + +#------------------------- the above is Uri's notes for his time series, mine below ---------------- +#--- doing PK2 run by run; there are 8 total runs, each with 226 TRs +#--- roughly %50 are null events, rest have an event in each run + +allinputs <- Sys.getenv("R_SWIFT_ARGS") +inputfilename <- noquote(strsplit(allinputs," ")[[1]][1]) +outobjectname <- noquote(strsplit(allinputs," ")[[1]][2]) +print(outobjectname) +outputPDFfilename <- paste(outobjectname,"pdf",sep="") +outputAfilename <- paste(outobjectname,"a",sep="") +outputBfilename <- paste(outobjectname,"b",sep="") + +dd <- read.table(inputfilename) +## make sure to change line below to fit the number of timepoints in ts +quickbase <- (sort(dd[,1])[113]) +adjusted.input = dd[,1]-quickbase +res_preprocess <- preprocess_1(input = adjusted.input); + +test2_res_D2_nlminb <- nlminb( + start = res_preprocess$guess, + objective = fcn_L2, + gradient = test_D_fcn_L2, +#hessian = test_D2_fcn_L2, + control = list(eval.max = 50000, iter.max = 5000), +# the lower line with fewer iterations is just for testing purposes. the upper line is the real one +# control = list(eval.max = 100, iter.max = 50), + lower = rep(c(0),length(res_preprocess$guess)), + fcall = f, + n = res_preprocess$n, + x = res_preprocess$interpolated_x, + y_and_noise = res_preprocess$interpolated_y) + +pdf(file=outputPDFfilename) +op <- par(mfrow=c(1,1)) +plot(res_preprocess$interpolated_x,res_preprocess$interpolated_y,xlab = "",ylab = "BOLD (1E-3)",type = "b"); +plot.est(par = test2_res_D2_nlminb$par, fcall = f, n = res_preprocess$n, res_preprocess$interpolated_x,col = "blue"); +plot.est.individual(par = test2_res_D2_nlminb$par, fcall = f, n = res_preprocess$n, res_preprocess$interpolated_x,col = "blue"); +dev.off() +par(op) + +result1<- est.stats.summary( + par = test2_res_D2_nlminb$par, + sse = test2_res_D2_nlminb$objective, + x = res_preprocess$interpolated_x, + y = res_preprocess$interpolated_y) + +result2 <- est.summary( + par = test2_res_D2_nlminb$par, + sse = test2_res_D2_nlminb$objective, + n = res_preprocess$n, + x = res_preprocess$interpolated_x, + y = res_preprocess$interpolated_y) + + +write.table(result1,outputAfilename,row.name = F,col.name = F) +write.table(result2,outputBfilename,row.name = F,col.name = F) Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/peakfit_2.1.R =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/peakfit_2.1.R (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/peakfit_2.1.R 2009-04-09 21:37:19 UTC (rev 2853) @@ -0,0 +1,385 @@ +# ---------------------------------------------------------------------- +# Savitzky-Golay Algorithm +# ---------------------------------------------------------------------- +# T2 <- sav.gol(T, fl, forder, dorder); +# +# Polynomial filtering method of Savitzky and Golay +# See Numerical Recipes, 1992, Chapter 14.8, for details. +# +# T = vector of signals to be filtered +# (the derivative is calculated for each ROW) +# fl = filter length (for instance fl = 51..151) +# forder = filter order (2 = quadratic filter, 4= quartic) +# dorder = derivative order (0 = smoothing, 1 = first derivative, etc.) +# +sav.gol <- function(T, fl, forder, dorder) +{ + m <- length(T) + dorder <- dorder + 1 + + # -- calculate filter coefficients -- + fc <- (fl-1)/2 # index: window left and right + X <- outer(-fc:fc, 0:forder, FUN="^") # polynomial terms and +coefficients + Y <- pinv(X); # pseudoinverse + + # -- filter via convolution and take care of the end points -- + T2 <- convolve(T, rev(Y[dorder,]), type="o") # convolve(...) + T2 <- T2[(fc+1):(length(T2)-fc)] +} +#----------------------------------------------------------------------- +# *** PseudoInvers of a Matrix *** +# using singular value decomposition +# +pinv <- function (A) +{ + s <- svd(A) + s$v %*% diag(1/s$d) %*% t(s$u) +} +#----------------------------------------------------------------------- +# +# +#------------gamma function used in peakfit------------------------------------------- +# +# | x-a1 | | (x-a1)/a2 + a3 -1 |^(a3-1) +# y = a0*exp|- -------|*| ------------------ | +# | a2 | | a3-1 | +# a0 = amplitude +# a1 = center +# a2 = width (>0) +# a3 = shape (> 1.01, < 167.92) +# + +f_peakfit <- function(a0,a1,a2,a3,n,x) { + expr <- rep(c(0),length(x)); + for (i in 1:n) + { + tmp <- rep(c(0),length(x)) + tmp <- a0[i]*exp(-(x - a1[i])/a2[i])*((x - a1[i])/(a2[i]*(a3[i] - 1)) + 1 )^(a3[i] - 1); +# when { b[i]/shape*(x -center[i]) + 1 } < 0, the power is meaningless. tmp[j] is "NaN" and will be replaced by zero + for (j in 1:length(x)) {if (tmp[j] == "NaN") {tmp[j] = 0} } + expr <- expr + tmp + } + eval(expr) +} + + +# define the function (a kind of simplified version from above) to be used to fit the data +# +# y = a*exp(-b*(x - center))*(b/shape*(x - center) + 1)^(shape) +# +# a = amplitude +# b = decay parameter (correspond to 1/a2) +# center = a1; +# shape = a3 - 1; later shape will be fixed as 8.6, which is a common number used in most of fMRI study +# +# +# ------------------------------------------------------------------------------------------- +# the following is a summation of gmma functions to fit raw data +# assumption: linear system +# x is the time points +# n is the number of peaks detected by second derivatives +# a, b and center are the parameters defined above and need to be estimated +# a, b and center are vectors with length corresponding to the number of peaks +# + + +# shape is fixed at 8.6 in all the following analysis +shape = 8.6 +f <- function(a,b,center,n,x) { + shape = 8.6; + expr <- rep(c(0),length(x)); + for (i in 1:n) + { + tmp <- rep(c(0),length(x)) + tmp <- a[i]*exp(-b[i]*(x - center[i]))*(b[i]/shape*(x -center[i]) + 1 )^shape +# when { b[i]/shape*(x -center[i]) + 1 } < 0, the power is meaningless. tmp[j] is "NaN" and will be replaced by zero + for (j in 1:length(x)) {if (tmp[j] == "NaN") {tmp[j] = 0} } + expr <- expr + tmp + } + eval(expr) +} + +# linear square fit, minmize funtion 0.5*|y - y_hat|^2. +# Use as an input function for constraint estimation +fcn_L2 <- function(par,n,x,y_and_noise,fcall) { + a <- par[1:n] + b <- par[(n+1):(2*n)] + center <- par[(2*n+1):(3*n)] + new_par <- c(list(a = a),list(b = b),list(center = center)) + res <- (y_and_noise - do.call("fcall",c( as.list(new_par), list(n = n), list(x = x) ) )); + 0.5*t(res) %*% res + } + + +# define the function (a kind of simplified version from above) to be used to fit the data +# +# y = a*exp(-b*(x - center))*(b/shape*(x - center) + 1)^(shape) +# +# a = amplitude +# b = decay parameter (correspond to 1/a2) +# center = a1; +# shape = a3 - 1; later shape will be fixed as 8.6, which is a common number used in most of fMRI study +# +# df_a = exp(-b*(x - center))*(b/shape*(x - center) + 1)^(shape) +# df_b = a*(-(x-center))*exp(-b*(x - center))*(b/shape*(x - center) + 1)^(shape) + a*exp(-b*(x - center))*shape*(b/shape*(x - center) + 1)^(shape-1)*(x - center)/shape +# df_center = a*b*exp(-b*(x - center))*(b/shape*(x - center) + 1)^(shape) + a*exp(-b*(x - center))*shape*(b/shape*(x - center) + 1)^(shape-1)*(-b/shape) + +D_fcn_L2 <- function(par,n,x,y_and_noise,fcall) { + df_a <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + df_b <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + df_center <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + a <- par[1:n] + b <- par[(n+1):(2*n)] + center <- par[(2*n+1):(3*n)] + new_par <- c(list(a = a),list(b = b),list(center = center)) + for (i in 1:n) { + df_a[,i] <- exp(-b[i]*(x - center[i]))*(b[i]/shape *(x -center[i]) + 1 )^shape + df_b[,i] <- a[i]*(-(x - center[i]))*exp(-b[i]*(x - center[i]))*(b[i]/shape *(x -center[i]) + 1 )^shape + a[i]*exp(-b[i]*(x - center[i]))*shape*(b[i]/shape*(x -center[i]) + 1 )^(shape - 1)*(x - center[i])/shape + df_center[,i] <- a[i]*b[i]*exp(-b[i]*(x - center[i]))*(b[i]/shape *(x -center[i]) + 1 )^shape + a[i]*exp(-b[i]*(x - center[i]))*shape*(b[i]/shape*(x -center[i]) + 1 )^(shape -1 )*(-b[i]/shape) + } + tmp <- cbind(df_a,df_b,df_center) + for (i in 1:(dim(tmp)[1]) ) { + for (j in 1:(dim(tmp)[2]) ) { + if (tmp[i,j] == "NaN") {tmp[i,j] = 0} + } + } + j_f <- tmp + y_eval <- do.call("fcall",c(list(a = a), list(b = b), list(center = center), list(n = n), list(x = x))) + t(j_f) %*% (y_eval - y_and_noise) + } + +#---- deriv 3------------------------------------------------------------- +D_individual_f <- deriv3( + ~a*exp(-b*(x - center))*(b/shape*(x -center) + 1 )^shape, + c("a","b","center"), + function(a,b,center,x,shape) {}) + +test_D_fcn_L2 <- function(par,n,x,y_and_noise,fcall) { + shape = 8.6 + df_a <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + df_b <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + df_center <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + a <- par[1:n] + b <- par[(n+1):(2*n)] + center <- par[(2*n+1):(3*n)] + new_par <- c(list(a = a),list(b = b),list(center = center)) + for (i in 1:n) { + tmp <- D_individual_f(a[i],b[i],center[i],x,shape) + g <- attr(tmp,"gradient") + df_a[,i] <- g[,1]; + df_b[,i] <- g[,2]; + df_center[,i] <- g[,3]; + } + tmp <- cbind(df_a,df_b,df_center) + for (i in 1:(dim(tmp)[1]) ) { + for (j in 1:(dim(tmp)[2]) ) { + if (tmp[i,j] == "NaN") {tmp[i,j] = 0} + } + } + j_f <- tmp + y_eval <- do.call("fcall",c(list(a = a), list(b = b), list(center = center), list(n = n), list(x = x))) + t(j_f) %*% (y_eval - y_and_noise) + } +#---------------------------------------------------------------------- + +# 2nd derivative +test_D2_fcn_L2 <- function(par,n,x,y_and_noise,fcall) { + shape = 8.6 + df_a <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + df_b <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + df_center <- matrix(rep(c(0),n*length(x)),ncol = n, nrow = length(x)) + a <- par[1:n] + b <- par[(n+1):(2*n)] + center <- par[(2*n+1):(3*n)] + new_par <- c(list(a = a),list(b = b),list(center = center)) + f <- do.call("fcall",c(list(a = a), list(b = b), list(center = center), list(n = n), list(x = x))) - y_and_noise + + for (i in 1:n) { + tmp <- D_individual_f(a[i],b[i],center[i],x,shape) + g <- attr(tmp,"gradient") + df_a[,i] <- g[,1]; + df_b[,i] <- g[,2]; + df_center[,i] <- g[,3]; + } + tmp <- cbind(df_a,df_b,df_center) + for (i in 1:(dim(tmp)[1]) ) { + for (j in 1:(dim(tmp)[2]) ) { + if (tmp[i,j] == "NaN") {tmp[i,j] = 0} + } + } + j_f <- tmp + term1 <- t(j_f) %*% j_f +# 1st term in the equation calculated, shall be a 3n x 3n matrix + + f_D2f = matrix(rep(c(0),9*n^2),ncol = 3*n,nrow = 3*n); + for (i in 1:n) { + tmp <- D_individual_f(a[i],b[i],center[i],x,shape) + h <- colSums(attr(tmp,"hessian")*f,1) + f_D2f[i,i] = h[1,1]; + f_D2f[n+i,i] = f_D2f[i,n+i] = h[1,2]; + f_D2f[2*n+i,i] = f_D2f[i,2*n+i] = h[1,3]; + f_D2f[n+i,n+i] = h[2,2]; + f_D2f[2*n+i,n+i] = f_D2f[n+i,2*n+i] = h[2,3]; + f_D2f[2*n+i,2*n+i] = h[3,3]; + } + for (i in 1:(dim(f_D2f)[1]) ) { + for (j in 1:(dim(f_D2f)[2]) ) { + if (f_D2f[i,j] == "NaN") {f_D2f[i,j] = 0} + } + } + + term1 + f_D2f + } + +#---------------------------------------------------------------------------------------- +# plot fitted curve with observed one +plot.est <- function(par,fcall,n,x,col) { + a <- par[1:n] + b <- par[(n+1):(2*n)] + center <- par[(2*n+1):(3*n)] + new_par <- c(list(a = a),list(b = b),list(center = center)) + y_est <- do.call("fcall",c( as.list(new_par), list(n = n), list(x = x) )) + lines(x,y_est,col = col,lwd = 2) + } + +# plot individual fitted curve +plot.est.individual <- function(par,fcall,n,x,col) { + a <- par[1:n] + b <- par[(n+1):(2*n)] + center <- par[(2*n+1):(3*n)] + for (i in 1:n) + { + new_par <- c(list(a = a[i]),list(b = b[i]),list(center = center[i])) + y_est <- do.call("fcall",c( as.list(new_par), list(n = 1), list(x = x) )) + lines(x,y_est,col = col,lwd = 1) + } +} + + +# fit summary + +# fit statistical summary for constraint estimation +est.stats.summary <- function(par,sse,x,y) { + n_x = length(x); # the number of observations + m = length(par); # the number of parameters + sse # sum of squares due to error + ave_y = mean(y); # the mean of observation + ssm = 0.5* t(y - ave_y) %*% (y - ave_y); # the sum of square about mean + r2 = 1 - sse/ssm; # R square, coefficient of determination + dof = n_x - m; # degree of freedom + r2_adjusted = 1 - sse*(n_x-1)/(ssm*(dof-1)); # adjusted R square + mse = sse/dof; # the mean square error + se = sqrt(mse); # the standard error of fit, the root mse + msr = (ssm - sse)/(m - 1); # mean square regression + f = msr/mse; # F-statistics + c(r2,dof,r2_adjusted,f,se) +# c("r2" = r2,"DF" = dof,"Adj r2" = r2_adjusted,"F" = f,"standard error of fit" = se) + } + +# fit summary includes two fitting parameters (amplitude and centerfwhm and analytical area under the curve +# -----fwhm------------ +# fwhm ~ 2.35*b^(1/2)*c +# b & c are from gamma variate function x^b*exp(-t/c) +# this approximation is found in ref "Event-related fMRI Contast When using Constant Interstimulus Interval: Theory and Experiment", pubished in Magnetic Resonance in Medicine 43:540-548 (2000) +# this approximation result is a little bit different from that from peakfit +# in our code,shape parameter is fixed at 8.6 & width related parameter is b +# fwhm ~ 2.35*sqrt(8.6)/b +# -----area under the curve------- +# after some derivations, area under the curve is a/b*exp(8.6)*8.6^(-8.6)*gamma(8.6+1); +# the derivation was confirmed with peakfit result +est.summary_1 <- function(fcall,par,sse,n,x,y) { + a <- par[1:n] # estimated a + b <- par[(n+1):(2*n)] # estimated b + center <- par[(2*n+1):(3*n)] # estimated center +# --calculate standard error of fit + n_x = length(x); # the number of observations + m = length(par); # the number of parameters + dof = n_x - m; # degree of freedom + sse # sum of squares due to error + mse = sse/dof; # the mean square error + se = sqrt(mse); # the standard error of fit, the root mse +#---standard error of parameters, se * 1/sqrt(diag(hessian matrix of 0.5*|y-y_hat|^2)) + std_par <- se * 1/sqrt(diag(test_D2_fcn_L2(par,n,x,y,f))) # std error of estimated parameters + std_a <- std_par[1:n]; + std_b <- std_par[(n+1):(2*n)]; + std_center <- std_par[(2*n+1):(3*n)]; +#---calculate fwhm, analytical area and percent area------ + shape <- 8.6 + fwhm <- matrix(c(0),ncol = 1,nrow = n); + area <- matrix(c(0),ncol = 1,nrow = n); + percent_area <- matrix(c(0),ncol = 1,nrow = n); for (i in 1:n) { + fwhm[i] <- 2.35*sqrt(shape)/b[i]; + area[i] <- a[i]/b[i]*exp(shape)*shape^(-shape)*gamma(shape+1); + } + percent_area <- area/sum(area)*100; +#---need to double check--------------------- + std_fwhm <- std_b * 2.35*sqrt(8.6) * 1/sqrt(b^4); +#---calculate partial F stats for each peak + y_est_col <- matrix(c(0),ncol = n, nrow = n_x); + for (i in 1:n) + { + new_par <- c(list(a = a[i]),list(b = b[i]),list(center = center[i])); + y_est_col[,i] <- do.call("fcall",c( as.list(new_par), list(n = 1), list(x = x) )); # each col corresponds to each peak fitted ts + } + + p_partial <- matrix(c(0),ncol = 1, nrow = n); + for (i in 1:n) + { + res_wo_i <- rowSums(y_est_col[,-i]) - y; # residual without ith peak + ssr <- 0.5*res_wo_i %*% res_wo_i; # sum square of error of residual + dof_diff <- 3; # degree of freedom difference between full model and reduced model (i.e. without ith peak) + F_partial <- ((ssr - sse)/dof_diff)/(sse/dof); # partial F stats for ith peak + p_partial[i] <- 1 - pf(F_partial,dof_diff,dof); + } + +#---summary of individual results + summary <- data.frame(a,std_a,center,std_center,fwhm,std_fwhm,area,percent_area,p_partial); + name_of_row <- paste("Peak",1:n,sep=""); + name_of_col <- c("Amplitude","e_Amplitude","Center","e_Center","FWHM","e_FWHM","Area","% Area","p_partial"); + dimnames(summary) <- list(name_of_row,name_of_col); + summary + } + +est.summary <- function(par,sse,n,x,y) { + a <- par[1:n] # estimated a + b <- par[(n+1):(2*n)] # estimated b + center <- par[(2*n+1):(3*n)] # estimated center +# --calculate standard error of fit + n_x = length(x); # the number of observations + m = length(par); # the number of parameters + dof = n_x - m; # degree of freedom + sse # sum of squares due to error + mse = sse/dof; # the mean square error + se = sqrt(mse); # the standard error of fit, the root mse +#---standard error of parameters, se * 1/sqrt(diag(hessian matrix of 0.5*|y-y_hat|^2)) + std_par <- se * 1/sqrt(diag(test_D2_fcn_L2(par,n,x,y,f))) # std error of estimated parameters + std_a <- std_par[1:n]; + std_b <- std_par[(n+1):(2*n)]; + std_center <- std_par[(2*n+1):(3*n)]; +#---calculate fwhm, analytical area and percent area------ + shape <- 8.6 + fwhm <- matrix(c(0),ncol = 1,nrow = n); + area <- matrix(c(0),ncol = 1,nrow = n); + percent_area <- matrix(c(0),ncol = 1,nrow = n); for (i in 1:n) { + fwhm[i] <- 2.35*sqrt(shape)/b[i]; + area[i] <- a[i]/b[i]*exp(shape)*shape^(-shape)*gamma(shape+1); + } + percent_area <- area/sum(area, na.rm=T)*100; +#---need to double check--------------------- + std_fwhm <- std_b * 2.35*sqrt(8.6) * 1/sqrt(b^4); +#---onset --------------- + onset <- center - shape/b; +#---summary of individual results + summary <- data.frame(a,std_a,center,std_center,fwhm,std_fwhm,area,percent_area,onset); + name_of_row <- paste("Peak",1:n,sep=""); + name_of_col <- c("Amplitude","e_Amplitude","Center","e_Center","FWHM","e_FWHM","Area","% Area","onset"); + dimnames(summary) <- list(name_of_row,name_of_col); + summary + } + + + + + + Property changes on: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/peakfit_2.1.R ___________________________________________________________________ Name: svn:executable + * Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/preprocess.Enewsmooth.R =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/preprocess.Enewsmooth.R (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/preprocess.Enewsmooth.R 2009-04-09 21:37:19 UTC (rev 2853) @@ -0,0 +1,76 @@ +preprocess_1 <- function(input) { + + # read file (read.table) and save as vector (as.matrix) + original_y <- input*1000; + + # original # of points before interpolation + original_x <- c(1:length(original_y)); + + #---------calculate 2nd derivative to find peaks------------------------------------ + # + # windowlength argument n + # p is order of polynimal + # D is the order of derivative + + windowlength <- 3 # 3 data points covered + # tested 4 data points, detail info might be lost + order_polynomial <- 2; # includes constant, linear and quadratic terms + order_derivative <- 2; # need the information of second derivatives + + derivative_original_y <- sav.gol(original_y, fl = windowlength, forder = order_polynomial, dorder = order_derivative); + + + # guess # of possible peaks by counting negative y derivatives and corresponding amplitude of y > noise level + noise_level <- 0; # so far noise level is set to 0 + + # guess peak location: use all negative second derivatives or use local min of second derivative + # peak_location = original_x[derivative_original_y < 0 & original_y > noise_level]; + + tmp <- rep(c(0),length(original_x)); + for ( i in 2:(length(original_x)-1) ) { + if ( (derivative_original_y[i] < derivative_original_y[i-1]) & (derivative_original_y[i] < derivative_original_y[i+1]) ) + { tmp[i] = i;} + } + peak_location = original_x[tmp > 0]; + + possible_peaks = length(peak_location) + + #------------------------------------------------------------------------------------- + + + #-----------------interpolation data--------------------------------------------------- + # + # always needs # of parameters < length of observation + # in the case that 2nd derivative detect many peaks, interpolation shall be used + stim_res = 2 # sympliy double the time points + + interpolation <- ksmooth(original_x,original_y,"normal",bandwidth = 1,x.points = seq(1,length(original_x),by = 1/stim_res)); + + #derivative_interpolation <- sav.gol(interpolation$y, fl = windowlength, forder = order_polynomial, dorder = order_derivative); + # calculatae new peak location + new_peak_location = stim_res*(peak_location - 1) + 1; + + # calculatae new peak amplitude + + peak_amplitude = rep(c(0),possible_peaks); + for (i in 1:possible_peaks ) { peak_amplitude[i] = interpolation$y[new_peak_location[i]] } + + #--------------------initializaation---------------------------------------- + # + # parameter a corresponds to peak amplitude + a_guess <- peak_amplitude; + + # parameter b corresponds to 1/peak width + b_guess <- rep(c(length(original_y)/possible_peaks), possible_peaks); + + #real_center_guess <- interpolation$x[new_peak_location]; + #center_guess <- b_guess*real_center_guess; + center_guess <- interpolation$x[new_peak_location]; + + guess <- c(a_guess,b_guess,center_guess); + n <- possible_peaks; + + out <- list(guess = guess,n = n,interpolated_x = interpolation$x,interpolated_y = interpolation$y); + out +} + \ No newline at end of file Property changes on: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/peak_scripts/preprocess.Enewsmooth.R ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Thu Apr 9 18:51:16 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 9 Apr 2009 18:51:16 -0500 (CDT) Subject: [Swift-commit] r2854 - usertools/cio/libexec Message-ID: <20090409235116.29D969D7AA@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-09 18:51:15 -0500 (Thu, 09 Apr 2009) New Revision: 2854 Modified: usertools/cio/libexec/hashserver.rb Log: updated ruby paths for bgp specificity Modified: usertools/cio/libexec/hashserver.rb =================================================================== --- usertools/cio/libexec/hashserver.rb 2009-04-09 21:37:19 UTC (rev 2853) +++ usertools/cio/libexec/hashserver.rb 2009-04-09 23:51:15 UTC (rev 2854) @@ -1,4 +1,4 @@ -#!/usr/bin/env ruby +#!/home/espinosa/local/bin/ruby require 'socket' require 'scanf' From noreply at svn.ci.uchicago.edu Sun Apr 12 10:59:28 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 12 Apr 2009 10:59:28 -0500 (CDT) Subject: [Swift-commit] r2855 - SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2 Message-ID: <20090412155928.866589D7A1@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-04-12 10:59:27 -0500 (Sun, 12 Apr 2009) New Revision: 2855 Modified: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/PK2.swift Log: created new peakfit types Modified: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/PK2.swift =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/PK2.swift 2009-04-09 23:51:15 UTC (rev 2854) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/PK2.swift 2009-04-12 15:59:27 UTC (rev 2855) @@ -1,35 +1,38 @@ -#---- doing peakfit on PK2 pilot subject +#---- doing peakfit on PK2 pilot subject and experimenting with code type file{} - +type Rscript; +type PeakfitR{ + Rscript ShellpeakPK2runbyrun; + Rscript preprocessEnewsmooth; + Rscript peakfitv2v1; +} type PeakResult{ file pdf; file a; file b; } -app (PeakResult outObj) peakScript (file scriptFile, file tsfile, file preprocR, file peakfitR){ - RInvoke @filename(scriptFile) @filename(tsfile) @filename(outObj); + +app (PeakResult outObj) peakScript (PeakfitR code, file tsfile){ + RInvoke @code.ShellpeakPK2runbyrun @filename(tsfile) @filename(outObj); } +PeakfitR code; -file script<"peak_scripts/Shellpeak_PK2runbyrun.R">; -file preprocR<"peak_scripts/preprocess.Enewsmooth.R">; -file peakfitR<"peak_scripts/peakfit_2.1.R">; +string declarelist[] = ["PK2"]; +int runs[] = [1:8]; +string hemilist[] = ["lh"]; +int inputRegionNo[] = [54]; -string declarelist[] = ["PK2"]; foreach subject in declarelist{ - int runs[] = [1:8]; foreach rn in runs{ - string hemilist[] = ["lh","rh"]; foreach h in hemilist{ - int inputRegionNo[] = [15,33,34,60]; foreach rr in inputRegionNo{ - string project_dir=@strcat("/disks/ci-gpfs/fmri/cnari/swift/projects/andric/peakfit_pilots/",subject); ## map inputs - file inFile; + file inFile; ## map results - PeakResult outObj; - (outObj)=peakScript(script,inFile, preprocR, peakfitR); + PeakResult outObj; + outObj=peakScript(code, inFile); } } } From noreply at svn.ci.uchicago.edu Tue Apr 14 10:29:35 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Apr 2009 10:29:35 -0500 (CDT) Subject: [Swift-commit] r2856 - in trunk/tests/sites: . gwynn.bsd.uchicago.edu Message-ID: <20090414152935.532D79D7B7@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-14 10:29:35 -0500 (Tue, 14 Apr 2009) New Revision: 2856 Added: trunk/tests/sites/gwynn.bsd.uchicago.edu/ trunk/tests/sites/gwynn.bsd.uchicago.edu/condor-provider.xml Log: site test to run on gwynn that tests local condor execution Added: trunk/tests/sites/gwynn.bsd.uchicago.edu/condor-provider.xml =================================================================== --- trunk/tests/sites/gwynn.bsd.uchicago.edu/condor-provider.xml (rev 0) +++ trunk/tests/sites/gwynn.bsd.uchicago.edu/condor-provider.xml 2009-04-14 15:29:35 UTC (rev 2856) @@ -0,0 +1,9 @@ + + + + + + /disks/gpfs/swift/site-test + + + From noreply at svn.ci.uchicago.edu Tue Apr 14 13:02:23 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Apr 2009 13:02:23 -0500 (CDT) Subject: [Swift-commit] r2857 - in usertools/cio/libexec: falkon zoid Message-ID: <20090414180223.B6FBF9D793@vm-125-59.ci.uchicago.edu> Author: aespinosa Date: 2009-04-14 13:02:23 -0500 (Tue, 14 Apr 2009) New Revision: 2857 Modified: usertools/cio/libexec/falkon/runworker-bgp.sh usertools/cio/libexec/zoid/logging-script.sh Log: Fixed lurking scripting errors from old falkon builds Modified: usertools/cio/libexec/falkon/runworker-bgp.sh =================================================================== --- usertools/cio/libexec/falkon/runworker-bgp.sh 2009-04-14 15:29:35 UTC (rev 2856) +++ usertools/cio/libexec/falkon/runworker-bgp.sh 2009-04-14 18:02:23 UTC (rev 2857) @@ -25,7 +25,7 @@ FALKON_ROOT_PATH=$7 - ln -s /fuse/gpfs1 /gpfs1 + #ln -s /fuse/gpfs1 /gpfs1 #FALKON_HOME=/gpfs1/falkon/falkon #HOME=/gpfs1/falkon Modified: usertools/cio/libexec/zoid/logging-script.sh =================================================================== --- usertools/cio/libexec/zoid/logging-script.sh 2009-04-14 15:29:35 UTC (rev 2856) +++ usertools/cio/libexec/zoid/logging-script.sh 2009-04-14 18:02:23 UTC (rev 2857) @@ -14,7 +14,7 @@ # end addition #startup - if [ "${1}" -eq "1" ]; then + if [ $1 -eq 1 ]; then #ERROR_LOG=/home/falkon/users/error.log ERROR_LOG=/dev/null @@ -91,7 +91,7 @@ #cleanup - if [ "${1}" -eq "0" ]; then + if [ $1 -eq 0 ]; then killall -9 java exit 0 From noreply at svn.ci.uchicago.edu Wed Apr 15 04:20:38 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 15 Apr 2009 04:20:38 -0500 (CDT) Subject: [Swift-commit] r2858 - trunk/libexec/log-processing Message-ID: <20090415092038.C081F9D7B7@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-15 04:20:38 -0500 (Wed, 15 Apr 2009) New Revision: 2858 Modified: trunk/libexec/log-processing/execute2.html.template trunk/libexec/log-processing/makefile trunk/libexec/log-processing/makefile.webpage Log: rename jobs-sites.table to .html so that it can be more easily generated and viewed independently Modified: trunk/libexec/log-processing/execute2.html.template =================================================================== --- trunk/libexec/log-processing/execute2.html.template 2009-04-14 18:02:23 UTC (rev 2857) +++ trunk/libexec/log-processing/execute2.html.template 2009-04-15 09:20:38 UTC (rev 2858) @@ -10,7 +10,7 @@

Breakdown of execute2 final statuses by site: (More information about execute2 events)

-include(`jobs-sites.table') +include(`jobs-sites.html')

Number of execute2 jobs at once - this is basically the number of execution attempts that have been submitted to some batch system such Modified: trunk/libexec/log-processing/makefile =================================================================== --- trunk/libexec/log-processing/makefile 2009-04-14 18:02:23 UTC (rev 2857) +++ trunk/libexec/log-processing/makefile 2009-04-15 09:20:38 UTC (rev 2858) @@ -17,7 +17,7 @@ cp *.event *.html *.png $(SDL) clean: - rm -f start-times.data kickstart-times.data start-time.tmp end-time.tmp threads.list tasks.list log *.data *.shifted *.png *.event *.coloured-event *.total *.tmp *.transitions *.last karatasks-type-counts.txt index.html *.lastsummary execstages.plot total.plot colour.plot jobs-sites.table jobs.retrycount.summary kickstart.stats execution-counts.txt site-duration.txt jobs.retrycount sp.plot karatasks.coloured-sorted-event *.cedps *.stats t.inf *.seenstates tmp-* clusterstats trname-summary sites-list.data.nm info-md5sums pse2d-tmp.eip karajan.html falkon.html execute2.html info.html execute.html kickstart.html scheduler.html assorted.html + rm -f start-times.data kickstart-times.data start-time.tmp end-time.tmp threads.list tasks.list log *.data *.shifted *.png *.event *.coloured-event *.total *.tmp *.transitions *.last karatasks-type-counts.txt index.html *.lastsummary execstages.plot total.plot colour.plot jobs-sites.html jobs.retrycount.summary kickstart.stats execution-counts.txt site-duration.txt jobs.retrycount sp.plot karatasks.coloured-sorted-event *.cedps *.stats t.inf *.seenstates tmp-* clusterstats trname-summary sites-list.data.nm info-md5sums pse2d-tmp.eip karajan.html falkon.html execute2.html info.html execute.html kickstart.html scheduler.html assorted.html t.inf: $(LOG) compute-t-inf > t.inf < $(LOG) @@ -67,8 +67,8 @@ sites-list.data.nm: sites-list.data number-sites-list < $< > $@ -jobs-sites.table: sites-list.data execute2.event - table-jobs-sites > jobs-sites.table +jobs-sites.html: sites-list.data execute2.event + table-jobs-sites > jobs-sites.html site-duration.txt: execute2.event sites-list.data per-site-execute2-durations > site-duration.txt Modified: trunk/libexec/log-processing/makefile.webpage =================================================================== --- trunk/libexec/log-processing/makefile.webpage 2009-04-14 18:02:23 UTC (rev 2857) +++ trunk/libexec/log-processing/makefile.webpage 2009-04-15 09:20:38 UTC (rev 2858) @@ -44,7 +44,7 @@ dostageout-duration-histogram.png execute-trails.png execute2-trails.png \ info-and-karajan-actives.png info-and-karajan-actives.2.png -htmldeps: $(SWIFT_PLOT_HOME)/index.html.template execution-counts.txt jobs-sites.table site-duration.txt execute2.lastsummary execute.lastsummary jobs.retrycount.summary trname-summary +htmldeps: $(SWIFT_PLOT_HOME)/index.html.template execution-counts.txt jobs-sites.html site-duration.txt execute2.lastsummary execute.lastsummary jobs.retrycount.summary trname-summary %.html: htmldeps m4 -I$(SWIFT_PLOT_HOME) $(SWIFT_PLOT_HOME)/$@.template > $@ From noreply at svn.ci.uchicago.edu Wed Apr 15 04:55:29 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 15 Apr 2009 04:55:29 -0500 (CDT) Subject: [Swift-commit] r2859 - provenancedb Message-ID: <20090415095529.400869D7B8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-15 04:55:28 -0500 (Wed, 15 Apr 2009) New Revision: 2859 Modified: provenancedb/provenance.xml Log: ongoing provenance notes Modified: provenancedb/provenance.xml =================================================================== --- provenancedb/provenance.xml 2009-04-15 09:20:38 UTC (rev 2858) +++ provenancedb/provenance.xml 2009-04-15 09:55:28 UTC (rev 2859) @@ -1943,6 +1943,30 @@ like a plain binary operator inside the Swift compiler and runtime) + +provenance of references vs provenance of the data in them: the array and +structure access operators can be used to acquire DSHandles +which have no value yet, and which are then subsequently assigned. In this +usage, the provenance of the containing structure should perhaps be that it +is constructed from the assignments made to its members, rather than the +other way round. There is some subtlety here that I have not fully figured +out. + + + +Piecewise construction of collections: arrays and structs can be +constructed piecewise using . = and [] =. +how is this to be represented in OPM? perhaps the closing operation maps +to the OPM process that creates the array, so that it ends up looking +like an explicit array construction, happening at the time of the close? + + + +Provenance of mapper parameters: mapper parameters are artifacts. We can +represent references to those in a Swift-specific part of an artifacts +value, perhaps. Probably not something OPM-generalisable. + +

From noreply at svn.ci.uchicago.edu Wed Apr 15 07:23:52 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 15 Apr 2009 07:23:52 -0500 (CDT) Subject: [Swift-commit] r2860 - SwiftApps/pc3 Message-ID: <20090415122352.4EAFE9D7B8@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-15 07:23:52 -0500 (Wed, 15 Apr 2009) New Revision: 2860 Modified: SwiftApps/pc3/CompactDatabase.sh SwiftApps/pc3/PSLoadExecutable.sh SwiftApps/pc3/PSLoadExecutableDB.sh SwiftApps/pc3/README Log: misc changes to simply execution of pc3 app Modified: SwiftApps/pc3/CompactDatabase.sh =================================================================== --- SwiftApps/pc3/CompactDatabase.sh 2009-04-15 09:55:28 UTC (rev 2859) +++ SwiftApps/pc3/CompactDatabase.sh 2009-04-15 12:23:52 UTC (rev 2860) @@ -1,7 +1,7 @@ #!/bin/bash INPUT=$1 -$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute CompactDatabase -f $INPUT +java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute CompactDatabase -f $INPUT - \ No newline at end of file + Modified: SwiftApps/pc3/PSLoadExecutable.sh =================================================================== --- SwiftApps/pc3/PSLoadExecutable.sh 2009-04-15 09:55:28 UTC (rev 2859) +++ SwiftApps/pc3/PSLoadExecutable.sh 2009-04-15 12:23:52 UTC (rev 2860) @@ -3,5 +3,5 @@ INPUT2=$2 OUTPUT=$3 -$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $INPUT2 -o $OUTPUT -f $INPUT1 +java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $INPUT2 -o $OUTPUT -f $INPUT1 Modified: SwiftApps/pc3/PSLoadExecutableDB.sh =================================================================== --- SwiftApps/pc3/PSLoadExecutableDB.sh 2009-04-15 09:55:28 UTC (rev 2859) +++ SwiftApps/pc3/PSLoadExecutableDB.sh 2009-04-15 12:23:52 UTC (rev 2860) @@ -4,4 +4,4 @@ SINPUT=$3 OUTPUT=$4 -$JAVA_HOME/bin/java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $SINPUT -o $OUTPUT -f $DB -f $INPUT +java -Dderby.system.home=/var/tmp/derby-pc3/ -cp "$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadExecutable.jar:$PC3LIB/Java/bin/info.ipaw.pc3.PSLoadWorkflow.jar:$PC3LIB/Java/lib/derby.jar:$PC3LIB/Java/lib/derbyclient.jar:$PC3LIB/Java/lib/derbynet.jar:$PC3LIB/Java/lib/derbyrun.jar:$PC3LIB/Java/lib/derbytools.jar" info.ipaw.pc3.PSLoadExecutable.Execute $SINPUT -o $OUTPUT -f $DB -f $INPUT Modified: SwiftApps/pc3/README =================================================================== --- SwiftApps/pc3/README 2009-04-15 09:55:28 UTC (rev 2859) +++ SwiftApps/pc3/README 2009-04-15 12:23:52 UTC (rev 2860) @@ -1,19 +1,36 @@ -* This is a work-space for the Swift entry to the third provenance challenge +* Download and extract PC3.tar.gz from the PC3 home page at + http://twiki.ipaw.info/bin/view/Challenge/ThirdProvenanceChallenge -* Download and extract PC3.tar.gz from the PC3 home page. - * Append tc.data.append to your tc.data. -* Requires xpath program (in Debian: $ apt-get install libxml-xpath-perl). +* Requires xpath program (in Debian: $ apt-get install libxml-xpath-perl) +* Requires Java 1.6 * Example: +$ cd swift-svn/SwiftApps/pc3 +$ export PATH=$PATH:`pwd` $ tar xvfz PC3.tar.gz $ cd PC3 -$ export JAVA_PATH= $ export PC3LIB=`pwd` -$ mkdir Java/tmp -$ cd Java/tmp -$ cp /pc3/* . -$ export PATH=$PATH:`pwd` -$ swift pc3.swift -csvpath=CSVRootPathInput.xml -jobid=JobIDInput.xml +$ cd Java/ +$ ant jar +$ mkdir tmp +$ cd tmp +$ swift -tc.data ../../../tc.data.append pc3.swift -csvpath=CSVRootPathInput.xml -jobid=JobIDInput.xml + +To re-run, you will need to delete the Derby database generated by the +run, like this: + +rm -rfv /var/tmp/derby-pc3/ + + +Extra notes for running on Ben's OS X 10.5 laptop with multiple Java versions +installed, the default of which does not work: + +Set these: +$ echo $JAVACMD +/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Commands//java +$ echo $JAVA_HOME +/System/Library/Frameworks/JavaVM.framework/Versions/1.6/ + From noreply at svn.ci.uchicago.edu Sun Apr 19 23:26:35 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 19 Apr 2009 23:26:35 -0500 (CDT) Subject: [Swift-commit] r2861 - in SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2: . output Message-ID: <20090420042635.D749E9D65E@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-04-19 23:26:34 -0500 (Sun, 19 Apr 2009) New Revision: 2861 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/output/ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/output/analysis/ Log: directory for peakfit output From noreply at svn.ci.uchicago.edu Sun Apr 19 23:30:46 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 19 Apr 2009 23:30:46 -0500 (CDT) Subject: [Swift-commit] r2862 - SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/output/analysis Message-ID: <20090420043046.C066A9D65E@vm-125-59.ci.uchicago.edu> Author: andric Date: 2009-04-19 23:30:46 -0500 (Sun, 19 Apr 2009) New Revision: 2862 Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/output/analysis/parse_designfile.R Log: R script for parsing design file from all runs into run by run Added: SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/output/analysis/parse_designfile.R =================================================================== --- SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/output/analysis/parse_designfile.R (rev 0) +++ SwiftApps/SIDGrid/swift/projects/andric/peakfit_pilots/PK2/output/analysis/parse_designfile.R 2009-04-20 04:30:46 UTC (rev 2862) @@ -0,0 +1,15 @@ +dd = read.table("for3dDecon.PK2_4condSHORTcat_all.1D") +tr = c(1:226) +runs = 8 +start = 1 +end = length(tr) + +for (i in 1:runs){ + design = dd[start:end,] + start = start + length(tr) + end = end + length(tr) + out = paste("run",i,"Design.txt",sep="") + write.table(design,file=out,col.names=F,quote=F) +} +print(start) +print(end) From noreply at svn.ci.uchicago.edu Tue Apr 21 02:53:15 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Apr 2009 02:53:15 -0500 (CDT) Subject: [Swift-commit] r2863 - SwiftApps/pc3 Message-ID: <20090421075315.C0D3F9CC7D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-21 02:53:15 -0500 (Tue, 21 Apr 2009) New Revision: 2863 Modified: SwiftApps/pc3/README Log: more notes for pc3 Modified: SwiftApps/pc3/README =================================================================== --- SwiftApps/pc3/README 2009-04-20 04:30:46 UTC (rev 2862) +++ SwiftApps/pc3/README 2009-04-21 07:53:15 UTC (rev 2863) @@ -17,7 +17,7 @@ $ ant jar $ mkdir tmp $ cd tmp -$ swift -tc.data ../../../tc.data.append pc3.swift -csvpath=CSVRootPathInput.xml -jobid=JobIDInput.xml +$ swift -tc.file ../../../tc.data.append pc3.swift -csvpath=CSVRootPathInput.xml -jobid=JobIDInput.xml To re-run, you will need to delete the Derby database generated by the run, like this: @@ -28,9 +28,6 @@ Extra notes for running on Ben's OS X 10.5 laptop with multiple Java versions installed, the default of which does not work: -Set these: -$ echo $JAVACMD -/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Commands//java -$ echo $JAVA_HOME -/System/Library/Frameworks/JavaVM.framework/Versions/1.6/ +Set this: +export PATH=/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Commands/:$PATH From noreply at svn.ci.uchicago.edu Tue Apr 21 02:54:04 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Apr 2009 02:54:04 -0500 (CDT) Subject: [Swift-commit] r2864 - nmi-build-test/submit-machine Message-ID: <20090421075404.6CF219CC7D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-21 02:54:04 -0500 (Tue, 21 Apr 2009) New Revision: 2864 Modified: nmi-build-test/submit-machine/kickstart.svn Log: r2847 did not correctly update kickstart URL Modified: nmi-build-test/submit-machine/kickstart.svn =================================================================== --- nmi-build-test/submit-machine/kickstart.svn 2009-04-21 07:53:15 UTC (rev 2863) +++ nmi-build-test/submit-machine/kickstart.svn 2009-04-21 07:54:04 UTC (rev 2864) @@ -1,2 +1,2 @@ method = svn -url = http://pegasus.isi.edu/svn/pegasus/trunk/src/tools/kickstart +url = https://pegasus.isi.edu/svn/pegasus/trunk/src/tools/kickstart From noreply at svn.ci.uchicago.edu Tue Apr 21 09:06:00 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Apr 2009 09:06:00 -0500 (CDT) Subject: [Swift-commit] r2865 - SwiftApps/pc3 Message-ID: <20090421140600.9AC5C9CC7F@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-21 09:06:00 -0500 (Tue, 21 Apr 2009) New Revision: 2865 Modified: SwiftApps/pc3/pc3.swift Log: make pc3.swift more swiftier Modified: SwiftApps/pc3/pc3.swift =================================================================== --- SwiftApps/pc3/pc3.swift 2009-04-21 07:54:04 UTC (rev 2864) +++ SwiftApps/pc3/pc3.swift 2009-04-21 14:06:00 UTC (rev 2865) @@ -1,6 +1,9 @@ type xmlfile; type textfile; +(external out) checkvalid(boolean b) { + if(!b) { stop(); } +} (xmlfile output) ps_load_executable(xmlfile input, string s) { app { @@ -8,6 +11,13 @@ } } +(xmlfile output) ps_load_executable_threaded(xmlfile input, string s, external thread) { + app { + ps_load_executable_app @input s @output; + } +} + + (textfile output) parse_xml_boolean_value(xmlfile input) { app { parse_xml_boolean_value_app @input @output; @@ -36,7 +46,8 @@ output = ps_load_executable_boolean(input, "IsMatchCSVFileTables"); } -(xmlfile output) create_empty_load_db(xmlfile input) { +// TODO I think this output never gets used? +(xmlfile output, external outthread) create_empty_load_db(xmlfile input) { output = ps_load_executable(input, "CreateEmptyLoadDB"); } @@ -56,42 +67,43 @@ output = ps_load_executable_boolean(input, "IsExistsCSVFile"); } -(xmlfile output) read_csv_file_column_names(xmlfile input) { - output = ps_load_executable(input, "ReadCSVFileColumnNames"); +(xmlfile output) read_csv_file_column_names(xmlfile input, external thread) { + output = ps_load_executable_threaded(input, "ReadCSVFileColumnNames", thread); } (boolean output) is_match_csv_file_column_names(xmlfile input) { output = ps_load_executable_boolean(input, "IsMatchCSVFileColumnNames"); } -(xmlfile output) ps_load_executable_db (xmlfile db, xmlfile input, string s) { +(xmlfile output) ps_load_executable_db (xmlfile db, xmlfile input, string s, external thread) { app { ps_load_executable_db_app @db @input s @output; } } -(boolean output) ps_load_executable_db_boolean(xmlfile db, xmlfile input, string s) { - xmlfile xml_out = ps_load_executable_db(db, input, s); +(boolean output) ps_load_executable_db_boolean(xmlfile db, xmlfile input, string s, external thread) { + xmlfile xml_out = ps_load_executable_db(db, input, s, thread); output = extract_boolean(xml_out); } -(boolean output) load_csv_file_into_table(xmlfile db, xmlfile input) { - output = ps_load_executable_db_boolean(db, input, "LoadCSVFileIntoTable"); +(boolean output) load_csv_file_into_table(xmlfile db, xmlfile input, external thread) { + output = ps_load_executable_db_boolean(db, input, "LoadCSVFileIntoTable", thread); } -(boolean output) update_computed_columns(xmlfile db, xmlfile input) { - output = ps_load_executable_db_boolean(db, input, "UpdateComputedColumns"); +(boolean output) update_computed_columns(xmlfile db, xmlfile input, external thread) { + output = ps_load_executable_db_boolean(db, input, "UpdateComputedColumns", thread); } -(boolean output) is_match_table_row_count(xmlfile db, xmlfile input) { - output = ps_load_executable_db_boolean(db, input, "IsMatchTableRowCount"); +(boolean output) is_match_table_row_count(xmlfile db, xmlfile input, external thread) { + output = ps_load_executable_db_boolean(db, input, "IsMatchTableRowCount", thread); } -(boolean output) is_match_table_column_ranges(xmlfile db, xmlfile input) { - output = ps_load_executable_db_boolean(db, input, "IsMatchTableColumnRanges"); +(boolean output) is_match_table_column_ranges(xmlfile db, xmlfile input, external thread) { + output = ps_load_executable_db_boolean(db, input, "IsMatchTableColumnRanges", thread); } -compact_database (xmlfile input) { +// TODO what is the input used for here? +compact_database (xmlfile input, external thread) { app { compact_database_app @input; } @@ -116,11 +128,12 @@ is_csv_ready_file_exists_output = is_csv_ready_file_exists(csv_root_path_input); -if(is_csv_ready_file_exists_output) { read_csv_ready_file_output = read_csv_ready_file(csv_root_path_input); is_match_csv_file_tables_output = is_match_csv_file_tables(read_csv_ready_file_output); if(is_match_csv_file_tables_output) { - create_empty_load_db_output = create_empty_load_db(job_id); + external db_over_time[]; + external dbinit; // some bug in analysis means can't use db_over_time for initial one + (create_empty_load_db_output, dbinit) = create_empty_load_db(job_id); count_entries_output = count_entries(read_csv_ready_file_output); entries = readData(count_entries_output); int entries_seq[] = [1:entries]; @@ -128,9 +141,11 @@ foreach i in entries_seq { split_list_output[i] = extract_entry(read_csv_ready_file_output, i); } - - iterate i { - + + +// TODO this can merge with merge with above foreach + + foreach i in entries_seq { boolean is_exists_csv_file_output; xmlfile read_csv_file_column_names_output; boolean is_match_csv_file_column_names_output; @@ -139,48 +154,27 @@ boolean is_match_table_row_count_output; boolean is_match_table_column_ranges_output; - is_exists_csv_file_output = is_exists_csv_file(split_list_output[i+1]); - if(is_exists_csv_file_output) { - read_csv_file_column_names_output = read_csv_file_column_names(split_list_output[i+1]); - is_match_csv_file_column_names_output = is_match_csv_file_column_names(read_csv_file_column_names_output); - if(is_match_csv_file_column_names_output) { - load_csv_file_into_table_output = load_csv_file_into_table(create_empty_load_db_output, read_csv_file_column_names_output); - if(load_csv_file_into_table_output) { - update_computed_columns_output = update_computed_columns(create_empty_load_db_output, read_csv_file_column_names_output); - if(update_computed_columns_output) { - is_match_table_row_count_output = is_match_table_row_count(create_empty_load_db_output, read_csv_file_column_names_output); - if(is_match_table_row_count_output) { - is_match_table_column_ranges_output = is_match_table_column_ranges(create_empty_load_db_output, read_csv_file_column_names_output); - if(!is_match_table_column_ranges_output) { - stop(); - } - } - else { - stop(); - } - } - else { - stop(); - } - } - else { - stop(); - } - } - else { - stop(); - } + is_exists_csv_file_output = is_exists_csv_file(split_list_output[i]); + external thread6 = checkvalid(is_exists_csv_file_output); +read_csv_file_column_names_output = read_csv_file_column_names(split_list_output[i], thread6); + is_match_csv_file_column_names_output = is_match_csv_file_column_names(read_csv_file_column_names_output); + external thread2 = checkvalid(is_match_csv_file_column_names_output); + + if(i==1) { // first element... + load_csv_file_into_table_output = load_csv_file_into_table(create_empty_load_db_output, read_csv_file_column_names_output, dbinit); + } else { + load_csv_file_into_table_output = load_csv_file_into_table(create_empty_load_db_output, read_csv_file_column_names_output, db_over_time[i]); } - else { - stop(); - } - } until ((i + 2) > entries); - compact_database(create_empty_load_db_output); + external thread3=checkvalid(load_csv_file_into_table_output); + update_computed_columns_output = update_computed_columns(create_empty_load_db_output, read_csv_file_column_names_output, thread3); + external thread4 = checkvalid(update_computed_columns_output); + is_match_table_row_count_output = is_match_table_row_count(create_empty_load_db_output, read_csv_file_column_names_output, thread4); + external thread1 = checkvalid(is_match_table_row_count_output); + is_match_table_column_ranges_output = is_match_table_column_ranges(create_empty_load_db_output, read_csv_file_column_names_output, thread1); + db_over_time[i+1] = checkvalid(is_match_table_column_ranges_output); + } + compact_database(create_empty_load_db_output, db_over_time[entries+1]); } else { stop(); } -} -else { - stop(); -} From noreply at svn.ci.uchicago.edu Tue Apr 21 14:14:11 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Apr 2009 14:14:11 -0500 (CDT) Subject: [Swift-commit] r2866 - trunk/tests/sites Message-ID: <20090421191411.653209CC7D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-21 14:14:11 -0500 (Tue, 21 Apr 2009) New Revision: 2866 Added: trunk/tests/sites/uj-fork-gram2.xml trunk/tests/sites/uj-pbs-gram2.xml Log: site tests for university of johannesburg cluster. these work for Swift 0.9rc2 and should work for anyone in the osgedu VO Added: trunk/tests/sites/uj-fork-gram2.xml =================================================================== --- trunk/tests/sites/uj-fork-gram2.xml (rev 0) +++ trunk/tests/sites/uj-fork-gram2.xml 2009-04-21 19:14:11 UTC (rev 2866) @@ -0,0 +1,9 @@ + + + + + + /nfs/home/benc/swifttest + + + Added: trunk/tests/sites/uj-pbs-gram2.xml =================================================================== --- trunk/tests/sites/uj-pbs-gram2.xml (rev 0) +++ trunk/tests/sites/uj-pbs-gram2.xml 2009-04-21 19:14:11 UTC (rev 2866) @@ -0,0 +1,9 @@ + + + + + + /nfs/home/benc/swifttest + + + From noreply at svn.ci.uchicago.edu Tue Apr 21 14:27:07 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Apr 2009 14:27:07 -0500 (CDT) Subject: [Swift-commit] r2867 - trunk/tests/sites/coaster Message-ID: <20090421192707.4779E9CC7D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-21 14:27:06 -0500 (Tue, 21 Apr 2009) New Revision: 2867 Added: trunk/tests/sites/coaster/uj-pbs-gram2.xml Log: coasters on the Univeristy of Johannesburg. these work with swift 0.9rc2 Added: trunk/tests/sites/coaster/uj-pbs-gram2.xml =================================================================== --- trunk/tests/sites/coaster/uj-pbs-gram2.xml (rev 0) +++ trunk/tests/sites/coaster/uj-pbs-gram2.xml 2009-04-21 19:27:06 UTC (rev 2867) @@ -0,0 +1,11 @@ + + + + + + + /nfs/home/benc/swifttest + 4 + + + From noreply at svn.ci.uchicago.edu Tue Apr 21 16:25:16 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Apr 2009 16:25:16 -0500 (CDT) Subject: [Swift-commit] r2868 - trunk/tests/sites Message-ID: <20090421212516.B78199CC7F@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-21 16:25:16 -0500 (Tue, 21 Apr 2009) New Revision: 2868 Added: trunk/tests/sites/README Log: notes on site tests Added: trunk/tests/sites/README =================================================================== --- trunk/tests/sites/README (rev 0) +++ trunk/tests/sites/README 2009-04-21 21:25:16 UTC (rev 2868) @@ -0,0 +1,28 @@ +Site tests +========== + +To run all tests in the root, say: + + $ ./run-all + +To run a single site test, say: + + $ ./run-site + +To run a subdirectory of tests, say: + + $ ./run-all subdirectory/ + +The trailing / is important. + +The tests in the root of sites/ should generally work for users associated +with Swift (for example, by being on Swift teragrid allocations or by being +in an OSG VO) + +Tests in wonky/ should run locally against a Swift built with provider-wonky. +These generally should test the reliabilty mechanisms of Swift. + +Tests defined in subdirectories named after hosts (eg tg-login.uc.teragrid.org) +should be run on that host, and should test functionality available on that +host (such as a local PBS or condor installation). + From noreply at svn.ci.uchicago.edu Tue Apr 21 17:31:04 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Apr 2009 17:31:04 -0500 (CDT) Subject: [Swift-commit] r2869 - in trunk/tests/sites: . communicado.ci.uchicago.edu Message-ID: <20090421223104.4372F9CC7F@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-21 17:31:03 -0500 (Tue, 21 Apr 2009) New Revision: 2869 Added: trunk/tests/sites/communicado.ci.uchicago.edu/ trunk/tests/sites/communicado.ci.uchicago.edu/local-condor.xml Log: communicado condor test - fails with condor whitespace problem; and needs condor_reschedule to be run to make condor on that machine actually run jobs Added: trunk/tests/sites/communicado.ci.uchicago.edu/local-condor.xml =================================================================== --- trunk/tests/sites/communicado.ci.uchicago.edu/local-condor.xml (rev 0) +++ trunk/tests/sites/communicado.ci.uchicago.edu/local-condor.xml 2009-04-21 22:31:03 UTC (rev 2869) @@ -0,0 +1,9 @@ + + + + + + /home/benc/swifttest + + + From noreply at svn.ci.uchicago.edu Thu Apr 23 03:13:36 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 03:13:36 -0500 (CDT) Subject: [Swift-commit] r2870 - trunk/libexec/log-processing Message-ID: <20090423081336.260979CC81@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 03:13:35 -0500 (Thu, 23 Apr 2009) New Revision: 2870 Modified: trunk/libexec/log-processing/log-to-initshareddir-transitions Log: make initshared transitions generator work even when there are no such log transitions Modified: trunk/libexec/log-processing/log-to-initshareddir-transitions =================================================================== --- trunk/libexec/log-processing/log-to-initshareddir-transitions 2009-04-21 22:31:03 UTC (rev 2869) +++ trunk/libexec/log-processing/log-to-initshareddir-transitions 2009-04-23 08:13:35 UTC (rev 2870) @@ -9,3 +9,4 @@ sed 's/\([^ ]*\) INFO vdl:initshareddir END host=\([^ ]*\) -.*$/\1 \2 END/' |\ grep -E '^[^$]' +exit 0 From noreply at svn.ci.uchicago.edu Thu Apr 23 03:14:14 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 03:14:14 -0500 (CDT) Subject: [Swift-commit] r2871 - trunk Message-ID: <20090423081414.F23FF9CC81@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 03:14:14 -0500 (Thu, 23 Apr 2009) New Revision: 2871 Removed: trunk/162-dot-on-array.swift Log: remove a misplaced test Deleted: trunk/162-dot-on-array.swift =================================================================== --- trunk/162-dot-on-array.swift 2009-04-23 08:13:35 UTC (rev 2870) +++ trunk/162-dot-on-array.swift 2009-04-23 08:14:14 UTC (rev 2871) @@ -1,36 +0,0 @@ -type messagefile; - -(messagefile t) greeting(string s) { - app { - echo s stdout=@filename(t); - } -} - -messagefile outfile <"162-dot-on-array.out">; - -type astruct { - string a; - string b; - string c; -}; - -astruct foo[]; - -foo[0].a = "zero-A"; -foo[0].b = "zero-B"; -foo[0].c = "zero-C"; - -foo[1].a = "one-A"; -foo[1].b = "one-B"; -foo[1].c = "one-C"; - -foo[2].a = "two-A"; -foo[2].b = "two-B"; -foo[2].c = "two-C"; - -string s[] = foo.c; - -string u = s[2]; - -outfile = greeting(u); - From noreply at svn.ci.uchicago.edu Thu Apr 23 03:15:04 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 03:15:04 -0500 (CDT) Subject: [Swift-commit] r2872 - trunk/libexec/log-processing Message-ID: <20090423081504.0BC239CC81@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 03:15:03 -0500 (Thu, 23 Apr 2009) New Revision: 2872 Modified: trunk/libexec/log-processing/log-to-execute-transitions Log: There might not be any executes in a successful run log (because the SwiftScript program may not have called any...). This commit makes the log processing code happier with that situation. Modified: trunk/libexec/log-processing/log-to-execute-transitions =================================================================== --- trunk/libexec/log-processing/log-to-execute-transitions 2009-04-23 08:14:14 UTC (rev 2871) +++ trunk/libexec/log-processing/log-to-execute-transitions 2009-04-23 08:15:03 UTC (rev 2872) @@ -20,6 +20,11 @@ sed 's/\([^ ]*\) INFO vdl:execute \([^ ]*\) .*thread=\([^ ]*\).*$/\1 \3 \2/' |\ grep -E '^[^$]' +# force a successful exit, rather than using the exit codes from pipeline +# components above (most probably grep?) + +exit 0 + #sed 's/^\([^ ]*\) [^ ]+ +vdl:execute \([^ ]*\) thread=\([^ ]*\).*/\1 \3 \2/' # | \ From noreply at svn.ci.uchicago.edu Thu Apr 23 03:16:16 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 03:16:16 -0500 (CDT) Subject: [Swift-commit] r2873 - trunk/src/org/griphyn/vdl/engine Message-ID: <20090423081616.D7FE59CC81@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 03:16:16 -0500 (Thu, 23 Apr 2009) New Revision: 2873 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java Log: correct missing quote in error message Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-04-23 08:15:03 UTC (rev 2872) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-04-23 08:16:16 UTC (rev 2873) @@ -756,7 +756,7 @@ || type.equals("boolean") || type.equals("boolean[]")) { appST.setAttribute("arguments", argumentST); } else { - throw new CompilationException("Cannot pass type '"+type+"' as a parameter to application '"+app.getExecutable()); + throw new CompilationException("Cannot pass type '"+type+"' as a parameter to application '"+app.getExecutable()+"'"); } } if(app.getStdin()!=null) From noreply at svn.ci.uchicago.edu Thu Apr 23 09:34:37 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 09:34:37 -0500 (CDT) Subject: [Swift-commit] r2874 - trunk/tests/sites/coaster Message-ID: <20090423143437.E1CA39CC81@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 09:34:37 -0500 (Thu, 23 Apr 2009) New Revision: 2874 Modified: trunk/tests/sites/coaster/teraport-gt2-gt2-pbs.xml Log: more recent teraport settings Modified: trunk/tests/sites/coaster/teraport-gt2-gt2-pbs.xml =================================================================== --- trunk/tests/sites/coaster/teraport-gt2-gt2-pbs.xml 2009-04-23 08:16:16 UTC (rev 2873) +++ trunk/tests/sites/coaster/teraport-gt2-gt2-pbs.xml 2009-04-23 14:34:37 UTC (rev 2874) @@ -2,8 +2,8 @@ fast 00:05:00 - - - /gpfs1/osg/data/swift-test + + + /gpfs/teraport/benc/swifttest From noreply at svn.ci.uchicago.edu Thu Apr 23 10:46:03 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 10:46:03 -0500 (CDT) Subject: [Swift-commit] r2875 - provenancedb Message-ID: <20090423154603.4A8AE9CC7D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 10:46:02 -0500 (Thu, 23 Apr 2009) New Revision: 2875 Modified: provenancedb/swift-prov-import-all-logs Log: Import all log files, not just ones for apparently successful runs. Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2009-04-23 14:34:37 UTC (rev 2874) +++ provenancedb/swift-prov-import-all-logs 2009-04-23 15:46:02 UTC (rev 2875) @@ -43,7 +43,11 @@ if [ "$EXISTING" -eq "0" ]; then echo IMPORTING -if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then + if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then + wfstatus="SUCCESS" + else + wfstatus="FAIL" + fi echo version $version in log file $filename echo ============= will import ============= @@ -59,11 +63,7 @@ fi # import-run-to-xml $filename - echo "INSERT INTO known_workflows (workflow_log_filename, version, importstatus) VALUES ('$filename','$version','imported');" | $SQLCMD - else - echo SKIP: Workflow does not report successful end. - echo "INSERT INTO known_workflows (workflow_log_filename, version, importstatus) VALUES ('$filename','$version','skipped - workflow did not end sucessfully');" | $SQLCMD - fi + echo "INSERT INTO known_workflows (workflow_log_filename, version, importstatus) VALUES ('$filename','$version','$wfstatus');" | $SQLCMD else echo SKIP: Already known in workflow fi From noreply at svn.ci.uchicago.edu Thu Apr 23 10:46:50 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 10:46:50 -0500 (CDT) Subject: [Swift-commit] r2876 - provenancedb Message-ID: <20090423154650.DA0CD9CC7D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 10:46:50 -0500 (Thu, 23 Apr 2009) New Revision: 2876 Modified: provenancedb/prov-init.sql provenancedb/prov-to-sql.sh Log: processes table to generalise executes, operators, @functions, compound procedures, internal procedures Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2009-04-23 15:46:02 UTC (rev 2875) +++ provenancedb/prov-init.sql 2009-04-23 15:46:50 UTC (rev 2876) @@ -2,6 +2,7 @@ -- this is the schema definition used for the main relational provenance -- implementation (in both sqlite3 and postgres) +DROP TABLE processes; DROP TABLE executes; DROP TABLE execute2s; DROP TABLE dataset_usage; @@ -21,12 +22,22 @@ execute_id char(128) ); +-- processes gives information about each process (in the OPM sense) +-- it is augmented by information in other tables +CREATE TABLE processes + (id char(128) PRIMARY KEY, -- a uri + type char(16) -- specifies the type of process. for any type, it + -- must be the case that the specific type table + -- has an entry for this process. + -- Having this type here seems poor normalisation, though? + ); + -- this gives information about each execute. -- each execute is identified by a unique URI. other information from --- swift logs is also stored here +-- swift logs is also stored here. an execute is an OPM process. CREATE TABLE executes - (id char(128) PRIMARY KEY, + (id char(128) PRIMARY KEY, -- actually foreign key to processes starttime numeric, duration numeric, finalstate char(128), @@ -40,14 +51,14 @@ CREATE TABLE execute2s (id char(128) PRIMARY KEY, - execute_id, -- secondary key to executes table + execute_id, -- secondary key to executes and processes tables starttime numeric, duration numeric, finalstate char(128), site char(128) ); --- dataset_usage records usage relationships between executes and datasets; +-- dataset_usage records usage relationships between processes and datasets; -- in SwiftScript terms, the input and output parameters for each -- application procedure invocation; in OPM terms, the artificts which are -- input to and output from each process that is a Swift execution @@ -56,12 +67,13 @@ -- dataset_id for common queries? maybe add arbitrary ID for sake of it? CREATE TABLE dataset_usage - (execute_id char(128), -- foreign key but not enforced because maybe execute - -- doesn't exist at time. same type as executes.id + (process_id char(128), -- foreign key but not enforced because maybe process + -- doesn't exist at time. same type as processes.id direction char(1), -- I or O for input or output dataset_id char(128), -- this will perhaps key against dataset table param_name char(128) -- the name of the parameter in this execute that - -- this dataset was bound to. + -- this dataset was bound to. sometimes this must + -- be contrived (for example, in positional varargs) ); Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-04-23 15:46:02 UTC (rev 2875) +++ provenancedb/prov-to-sql.sh 2009-04-23 15:46:50 UTC (rev 2876) @@ -10,6 +10,7 @@ rm -f tmp-import.sql while read time duration thread endstate app scratch; do + echo "INSERT INTO processes (id, type) VALUES ('$thread', 'execute');" >> tmp-import.sql echo "INSERT INTO executes (id, starttime, duration, finalstate, app, scratch) VALUES ('$thread', $time, $duration, '$endstate', '$app', '$scratch');" >> tmp-import.sql done < execute.global.event @@ -22,14 +23,13 @@ echo "INSERT INTO execute2s (id, execute_id, starttime, duration, finalstate, site) VALUES ('$globalid', '$execute_id', '$starttime', '$duration', '$endstate', '$site');" >> tmp-import.sql done < execute2.global.event - while read thread direction dataset variable rest; do if [ "$direction" == "input" ] ; then dir=I else dir=O fi - echo "INSERT INTO dataset_usage (execute_id, direction, dataset_id, param_name) VALUES ('$thread', '$dir', '$dataset', '$variable');" >> tmp-import.sql + echo "INSERT INTO dataset_usage (process_id, direction, dataset_id, param_name) VALUES ('$thread', '$dir', '$dataset', '$variable');" >> tmp-import.sql done < tie-data-invocs.txt while read thread appname; do From noreply at svn.ci.uchicago.edu Thu Apr 23 10:47:39 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 10:47:39 -0500 (CDT) Subject: [Swift-commit] r2877 - provenancedb Message-ID: <20090423154739.926C99CC7D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 10:47:39 -0500 (Thu, 23 Apr 2009) New Revision: 2877 Modified: provenancedb/prepare-provenance-chart provenancedb/prov-init.sql provenancedb/prov-to-sql.sh Log: Imports the values of primitive-typed datasets. Modified: provenancedb/prepare-provenance-chart =================================================================== --- provenancedb/prepare-provenance-chart 2009-04-23 15:46:50 UTC (rev 2876) +++ provenancedb/prepare-provenance-chart 2009-04-23 15:47:39 UTC (rev 2877) @@ -21,6 +21,8 @@ #AbstractDataNode dataset 3814442 has filename file://localhost/q.out cat $1 | grep ' FILENAME ' | sed 's/^.*dataset=\([^ ]*\) filename=\([^ ]*\).*$/\1 \2/' | sort | uniq > dataset-filenames.txt +cat $1 | grep ' VALUE ' | sed 's/^.*dataset=\([^ ]*\) VALUE=\(.*\)$/\1 \2/' | sort | uniq > dataset-values.txt + cat $1 | grep ' PROCEDURE ' | sed "s/^.*thread=\([^ ]*\) name=\([^ ]*\)\$/${WFID}\1 \2/" > invocation-procedure-names.txt info-to-extrainfo > extrainfo.txt Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2009-04-23 15:46:50 UTC (rev 2876) +++ provenancedb/prov-init.sql 2009-04-23 15:47:39 UTC (rev 2877) @@ -10,6 +10,7 @@ DROP TABLE dataset_containment; DROP TABLE dataset_filenames; DROP TABLE executes_in_workflows; +DROP TABLE dataset_values; DROP TABLE known_workflows; DROP TABLE workflow_events; DROP TABLE extrainfo; @@ -115,6 +116,15 @@ filename char(128) ); +-- dataset_values stores the value for each dataset which is known to have +-- a value (which is all assigned primitive types). No attempt is made here +-- to expose that value as an SQL type other than a string, and so (for +-- example) SQL numerical operations should not be expected to work, even +-- though the user knows that a particular dataset stores a numeric value. +CREATE TABLE dataset_values + ( dataset_id char(128), -- should be primary key + value char(128) + ); -- known_workflows stores some information about each workflow log that has -- been seen by the importer: the log filename, swift version and import Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-04-23 15:46:50 UTC (rev 2876) +++ provenancedb/prov-to-sql.sh 2009-04-23 15:47:39 UTC (rev 2877) @@ -45,6 +45,10 @@ echo "INSERT INTO dataset_filenames (dataset_id, filename) VALUES ('$dataset', '$filename');" >> tmp-import.sql done < dataset-filenames.txt +while read dataset value; do + echo "INSERT INTO dataset_values (dataset_id, value) VALUES ('$dataset', '$value');" >> tmp-import.sql +done < dataset-values.txt + while read start duration wfid; do echo "INSERT INTO workflow_events (workflow_id,starttime, duration) VALUES ('$wfid', '$start', '$duration');" >> tmp-import.sql done < workflow.event From noreply at svn.ci.uchicago.edu Thu Apr 23 11:14:53 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 11:14:53 -0500 (CDT) Subject: [Swift-commit] r2878 - provenancedb Message-ID: <20090423161453.56D319CC7D@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 11:14:52 -0500 (Thu, 23 Apr 2009) New Revision: 2878 Modified: provenancedb/prepare-provenance-chart provenancedb/prov-to-sql.sh Log: import compound procedures into provdb Modified: provenancedb/prepare-provenance-chart =================================================================== --- provenancedb/prepare-provenance-chart 2009-04-23 15:47:39 UTC (rev 2877) +++ provenancedb/prepare-provenance-chart 2009-04-23 16:14:52 UTC (rev 2878) @@ -27,3 +27,4 @@ info-to-extrainfo > extrainfo.txt +cat $1 | grep ' STARTCOMPOUND ' | sed 's/^.* thread=\([^ ]*\) name=\([^ ]*\).*$/\1 \2/' > compounds.txt Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-04-23 15:47:39 UTC (rev 2877) +++ provenancedb/prov-to-sql.sh 2009-04-23 16:14:52 UTC (rev 2878) @@ -58,6 +58,13 @@ echo "INSERT INTO extrainfo (execute2id, extrainfo) VALUES ('$id', '$extrainfo');" >> tmp-import.sql done < extrainfo.txt +# TODO this could merge with other naming tables +while read thread procname ; do + compoundid=$WFID$thread + echo "INSERT INTO processes (id, type) VALUES ('$compoundid', 'compound');" >> tmp-import.sql + echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$compoundid', '$procname');" >> tmp-import.sql +done < compounds.txt + echo Sending SQL to DB $SQLCMD < tmp-import.sql From noreply at svn.ci.uchicago.edu Thu Apr 23 11:37:47 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 11:37:47 -0500 (CDT) Subject: [Swift-commit] r2879 - SwiftApps/pc3 Message-ID: <20090423163747.55BE69CC81@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 11:37:47 -0500 (Thu, 23 Apr 2009) New Revision: 2879 Modified: SwiftApps/pc3/pc3.swift Log: notes in pc3.swift Modified: SwiftApps/pc3/pc3.swift =================================================================== --- SwiftApps/pc3/pc3.swift 2009-04-23 16:14:52 UTC (rev 2878) +++ SwiftApps/pc3/pc3.swift 2009-04-23 16:37:47 UTC (rev 2879) @@ -126,24 +126,44 @@ int entries; xmlfile split_list_output[]; +// check that the batch directory and manifest are ready is_csv_ready_file_exists_output = is_csv_ready_file_exists(csv_root_path_input); - read_csv_ready_file_output = read_csv_ready_file(csv_root_path_input); - is_match_csv_file_tables_output = is_match_csv_file_tables(read_csv_ready_file_output); - if(is_match_csv_file_tables_output) { +// read_csv_ready_file_output will contain a list of csv file entries, +// with each containing the path of a csv, a path to a header file, +// the rowcount for that file, the target table name, and a checksum +read_csv_ready_file_output = read_csv_ready_file(csv_root_path_input); + +// check that for each csv file entry, we have a corresponding csv file +is_match_csv_file_tables_output = is_match_csv_file_tables(read_csv_ready_file_output); +if(is_match_csv_file_tables_output) { + + // These variables are used to sequence database accesses in a dataflow + // style, rather than using imperative sequencing operations. With a + // database that allowed parallel accesses, we would not need to have + // such tight sequencing, though we would still need some. external db_over_time[]; - external dbinit; // some bug in analysis means can't use db_over_time for initial one + external dbinit; // some bug in Swift static analysis means can't use db_over_time for initial one + + + // create_empty_load_db_output contains a reference to the + // newly created database (create_empty_load_db_output, dbinit) = create_empty_load_db(job_id); + + // count the entries in read_csv_ready_file_output + // TODO could this table be read into a Swift struct array? count_entries_output = count_entries(read_csv_ready_file_output); entries = readData(count_entries_output); int entries_seq[] = [1:entries]; - + + // splitlistoutput[] will contain one element for each entry in + // read_csv_ready_file_output foreach i in entries_seq { split_list_output[i] = extract_entry(read_csv_ready_file_output, i); } - -// TODO this can merge with merge with above foreach +// TODO this can merge with merge with above foreach, and split_list_output +// does not need to be an array then foreach i in entries_seq { boolean is_exists_csv_file_output; @@ -153,28 +173,57 @@ boolean update_computed_columns_output; boolean is_match_table_row_count_output; boolean is_match_table_column_ranges_output; - + + // check that the data files exist, and use an external to + // sequence this before read_csv_file_column_names is_exists_csv_file_output = is_exists_csv_file(split_list_output[i]); external thread6 = checkvalid(is_exists_csv_file_output); -read_csv_file_column_names_output = read_csv_file_column_names(split_list_output[i], thread6); + + // update the CSVFileEntry with column names read from the header file + // (which is itself listed in the CSVFileEntry) + read_csv_file_column_names_output = read_csv_file_column_names(split_list_output[i], thread6); + + // check that the table column names match the csv column names + // TODO where is the database reference passed in? is there enough + // information in the CSVFileEntry to determine that? there is a + // table name at least... I guess I should look inside is_match_csv_file_column_names_output = is_match_csv_file_column_names(read_csv_file_column_names_output); external thread2 = checkvalid(is_match_csv_file_column_names_output); - if(i==1) { // first element... + // explicit external-based sequencing between checking table validity and + // loading into the DB + + if(i==1) { // first element... see above note about not being + // able to use db_over_time[1] for this. load_csv_file_into_table_output = load_csv_file_into_table(create_empty_load_db_output, read_csv_file_column_names_output, dbinit); } else { load_csv_file_into_table_output = load_csv_file_into_table(create_empty_load_db_output, read_csv_file_column_names_output, db_over_time[i]); } external thread3=checkvalid(load_csv_file_into_table_output); + + // explicitly sequence the database load and the computed-column-update + + // update the computed columns, and fail the workflow if this fails update_computed_columns_output = update_computed_columns(create_empty_load_db_output, read_csv_file_column_names_output, thread3); external thread4 = checkvalid(update_computed_columns_output); + + + // now check that we loaded in the right number of rows, and ranges + // TODO With a parallelised database, we could do these two checks in + // parallel, I think. But for now, we use 'thread1' to sequence them + // explicitly + is_match_table_row_count_output = is_match_table_row_count(create_empty_load_db_output, read_csv_file_column_names_output, thread4); external thread1 = checkvalid(is_match_table_row_count_output); is_match_table_column_ranges_output = is_match_table_column_ranges(create_empty_load_db_output, read_csv_file_column_names_output, thread1); db_over_time[i+1] = checkvalid(is_match_table_column_ranges_output); + } + + // TODO if we did this in a more parallel fashion, could wait for the + // whole db_over_time array to be closed... compact_database(create_empty_load_db_output, db_over_time[entries+1]); - } - else { +} +else { stop(); - } +} From noreply at svn.ci.uchicago.edu Thu Apr 23 11:48:22 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Apr 2009 11:48:22 -0500 (CDT) Subject: [Swift-commit] r2880 - trunk/tests/sites/broken Message-ID: <20090423164822.190C39CC81@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-23 11:48:21 -0500 (Thu, 23 Apr 2009) New Revision: 2880 Added: trunk/tests/sites/broken/tp-pbs-gram4.xml Log: a broken teraport pbs+gram4 site test Added: trunk/tests/sites/broken/tp-pbs-gram4.xml =================================================================== --- trunk/tests/sites/broken/tp-pbs-gram4.xml (rev 0) +++ trunk/tests/sites/broken/tp-pbs-gram4.xml 2009-04-23 16:48:21 UTC (rev 2880) @@ -0,0 +1,9 @@ + + + + + + /home/benc/swifttest + + + From noreply at svn.ci.uchicago.edu Fri Apr 24 06:57:45 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 24 Apr 2009 06:57:45 -0500 (CDT) Subject: [Swift-commit] r2881 - trunk Message-ID: <20090424115745.8DB589CC82@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-24 06:57:44 -0500 (Fri, 24 Apr 2009) New Revision: 2881 Modified: trunk/CHANGES.txt Log: fix type in CHANGES note Modified: trunk/CHANGES.txt =================================================================== --- trunk/CHANGES.txt 2009-04-23 16:48:21 UTC (rev 2880) +++ trunk/CHANGES.txt 2009-04-24 11:57:44 UTC (rev 2881) @@ -65,7 +65,7 @@ made in the same statement. (02/05/09) -*** TIghter type checking on app blocks. Primitive types and arrays of +*** Tighter type checking on app blocks. Primitive types and arrays of primitive types are permitted. Other types are prohibited. *** Arrays of primitive types can be passed to app blocks, and will be From noreply at svn.ci.uchicago.edu Mon Apr 27 08:05:32 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Apr 2009 08:05:32 -0500 (CDT) Subject: [Swift-commit] r2882 - www/downloads Message-ID: <20090427130533.099199CC8E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-27 08:05:32 -0500 (Mon, 27 Apr 2009) New Revision: 2882 Added: www/downloads/release-notes-0.9.txt Modified: www/downloads/index.php Log: Web page updates for 0.9 release Modified: www/downloads/index.php =================================================================== --- www/downloads/index.php 2009-04-24 11:57:44 UTC (rev 2881) +++ www/downloads/index.php 2009-04-27 13:05:32 UTC (rev 2882) @@ -27,16 +27,17 @@

DOWNLOADS

Latest Release

-

Swift 0.8 - 2009/01/30

-

Swift v0.8 is a development release intended to release functionality -and fixes that have gone in to trunk since v0.7. -[swift-0.8.tar.gz] +

Swift 0.9 - 2009/04/27

+

Swift v0.9 is a development release intended to release functionality +and fixes that have gone in to trunk since v0.8. +[swift-0.9.tar.gz] + +[release-notes-0.8.txt]

@@ -124,6 +125,44 @@

Historical releases

+

Swift 0.8 - 2009/01/30

+

Swift v0.8 is a development release intended to release functionality +and fixes that have gone in to trunk since v0.7. +[swift-0.8.tar.gz] +[swift-0.8-stripped.tar.gz - for installation on systems with existing grid stacks] +[release-notes-0.8.txt] +

+ + + +

Swift 0.7 - 2008/11/11

Swift v0.7 is a development release intended to release functionality and fixes that have gone in to trunk since v0.7. Added: www/downloads/release-notes-0.9.txt =================================================================== --- www/downloads/release-notes-0.9.txt (rev 0) +++ www/downloads/release-notes-0.9.txt 2009-04-27 13:05:32 UTC (rev 2882) @@ -0,0 +1,139 @@ + +User interface +============== + +*** Added console text interface to provide live information about swift + runs, which can be enabled with the -tui commandline parameter + +*** when replication is enabled, swift will locally kill jobs that have + run for twice their specified walltime + +Execution modes +=============== + +*** Support for Condor-G submissions, by setting a job specification attribute + of "grid" and specifying a gridResource attribute containing the string + to be placed into the Condor-G grid_resource classad. + +*** Support for submissions to a local condor pool + +*** Coasters: substantial ongoing development work + +Configuration parameters +======================== + +*** Environment variable SWIFT_EXTRA_INFO, when set in an environment profile, + is executed with the result being recorded in wrapper logs. This allows + arbitrary information about the remote environment to be gathered and + returned to the submit side. + +*** New configuration option wrapper.invocation.mode, specifiable either + globally in the configuration file or per-site as a profile key, + that configures whether wrapper script invocations are made with an + absolute path (as was the behaviour in Swift 0.8) or with a relative + path (as was behaviour in previous versions of Swift). + +*** New profile key coasterWorkerMaxwalltime - a coaster parameter to + explicitly set worker maxwalltime, overriding the default computation + based on job walltimes. this should be useful when it is desirable + to specify worker parameters based on the known properties of the + target queue rather than on properties of the jobs to be executed + inside coasters. + +*** New profile key coasterInternalIP parameter that allows the IP address + used by coaster workers to communicate to the coaster head job to be set + explicitly. This can be used when running on a cluster which an internal + network which cannot access the IP address that the head node picks + +*** configuration property ticker.disable to disable runtime ticker display + +New commands +============ + +*** The log-processing code, primarily exposed as the swift-plot-log command, + has been merged into the main Swift release, rather than being a separate + download. + +*** There is a new command swift-osg-ress-site-catalog which will generate + a site catalog based on data gathered from OSG's ReSS information + system. This can be used by OSG users to easily generate a large number + of site catalog entries for sites that are likely to work. + +Language changes +================ + +*** Procedure invocations can be made in any expression context now, rather + than only directly in an assignment. + +*** Mappings can now be made in any declaration, whether it has an assignment + or not. Previous a procedure call assignment and a mapping could not be + made in the same statement. + +*** Handling of [*] and . has changed. [*] has been broken for several + Swift releases, but now has some approximation of its former behaviour, + in a cleaner implementation. + [*] is now an identity operation, that is array[*] == array. + The structure access operator . can now take an array on the left + hand side. In situations where a[*].img was used, it is permissible + to continue to write a[*].img, or to write a.img - both of these will + return an array of img elements. + +*** Tighter type checking on app blocks. Primitive types and arrays of + primitive types are permitted. Other types are prohibited. + +*** Arrays of primitive types can be passed to app blocks, and will be + expanded into multiple command-line parameters, one parameter per + array element. + +*** != operator had been broken forever. It works now. + +*** output of trace() is changed for non-primitive datasets: + + * Array handling: + Previous to this, trace would show the internal Java object representation + of an arrays, which is fairly useless for a user. This commit makes + trace show more about the array, and only emit the trace when the array + is closed. + + * Other datasets: + Trace will wait for those datasets to be closed, and emit their default + string representation (including the variable name and path used in + SwiftScript) + +*** loop condition in iterate statements can now refer to variables declared + within the iteration body (this was bug 177) + +*** Mappings can now be made in any declaration, whether it has an assignment + or not. + +*** strsplit function which will split the input string based on separators + that match the given pattern and return a string array. + + +Deprecations and removal of old functionality +============================================= + +*** Removed support for .dtm file extension which was deprecated in Swift 0.4 + +Internal changes +================ + +*** The wrapper.sh and seq.sh scripts that are deployed to remote sites to + help with execution have been renamed to more Swift specific names, to + avoid collision with user-supplied files of the same name. The new names + are _swiftwrap and _swiftseq + +*** Recompilation will happen if a .kml file was compiled with a different + version of Swift to the version being invoked. This is in addition to the + existing behaviour where a .swift file will be recompiled if it is newer + than the corresponding .kml file. + +*** Added a throttling parallelFor and changed the swift foreach underlying + implementation to it. The throttling parallelFor limits the number of + concurrent iterations allowed, which should allow swift to scale better + in certain cases. This is controlled by the configuration property + foreach.max.threads + +*** Default JVM heap size is now 256M + + From noreply at svn.ci.uchicago.edu Mon Apr 27 08:09:28 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Apr 2009 08:09:28 -0500 (CDT) Subject: [Swift-commit] r2883 - www/downloads Message-ID: <20090427130928.7D4AF9CC8E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-27 08:09:28 -0500 (Mon, 27 Apr 2009) New Revision: 2883 Modified: www/downloads/index.php Log: fix release notes labelling Modified: www/downloads/index.php =================================================================== --- www/downloads/index.php 2009-04-27 13:05:32 UTC (rev 2882) +++ www/downloads/index.php 2009-04-27 13:09:28 UTC (rev 2883) @@ -38,8 +38,8 @@ onClick="javascript: pageTracker._trackPageview('/swift/packages/swift-0.8-stripped.tar.gz');" >swift-0.8-stripped.tar.gz - for installation on systems with existing grid stacks] --> [release-notes-0.8.txt] +onClick="javascript: pageTracker._trackPageview('/swift/packages/release-notes-0.9.txt');" +>release-notes-0.9.txt]

+ +[swift-0.9-stripped.tar.gz - for installation on systems with existing grid stacks] [release-notes-0.9.txt] From noreply at svn.ci.uchicago.edu Tue Apr 28 02:11:14 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 28 Apr 2009 02:11:14 -0500 (CDT) Subject: [Swift-commit] r2887 - in trunk: libexec src/org/griphyn/vdl/karajan/lib Message-ID: <20090428071114.7D52D9CC8E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-28 02:11:13 -0500 (Tue, 28 Apr 2009) New Revision: 2887 Modified: trunk/libexec/vdl-int.k trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java Log: new progress ticker stage for exit status checks, so that finished jobs do not stay in Active state when file rate limiting is happening Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2009-04-27 13:57:37 UTC (rev 2886) +++ trunk/libexec/vdl-int.k 2009-04-28 07:11:13 UTC (rev 2887) @@ -421,7 +421,8 @@ replicationChannel=replicationChannel jobid=jobid ) - + + vdl:setprogress("Checking status") if(statusMode == "files" checkJobStatus(rhost, wfdir, jobid, tr, jobdir) ) Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2009-04-27 13:57:37 UTC (rev 2886) +++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2009-04-28 07:11:13 UTC (rev 2887) @@ -36,6 +36,7 @@ "Submitting", "Submitted", "Active", + "Checking status", "Stage out", "Failed", "Replicating", From noreply at svn.ci.uchicago.edu Tue Apr 28 06:19:49 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 28 Apr 2009 06:19:49 -0500 (CDT) Subject: [Swift-commit] r2888 - trunk/resources Message-ID: <20090428111949.8F99A9CC8F@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-28 06:19:49 -0500 (Tue, 28 Apr 2009) New Revision: 2888 Modified: trunk/resources/Karajan.stg Log: Log input and output parameters for all procedures, not just application procedures Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-04-28 07:11:13 UTC (rev 2887) +++ trunk/resources/Karajan.stg 2009-04-28 11:19:49 UTC (rev 2888) @@ -58,6 +58,8 @@ $optargs:default_arg();separator="\n"$ $outputs:typecheck();separator="\n"$ $inputs:typecheck();separator="\n"$ + $inputs:vdl_log_input();separator="\n"$ + $outputs:vdl_log_output();separator="\n"$ $if(binding)$ $vdl_execute(outputs=outputs,inputs=inputs,application=binding.application, name=name,line=line)$ $else$ @@ -99,8 +101,6 @@ vdl_execute(outputs,inputs,application,name,line) ::= << -$inputs:vdl_log_input();separator="\n"$ -$outputs:vdl_log_output();separator="\n"$ $application.exec$ $inputs:vdl_stagein();separator="\n"$ From noreply at svn.ci.uchicago.edu Tue Apr 28 07:39:19 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 28 Apr 2009 07:39:19 -0500 (CDT) Subject: [Swift-commit] r2889 - in trunk: src/org/griphyn/vdl/karajan/lib tests/language-behaviour Message-ID: <20090428123919.1F45F9CC8F@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-28 07:39:18 -0500 (Tue, 28 Apr 2009) New Revision: 2889 Modified: trunk/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java trunk/tests/language-behaviour/run Log: make iterate give each iteration a unique thread ID, without launching an entire karajan thread for each iteration. add language-behaviour test to check that execute IDs are unique after every test Modified: trunk/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java 2009-04-28 11:19:49 UTC (rev 2888) +++ trunk/src/org/griphyn/vdl/karajan/lib/InfiniteCountingWhile.java 2009-04-28 12:39:18 UTC (rev 2889) @@ -9,6 +9,7 @@ import java.util.Arrays; import java.util.List; +import org.globus.cog.karajan.util.ThreadingContext; import org.globus.cog.karajan.workflow.nodes.*; import org.globus.cog.karajan.stack.VariableStack; import org.globus.cog.karajan.util.TypeUtil; @@ -26,6 +27,9 @@ public void pre(VariableStack stack) throws ExecutionException { stack.setVar("#condition", new Condition()); + ThreadingContext tc = (ThreadingContext)stack.getVar("#thread"); + stack.setVar("#iteratethread", tc); + stack.setVar("#thread", tc.split(0)); stack.setVar(VAR, "$"); String counterName = (String)stack.getVar(VAR); stack.setVar(counterName, Arrays.asList(new Integer[] {new Integer(0)})); @@ -62,6 +66,8 @@ Integer wrappedi = (Integer)l.get(0); int i = wrappedi.intValue(); i++; + ThreadingContext tc = (ThreadingContext)stack.getVar("#iteratethread"); + stack.setVar("#thread", tc.split(i)); stack.setVar(counterName, Arrays.asList(new Integer[] {new Integer(i)})); } else { Modified: trunk/tests/language-behaviour/run =================================================================== --- trunk/tests/language-behaviour/run 2009-04-28 11:19:49 UTC (rev 2888) +++ trunk/tests/language-behaviour/run 2009-04-28 12:39:18 UTC (rev 2889) @@ -87,6 +87,15 @@ fi fi + # check that all execute and execute2 IDs are unique + swift-plot-log ${t}-*.log execute.transitions + if [ "$( cat execute.transitions | grep ' START ' | cut -d ' ' -f 2 | sort | uniq -d)" != "" ]; then + echo EXECUTE IDS NOT UNIQUE + exit 3 + fi + + + done echo "Test passed at $(date)" if [ "$SUB" == "" ]; then From noreply at svn.ci.uchicago.edu Tue Apr 28 07:57:59 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 28 Apr 2009 07:57:59 -0500 (CDT) Subject: [Swift-commit] r2890 - www/papers Message-ID: <20090428125759.5BB3F9CC8F@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-28 07:57:59 -0500 (Tue, 28 Apr 2009) New Revision: 2890 Modified: www/papers/index.php Log: add W&M tech report on neurosurgery Modified: www/papers/index.php =================================================================== --- www/papers/index.php 2009-04-28 12:39:18 UTC (rev 2889) +++ www/papers/index.php 2009-04-28 12:57:59 UTC (rev 2890) @@ -68,6 +68,17 @@ [ pdf ] +
+ Andriy Fedorov, Benjamin Clifford, Simon K. War?eld, Ron Kikinis, Nikos Chrisochoides + + Non-Rigid Registration for Image-Guided Neurosurgery on the TeraGrid: A Case Study + + College of William and Mary Technical Report + 2009 + [ pdf ] +
+ +

Research leading to Swift and SwiftScript

From noreply at svn.ci.uchicago.edu Wed Apr 29 04:55:04 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 29 Apr 2009 04:55:04 -0500 (CDT) Subject: [Swift-commit] r2891 - in trunk: . libexec/log-processing Message-ID: <20090429095504.E15629CC90@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-29 04:55:04 -0500 (Wed, 29 Apr 2009) New Revision: 2891 Added: trunk/libexec/log-processing/log-to-compound-transitions Modified: trunk/build.xml trunk/libexec/log-processing/makefile Log: log-processing of compound procedure logs Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2009-04-28 12:57:59 UTC (rev 2890) +++ trunk/build.xml 2009-04-29 09:55:04 UTC (rev 2891) @@ -154,6 +154,7 @@ + Added: trunk/libexec/log-processing/log-to-compound-transitions =================================================================== --- trunk/libexec/log-processing/log-to-compound-transitions (rev 0) +++ trunk/libexec/log-processing/log-to-compound-transitions 2009-04-29 09:55:04 UTC (rev 2891) @@ -0,0 +1,15 @@ +#!/bin/bash + +# this will take a log file and produce a set of state transition +# events for every compound procedure + +# 2009-04-29 10:46:36,072+0200 INFO compound STARTCOMPOUND thread=0 name=compound +# 2009-04-29 10:46:36,398+0200 INFO compound ENDCOMPOUND thread=0 + +grep ' STARTCOMPOUND ' $1 | iso-to-secs | \ +sed 's/^\(.*\) INFO [^ ]* STARTCOMPOUND thread=\([0-9\-]*\) name=\([^ ]*\).*$/\1 \2 START \3/' + +grep ' ENDCOMPOUND ' $1 | iso-to-secs | \ +sed 's/^\(.*\) INFO [^ ]* ENDCOMPOUND thread=\([0-9\-]*\).*$/\1 \2 END/' + + Property changes on: trunk/libexec/log-processing/log-to-compound-transitions ___________________________________________________________________ Name: svn:executable + * Modified: trunk/libexec/log-processing/makefile =================================================================== --- trunk/libexec/log-processing/makefile 2009-04-28 12:57:59 UTC (rev 2890) +++ trunk/libexec/log-processing/makefile 2009-04-29 09:55:04 UTC (rev 2891) @@ -97,6 +97,9 @@ dostageout.transitions: $(LOG) log-to-dostageout-transitions < $(LOG) > dostageout.transitions +compound.transitions: $(LOG) + log-to-compound-transitions $(LOG) > compound.transitions + ## retry analysis jobs.retrycount.summary: execute2.event From noreply at svn.ci.uchicago.edu Wed Apr 29 04:58:49 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 29 Apr 2009 04:58:49 -0500 (CDT) Subject: [Swift-commit] r2892 - in trunk: libexec resources src/org/griphyn/vdl/karajan/lib tests/language-behaviour Message-ID: <20090429095849.925059CC90@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-29 04:58:49 -0500 (Wed, 29 Apr 2009) New Revision: 2892 Added: trunk/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java trunk/tests/language-behaviour/00241-nested-single-compound.out.expected trunk/tests/language-behaviour/00241-nested-single-compound.swift Modified: trunk/libexec/vdl-lib.xml trunk/resources/Karajan.stg trunk/tests/language-behaviour/run Log: Make nested compound procedures and application procedures nested inside compound procedures get a unique thread ID. Test that this is the case. Modified: trunk/libexec/vdl-lib.xml =================================================================== --- trunk/libexec/vdl-lib.xml 2009-04-29 09:55:04 UTC (rev 2891) +++ trunk/libexec/vdl-lib.xml 2009-04-29 09:58:49 UTC (rev 2892) @@ -29,6 +29,7 @@ + Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-04-29 09:55:04 UTC (rev 2891) +++ trunk/resources/Karajan.stg 2009-04-29 09:58:49 UTC (rev 2892) @@ -166,7 +166,9 @@ $statements;separator="\n"$ $else$ -$statements$ + + $statements$ + $endif$ >> Added: trunk/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/SequentialWithID.java 2009-04-29 09:58:49 UTC (rev 2892) @@ -0,0 +1,22 @@ +package org.griphyn.vdl.karajan.lib; + +import org.globus.cog.karajan.workflow.nodes.Sequential; +import org.globus.cog.karajan.stack.VariableStack; +import org.globus.cog.karajan.util.ThreadingContext; +import org.globus.cog.karajan.workflow.ExecutionException; + + +/** launch a single child, giving it a new thread ID, but without starting + an entire karajan thread +*/ + +public class SequentialWithID extends Sequential { + + protected void executeChildren(VariableStack stack) throws ExecutionException { + ThreadingContext tc = (ThreadingContext)stack.getVar("#thread"); + stack.setVar("#thread", tc.split(1)); + super.executeChildren(stack); + } + +} + Added: trunk/tests/language-behaviour/00241-nested-single-compound.out.expected =================================================================== --- trunk/tests/language-behaviour/00241-nested-single-compound.out.expected (rev 0) +++ trunk/tests/language-behaviour/00241-nested-single-compound.out.expected 2009-04-29 09:58:49 UTC (rev 2892) @@ -0,0 +1 @@ +f Added: trunk/tests/language-behaviour/00241-nested-single-compound.swift =================================================================== --- trunk/tests/language-behaviour/00241-nested-single-compound.swift (rev 0) +++ trunk/tests/language-behaviour/00241-nested-single-compound.swift 2009-04-29 09:58:49 UTC (rev 2892) @@ -0,0 +1,21 @@ +type file; + +(file t) greeting(string m) { + app { + echo m stdout=@filename(t); + } +} + +(file first) compound() { + first = greeting("f"); +} + +(file first) compoundB() { + first = compound(); +} + +file a <"00241-nested-single-compound.out">; + +a = compoundB(); + + Modified: trunk/tests/language-behaviour/run =================================================================== --- trunk/tests/language-behaviour/run 2009-04-29 09:55:04 UTC (rev 2891) +++ trunk/tests/language-behaviour/run 2009-04-29 09:58:49 UTC (rev 2892) @@ -88,14 +88,13 @@ fi # check that all execute and execute2 IDs are unique - swift-plot-log ${t}-*.log execute.transitions - if [ "$( cat execute.transitions | grep ' START ' | cut -d ' ' -f 2 | sort | uniq -d)" != "" ]; then - echo EXECUTE IDS NOT UNIQUE + swift-plot-log ${t}-*.log execute.transitions compound.transitions + if [ "$( cat compound.transitions execute.transitions | grep ' START ' | cut -d ' ' -f 2 | sort | uniq -d)" != "" ]; then + echo EXECUTE AND COMPOUND IDS NOT UNIQUE exit 3 fi - done echo "Test passed at $(date)" if [ "$SUB" == "" ]; then From noreply at svn.ci.uchicago.edu Wed Apr 29 05:21:39 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 29 Apr 2009 05:21:39 -0500 (CDT) Subject: [Swift-commit] r2893 - provenancedb Message-ID: <20090429102139.B6F1A9CC90@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-29 05:21:39 -0500 (Wed, 29 Apr 2009) New Revision: 2893 Modified: provenancedb/prepare-for-import provenancedb/prepare-provenance-chart provenancedb/prov-to-sql.sh Log: Make provenancedb use log-processing compound stuff introduced in r2891 Modified: provenancedb/prepare-for-import =================================================================== --- provenancedb/prepare-for-import 2009-04-29 09:58:49 UTC (rev 2892) +++ provenancedb/prepare-for-import 2009-04-29 10:21:39 UTC (rev 2893) @@ -4,7 +4,7 @@ # workflow has finished and will do everything necessary to import all # of the provenance information into the SQL provenance database. -swift-plot-log $1 execute.global.event execute2.event workflow.event execute2.global.event +swift-plot-log $1 execute.global.event execute2.event workflow.event execute2.global.event compound.event prepare-provenance-chart $1 Modified: provenancedb/prepare-provenance-chart =================================================================== --- provenancedb/prepare-provenance-chart 2009-04-29 09:58:49 UTC (rev 2892) +++ provenancedb/prepare-provenance-chart 2009-04-29 10:21:39 UTC (rev 2893) @@ -27,4 +27,3 @@ info-to-extrainfo > extrainfo.txt -cat $1 | grep ' STARTCOMPOUND ' | sed 's/^.* thread=\([^ ]*\) name=\([^ ]*\).*$/\1 \2/' > compounds.txt Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2009-04-29 09:58:49 UTC (rev 2892) +++ provenancedb/prov-to-sql.sh 2009-04-29 10:21:39 UTC (rev 2893) @@ -59,11 +59,11 @@ done < extrainfo.txt # TODO this could merge with other naming tables -while read thread procname ; do +while read start duration thread finalstate procname ; do compoundid=$WFID$thread echo "INSERT INTO processes (id, type) VALUES ('$compoundid', 'compound');" >> tmp-import.sql echo "INSERT INTO invocation_procedure_names (execute_id, procedure_name) VALUES ('$compoundid', '$procname');" >> tmp-import.sql -done < compounds.txt +done < compound.event echo Sending SQL to DB From noreply at svn.ci.uchicago.edu Wed Apr 29 06:39:28 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 29 Apr 2009 06:39:28 -0500 (CDT) Subject: [Swift-commit] r2894 - provenancedb Message-ID: <20090429113928.4EEAE9CC8E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-29 06:39:27 -0500 (Wed, 29 Apr 2009) New Revision: 2894 Added: provenancedb/db-to-opm.sh Modified: provenancedb/prov-to-opm.sh Log: database-backed opm exporter (rather than using the log-processing based OPM exporter). this is part of a shift towards using the db as the canonical source of provenance information, rather than log-processing output files. Added: provenancedb/db-to-opm.sh =================================================================== --- provenancedb/db-to-opm.sh (rev 0) +++ provenancedb/db-to-opm.sh 2009-04-29 11:39:27 UTC (rev 2894) @@ -0,0 +1,125 @@ +#!/bin/bash + +echo Generating OPM for entire sqlite3 database + +rm -f ids.txt +touch ids.txt + +mkid() { + if ! grep --silent "^$1\$" ids.txt ; then + echo $1 >> ids.txt + fi + echo -n x + grep -n "^$1\$" ids.txt | cut -f 1 -d ':' +} + +rm -f opm.xml + +echo "" > opm.xml + +# TODO - there are actually many accounts here, if compound procedure +# nesting is regarded as presenting multiple accounts. +# For now, emit everything into a single account, which probably +# violates some (explicit or implicit) integrity rules. +echo "" >> opm.xml + +echo "" >> opm.xml + +sqlite3 -separator ' ' -batch provdb "select * from processes;" | + while read id type ; do + flatid=$(mkid $id) + echo " " + echo " " + echo " " + echo " $type" + echo " $id" + + + if [ "$type" == "execute" ]; then + sqlite3 -separator ' ' -batch provdb "select * from executes where id='$id';" | ( read id starttime duration finalstate app scratch; echo " " ) + fi + + sqlite3 -separator ' ' -batch provdb "select procedure_name from invocation_procedure_names where execute_id='$id';" | ( read pn ; echo " $pn") + + # TODO type handling for other types + + echo " " + echo " " + done >> opm.xml + +echo "" >> opm.xml + +echo "" >> opm.xml + +# we need a list of all artifacts here. for now, take everything we can +# find in the tie-data-invocs and containment tables, uniquefied. +# This is probably the wrong thing to do? + +sqlite3 -separator ' ' -batch provdb "select outer_dataset_id from dataset_containment;" > tmp-dshandles.txt +sqlite3 -separator ' ' -batch provdb "select inner_dataset_id from dataset_containment;" >> tmp-dshandles.txt +sqlite3 -separator ' ' -batch provdb "select dataset_id from dataset_usage;" >> tmp-dshandles.txt + +cat tmp-dshandles.txt | sort | uniq > tmp-dshandles2.txt + +while read artifact ; do +artifactid=$(mkid $artifact) +echo " " +echo " " +echo " $artifact" + +sqlite3 -separator ' ' -batch provdb "select inner_dataset_id from dataset_containment where outer_dataset_id='$artifact';" | while read innerartifact ; do + innerflat=$(mkid $innerartifact) + echo "" + done + +sqlite3 -separator ' ' -batch provdb "select filename from dataset_filenames where dataset_id='$artifact';" | while read fn ; do + echo "$fn" + done + +sqlite3 -separator ' ' -batch provdb "select value from dataset_values where dataset_id='$artifact';" | while read value ; do + echo "$value" + done + +echo " " +echo " " +echo " " +done < tmp-dshandles2.txt >> opm.xml + +echo "" >> opm.xml + +echo "" >> opm.xml + +# other stuff can do this in any order, but here we must probably do it +# in two passes, one for each relation, in order to satisfy schema. +# but for now do it in a single pass... + +sqlite3 -separator ' ' -batch provdb "select * from dataset_usage;" | + while read thread direction dataset variable rest; do + datasetid=$(mkid $dataset) + threadid=$(mkid $thread) + if [ "$direction" == "I" ] ; then + echo " " + echo " " + echo " " + echo " " + echo " " + echo " " + elif [ "$direction" == "O" ] ; then + echo " " + echo " " + echo " " + echo " " + echo " " + echo " " + else + echo ERROR: unknown dataset usage direction: $direction + fi +done >> opm.xml + + + +echo "" >> opm.xml + +echo "" >> opm.xml +echo Finished generating OPM, in opm.xml + Property changes on: provenancedb/db-to-opm.sh ___________________________________________________________________ Name: svn:executable + * Modified: provenancedb/prov-to-opm.sh =================================================================== --- provenancedb/prov-to-opm.sh 2009-04-29 10:21:39 UTC (rev 2893) +++ provenancedb/prov-to-opm.sh 2009-04-29 11:39:27 UTC (rev 2894) @@ -23,7 +23,7 @@ # really know how should be mapped from Swift echo " " -done < $LOGDIR/execute.global.event >> opm.xml +done < execute.global.event >> opm.xml echo "" >> opm.xml @@ -38,11 +38,11 @@ while read outer inner; do echo $input echo $output -done < $LOGDIR/tie-containers.txt > tmp-dshandles.txt +done < tie-containers.txt > tmp-dshandles.txt while read t d dataset rest ; do echo $dataset -done < $LOGDIR/tie-data-invocs.txt >> tmp-dshandles.txt +done < tie-data-invocs.txt >> tmp-dshandles.txt cat tmp-dshandles.txt | sort | uniq > tmp-dshandles2.txt @@ -81,7 +81,7 @@ echo " " echo " " fi -done < $LOGDIR/tie-data-invocs.txt >> opm.xml +done < tie-data-invocs.txt >> opm.xml From noreply at svn.ci.uchicago.edu Wed Apr 29 06:40:47 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 29 Apr 2009 06:40:47 -0500 (CDT) Subject: [Swift-commit] r2895 - provenancedb Message-ID: <20090429114047.576389CC8E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-29 06:40:47 -0500 (Wed, 29 Apr 2009) New Revision: 2895 Modified: provenancedb/db-to-opm.sh Log: Add OPM OTimes into opm exporter Modified: provenancedb/db-to-opm.sh =================================================================== --- provenancedb/db-to-opm.sh 2009-04-29 11:39:27 UTC (rev 2894) +++ provenancedb/db-to-opm.sh 2009-04-29 11:40:47 UTC (rev 2895) @@ -13,6 +13,15 @@ grep -n "^$1\$" ids.txt | cut -f 1 -d ':' } +mkxmldatetime() { +TZ=UTC date -j -f %s $1 +"%Y-%m-%dT%H:%M:%SZ" + +# this includes the TZ, but not in the correct format for xsd:dateTime +# - xsd dateTime requires a colon separating the TZ hours and minutes +# date -j -f %s $1 +"%Y-%m-%dT%H:%M:%S%z" + +} + rm -f opm.xml echo "" > opm.xml @@ -87,6 +96,12 @@ echo "" >> opm.xml +# this agent is the Swift command-line client +# TODO other agents - the wrapper script invocations at least +echo "" >> opm.xml +echo " " >> opm.xml +echo "" >> opm.xml + echo "" >> opm.xml # other stuff can do this in any order, but here we must probably do it @@ -116,10 +131,27 @@ fi done >> opm.xml +# attach timings of executes +#sqlite3 -separator ' ' -batch provdb "select * from executes where id='$id';" | ( read id starttime duration finalstate app scratch; echo " " ) +# TODO for now, don't put any different between the no-later-than and +# no-earlier-than times. in reality, we know the swift log timestamp +# resolution and can take that into account + +sqlite3 -separator ' ' -batch provdb "select * from executes;" | while read id starttime duration finalstate app scratch ; do + ( echo "" ; + echo "" ; + export XMLSTART=$(mkxmldatetime $starttime); + echo "$XMLSTARTswiftclient" ; + export E=$(mkxmldatetime $(echo $starttime + $duration | bc -l)) ; + echo "$Eswiftclient" ; + echo "" ) >> opm.xml +done + echo "" >> opm.xml + echo "" >> opm.xml echo Finished generating OPM, in opm.xml From noreply at svn.ci.uchicago.edu Wed Apr 29 06:48:57 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 29 Apr 2009 06:48:57 -0500 (CDT) Subject: [Swift-commit] r2896 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20090429114857.58A7B9CC8E@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-04-29 06:48:56 -0500 (Wed, 29 Apr 2009) New Revision: 2896 Modified: trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java Log: Use a different large constant for external ID numbers so that they are less easily confused with AbstractDataNode ID numbers. Modified: trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java 2009-04-29 11:40:47 UTC (rev 2895) +++ trunk/src/org/griphyn/vdl/mapping/ExternalDataNode.java 2009-04-29 11:48:56 UTC (rev 2896) @@ -35,7 +35,7 @@ (or any other) particular value. Note that this introduces a maximum on the number of datasets which can be dealt with in any run to be about 2^62. */ - private static long datasetIDCounter = 720000000000l; + private static long datasetIDCounter = 850000000000l; /** This is used to provide a (hopefully) globally unique identifier for each time the datasetIDCounter is reset (whenever this class is