[Swift-commit] r2824 - usertools/cio/bin

[noreply at svn.ci.uchicago.edu]

Author: zzhang
Date: 2009-04-04 21:09:57 -0500 (Sat, 04 Apr 2009)
New Revision: 2824

Modified:
   usertools/cio/bin/broadcast.sh
   usertools/cio/bin/cnip.sh
   usertools/cio/bin/wrapper.sh
Log:
DHT integrated

Modified: usertools/cio/bin/broadcast.sh
===================================================================
--- usertools/cio/bin/broadcast.sh	2009-04-04 22:54:25 UTC (rev 2823)
+++ usertools/cio/bin/broadcast.sh	2009-04-05 02:09:57 UTC (rev 2824)
@@ -1,5 +1,6 @@
 #!/fuse/bin/bash
 
+LD_LIBRARY_PATH=/lib:/fuse/lib:/usr/lib
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/zzhang/cio/lib
 PATH=/fuse/bin:/fuse/usr/bin:$PATH
 

Modified: usertools/cio/bin/cnip.sh
===================================================================
--- usertools/cio/bin/cnip.sh	2009-04-04 22:54:25 UTC (rev 2823)
+++ usertools/cio/bin/cnip.sh	2009-04-05 02:09:57 UTC (rev 2824)
@@ -2,16 +2,29 @@
 LD_LIBRARY_PATH=/lib:/fuse/lib:/fuse/usr/lib
 PATH=/fuse/bin:/fuse/usr/bin:$PATH
 
-/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd.2409 &
+/home/iskra/ZeptoOS/packages/cnip/prebuilt/cn-ipfwd &
 
 while [ ! -f /tmp/ifconfig.cmd ]; do
     sleep 1
 done
 
 . /tmp/ifconfig.cmd
-
+ALLOCATION=$BG_SIZE
 RANK=`echo $CONTROL_INIT | awk -F, '{print $4}'`
+ITER=`expr $ALLOCATION / 128`
+if [ $ITER = "0" ]
+then
+    echo 10.128.0.0 >> /dev/shm/DHTlist
+fi
+for (( i=0; i<$ITER; i++))
+do
+  J=`expr $i / 2`
+  X=`expr $i \* 128`
+  K=`expr $X \% 256`
+  echo 10.128.$J.$K >> /dev/shm/DHTlist
+done
 
+
 if [ -d /dev/shm/share ];
     then
     rm -rf /dev/shm/share 
@@ -23,10 +36,10 @@
 
 /home/zzhang/chirp/bin/chirp_server -r /dev/shm/share &
 
-if [ "$RANK" = "0" ];
+if [ "$RANK" = "0" ] || [ "$RANK" = "64" ];
 then
     /home/zzhang/cio/bin/hashserver.rb &
-else    
+#else    
     /home/zzhang/falkon/worker/run.worker-c-bgp.sh $1 $2 $3 $4 $5 $6 $7
 fi
 

Modified: usertools/cio/bin/wrapper.sh
===================================================================
--- usertools/cio/bin/wrapper.sh	2009-04-04 22:54:25 UTC (rev 2823)
+++ usertools/cio/bin/wrapper.sh	2009-04-05 02:09:57 UTC (rev 2824)
@@ -1,3 +1,4 @@
+#!/fuse/bin/bash
 # this script must be invoked inside of bash, not plain sh
 PATH=/fuse/bin:/fuse/usr/bin:$PATH
 infosection() {
@@ -94,11 +95,12 @@
 	ln -s "/dev/shm/share/$FILENAME" "$DIR/$L"
     elif [ "$TYPE" == "_concurrent" ]; then
 	echo "cioinput(): toruscp for intermediate data $INPUT" >> /dev/shm/cio
-	mkdir -p $FILEPATH
-	echo "cioinput(): $DIR/$FILEPATH" >> /dev/shm/cio
-	/home/zzhang/cio/bin/toruscp.sh $FILENAME $DIR/$FILEPATH
+	echo DIR: `pwd` >> /dev/shm/torus
+	mkdir -p $DIR/$FILEPATH
+	echo "cioinput(): $INPUT" >> /dev/shm/cio
+	/home/zzhang/DHT/bin/DHTcp.sh $INPUT $DIR
     else
-	echo "cioinput(): copy from GPFS $INPUT" >> /dev/shm/cio
+	echo "cioinput(): copy from GPFS $INPUT pwd:`pwd` " >> /dev/shm/cio
 	cp "$PWD/shared/$L" "$DIR/$L"
     fi
 }
@@ -111,18 +113,79 @@
 
     if [ "$TYPE" == "_concurrent" ]; then
 	echo "ciooutput(): write intermediate data $OUTPUT" >> /dev/shm/cio
-	/home/zzhang/cio/bin/chtput.rb $FILENAME $RANK
-	cp $OUTPUT /dev/shm/share/
+	echo `pwd` >> /dev/shm/cio
+	/home/zzhang/DHT/bin/DHTregister.sh $OUTPUT $RANK
     else
 	echo "ciooutput(): write regular data $OUTPUT" >> /dev/shm/cio
 	#dd if="$OUTPUT" of="$WFDIR/shared/$OUTPUT" bs=128k
-	echo "$OUTPUT /chirp/multi/${CHIRP_ADD}@stripe/" >> /dev/shm/chirp_add
+	#echo "$OUTPUT /chirp/multi/${CHIRP_ADD}@stripe/" >> /dev/shm/chirp_add
 	cp "$OUTPUT" /chirp/multi/${CHIRP_ADD}@stripe/
     fi
 }
+
+extractfile()
+{
+    ARCHIVE="$1"
+    START_LOCK=$tmp/mtio-lock-started
+    END_LOCK=$tmp/mtio-lock-finished
+    current=`pwd`
+    cd $tmp
+    mkdir ${START_LOCK}
+
+    EXIT_CODE=$?
+    # EXIT_CODE=0 ###### FOR TESTING - forces creation of new dir
+
+    if [ "${EXIT_CODE}" -ne "0" ]; then
+        echo "waiting for data to be extracted"
+        ((i = 0))
+        while (( i == 0 ))
+          do
+          if [ -d "${END_LOCK}" ]; then
+              ((i = 1))
+          fi
+
+          if [ ! -d "${END_LOCK}" ]; then
+              /bin/sleep 1
+          fi
+        done
+    else
+        if [ ! -f "${ARCHIVE}" ]; then
+            echo "archive doesn't exist... exiting"
+            rmdir $START_LOCK
+            exit -2
+        fi
+
+        echo "extract the archive"
+        cd $tmp
+        echo "extracted" >> /dev/shm/extract
+        tar xf $ARCHIVE
+
+        EXIT_CODE=$?
+
+        if [ "${EXIT_CODE}" -ne "0" ]; then
+            echo "Error in untar of ${ARCHIVE} /... exit code ${EXIT_CODE}"
+            exit ${EXIT_CODE}
+            rmdir $START_LOCK
+        fi
+
+        mkdir -p ${END_LOCK}
+
+        EXIT_CODE=$?
+        if [ "${EXIT_CODE}" -ne "0" ]; then
+            echo "Error in mkdir ${END_LOCK}... exit code ${EXIT_CODE}"
+            rmdir $START_LOCK
+            exit ${EXIT_CODE}
+        fi
+    fi
+    cd $current
+}
+#/home/zzhang/bashtest/tar.sh
+tmp=/dev/shm/share
+extractfile common.tar
+cd $PWD
+
+RANK=`echo $CONTROL_INIT | awk -F, '{print $4}'`
 echo $@ >> /dev/shm/log
-RANK=`echo $CONTROL_INIT | awk -F, '{print $4}'`
-echo "RANK: $RANK" >> /dev/shm/log
 COMMANDLINE=$@
 WFDIR=$PWD
 ID=$1
@@ -216,7 +279,7 @@
 	else
 	    cioinput $L
 	        #cp "$PWD/shared/$L" "$DIR/$L"
-		checkError 254 "Failed to link input file $L"
+	    checkError 254 "Failed to link input file $L `ls -l $DIR/$L`"
 	fi
 done