[Swift-commit] r7132 - SwiftApps/Swift-MapRed/mapred_chirp
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Fri Oct 4 11:33:52 CDT 2013
Author: yadunandb
Date: 2013-10-04 11:33:52 -0500 (Fri, 04 Oct 2013)
New Revision: 7132
Modified:
SwiftApps/Swift-MapRed/mapred_chirp/beagle.xml
SwiftApps/Swift-MapRed/mapred_chirp/combiner.sh
SwiftApps/Swift-MapRed/mapred_chirp/parcombiner.sh
SwiftApps/Swift-MapRed/mapred_chirp/run.sh
SwiftApps/Swift-MapRed/mapred_chirp/teragen.kml
SwiftApps/Swift-MapRed/mapred_chirp/teragen.swift
SwiftApps/Swift-MapRed/mapred_chirp/teragen_wrap.sh
SwiftApps/Swift-MapRed/mapred_chirp/test
Log:
Minor updates
Modified: SwiftApps/Swift-MapRed/mapred_chirp/beagle.xml
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/beagle.xml 2013-10-04 16:32:50 UTC (rev 7131)
+++ SwiftApps/Swift-MapRed/mapred_chirp/beagle.xml 2013-10-04 16:33:52 UTC (rev 7132)
@@ -1,18 +1,23 @@
<config>
<pool handle="beagle">
- <execution provider="coaster" jobmanager="ssh-cl:pbs" url="login4.beagle.ci.uchicago.edu"/>
+ <execution provider="coaster" jobmanager="local:pbs"/>
<profile namespace="globus" key="jobsPerNode">24</profile>
<profile namespace="globus" key="lowOverAllocation">100</profile>
<profile namespace="globus" key="highOverAllocation">100</profile>
<profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24</profile>
+ <!-- to use a beage reservation, eg:
+ <profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24;pbs.resource_list=advres=wilde.1768</profile>
+ -->
<profile namespace="globus" key="maxtime">3600</profile>
- <profile namespace="globus" key="maxWalltime">00:10:00</profile>
+ <profile namespace="globus" key="maxWalltime">00:15:00</profile>
<profile namespace="globus" key="userHomeOverride">/lustre/beagle/yadunandb/swiftwork</profile>
- <profile namespace="globus" key="slots">20</profile>
+ <profile namespace="globus" key="slots">4</profile>
<profile namespace="globus" key="maxnodes">1</profile>
<profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="karajan" key="jobThrottle">1</profile>
+ <profile namespace="karajan" key="jobThrottle">4.80</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
+ <!-- <filesystem provider="local"/> -->
+ <!-- <workdirectory>/lustre/beagle/yadunandb/swiftwork</workdirectory> -->
<workdirectory>/tmp/yadunandb/swiftwork</workdirectory>
</pool>
-</config>
\ No newline at end of file
+</config>
Modified: SwiftApps/Swift-MapRed/mapred_chirp/combiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/combiner.sh 2013-10-04 16:32:50 UTC (rev 7131)
+++ SwiftApps/Swift-MapRed/mapred_chirp/combiner.sh 2013-10-04 16:33:52 UTC (rev 7132)
@@ -13,10 +13,10 @@
chirp_get $HOST $TARGET $TARGET.imd
RES=($(awk '{ sum += $1 } END { print sum,NR }' $TARGET.imd))
echo "$TARGET.imd"
-
SUM=$(($SUM+${RES[0]}))
COUNT=$(($COUNT+${RES[1]}))
done
echo "SUM : $SUM"
echo "COUNT: $COUNT"
+hostname -f
exit 0
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/mapred_chirp/parcombiner.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/parcombiner.sh 2013-10-04 16:32:50 UTC (rev 7131)
+++ SwiftApps/Swift-MapRed/mapred_chirp/parcombiner.sh 2013-10-04 16:33:52 UTC (rev 7132)
@@ -12,6 +12,7 @@
TARGET=$(basename $FILE)
chirp_get $HOST $TARGET $TARGET.imd
RES=($(awk '{ sum += $1 } END { print sum,NR }' $TARGET.imd))
+ echo "For $HOST:$FILE$ {RES[*]}"
echo "${RES[0]} ${RES[1]}" > "$2.$RANDOM.imd2"
}
@@ -28,28 +29,3 @@
echo "SUM : $SUM"
echo "COUNT: $COUNT"
exit 0
-
-
-
-
-
-exit 0
-
-
-
-for file_token in $*
-do
- TOKEN=($(cat $file_token))
- HOST=${TOKEN[0]}
- FILE=${TOKEN[1]}
- TARGET=$(basename $FILE)
- chirp_get $HOST $TARGET $TARGET.imd
- RES=($(awk '{ sum += $1 } END { print sum,NR }' $TARGET.imd))
- echo "$TARGET.imd"
-
- SUM=$(($SUM+${RES[0]}))
- COUNT=$(($COUNT+${RES[1]}))
-done
-echo "SUM : $SUM"
-echo "COUNT: $COUNT"
-exit 0
Modified: SwiftApps/Swift-MapRed/mapred_chirp/run.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/run.sh 2013-10-04 16:32:50 UTC (rev 7131)
+++ SwiftApps/Swift-MapRed/mapred_chirp/run.sh 2013-10-04 16:33:52 UTC (rev 7132)
@@ -5,12 +5,14 @@
#echo "Running run_catsn.swift"
#swift -tc.file tc.data -sites.file sites.xml run_catsn.swift
+APPS=$1; shift;
+SIZE=$1; shift;
if [ "$1" != "" ]
then
SITE="$1"
else
- SITE="anl"
+ SITE="beagle"
fi
#Call as check_error $? <ERR_CODE> <Error message>
@@ -31,20 +33,20 @@
RECSIZE=$2
ACTUAL_RESULT=($(tail -n 1 final_result))
EXPECTED_RESULT=$(($LOOPS * $RECSIZE * 10000 ))
- if [ "$EXPECTED_RESULT" == "${ACTUAL_RESULT[1]}" ]
+ if [ "$EXPECTED_RESULT" == "${ACTUAL_RESULT[1]}" ]
then
- echo "---------------------------------------------"
- echo "COUNT matches expected records"
- echo "ACTUAL COUNT : ${ACTUAL_RESULT[1]}"
- echo "EXPECTED COUNT: $EXPECTED_RESULT"
- echo "---------------------------------------------"
+ echo "---------------------------------------------"
+ echo "COUNT matches expected records"
+ echo "ACTUAL COUNT : ${ACTUAL_RESULT[1]}"
+ echo "EXPECTED COUNT: $EXPECTED_RESULT"
+ echo "---------------------------------------------"
else
- echo "---------------------------------------------"
- echo "COUNT does *NOT* match expected records"
- echo "ACTUAL COUNT : ${ACTUAL_RESULT[1]}"
- echo "EXPECTED COUNT: $EXPECTED_RESULT"
- cat err_file
- echo "---------------------------------------------"
+ echo "---------------------------------------------"
+ echo "COUNT does *NOT* match expected records"
+ echo "ACTUAL COUNT : ${ACTUAL_RESULT[1]}"
+ echo "EXPECTED COUNT: $EXPECTED_RESULT"
+ cat err_file
+ echo "---------------------------------------------"
fi
}
@@ -74,91 +76,5 @@
echo "================================================================"
}
-#run_swift_chirp 20 20;
-run_swift_chirp 10 100;
+run_swift_chirp 1000 10;
exit 0
-
-####################SWIFT + CHIRP VERSION############################
-LOOPS=10
-RECSIZE=100
-for LOOPS in `seq 10 10 50`
-do
- for RECSIZE in `seq 0 25 200`
- do
- [ $RECSIZE -eq 0 ] && RECSIZE=1
- run_swift_chirp $LOOPS $RECSIZE
- sleep 300;
- done
-done;
-exit 0
-####################PROVIDER STAGING VERSION######################
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-TOTALSIZE=1000000000 # Totalsize/chunksize is the
-CHUNKSIZE=100000000 # 10^8 records in each chunk
-NUMCHUNKS=$(($TOTALSIZE / $CHUNKSIZE))
-LOOP=1
-TIMEOUT=600
-time {
- echo "RUNTYPE :PROVIDER STAGING VERSION, SITE: $SITE, CHUNKS: $NUMCHUNKS"
- echo "timeout $TIMEOUT swift -tc.file tc.data -config cf -sites.file $SITE.xml teragen.swift -loops=$NUMCHUNKS"
- timeout $TIMEOUT swift -tc.file tc.data -config cf -sites.file $SITE.xml teragen.swift -loops=$NUMCHUNKS
- check_error $? 124 "Run terminated by timeout of $((TIMEOUT/60)) minute(s)"
-} | tee -a LAB_RECORDS
-exit 0
-
-
-#######################CDM VERSION##############################
-
-
-TOTALSIZE=1000000000 # Totalsize/chunksize is the
-CHUNKSIZE=100000000 # 10^8 records in each chunk: THIS IS A DEFAULT
-NUMCHUNKS=$(($TOTALSIZE / $CHUNKSIZE))
-LOOP=1
-TIMEOUT=600
-time {
- echo "RUNTYPE :CDM VERSION, SITE: $SITE, CHUNKS: $NUMCHUNKS"
- echo "timeout $TIMEOUT swift -cdm.file $SITE.cdm -tc.file tc.data -config cf -sites.file $SITE.xml teragen.swift -loops=$NUMCHUNKS"
- timeout $TIMEOUT swift -cdm.file $SITE.cdm -tc.file tc.data -config cf -sites.file $SITE.xml teragen.swift -loops=$NUMCHUNKS
- check_error $? 124 "Run terminated by timeout of $((TIMEOUT/60)) minute(s)"
-} | tee -a LAB_RECORDS
-exit 0
-
Modified: SwiftApps/Swift-MapRed/mapred_chirp/teragen.kml
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/teragen.kml 2013-10-04 16:32:50 UTC (rev 7131)
+++ SwiftApps/Swift-MapRed/mapred_chirp/teragen.kml 2013-10-04 16:33:52 UTC (rev 7132)
@@ -1,4 +1,4 @@
-<project><!-- CACHE ID 9d52f32c-4127-4505-8c1f-1383cafe01de-no-provenance -->
+<project><!-- CACHE ID 776e318b-73ca-4b57-9190-8b70016a4560-no-provenance -->
<import file="sys.xml"/>
<import file="scheduler.xml"/>
<import file="rlog.xml"/>
Modified: SwiftApps/Swift-MapRed/mapred_chirp/teragen.swift
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/teragen.swift 2013-10-04 16:32:50 UTC (rev 7131)
+++ SwiftApps/Swift-MapRed/mapred_chirp/teragen.swift 2013-10-04 16:33:52 UTC (rev 7132)
@@ -25,10 +25,10 @@
}
-#script combine <"combiner2.sh">;
-#script combine <"combiner.sh">;
+//script combine <"combiner2.sh">;
+//script combine <"combiner.sh">;
script combine <"parcombiner.sh">;
file final <"final_result">;
file errs <"err_file">;
-(final, errs) = comb_data(combine, tgen_out);
\ No newline at end of file
+(final, errs) = comb_data(combine, tgen_out);
Modified: SwiftApps/Swift-MapRed/mapred_chirp/teragen_wrap.sh
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/teragen_wrap.sh 2013-10-04 16:32:50 UTC (rev 7131)
+++ SwiftApps/Swift-MapRed/mapred_chirp/teragen_wrap.sh 2013-10-04 16:33:52 UTC (rev 7132)
@@ -1,6 +1,5 @@
#!/bin/bash
-
# By default with ARG1:100 and SLICESIZE=10000, this script will generate
# 10^6 records.
ARG1=1
@@ -8,13 +7,14 @@
FILE="input_$RANDOM.txt"
LOWERLIMIT=0
-UPPERLIMIT=1000000000 # 10^9
+UPPERLIMIT=10000000 # 10^8
SLICESIZE=10000 # 10^4 records padded to 100B would result in 1MB file
#SLICESIZE=1000 # 10^3 If padded to 100B would result
-FOLDER="/sandbox/$USER"
-TIMEOUT=600
+#FOLDER="/lustre/beagle/yadunand"
+FOLDER="/tmp/yadunandb"
+TIMEOUT=100
# clean_folder SLEEP_TIME FOLDER_NAME
clean_folder()
@@ -32,7 +32,8 @@
killall -u $USER chirp_server;
echo "unix:$USER rwlds" > $FOLDER/acl.conf
echo "hostname:* rwl" >> $FOLDER/acl.conf
- timeout $TIMEOUT chirp_server -A $FOLDER/acl.conf -r $FOLDER &
+# timeout $TIMEOUT chirp_server -A $FOLDER/acl.conf -r $FOLDER
+ timeout $TIMEOUT chirp_server -d -o $FOLDER/$FILE -A $FOLDER/acl.conf -r $FOLDER &
fi
FILE=$FOLDER/$FILE;
@@ -41,5 +42,7 @@
fi
shuf -i $LOWERLIMIT-$UPPERLIMIT -n $(($SLICESIZE*$ARG1)) | awk '{printf "%-99s\n", $0}' > $FILE
-echo "$HOSTNAME $FILE"
+
+HOST=$(hostname -f)
+echo "$HOST $FILE"
exit 0
\ No newline at end of file
Modified: SwiftApps/Swift-MapRed/mapred_chirp/test
===================================================================
--- SwiftApps/Swift-MapRed/mapred_chirp/test 2013-10-04 16:32:50 UTC (rev 7131)
+++ SwiftApps/Swift-MapRed/mapred_chirp/test 2013-10-04 16:33:52 UTC (rev 7132)
@@ -1,5 +1 @@
-stomp /sandbox/yadunand/input_32754.txt
-thwomp /sandbox/yadunand/input_15959.txt
-stomp /sandbox/yadunand/input_21602.txt
-vanquish /sandbox/yadunand/input_18357.txt
-thrash /sandbox/yadunand/input_27031.txt
+login5.beagle.ci.uchicago.edu /home/yadunandb/Swift-MapRed/mapred_chirp/input_28579.txt
More information about the Swift-commit
mailing list