[Swift-commit] r6896 - branches/release-0.94/tests/multi_remote
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Tue Aug 20 11:06:51 CDT 2013
Author: yadunandb
Date: 2013-08-20 11:06:51 -0500 (Tue, 20 Aug 2013)
New Revision: 6896
Added:
branches/release-0.94/tests/multi_remote/remote_driver.check.sh
Modified:
branches/release-0.94/tests/multi_remote/README
branches/release-0.94/tests/multi_remote/remote_driver.setup.sh
branches/release-0.94/tests/multi_remote/remote_driver.swift
branches/release-0.94/tests/multi_remote/remote_driver.timeout
branches/release-0.94/tests/multi_remote/sites.template.xml
branches/release-0.94/tests/multi_remote/swift.properties
branches/release-0.94/tests/multi_remote/tc.template.data
Log:
Committing latest changes to multi_remote
Modified: branches/release-0.94/tests/multi_remote/README
===================================================================
--- branches/release-0.94/tests/multi_remote/README 2013-08-20 15:51:22 UTC (rev 6895)
+++ branches/release-0.94/tests/multi_remote/README 2013-08-20 16:06:51 UTC (rev 6896)
@@ -1,5 +1,3 @@
-
-
What
====
Added: branches/release-0.94/tests/multi_remote/remote_driver.check.sh
===================================================================
--- branches/release-0.94/tests/multi_remote/remote_driver.check.sh (rev 0)
+++ branches/release-0.94/tests/multi_remote/remote_driver.check.sh 2013-08-20 16:06:51 UTC (rev 6896)
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+if [ -d "swift" ]
+then
+ rm -rf swift swift.tar
+fi
+
+SITES_PASSED=`ls out/*out | wc -l`
+HOME=$PWD
+MAIL="$HOME/Mail.content"
+
+cd out;
+echo "Test-Results" > $MAIL
+echo "=====================================">> $MAIL
+echo "Test home : $PWD" >> $MAIL
+echo "Sites successful : $SITES_PASSED" >> $MAIL
+VERSION=`swift -version | cut -d " " -f -2 | sed 's/\ /-/g' | tr '[A-Z]' '[a-z]'`
+for i in `ls *tar`
+do
+ scp $i ci:~/public_html/$VERSION/
+ tar -xf $i
+ folder=`tar -tf $i | head -n 1`
+ echo "Results for site : $folder" >> $MAIL
+ grep "Tests run" $folder/run*/tests*html >> $MAIL
+ grep "Tests succeeded" $folder/run*/tests*html >> $MAIL
+ grep "Tests failed" $folder/run*/tests*html >> $MAIL
+ grep "Total Time" $folder/run*/tests*html >> $MAIL
+ echo ""
+ rm -rf $folder
+done
+cd $HOME;
+cd ..;
+RUN_DIR=$(basename $PWD);
+cd ..;
+mv $RUN_DIR "$RUN_DIR-local"
+tar -cf Local.tar "$RUN_DIR-local"
+scp Local.tar ci:~/public_html/$VERSION/
+
+echo "Running update and maintenance script on publish server";
+ssh ci "cd ~/public_html/$VERSION/; ./maint.sh"
+mv "$RUN_DIR-local" $RUN_DIR
+rm Local.tar
+
+echo "=====================================">> $MAIL
+cd $HOME;
+cat $MAIL
+mailx -s "Test results from $HOSTNAME" -r $FROM_MAIL $TO_MAIL < $MAIL
+
Property changes on: branches/release-0.94/tests/multi_remote/remote_driver.check.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: branches/release-0.94/tests/multi_remote/remote_driver.setup.sh
===================================================================
--- branches/release-0.94/tests/multi_remote/remote_driver.setup.sh 2013-08-20 15:51:22 UTC (rev 6895)
+++ branches/release-0.94/tests/multi_remote/remote_driver.setup.sh 2013-08-20 16:06:51 UTC (rev 6896)
@@ -3,70 +3,80 @@
#CLEAN_CHECKOUT="yes"
CLEAN_CHECKOUT="no"
-[ ! -z $COG_URL ] || COG_URL=https://cogkit.svn.sourceforge.net/svnroot/cogkit/branches/4.1.10/src/cog
-[ ! -z $SWIFT_URL ] || SWIFT_URL=https://svn.ci.uchicago.edu/svn/vdl2/branches/release-0.94
+#[ ! -z $COG_URL ] || COG_URL=https://cogkit.svn.sourceforge.net/svnroot/cogkit/branches/4.1.10/src/cog
+#[ ! -z $SWIFT_URL ] || SWIFT_URL=https://svn.ci.uchicago.edu/svn/vdl2/branches/release-0.94
+[ ! -z $COG_URL ] || COG_URL=https://svn.code.sf.net/p/cogkit/svn/branches/4.1.10/src/cog
+[ ! -z $SWIFT_URL ] || SWIFT_URL=https://svn.ci.uchicago.edu/svn/vdl2/branches/release-0.94
[ ! -z $SWIFT_VERSION ] || SWIFT_VERSION=0.94
[ ! -z $BEAGLE_USERNAME ] || BEAGLE_USERNAME="yadunandb"
[ ! -z $MIDWAY_USERNAME ] || MIDWAY_USERNAME="yadunand"
[ ! -z $UC3_USERNAME ] || UC3_USERNAME="yadunand"
[ ! -z $MCS_USERNAME ] || MCS_USERNAME="yadunand"
+[ ! -z $PUBLISH_FOLDER ] || PUBLISH_FOLDER="\/home\/yadunandb\/public_html\/results"
+[ ! -z $SWIFT_SOURCE ] || SWIFT_SOURCE="/home/yadunand/swift"
+[ ! -z $RUN_TYPE ] || RUN_TYPE="daily"
-
-export GLOBUS_HOSTNAME="swift.rcc.uchicago.edu"
-
-
SITES="sites.xml"
-
cp $SITES $SITES.bak
cat $SITES | sed "s/BEAGLE_USERNAME/$BEAGLE_USERNAME/g" > tmp && mv tmp $SITES
cat $SITES | sed "s/MIDWAY_USERNAME/$MIDWAY_USERNAME/g" > tmp && mv tmp $SITES
cat $SITES | sed "s/UC3_USERNAME/$UC3_USERNAME/g" > tmp && mv tmp $SITES
cat $SITES | sed "s/MCS_USERNAME/$MCS_USERNAME/g" > tmp && mv tmp $SITES
+cat $SITES | sed "s/PUBLISH_FOLDER/$PUBLISH_FOLDER/g" > tmp && mv tmp $SITES
+
+export GLOBUS_HOSTNAME="swift.rcc.uchicago.edu"
+
BASE=$PWD
-# Make clean checkout if no swift dir is present or
-# Clean checkout requested
-cp /home/yadunand/swift/cog/modules/swift/tests/multi_remote/swift.tar ./
-if [ -f "swift.tar" ]
-then
- echo "Found swift.tar. Extracting.."
- tar -xf swift.tar
-fi
-
-if [ "CLEAN_CHECKOUT" == "yes" ] || [ ! -d "swift" ]
-then
- echo "Cleaning and making fresh checkout"
- rm -rf swift &> /dev/null
- mkdir swift && cd swift
- svn co $COG_URL
- cd cog/modules
- svn co $SWIFT_URL swift
- cd swift
-else
- echo "Updating Cog sources"
- cd swift/
- svn up *
- echo "Updating Swift sources"
- cd cog/modules/swift
- svn up *
-fi
+# Make clean checkout if no swift dir is present or
+# Clean checkout requested
-echo "$PWD : Starting compile"
-ant redist | tee $BASE/compile.log
-if [ "$?" != "0" ]
-then
- echo "Swift compile failed. Cannot proceed"
- exit 1
-fi
+[ -f "/scratch/midway/yadunand/swift.tar" ] && cp /scratch/midway/yadunand/swift.tar ./
-cd $BASE
-if [ -d "swift" ]
+if [ "$REMOTE_DRIVER_FASTSETUP" == "true" ]
then
- tar -cf swift.tar.tmp ./swift && mv swift.tar.tmp swift.tar && echo "Tarred successfully"
+ echo "FASTSETUP: Skipping svn update and rebuild"
else
- echo "Could not find swift folder to tar"
-fi;
+ if [ -f "swift.tar" ]
+ then
+ echo "Found swift.tar. Extracting.."
+ tar -xf swift.tar
+ fi
+
+ if [ "CLEAN_CHECKOUT" == "yes" ] || [ ! -d "swift" ]
+ then
+ echo "Cleaning and making fresh checkout"
+ rm -rf swift &> /dev/null
+ mkdir swift && cd swift
+ svn co $COG_URL
+ cd cog/modules
+ svn co $SWIFT_URL swift
+ cd swift
+ else
+ echo "Updating Cog sources"
+ cd swift/
+ svn up *
+ echo "Updating Swift sources"
+ cd cog/modules/swift
+ svn up *
+ fi
+
+ echo "$PWD : Starting compile"
+ ant redist | tee $BASE/compile.log
+ if [ "$?" != "0" ]
+ then
+ echo "Swift compile failed. Cannot proceed"
+ exit 1
+ fi
+ cd $BASE
+ if [ -d "swift" ]
+ then
+ tar -cf swift.tar.tmp ./swift && mv swift.tar.tmp swift.tar && echo "Tarred successfully"
+ else
+ echo "Could not find swift folder to tar"
+ fi;
+fi
# Wrapper is the script that gets executed on the remote nodes
# The outputs go to the out directory
@@ -77,7 +87,7 @@
SWIFT_TARBALL=$1
LOG_TARBALL=$2
RUN_HOME=$PWD
-echo "Wrapper running on : $HOSTNAME "
+echo "HOSTNAME: $HOSTNAME"
if [ -f $SWIFT_TARBALL ]
then
@@ -129,15 +139,63 @@
fi;
cd tests/
-#./suite.sh -l 1 -t $PWD/groups/group-stress-heavy.sh 2>&1 | tee $RUN_HOME/TEST.log
+
+EOF
+
+
+if [ "$RUN_TYPE" == "daily" ]; then
+cat <<'EOF' >> wrapper.sh
+#./suite.sh -l 1 -t $PWD/groups/group-functions.sh 2>&1 | tee $RUN_HOME/TEST.log
+#./suite.sh -l 1 -t $PWD/groups/group-remote-site.sh 2>&1 | tee $RUN_HOME/TEST.log
+./suite.sh -l 1 -t $PWD/groups/group-all-local.sh 2>&1 | tee $RUN_HOME/TEST.log
+EOF
+elif [ "$RUN_TYPE" == "weekly" ]; then
+cat <<'EOF' >> wrapper.sh
+#./suite.sh -l 1 -t $PWD/groups/group-functions.sh 2>&1 | tee $RUN_HOME/TEST.log
+#./suite.sh -l 1 -t $PWD/groups/group-remote-site.sh 2>&1 | tee $RUN_HOME/TEST.log
+./suite.sh -l 1 -t $PWD/groups/group-all-local.sh 2>&1 | tee $RUN_HOME/TEST.log
+EOF
+elif [ "$RUN_TYPE" == "manual" ]; then
+cat <<'EOF' >> wrapper.sh
+#./suite.sh -l 1 -t $PWD/groups/group-functions.sh 2>&1 | tee $RUN_HOME/TEST.log
+#./suite.sh -l 1 -t $PWD/groups/group-remote-site.sh 2>&1 | tee $RUN_HOME/TEST.log
+./suite.sh -l 1 -t $PWD/groups/group-all-local.sh 2>&1 | tee $RUN_HOME/TEST.log
+EOF
+elif [ "$RUN_TYPE" == "test" ]; then
+cat <<'EOF' >> wrapper.sh
./suite.sh -l 1 -t $PWD/groups/group-functions.sh 2>&1 | tee $RUN_HOME/TEST.log
-#./suite.sh -l 1 -t $PWD/groups/group-all-local.sh 2>&1 | tee $RUN_HOME/TEST.log
+EOF
+fi
+cat <<'EOF' >> wrapper.sh
+
cd $RUN_HOME
-tar -cvf $LOG_TARBALL *log $BASENAME/run*/tests-*{html,log}
-rm $BASENAME* -rf
+echo "Current dir : $PWD"
+rm -rf $BASENAME/cog
+
+RUNDIR=`ls $BASENAME | grep run`
+
+cp *log $BASENAME/$RUNDIR/
+
+mv $BASENAME/$RUNDIR "$RUNDIR-$HOSTNAME"
+tar -cvf $LOG_TARBALL "$RUNDIR-$HOSTNAME" &> /dev/null
+rm -rf $BASENAME* "$RUNDIR-$HOSTNAME" &> /dev/null
echo "Folder cleaned"
-ls -lah
+
+PID=$$
+CPIDS=$(pgrep -P $PID); (sleep 20 && kill -KILL $CPIDS &); kill -TERM $CPIDS;
+exit 0
+
EOF
+
+cat<<'EOF' > publish.sh
+#!/bin/bash
+echo "Hostname : $HOSTNAME"
+echo "PWD : $PWD"
+ls
+
+EOF
+
+echo "$0 : Completed"
exit 0
Modified: branches/release-0.94/tests/multi_remote/remote_driver.swift
===================================================================
--- branches/release-0.94/tests/multi_remote/remote_driver.swift 2013-08-20 15:51:22 UTC (rev 6895)
+++ branches/release-0.94/tests/multi_remote/remote_driver.swift 2013-08-20 16:06:51 UTC (rev 6896)
@@ -3,6 +3,7 @@
file swift_package <"swift.tar">;
script wrapper <"wrapper.sh">;
+script publish <"publish.sh">;
file out[] <simple_mapper;prefix="out/driver", suffix=".out">;
file err[] <simple_mapper;prefix="out/driver", suffix=".err">;
@@ -25,9 +26,9 @@
mid @run @filename(tar) @log stdout=@filename(out) stderr=@filename(err);
}
-app (file out, file err, file log) remote_crush (script run, file tar)
+app (file out, file err, file log) remote_mcs (script run, file tar)
{
- csh @run @filename(tar) @log stdout=@filename(out) stderr=@filename(err);
+ mcs @run @filename(tar) @log stdout=@filename(out) stderr=@filename(err);
}
app (file out, file err, file log) remote_frisbee (script run, file tar)
@@ -35,6 +36,12 @@
fsb @run @filename(tar) @log stdout=@filename(out) stderr=@filename(err);
}
+app (file out, file err) publish_results (script publish, file tar, file out, file err)
+{
+ pub @publish stdout=@out stderr=@err;
+}
+
+
tracef("Filename of the wraper : %s \n", @filename(wrapper));
tracef("Filename of the package : %s \n", @filename(swift_package));
@@ -45,19 +52,19 @@
{
case 0:
tracef("Calling uc3\n");
- (out[i], err[i], log[i]) = remote_uc3 (wrapper, swift_package);
+ //(out[i], err[i], log[i]) = remote_uc3 (wrapper, swift_package);
case 1:
tracef("Calling beagle \n");
- (out[i], err[i], log[i]) = remote_beagle (wrapper, swift_package);
+ //(out[i], err[i], log[i]) = remote_beagle (wrapper, swift_package);
case 2:
tracef("Calling Midway\n");
- (out[i], err[i], log[i]) = remote_midway (wrapper, swift_package);
+ //(out[i], err[i], log[i]) = remote_midway (wrapper, swift_package);
case 3:
- tracef("Skipping Crush on MCS\n");
-// (out[i], err[i], log[i]) = remote_crush(wrapper, swift_package);
+ tracef("Calling thwomp on MCS\n");
+ (out[i], err[i], log[i]) = remote_mcs (wrapper, swift_package);
case 4:
tracef("Skipping Frisbee on MCS (Will fail due to BUG:1030)\n");
-// (out[i], err[i], log[i]) = remote_frisbee(wrapper, swift_package);
+ //(out[i], err[i], log[i]) = remote_frisbee(wrapper, swift_package);
default:
tracef("Fail: Unknown site %s : %i \n", site, i);
}
Modified: branches/release-0.94/tests/multi_remote/remote_driver.timeout
===================================================================
--- branches/release-0.94/tests/multi_remote/remote_driver.timeout 2013-08-20 15:51:22 UTC (rev 6895)
+++ branches/release-0.94/tests/multi_remote/remote_driver.timeout 2013-08-20 16:06:51 UTC (rev 6896)
@@ -1 +1 @@
-7200
\ No newline at end of file
+3600
\ No newline at end of file
Modified: branches/release-0.94/tests/multi_remote/sites.template.xml
===================================================================
--- branches/release-0.94/tests/multi_remote/sites.template.xml 2013-08-20 15:51:22 UTC (rev 6895)
+++ branches/release-0.94/tests/multi_remote/sites.template.xml 2013-08-20 16:06:51 UTC (rev 6896)
@@ -15,6 +15,20 @@
<workdirectory>.</workdirectory>
</pool>
+ <pool handle="ci">
+ <execution provider="coaster" jobmanager="ssh-cl:local" url="login.ci.uchicago.edu"/>
+ <profile namespace="globus" key="jobsPerNode">24</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="maxtime">120</profile>
+ <profile namespace="globus" key="maxWalltime">00:02:00</profile>
+ <profile namespace="globus" key="slots">5</profile>
+ <profile namespace="globus" key="maxnodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">4.80</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>PUBLISH_FOLDER</workdirectory>
+ </pool>
<pool handle="beagle">
<execution provider="coaster" jobmanager="ssh-cl:local" url="login4.beagle.ci.uchicago.edu"/>
@@ -68,15 +82,16 @@
<workdirectory>/home/UC3_USERNAME/swiftwork</workdirectory>
</pool>
- <pool handle="crush">
- <execution provider="coaster" jobmanager="ssh-cl:local" url="crush.mcs.anl.gov"/>
+ <pool handle="mcs">
+ <execution provider="coaster" jobmanager="ssh-cl:local" url="thwomp.mcs.anl.gov"/>
<profile namespace="globus" key="jobsPerNode">8</profile>
<profile namespace="globus" key="lowOverAllocation">100</profile>
<profile namespace="globus" key="highOverAllocation">100</profile>
- <profile namespace="globus" key="maxtime">3600</profile>
- <profile namespace="globus" key="maxWalltime">00:05:00</profile>
+ <profile namespace="globus" key="maxtime">36000</profile>
+ <profile namespace="globus" key="maxWalltime">01:05:00</profile>
<profile namespace="karajan" key="jobThrottle">0.0799</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="userHomeOverride">/sandbox/MCS_USERNAME/</profile>
<workdirectory>/sandbox/MCS_USERNAME/swiftwork</workdirectory>
</pool>
Modified: branches/release-0.94/tests/multi_remote/swift.properties
===================================================================
--- branches/release-0.94/tests/multi_remote/swift.properties 2013-08-20 15:51:22 UTC (rev 6895)
+++ branches/release-0.94/tests/multi_remote/swift.properties 2013-08-20 16:06:51 UTC (rev 6896)
@@ -5,4 +5,5 @@
lazy.errors=true
provider.staging.pin.swiftfiles=false
sitedir.keep=true
-tcp.port.range=50000,51000
\ No newline at end of file
+tcp.port.range=50000,51000
+status.mode=provider
\ No newline at end of file
Modified: branches/release-0.94/tests/multi_remote/tc.template.data
===================================================================
--- branches/release-0.94/tests/multi_remote/tc.template.data 2013-08-20 15:51:22 UTC (rev 6895)
+++ branches/release-0.94/tests/multi_remote/tc.template.data 2013-08-20 16:06:51 UTC (rev 6896)
@@ -1,6 +1,6 @@
beagle bgl /bin/bash null null null
uc3 uc3 /bin/bash null null null
midway mid /bin/bash null null null
-frisbee fsb /bin/bash null null null
-crush csh /bin/bash null null null
+mcs mcs /bin/bash null null null
local l_bash /bin/bash null null null
+ci pub /bin/bash null null null
\ No newline at end of file
More information about the Swift-commit
mailing list