[Swift-commit] r7906 - in branches/release-0.95/tests: . groups multi_remote sites/uc3 stress stress/IO/beagle stress/IO/multiple stress/IO/uc3 stress/apps/modis_uc3 stress/internals stress/jobs_per_node stress/random_fail
yadunandb at ci.uchicago.edu
yadunandb at ci.uchicago.edu
Mon Jun 9 12:03:05 CDT 2014
Author: yadunandb
Date: 2014-06-09 12:03:04 -0500 (Mon, 09 Jun 2014)
New Revision: 7906
Added:
branches/release-0.95/tests/stress/jobs_per_node/
branches/release-0.95/tests/stress/jobs_per_node/combiner.sh
branches/release-0.95/tests/stress/jobs_per_node/count_jobs.sh
branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.args
branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.check.sh
branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.setup.sh
branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.stdout
branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.swift
branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.timeout
branches/release-0.95/tests/stress/jobs_per_node/run
branches/release-0.95/tests/stress/jobs_per_node/sites.template.xml
branches/release-0.95/tests/stress/jobs_per_node/swift.log
branches/release-0.95/tests/stress/jobs_per_node/swift.properties
branches/release-0.95/tests/stress/jobs_per_node/tc.data
branches/release-0.95/tests/stress/jobs_per_node/teragen_wrap.sh
branches/release-0.95/tests/stress/jobs_per_node/title.txt
branches/release-0.95/tests/stress/random_fail/
branches/release-0.95/tests/stress/random_fail/Bug_info
branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.check.sh
branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.setup.sh
branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.swift
branches/release-0.95/tests/stress/random_fail/run
branches/release-0.95/tests/stress/random_fail/sites.template.xml
branches/release-0.95/tests/stress/random_fail/swift.properties
branches/release-0.95/tests/stress/random_fail/tc.template.data
branches/release-0.95/tests/stress/random_fail/title.txt
Modified:
branches/release-0.95/tests/groups/group-daily-remote.sh
branches/release-0.95/tests/multi_remote/remote_driver.check.sh
branches/release-0.95/tests/multi_remote/remote_driver.setup.sh
branches/release-0.95/tests/sites/uc3/sites.template.xml
branches/release-0.95/tests/stress/IO/beagle/sites.template.xml
branches/release-0.95/tests/stress/IO/multiple/sites.template.xml
branches/release-0.95/tests/stress/IO/uc3/sites.template.xml
branches/release-0.95/tests/stress/apps/modis_uc3/sites.template.xml
branches/release-0.95/tests/stress/internals/tc.template.data
branches/release-0.95/tests/user_test.sh
branches/release-0.95/tests/user_yadu.sh
Log:
Updated sites.xml for UC3.
Several old updates
Modified: branches/release-0.95/tests/groups/group-daily-remote.sh
===================================================================
--- branches/release-0.95/tests/groups/group-daily-remote.sh 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/groups/group-daily-remote.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -33,7 +33,7 @@
$TESTDIR/sites/mac-frisbee \
$TESTDIR/sites/blues \
$TESTDIR/sites/fusion \
- $TESTDIR/sites/raven \
+ #$TESTDIR/sites/raven \
# Remote-cluster IO tests
$TESTDIR/stress/IO/beagle \
@@ -51,8 +51,6 @@
$TESTDIR/stress/apps/modis_uc3 \
# $TESTDIR/stress/apps/modis_multiple\
- # Local stress tests
- $TESTDIR/stress/internals \
# Local cluster tests.
$TESTDIR/stress/local_cluster \
$TESTDIR/stress/random_fail \
Modified: branches/release-0.95/tests/multi_remote/remote_driver.check.sh
===================================================================
--- branches/release-0.95/tests/multi_remote/remote_driver.check.sh 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/multi_remote/remote_driver.check.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -15,8 +15,8 @@
SITES_PASSED=`ls out/*out | wc -l`
HOME=$PWD
MAIL="$HOME/Mail.content"
-#VERSION=`grep "^Swift.*swift-.*cog.*" remote_driver.stdout | cut -d " " -f -2 | sed 's/\ /-/g' | tr '[A-Z]' '[a-z]'`
-VERSION="swift-trunk"
+VERSION=`grep "^Swift.*swift-.*cog.*" remote_driver.stdout | cut -d " " -f -2 | sed 's/\ /-/g' | tr '[A-Z]' '[a-z]'`
+#VERSION="swift-trunk"
cd out;
echo "Test-Results" > $MAIL
Modified: branches/release-0.95/tests/multi_remote/remote_driver.setup.sh
===================================================================
--- branches/release-0.95/tests/multi_remote/remote_driver.setup.sh 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/multi_remote/remote_driver.setup.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -1,8 +1,8 @@
#!/bin/bash
-[ ! -z $COG_URL ] || COG_URL=https://svn.code.sf.net/p/cogkit/svn/trunk/src/cog
-[ ! -z $SWIFT_URL ] || SWIFT_URL=https://svn.ci.uchicago.edu/svn/vdl2/trunk
-[ ! -z $SWIFT_VERSION ] || SWIFT_VERSION=trunk
+[ ! -z $COG_URL ] || COG_URL=https://svn.code.sf.net/p/cogkit/svn/branches/4.1.11/src/cog
+[ ! -z $SWIFT_URL ] || SWIFT_URL=https://svn.ci.uchicago.edu/svn/vdl2/branches/release-0.95
+[ ! -z $SWIFT_VERSION ] || SWIFT_VERSION=0.95
[ ! -z $BEAGLE_USERNAME ] || BEAGLE_USERNAME="yadunandb"
[ ! -z $MIDWAY_USERNAME ] || MIDWAY_USERNAME="yadunand"
[ ! -z $UC3_USERNAME ] || UC3_USERNAME="yadunand"
Modified: branches/release-0.95/tests/sites/uc3/sites.template.xml
===================================================================
--- branches/release-0.95/tests/sites/uc3/sites.template.xml 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/sites/uc3/sites.template.xml 2014-06-09 17:03:04 UTC (rev 7906)
@@ -2,7 +2,7 @@
<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
<pool handle="uc3">
- <execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
+ <execution provider="coaster" url="login01.osgconnect.net" jobmanager="ssh-cl:condor"/>
<profile namespace="karajan" key="jobThrottle">10.00</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<profile namespace="globus" key="jobsPerNode">1</profile>
@@ -13,7 +13,7 @@
<profile namespace="globus" key="slots">1</profile>
<profile namespace="globus" key="maxNodes">1</profile>
<profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="jobType">nonshared</profile>
- <workdirectory>.</workdirectory>
+ <profile namespace="globus" key="condor.+ProjectName">"Swift"</profile>
+ <workdirectory>/tmp/swiftwork</workdirectory>
</pool>
-</config>
\ No newline at end of file
+</config>
Modified: branches/release-0.95/tests/stress/IO/beagle/sites.template.xml
===================================================================
--- branches/release-0.95/tests/stress/IO/beagle/sites.template.xml 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/stress/IO/beagle/sites.template.xml 2014-06-09 17:03:04 UTC (rev 7906)
@@ -8,10 +8,10 @@
<profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24</profile>
<profile namespace="globus" key="maxtime">3700</profile>
<profile namespace="globus" key="maxWalltime">00:25:00</profile>
- <profile namespace="globus" key="queue">development</profile>
+ <!-- <profile namespace="globus" key="queue">development</profile> -->
<profile namespace="globus" key="userHomeOverride">/lustre/beagle/{env.USER}/swiftwork</profile>
- <profile namespace="globus" key="slots">20</profile>
- <profile namespace="globus" key="maxnodes">10</profile>
+ <profile namespace="globus" key="slots">2</profile>
+ <profile namespace="globus" key="maxnodes">4</profile>
<profile namespace="globus" key="nodeGranularity">1</profile>
<profile namespace="karajan" key="jobThrottle">4.80</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
@@ -19,4 +19,4 @@
<!-- <workdirectory>/lustre/beagle/yadunandb/swiftwork</workdirectory> -->
<workdirectory>/tmp/{env.USER}/swiftwork</workdirectory>
</pool>
-</config>
\ No newline at end of file
+</config>
Modified: branches/release-0.95/tests/stress/IO/multiple/sites.template.xml
===================================================================
--- branches/release-0.95/tests/stress/IO/multiple/sites.template.xml 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/stress/IO/multiple/sites.template.xml 2014-06-09 17:03:04 UTC (rev 7906)
@@ -1,5 +1,5 @@
-<config>
-
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
<pool handle="uc3">
<execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
<profile namespace="karajan" key="jobThrottle">10.00</profile>
@@ -24,7 +24,6 @@
<profile namespace="globus" key="lowOverAllocation">100</profile>
<profile namespace="globus" key="highOverAllocation">100</profile>
<!-- <profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24</profile> -->
- <profile namespace="globus" key="providerAttributes">pbs.aprun;pbs.mpp;depth=24;pbs.resource_list=advres=wilde.1768</profile>
<profile namespace="globus" key="maxtime">3600</profile>
<profile namespace="globus" key="maxWalltime">00:05:00</profile>
<profile namespace="globus" key="userHomeOverride">/lustre/beagle/{beagle.USER}/swiftwork</profile>
Modified: branches/release-0.95/tests/stress/IO/uc3/sites.template.xml
===================================================================
--- branches/release-0.95/tests/stress/IO/uc3/sites.template.xml 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/stress/IO/uc3/sites.template.xml 2014-06-09 17:03:04 UTC (rev 7906)
@@ -1,17 +1,21 @@
-<config>
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+
<pool handle="uc3">
<execution provider="coaster" url="uc3-sub.uchicago.edu" jobmanager="ssh-cl:condor"/>
<profile namespace="karajan" key="jobThrottle">10.00</profile>
<profile namespace="karajan" key="initialScore">10000</profile>
<profile namespace="globus" key="jobsPerNode">1</profile>
<profile namespace="globus" key="maxtime">3600</profile>
- <profile namespace="globus" key="maxWalltime">00:30:00</profile>
+ <profile namespace="globus" key="maxWalltime">00:10:00</profile>
<profile namespace="globus" key="highOverAllocation">100</profile>
<profile namespace="globus" key="lowOverAllocation">100</profile>
- <profile namespace="globus" key="slots">1000</profile>
+ <profile namespace="globus" key="slots">2</profile>
<profile namespace="globus" key="maxNodes">1</profile>
<profile namespace="globus" key="nodeGranularity">1</profile>
<profile namespace="globus" key="jobType">nonshared</profile>
+ <profile namespace="globus" key="condor.+ProjectName">"Swift"</profile>
<workdirectory>.</workdirectory>
</pool>
-</config>
\ No newline at end of file
+
+</config>
Modified: branches/release-0.95/tests/stress/apps/modis_uc3/sites.template.xml
===================================================================
--- branches/release-0.95/tests/stress/apps/modis_uc3/sites.template.xml 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/stress/apps/modis_uc3/sites.template.xml 2014-06-09 17:03:04 UTC (rev 7906)
@@ -14,6 +14,7 @@
<profile namespace="globus" key="maxNodes">1</profile>
<profile namespace="globus" key="nodeGranularity">1</profile>
<profile namespace="globus" key="condor.+AccountingGroup">"group_friends.{env.USER}"</profile>
+ <profile namespace="globus" key="condor.+ProjectName">"Swift"</profile>
<profile namespace="globus" key="jobType">nonshared</profile>
<workdirectory>.</workdirectory>
</pool>
Modified: branches/release-0.95/tests/stress/internals/tc.template.data
===================================================================
--- branches/release-0.95/tests/stress/internals/tc.template.data 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/stress/internals/tc.template.data 2014-06-09 17:03:04 UTC (rev 7906)
@@ -1,2 +1,2 @@
-local bash /bin/bash null null null
-local sleep /bin/sleep null null null
+localhost bash /bin/bash null null null
+localhost sleep /bin/sleep null null null
Added: branches/release-0.95/tests/stress/jobs_per_node/combiner.sh
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/combiner.sh (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/combiner.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+FILES=$*
+SUM=0
+COUNT=0
+
+for file in $*
+do
+ RES=($(awk '{ sum += $1 } END { print sum,NR }' $file))
+ echo "${RES[0]} ${RES[1]}"
+ SUM=$(($SUM+${RES[0]}))
+ COUNT=$(($COUNT+${RES[1]}))
+done
+echo "SUM : $SUM"
+echo "COUNT: $COUNT"
+exit 0
Added: branches/release-0.95/tests/stress/jobs_per_node/count_jobs.sh
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/count_jobs.sh (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/count_jobs.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+SELF="count_jobs.sh"
+SLEEPTIME=60
+
+
+BEFORE=$(($RANDOM%$SLEEPTIME))
+AFTER=$(($SLEEPTIME-$BEFORE))
+
+sleep $BEFORE
+echo "NODE $(hostname -f)"
+ACTIVE=`ps -u $USER | grep $SELF | wc -l`
+echo "ps -u $USER | grep | wc -l"
+echo "ACTIVE $ACTIVE"
+echo "SPLIT $BEFORE:$AFTER"
+sleep $AFTER
\ No newline at end of file
Property changes on: branches/release-0.95/tests/stress/jobs_per_node/count_jobs.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.args
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.args (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.args 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1 @@
+-loops=10
Added: branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.check.sh
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.check.sh (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.check.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+BASE=${0%.check.sh}
+ARGS=`cat $BASE.args | sed 's/-loops=//'`
+
+exit 0
+EXPECTED=$(($ARGS * 10000))
+
+if [ -f "final_result" ];then
+ RESULT=($(tail -n 1 final_result))
+ echo "RESULT line : ${RESULT[*]}"
+ echo "EXPECTED = $EXPECTED"
+ echo "ACTUAL = ${RESULT[1]}"
+fi
+
+if [[ "${RESULT[1]}" == "$EXPECTED" ]]
+then
+ echo "Result matched"
+else
+ echo "Result does not match expectation" >&2
+ exit 1
+fi
Property changes on: branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.check.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.setup.sh
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.setup.sh (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.setup.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,100 @@
+#!/bin/bash
+
+jobspernode=$(( $RANDOM % 7 + 1 ))
+LOOPS=$(( $(($RANDOM%4 + 1)) * 2 ))
+
+HOST=$(hostname -f)
+
+ARGS_FILE=${0%.setup.sh}.args
+
+if [[ "$HOST" == *midway* ]]; then
+ echo "On Midway"
+ echo "midway bash /bin/bash null null null" > tc.data
+elif [[ "$HOST" == *beagle* ]]; then
+ echo "On Beagle"
+ echo "beagle bash /bin/bash null null null" > tc.data
+elif [[ "$HOST" == *mcs* ]]; then
+ echo "On MCS"
+ echo "mcs bash /bin/bash null null null" > tc.data
+elif [[ "$HOST" == *uc3* ]]; then
+ echo "On UC3"
+ echo "uc3 bash /bin/bash null null null" > tc.data
+elif [[ "$HOST" == blogin*lcrc* ]]; then
+ echo "On Blues at LCRC "
+ echo "blues bash /bin/bash null null null" > tc.data
+elif [[ "$HOST" == flogin*lcrc* ]]; then
+ echo "On Fusion at LCRC "
+ echo "fusion bash /bin/bash null null null" > tc.data
+else
+ echo "On unidentified machine, using defaults"
+ echo "local bash /bin/bash null null null" > tc.data
+fi
+
+echo "-loops=$(( $jobspernode * $LOOPS ))" > $ARGS_FILE
+cat sites.xml | sed "s/JOBSPERNODE/$jobspernode/" > tmp && mv tmp sites.xml
+
+cp sites.xml sites.xml.before
+if [[ -z $MIDWAY_USERNAME ]]
+then
+ echo "Remote username not provided. Skipping sites configs"
+else
+ cat sites.xml | sed "s/MIDWAY_USERNAME/$MIDWAY_USERNAME/" > tmp && mv tmp\
+ sites.xml
+fi
+if [[ -z $UC3_USERNAME ]]
+then
+ echo "Remote username not provided. Skipping sites configs"
+else
+ cat sites.xml | sed "s/UC3_USERNAME/$UC3_USERNAME/" > tmp && mv tmp si\
+tes.xml
+fi
+if [[ -z $BEAGLE_USERNAME ]]
+then
+ echo "Remote username not provided. Skipping sites configs"
+else
+ cat sites.xml | sed "s/BEAGLE_USERNAME/$BEAGLE_USERNAME/" > tmp && mv \
+tmp sites.xml
+fi
+if [[ -z $MCS_USERNAME ]]
+then
+ echo "Remote username not provided. Skipping sites configs"
+else
+ cat sites.xml | sed "s/MCS_USERNAME/$MCS_USERNAME/" > tmp && mv \
+tmp sites.xml
+fi
+if [[ -z $BLUES_USERNAME ]]
+then
+ echo "Remote username not provided. Skipping sites configs"
+else
+ cat sites.xml | sed "s/BLUES_USERNAME/$BLUES_USERNAME/" > tmp && mv \
+tmp sites.xml
+fi
+if [[ -z $FUSION_USERNAME ]]
+then
+ echo "Remote username not provided. Skipping sites configs"
+else
+ cat sites.xml | sed "s/FUSION_USERNAME/$FUSION_USERNAME/" > tmp && mv \
+tmp sites.xml
+fi
+cp sites.xml sites.xml.after
+
+cat<<'EOF' > count_jobs.sh
+#!/bin/bash
+
+SELF="count_jobs.sh"
+SLEEPTIME=60
+
+
+BEFORE=$(($RANDOM%$SLEEPTIME))
+AFTER=$(($SLEEPTIME-$BEFORE))
+
+sleep $BEFORE
+echo "NODE $(hostname -f)"
+ACTIVE=`ps -u $USER | grep $SELF | wc -l`
+ps -u $USER
+echo "ps -u $USER | grep | wc -l"
+echo "ACTIVE $ACTIVE"
+echo "SPLIT $BEFORE:$AFTER"
+sleep $AFTER
+EOF
+
Property changes on: branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.setup.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.stdout
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.stdout (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.stdout 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,3 @@
+Swift 0.94 swift-r6888 cog-r3762
+
+RunID: 20130822-2202-c5ih1pj1
Added: branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.swift
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.swift (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.swift 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,18 @@
+type file;
+type script;
+
+app (file out, file err) app_run (script run)
+{
+ bash @run stdout=@out stderr=@err;
+}
+
+file f_out[] <simple_mapper; prefix="output/t", suffix=".out">;
+file f_err[] <simple_mapper; prefix="output/t", suffix=".err">;
+
+script wrapper <"count_jobs.sh">;
+
+int loop = @toInt(@arg("loops","10"));
+
+foreach item,i in [0:loop-1] {
+ (f_out[i], f_err[i]) = app_run(wrapper);
+}
Added: branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.timeout
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.timeout (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/jobs_per_node.timeout 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1 @@
+300
\ No newline at end of file
Added: branches/release-0.95/tests/stress/jobs_per_node/run
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/run (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/run 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#PATH=/scratch/midway/yadunand/swift-0.94RC2/cog/modules/swift/dist/swift-svn/bin:$PATH
+
+echo "Swift location: "; which swift
+echo "Swift version : "; swift -version
+
+export MIDWAY_USERNAME=yadunand
+export BEAGLE_USERNAME=yadunandb
+export MCS_USERNAME=yadunand
+export UC3_USERNAME=yadunand
+
+SCRIPT=jobs_per_node.swift
+BASE=${SCRIPT%.swift}
+
+rm $BASE.stdout
+cat title.txt
+
+cp sites.template.xml sites.xml
+./$BASE.setup.sh
+
+ARGS=$(cat $BASE.args)
+swift -tc.file tc.data -config swift.properties -sites.file sites.xml $BASE.swift ${ARGS[*]} | tee -a $BASE.stdout
+
+rm -rf *{swiftx,kml} $BASE-* _concurrent* failed* &> /dev/null
+
+./$BASE.check.sh
\ No newline at end of file
Property changes on: branches/release-0.95/tests/stress/jobs_per_node/run
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.95/tests/stress/jobs_per_node/sites.template.xml
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/sites.template.xml (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/sites.template.xml 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,128 @@
+<config>
+
+ <pool handle="local">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <profile namespace="globus" key="jobsPerNode">JOBSPERNODE</profile>
+ <profile namespace="globus" key="maxWalltime">00:15:00</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">0.03</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>.</workdirectory>
+ </pool>
+
+ <pool handle="beagle">
+ <execution provider="coaster" jobmanager="local:pbs"/>
+ <profile namespace="globus" key="jobsPerNode">JOBSPERNODE</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="providerAttributes"></profile>
+ <profile namespace="globus" key="maxtime">36000</profile>
+ <profile namespace="globus" key="maxWalltime">01:05:00</profile>
+ <profile namespace="globus" key="userHomeOverride">/lustre/beagle/BEAGLE_USERNAME/swiftwork</profile>
+ <profile namespace="globus" key="slots">4</profile>
+ <profile namespace="globus" key="maxnodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">4.80</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/BEAGLE_USERNAME/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="blues">
+ <execution jobmanager="local:pbs" provider="coaster"/>
+ <filesystem provider="local" url="none" />
+ <profile namespace="globus" key="maxtime">3600</profile>
+ <profile namespace="globus" key="jobsPerNode">JOBSPERNODE</profile>
+ <profile namespace="globus" key="ppn">8</profile>
+ <profile namespace="globus" key="queue">route</profile>
+ <profile namespace="globus" key="nodeGranularity">2</profile>
+ <profile namespace="globus" key="slots">4</profile>
+ <profile namespace="globus" key="maxNodes">4</profile>
+ <profile namespace="karajan" key="jobThrottle">2.20</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="HighOverAllocation">1000</profile>
+ <profile namespace="globus" key="LowOverAllocation">1000</profile>
+ <workdirectory>/tmp/BLUES_USERNAME/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="fusion">
+ <execution jobmanager="local:pbs" provider="coaster"/>
+ <filesystem provider="local" url="none" />
+ <profile namespace="globus" key="maxtime">3600</profile>
+ <profile namespace="globus" key="jobsPerNode">JOBSPERNODE</profile>
+ <profile namespace="globus" key="slots">4</profile>
+ <profile namespace="globus" key="nodeGranularity">2</profile>
+ <profile namespace="globus" key="maxNodes">2</profile>
+ <profile namespace="globus" key="queue">shared</profile>
+ <profile namespace="karajan" key="jobThrottle">5.99</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="HighOverAllocation">1000</profile>
+ <profile namespace="globus" key="LowOverAllocation">1000</profile>
+ <workdirectory>/homes/FUSION_USERNAME/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="midway">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">JOBSPERNODE</profile>
+ <profile namespace="globus" key="maxTime">36000</profile>
+ <profile namespace="globus" key="maxWalltime">01:05:00</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">4</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.64</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/MIDWAY_USERNAME</workdirectory>
+ </pool>
+
+
+ <pool handle="uc3">
+ <execution provider="coaster" jobmanager="local:condor"/>
+ <profile namespace="karajan" key="jobThrottle">10.00</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobsPerNode">JOBSPERNODE</profile>
+ <profile namespace="globus" key="maxtime">36000</profile>
+ <profile namespace="globus" key="maxWalltime">01:05:00</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">1000</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <!-- <profile namespace="globus" key="condor.+AccountingGroup">"group_friends.yadunand"</profile> -->
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <workdirectory>/home/UC3_USERNAME/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="mcs">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <profile namespace="globus" key="jobsPerNode">JOBSPERNODE</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="maxtime">36000</profile>
+ <profile namespace="globus" key="maxWalltime">01:05:00</profile>
+ <profile namespace="karajan" key="jobThrottle">0.0799</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="userHomeOverride">/sandbox/MCS_USERNAME/</profile>
+ <workdirectory>/sandbox/MCS_USERNAME/swiftwork</workdirectory>
+ </pool>
+
+
+ <pool handle="frisbee">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <profile namespace="globus" key="jobsPerNode">JOBSPERNODE</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="maxtime">3600</profile>
+ <profile namespace="globus" key="maxWalltime">00:05:00</profile>
+ <profile namespace="karajan" key="jobThrottle">0.0799</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/sandbox/MCS_USERNAME/swiftwork</workdirectory>
+ </pool>
+
+</config>
Added: branches/release-0.95/tests/stress/jobs_per_node/swift.log
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/swift.log (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/swift.log 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,4 @@
+2013-08-22 22:01:41,282+0000 DEBUG Loader Swift started
+2013-08-22 22:01:41,557+0000 DEBUG Loader Swift started
+2013-08-22 22:02:03,942+0000 DEBUG Loader Swift started
+2013-08-22 22:02:04,214+0000 DEBUG Loader Swift started
Added: branches/release-0.95/tests/stress/jobs_per_node/swift.properties
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/swift.properties (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/swift.properties 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,8 @@
+use.provider.staging=true
+use.wrapper.staging=false
+wrapperlog.always.transfer=true
+execution.retries=0
+lazy.errors=false
+provider.staging.pin.swiftfiles=false
+sitedir.keep=true
+tcp.port.range=50000,51000
\ No newline at end of file
Added: branches/release-0.95/tests/stress/jobs_per_node/tc.data
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/tc.data (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/tc.data 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1 @@
+midway bash /bin/bash null null null
Added: branches/release-0.95/tests/stress/jobs_per_node/teragen_wrap.sh
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/teragen_wrap.sh (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/teragen_wrap.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# By default with ARG1:100 and SLICESIZE=10000, this script will generate
+# 10^6 records.
+ARG1=1
+[ ! -z $1 ] && ARG1=$1
+
+FILE="input_$RANDOM.txt"
+LOWERLIMIT=0
+UPPERLIMIT=1000000 # 10^9
+SLICESIZE=10000 # 10^4 records padded to 100B would result in 1MB file
+#SLICESIZE=1000 # 10^3 If padded to 100B would result
+
+shuf -i $LOWERLIMIT-$UPPERLIMIT -n $(($SLICESIZE*$ARG1)) | awk '{printf "%-99s\n", $0}'
+exit 0
Added: branches/release-0.95/tests/stress/jobs_per_node/title.txt
===================================================================
--- branches/release-0.95/tests/stress/jobs_per_node/title.txt (rev 0)
+++ branches/release-0.95/tests/stress/jobs_per_node/title.txt 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,2 @@
+Test for the jobspernode swift config attribute
+
Added: branches/release-0.95/tests/stress/random_fail/Bug_info
===================================================================
--- branches/release-0.95/tests/stress/random_fail/Bug_info (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/Bug_info 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,22 @@
+Regression tests for Bug 1067
+| Link -> http://bugzilla.mcs.anl.gov/swift/show_bug.cgi?id=1067
+Swift 0.94 swift-r6888 cog-r3762
+
+Exception in sh:
+ Arguments: [randfail.sh, 50, 0]
+ Host: local
+ Directory: rand_fail_Bug1067-20130820-1749-6n7zbux4/jobs/6/sh-6abm13el
+ stderr.txt: Failing 11 < 50
+ stdout.txt:
+Caused by: Application /bin/bash failed with an exit code of 255
+
+Exception in sh:
+ Arguments: [randfail.sh, 50, 0]
+ Host: local
+ Directory: rand_fail_Bug1067-20130820-1749-6n7zbux4/jobs/8/sh-8abm13el
+ stderr.txt: Failing 19 < 50
+ stdout.txt:
+Caused by: Application /bin/bash failed with an exit code of 255
+
+Execution failed:
+ Got one name (derr) and 0 values: []
\ No newline at end of file
Added: branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.check.sh
===================================================================
--- branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.check.sh (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.check.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ! -f ${0%.check.sh}.stdout ]
+then
+ echo "${$0%.check.sh}.stdout missing"
+ exit -1
+fi
+
+grep "Got one name (derr)" ${0%.check.sh}.stdout
+if [ "$?" == 0 ]
+then
+ echo "EXIT : REGRESSION FOUND!" >&2
+ exit -1
+else
+ echo "Test passed"
+fi
+exit 0
\ No newline at end of file
Property changes on: branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.check.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.setup.sh
===================================================================
--- branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.setup.sh (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.setup.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Setup script will just output the following file
+
+cat<<'EOF' > randfail.sh
+#!/bin/bash
+
+FAIL_PROBABILITY=$1
+DELAY=$2
+
+ITEM=$(($RANDOM%100))
+sleep $2
+
+if (( "$ITEM" <= "$FAIL_PROBABILITY" ))
+then
+ echo "Failing $ITEM < $FAIL_PROBABILITY" >&2
+ exit -1
+fi
+echo "Not failing $ITEM > $FAIL_PROBABILITY"
+exit 0
+EOF
+
Property changes on: branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.setup.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.swift
===================================================================
--- branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.swift (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/rand_fail_Bug1067.swift 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,34 @@
+type file;
+
+file script<"randfail.sh">;
+
+app (file ofile1, file ofile2) quicklyFailingApp(file script, int failchance,
+int delay)
+{
+ sh @script failchance delay stdout=@ofile1 stderr=@ofile2;
+}
+
+app (file ofile) someApp3(file ifile, file jfile, file kfile)
+{
+ sh "-c" @strcat("cat ", at filename(ifile)) stdout=@ofile;
+}
+
+app (file ofile) someApp(file ifile)
+{
+ sh "-c" @strcat("cat ", at filename(ifile)) stdout=@ofile;
+}
+
+app sleep (int sec)
+{
+ sh "-c" @strcat("sleep ",sec);
+}
+
+int sufficientlyLargeNumber = 100;
+
+file a[];
+foreach i in [0:sufficientlyLargeNumber] {
+ file f1<single_file_mapper; file=@strcat("failed1.",i,".out")>;
+ file f2<single_file_mapper; file=@strcat("failed2.",i,".out")>;
+ (f1,f2) = quicklyFailingApp(script,50,0);
+ a[i] = someApp(f2);
+}
Added: branches/release-0.95/tests/stress/random_fail/run
===================================================================
--- branches/release-0.95/tests/stress/random_fail/run (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/run 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+PATH=/scratch/midway/yadunand/swift-0.94RC2/cog/modules/swift/dist/swift-svn/bin:$PATH
+
+
+echo "Swift location: "; which swift
+echo "Swift version : "; swift -version
+
+rm rand_fail_Bug1067.stdout
+cat title.txt
+
+./rand_fail_Bug1067.setup.sh
+
+for i in `seq 1 10`
+do
+swift -tc.file tc.template.data -config swift.properties -sites.file sites.template.xml rand_fail_Bug1067.swift | tee -a rand_fail_Bug1067.stdout
+
+rm -rf *{swiftx,kml} rand_fail_Bug1067-* _concurrent* failed*
+done
+
+./rand_fail_Bug1067.check.sh
\ No newline at end of file
Property changes on: branches/release-0.95/tests/stress/random_fail/run
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.95/tests/stress/random_fail/sites.template.xml
===================================================================
--- branches/release-0.95/tests/stress/random_fail/sites.template.xml (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/sites.template.xml 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+
+ <pool handle="midway">
+ <execution provider="coaster" jobmanager="local:local"/>
+ <profile namespace="globus" key="queue">sandyb</profile>
+ <profile namespace="globus" key="jobsPerNode">16</profile>
+ <profile namespace="globus" key="maxtime">36000</profile>
+ <profile namespace="globus" key="maxWalltime">00:10:00</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">4</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="karajan" key="jobThrottle">.64</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <workdirectory>/tmp/MIDWAY_USERNAME</workdirectory>
+ </pool>
+
+</config>
Added: branches/release-0.95/tests/stress/random_fail/swift.properties
===================================================================
--- branches/release-0.95/tests/stress/random_fail/swift.properties (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/swift.properties 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,11 @@
+wrapperlog.always.transfer=true
+sitedir.keep=true
+file.gc.enabled=false
+status.mode=provider
+
+execution.retries=5
+lazy.errors=true
+
+use.wrapper.staging=false
+use.provider.staging=false
+provider.staging.pin.swiftfiles=falsewrapperlog.always.transfer=true
Added: branches/release-0.95/tests/stress/random_fail/tc.template.data
===================================================================
--- branches/release-0.95/tests/stress/random_fail/tc.template.data (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/tc.template.data 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,2 @@
+local perl /usr/bin/perl null null null
+local sh /bin/bash
Added: branches/release-0.95/tests/stress/random_fail/title.txt
===================================================================
--- branches/release-0.95/tests/stress/random_fail/title.txt (rev 0)
+++ branches/release-0.95/tests/stress/random_fail/title.txt 2014-06-09 17:03:04 UTC (rev 7906)
@@ -0,0 +1,2 @@
+Regression tests for Bug 1067
+| Link -> http://bugzilla.mcs.anl.gov/swift/show_bug.cgi?id=1067
Modified: branches/release-0.95/tests/user_test.sh
===================================================================
--- branches/release-0.95/tests/user_test.sh 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/user_test.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -26,4 +26,4 @@
then
export GLOBUS_HOSTNAME=swift.rcc.uchicago.edu
export GLOBUS_TCP_PORT_RANGE=50000,51000
-fi;
\ No newline at end of file
+fi;
Modified: branches/release-0.95/tests/user_yadu.sh
===================================================================
--- branches/release-0.95/tests/user_yadu.sh 2014-06-03 20:40:12 UTC (rev 7905)
+++ branches/release-0.95/tests/user_yadu.sh 2014-06-09 17:03:04 UTC (rev 7906)
@@ -7,12 +7,21 @@
export COMM_USERNAME="yadunandb"
export BRID_USERNAME="yadunandb"
export FROM_MAIL="Test-Engine at midway001"
-export TO_MAIL="yadudoc1729 at gmail.com"
+export TO_MAIL="yadudoc1729 at gmail.com davidkelly999 at uchicago.edu wilde at mcs.anl.gov"
+#export TO_MAIL="yadudoc1729 at gmail.com"
-#export REMOTE_DRIVER_FASTSETUP="false"
-export REMOTE_DRIVER_FASTSETUP="true"
-export SWIFT_TAR_FILE="/scratch/midway/yadunand/swift-trunk.tar"
+################# MUST UPDATE FOR EACH REVISION #########################
+COG_URL=https://svn.code.sf.net/p/cogkit/svn/branches/4.1.11/src/cog
+SWIFT_URLk=https://svn.ci.uchicago.edu/svn/vdl2/branches/release-0.95
+export SWIFT_TAR_FILE=""
+export SWIFT_VERSION_OVERRIDE="swift-0.95"
+export SWIFT_VERSION="swift-0.95"
+################# MUST UPDATE FOR EACH REVISION #########################
+export REMOTE_DRIVER_FASTSETUP="false"
+#export REMOTE_DRIVER_FASTSETUP="true"
+export CLEAN_CHECKOUT="yes"
+
export KILL_JAVA="false"
# Extra performance stats
@@ -25,4 +34,4 @@
then
export GLOBUS_HOSTNAME=swift.rcc.uchicago.edu
export GLOBUS_TCP_PORT_RANGE=50000,51000
-fi;
\ No newline at end of file
+fi;
More information about the Swift-commit
mailing list