From noreply at svn.ci.uchicago.edu Fri Sep 5 07:19:23 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 07:19:23 -0500 (CDT) Subject: [Swift-commit] r2212 - provenancedb Message-ID: <20080905121923.7AE079B4001@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 07:19:22 -0500 (Fri, 05 Sep 2008) New Revision: 2212 Added: provenancedb/import-ci Log: script to import bencs log repo to db at CI Added: provenancedb/import-ci =================================================================== --- provenancedb/import-ci (rev 0) +++ provenancedb/import-ci 2008-09-05 12:19:22 UTC (rev 2212) @@ -0,0 +1,4 @@ +#!/bin/bash +cd /home/benc/swift-svn/provenancedb +./import-all $@ + Property changes on: provenancedb/import-ci ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Fri Sep 5 07:28:42 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 07:28:42 -0500 (CDT) Subject: [Swift-commit] r2213 - provenancedb Message-ID: <20080905122842.822379B4001@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 07:28:41 -0500 (Fri, 05 Sep 2008) New Revision: 2213 Modified: provenancedb/import-all provenancedb/import-ci Log: PROVDB env variable to allow script to use different databases Modified: provenancedb/import-all =================================================================== --- provenancedb/import-all 2008-09-05 12:19:22 UTC (rev 2212) +++ provenancedb/import-all 2008-09-05 12:28:41 UTC (rev 2213) @@ -3,8 +3,10 @@ # we need to keep this out of the log-proceesing dir because import # of individual runs will clean other files. +PROVDB=provdb source paths.source + pushd $LOGDIR ./create-everylog-vs-versions-data $LOGREPO popd @@ -24,7 +26,7 @@ if [ $version -ge 1538 ]; then echo -n "Log: $filename ... " -EXISTING=$(psql -p 5435 -d provdb -U benc --tuples-only -c "select count(*) from known_workflows where workflow_log_filename='$filename';") +EXISTING=$(psql -p 5435 -d $PROVDB -U benc --tuples-only -c "select count(*) from known_workflows where workflow_log_filename='$filename';") # echo EXISTING is X${EXISTING}X if [ $EXISTING -eq 0 ]; then Modified: provenancedb/import-ci =================================================================== --- provenancedb/import-ci 2008-09-05 12:19:22 UTC (rev 2212) +++ provenancedb/import-ci 2008-09-05 12:28:41 UTC (rev 2213) @@ -1,4 +1,12 @@ #!/bin/bash + +if [ "X$PROVDB" == "X" ]; then + echo PROVDB is not set. Defaulting to provdb + export PROVDB=provdb +else + echo PROVDB is set to $PROVDB +fi + cd /home/benc/swift-svn/provenancedb ./import-all $@ From noreply at svn.ci.uchicago.edu Fri Sep 5 08:55:10 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 08:55:10 -0500 (CDT) Subject: [Swift-commit] r2214 - log-processing Message-ID: <20080905135510.B9E851AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 08:55:09 -0500 (Fri, 05 Sep 2008) New Revision: 2214 Modified: log-processing/everylog-vs-versions Log: typo Modified: log-processing/everylog-vs-versions =================================================================== --- log-processing/everylog-vs-versions 2008-09-05 12:28:41 UTC (rev 2213) +++ log-processing/everylog-vs-versions 2008-09-05 13:55:09 UTC (rev 2214) @@ -1,6 +1,6 @@ #!/bin/bash -./create-everylog-vs-versions.data $1 +./create-everylog-vs-versions-data $1 gnuplot everylog-vs-versions.plot From noreply at svn.ci.uchicago.edu Fri Sep 5 08:57:46 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 08:57:46 -0500 (CDT) Subject: [Swift-commit] r2215 - log-processing Message-ID: <20080905135746.7B6DD1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 08:57:45 -0500 (Fri, 05 Sep 2008) New Revision: 2215 Modified: log-processing/makefile log-processing/whole-workflow-event Log: put workflow ID into workflow event file, for provenancedb Modified: log-processing/makefile =================================================================== --- log-processing/makefile 2008-09-05 13:55:09 UTC (rev 2214) +++ log-processing/makefile 2008-09-05 13:57:45 UTC (rev 2215) @@ -55,7 +55,7 @@ ./list-known-threads > threads.list workflow.event: start-time.tmp end-time.tmp - ./whole-workflow-event > workflow.event + ./whole-workflow-event $(LOG) > workflow.event execute-start-times.data: log ./extract-execute-start-times > execute-start-times.data Modified: log-processing/whole-workflow-event =================================================================== --- log-processing/whole-workflow-event 2008-09-05 13:55:09 UTC (rev 2214) +++ log-processing/whole-workflow-event 2008-09-05 13:57:45 UTC (rev 2215) @@ -3,5 +3,13 @@ export st=$(cat start-time.tmp) export et=$(cat end-time.tmp) -echo $st $(echo $et - $st | bc ) whole-workflow +#2008-08-30 08:59:26,994-0500 INFO unknown RUNID id=tag:benc at ci.uchicago.edu,2007:swift:run:20080830-0859-q9bqd8r4 +WFID=$(grep -E '^[^ ]* [^ ]* INFO unknown RUNID id=' $1 | sed 's/^.*INFO unknown RUNID id=\(.*\)$/\1/') + +if [ "X$WFID" == "X" ] ; then + WFID="unknown-run-id" +fi + +echo $st $(echo $et - $st | bc ) $WFID + From noreply at svn.ci.uchicago.edu Fri Sep 5 08:59:16 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 08:59:16 -0500 (CDT) Subject: [Swift-commit] r2216 - log-processing Message-ID: <20080905135916.58B9A1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 08:59:16 -0500 (Fri, 05 Sep 2008) New Revision: 2216 Modified: log-processing/create-everylog-vs-versions-data Log: allow subsets of log files in repo to be selected using /usr/bin/find predicates, for provenancedb Modified: log-processing/create-everylog-vs-versions-data =================================================================== --- log-processing/create-everylog-vs-versions-data 2008-09-05 13:57:45 UTC (rev 2215) +++ log-processing/create-everylog-vs-versions-data 2008-09-05 13:59:16 UTC (rev 2216) @@ -5,7 +5,7 @@ rm -f everylog-vs-versions.data -for logfile in `find $1 -name \*.log -and -not -name cluster-\* -and -not -name swift.log`; do +for logfile in `find $1 -name \*.log -and -not -name cluster-\* -and -not -name swift.log $EXTRALOGRESTRICTION`; do VERSION=$(grep -m 1 -E 'Swift v[^ ]* r[0-9]*' $logfile | sed 's/.*Swift v[^ ]* r\([0-9]*\).*/\1/') if [ "X$VERSION" == "X" ]; then # try newer format version info From noreply at svn.ci.uchicago.edu Fri Sep 5 09:35:24 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 09:35:24 -0500 (CDT) Subject: [Swift-commit] r2217 - provenancedb Message-ID: <20080905143524.6BBB69B4001@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 09:35:23 -0500 (Fri, 05 Sep 2008) New Revision: 2217 Modified: provenancedb/prov-init.sql Log: workflow timing information, and security changes Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2008-09-05 13:59:16 UTC (rev 2216) +++ provenancedb/prov-init.sql 2008-09-05 14:35:23 UTC (rev 2217) @@ -8,6 +8,7 @@ DROP TABLE dataset_filenames; DROP TABLE executes_in_workflows; DROP TABLE known_workflows; +DROP TABLE workflow_events; CREATE TABLE executes_in_workflows (workflow_id char(128), @@ -57,6 +58,15 @@ CREATE TABLE known_workflows ( workflow_log_filename char(128), - version char(128) + version char(128), + importstatus char(128) ); +CREATE TABLE workflow_events + ( workflow_id char(128), + starttime numeric, + duration numeric + ); + +grant all on dataset_containment, dataset_filenames, dataset_usage, executes_in_workflows, invocation_procedure_names, known_workflows, workflow_events to public; + From noreply at svn.ci.uchicago.edu Fri Sep 5 09:52:00 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 09:52:00 -0500 (CDT) Subject: [Swift-commit] r2218 - provenancedb Message-ID: <20080905145200.AFF259B4001@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 09:51:57 -0500 (Fri, 05 Sep 2008) New Revision: 2218 Modified: provenancedb/import-all Log: more information about how a log file is known Modified: provenancedb/import-all =================================================================== --- provenancedb/import-all 2008-09-05 14:35:23 UTC (rev 2217) +++ provenancedb/import-all 2008-09-05 14:51:57 UTC (rev 2218) @@ -3,7 +3,6 @@ # we need to keep this out of the log-proceesing dir because import # of individual runs will clean other files. -PROVDB=provdb source paths.source @@ -33,15 +32,16 @@ echo IMPORTING if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then - $SQLCMD -c "INSERT INTO known_workflows (workflow_log_filename, version) VALUES ('$filename','$version');" echo version $version in log file $filename echo ============= will import ============= ./prepare-for-import $filename ./import-run-to-sql $filename ./import-run-to-xml $filename + $SQLCMD -c "INSERT INTO known_workflows (workflow_log_filename, version, importstatus) VALUES ('$filename','$version','imported');" else echo SKIP: Workflow does not report successful end. + $SQLCMD -c "INSERT INTO known_workflows (workflow_log_filename, version, importstatus) VALUES ('$filename','$version','skipped - workflow did not end sucessfully');" fi else echo SKIP: Already known in workflow From noreply at svn.ci.uchicago.edu Fri Sep 5 09:54:34 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 09:54:34 -0500 (CDT) Subject: [Swift-commit] r2219 - provenancedb Message-ID: <20080905145434.DC9F19B4001@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 09:54:33 -0500 (Fri, 05 Sep 2008) New Revision: 2219 Modified: provenancedb/prepare-for-import Log: need to generate workflow.event for workflow IDs Modified: provenancedb/prepare-for-import =================================================================== --- provenancedb/prepare-for-import 2008-09-05 14:51:57 UTC (rev 2218) +++ provenancedb/prepare-for-import 2008-09-05 14:54:33 UTC (rev 2219) @@ -5,7 +5,7 @@ # of the provenance information into the SQL provenance database. pushd $LOGDIR -make LOG=$1 clean execute.global.event execute2.event +make LOG=$1 clean execute.global.event execute2.event workflow.event popd ./prepare-provenance-chart $1 From noreply at svn.ci.uchicago.edu Fri Sep 5 09:57:13 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 5 Sep 2008 09:57:13 -0500 (CDT) Subject: [Swift-commit] r2220 - provenancedb Message-ID: <20080905145713.8590C9B4001@www.ci.uchicago.edu> Author: benc Date: 2008-09-05 09:57:10 -0500 (Fri, 05 Sep 2008) New Revision: 2220 Modified: provenancedb/prov-to-sql.sh Log: workflow_events table population Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2008-09-05 14:54:33 UTC (rev 2219) +++ provenancedb/prov-to-sql.sh 2008-09-05 14:57:10 UTC (rev 2220) @@ -1,5 +1,6 @@ #!/bin/bash +echo Generating SQL rm -f tmp-import.sql @@ -29,5 +30,12 @@ echo "INSERT INTO dataset_filenames (dataset_id, filename) VALUES ('$dataset', '$filename');" >> tmp-import.sql done < $LOGDIR/dataset-filenames.txt +while read start duration wfid; do + echo "INSERT INTO workflow_events (workflow_id,starttime, duration) VALUES ('$wfid', '$start', '$duration');" >> tmp-import.sql +done < $LOGDIR/workflow.event +echo Sending SQL to DB + $SQLCMD < tmp-import.sql +echo Finished sending SQL to DB + From noreply at svn.ci.uchicago.edu Mon Sep 8 14:56:47 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 8 Sep 2008 14:56:47 -0500 (CDT) Subject: [Swift-commit] r2221 - nmi-build-test Message-ID: <20080908195647.6D3EE1360037@www.ci.uchicago.edu> Author: benc Date: 2008-09-08 14:56:46 -0500 (Mon, 08 Sep 2008) New Revision: 2221 Added: nmi-build-test/debug-x86_64-hang Log: test script for debugging hang on x86_64 Added: nmi-build-test/debug-x86_64-hang =================================================================== --- nmi-build-test/debug-x86_64-hang (rev 0) +++ nmi-build-test/debug-x86_64-hang 2008-09-08 19:56:46 UTC (rev 2221) @@ -0,0 +1,17 @@ +#!/bin/bash +which ant +which javac +#locate ant +#locate javac +mv trunk cog/modules/vdsk +cd cog/modules/vdsk/ + +echo building... +ant redist || exit 1 + +echo testing... +export PATH=`pwd`/dist/vdsk-svn/bin:$PATH + +cd ../language-behaviour +./run 001-echo || exit 3 + Property changes on: nmi-build-test/debug-x86_64-hang ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Mon Sep 8 15:13:28 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 8 Sep 2008 15:13:28 -0500 (CDT) Subject: [Swift-commit] r2222 - nmi-build-test Message-ID: <20080908201328.3E2081360037@www.ci.uchicago.edu> Author: benc Date: 2008-09-08 15:13:27 -0500 (Mon, 08 Sep 2008) New Revision: 2222 Modified: nmi-build-test/debug-x86_64-hang Log: download patch and apply for testing Modified: nmi-build-test/debug-x86_64-hang =================================================================== --- nmi-build-test/debug-x86_64-hang 2008-09-08 19:56:46 UTC (rev 2221) +++ nmi-build-test/debug-x86_64-hang 2008-09-08 20:13:27 UTC (rev 2222) @@ -4,6 +4,11 @@ #locate ant #locate javac mv trunk cog/modules/vdsk + +wget http://www.ci.uchicago.edu/~benc/tmp/more-debug-hanging-wh + +patch -p1 < more-debug-hanging-wh && echo patch was successful + cd cog/modules/vdsk/ echo building... From noreply at svn.ci.uchicago.edu Mon Sep 8 16:21:28 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 8 Sep 2008 16:21:28 -0500 (CDT) Subject: [Swift-commit] r2223 - nmi-build-test Message-ID: <20080908212128.6FD6F1360037@www.ci.uchicago.edu> Author: benc Date: 2008-09-08 16:21:27 -0500 (Mon, 08 Sep 2008) New Revision: 2223 Modified: nmi-build-test/debug-x86_64-hang Log: incorrect relative path to test dir Modified: nmi-build-test/debug-x86_64-hang =================================================================== --- nmi-build-test/debug-x86_64-hang 2008-09-08 20:13:27 UTC (rev 2222) +++ nmi-build-test/debug-x86_64-hang 2008-09-08 21:21:27 UTC (rev 2223) @@ -17,6 +17,6 @@ echo testing... export PATH=`pwd`/dist/vdsk-svn/bin:$PATH -cd ../language-behaviour +cd tests/language-behaviour ./run 001-echo || exit 3 From noreply at svn.ci.uchicago.edu Mon Sep 8 16:56:49 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 8 Sep 2008 16:56:49 -0500 (CDT) Subject: [Swift-commit] r2224 - nmi-build-test Message-ID: <20080908215649.C7D251360037@www.ci.uchicago.edu> Author: benc Date: 2008-09-08 16:56:48 -0500 (Mon, 08 Sep 2008) New Revision: 2224 Modified: nmi-build-test/debug-x86_64-hang Log: do not need to strip a path component... Modified: nmi-build-test/debug-x86_64-hang =================================================================== --- nmi-build-test/debug-x86_64-hang 2008-09-08 21:21:27 UTC (rev 2223) +++ nmi-build-test/debug-x86_64-hang 2008-09-08 21:56:48 UTC (rev 2224) @@ -7,7 +7,7 @@ wget http://www.ci.uchicago.edu/~benc/tmp/more-debug-hanging-wh -patch -p1 < more-debug-hanging-wh && echo patch was successful +patch < more-debug-hanging-wh && echo patch was successful cd cog/modules/vdsk/ From noreply at svn.ci.uchicago.edu Tue Sep 9 09:08:01 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 9 Sep 2008 09:08:01 -0500 (CDT) Subject: [Swift-commit] r2225 - nmi-build-test Message-ID: <20080909140801.88A201AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-09 09:08:00 -0500 (Tue, 09 Sep 2008) New Revision: 2225 Modified: nmi-build-test/debug-x86_64-hang Log: more x86_64 hang debugging Modified: nmi-build-test/debug-x86_64-hang =================================================================== --- nmi-build-test/debug-x86_64-hang 2008-09-08 21:56:48 UTC (rev 2224) +++ nmi-build-test/debug-x86_64-hang 2008-09-09 14:08:00 UTC (rev 2225) @@ -7,7 +7,7 @@ wget http://www.ci.uchicago.edu/~benc/tmp/more-debug-hanging-wh -patch < more-debug-hanging-wh && echo patch was successful +patch -p0 < more-debug-hanging-wh && echo patch was successful cd cog/modules/vdsk/ @@ -18,5 +18,8 @@ export PATH=`pwd`/dist/vdsk-svn/bin:$PATH cd tests/language-behaviour -./run 001-echo || exit 3 +while true; do + ./run 001-echo || exit 3 +done + From noreply at svn.ci.uchicago.edu Wed Sep 17 14:00:53 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Sep 2008 14:00:53 -0500 (CDT) Subject: [Swift-commit] r2226 - trunk/libexec Message-ID: <20080917190053.C86169B4001@www.ci.uchicago.edu> Author: benc Date: 2008-09-17 14:00:52 -0500 (Wed, 17 Sep 2008) New Revision: 2226 Modified: trunk/libexec/wrapper.sh Log: If supplied executable is not an absolute path, look up the path using "which" Modified: trunk/libexec/wrapper.sh =================================================================== --- trunk/libexec/wrapper.sh 2008-09-09 14:08:00 UTC (rev 2225) +++ trunk/libexec/wrapper.sh 2008-09-17 19:00:52 UTC (rev 2226) @@ -163,6 +163,10 @@ export PATH=$PATHPREFIX:$PATH fi +if [ "X${EXEC:1:1}" != "X/" ] ; then +export EXEC=$(which $EXEC) +fi + log "DIR=$DIR" log "EXEC=$EXEC" log "STDIN=$STDIN" From noreply at svn.ci.uchicago.edu Wed Sep 17 14:42:42 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 17 Sep 2008 14:42:42 -0500 (CDT) Subject: [Swift-commit] r2227 - trunk/docs Message-ID: <20080917194242.89B9A9B4001@www.ci.uchicago.edu> Author: benc Date: 2008-09-17 14:42:41 -0500 (Wed, 17 Sep 2008) New Revision: 2227 Modified: trunk/docs/userguide.xml Log: missing profile name for coastersPerNode Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2008-09-17 19:00:52 UTC (rev 2226) +++ trunk/docs/userguide.xml 2008-09-17 19:42:41 UTC (rev 2227) @@ -2291,7 +2291,7 @@ condor_requirements allows a requirements string to be specified when Condor is used as an LRM behind GRAM2. Example: <profile namespace="globus" key="condor_requirements">Arch == "X86_64" || Arch="INTEL"</profile> - Specifies + coastersPerNode specifies the number of coaster workers to be run on each node. This profile entry is used by the coaster provider. From noreply at svn.ci.uchicago.edu Fri Sep 19 10:19:32 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Sep 2008 10:19:32 -0500 (CDT) Subject: [Swift-commit] r2228 - SwiftApps Message-ID: <20080919151932.8E4C81AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-19 10:19:30 -0500 (Fri, 19 Sep 2008) New Revision: 2228 Added: SwiftApps/benctest/ Log: demo of svn From noreply at svn.ci.uchicago.edu Fri Sep 19 10:20:32 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Sep 2008 10:20:32 -0500 (CDT) Subject: [Swift-commit] r2229 - SwiftApps Message-ID: <20080919152032.8331F1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-19 10:20:30 -0500 (Fri, 19 Sep 2008) New Revision: 2229 Removed: SwiftApps/benctest/ Log: remove demo svn From noreply at svn.ci.uchicago.edu Fri Sep 19 11:35:20 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Sep 2008 11:35:20 -0500 (CDT) Subject: [Swift-commit] r2230 - in trunk/docs: . formatting Message-ID: <20080919163520.8402E1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-19 11:35:19 -0500 (Fri, 19 Sep 2008) New Revision: 2230 Added: trunk/docs/build-chunked-userguide.sh trunk/docs/formatting/swiftsh_html_chunked.xsl Modified: trunk/docs/Makefile trunk/docs/buildguides.sh Log: Chunked userguide for easier online use Modified: trunk/docs/Makefile =================================================================== --- trunk/docs/Makefile 2008-09-19 15:20:30 UTC (rev 2229) +++ trunk/docs/Makefile 2008-09-19 16:35:19 UTC (rev 2230) @@ -6,6 +6,9 @@ pdfs: userguide.pdf tutorial.pdf tutorial-live.pdf quickstartguide.pdf reallyquickstartguide.pdf languagespec.pdf languagespec-0.6.pdf log-processing.pdf +chunked-userguide: userguide.xml + ./build-chunked-userguide.sh + %.php: %.xml xsltproc --nonet formatting/swiftsh_html.xsl $< sed -e "s/index.html#/#/g" index.html >$@ Added: trunk/docs/build-chunked-userguide.sh =================================================================== --- trunk/docs/build-chunked-userguide.sh (rev 0) +++ trunk/docs/build-chunked-userguide.sh 2008-09-19 16:35:19 UTC (rev 2230) @@ -0,0 +1,12 @@ +#!/bin/sh + +mkdir -p userguide/ || exit 1 +cd userguide/ || exit 2 +rm -f *.html *.php + +xsltproc --nonet ../formatting/swiftsh_html_chunked.xsl ../userguide.xml + +for a in *.html; do + B=$(basename -s .html $a) + mv $a ${B}.php +done Modified: trunk/docs/buildguides.sh =================================================================== --- trunk/docs/buildguides.sh 2008-09-19 15:20:30 UTC (rev 2229) +++ trunk/docs/buildguides.sh 2008-09-19 16:35:19 UTC (rev 2230) @@ -32,3 +32,5 @@ process "languagespec.xml" "swift_html.xsl" process "languagespec-0.6.xml" "swift_html.xsl" process "log-processing.xml" "swift_html.xsl" +./build-chunked-userguide + Added: trunk/docs/formatting/swiftsh_html_chunked.xsl =================================================================== --- trunk/docs/formatting/swiftsh_html_chunked.xsl (rev 0) +++ trunk/docs/formatting/swiftsh_html_chunked.xsl 2008-09-19 16:35:19 UTC (rev 2230) @@ -0,0 +1,253 @@ + + + + + + + + + + + + + 1 + + + 1 + + /docbook-images/ + + + + + + /docbook-images/ + + 0 + + + + 0 + + 1 + + 2 + + 4 + + 1 + + + + + + + + + + + + + + + + + + + + + + + 1 + + 1 + + + yes + + + + + + + + + + + + + + + + + + + + + + + + + + + section-3 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + title + + + + + + + + + + + + + + + +
+ + + + + + + +
+ ]]> + + + + + + + + + + +
+ + + ]]> + + + From noreply at svn.ci.uchicago.edu Fri Sep 19 11:38:08 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Sep 2008 11:38:08 -0500 (CDT) Subject: [Swift-commit] r2231 - trunk/docs Message-ID: <20080919163808.61F791AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-19 11:38:07 -0500 (Fri, 19 Sep 2008) New Revision: 2231 Modified: trunk/docs/build-chunked-userguide.sh Log: make build-chunked-userguide executable Property changes on: trunk/docs/build-chunked-userguide.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Fri Sep 19 11:59:07 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Sep 2008 11:59:07 -0500 (CDT) Subject: [Swift-commit] r2232 - in trunk/docs: . formatting Message-ID: <20080919165907.E4A0A1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-19 11:59:07 -0500 (Fri, 19 Sep 2008) New Revision: 2232 Modified: trunk/docs/build-chunked-userguide.sh trunk/docs/formatting/swiftsh_html_chunked.xsl Log: generate PHP extension inside docbook so that inter-chunk links work. Modified: trunk/docs/build-chunked-userguide.sh =================================================================== --- trunk/docs/build-chunked-userguide.sh 2008-09-19 16:38:07 UTC (rev 2231) +++ trunk/docs/build-chunked-userguide.sh 2008-09-19 16:59:07 UTC (rev 2232) @@ -6,7 +6,3 @@ xsltproc --nonet ../formatting/swiftsh_html_chunked.xsl ../userguide.xml -for a in *.html; do - B=$(basename -s .html $a) - mv $a ${B}.php -done Modified: trunk/docs/formatting/swiftsh_html_chunked.xsl =================================================================== --- trunk/docs/formatting/swiftsh_html_chunked.xsl 2008-09-19 16:38:07 UTC (rev 2231) +++ trunk/docs/formatting/swiftsh_html_chunked.xsl 2008-09-19 16:59:07 UTC (rev 2232) @@ -5,6 +5,7 @@ + .php From noreply at svn.ci.uchicago.edu Fri Sep 19 12:08:03 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Sep 2008 12:08:03 -0500 (CDT) Subject: [Swift-commit] r2233 - trunk/docs Message-ID: <20080919170803.832D11AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-19 12:08:02 -0500 (Fri, 19 Sep 2008) New Revision: 2233 Modified: trunk/docs/userguide.xml Log: IDs on top level sections, useful for chunked user guide Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2008-09-19 16:59:07 UTC (rev 2232) +++ trunk/docs/userguide.xml 2008-09-19 17:08:02 UTC (rev 2233) @@ -8,7 +8,7 @@ Source control $LastChangedRevision$ -
+
Overview This manual provides reference material for Swift: the SwiftScript language @@ -33,7 +33,7 @@ files to and from the chosen sites and remote execution of program code.
-
+
The SwiftScript Language
Data model Data processed by Swift is strongly typed. It may be take the form @@ -1953,7 +1953,7 @@
-
Invoking an application from Swift +
Invoking an application from Swift There are certain requirements on the behaviour of application programs used in SwiftScript programs. These requirements are primarily to ensure @@ -1996,7 +1996,8 @@
-
Technical overview of the Swift architecture +
+ Technical overview of the Swift architecture This section attempts to provide a technical overview of the Swift architecture. @@ -2177,7 +2178,7 @@
-
Built-in procedure reference +
Built-in procedure reference This section details built-in procedures that are available for use in the SwiftScript language. @@ -2418,7 +2419,7 @@
-
The Site Catalog - sites.xml +
The Site Catalog - sites.xml The site catalog lists details of each site that Swift can use. The default file contains one entry for local execution, and a large number of @@ -2586,7 +2587,7 @@
-
The Transformation Catalog - tc.data +
The Transformation Catalog - tc.data The transformation catalog lists where application executables are located on remote sites. @@ -2631,7 +2632,7 @@
-
Environment variables +
Environment variables There are a number of environment variables used by Swift. Some of them are documented in this section: @@ -2666,7 +2667,7 @@ site-shared filesystem directly.
-
Build options +
Build options See the Swift download page for instructions on downloading and From noreply at svn.ci.uchicago.edu Fri Sep 19 12:19:50 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Sep 2008 12:19:50 -0500 (CDT) Subject: [Swift-commit] r2234 - trunk/docs Message-ID: <20080919171950.5FE3E1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-19 12:19:49 -0500 (Fri, 19 Sep 2008) New Revision: 2234 Modified: trunk/docs/buildguides.sh Log: correct filename for buildguides Modified: trunk/docs/buildguides.sh =================================================================== --- trunk/docs/buildguides.sh 2008-09-19 17:08:02 UTC (rev 2233) +++ trunk/docs/buildguides.sh 2008-09-19 17:19:49 UTC (rev 2234) @@ -32,5 +32,5 @@ process "languagespec.xml" "swift_html.xsl" process "languagespec-0.6.xml" "swift_html.xsl" process "log-processing.xml" "swift_html.xsl" -./build-chunked-userguide +./build-chunked-userguide.sh From noreply at svn.ci.uchicago.edu Fri Sep 19 13:54:23 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 19 Sep 2008 13:54:23 -0500 (CDT) Subject: [Swift-commit] r2235 - trunk/docs/formatting Message-ID: <20080919185423.774E61AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-19 13:54:22 -0500 (Fri, 19 Sep 2008) New Revision: 2235 Modified: trunk/docs/formatting/swiftsh_html_chunked.xsl Log: ugly looking but functional navigation buttons for chunked userguide Modified: trunk/docs/formatting/swiftsh_html_chunked.xsl =================================================================== --- trunk/docs/formatting/swiftsh_html_chunked.xsl 2008-09-19 17:19:49 UTC (rev 2234) +++ trunk/docs/formatting/swiftsh_html_chunked.xsl 2008-09-19 18:54:22 UTC (rev 2235) @@ -179,13 +179,24 @@ + + + + + - + + + + + + + @@ -216,7 +227,7 @@ - + @@ -251,4 +262,28 @@ ]]> + + + +
+ + + + + + + + + + + + + > + + + + + +
+ From noreply at svn.ci.uchicago.edu Sat Sep 20 10:47:37 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 20 Sep 2008 10:47:37 -0500 (CDT) Subject: [Swift-commit] r2236 - www/docs Message-ID: <20080920154737.1FA981AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-20 10:47:35 -0500 (Sat, 20 Sep 2008) New Revision: 2236 Modified: www/docs/index.php Log: link to multi-page (docbook chunked) user guide Modified: www/docs/index.php =================================================================== --- www/docs/index.php 2008-09-19 18:54:22 UTC (rev 2235) +++ www/docs/index.php 2008-09-20 15:47:35 UTC (rev 2236) @@ -62,7 +62,7 @@

Swift User Guide

From noreply at svn.ci.uchicago.edu Sun Sep 21 14:20:55 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 21 Sep 2008 14:20:55 -0500 (CDT) Subject: [Swift-commit] r2237 - log-processing Message-ID: <20080921192055.B9A651360002@www.ci.uchicago.edu> Author: benc Date: 2008-09-21 14:20:54 -0500 (Sun, 21 Sep 2008) New Revision: 2237 Added: log-processing/queue-state-transitions Modified: log-processing/index.html.template log-processing/makefile.karatasks log-processing/makefile.webpage Log: Generate a transitions file for job queue times, and associated (not working) graphs. Patch supplied by: Andriy Fedorov Modified: log-processing/index.html.template =================================================================== --- log-processing/index.html.template 2008-09-20 15:47:35 UTC (rev 2236) +++ log-processing/index.html.template 2008-09-21 19:20:54 UTC (rev 2237) @@ -167,7 +167,15 @@

karatasks JOB_SUBMISSION trails:

+

Number of karajan level job submissions that are 'Submitted' - this means +that the underlying execution layer (such as GRAM) has reported that the +job is submitted for execution, but not executing (queued).

+ +

karajan queued JOB_SUBMISSION tasks

+ + +

falkon

Falkon taskperf log:

Modified: log-processing/makefile.karatasks =================================================================== --- log-processing/makefile.karatasks 2008-09-20 15:47:35 UTC (rev 2236) +++ log-processing/makefile.karatasks 2008-09-21 19:20:54 UTC (rev 2237) @@ -35,3 +35,7 @@ karatasks.JOB_SUBMISSION.Active.transitions: karatasks.JOB_SUBMISSION.transitions cat karatasks.JOB_SUBMISSION.transitions | ./swap-and-sort-and-swap | ./active-state-transitions > karatasks.JOB_SUBMISSION.Active.transitions + +karatasks.JOB_SUBMISSION.Queue.transitions: karatasks.JOB_SUBMISSION.transitions + cat karatasks.JOB_SUBMISSION.transitions | ./swap-and-sort-and-swap | ./queue-state-transitions > karatasks.JOB_SUBMISSION.Queue.transitions + Modified: log-processing/makefile.webpage =================================================================== --- log-processing/makefile.webpage 2008-09-20 15:47:35 UTC (rev 2236) +++ log-processing/makefile.webpage 2008-09-21 19:20:54 UTC (rev 2237) @@ -10,7 +10,9 @@ karatasks.FILE_OPERATION-trails.png karatasks.FILE_TRANSFER-trails.png \ karatasks.FILE_TRANSFER.stats karatasks.FILE_OPERATION.stats \ karatasks.JOB_SUBMISSION.stats karatasks-total.png \ -karatasks.JOB_SUBMISSION.Active-total.png index.html.kara +karatasks.JOB_SUBMISSION.Active-total.png index.html.kara \ +karatasks.JOB_SUBMISSION.Queue-total.png \ +karatasks.JOB_SUBMISSION.Queue.sorted-start.png webpage.clusters: karatasks.clusters.png karatasks.clusters-total.png \ karatasks.clusters.sorted-start.png karatasks.clusters-duration-histogram.png Added: log-processing/queue-state-transitions =================================================================== --- log-processing/queue-state-transitions (rev 0) +++ log-processing/queue-state-transitions 2008-09-21 19:20:54 UTC (rev 2237) @@ -0,0 +1,34 @@ +#!/usr/bin/env perl + +# pull out transitions that go to submitted state and the immediately +# following transition + +use Time::Local; + +$prevjob = "queue-state-transitions no previous event"; + +$prevline = ""; + +$active = 0; + +foreach $n () { + $n =~ /^([^ ]*) ([^ ]*) ([^ ]*) (.*)$/; + $ts = $1; + $job = $2; + $state = $3; + + if($state eq "Submitted") { + print "$n"; + $active = 1; + } elsif( ($job eq $prevjob) && ($active) ) { + if($active == 1) { + print $n; + $active = 0; + } + } else { + $prevjob = $job; + }; + $prevline = $n; +} + +# print $prevline; Property changes on: log-processing/queue-state-transitions ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Sun Sep 21 14:46:31 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 21 Sep 2008 14:46:31 -0500 (CDT) Subject: [Swift-commit] r2238 - log-processing Message-ID: <20080921194631.D6BA91360002@www.ci.uchicago.edu> Author: benc Date: 2008-09-21 14:46:31 -0500 (Sun, 21 Sep 2008) New Revision: 2238 Modified: log-processing/queue-state-transitions Log: queue delay graphs had all events ending at t=infinity rather than at point job becomes active/fails Modified: log-processing/queue-state-transitions =================================================================== --- log-processing/queue-state-transitions 2008-09-21 19:20:54 UTC (rev 2237) +++ log-processing/queue-state-transitions 2008-09-21 19:46:31 UTC (rev 2238) @@ -16,6 +16,7 @@ $ts = $1; $job = $2; $state = $3; + $linerest = $4; if($state eq "Submitted") { print "$n"; @@ -23,6 +24,7 @@ } elsif( ($job eq $prevjob) && ($active) ) { if($active == 1) { print $n; + print "$ts $job DoneQueueing $linerest\n"; $active = 0; } } else { From noreply at svn.ci.uchicago.edu Mon Sep 22 15:16:15 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Sep 2008 15:16:15 -0500 (CDT) Subject: [Swift-commit] r2239 - log-processing Message-ID: <20080922201616.11CF71AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-22 15:16:14 -0500 (Mon, 22 Sep 2008) New Revision: 2239 Added: log-processing/trail-freex.plot.template Log: template was uncommitted causing incorrect log to be plotted in conspiracy with a temp-file reuse bug Added: log-processing/trail-freex.plot.template =================================================================== --- log-processing/trail-freex.plot.template (rev 0) +++ log-processing/trail-freex.plot.template 2008-09-22 20:16:14 UTC (rev 2239) @@ -0,0 +1,12 @@ + +set terminal png +set output 'tmp-trail.png' + +set key outside below +set ylabel 'total events' +set xlabel 'realtime / s' + +set xrange [0:] + + +plot \ From noreply at svn.ci.uchicago.edu Mon Sep 22 15:18:10 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Sep 2008 15:18:10 -0500 (CDT) Subject: [Swift-commit] r2240 - log-processing Message-ID: <20080922201810.CE6B11AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-22 15:18:10 -0500 (Mon, 22 Sep 2008) New Revision: 2240 Modified: log-processing/trail-freex Log: temporary file name reuse caused old plots to be output rather than no output to be made (this is the bug referenced in r2239 commit message) Modified: log-processing/trail-freex =================================================================== --- log-processing/trail-freex 2008-09-22 20:16:14 UTC (rev 2239) +++ log-processing/trail-freex 2008-09-22 20:18:10 UTC (rev 2240) @@ -1,5 +1,6 @@ #!/bin/bash +rm tmp-trail.png # maxtime=$(./max-duration) From noreply at svn.ci.uchicago.edu Mon Sep 22 15:24:13 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Sep 2008 15:24:13 -0500 (CDT) Subject: [Swift-commit] r2241 - log-processing Message-ID: <20080922202413.7C3F61AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-22 15:24:12 -0500 (Mon, 22 Sep 2008) New Revision: 2241 Modified: log-processing/makefile Log: zeroed transitions need start-time file to exist Modified: log-processing/makefile =================================================================== --- log-processing/makefile 2008-09-22 20:18:10 UTC (rev 2240) +++ log-processing/makefile 2008-09-22 20:24:12 UTC (rev 2241) @@ -114,7 +114,7 @@ info.transitions: ./info-to-transitions $(IDIR) > $@ -info.zeroed.transitions: +info.zeroed.transitions: start-time.tmp ./info-to-zeroed-transitions $(IDIR) > $@ info-md5sums: execute2.event From noreply at svn.ci.uchicago.edu Mon Sep 22 15:48:45 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Sep 2008 15:48:45 -0500 (CDT) Subject: [Swift-commit] r2242 - log-processing/bin Message-ID: <20080922204845.EF3A41AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-22 15:48:45 -0500 (Mon, 22 Sep 2008) New Revision: 2242 Modified: log-processing/bin/swift-plot-log Log: build -info logs if there is a .d subdirectory for a log Modified: log-processing/bin/swift-plot-log =================================================================== --- log-processing/bin/swift-plot-log 2008-09-22 20:24:12 UTC (rev 2241) +++ log-processing/bin/swift-plot-log 2008-09-22 20:48:45 UTC (rev 2242) @@ -20,9 +20,18 @@ echo Log is in directory $LOG_DIRECTORY echo Log basename is $LOG_FILE_BASE +MAKEENV="" +MAKETARGETS="" +INFODIR=${LOG_DIRECTORY}/${LOG_FILE_BASE}.d + +if [ -d $INFODIR ]; then + MAKEENV="$MAKEENV IDIR=${LOG_DIRECTORY}/${LOG_FILE_BASE}.d/" + MAKETARGETS="webpage.info" +fi + cd $LOG_CODE_HOME -make SDL=$LOG_FILE_BASE LOG=$LOG_FILE_PATH clean webpage.kara webpage.weights webpage distributable distribute +make $MAKEENV SDL=$LOG_FILE_BASE LOG=$LOG_FILE_PATH clean webpage.kara webpage.weights karatasks.JOB_SUBMISSION.Queue.transitions karatasks.JOB_SUBMISSION.Queue.event $MAKETARGETS webpage distribute mv report-$LOG_FILE_BASE $LOG_DIRECTORY/ From noreply at svn.ci.uchicago.edu Mon Sep 22 18:11:17 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 22 Sep 2008 18:11:17 -0500 (CDT) Subject: [Swift-commit] r2243 - trunk/tests/misc Message-ID: <20080922231117.9628C1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-22 18:11:16 -0500 (Mon, 22 Sep 2008) New Revision: 2243 Modified: trunk/tests/misc/wonky-replication.sh Log: better sites file for replication test Modified: trunk/tests/misc/wonky-replication.sh =================================================================== --- trunk/tests/misc/wonky-replication.sh 2008-09-22 20:48:45 UTC (rev 2242) +++ trunk/tests/misc/wonky-replication.sh 2008-09-22 23:11:16 UTC (rev 2243) @@ -16,11 +16,17 @@ cat tmp.tc.data.wonky-twosite | sed 's/localhost/wonkyA/' >> tmp.tc.data.sites cat tmp.tc.data.wonky-twosite | sed 's/localhost/wonkyB/' >> tmp.tc.data.sites -cat $(dirname $(which swift))/../etc/swift.properties | grep --invert-match -E '^replication.enabled=' > $CF +cat $(dirname $(which swift))/../etc/swift.properties | \ + grep --invert-match -E '^replication.enabled=' | \ + grep --invert-match -E '^tc.file=' | \ + grep --invert-match -E '^sites.file=' \ + > $CF echo replication.enabled=true >> $CF +echo tc.file=tmp.tc.data.sites >> $CF +echo sites.file=../sites/wonky/slow-queue-fast-queue.xml >> $CF -export SWIFT_TEST_PARAMS="-config $CF -tc.file tmp.tc.data.sites -sites.file ../sites/wonky/slow-queue-fast-queue.xml -debug" +export SWIFT_TEST_PARAMS="-config $CF" -./run 066-many.swift +./run 0651-several-delay.swift 066-many.swift From noreply at svn.ci.uchicago.edu Tue Sep 23 09:55:28 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 23 Sep 2008 09:55:28 -0500 (CDT) Subject: [Swift-commit] r2244 - log-processing Message-ID: <20080923145528.9302E1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-23 09:55:27 -0500 (Tue, 23 Sep 2008) New Revision: 2244 Modified: log-processing/whole-workflow-event Log: add workflow filename on to the end of workflow.event Modified: log-processing/whole-workflow-event =================================================================== --- log-processing/whole-workflow-event 2008-09-22 23:11:16 UTC (rev 2243) +++ log-processing/whole-workflow-event 2008-09-23 14:55:27 UTC (rev 2244) @@ -11,5 +11,5 @@ WFID="unknown-run-id" fi -echo $st $(echo $et - $st | bc ) $WFID +echo $st $(echo $et - $st | bc ) $WFID $1 From noreply at svn.ci.uchicago.edu Tue Sep 23 09:58:00 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 23 Sep 2008 09:58:00 -0500 (CDT) Subject: [Swift-commit] r2245 - log-processing Message-ID: <20080923145800.2B59D1AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-23 09:57:59 -0500 (Tue, 23 Sep 2008) New Revision: 2245 Modified: log-processing/logs-for-all Log: path for use at CI Modified: log-processing/logs-for-all =================================================================== --- log-processing/logs-for-all 2008-09-23 14:55:27 UTC (rev 2244) +++ log-processing/logs-for-all 2008-09-23 14:57:59 UTC (rev 2245) @@ -1,13 +1,13 @@ #!/bin/bash -for a in `find ~/work/everylog/wilde -name \*.log -and -not -name cluster-\* ` ; do +for a in `find ~/swift-logs -name Mol\*.log -and -not -name cluster-\* ` ; do echo Processing log file $a SDL=$(basename $a .log) D=$(dirname $a) if [ -d report-$SDL ]; then echo Skipping else - make KDIR=$D LOG=$a SDL=$SDL clean webpage.weights webpage.kara webpage.kickstart webpage + make KDIR=$D LOG=$a SDL=$SDL clean webpage.kara webpage.kickstart webpage make KDIR=$D LOG=$a SDL=$SDL distributable distribute fi done From noreply at svn.ci.uchicago.edu Tue Sep 23 13:09:34 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 23 Sep 2008 13:09:34 -0500 (CDT) Subject: [Swift-commit] r2246 - log-processing Message-ID: <20080923180934.E45091AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-23 13:09:34 -0500 (Tue, 23 Sep 2008) New Revision: 2246 Modified: log-processing/index.html.template log-processing/makefile.webpage Log: more information about queue time and active time according to karajan status Modified: log-processing/index.html.template =================================================================== --- log-processing/index.html.template 2008-09-23 14:57:59 UTC (rev 2245) +++ log-processing/index.html.template 2008-09-23 18:09:34 UTC (rev 2246) @@ -175,8 +175,29 @@

karajan queued JOB_SUBMISSION tasks

+

karajan queued JOB_SUBMISSION cumulative duration

+ +
+include(`karatasks.JOB_SUBMISSION.Queue.stats')
+
+ +

Number of tasks in Active state

+ + +

karajan active JOB_SUBMISSION tasks

+ + +

karajan active JOB_SUBMISSION cumulative duration

+ + +
+include(`karatasks.JOB_SUBMISSION.Active.stats')
+
+
+ +

falkon

Falkon taskperf log:

Modified: log-processing/makefile.webpage =================================================================== --- log-processing/makefile.webpage 2008-09-23 14:57:59 UTC (rev 2245) +++ log-processing/makefile.webpage 2008-09-23 18:09:34 UTC (rev 2246) @@ -11,8 +11,13 @@ karatasks.FILE_TRANSFER.stats karatasks.FILE_OPERATION.stats \ karatasks.JOB_SUBMISSION.stats karatasks-total.png \ karatasks.JOB_SUBMISSION.Active-total.png index.html.kara \ +karatasks.JOB_SUBMISSION.Active.sorted-start.png \ +karatasks.JOB_SUBMISSION.Active.stats \ +karatasks.JOB_SUBMISSION.Active-duration-histogram.png \ karatasks.JOB_SUBMISSION.Queue-total.png \ -karatasks.JOB_SUBMISSION.Queue.sorted-start.png +karatasks.JOB_SUBMISSION.Queue.sorted-start.png \ +karatasks.JOB_SUBMISSION.Queue.stats \ +karatasks.JOB_SUBMISSION.Queue-duration-histogram.png webpage.clusters: karatasks.clusters.png karatasks.clusters-total.png \ karatasks.clusters.sorted-start.png karatasks.clusters-duration-histogram.png From noreply at svn.ci.uchicago.edu Tue Sep 23 13:21:31 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 23 Sep 2008 13:21:31 -0500 (CDT) Subject: [Swift-commit] r2247 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20080923182131.607601AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-23 13:21:30 -0500 (Tue, 23 Sep 2008) New Revision: 2247 Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java Log: uninitialised state was missing from list of known states in runtime ticker, causing it to appear at the end of status list rather than its more natural position at the start. Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2008-09-23 18:09:34 UTC (rev 2246) +++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2008-09-23 18:21:30 UTC (rev 2247) @@ -22,6 +22,7 @@ public static final int MAX_PERIOD_MS=60000; public static final String[] preferredOutputOrder = { + "uninitialized", "Initializing", "Selecting site", "Stage in", From noreply at svn.ci.uchicago.edu Tue Sep 23 14:25:22 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 23 Sep 2008 14:25:22 -0500 (CDT) Subject: [Swift-commit] r2248 - in trunk/tests: misc sites/wonky Message-ID: <20080923192522.D06291AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-23 14:25:21 -0500 (Tue, 23 Sep 2008) New Revision: 2248 Added: trunk/tests/misc/wonky-failfirst.sh trunk/tests/sites/wonky/wonky-failfirst.xml Log: test with jobs failing first Added: trunk/tests/misc/wonky-failfirst.sh =================================================================== --- trunk/tests/misc/wonky-failfirst.sh (rev 0) +++ trunk/tests/misc/wonky-failfirst.sh 2008-09-23 19:25:21 UTC (rev 2248) @@ -0,0 +1,16 @@ +#!/bin/bash + +cd ../language-behaviour + +./generate-tc.data + +cat tc.data ../sites/tc.data > tmp.tc.data.sites + +SITE=wonky/wonky-failfirst.xml + +echo testing site configuration: $SITE + +export SWIFT_TEST_PARAMS="-sites.file ../sites/${SITE} -tc.file tmp.tc.data.sites" + +./run 001-echo + Property changes on: trunk/tests/misc/wonky-failfirst.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/sites/wonky/wonky-failfirst.xml =================================================================== --- trunk/tests/sites/wonky/wonky-failfirst.xml (rev 0) +++ trunk/tests/sites/wonky/wonky-failfirst.xml 2008-09-23 19:25:21 UTC (rev 2248) @@ -0,0 +1,11 @@ + + + + + + /var/tmp + 0 + 0 + + + From noreply at svn.ci.uchicago.edu Tue Sep 23 14:29:17 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 23 Sep 2008 14:29:17 -0500 (CDT) Subject: [Swift-commit] r2249 - provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky Message-ID: <20080923192917.6E5441AB0193@www.ci.uchicago.edu> Author: benc Date: 2008-09-23 14:29:16 -0500 (Tue, 23 Sep 2008) New Revision: 2249 Modified: provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java Log: fail-first jobs mode Modified: provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java =================================================================== --- provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java 2008-09-23 19:25:21 UTC (rev 2248) +++ provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java 2008-09-23 19:29:16 UTC (rev 2249) @@ -17,8 +17,10 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Random; import org.apache.log4j.Logger; @@ -45,6 +47,7 @@ static Random r = new Random(); static List firstJobList = Collections.synchronizedList(new ArrayList()); +static Map siteStates = Collections.synchronizedMap(new HashMap()); static int globalJobNumber = 0; int jobNumber = globalJobNumber++; // racy? @@ -383,6 +386,27 @@ boolean failDelay(String name) { double failRate = getWonkyParam(name+"fail"); double delayTime = getWonkyParam(name+"delay"); + double failFirstCount = getWonkyParam(name+"failfirstcount"); + + if(failFirstCount > 0) { + System.out.println("State "+name+" will fail first "+failFirstCount+" times"); + String key = getSiteName() + ":" + name + "failfirstcountsofar"; + Integer iObj = (Integer)siteStates.get(key); + if(iObj == null) { + iObj = new Integer(0); + siteStates.put(key,iObj); + + } + int i = iObj.intValue(); + i++; + siteStates.put(key,new Integer(i)); + System.out.println("new value is "+i); + if(i<=failFirstCount) { // not failed enough + System.out.println("Failing because early job"); + return false; + } + } + System.out.print("State "+name+" with probability of failure "+failRate+": "); if(Math.random()