Function reference +

Function reference This section details functions that are available for use in the SwiftScript language. From noreply at svn.ci.uchicago.edu Sat Jun 7 15:54:32 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 7 Jun 2008 15:54:32 -0500 (CDT) Subject: [Swift-commit] r2055 - in trunk/tests/sites: . broken Message-ID: <20080607205432.3B4509B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-07 15:54:31 -0500 (Sat, 07 Jun 2008) New Revision: 2055 Added: trunk/tests/sites/broken/ trunk/tests/sites/broken/osg-ucsdt2-condor.xml trunk/tests/sites/broken/osg-ucsdt2-fork.xml Log: site definitions for sites that seem broken but are interesting to run tests against Added: trunk/tests/sites/broken/osg-ucsdt2-condor.xml =================================================================== --- trunk/tests/sites/broken/osg-ucsdt2-condor.xml (rev 0) +++ trunk/tests/sites/broken/osg-ucsdt2-condor.xml 2008-06-07 20:54:31 UTC (rev 2055) @@ -0,0 +1,10 @@ + + + + + + + /osgfs/data + + + Added: trunk/tests/sites/broken/osg-ucsdt2-fork.xml =================================================================== --- trunk/tests/sites/broken/osg-ucsdt2-fork.xml (rev 0) +++ trunk/tests/sites/broken/osg-ucsdt2-fork.xml 2008-06-07 20:54:31 UTC (rev 2055) @@ -0,0 +1,10 @@ + + + + + + + /osgfs/data + + + From noreply at svn.ci.uchicago.edu Mon Jun 9 11:07:57 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 9 Jun 2008 11:07:57 -0500 (CDT) Subject: [Swift-commit] r2056 - trunk/tests/sites/broken Message-ID: <20080609160757.3E83F9B4002@www.ci.uchicago.edu> Author: benc Date: 2008-06-09 11:07:56 -0500 (Mon, 09 Jun 2008) New Revision: 2056 Added: trunk/tests/sites/broken/osg-ucsdt2-b2-condor.xml trunk/tests/sites/broken/osg-ucsdt2-b2-fork.xml Log: more suspect sites Added: trunk/tests/sites/broken/osg-ucsdt2-b2-condor.xml =================================================================== --- trunk/tests/sites/broken/osg-ucsdt2-b2-condor.xml (rev 0) +++ trunk/tests/sites/broken/osg-ucsdt2-b2-condor.xml 2008-06-09 16:07:56 UTC (rev 2056) @@ -0,0 +1,9 @@ + + + + + /osgfs/data/lixi + .542 + 0.05 + + Added: trunk/tests/sites/broken/osg-ucsdt2-b2-fork.xml =================================================================== --- trunk/tests/sites/broken/osg-ucsdt2-b2-fork.xml (rev 0) +++ trunk/tests/sites/broken/osg-ucsdt2-b2-fork.xml 2008-06-09 16:07:56 UTC (rev 2056) @@ -0,0 +1,9 @@ + + + + + /osgfs/data/lixi + .542 + 0.05 + + From noreply at svn.ci.uchicago.edu Wed Jun 11 16:40:11 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 11 Jun 2008 16:40:11 -0500 (CDT) Subject: [Swift-commit] r2057 - / Message-ID: <20080611214011.4E8629B4001@www.ci.uchicago.edu> Author: houzx Date: 2008-06-11 16:40:09 -0500 (Wed, 11 Jun 2008) New Revision: 2057 Added: gasateria/ Log: Making a new dir. From noreply at svn.ci.uchicago.edu Thu Jun 12 15:57:23 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 12 Jun 2008 15:57:23 -0500 (CDT) Subject: [Swift-commit] r2059 - gasateria/bin Message-ID: <20080612205723.1D89C9B4001@www.ci.uchicago.edu> Author: houzx Date: 2008-06-12 15:57:22 -0500 (Thu, 12 Jun 2008) New Revision: 2059 Removed: gasateria/bin/update-sites-cache- Log: Deleting file 'update-sites-cache-' Deleted: gasateria/bin/update-sites-cache- =================================================================== --- gasateria/bin/update-sites-cache- 2008-06-11 21:43:56 UTC (rev 2058) +++ gasateria/bin/update-sites-cache- 2008-06-12 20:57:22 UTC (rev 2059) @@ -1,59 +0,0 @@ -#/bin/bash -CDATE=`date +%F-%R` -DIR=`pwd` -USER=`whoami` -# get the grid computing sites -GRID=$1 -VO=$2 -if [ "$GRID" == "" ]; then - echo "Please input the GRID name as the first parameter." - echo "The GRID can be:osg osg-itb (teragrid?)" - echo "If necessary, please input the virtual organization name as the second parameter.Just use a space to separate the two parameters." - echo "For osg, vo can be:CDF CMS CompBioGrid DES DOSAR DZero Engage Fermilab fMRI GADU geant4 GLOW GPN GRASE GridChem GridEx GROW i2u2 iVDGL LIGO mariachi MIS nanoHUB NWICG Ops OSG OSGEDU SDSS STAR USATLAS" - exit -fi - - if [ "$VO" == "" ]; then - { - /usr/bin/wget -O $DIR/tmp/$VO-vo-sitelist.txt "http://vors.grid.iu.edu/cgi-bin/tindex.cgi?grid=$GRID" 1>/dev/null 2>/dev/null - } - else - { - /usr/bin/wget -O $DIR/tmp/$VO-vo-sitelist.txt "http://vors.grid.iu.edu/cgi-bin/tindex.cgi?grid=$GRID&VO=$VO" 1>/dev/null 2>/dev/null - } - fi - -/bin/cat $DIR/tmp/$VO-vo-sitelist.txt |grep compute |grep PASS |awk -F, {'print $3'} > $DIR/tmp/$GRID-$VO-compute-sites.txt - -#touch sites-$CDATE.txt -#for i in $(cat $DIR/tmp/$GRID-$VO-compute-sites.txt) -for i in $(/bin/awk -F: '{print $1}' $DIR/tmp/$GRID-$VO-compute-sites.txt) -do - echo $i >> $DIR/tmp/sites-$CDATE.txt - globusrun -a -r $i/jobmanager-fork >> $DIR/tmp/sites-$CDATE.txt -done - -if [ -f $DIR/tmp/sites-$CDATE.txt ]; then - m=0 - for i in $(sed -n '/success/=' $DIR/tmp/sites-$CDATE.txt) -do - #echo $i - m=`expr $m + 1` - j=`expr $i - 2` - S=`sed -n "$j"p $DIR/tmp/sites-$CDATE.txt` - echo $S >> $DIR/tmp/$GRID-$VO-sites-success-$CDATE.txt -done - echo " Now it's `date +%D-%R`.Altogether, there are $m successful available grid sites." -fi - -#If there is new sites, update sites cache -if [ $m > 24 ]; then - for SITE in $(cat $DIR/tmp/$GRID-$VO-sites-success-$CDATE.txt) - do - /bin/grep $SITE $DIR/.osg-sites.cache >> $DIR/tmp/$GRID-$VO-avail-sites-$CDATE.txt - done - -fi - -#clean the files -/bin/rm -rf $DIR/tmp/sites-$CDATE.txt $DIR/tmp/$GRID-$VO-sites-success-$CDATE.txt From noreply at svn.ci.uchicago.edu Sun Jun 15 06:31:04 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 06:31:04 -0500 (CDT) Subject: [Swift-commit] r2060 - log-processing Message-ID: <20080615113104.6F7C29B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 06:31:01 -0500 (Sun, 15 Jun 2008) New Revision: 2060 Modified: log-processing/makefile Log: another file to clean Modified: log-processing/makefile =================================================================== --- log-processing/makefile 2008-06-12 20:57:22 UTC (rev 2059) +++ log-processing/makefile 2008-06-15 11:31:01 UTC (rev 2060) @@ -16,7 +16,7 @@ cp *.event *.html *.png report-$(SDL) clean: - rm -f start-times.data kickstart-times.data start-time.tmp end-time.tmp threads.list tasks.list log *.data *.shifted *.png *.event *.coloured-event *.total *.tmp *.transitions *.last karatasks-type-counts.txt index.html *.lastsummary execstages.plot total.plot colour.plot jobs-sites.table jobs.retrycount.summary kickstart.stats execution-counts.txt site-duration.txt jobs.retrycount sp.plot karatasks.coloured-sorted-event *.cedps *.stats t.inf *.seenstates tmp-* info.* clusterstats trname-summary sites-list.data.nm info-md5sums + rm -f start-times.data kickstart-times.data start-time.tmp end-time.tmp threads.list tasks.list log *.data *.shifted *.png *.event *.coloured-event *.total *.tmp *.transitions *.last karatasks-type-counts.txt index.html *.lastsummary execstages.plot total.plot colour.plot jobs-sites.table jobs.retrycount.summary kickstart.stats execution-counts.txt site-duration.txt jobs.retrycount sp.plot karatasks.coloured-sorted-event *.cedps *.stats t.inf *.seenstates tmp-* info.* clusterstats trname-summary sites-list.data.nm info-md5sums pse2d-tmp.eip t.inf: $(LOG) ./compute-t-inf > t.inf < $(LOG) From noreply at svn.ci.uchicago.edu Sun Jun 15 07:16:26 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 07:16:26 -0500 (CDT) Subject: [Swift-commit] r2061 - in log-processing: . bin Message-ID: <20080615121626.26EF69B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 07:16:25 -0500 (Sun, 15 Jun 2008) New Revision: 2061 Added: log-processing/bin/ log-processing/bin/swift-plot-log Log: command to generate webpage for specified log Added: log-processing/bin/swift-plot-log =================================================================== --- log-processing/bin/swift-plot-log (rev 0) +++ log-processing/bin/swift-plot-log 2008-06-15 12:16:25 UTC (rev 2061) @@ -0,0 +1,28 @@ +#!/bin/bash + +# $1 should be the pathname of the log file to plot + +LOG_CODE_HOME="`dirname $0`/../" + +FC="${1:0:1}" + +if [ "$FC" = "/" ]; then # absolute path + LOG_FILE_PATH=$1 +else + LOG_FILE_PATH="`pwd`/$1" +fi + +LOG_DIRECTORY=`dirname $LOG_FILE_PATH` + +LOG_FILE_BASE=`basename $LOG_FILE_PATH .log` + +echo Log file path is $LOG_FILE_PATH +echo Log is in directory $LOG_DIRECTORY +echo Log basename is $LOG_FILE_BASE + + +cd $LOG_CODE_HOME + +make SDL=$LOG_FILE_BASE LOG=$LOG_FILE_PATH clean webpage distributable distribute + +mv report-$LOG_FILE_BASE $LOG_DIRECTORY/ Property changes on: log-processing/bin/swift-plot-log ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Sun Jun 15 07:49:17 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 07:49:17 -0500 (CDT) Subject: [Swift-commit] r2062 - log-processing Message-ID: <20080615124917.D0D5D9B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 07:49:17 -0500 (Sun, 15 Jun 2008) New Revision: 2062 Modified: log-processing/index.html.template Log: move more interesting graph to the front of the report Modified: log-processing/index.html.template =================================================================== --- log-processing/index.html.template 2008-06-15 12:16:25 UTC (rev 2061) +++ log-processing/index.html.template 2008-06-15 12:49:17 UTC (rev 2062) @@ -3,37 +3,26 @@

Swift log analysis

This is an automatically generated post-mortem log file -analysis for Swift and related tools.

(TODO indicate log file name here; indicate report generation time here; indicate version of swift that was used - if locally modified print a big red warning)

1. Whole workflow
2. execute - application calls from SwiftScript
3. execute2 - attempts by Swift to invoke applications
4. kickstart - records from the worker nodes
5. karajan - statistics on the guts of the execution engine
6. falkon
7. per-site statistics

Summary of execution

This plot shows the state of executions over time. When a SwiftScript +program makes a procedure call, an execution will appear on this plot +in START state. Over time, jobs should end up either in END_SUCCESS or +END_FAILURE states. +

In this document, text formatted like this is boilerplate explanatory -text and will not change from run to run...

... whilst text formatted like this is generated based on the
-supplied run log files.
+Final status totals:
++include(`execute.lastsummary')
 
 
-Workflow
 
-The workflow itself
-(text file of event(s))
- - there should be just one of them and it should
-take the whole span of execution time:
-
-
+
 execute
 Executions 
 (text file of events)
@@ -286,5 +275,14 @@
 

 5. 
 

+
+Workflow
+
+The workflow itself
+(text file of event(s))
+ - there should be just one of them and it should
+take the whole span of execution time:
+
+
 end
 



From noreply at svn.ci.uchicago.edu  Sun Jun 15 09:42:06 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 15 Jun 2008 09:42:06 -0500 (CDT)
Subject: [Swift-commit] r2063 - log-processing
Message-ID: <20080615144206.3B8869B4001@www.ci.uchicago.edu>

Author: benc
Date: 2008-06-15 09:42:05 -0500 (Sun, 15 Jun 2008)
New Revision: 2063

Modified:
   log-processing/index.html.template
Log:
more rearrangement of report page

Modified: log-processing/index.html.template
===================================================================
--- log-processing/index.html.template	2008-06-15 12:49:17 UTC (rev 2062)
+++ log-processing/index.html.template	2008-06-15 14:42:05 UTC (rev 2063)
@@ -12,7 +12,8 @@
 This plot shows the state of executions over time. When a SwiftScript
 program makes a procedure call, an execution will appear on this plot
 in START state. Over time, jobs should end up either in END_SUCCESS or
-END_FAILURE states.
+END_FAILURE states. (More information about
+execute events)
 
 
 
@@ -21,7 +22,17 @@
 include(`execute.lastsummary')

Breakdown of execute2 final statuses by site: +(More information about execute2 events)

+include(`jobs-sites.table') + + +

Number of execute2 jobs at once - this is basically the number of +execution attempts that have been submitted to some batch system such +as GRAM (so are either queued or executing)

execute

Executions From noreply at svn.ci.uchicago.edu Sun Jun 15 12:10:25 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 12:10:25 -0500 (CDT) Subject: [Swift-commit] r2064 - log-processing/bin Message-ID: <20080615171025.2084D9B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 12:10:24 -0500 (Sun, 15 Jun 2008) New Revision: 2064 Modified: log-processing/bin/swift-plot-log Log: make karajan and scheduler plots Modified: log-processing/bin/swift-plot-log =================================================================== --- log-processing/bin/swift-plot-log 2008-06-15 14:42:05 UTC (rev 2063) +++ log-processing/bin/swift-plot-log 2008-06-15 17:10:24 UTC (rev 2064) @@ -23,6 +23,6 @@ cd $LOG_CODE_HOME -make SDL=$LOG_FILE_BASE LOG=$LOG_FILE_PATH clean webpage distributable distribute +make SDL=$LOG_FILE_BASE LOG=$LOG_FILE_PATH clean webpage.kara webpage.weights webpage distributable distribute mv report-$LOG_FILE_BASE $LOG_DIRECTORY/ From noreply at svn.ci.uchicago.edu Sun Jun 15 12:10:55 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 12:10:55 -0500 (CDT) Subject: [Swift-commit] r2065 - log-processing Message-ID: <20080615171055.CDB7D9B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 12:10:55 -0500 (Sun, 15 Jun 2008) New Revision: 2065 Modified: log-processing/table-jobs-sites Log: right-align numbers in site table Modified: log-processing/table-jobs-sites =================================================================== --- log-processing/table-jobs-sites 2008-06-15 17:10:24 UTC (rev 2064) +++ log-processing/table-jobs-sites 2008-06-15 17:10:55 UTC (rev 2065) @@ -6,7 +6,7 @@ STATES="JOB_START JOB_END APPLICATION_EXCEPTION" -echo "" +echo "
" echo " " @@ -24,12 +24,12 @@ ACC=0 -echo "" +echo "" for status in $STATES; do count=$(grep $status execute2.event | grep $site | wc -l) ACC=$(( $ACC + $count )) -echo "" +echo "" done count=$(grep $site execute2.event | wc -l) From noreply at svn.ci.uchicago.edu Sun Jun 15 12:11:33 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 12:11:33 -0500 (CDT) Subject: [Swift-commit] r2066 - log-processing Message-ID: <20080615171133.5828C9B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 12:11:33 -0500 (Sun, 15 Jun 2008) New Revision: 2066 Added: log-processing/active-state-transitions Modified: log-processing/index.html.template log-processing/makefile.karatasks log-processing/makefile.webpage Log: plot of jobs in active state Added: log-processing/active-state-transitions =================================================================== --- log-processing/active-state-transitions (rev 0) +++ log-processing/active-state-transitions 2008-06-15 17:11:33 UTC (rev 2066) @@ -0,0 +1,34 @@ +#!/usr/bin/env perl + +# pull out transitions that go to active state and the immediately +# following transition + +use Time::Local; + +$prevjob = "active-state-transitions no previous event"; + +$prevline = ""; + +$active = 0; + +foreach $n () { + $n =~ /^([^ ]*) ([^ ]*) ([^ ]*) (.*)$/; + $ts = $1; + $job = $2; + $state = $3; + + if($state eq "Active") { + print "$n"; + $active = 1; + } elsif( ($job eq $prevjob) && ($active) ) { + if($active == 1) { + print $n; + $active = 0; + } + } else { + $prevjob = $job; + }; + $prevline = $n; +} + +# print $prevline; Property changes on: log-processing/active-state-transitions ___________________________________________________________________ Name: svn:executable + * Modified: log-processing/index.html.template =================================================================== --- log-processing/index.html.template 2008-06-15 17:10:55 UTC (rev 2065) +++ log-processing/index.html.template 2008-06-15 17:11:33 UTC (rev 2066) @@ -33,6 +33,11 @@ as GRAM (so are either queued or executing)
+
Number of karajan level job submissions that are 'Active' - this means +that the underlying execution layer (such as GRAM) has reported that the +job is actually executing
+ +

execute

Executions Modified: log-processing/makefile.karatasks =================================================================== --- log-processing/makefile.karatasks 2008-06-15 17:10:55 UTC (rev 2065) +++ log-processing/makefile.karatasks 2008-06-15 17:11:33 UTC (rev 2066) @@ -33,3 +33,5 @@ karatasks.JOB_SUBMISSION.transitions: karatasks.transitions ./karatasks-only JOB_SUBMISSION > $@ +karatasks.JOB_SUBMISSION.Active.transitions: karatasks.JOB_SUBMISSION.transitions + cat karatasks.JOB_SUBMISSION.transitions | ./swap-and-sort-and-swap | ./active-state-transitions > karatasks.JOB_SUBMISSION.Active.transitions Modified: log-processing/makefile.webpage =================================================================== --- log-processing/makefile.webpage 2008-06-15 17:10:55 UTC (rev 2065) +++ log-processing/makefile.webpage 2008-06-15 17:11:33 UTC (rev 2066) @@ -9,7 +9,8 @@ karatasks.FILE_OPERATION-total.png karatasks.JOB_SUBMISSION-trails.png \ karatasks.FILE_OPERATION-trails.png karatasks.FILE_TRANSFER-trails.png \ karatasks.FILE_TRANSFER.stats karatasks.FILE_OPERATION.stats \ -karatasks.JOB_SUBMISSION.stats karatasks-total.png index.html.kara +karatasks.JOB_SUBMISSION.stats karatasks-total.png \ +karatasks.JOB_SUBMISSION.Active-total.png index.html.kara webpage.clusters: karatasks.clusters.png karatasks.clusters-total.png \ karatasks.clusters.sorted-start.png karatasks.clusters-duration-histogram.png From noreply at svn.ci.uchicago.edu Sun Jun 15 14:51:16 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 14:51:16 -0500 (CDT) Subject: [Swift-commit] r2067 - in trunk: libexec src/org/griphyn/vdl/type Message-ID: <20080615195117.04DE99B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 14:51:16 -0500 (Sun, 15 Jun 2008) New Revision: 2067 Modified: trunk/libexec/vdl-int.k trunk/src/org/griphyn/vdl/type/Types.java Log: minor whitespace fixes Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2008-06-15 17:11:33 UTC (rev 2066) +++ trunk/libexec/vdl-int.k 2008-06-15 19:51:16 UTC (rev 2067) @@ -246,9 +246,9 @@ task:transfer(srchost=host, srcfile=bname, srcdir=rdir, destdir=ldir, desthost=dhost, destprovider=provider) ) - vdl:logvar(var, path, host, dir, bname) - log(LOG:DEBUG, "FILE_STAGE_OUT_END srcname={bname} srcdir={rdir} srchost={host} ", - "destdir={ldir} desthost={dhost} provider={provider}") + vdl:logvar(var, path, host, dir, bname) + log(LOG:DEBUG, "FILE_STAGE_OUT_END srcname={bname} srcdir={rdir} srchost={host} ", + "destdir={ldir} desthost={dhost} provider={provider}") list(bname, rdir, host, file:size(fullLocal)) ) Modified: trunk/src/org/griphyn/vdl/type/Types.java =================================================================== --- trunk/src/org/griphyn/vdl/type/Types.java 2008-06-15 17:11:33 UTC (rev 2066) +++ trunk/src/org/griphyn/vdl/type/Types.java 2008-06-15 19:51:16 UTC (rev 2067) @@ -52,11 +52,11 @@ private static Type addPrimitiveType(String name) { Type type = Type.Factory.createType(name, true); addType(type); - return type; + return type; } - - public static final Type INT, STRING, FLOAT, BOOLEAN, ANY; + public static final Type INT, STRING, FLOAT, BOOLEAN, ANY; + // add built-in primitive types static { STRING = addPrimitiveType("string"); From noreply at svn.ci.uchicago.edu Sun Jun 15 15:35:54 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 15:35:54 -0500 (CDT) Subject: [Swift-commit] r2068 - trunk/tests/misc Message-ID: <20080615203554.529819B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 15:35:53 -0500 (Sun, 15 Jun 2008) New Revision: 2068 Added: trunk/tests/misc/restart-iterate.sh Modified: trunk/tests/misc/run Log: test of iterate with restarts Added: trunk/tests/misc/restart-iterate.sh =================================================================== --- trunk/tests/misc/restart-iterate.sh (rev 0) +++ trunk/tests/misc/restart-iterate.sh 2008-06-15 20:35:53 UTC (rev 2068) @@ -0,0 +1,95 @@ +#!/bin/bash + +export CF=swift.properties.restart-iterate +cat $(dirname $(which swift))/../etc/swift.properties | grep --invert-match -E '^lazy.errors=' > $CF +echo lazy.errors=true >> $CF + +rm -f *.rlog restart-*.out restart-iterate.kml restart-iterate.xml restart-iterate.*.out restart-iterate.*.out.single-run + +rm -rf _concurrent + +echo "localhost helperA $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" > tmp.restartOK.tc.data +echo "localhost helperB $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartOK.tc.data +echo "localhost helperC $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartOK.tc.data + +swift -config $CF -tc.file tmp.restartOK.tc.data restart-iterate.swift + +PRECHECKEXIT=$? + + +if [ "$PRECHECKEXIT" != 0 ]; then + echo Failed - attempt to run workflow without restart configuration failed + exit 1 +fi + +for fn in restart-iterate.*.out; do +mv $fn ${fn}.single-run +done + +rm -f *.rlog restart-*.out restart-iterate.kml restart-iterate.xml +rm -rf _concurrent + +echo "localhost helperA $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" > tmp.restartA.tc.data +echo "localhost helperB $(pwd)/restart5-helper-fail INSTALLED INTEL32::LINUX null" >> tmp.restartA.tc.data +echo "localhost helperC $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartA.tc.data + +swift -config $CF -tc.file tmp.restartA.tc.data restart-iterate.swift + +FIRSTEXIT=$? + +# this invocation should fail, with restart-1.out in existence but +# not the others + +if [ "$FIRSTEXIT" == 0 ]; then + echo Failed - workflow was indicated as successfully completed the first time round. + exit 2 +fi + +for i in 0 1 2 3 4 ; do + if [ ! -f restart-iterate.000$i.out ] ; then + echo Some first-time output files were missing + exit 4 + fi +done + +for i in 05 06 07 08 09 10 ; do + if [ -f restart-iterate.00$i.out ] ; then + echo Second-time output file appeared in first-time run + exit 5 + fi +done + +# now make A fail - we should have run it already, and so we want to make +# sure it does not run again; and make B succeed this time round. + +echo "localhost helperA $(pwd)/restart5-helper-fail INSTALLED INTEL32::LINUX null" > tmp.restartB.tc.data +echo "localhost helperB $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartB.tc.data +echo "localhost helperC $(pwd)/restart5-helper-success INSTALLED INTEL32::LINUX null" >> tmp.restartB.tc.data + +# there should be only a single rlog here, because we deleted them all +# at the start of this script. +swift -config $CF -resume *.rlog -tc.file tmp.restartB.tc.data restart-iterate.swift + +SECONDEXIT=$? + +if [ "$SECONDEXIT" != "0" ]; then + echo Failed - second round failed + exit 3 +fi + +if [ ! -f restart-iterate.0002.out ] ; then + echo Second-time output file missing + exit 5 +fi + +for i in 1 2 3; do + diff -q restart-iterate.000$i.out restart-iterate.000$i.out.single-run + if [ "$?" -ne "0" ]; then + echo restart-based output file $i differs from non-restarted output file + exit 6 + fi +done + +echo restart-iterate success +exit 0 + Property changes on: trunk/tests/misc/restart-iterate.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/misc/run =================================================================== --- trunk/tests/misc/run 2008-06-15 19:51:16 UTC (rev 2067) +++ trunk/tests/misc/run 2008-06-15 20:35:53 UTC (rev 2068) @@ -1,5 +1,5 @@ #!/bin/sh -for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 workernode-local; do +for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 restart-iterate workernode-local; do ./${a}.sh R=$? echo test $a ended with return value $R From noreply at svn.ci.uchicago.edu Sun Jun 15 16:03:09 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 16:03:09 -0500 (CDT) Subject: [Swift-commit] r2069 - trunk/libexec Message-ID: <20080615210309.8F7B39B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 16:03:09 -0500 (Sun, 15 Jun 2008) New Revision: 2069 Modified: trunk/libexec/vdl-int.k Log: my sick mind picks up the wrong meaning from this function name and makes me laugh; so I renamed it Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2008-06-15 20:35:53 UTC (rev 2068) +++ trunk/libexec/vdl-int.k 2008-06-15 21:03:09 UTC (rev 2069) @@ -298,7 +298,7 @@ ) ) - element(transferSTDs, [rhost, tmpdir, jobid, stdout, stderr] + element(transferStandardFiles, [rhost, tmpdir, jobid, stdout, stderr] concat( for(f, list(list("stderr.txt", stderr), list("stdout.txt", stdout)) [name, file] := each(f) @@ -443,7 +443,7 @@ cleanupFiles(cacheFilesToRemove, rhost) ) - outs := transferSTDs(rhost, tmpdir, jobid, stdout, stderr) + outs := transferStandardFiles(rhost, tmpdir, jobid, stdout, stderr) discard(transferWrapperLog(rhost, wfdir, jobid, jobdir)) kickstartRec := if( From noreply at svn.ci.uchicago.edu Sun Jun 15 16:10:27 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 16:10:27 -0500 (CDT) Subject: [Swift-commit] r2070 - trunk/src/org/griphyn/vdl/engine Message-ID: <20080615211027.65F529B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 16:10:26 -0500 (Sun, 15 Jun 2008) New Revision: 2070 Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java trunk/src/org/griphyn/vdl/engine/VariableScope.java Log: remove old constructor that has been superceded Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Karajan.java 2008-06-15 21:03:09 UTC (rev 2069) +++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2008-06-15 21:10:26 UTC (rev 2070) @@ -183,7 +183,7 @@ public void procedure(Procedure proc, VariableScope containingScope) throws CompilationException { VariableScope outerScope = new VariableScope(this, null); - VariableScope innerScope = new VariableScope(this, outerScope, false); + VariableScope innerScope = new VariableScope(this, outerScope, VariableScope.ENCLOSURE_NONE); StringTemplate procST = template("procedure"); containingScope.bodyTemplate.setAttribute("procedures", procST); procST.setAttribute("name", proc.getName()); Modified: trunk/src/org/griphyn/vdl/engine/VariableScope.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/VariableScope.java 2008-06-15 21:03:09 UTC (rev 2069) +++ trunk/src/org/griphyn/vdl/engine/VariableScope.java 2008-06-15 21:10:26 UTC (rev 2070) @@ -42,39 +42,13 @@ public StringTemplate bodyTemplate; public VariableScope(Karajan c, VariableScope parent) { - this(c,parent,true); + this(c,parent,ENCLOSURE_ALL); } - /** Creates a new variable scope. @param c the compiler scope in which this variable lives @param parent the enclosing scope, or null if this is a top level scope - @param a if true, assignments made in this scope - to variables in an enclosing scope will be permitted. if false, - this will be prohibited, which is desirable in loops to prohibit - multiple assignment - @deprecated use explicit upwards parameter rather than boolean - */ - public VariableScope(Karajan c, VariableScope parent, boolean a) { - if(parentScope!=null) { - logger.info("New scope "+hashCode()+" with parent scope "+parentScope.hashCode()); - } else { - logger.info("New scope "+hashCode()+" with no parent."); - } - compiler = c; - parentScope = parent; - if(a) { - enclosureType = ENCLOSURE_ALL; - } else { - enclosureType = ENCLOSURE_NONE; - } - } - - /** Creates a new variable scope. - @param c the compiler scope in which this variable lives - @param parent the enclosing scope, or null if this is a - top level scope @param a specifies how assignments to variables made in enclosing scopes will be handled. */ From noreply at svn.ci.uchicago.edu Sun Jun 15 18:23:22 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 18:23:22 -0500 (CDT) Subject: [Swift-commit] r2071 - trunk/tests/misc Message-ID: <20080615232322.35A149B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 18:23:21 -0500 (Sun, 15 Jun 2008) New Revision: 2071 Added: trunk/tests/misc/restart-iterate.swift Log: forgot to add this file for test Added: trunk/tests/misc/restart-iterate.swift =================================================================== --- trunk/tests/misc/restart-iterate.swift (rev 0) +++ trunk/tests/misc/restart-iterate.swift 2008-06-15 23:23:21 UTC (rev 2071) @@ -0,0 +1,40 @@ +type file; + +(file t) a(file i) { + app { + helperA @t @i "IN A"; + } +} + +(file t) b(file i) { + app { + helperB @t @i "IN B"; + } +} + +(file t) c(file i) { + app { + helperC @t @i "IN C"; + } +} + +(file r) q(file i, int n) { + file t; + if(n<5) { t=a(i); } else { t=b(i); } + r=c(t); +} + +file J <"restart.in">; + +file X[]; + +file Y[] ; + +iterate iv { + X[iv] = q(J,iv); +} until(iv>10); + +foreach x,j in X { + Y[j] = q(x,j); +} + From noreply at svn.ci.uchicago.edu Sun Jun 15 18:38:44 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 18:38:44 -0500 (CDT) Subject: [Swift-commit] r2072 - trunk/libexec Message-ID: <20080615233844.B66979B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 18:38:44 -0500 (Sun, 15 Jun 2008) New Revision: 2072 Modified: trunk/libexec/wrapper.sh Log: log command line arguments in wrapper Modified: trunk/libexec/wrapper.sh =================================================================== --- trunk/libexec/wrapper.sh 2008-06-15 23:23:21 UTC (rev 2071) +++ trunk/libexec/wrapper.sh 2008-06-15 23:38:44 UTC (rev 2072) @@ -22,6 +22,8 @@ cat /proc/cpuinfo 2>&1 >& "$INFO" infosection "/proc/meminfo" cat /proc/meminfo 2>&1 >& "$INFO" + infosection "command line" + echo $COMMANDLINE 2>&1 >& "$INFO" } logstate() { @@ -84,6 +86,7 @@ exec 3>&- } +COMMANDLINE=$@ WFDIR=$PWD openinfo "wrapper.log" ID=$1 From noreply at svn.ci.uchicago.edu Sun Jun 15 18:41:16 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 18:41:16 -0500 (CDT) Subject: [Swift-commit] r2073 - trunk/tests/sites/coaster Message-ID: <20080615234116.6E3629B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 18:41:16 -0500 (Sun, 15 Jun 2008) New Revision: 2073 Added: trunk/tests/sites/coaster/fletch-coaster-gram2-gram2-condor.xml trunk/tests/sites/coaster/fletch-coaster-gram2-gram2-fork.xml Log: two coaster tests for the BSD cluster. fork works; condor fails with a command-line truncation problem in the middle job of 130-fmri Added: trunk/tests/sites/coaster/fletch-coaster-gram2-gram2-condor.xml =================================================================== --- trunk/tests/sites/coaster/fletch-coaster-gram2-gram2-condor.xml (rev 0) +++ trunk/tests/sites/coaster/fletch-coaster-gram2-gram2-condor.xml 2008-06-15 23:41:16 UTC (rev 2073) @@ -0,0 +1,9 @@ + + + + + + /disks/gpfs/swift/site-test + + + Added: trunk/tests/sites/coaster/fletch-coaster-gram2-gram2-fork.xml =================================================================== --- trunk/tests/sites/coaster/fletch-coaster-gram2-gram2-fork.xml (rev 0) +++ trunk/tests/sites/coaster/fletch-coaster-gram2-gram2-fork.xml 2008-06-15 23:41:16 UTC (rev 2073) @@ -0,0 +1,9 @@ + + + + + + /disks/gpfs/swift/site-test + + + From noreply at svn.ci.uchicago.edu Sun Jun 15 18:45:15 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 15 Jun 2008 18:45:15 -0500 (CDT) Subject: [Swift-commit] r2074 - trunk/tests/sites Message-ID: <20080615234515.DCB799B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-15 18:45:15 -0500 (Sun, 15 Jun 2008) New Revision: 2074 Modified: trunk/tests/sites/run-site Log: restore running lots of tests on a site - this had got removed accidentally Modified: trunk/tests/sites/run-site =================================================================== --- trunk/tests/sites/run-site 2008-06-15 23:41:16 UTC (rev 2073) +++ trunk/tests/sites/run-site 2008-06-15 23:45:15 UTC (rev 2074) @@ -12,5 +12,5 @@ export SWIFT_TEST_PARAMS="-sites.file ../sites/${SITE} -tc.file tmp.tc.data.sites" -#./run 061-cattwo 130-fmri 103-quote.swift 1032-singlequote.swift 1031-quote.swift 1033-singlequote.swift 141-space-in-filename 142-space-and-quotes -./run 061-cattwo +./run 061-cattwo 130-fmri 103-quote.swift 1032-singlequote.swift 1031-quote.swift 1033-singlequote.swift 141-space-in-filename 142-space-and-quotes +#./run 061-cattwo From noreply at svn.ci.uchicago.edu Mon Jun 16 09:35:09 2008 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 16 Jun 2008 09:35:09 -0500 (CDT) Subject: [Swift-commit] r2075 - provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky Message-ID: <20080616143509.0EA5F9B4001@www.ci.uchicago.edu> Author: benc Date: 2008-06-16 09:35:08 -0500 (Mon, 16 Jun 2008) New Revision: 2075 Modified: provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java Log: more delay parameters Modified: provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java =================================================================== --- provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java 2008-06-15 23:45:15 UTC (rev 2074) +++ provider-wonky/src/org/globus/cog/abstraction/impl/execution/wonky/JobSubmissionTaskHandler.java 2008-06-16 14:35:08 UTC (rev 2075) @@ -13,9 +13,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Random; import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.StatusImpl; @@ -29,22 +33,26 @@ import org.globus.cog.abstraction.interfaces.DelegatedTaskHandler; import org.globus.cog.abstraction.interfaces.FileLocation; import org.globus.cog.abstraction.interfaces.JobSpecification; +import org.globus.cog.abstraction.interfaces.ServiceContact; import org.globus.cog.abstraction.interfaces.Status; import org.globus.cog.abstraction.interfaces.Task; -/** - * @author Kaizar Amin (amin at mcs.anl.gov) - * - */ public class JobSubmissionTaskHandler implements DelegatedTaskHandler, Runnable { private static Logger logger = Logger .getLogger(JobSubmissionTaskHandler.class); -static boolean firstJob = true; +static Random r = new Random(); + +static List firstJobList = Collections.synchronizedList(new ArrayList()); + static int globalJobNumber = 0; -int jobNumber = globalJobNumber++; // racy +int jobNumber = globalJobNumber++; // racy? +List siteOptions; + +String siteName; + public static final int BUFFER_SIZE = 1024; private Task task = null; @@ -63,8 +71,25 @@ throw new TaskSubmissionException("Task is not in unsubmitted state"); } else { -System.out.println("Submitting wonky job "+jobNumber); +logger.info("Submitting wonky job "+jobNumber); this.task = task; + + + ServiceContact serviceContact = this.task.getService(0) + .getServiceContact(); + String server = serviceContact.getContact(); + +logger.debug("Contact is: "+server); + String[] splitServer = server.split("/"); + siteName = splitServer[0]; + siteOptions = Arrays.asList(splitServer); + + + + if(!failDelay("submitting")) { + this.task.setStatus(Status.FAILED); + return; + } task.setStatus(Status.SUBMITTING); JobSpecification spec; try { @@ -85,6 +110,10 @@ synchronized(this) { this.thread = new Thread(this); if (this.task.getStatus().getStatusCode() != Status.CANCELED) { + if(!failDelay("submitted")) { + this.task.setStatus(Status.FAILED); + return; + } this.task.setStatus(Status.SUBMITTED); this.thread.start(); if (spec.isBatchJob()) { @@ -100,17 +129,17 @@ public void suspend() throws InvalidSecurityContextException, TaskSubmissionException { -System.out.println("Suspend called on wonky job "+jobNumber); +logger.info("Suspend called on wonky job "+jobNumber); } public void resume() throws InvalidSecurityContextException, TaskSubmissionException { -System.out.println("Resume called on wonky job "+jobNumber); +logger.info("Resume called on wonky job "+jobNumber); } public void cancel() throws InvalidSecurityContextException, TaskSubmissionException { -System.out.println("Cancel called on wonky job "+jobNumber); +logger.info("Cancel called on wonky job "+jobNumber); synchronized(this) { killed = true; process.destroy(); @@ -123,30 +152,52 @@ public void run() { try { -System.out.println("Wonky job in queue, job number "+jobNumber); + JobSpecification spec = (JobSpecification) this.task + .getSpecification(); + + File dir = null; + if (spec.getDirectory() != null) { + dir = new File(spec.getDirectory()); + } +logger.info("Wonky job in queue, job number "+jobNumber); +String[] cmdarray = buildCmdArray(spec, jobNumber); try { -if(firstJob) { -System.out.println("not sleeping - this is the first job, "+jobNumber); -firstJob=false; // mmmm racy +logger.info("wonky site is "+siteName); + if(siteOptions.contains("firstfast") && !firstJobList.contains(siteName)) { +logger.info("not sleeping - this is the first job on site "+siteName+", "+jobNumber); +firstJobList.add(siteName); } else { -Thread.sleep(120*1000); +double qmean=0; +double qstddev=0; + +Iterator it = siteOptions.iterator(); +while(it.hasNext()) { +String op = (String) it.next(); +logger.debug("Checking: "+op); +if(op.startsWith("qmean=")) { +qmean = Double.parseDouble(op.substring(6)); +logger.debug("qmean = "+qmean); +} else if(op.startsWith("qstddev=")) { +qstddev = Double.parseDouble(op.substring(8)); +logger.debug("qstddev = "+qstddev); } +} + +double qdelay = qmean + qstddev * r.nextGaussian(); + +Thread.sleep((long)(qdelay*1000)); +} } catch(InterruptedException ie) { System.out.println("fake queue delay interrupted for job "+jobNumber); } System.out.println("Wonky job running now"); - // TODO move away from the multi-threaded approach - JobSpecification spec = (JobSpecification) this.task - .getSpecification(); - - File dir = null; - if (spec.getDirectory() != null) { - dir = new File(spec.getDirectory()); + process = Runtime.getRuntime().exec(cmdarray, + buildEnvp(spec), dir); + if(!failDelay("active")) { + this.task.setStatus(Status.FAILED); + return; } - - process = Runtime.getRuntime().exec(buildCmdArray(spec), - buildEnvp(spec), dir); this.task.setStatus(Status.ACTIVE); // reusable byte buffer @@ -195,7 +246,11 @@ } System.err.println("Wonky job completed with exitCode "+exitCode); if (exitCode == 0) { - this.task.setStatus(Status.COMPLETED); + if(failDelay("completed")) { + this.task.setStatus(Status.COMPLETED); + } else { + this.task.setStatus(Status.FAILED); + } System.err.println("Wonky job status COMPLETED "+jobNumber); } else { throw new JobException(exitCode); @@ -274,16 +329,21 @@ } } - private String[] buildCmdArray(JobSpecification spec) { + private String[] buildCmdArray(JobSpecification spec, int jobid) { List arguments = spec.getArgumentsAsList(); String[] cmdarray = new String[arguments.size() + 1]; + logger.debug("jobid "+jobid+" :"); // this loop is a console race... cmdarray[0] = spec.getExecutable(); Iterator i = arguments.iterator(); int index = 1; while (i.hasNext()) { - cmdarray[index++] = (String) i.next(); + String s = (String)i.next(); + logger.debug(s); + cmdarray[index++] = s; + } + logger.debug("end of args"); return cmdarray; } @@ -307,4 +367,36 @@ } return envp; } + + boolean failDelay(String name) { + double failRate = getWonkyParam(name+"fail"); + double delayTime = getWonkyParam(name+"delay"); + System.out.println("Fail probability is: "+failRate); + if(Math.random()
site
$site
$site $count $count