From noreply at svn.ci.uchicago.edu Mon May 4 08:48:51 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 4 May 2009 08:48:51 -0500 (CDT) Subject: [Swift-commit] r2897 - trunk/resources Message-ID: <20090504134851.90BF69CC83@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-04 08:48:50 -0500 (Mon, 04 May 2009) New Revision: 2897 Modified: trunk/resources/Karajan.stg Log: move TODO around so it does not appear in generated kml Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2009-04-29 11:48:56 UTC (rev 2896) +++ trunk/resources/Karajan.stg 2009-05-04 13:48:50 UTC (rev 2897) @@ -1,5 +1,7 @@ group Karajan; +// TODO can move progress ticker start into vdl:mains so karajan files +// are smaller program(types,procedures,declarations,statements,constants,buildversion) ::= << @@ -19,7 +21,6 @@ $if(statements)$ - $parallel(statements=statements)$ From noreply at svn.ci.uchicago.edu Tue May 5 11:19:12 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 5 May 2009 11:19:12 -0500 (CDT) Subject: [Swift-commit] r2899 - in trunk: . docs etc libexec src/org/griphyn/vdl/mapping src/org/griphyn/vdl/util Message-ID: <20090505161912.DC69A9CC83@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-05 11:19:12 -0500 (Tue, 05 May 2009) New Revision: 2899 Modified: trunk/CHANGES.txt trunk/docs/userguide.xml trunk/etc/swift.properties trunk/libexec/vdl.k trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java trunk/src/org/griphyn/vdl/util/VDL2Config.java trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java Log: New configuration property provenance.log which controls whether verbose provenance-related log information is recorded in the run log file or not. By default, this is false, which means that less information is output by default now than previously. Modified: trunk/CHANGES.txt =================================================================== --- trunk/CHANGES.txt 2009-05-05 15:31:26 UTC (rev 2898) +++ trunk/CHANGES.txt 2009-05-05 16:19:12 UTC (rev 2899) @@ -1,3 +1,9 @@ +(05/05/09) +*** New configuration property provenance.log which controls whether verbose + provenance-related log information is recorded in the run log file or + not. By default, this is false, which means that less information is + output by default now than previously. + (04/27/09) *** Swift 0.9 built from Swift SVN r2860 and cog SVN r2388 Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-05 15:31:26 UTC (rev 2898) +++ trunk/docs/userguide.xml 2009-05-05 16:19:12 UTC (rev 2899) @@ -2968,6 +2968,23 @@ + + + + provenance.log + + + + Valid values: true, false + + + Default value: false + + + This property controls whether the log file will contain provenance information enabling this will increase the size of log files, sometimes significantly. + + + Modified: trunk/etc/swift.properties =================================================================== --- trunk/etc/swift.properties 2009-05-05 15:31:26 UTC (rev 2898) +++ trunk/etc/swift.properties 2009-05-05 16:19:12 UTC (rev 2899) @@ -312,3 +312,10 @@ # foreach.max.threads=1024 + +# controls whether the log file will contain provenance information +# enabling this will increase the size of log files, sometimes +# significantly. + +provenance.log=false + Modified: trunk/libexec/vdl.k =================================================================== --- trunk/libexec/vdl.k 2009-05-05 15:31:26 UTC (rev 2898) +++ trunk/libexec/vdl.k 2009-05-05 16:19:12 UTC (rev 2899) @@ -24,7 +24,11 @@ export( element(parameterlog, [direction, variable, id], - log("info","PARAM thread={#thread} direction={direction} variable={variable} provenanceid={id}") + + if( + vdl:configProperty("provenance.log") == "true" + log("info","PARAM thread={#thread} direction={direction} variable={variable} provenanceid={id}") + ) ) element(split, [var], each(str:split(vdl:getFieldValue(var), " "))) Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-05-05 15:31:26 UTC (rev 2898) +++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-05-05 16:19:12 UTC (rev 2899) @@ -20,6 +20,9 @@ import org.griphyn.vdl.karajan.VDL2FutureException; +import org.griphyn.vdl.util.VDL2Config; + + public abstract class AbstractDataNode implements DSHandle { static final String DATASET_URI_PREFIX = "tag:benc at ci.uchicago.edu,2008:swift:dataset:"; @@ -392,12 +395,15 @@ notifyListeners(); logger.info("closed " + this.getIdentifyingString()); // so because its closed, we can dump the contents + try { - logContent(); + if(VDL2Config.getConfig().getProvenanceLog()) { + logContent(); + } } catch (Exception e) { logger.warn("Exception whilst logging dataset content for " + this, - e); + e); } // TODO record retrospective provenance information for this dataset // here Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2009-05-05 15:31:26 UTC (rev 2898) +++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2009-05-05 16:19:12 UTC (rev 2899) @@ -83,6 +83,7 @@ put("throttle.file.operations", "8"); put("throttle.score.job.factor", "4"); put(VDL2ConfigProperties.SITEDIR_KEEP, "false"); + put(VDL2ConfigProperties.PROVENANCE_LOG, "false"); put("replication.enabled", "false"); put("replication.min.queue.time", "60"); @@ -202,6 +203,10 @@ return Boolean.valueOf(getProperty(VDL2ConfigProperties.SITEDIR_KEEP, "true")).booleanValue(); } + public boolean getProvenanceLog() { + return Boolean.valueOf(getProperty(VDL2ConfigProperties.PROVENANCE_LOG, "false")).booleanValue(); + } + public String toString() { return "Swift configuration " + files; } Modified: trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2009-05-05 15:31:26 UTC (rev 2898) +++ trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2009-05-05 16:19:12 UTC (rev 2899) @@ -24,6 +24,7 @@ public static final String KICKSTART_ALWAYS_TRANSFER = "kickstart.always.transfer"; public static final String WRAPPERLOG_ALWAYS_TRANSFER = "wrapperlog.always.transfer"; public static final String SITEDIR_KEEP = "sitedir.keep"; + public static final String PROVENANCE_LOG = "provenance.log"; public static final Map PROPERTIES; static { @@ -109,6 +110,12 @@ "If set to true, keeps remote site run directory after execution has completed.")); + PROPERTIES.put( + PROVENANCE_LOG, + new PropInfo( + "", + "If set to true, will record provenance information in the log file")); + } public static Map getPropertyDescriptions() { From noreply at svn.ci.uchicago.edu Tue May 5 12:57:59 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 5 May 2009 12:57:59 -0500 (CDT) Subject: [Swift-commit] r2900 - SwiftApps/SIDGrid/config Message-ID: <20090505175759.6FD539CCA0@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-05-05 12:57:59 -0500 (Tue, 05 May 2009) New Revision: 2900 Modified: SwiftApps/SIDGrid/config/sites_ranger.xml Log: using gridftp for file transfer Modified: SwiftApps/SIDGrid/config/sites_ranger.xml =================================================================== --- SwiftApps/SIDGrid/config/sites_ranger.xml 2009-05-05 16:19:12 UTC (rev 2899) +++ SwiftApps/SIDGrid/config/sites_ranger.xml 2009-05-05 17:57:59 UTC (rev 2900) @@ -9,8 +9,9 @@ 1 8 - TG-DBS080005N - + TG-DBS080004N + normal + 16 /work/00926/tg459516/sidgrid_out/{username} From noreply at svn.ci.uchicago.edu Tue May 5 12:58:41 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 5 May 2009 12:58:41 -0500 (CDT) Subject: [Swift-commit] r2901 - SwiftApps/SIDGrid/config Message-ID: <20090505175841.4B1079CCA0@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-05-05 12:58:41 -0500 (Tue, 05 May 2009) New Revision: 2901 Modified: SwiftApps/SIDGrid/config/sites_ucanl32.xml Log: using gram 2 for default while gram 4 broken on site Modified: SwiftApps/SIDGrid/config/sites_ucanl32.xml =================================================================== --- SwiftApps/SIDGrid/config/sites_ucanl32.xml 2009-05-05 17:57:59 UTC (rev 2900) +++ SwiftApps/SIDGrid/config/sites_ucanl32.xml 2009-05-05 17:58:41 UTC (rev 2901) @@ -10,9 +10,11 @@ + 1 + 2 ia32-compute - + /scratch/gpfs/local/sidgrid_out/{username} From noreply at svn.ci.uchicago.edu Tue May 5 12:59:22 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 5 May 2009 12:59:22 -0500 (CDT) Subject: [Swift-commit] r2902 - SwiftApps/SIDGrid/config Message-ID: <20090505175922.385E49CCA0@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-05-05 12:59:22 -0500 (Tue, 05 May 2009) New Revision: 2902 Added: SwiftApps/SIDGrid/config/sites_ucanl64.xml Log: using gram 2 for default while gram 4 broken on site Added: SwiftApps/SIDGrid/config/sites_ucanl64.xml =================================================================== --- SwiftApps/SIDGrid/config/sites_ucanl64.xml (rev 0) +++ SwiftApps/SIDGrid/config/sites_ucanl64.xml 2009-05-05 17:59:22 UTC (rev 2902) @@ -0,0 +1,20 @@ + + + + + + + + + + + + + 1 + 2 + ia64-compute + + + /scratch/gpfs/local/sidgrid_out/{username} + + From noreply at svn.ci.uchicago.edu Wed May 6 00:54:52 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 6 May 2009 00:54:52 -0500 (CDT) Subject: [Swift-commit] r2903 - trunk/docs Message-ID: <20090506055452.C01C39CCA1@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-06 00:54:51 -0500 (Wed, 06 May 2009) New Revision: 2903 Modified: trunk/docs/userguide.xml Log: provenance.log option was added out of order in the user guide. fix that. Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-05 17:59:22 UTC (rev 2902) +++ trunk/docs/userguide.xml 2009-05-06 05:54:51 UTC (rev 2903) @@ -2568,6 +2568,23 @@ + provenance.log + + + + Valid values: true, false + + + Default value: false + + + This property controls whether the log file will contain provenance information enabling this will increase the size of log files, sometimes significantly. + + + + + + replication.enabled @@ -2968,23 +2985,6 @@ - - - - provenance.log - - - - Valid values: true, false - - - Default value: false - - - This property controls whether the log file will contain provenance information enabling this will increase the size of log files, sometimes significantly. - - - From noreply at svn.ci.uchicago.edu Wed May 6 05:37:40 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 6 May 2009 05:37:40 -0500 (CDT) Subject: [Swift-commit] r2904 - nmi-build-test/submit-machine Message-ID: <20090506103740.9818A9CC83@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-06 05:37:39 -0500 (Wed, 06 May 2009) New Revision: 2904 Modified: nmi-build-test/submit-machine/build-hourly Log: fix up another URL Modified: nmi-build-test/submit-machine/build-hourly =================================================================== --- nmi-build-test/submit-machine/build-hourly 2009-05-06 05:54:51 UTC (rev 2903) +++ nmi-build-test/submit-machine/build-hourly 2009-05-06 10:37:39 UTC (rev 2904) @@ -20,7 +20,7 @@ nmi_submit swift-hourly.tmp fi -svn info https://smarty.isi.edu/svn/repo1/pegasus/trunk/src/tools/kickstart | grep -E '^Last Changed Rev: ' > kickstart.repo.latest +svn info https://pegasus.isi.edu/svn/pegasus/trunk/src/tools/kickstart | grep -E '^Last Changed Rev: ' > kickstart.repo.latest if (! diff kickstart.repo.latest kickstart.repo.last ) ; then KSREBUILD="kickstart >=r$(cat kickstart.repo.latest | sed 's/Last Changed Rev: //'). $KSREBUILD" fi From noreply at svn.ci.uchicago.edu Wed May 6 13:48:57 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 6 May 2009 13:48:57 -0500 (CDT) Subject: [Swift-commit] r2905 - in trunk: libexec src/org/griphyn/vdl/karajan/lib Message-ID: <20090506184857.353D79CCA1@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-06 13:48:55 -0500 (Wed, 06 May 2009) New Revision: 2905 Modified: trunk/libexec/execute-default.k trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java Log: progress ticker state for jobs that finished in previous run Modified: trunk/libexec/execute-default.k =================================================================== --- trunk/libexec/execute-default.k 2009-05-06 10:37:39 UTC (rev 2904) +++ trunk/libexec/execute-default.k 2009-05-06 18:48:55 UTC (rev 2905) @@ -9,6 +9,9 @@ if( derr == false then( + if(done + vdl:setprogress("Finished in previous run") + ) if( sys:not(done) try( sequential( Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2009-05-06 10:37:39 UTC (rev 2904) +++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2009-05-06 18:48:55 UTC (rev 2905) @@ -40,6 +40,7 @@ "Stage out", "Failed", "Replicating", + "Finished in previous run", "Finished successfully" }; From noreply at svn.ci.uchicago.edu Thu May 7 09:49:59 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 May 2009 09:49:59 -0500 (CDT) Subject: [Swift-commit] r2906 - in trunk: . bin Message-ID: <20090507144959.62D9E9CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-07 09:49:59 -0500 (Thu, 07 May 2009) New Revision: 2906 Removed: trunk/bin/swift-log-to-cedps Modified: trunk/CHANGES.txt trunk/build.xml Log: swift-log-to-cedps is removed. The functionality of generating .cedps format files is available by specifying the desired .cedps files in an invocation of swift-plot-log. Lack of widespread use of this command means a deprecation period is probably unnecessary. Modified: trunk/CHANGES.txt =================================================================== --- trunk/CHANGES.txt 2009-05-06 18:48:55 UTC (rev 2905) +++ trunk/CHANGES.txt 2009-05-07 14:49:59 UTC (rev 2906) @@ -1,3 +1,9 @@ +(05/07/09) +*** swift-log-to-cedps is removed. The functionality of generating + .cedps format files is available by specifying the desired .cedps files + in an invocation of swift-plot-log. Lack of widespread use of this + command means a deprecation period is probably unnecessary. + (05/05/09) *** New configuration property provenance.log which controls whether verbose provenance-related log information is recorded in the run log file or Deleted: trunk/bin/swift-log-to-cedps =================================================================== --- trunk/bin/swift-log-to-cedps 2009-05-06 18:48:55 UTC (rev 2905) +++ trunk/bin/swift-log-to-cedps 2009-05-07 14:49:59 UTC (rev 2906) @@ -1,52 +0,0 @@ -#!/bin/bash - -ORIGDIR=$(pwd) -WORKINGDIR=$(mktemp -d /tmp/swift-plot-log-XXXXXXXXXXXXXXXX) - - -# $1 should be the pathname of the log file to plot - -LOG_CODE_HOME="`dirname $0`/../libexec/log-processing/" -export SWIFT_PLOT_HOME=$LOG_CODE_HOME - -export PATH=${LOG_CODE_HOME}:${PATH} - -FC="${1:0:1}" - -if [ "$FC" = "/" ]; then # absolute path - LOG_FILE_PATH=$1 -else - LOG_FILE_PATH="`pwd`/$1" -fi - -LOG_DIRECTORY=`dirname $LOG_FILE_PATH` - -LOG_FILE_BASE=`basename $LOG_FILE_PATH .log` - -echo Log file path is $LOG_FILE_PATH -echo Log is in directory $LOG_DIRECTORY -echo Log basename is $LOG_FILE_BASE - -cd $WORKINGDIR -echo Now in directory $(pwd) - -MAKEENV="-f ${LOG_CODE_HOME}/makefile -I ${LOG_CODE_HOME}" -MAKETARGETS="" - -INFODIR=${LOG_DIRECTORY}/${LOG_FILE_BASE}.d - -if [ -d $INFODIR ]; then - MAKEENV="$MAKEENV IDIR=${LOG_DIRECTORY}/${LOG_FILE_BASE}.d/" - MAKETARGETS="webpage.info" -fi - -make $MAKEENV SDL=${ORIGDIR}/report-${LOG_FILE_BASE} LOG=$LOG_FILE_PATH clean execute.cedps execute2.cedps karatasks.cedps - -cp -v *.cedps $ORIGDIR - -# webpage.kara webpage.weights karatasks.JOB_SUBMISSION.Queue.transitions karatasks.JOB_SUBMISSION.Queue.event karatasks.JOB_SUBMISSION.eip $MAKETARGETS webpage distribute - -# mv report-$LOG_FILE_BASE $LOG_DIRECTORY/ - -# rm -r $WORKINGDIR -echo not removing $WORKINGDIR Modified: trunk/build.xml =================================================================== --- trunk/build.xml 2009-05-06 18:48:55 UTC (rev 2905) +++ trunk/build.xml 2009-05-07 14:49:59 UTC (rev 2906) @@ -81,7 +81,6 @@ - From noreply at svn.ci.uchicago.edu Thu May 7 10:08:09 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 May 2009 10:08:09 -0500 (CDT) Subject: [Swift-commit] r2907 - trunk/docs Message-ID: <20090507150809.A2ACE9CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-07 10:08:09 -0500 (Thu, 07 May 2009) New Revision: 2907 Modified: trunk/docs/userguide.xml Log: Userguide editing Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-07 14:49:59 UTC (rev 2906) +++ trunk/docs/userguide.xml 2009-05-07 15:08:09 UTC (rev 2907) @@ -27,10 +27,11 @@ SwiftScript programs are dataflow oriented - they are primarily -concerned with processing (possibly large) data files, by invoking -programs to do that processing. Swift handles execution of such programs -on remote sites by choosing sites, handling the staging of input and output -files to and from the chosen sites and remote execution of program code. +concerned with processing (possibly large) collections of data files, +by invoking programs to do that processing. Swift handles execution of +such programs on remote sites by choosing sites, handling the staging +of input and output files to and from the chosen sites and remote execution +of program code.
@@ -43,7 +44,9 @@ Data is represented in a script by strongly-typed single-assignment -variables, using a C-like syntax. +variables. The syntax superficially resembles C and Java. For example, +{ and } characters are used to +enclose blocks of statements. Types in Swift can be atomic or @@ -76,13 +79,6 @@ -Conceptually, a parallel can be drawn between Swift mapped variables -and Java reference types. In both cases there is no syntactic distinction -between primitive types and mapped types or reference types respectively. -Additionally, the semantic distinction is also kept to a minimum. - - - Component programs of scripts are declared in an app declaration, with the description of the command line syntax for that program and a list of input and output data. An app @@ -120,7 +116,7 @@ -The examples above have used the type image with out any +The examples above have used the type image without any definition of that type. We can declare it as a marker type which has no structure exposed to SwiftScript: @@ -254,7 +250,7 @@ while in this fragment, execution is serialised by the variable y, with procedure p executing -before q: +before q. y=p(x); @@ -262,11 +258,13 @@ -Arrays in SwiftScript are more generally -monotonic; that is, knowledge about the +Arrays in SwiftScript are more +monotonic - a generalisation of being +assignment. Knowledge about the content of an array increases during execution, but cannot otherwise -change. Each element of the array is single assignment. -Eventually, all values for an array are known, and that array +change. Each element of the array is itself single assignment or monotonic +(depending on its type). +During a run all values for an array are eventually known, and that array is regarded as closed. @@ -515,7 +513,7 @@
- Syntax + More technical details about SwiftScript The syntax of SwiftScript has a superficial resemblance to C and Java. For example, { and } characters are used to enclose blocks of statements. @@ -810,8 +808,7 @@ Swift provides a number of mappers that are useful in common cases. This section details those standard mappers. For more complex cases, it is possible to write application-specific mappers in Java and -use them within a SwiftScript program. For more information on writing -an application specific mapper, see the tutorial module on mappers. +use them within a SwiftScript program.
The single file mapper @@ -1348,8 +1345,15 @@
+
Commands + +The commands detailed in this section are available in the +bin/ directory of a Swift installation and can +by run from the commandline if that directory is placed on the +PATH. +
- The swift command + swift The swift command is the main command line tool for executing SwiftScript programs. @@ -1487,6 +1491,70 @@
+ +
swift-log-to-cedps
+
swift-osg-ress-site-catalog + +The swift-osg-ress-site-catalog command generates a site +catalog based on OSG's +ReSS information system (since Swift 0.9) + + +Usage: swift-osg-ress-site-catalog [options] + + + --help + +Show help message + + + + --vo=[name] + +Set what VO to query ReSS for + + + + --engage-verified + +Only retrieve sites verified by the Engagement VO site +verification tests This can not be used together with --vo, +as the query will only work for sites advertising support for the +Engagement VO. + +This option means information will be retrieved from the +Engagement collector instead of the top-level ReSS collector. + + + + + --out=[filename] + +Write to [filename] instead of stdout + + + + + +
+ + +
swift-plot-log + +swift-plot-log generates summaries of Swift run log +files. + + +Usage: swift-plot-log [logfile] [targets] + + +When no targets are specified, swift-plog-log will +generate an HTML report for the run. When targets are specified, only +those named targets will be generated. + +
+ +
Executing <literal>app</literal> procedures This section describes how Swift executes app procedures, @@ -1670,7 +1738,7 @@
-notes on how swift implements file input and output +How Swift implements the site execution model @@ -1741,7 +1809,7 @@ -jobs/ //application workspace directories// (optionally placed here - +jobs/ - application workspace directories (optionally placed here - see below) @@ -1774,7 +1842,7 @@ The Swift client side then checks for the presence of and deletes a status -file indicating success; copies files from the site shared directory to +file indicating success; and copies files from the site shared directory to the appropriate client side location. @@ -2508,7 +2576,7 @@ The generated dot file can be rendered into a graphical form using Graphviz, + url="http://www.graphviz.org/">Graphviz, for example with a command-line such as: @@ -3002,6 +3070,17 @@
Profiles + +Profiles are configuration parameters than can be specified either for +sites or for transformation catalog entries. They influence the behaviour +of Swift towards that site (for example, by changing the load Swift will +place on that sites) or when running a particular procedure. + + +Profile entries for a site are specified in the site catalog. Profile +entries for specific procedures are specified in the transformation +catalog. +
Karajan namespace maxSubmitRate limits the maximum rate of job submission, in jobs per second. For example: @@ -3695,36 +3774,6 @@ mpirun command should be explicitly invoked.
-
Generating a sites.xml for OSG - -The Swift distribution contains a command -swift-osg-ress-site-catalog that can generate a site -catalog based on OSG's ReSS information system (since Swift 0.9) - - - -Usage: - swift-osg-ress-site-catalog [options] - -Options: - --help Show this help message - - --vo=[name] - Set what VO to query ReSS for - - --engage-verified - Only retrieve sites verified by the Engagement VO site - verification tests This can not be used together with --vo, as - the query will only work for sites advertising support for the - Engagement VO. - - This option means information will be retrieved from the - Engagement collector instead of the top-level ReSS collector. - - --out=[filename] - Write to [filename] instead of stdout - -
From noreply at svn.ci.uchicago.edu Thu May 7 10:20:34 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 May 2009 10:20:34 -0500 (CDT) Subject: [Swift-commit] r2908 - trunk/docs Message-ID: <20090507152034.D4D239CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-07 10:20:34 -0500 (Thu, 07 May 2009) New Revision: 2908 Modified: trunk/docs/userguide.xml Log: remove swift-log-to-cedps documentation, as that command was removed in r2906 Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-07 15:08:09 UTC (rev 2907) +++ trunk/docs/userguide.xml 2009-05-07 15:20:34 UTC (rev 2908) @@ -1492,7 +1492,6 @@
-
swift-log-to-cedps
swift-osg-ress-site-catalog The swift-osg-ress-site-catalog command generates a site From noreply at svn.ci.uchicago.edu Thu May 7 12:14:32 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 May 2009 12:14:32 -0500 (CDT) Subject: [Swift-commit] r2909 - trunk/docs Message-ID: <20090507171432.850D79CC83@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-07 12:14:32 -0500 (Thu, 07 May 2009) New Revision: 2909 Modified: trunk/docs/userguide.xml Log: Userguide sections on retries and replication. Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-07 15:20:34 UTC (rev 2908) +++ trunk/docs/userguide.xml 2009-05-07 17:14:32 UTC (rev 2909) @@ -3539,12 +3539,40 @@
-
Workflow restart/recovery +
Reliability mechanisms + +This section details reliabilty mechanisms in Swift: retries, restarts +and replication. + + +
Retries +If an application procedure execution fails, Swift will attempt that +execution again repeatedly until it succeeds, up until the limit +defined in the execution.retries configuration +property. + + +Site selection will occur for retried jobs in the same way that it happens +for new jobs. Retried jobs may run on the same site or may run on a +different site. + + +If the retry limit execution.retries is reached for an +application procedure, then that application procedure will fail. This will +cause the entire run to fail - either immediately (if the +lazy.errors property is false) or +after all other possible work has been attempted (if the +lazy.errors property is true). + +
+ +
Restarts + If a run fails, Swift can resume the program from the point of failure. When a run fails, a restart log file will be left behind in -a file named using the unique job ID and a .rlog extension. This restart log -can then be passed to a subsequent Swift invocation using the -resume +a file named using the unique job ID and a .rlog extension. This restart log +can then be passed to a subsequent Swift invocation using the -resume parameter. Swift will resume execution, avoiding execution of invocations that have previously completed successfully. The SwiftScript source file and input data files should not be modified between runs. @@ -3575,7 +3603,30 @@
+
Replication + +When an execution job has been waiting in a site queue for a certain +period of time, Swift can resubmit replicas of that job (up to the limit +defined in the replication.limit configuration property). +When any of those jobs moves from queued to active state, all of the +other replicas will be cancelled. + + +This is intended to deal with situations where some sites have a substantially +longer (sometimes effectively infinite) queue time than other sites. +Selecting those slower sites can cause a very large delay in overall run time. + + +Replication can be enabled by setting the +replication.enabled configuration property to +true. The maximum number of replicas that will be +submitted for a job is controlled by the replication.limit +configuration property. + +
+
+
Clustering Swift can group a number of short job submissions into a single larger From noreply at svn.ci.uchicago.edu Thu May 7 12:23:34 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 May 2009 12:23:34 -0500 (CDT) Subject: [Swift-commit] r2910 - trunk/docs Message-ID: <20090507172335.09A759CC83@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-07 12:23:34 -0500 (Thu, 07 May 2009) New Revision: 2910 Modified: trunk/docs/userguide.xml Log: note on local maxwalltime enforcement Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-07 17:14:32 UTC (rev 2909) +++ trunk/docs/userguide.xml 2009-05-07 17:23:34 UTC (rev 2910) @@ -3123,6 +3123,10 @@ Hours:Minutes:Seconds + When replication is enabled (see replication), then walltime will also be enforced at the Swift client side: when +a job has been active for more than twice the maxwalltime, Swift will kill the +job and regard it as failed. + queue is used by the PBS, GRAM2 and GRAM4 providers. This profile entry specifies which queue jobs will be submitted to. The valid queue names From noreply at svn.ci.uchicago.edu Thu May 7 12:24:18 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 7 May 2009 12:24:18 -0500 (CDT) Subject: [Swift-commit] r2911 - trunk/docs Message-ID: <20090507172418.CDC5F9CC83@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-07 12:24:18 -0500 (Thu, 07 May 2009) New Revision: 2911 Modified: trunk/docs/userguide.xml Log: most other prompts are $ not >, so make the restart example prompt like that Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-07 17:23:34 UTC (rev 2910) +++ trunk/docs/userguide.xml 2009-05-07 17:24:18 UTC (rev 2911) @@ -3599,7 +3599,7 @@ used after the SwiftScript program file name. Example: -> swift From noreply at svn.ci.uchicago.edu Tue May 12 15:05:31 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 12 May 2009 15:05:31 -0500 (CDT) Subject: [Swift-commit] r2912 - SwiftApps/SIDGrid/config Message-ID: <20090512200531.40B7E9CCA7@vm-125-59.ci.uchicago.edu> Author: skenny Date: 2009-05-12 15:05:30 -0500 (Tue, 12 May 2009) New Revision: 2912 Modified: SwiftApps/SIDGrid/config/sites_bsd.xml Log: fixed path for work dir Modified: SwiftApps/SIDGrid/config/sites_bsd.xml =================================================================== --- SwiftApps/SIDGrid/config/sites_bsd.xml 2009-05-07 17:24:18 UTC (rev 2911) +++ SwiftApps/SIDGrid/config/sites_bsd.xml 2009-05-12 20:05:30 UTC (rev 2912) @@ -10,10 +10,12 @@ + 1 + 8 2000 Arch == "X86_64" || Arch == "INTEL" - /disks/gpfs/fmri/cnari/swift/work/{username} + /disks/ci-gpfs/fmri/cnari/swift/work/{username} From noreply at svn.ci.uchicago.edu Wed May 13 08:26:36 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:26:36 -0500 (CDT) Subject: [Swift-commit] r2915 - trunk/etc Message-ID: <20090513132636.C65A89CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:26:36 -0500 (Wed, 13 May 2009) New Revision: 2915 Modified: trunk/etc/sites.xml Log: Make default sites file use element not Modified: trunk/etc/sites.xml =================================================================== --- trunk/etc/sites.xml 2009-05-12 21:47:12 UTC (rev 2914) +++ trunk/etc/sites.xml 2009-05-13 13:26:36 UTC (rev 2915) @@ -18,7 +18,7 @@ - + /var/tmp 0 From noreply at svn.ci.uchicago.edu Wed May 13 08:27:38 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:27:38 -0500 (CDT) Subject: [Swift-commit] r2916 - trunk/docs Message-ID: <20090513132738.D553F9CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:27:38 -0500 (Wed, 13 May 2009) New Revision: 2916 Modified: trunk/docs/userguide.xml Log: maxwalltime notes Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-13 13:26:36 UTC (rev 2915) +++ trunk/docs/userguide.xml 2009-05-13 13:27:38 UTC (rev 2916) @@ -3112,8 +3112,7 @@
Globus namespace - maxwalltime specifies a walltime limit for each job, in minutes. This profile setting also interacts -with the clustering mechanism. + maxwalltime specifies a walltime limit for each job, in minutes. The following formats are recognized: @@ -3127,6 +3126,17 @@ a job has been active for more than twice the maxwalltime, Swift will kill the job and regard it as failed. + +When clustering is used, maxwalltime will be used to +select which jobs will be clustered together. More information on this is +available in the clustering section. + + +When coasters as used, maxwalltime influences the default +coaster worker maxwalltime, and which jobs will be sent to which workers. +More information on this is available in the coasters +section. + queue is used by the PBS, GRAM2 and GRAM4 providers. This profile entry specifies which queue jobs will be submitted to. The valid queue names @@ -3627,6 +3637,11 @@ submitted for a job is controlled by the replication.limit configuration property. + +When replication is enabled, Swift will also enforce the +maxwalltime profile setting for jobs as documented in +the profiles section. +
From noreply at svn.ci.uchicago.edu Wed May 13 08:42:23 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:42:23 -0500 (CDT) Subject: [Swift-commit] r2917 - trunk/docs Message-ID: <20090513134223.EF0E49CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:42:23 -0500 (Wed, 13 May 2009) New Revision: 2917 Modified: trunk/docs/userguide.xml Log: adjust site catalog headings Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-13 13:27:38 UTC (rev 2916) +++ trunk/docs/userguide.xml 2009-05-13 13:42:23 UTC (rev 2917) @@ -3257,7 +3257,7 @@ -
File transfer method specification +
File transfer method Transfer methods are specified with either @@ -3292,7 +3292,7 @@
-
Execution method specification +
Execution method Execution methods may be specified either with the <jobmanager> @@ -3368,7 +3368,7 @@ section on coasters.
-
Other site catalog parameters +
Work directory The workdirectory element specifies where on the site files can be @@ -3382,6 +3382,9 @@ appropriate for this. +
+
Profiles + Profile keys can be specified using the <profile> element. For example: From noreply at svn.ci.uchicago.edu Wed May 13 08:43:28 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:43:28 -0500 (CDT) Subject: [Swift-commit] r2918 - trunk/docs Message-ID: <20090513134328.E9F479CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:43:28 -0500 (Wed, 13 May 2009) New Revision: 2918 Modified: trunk/docs/userguide.xml Log: disperse content of environment variables section into commandline and profiles sections Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-13 13:42:23 UTC (rev 2917) +++ trunk/docs/userguide.xml 2009-05-13 13:43:28 UTC (rev 2918) @@ -1490,8 +1490,31 @@
+
Environment variables + The swift is influenced by the +following environment variables: + + +GLOBUS_HOSTNAME, GLOBUS_TCP_PORT_RANGE - set in the environment before running +Swift. These can be set to inform Swift of the +configuration of your local firewall. More information can be found in +the Globus firewall +How-to. + + +COG_OPTS - set in the environment before running Swift. Options set in this +variable will be passed as parameters to the Java Virtual Machine which +will run Swift. The parameters vary between virtual machine imlementations, +but can usually be used to alter settings such as maximum heap size. +Typing 'java -help' will sometimes give a list of commands. The Sun Java +1.4.2 command line options are documented here. + + +
+
+
swift-osg-ress-site-catalog The swift-osg-ress-site-catalog command generates a site @@ -3172,46 +3195,24 @@
env namespace Profile keys set in the env namespace will be set in the unix environment of the -executed job. +executed job. Some environment variables influence the worker-side +behaviour of Swift: -
-
- -
Environment variables - -There are a number of environment variables used by Swift. Some of them are -documented in this section: - - + PATHPREFIX - set in env namespace profiles. This path is prefixed onto the start of the PATH when jobs are executed. It can be more useful than setting the PATH environment variable directly, because setting PATH will cause the execution site's default path to be lost. - - -GLOBUS_HOSTNAME, GLOBUS_TCP_PORT_RANGE - set in the environment before running -Swift. These can be set to inform Swift of the -configuration of your local firewall. More information can be found in -the Globus firewall -How-to. - - -COG_OPTS - set in the environment before running Swift. Options set in this -variable will be passed as parameters to the Java Virtual Machine which -will run Swift. The parameters vary between virtual machine imlementations, -but can usually be used to alter settings such as maximum heap size. -Typing 'java -help' will sometimes give a list of commands. The Sun Java -1.4.2 command line options are documented here. - - + + SWIFT_JOBDIR_PATH - set in env namespace profiles. If set, then Swift will use the path specified here as a worker-node local temporary directory to copy input files to before running a job. If unset, Swift will keep input files on the site-shared filesystem. In some cases, copying to a worker-node local directory can be much faster than having applications access the site-shared filesystem directly. - - + + SWIFT_EXTRA_INFO - set in env namespace profiles. If set, then Swift will execute the command specified in SWIFT_EXTRA_INFO on execution sites immediately before @@ -3219,8 +3220,10 @@ wrapper info log file for that job. This is intended to allow software version and other arbitrary information about the remote site to be gathered and returned to the submit side. (since Swift 0.9) - + +
+
The Site Catalog - sites.xml The site catalog lists details of each site that Swift can use. The default From noreply at svn.ci.uchicago.edu Wed May 13 08:44:34 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:44:34 -0500 (CDT) Subject: [Swift-commit] r2919 - trunk/libexec Message-ID: <20090513134434.C24679CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:44:34 -0500 (Wed, 13 May 2009) New Revision: 2919 Modified: trunk/libexec/vdl-sc.k Log: tidy whitespace in site catalog processing Modified: trunk/libexec/vdl-sc.k =================================================================== --- trunk/libexec/vdl-sc.k 2009-05-13 13:43:28 UTC (rev 2918) +++ trunk/libexec/vdl-sc.k 2009-05-13 13:44:34 UTC (rev 2919) @@ -3,9 +3,9 @@ namespace("vdl" export(siteCatalog element([file] - + import("task.k") - + element(pool, [handle, optional(gridlaunch), optional(sysinfo), optional(lrc), ..., channel(properties)] host(name = handle each(...) @@ -15,7 +15,7 @@ ) ) ) - + element(gridftp, [url, optional(storage), optional(major), optional(minor), optional(patch)] if( url == "local://localhost" @@ -23,7 +23,7 @@ service(type="file", provider="gsiftp", url=url) ) ) - + element(jobmanager, [url, major, optional(universe), optional(minor), optional(patch)] provider := if( url == "local://localhost" "local" @@ -34,15 +34,15 @@ ) service(type="execution", provider=provider, url=url) ) - + element(execution, [provider, url, optional(jobManager)] service(type="execution", provider=provider, maybe(url=url), maybe(jobManager=jobManager)) ) - + element(filesystem, [provider, url, optional(storage)] service(type="file", provider=provider, url=url) ) - + element(profile, [namespace, key, value] if( namespace == "karajan" @@ -50,11 +50,11 @@ property("{namespace}:{key}", value) ) ) - + element(workdirectory, [dir] property(name="workdir", value = dir) ) - + resources( executeFile(file) ) From noreply at svn.ci.uchicago.edu Wed May 13 08:45:53 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:45:53 -0500 (CDT) Subject: [Swift-commit] r2920 - trunk/tests/language-behaviour Message-ID: <20090513134553.DE9659CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:45:53 -0500 (Wed, 13 May 2009) New Revision: 2920 Modified: trunk/tests/language-behaviour/run Log: Some language-behaviour testing was run per-expected-file when it should be running per-script. Modified: trunk/tests/language-behaviour/run =================================================================== --- trunk/tests/language-behaviour/run 2009-05-13 13:44:34 UTC (rev 2919) +++ trunk/tests/language-behaviour/run 2009-05-13 13:45:53 UTC (rev 2920) @@ -77,25 +77,25 @@ echo "No expected output files specified for this test case - not checking output." fi - if [ "$SWIFT_TEST_EXCEPTIONS" == "no" ]; then - echo Skipping exception test due to test configuration - else - grep --quiet " at " ${t}-*.log - if [ "$?" == "0" ]; then - echo EXCEPTIONS - stacktrace-like log lines found in output for ${t} - exit 2 - fi - fi + done - # check that all execute and execute2 IDs are unique - swift-plot-log ${t}-*.log execute.transitions compound.transitions - if [ "$( cat compound.transitions execute.transitions | grep ' START ' | cut -d ' ' -f 2 | sort | uniq -d)" != "" ]; then - echo EXECUTE AND COMPOUND IDS NOT UNIQUE - exit 3 + if [ "$SWIFT_TEST_EXCEPTIONS" == "no" ]; then + echo Skipping exception test due to test configuration + else + grep --quiet " at " ${t}-*.log + if [ "$?" == "0" ]; then + echo EXCEPTIONS - stacktrace-like log lines found in output for ${t} + exit 2 fi + fi + # check that all execute and execute2 IDs are unique + swift-plot-log ${t}-*.log execute.transitions compound.transitions + if [ "$( cat compound.transitions execute.transitions | grep ' START ' | cut -d ' ' -f 2 | sort | uniq -d)" != "" ]; then + echo EXECUTE AND COMPOUND IDS NOT UNIQUE + exit 3 + fi - done echo "Test passed at $(date)" if [ "$SUB" == "" ]; then echo ----------===========================---------- From noreply at svn.ci.uchicago.edu Wed May 13 08:47:53 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:47:53 -0500 (CDT) Subject: [Swift-commit] r2921 - in trunk: docs src/org/griphyn/vdl/engine Message-ID: <20090513134753.1D7DB9CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:47:52 -0500 (Wed, 13 May 2009) New Revision: 2921 Modified: trunk/docs/userguide.xml trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java Log: remove print internal procedure which was deprecated in r1344 Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-13 13:45:53 UTC (rev 2920) +++ trunk/docs/userguide.xml 2009-05-13 13:47:52 UTC (rev 2921) @@ -2162,16 +2162,6 @@ (since Swift 0.7)
-
print - -Deprecated - use trace instead. - - -print will print its parameters to stdout; but will do this at a point -in execution that is undefined. Specifically, it will not necessarily -wait for its parameters to be properly set. - -
trace trace will log its parameters. By default these will appear on both stdout Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2009-05-13 13:45:53 UTC (rev 2920) +++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2009-05-13 13:47:52 UTC (rev 2921) @@ -116,10 +116,6 @@ readData2.addOutputArg(rd2OutputArg); proceduresMap.put("readData2", readData2); - ProcedureSignature print = new ProcedureSignature("print"); - print.setAnyNumOfInputArgs(); - proceduresMap.put("print", print); - ProcedureSignature trace = new ProcedureSignature("trace"); trace.setAnyNumOfInputArgs(); proceduresMap.put("trace", trace); From noreply at svn.ci.uchicago.edu Wed May 13 08:48:57 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:48:57 -0500 (CDT) Subject: [Swift-commit] r2922 - trunk/docs Message-ID: <20090513134857.53EA39CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:48:57 -0500 (Wed, 13 May 2009) New Revision: 2922 Modified: trunk/docs/userguide.xml Log: tidy since message Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-13 13:47:52 UTC (rev 2921) +++ trunk/docs/userguide.xml 2009-05-13 13:48:57 UTC (rev 2922) @@ -2166,7 +2166,7 @@ trace will log its parameters. By default these will appear on both stdout and in the run log file. Some formatting occurs to produce the log message. -The particular output format should not be relied upon. (new in 0.4) +The particular output format should not be relied upon. (since Swift 0.4)
From noreply at svn.ci.uchicago.edu Wed May 13 08:49:58 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:49:58 -0500 (CDT) Subject: [Swift-commit] r2923 - trunk/libexec Message-ID: <20090513134958.D94E29CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:49:58 -0500 (Wed, 13 May 2009) New Revision: 2923 Modified: trunk/libexec/_swiftwrap Log: Check for existence of linuxy /proc files before catting them so that we get a more elegant error message in their absence. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2009-05-13 13:48:57 UTC (rev 2922) +++ trunk/libexec/_swiftwrap 2009-05-13 13:49:58 UTC (rev 2923) @@ -18,10 +18,14 @@ env 2>&1 >& "$INFO" infosection "df" df 2>&1 >& "$INFO" - infosection "/proc/cpuinfo" - cat /proc/cpuinfo 2>&1 >& "$INFO" - infosection "/proc/meminfo" - cat /proc/meminfo 2>&1 >& "$INFO" + if [ -e "/proc/cpuinfo" ]; then + infosection "/proc/cpuinfo" + cat /proc/cpuinfo 2>&1 >& "$INFO" + fi + if [ -e "/proc/meminfo" ]; then + infosection "/proc/meminfo" + cat /proc/meminfo 2>&1 >& "$INFO" + fi infosection "command line" echo $COMMANDLINE 2>&1 >& "$INFO" if [ -f "$STDOUT" ] ; then From noreply at svn.ci.uchicago.edu Wed May 13 08:51:04 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:51:04 -0500 (CDT) Subject: [Swift-commit] r2924 - trunk/docs Message-ID: <20090513135104.35A999CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:51:04 -0500 (Wed, 13 May 2009) New Revision: 2924 Modified: trunk/docs/userguide.xml Log: docs for condor provider Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-13 13:49:58 UTC (rev 2923) +++ trunk/docs/userguide.xml 2009-05-13 13:51:04 UTC (rev 2924) @@ -3336,6 +3336,28 @@ +For execution through a local Condor installation, the condor +provider should be used. This provider can run jobs either in the default +vanilla universe, or can use Condor-G to run jobs on remote sites. + + +When running locally, only the <execution> element +needs to be specified: + +<execution provider="condor" url="none" /> + + + +When running with Condor-G, it is necessary to specify the Condor grid +universe and the contact string for the remote site. For example: + + <execution provider="condor" /> + <profile namespace="globus" key="jobType">grid</profile> + <profile namespace="globus" key="gridResource">gt2 belhaven-1.renci.org/jobmanager-fork</profile> + + + + For execution through SSH, the ssh provider should be used: <execution url="www11.i2u2.org" provider="ssh"/> From noreply at svn.ci.uchicago.edu Wed May 13 08:52:50 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 13 May 2009 08:52:50 -0500 (CDT) Subject: [Swift-commit] r2925 - trunk/libexec/log-processing Message-ID: <20090513135250.A2B789CCAF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-13 08:52:50 -0500 (Wed, 13 May 2009) New Revision: 2925 Modified: trunk/libexec/log-processing/colour.plot.template trunk/libexec/log-processing/slt2.plot trunk/libexec/log-processing/sp.plot.template trunk/libexec/log-processing/start-last-times-and-kickstart.plot trunk/libexec/log-processing/start-last-times.plot trunk/libexec/log-processing/weights.plot Log: correct spelling of arbitrary in log plot axes, and change one axes to score Modified: trunk/libexec/log-processing/colour.plot.template =================================================================== --- trunk/libexec/log-processing/colour.plot.template 2009-05-13 13:51:04 UTC (rev 2924) +++ trunk/libexec/log-processing/colour.plot.template 2009-05-13 13:52:50 UTC (rev 2925) @@ -4,7 +4,7 @@ set key outside below -set ylabel 'arbitary event number' +set ylabel 'arbitrary event number' set xlabel 'realtime / s' set style arrow 1 nohead lt 1 Modified: trunk/libexec/log-processing/slt2.plot =================================================================== --- trunk/libexec/log-processing/slt2.plot 2009-05-13 13:51:04 UTC (rev 2924) +++ trunk/libexec/log-processing/slt2.plot 2009-05-13 13:52:50 UTC (rev 2925) @@ -4,7 +4,7 @@ set output 'slt2.png' -set ylabel 'arbitary job number' +set ylabel 'arbitrary job number' set xlabel 'realtime / s' set style arrow 1 nohead Modified: trunk/libexec/log-processing/sp.plot.template =================================================================== --- trunk/libexec/log-processing/sp.plot.template 2009-05-13 13:51:04 UTC (rev 2924) +++ trunk/libexec/log-processing/sp.plot.template 2009-05-13 13:52:50 UTC (rev 2925) @@ -4,7 +4,7 @@ set key outside below -set ylabel 'arbitary event number' +set ylabel 'arbitrary event number' set xlabel 'realtime / s' set style arrow 1 nohead lt 1 Modified: trunk/libexec/log-processing/start-last-times-and-kickstart.plot =================================================================== --- trunk/libexec/log-processing/start-last-times-and-kickstart.plot 2009-05-13 13:51:04 UTC (rev 2924) +++ trunk/libexec/log-processing/start-last-times-and-kickstart.plot 2009-05-13 13:52:50 UTC (rev 2925) @@ -4,7 +4,7 @@ set key outside below -set ylabel 'arbitary job number' +set ylabel 'arbitrary job number' set xlabel 'realtime / s' set style arrow 1 nohead lt 1 Modified: trunk/libexec/log-processing/start-last-times.plot =================================================================== --- trunk/libexec/log-processing/start-last-times.plot 2009-05-13 13:51:04 UTC (rev 2924) +++ trunk/libexec/log-processing/start-last-times.plot 2009-05-13 13:52:50 UTC (rev 2925) @@ -4,7 +4,7 @@ set key outside below -set ylabel 'arbitary job number' +set ylabel 'arbitrary job number' set xlabel 'realtime / s' set style arrow 1 nohead lt 1 Modified: trunk/libexec/log-processing/weights.plot =================================================================== --- trunk/libexec/log-processing/weights.plot 2009-05-13 13:51:04 UTC (rev 2924) +++ trunk/libexec/log-processing/weights.plot 2009-05-13 13:52:50 UTC (rev 2925) @@ -4,7 +4,7 @@ set key outside below -set ylabel 'arbitary event number' +set ylabel 'score' set xlabel 'realtime / s' set style arrow 1 nohead lt 1 From noreply at svn.ci.uchicago.edu Thu May 14 03:54:07 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 May 2009 03:54:07 -0500 (CDT) Subject: [Swift-commit] r2926 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20090514085407.D47779CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-14 03:54:07 -0500 (Thu, 14 May 2009) New Revision: 2926 Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftArg.java Log: Raw accessor for Swift arguments that allow allow access to the underlying DSHandle objects without unwrapping or waiting for the DSHandle to be assigned/closed. Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftArg.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SwiftArg.java 2009-05-13 13:52:50 UTC (rev 2925) +++ trunk/src/org/griphyn/vdl/karajan/lib/SwiftArg.java 2009-05-14 08:54:07 UTC (rev 2926) @@ -84,6 +84,15 @@ throw new ExecutionException("Internal type error. Expected a Double. Got " + classOf(dbl)); } } + + public DSHandle getRawValue(VariableStack stack) throws ExecutionException { + Object v = super.getValue(stack); + if(v instanceof DSHandle) { + return (DSHandle)v; + } else { + throw new ExecutionException("Expected Swift data, but got some primitive type (" + v + ")"); + } + } private static Class classOf(Object object) { if (object == null) { From noreply at svn.ci.uchicago.edu Thu May 14 03:55:40 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 May 2009 03:55:40 -0500 (CDT) Subject: [Swift-commit] r2927 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20090514085540.31B5D9CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-14 03:55:40 -0500 (Thu, 14 May 2009) New Revision: 2927 Modified: trunk/src/org/griphyn/vdl/karajan/lib/Operators.java Log: Add logging of provenance information for operators. Modified: trunk/src/org/griphyn/vdl/karajan/lib/Operators.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Operators.java 2009-05-14 08:54:07 UTC (rev 2926) +++ trunk/src/org/griphyn/vdl/karajan/lib/Operators.java 2009-05-14 08:55:40 UTC (rev 2927) @@ -1,5 +1,8 @@ package org.griphyn.vdl.karajan.lib; +import java.io.IOException; + +import org.apache.log4j.Logger; import org.globus.cog.karajan.arguments.Arg; import org.globus.cog.karajan.stack.VariableStack; import org.globus.cog.karajan.workflow.ExecutionException; @@ -8,6 +11,7 @@ import org.griphyn.vdl.mapping.RootDataNode; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.util.VDL2Config; public class Operators extends FunctionsCollection { @@ -15,6 +19,8 @@ public static final SwiftArg R = new SwiftArg.Positional("right"); public static final SwiftArg U = new SwiftArg.Positional("unaryArg"); + public static final Logger provenanceLogger = Logger.getLogger("org.globus.swift.provenance.operators"); + private DSHandle newNum(Type type, double value) throws ExecutionException { try { DSHandle handle = new RootDataNode(type); @@ -80,24 +86,31 @@ public Object vdlop_sum(VariableStack stack) throws ExecutionException { Object l = L.getValue(stack); Object r = R.getValue(stack); + DSHandle ret; if (l instanceof String || r instanceof String) { - return newString(((String) l) + ((String) r)); + ret = newString(((String) l) + ((String) r)); } else { - return newNum(type(stack), SwiftArg.checkDouble(l) + SwiftArg.checkDouble(r)); + ret = newNum(type(stack), SwiftArg.checkDouble(l) + SwiftArg.checkDouble(r)); } + logBinaryProvenance(stack, "sum", ret); + return ret; } public Object vdlop_subtraction(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newNum(type(stack), l - r); + DSHandle ret = newNum(type(stack), l - r); + logBinaryProvenance(stack, "subtraction", ret); + return ret; } public Object vdlop_product(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newNum(type(stack), l * r); + DSHandle ret = newNum(type(stack), l * r); + logBinaryProvenance(stack, "product", ret); + return ret; } public Object vdlop_quotient(VariableStack stack) throws ExecutionException { @@ -108,43 +121,57 @@ public Object vdlop_fquotient(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newNum(Types.FLOAT, l / r); + DSHandle ret = newNum(Types.FLOAT, l / r); + logBinaryProvenance(stack, "fquotient", ret); + return ret; } public Object vdlop_iquotient(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newNum(Types.INT, l / r); + DSHandle ret = newNum(Types.INT, l / r); + logBinaryProvenance(stack, "iquotient", ret); + return ret; } public Object vdlop_remainder(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newNum(type(stack), l % r); + DSHandle ret = newNum(type(stack), l % r); + logBinaryProvenance(stack, "remainder", ret); + return ret; } public Object vdlop_le(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newBool(l <= r); + DSHandle ret = newBool(l <= r); + logBinaryProvenance(stack, "le", ret); + return ret; } public Object vdlop_ge(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newBool(l >= r); + DSHandle ret = newBool(l >= r); + logBinaryProvenance(stack, "ge", ret); + return ret; } public Object vdlop_gt(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newBool(l > r); + DSHandle ret = newBool(l > r); + logBinaryProvenance(stack, "gt", ret); + return ret; } public Object vdlop_lt(VariableStack stack) throws ExecutionException { double l = L.getDoubleValue(stack); double r = R.getDoubleValue(stack); - return newBool(l < r); + DSHandle ret = newBool(l < r); + logBinaryProvenance(stack, "lt", ret); + return ret; } public Object vdlop_eq(VariableStack stack) throws ExecutionException { @@ -156,7 +183,9 @@ if (r == null) { throw new ExecutionException("Second operand is null"); } - return newBool(l.equals(r)); + DSHandle ret = newBool(l.equals(r)); + logBinaryProvenance(stack, "eq", ret); + return ret; } public Object vdlop_ne(VariableStack stack) throws ExecutionException { @@ -168,23 +197,59 @@ if (r == null) { throw new ExecutionException("Second operand is null"); } - return newBool(!(l.equals(r))); + DSHandle ret = newBool(!(l.equals(r))); + logBinaryProvenance(stack, "ne", ret); + return ret; } public Object vdlop_and(VariableStack stack) throws ExecutionException { boolean l = ((Boolean)L.getValue(stack)).booleanValue(); boolean r = ((Boolean)R.getValue(stack)).booleanValue(); - return newBool(l && r); + DSHandle ret = newBool(l && r); + logBinaryProvenance(stack, "and", ret); + return ret; } public Object vdlop_or(VariableStack stack) throws ExecutionException { boolean l = ((Boolean)L.getValue(stack)).booleanValue(); boolean r = ((Boolean)R.getValue(stack)).booleanValue(); - return newBool(l || r); + DSHandle ret = newBool(l || r); + logBinaryProvenance(stack, "or", ret); + return ret; } public Object vdlop_not(VariableStack stack) throws ExecutionException { boolean u = ((Boolean)U.getValue(stack)).booleanValue(); - return newBool(!u); + DSHandle ret = newBool(!u); + logUnaryProvenance(stack, "not", ret); + return ret; } + + private void logBinaryProvenance(VariableStack stack, String name, DSHandle resultDataset) throws ExecutionException { + try { + if(VDL2Config.getConfig().getProvenanceLog()) { + String thread = stack.getVar("#thread").toString(); + String lhs = L.getRawValue(stack).getIdentifier(); + String rhs = R.getRawValue(stack).getIdentifier(); + String result = resultDataset.getIdentifier(); + provenanceLogger.info("OPERATOR thread="+thread+" operator="+name+" lhs="+lhs+" rhs="+rhs+" result="+result); + } + } catch(IOException ioe) { + throw new ExecutionException("Exception when logging provenance for binary operator", ioe); + } + } + + private void logUnaryProvenance(VariableStack stack, String name, DSHandle resultDataset) throws ExecutionException { + try { + if(VDL2Config.getConfig().getProvenanceLog()) { + String thread = stack.getVar("#thread").toString(); + String lhs = U.getRawValue(stack).getIdentifier(); + String result = resultDataset.getIdentifier(); + provenanceLogger.info("UNARYOPERATOR thread="+thread+" operator="+name+" operand="+lhs+" result="+result); + } + } catch(IOException ioe) { + throw new ExecutionException("Exception when logging provenance for unary operator", ioe); + } + } } + From noreply at svn.ci.uchicago.edu Thu May 14 04:24:48 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 May 2009 04:24:48 -0500 (CDT) Subject: [Swift-commit] r2929 - provenancedb Message-ID: <20090514092448.8713E9CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-14 04:24:48 -0500 (Thu, 14 May 2009) New Revision: 2929 Modified: provenancedb/prov-sql-generate-transitive-closures.sql provenancedb/prov-sql-iterate-transitive-closures.sql Log: fix up sql transitivity script: execute_id is now process_id; addition of compounds to process table means that we get duplicates which need eliminating with DISTINCT; and some CONSTRAINTs to check that we aren't putting duplicates in any of the transitivity tables. Modified: provenancedb/prov-sql-generate-transitive-closures.sql =================================================================== --- provenancedb/prov-sql-generate-transitive-closures.sql 2009-05-14 09:02:27 UTC (rev 2928) +++ provenancedb/prov-sql-generate-transitive-closures.sql 2009-05-14 09:24:48 UTC (rev 2929) @@ -11,12 +11,13 @@ CREATE TABLE trans (before char(128), - after char(128) - ); + after char(128), + CONSTRAINT no_duplicate_arcs_in_trans UNIQUE (before, after)); CREATE TABLE transbase (before char(128), - after char(128) + after char(128), + CONSTRAINT no_duplicate_arcs_in_transbase UNIQUE (before, after) ); CREATE INDEX transindex ON trans (before); @@ -24,11 +25,10 @@ INSERT INTO transbase SELECT inner_dataset_id AS before, outer_dataset_id AS after FROM dataset_containment; -INSERT INTO transbase SELECT execute_id AS before, dataset_id AS after FROM dataset_usage where direction='O'; +INSERT INTO transbase SELECT process_id AS before, dataset_id AS after FROM dataset_usage where direction='O'; -INSERT INTO transbase SELECT dataset_id AS before, execute_id AS after FROM dataset_usage where direction='I'; +INSERT INTO transbase SELECT dataset_id AS before, process_id AS after FROM dataset_usage where direction='I'; -- now seed the transitive closure table - INSERT into trans SELECT before, after from transbase; Modified: provenancedb/prov-sql-iterate-transitive-closures.sql =================================================================== --- provenancedb/prov-sql-iterate-transitive-closures.sql 2009-05-14 09:02:27 UTC (rev 2928) +++ provenancedb/prov-sql-iterate-transitive-closures.sql 2009-05-14 09:24:48 UTC (rev 2929) @@ -3,12 +3,13 @@ CREATE TEMPORARY TABLE transtemp (before char(128), - after char(128) + after char(128), +CONSTRAINT no_duplicate_arcs_in_transtemp UNIQUE (before, after) ); -- existing path before existing arc -INSERT INTO transtemp SELECT transbase.before, trans.after FROM transbase, trans +INSERT INTO transtemp SELECT DISTINCT transbase.before, trans.after FROM transbase, trans WHERE transbase.after = trans.before; INSERT INTO trans SELECT transtemp.before, transtemp.after FROM transtemp From noreply at svn.ci.uchicago.edu Thu May 14 11:13:50 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 May 2009 11:13:50 -0500 (CDT) Subject: [Swift-commit] r2930 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20090514161350.624FB9CCA2@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-14 11:13:49 -0500 (Thu, 14 May 2009) New Revision: 2930 Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java Log: missing field when dataset contains a throwable Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-05-14 09:24:48 UTC (rev 2929) +++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-05-14 16:13:49 UTC (rev 2930) @@ -126,6 +126,7 @@ else if (value instanceof Throwable) { sb.append(" containing throwable "); sb.append(value.getClass()); + sb.append(" dataset="); } else { sb.append(" value="); From noreply at svn.ci.uchicago.edu Thu May 14 11:45:22 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 May 2009 11:45:22 -0500 (CDT) Subject: [Swift-commit] r2931 - SwiftApps/pc3 Message-ID: <20090514164522.2B9AE9CCBF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-14 11:45:21 -0500 (Thu, 14 May 2009) New Revision: 2931 Added: SwiftApps/pc3/q2.sh Modified: SwiftApps/pc3/pc3.swift Log: core query 2 Modified: SwiftApps/pc3/pc3.swift =================================================================== --- SwiftApps/pc3/pc3.swift 2009-05-14 16:13:49 UTC (rev 2930) +++ SwiftApps/pc3/pc3.swift 2009-05-14 16:45:21 UTC (rev 2931) @@ -43,6 +43,7 @@ } (boolean output) is_match_csv_file_tables(xmlfile input) { + string fn = readData(input); output = ps_load_executable_boolean(input, "IsMatchCSVFileTables"); } @@ -99,6 +100,8 @@ } (boolean output) is_match_table_column_ranges(xmlfile db, xmlfile input, external thread) { + string dbcontent = readData(db); + string inputcontent = readData(input); output = ps_load_executable_db_boolean(db, input, "IsMatchTableColumnRanges", thread); } Added: SwiftApps/pc3/q2.sh =================================================================== --- SwiftApps/pc3/q2.sh (rev 0) +++ SwiftApps/pc3/q2.sh 2009-05-14 16:45:21 UTC (rev 2931) @@ -0,0 +1,23 @@ +#!/bin/bash + +# this will get all the process IDs for IsMatchTableColumnRanges calls... +# select execute_id from invocation_procedure_names where procedure_name='is_match_table_column_ranges'; + +# need to do two things: + +# augment with a success/failure value + +# select * from processes where id like 'tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:pc3-20090429-1456-ssftbvsa:0-5-5-3-11-%'; +# this will get the compound and execute (but not operator) subprocesses +# of a compound process +# we can probably ask if there is a sub-process with 'stop' in it... +# although a more positive measure of success might be better +# should generalise process containment... + + +# augment with the table name +# perhaps the workflow needs to bring more of the parameter structure into +# swiftscript rather than dealing with opaque files + +echo " select processes.id, dataset_usage.dataset_id, dataset_usage.param_name, dataset_values.value from processes,invocation_procedure_names,dataset_usage, dataset_values where type='compound' and processes.id = invocation_procedure_names.execute_id and procedure_name='is_match_table_column_ranges' and dataset_usage.process_id = processes.id and dataset_usage.direction='O' and dataset_usage.param_name='inputcontent' and dataset_usage.dataset_id = dataset_values.dataset_id;" | sqlite3 provdb + Property changes on: SwiftApps/pc3/q2.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Thu May 14 11:46:56 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 May 2009 11:46:56 -0500 (CDT) Subject: [Swift-commit] r2932 - SwiftApps/pc3 Message-ID: <20090514164656.395FC9CCBF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-14 11:46:56 -0500 (Thu, 14 May 2009) New Revision: 2932 Added: SwiftApps/pc3/oq5-query SwiftApps/pc3/oq5-run.sh Modified: SwiftApps/pc3/pc3.swift Log: optional query 5 Added: SwiftApps/pc3/oq5-query =================================================================== --- SwiftApps/pc3/oq5-query (rev 0) +++ SwiftApps/pc3/oq5-query 2009-05-14 16:46:56 UTC (rev 2932) @@ -0,0 +1,19 @@ + + select known_workflows.importstatus, value from known_workflows, processes_i +n_workflows, dataset_usage, dataset_values where param_name='job_id_content' and + dataset_usage.dataset_id = dataset_values.dataset_id and dataset_usage.process_ +id=processes_in_workflows.process_id and known_workflows.workflow_id = processes +_in_workflows.workflow_id; + + SUCCESS| J062941 + SUCCESS| J062942 + SUCCESS| J062943 + SUCCESS| J062944 + SUCCESS| J062945 + sqlite> + Added: SwiftApps/pc3/oq5-run.sh =================================================================== --- SwiftApps/pc3/oq5-run.sh (rev 0) +++ SwiftApps/pc3/oq5-run.sh 2009-05-14 16:46:56 UTC (rev 2932) @@ -0,0 +1,27 @@ +#!/bin/bash + +rm -rfv /var/tmp/derby-pc3/ + +for ID in J062941 J062942 J062943 J062944 J062945 ; do + + echo Swift run for $ID + +cat > oc5CSVRootPathInput.xml << UUUUUUU + + + /Users/benc/work/swift-svn.git/SwiftApps/pc3/PC3/SampleData/$ID + +UUUUUUU + +cat > oc5JobIDInput.xml << UUUUUU + + + $ID + +UUUUUU + + swift -tc.file tc.data.append pc3.swift -csvpath=oc5CSVRootPathInput.xml -jobid=oc5JobIDInput.xml + + +done + Property changes on: SwiftApps/pc3/oq5-run.sh ___________________________________________________________________ Name: svn:executable + * Modified: SwiftApps/pc3/pc3.swift =================================================================== --- SwiftApps/pc3/pc3.swift 2009-05-14 16:45:21 UTC (rev 2931) +++ SwiftApps/pc3/pc3.swift 2009-05-14 16:46:56 UTC (rev 2932) @@ -119,6 +119,7 @@ } string csv_root_path_input_arg = @arg("csvpath"); string job_id_arg = @arg("jobid"); +string job_id_content = readData(job_id_arg); xmlfile csv_root_path_input ; xmlfile job_id ; boolean is_csv_ready_file_exists_output; From noreply at svn.ci.uchicago.edu Thu May 14 11:48:04 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 14 May 2009 11:48:04 -0500 (CDT) Subject: [Swift-commit] r2933 - provenancedb Message-ID: <20090514164804.54AB39CCBF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-14 11:48:04 -0500 (Thu, 14 May 2009) New Revision: 2933 Modified: provenancedb/prov-init.sql provenancedb/swift-prov-import-all-logs Log: Add URI-format run ID to known_workflow table Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2009-05-14 16:46:56 UTC (rev 2932) +++ provenancedb/prov-init.sql 2009-05-14 16:48:04 UTC (rev 2933) @@ -130,7 +130,9 @@ -- been seen by the importer: the log filename, swift version and import -- status. CREATE TABLE known_workflows - ( workflow_log_filename char(128), + ( + workflow_id char(128), + workflow_log_filename char(128), version char(128), importstatus char(128) ); Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2009-05-14 16:46:56 UTC (rev 2932) +++ provenancedb/swift-prov-import-all-logs 2009-05-14 16:48:04 UTC (rev 2933) @@ -63,7 +63,11 @@ fi # import-run-to-xml $filename - echo "INSERT INTO known_workflows (workflow_log_filename, version, importstatus) VALUES ('$filename','$version','$wfstatus');" | $SQLCMD + + export RUNID=$(basename $filename .log) + export WF="tag:benc at ci.uchicago.edu,2008:swiftlogs:execute:${RUNID}:run" + + echo "INSERT INTO known_workflows (workflow_id, workflow_log_filename, version, importstatus) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD else echo SKIP: Already known in workflow fi From noreply at svn.ci.uchicago.edu Fri May 15 06:23:18 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 15 May 2009 06:23:18 -0500 (CDT) Subject: [Swift-commit] r2934 - trunk/tests/language-behaviour Message-ID: <20090515112318.2F2C79CCBF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-15 06:23:17 -0500 (Fri, 15 May 2009) New Revision: 2934 Added: trunk/tests/language-behaviour/1102-array-range-step.swift Log: another test for array ranges Added: trunk/tests/language-behaviour/1102-array-range-step.swift =================================================================== --- trunk/tests/language-behaviour/1102-array-range-step.swift (rev 0) +++ trunk/tests/language-behaviour/1102-array-range-step.swift 2009-05-15 11:23:17 UTC (rev 2934) @@ -0,0 +1,8 @@ + +int i[] = [0:9]; + +int j[] = [0:19:2]; + +foreach ix in i { + trace(j[ix]); +} From noreply at svn.ci.uchicago.edu Fri May 15 06:24:49 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 15 May 2009 06:24:49 -0500 (CDT) Subject: [Swift-commit] r2935 - in trunk/tests/language: working working-base Message-ID: <20090515112449.9B8219CCBF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-15 06:24:49 -0500 (Fri, 15 May 2009) New Revision: 2935 Modified: trunk/tests/language/working-base/arraymapper.xml trunk/tests/language/working-base/range.xml trunk/tests/language/working/arraymapper.swift trunk/tests/language/working/range.swift Log: Remove uses of print in language tests, as this was removed in r2921 Modified: trunk/tests/language/working/arraymapper.swift =================================================================== --- trunk/tests/language/working/arraymapper.swift 2009-05-15 11:23:17 UTC (rev 2934) +++ trunk/tests/language/working/arraymapper.swift 2009-05-15 11:24:49 UTC (rev 2935) @@ -3,6 +3,6 @@ file files[]; foreach f in files { - print(f); + trace(f); } Modified: trunk/tests/language/working/range.swift =================================================================== --- trunk/tests/language/working/range.swift 2009-05-15 11:23:17 UTC (rev 2934) +++ trunk/tests/language/working/range.swift 2009-05-15 11:24:49 UTC (rev 2935) @@ -1,5 +1,5 @@ int nums[] = [0:20:2]; // generate a list 0, 2, 4, 6, 8 ... foreach num in nums { - print(num); + trace(num); } Modified: trunk/tests/language/working-base/arraymapper.xml =================================================================== --- trunk/tests/language/working-base/arraymapper.xml 2009-05-15 11:23:17 UTC (rev 2934) +++ trunk/tests/language/working-base/arraymapper.xml 2009-05-15 11:24:49 UTC (rev 2935) @@ -16,7 +16,7 @@ files - + f Modified: trunk/tests/language/working-base/range.xml =================================================================== --- trunk/tests/language/working-base/range.xml 2009-05-15 11:23:17 UTC (rev 2934) +++ trunk/tests/language/working-base/range.xml 2009-05-15 11:24:49 UTC (rev 2935) @@ -14,7 +14,7 @@ nums - + num From noreply at svn.ci.uchicago.edu Fri May 15 06:26:34 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 15 May 2009 06:26:34 -0500 (CDT) Subject: [Swift-commit] r2936 - trunk/src/org/griphyn/vdl/karajan/lib/swiftscript Message-ID: <20090515112634.465FD9CCBF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-15 06:26:33 -0500 (Fri, 15 May 2009) New Revision: 2936 Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java Log: make strsplit whitespace consistent with rest of file Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2009-05-15 11:24:49 UTC (rev 2935) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2009-05-15 11:26:33 UTC (rev 2936) @@ -99,21 +99,21 @@ } public DSHandle swiftscript_strsplit(VariableStack stack) throws ExecutionException, NoSuchTypeException, - InvalidPathException { - String str = TypeUtil.toString(PA_INPUT.getValue(stack)); - String pattern = TypeUtil.toString(PA_PATTERN.getValue(stack)); - - String[] split = str.split(pattern); - - DSHandle handle = new RootArrayDataNode(Types.STRING.arrayType()); - for (int i = 0; i < split.length; i++) { - DSHandle el = handle.getField(Path.EMPTY_PATH.addFirst(String.valueOf(i), true)); - el.setValue(split[i]); - } - handle.closeDeep(); - return handle; - } + InvalidPathException { + String str = TypeUtil.toString(PA_INPUT.getValue(stack)); + String pattern = TypeUtil.toString(PA_PATTERN.getValue(stack)); + String[] split = str.split(pattern); + + DSHandle handle = new RootArrayDataNode(Types.STRING.arrayType()); + for (int i = 0; i < split.length; i++) { + DSHandle el = handle.getField(Path.EMPTY_PATH.addFirst(String.valueOf(i), true)); + el.setValue(split[i]); + } + handle.closeDeep(); + return handle; + } + public DSHandle swiftscript_regexp(VariableStack stack) throws ExecutionException, NoSuchTypeException, InvalidPathException { String inputString = TypeUtil.toString(PA_INPUT.getValue(stack)); From noreply at svn.ci.uchicago.edu Fri May 15 06:28:03 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 15 May 2009 06:28:03 -0500 (CDT) Subject: [Swift-commit] r2937 - trunk/tests/misc Message-ID: <20090515112803.C597F9CCBF@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-15 06:28:03 -0500 (Fri, 15 May 2009) New Revision: 2937 Added: trunk/tests/misc/arg-default.swift trunk/tests/misc/arg-nodefault.swift trunk/tests/misc/arg.sh Modified: trunk/tests/misc/run Log: tests for @arg Added: trunk/tests/misc/arg-default.swift =================================================================== --- trunk/tests/misc/arg-default.swift (rev 0) +++ trunk/tests/misc/arg-default.swift 2009-05-15 11:28:03 UTC (rev 2937) @@ -0,0 +1,5 @@ + +string s = @arg("foo","bar"); + +trace(s); + Added: trunk/tests/misc/arg-nodefault.swift =================================================================== --- trunk/tests/misc/arg-nodefault.swift (rev 0) +++ trunk/tests/misc/arg-nodefault.swift 2009-05-15 11:28:03 UTC (rev 2937) @@ -0,0 +1,5 @@ + +string s = @arg("foo"); + +trace(s); + Added: trunk/tests/misc/arg.sh =================================================================== --- trunk/tests/misc/arg.sh (rev 0) +++ trunk/tests/misc/arg.sh 2009-05-15 11:28:03 UTC (rev 2937) @@ -0,0 +1,17 @@ +#!/bin/bash + +swift arg-default.swift -foo=bar + +if [ "$?" != "0" ]; then + echo default arg test FAIL + exit 1 +fi + +swift arg-nodefault.swift -foo=bar + +if [ "$?" != "0" ]; then + echo no-default arg test FAIL + exit 2 +fi + +echo SUCCESS for arg tests Property changes on: trunk/tests/misc/arg.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/misc/run =================================================================== --- trunk/tests/misc/run 2009-05-15 11:26:33 UTC (rev 2936) +++ trunk/tests/misc/run 2009-05-15 11:28:03 UTC (rev 2937) @@ -2,7 +2,7 @@ for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 restart-iterate workernode-local \ ordering-extern-notlazy restart-extern ordering-extern \ external-mapper-args extract-int-delayed asserts \ -relative-wrapper \ +relative-wrapper arg \ ; do ./${a}.sh R=$? From noreply at svn.ci.uchicago.edu Sun May 17 03:49:27 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 17 May 2009 03:49:27 -0500 (CDT) Subject: [Swift-commit] r2938 - trunk/tests/language-behaviour Message-ID: <20090517084927.85EBF9CCA3@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-17 03:49:26 -0500 (Sun, 17 May 2009) New Revision: 2938 Modified: trunk/tests/language-behaviour/091-case.swift Log: Remove uses of print in language-behaviour test, as this was removed in r2921 Modified: trunk/tests/language-behaviour/091-case.swift =================================================================== --- trunk/tests/language-behaviour/091-case.swift 2009-05-15 11:28:03 UTC (rev 2937) +++ trunk/tests/language-behaviour/091-case.swift 2009-05-17 08:49:26 UTC (rev 2938) @@ -10,7 +10,7 @@ int selector = 8; -print(selector); +trace(selector); string message; From noreply at svn.ci.uchicago.edu Sun May 17 07:00:03 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 17 May 2009 07:00:03 -0500 (CDT) Subject: [Swift-commit] r2939 - trunk/libexec Message-ID: <20090517120003.B2A9A9CCA7@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-17 07:00:03 -0500 (Sun, 17 May 2009) New Revision: 2939 Modified: trunk/libexec/_swiftwrap Log: Move away from using $@ for application parameters and instead use an in-swift array. This makes it easier later on to get parameters from a file instead of the wrapper commandline. Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2009-05-17 08:49:26 UTC (rev 2938) +++ trunk/libexec/_swiftwrap 2009-05-17 12:00:03 UTC (rev 2939) @@ -178,6 +178,9 @@ fail 254 "Missing arguments (-a option)" fi +declare -a CMDARGS +CMDARGS=("$@") + if [ "$STATUSMODE" = "files" ]; then mkdir -p $WFDIR/status/$JOBDIR fi @@ -262,9 +265,9 @@ fi if [ "$KICKSTART" == "" ]; then if [ "$STDIN" == "" ]; then - "$EXEC" "$@" 1>"$STDOUT" 2>"$STDERR" + "$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" else - "$EXEC" "$@" 1>"$STDOUT" 2>"$STDERR" <"$STDIN" + "$EXEC" "${CMDARGS[@]}" 1>"$STDOUT" 2>"$STDERR" <"$STDIN" fi checkError $? "Exit code $?" else From noreply at svn.ci.uchicago.edu Sun May 17 07:03:40 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 17 May 2009 07:03:40 -0500 (CDT) Subject: [Swift-commit] r2940 - in trunk: . docs etc libexec src/org/griphyn/vdl/util tests/misc tests/sites Message-ID: <20090517120340.2932F9CCA7@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-17 07:03:39 -0500 (Sun, 17 May 2009) New Revision: 2940 Added: trunk/tests/misc/arg-mode.sh trunk/tests/sites/local-wrapper-args-file.xml Modified: trunk/CHANGES.txt trunk/docs/userguide.xml trunk/etc/swift.properties trunk/libexec/_swiftwrap trunk/libexec/vdl-int.k trunk/src/org/griphyn/vdl/util/VDL2Config.java trunk/tests/misc/run Log: Allow wrapper parameters to be specific in a file rather than on the commandline. This trades commandline length for an extra file to stage-in per job. A new parameter, wrapper.parameter.mode, is introduced to control this. Modified: trunk/CHANGES.txt =================================================================== --- trunk/CHANGES.txt 2009-05-17 12:00:03 UTC (rev 2939) +++ trunk/CHANGES.txt 2009-05-17 12:03:39 UTC (rev 2940) @@ -1,3 +1,10 @@ +(05/17/09) +*** Allow wrapper parameters to be specific in a file rather than on the + commandline. This trades commandline length for an extra file to stage-in + per job. + A new parameter, wrapper.parameter.mode, is introduced to control this. + This was feature request bug 201. + (05/07/09) *** swift-log-to-cedps is removed. The functionality of generating .cedps format files is available by specifying the desired .cedps files Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-17 12:00:03 UTC (rev 2939) +++ trunk/docs/userguide.xml 2009-05-17 12:03:39 UTC (rev 2940) @@ -3039,6 +3039,25 @@ + wrapper.parameter.mode + + + +Controls how Swift will supply parameters to the remote wrapper script. +args mode will pass parameters on the command line. Some +execution systems do not pass commandline parameters sufficiently cleanly +for Swift to operate correctly. +files mode will pass parameters through an additional +input file (since Swift 0.95). This provides a cleaner communication channel +for parameters, at the expense of transferring an additional file for each +job invocation. + + + + + + + wrapperlog.always.transfer Modified: trunk/etc/swift.properties =================================================================== --- trunk/etc/swift.properties 2009-05-17 12:00:03 UTC (rev 2939) +++ trunk/etc/swift.properties 2009-05-17 12:03:39 UTC (rev 2940) @@ -295,6 +295,14 @@ # # status.mode=files +# Controls how swift will supply parameters to the remote wrapper script. +# 'args' mode will pass parameters on the command line +# 'files' mode will pass parameters through an additional input file +# +# valid values: args, files +# Default: files +# +# wrapper.parameter.mode=args # Determines if Swift remote wrappers will be executed by specifying an # absolute path, or a path relative to the job initial working directory Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2009-05-17 12:00:03 UTC (rev 2939) +++ trunk/libexec/_swiftwrap 2009-05-17 12:03:39 UTC (rev 2940) @@ -74,12 +74,21 @@ fi } +checkparamfile() { + log "checking for paramfile" + if [ "$1" == "-p" ]; then + JOBDIR=$2 + PARAMFILE=${WFDIR}/parameters/${JOBDIR}/param-${ID} + fi + log "paramfile is: $PARAMFILE" +} + getarg() { NAME=$1 shift VALUE="" SHIFTCOUNT=0 - if [ "$1" == "$NAME" ]; then + if [ "$PARAMFILE" == "" ] && [ "$1" == "$NAME" ]; then shift let "SHIFTCOUNT=$SHIFTCOUNT+1" while [ "${1:0:1}" != "-" ] && [ "$#" != "0" ]; do @@ -87,10 +96,12 @@ shift let "SHIFTCOUNT=$SHIFTCOUNT+1" done + VALUE="${VALUE:1}" + elif [ "$PARAMFILE" != "" ] && grep -E "^$NAME " $PARAMFILE ; then + VALUE=$(grep -E "^$NAME " $PARAMFILE | cut -d ' ' -f 2-) else fail 254 "Missing $NAME argument" fi - VALUE="${VALUE:1}" } openinfo() { @@ -115,6 +126,7 @@ # make the WFDIR absolute WFDIR=$(pwd) +PARAMFILE= openinfo "wrapper.log" ID=$1 @@ -122,10 +134,16 @@ shift -getarg "-jobdir" "$@" -JOBDIR=$VALUE -shift $SHIFTCOUNT +checkparamfile "$@" +# JOBDIR might have been assigned through the -p option, or might +# be a parameter here +if [ "$JOBDIR" == "" ] ; then + getarg "-jobdir" "$@" + JOBDIR=$VALUE + shift $SHIFTCOUNT +fi + checkEmpty "$JOBDIR" "Missing job directory prefix" mkdir -p $WFDIR/info/$JOBDIR closeinfo @@ -172,15 +190,25 @@ STATUSMODE=$VALUE shift $SHIFTCOUNT -if [ "$1" == "-a" ]; then +declare -a CMDARGS +if [ "$PARAMFILE" == "" ] && [ "$1" == "-a" ] ; then shift + CMDARGS=("$@") +elif [ "$PARAMFILE" != "" ] ; then + CMDARGS=() + FIRST=1 + while read line ; do + if [ "$FIRST" == "1" ] ; then + CMDARGS=("$line") + FIRST=0 + else + CMDARGS=("${CMDARGS[*]}" "$line") + fi + done < <(grep -E "^-a " $PARAMFILE | cut -d " " -f 2-) else fail 254 "Missing arguments (-a option)" fi -declare -a CMDARGS -CMDARGS=("$@") - if [ "$STATUSMODE" = "files" ]; then mkdir -p $WFDIR/status/$JOBDIR fi Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2009-05-17 12:00:03 UTC (rev 2939) +++ trunk/libexec/vdl-int.k 2009-05-17 12:03:39 UTC (rev 2940) @@ -113,6 +113,11 @@ dir:make(dircat(wfdir, "status"), host=rhost) ) + wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) + if(wrapperMode == "files" + dir:make(dircat(wfdir, "parameters"), host=rhost) + ) + dir:make(dircat(wfdir, "info"), host=rhost) wfdir, sharedDir //we send the cleanup data to vdl:main() @@ -204,6 +209,24 @@ ) ) + + element(stageWrapperParams, [jobid, jobdir, wrapfile, dir, host] + log(LOG:INFO, "START jobid={jobid} - staging in wrapper params") + provider := provider(wrapfile) + srchost := hostname(wrapfile) + srcdir := dirname(wrapfile) + destdir := dircat(dir, "/parameters/{jobdir}/") + filename := basename(wrapfile) + + cacheOn(list(destdir, host) + dir:make(destdir, host=host, provider=provider) + ) + + task:transfer(srcprovider=provider, srchost=srchost, srcfile=filename, srcdir=srcdir, desthost=host, destdir=destdir) + log(LOG:INFO, "END jobid={jobid}") + + ) + element(doStagein, [jobid, files, dir, host] log(LOG:INFO, "START jobid={jobid} - Staging in files") uParallelFor(file, files @@ -381,27 +404,71 @@ log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") statusMode := configProperty("status.mode",host=rhost) + wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) + ddir := "{VDL:SCRIPTNAME}-{VDL:RUNID}.d" + if(sys:not(file:exists(ddir)) + task:dir:make(ddir) + ) + wrapfile := "{ddir}/param-{jobid}" + + stdout := try(stdout, "stdout.txt") + stderr := try(stderr, "stderr.txt") + kickstart := vdl:kickstart(rhost) + fileDirs := fileDirs(stagein, stageout) + + if(wrapperMode == "files" + sequential( + sys:file:write(wrapfile, + "-e ",vdl:executable(tr, rhost), nl(), + "-out ",stdout,nl(), + "-err ",stderr,nl(), + "-i ",maybe(stdin),nl(), + "-d ",flatten(each(fileDirs)),nl(), + "-if ",flatten(infiles(stagein)),nl(), + "-of ",flatten(outfiles(stageout)),nl(), + "-k ",kickstart,nl(), + "-status ",statusMode,nl()) + for(argiterator, arguments + sys:file:write(wrapfile,append=true,"-a ",argiterator,nl()) + ) + ) + ) + vdl:setprogress("Stage in") tmpdir := dircat(concat(wfdir, "/jobs/", jobdir), jobid) - stdout := try(stdout, "stdout.txt") - stderr := try(stderr, "stderr.txt") - kickstart := vdl:kickstart(rhost) try( sequential( - fileDirs := fileDirs(stagein, stageout) createDirSet(jobid, sharedDir, rhost, fileDirs) doStagein(jobid, stagein, sharedDir, rhost) + if(wrapperMode == "files" + stageWrapperParams(jobid, jobdir, wrapfile, wfdir, rhost) + ) + log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}") vdl:setprogress("Submitting") - vdl:execute("/bin/bash", - list("shared/_swiftwrap", jobid, + + if(wrapperMode == "files" + vdl:execute("/bin/bash", + list("shared/_swiftwrap", jobid, "-p", jobdir) + directory=wfdir + redirect=false + host=rhost + vdl:tcprofile(tr, rhost) //this gets various app params from the tc, such as environment, walltime, etc + replicationGroup=replicationGroup + replicationChannel=replicationChannel + jobid=jobid + ) + ) + if(wrapperMode == "args" + vdl:execute("/bin/bash", + list("shared/_swiftwrap", jobid, "-jobdir", jobdir, "-e", vdl:executable(tr, rhost), "-out", stdout, @@ -412,7 +479,8 @@ "-of", flatten(outfiles(stageout)), "-k", kickstart, "-status", statusMode - "-a", maybe(each(arguments))) + "-a", maybe(each(arguments)) + ) directory=wfdir redirect=false host=rhost @@ -420,13 +488,18 @@ replicationGroup=replicationGroup replicationChannel=replicationChannel jobid=jobid + ) ) vdl:setprogress("Checking status") if(statusMode == "files" checkJobStatus(rhost, wfdir, jobid, tr, jobdir) ) - + + if(wrapperMode == "files" + file:remove(wrapfile) + ) + log(LOG:DEBUG, "STAGING_OUT jobid={jobid}") @@ -497,7 +570,7 @@ ) ) ) - + element(generateProvenanceGraph, [gdata] pgraph := vdl:configProperty("pgraph") gname := if(pgraph == "true" "{VDL:SCRIPTNAME}-{VDL:RUNID}.dot" pgraph) Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2009-05-17 12:00:03 UTC (rev 2939) +++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2009-05-17 12:03:39 UTC (rev 2940) @@ -89,6 +89,7 @@ put("replication.min.queue.time", "60"); put("replication.limit", "3"); put("status.mode", "files"); + put("wrapper.parameter.mode", "args"); put("wrapper.invocation.mode", "absolute"); } Added: trunk/tests/misc/arg-mode.sh =================================================================== --- trunk/tests/misc/arg-mode.sh (rev 0) +++ trunk/tests/misc/arg-mode.sh 2009-05-17 12:03:39 UTC (rev 2940) @@ -0,0 +1,19 @@ +#!/bin/bash + +INITDIR=$(pwd) + +cd ../language-behaviour + +./generate-tc.data + +cat tc.data ../sites/tc.data > tmp.tc.data.sites + +SITE=local-wrapper-args-file.xml + +echo testing site configuration: $SITE + +export SWIFT_TEST_PARAMS="-sites.file ../sites/${SITE} -tc.file tmp.tc.data.sites" + +./run 001-echo 066-many 141-space-in-filename 142-space-and-quotes 1421-space-and-quotes + + Property changes on: trunk/tests/misc/arg-mode.sh ___________________________________________________________________ Name: svn:executable + * Modified: trunk/tests/misc/run =================================================================== --- trunk/tests/misc/run 2009-05-17 12:00:03 UTC (rev 2939) +++ trunk/tests/misc/run 2009-05-17 12:03:39 UTC (rev 2940) @@ -2,7 +2,7 @@ for a in clusters no-retries dryrun typecheck path-prefix restart restart2 restart3 restart4 restart5 restart-iterate workernode-local \ ordering-extern-notlazy restart-extern ordering-extern \ external-mapper-args extract-int-delayed asserts \ -relative-wrapper arg \ +relative-wrapper arg arg-mode \ ; do ./${a}.sh R=$? Added: trunk/tests/sites/local-wrapper-args-file.xml =================================================================== --- trunk/tests/sites/local-wrapper-args-file.xml (rev 0) +++ trunk/tests/sites/local-wrapper-args-file.xml 2009-05-17 12:03:39 UTC (rev 2940) @@ -0,0 +1,12 @@ + + + + + + /var/tmp + 0 + 0 + files + + + From noreply at svn.ci.uchicago.edu Sun May 17 10:27:08 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 17 May 2009 10:27:08 -0500 (CDT) Subject: [Swift-commit] r2941 - provenancedb Message-ID: <20090517152708.964A49CCA7@vm-125-59.ci.uchicago.edu> Author: benc Date: 2009-05-17 10:27:08 -0500 (Sun, 17 May 2009) New Revision: 2941 Modified: provenancedb/db-to-opm.sh Log: OPM exporter needs to at least encode < as < Modified: provenancedb/db-to-opm.sh =================================================================== --- provenancedb/db-to-opm.sh 2009-05-17 12:03:39 UTC (rev 2940) +++ provenancedb/db-to-opm.sh 2009-05-17 15:27:08 UTC (rev 2941) @@ -86,7 +86,8 @@ done sqlite3 -separator ' ' -batch provdb "select value from dataset_values where dataset_id='$artifact';" | while read value ; do - echo "$value" + encvalue=$(echo $value | sed 's/$encvalue" done echo " " From noreply at svn.ci.uchicago.edu Sun May 24 15:39:49 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 May 2009 15:39:49 -0500 (CDT) Subject: [Swift-commit] r2942 - trunk/docs Message-ID: <20090524203949.25AD89CC7F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-05-24 15:39:48 -0500 (Sun, 24 May 2009) New Revision: 2942 Modified: trunk/docs/userguide.xml Log: updated userguide Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-17 15:27:08 UTC (rev 2941) +++ trunk/docs/userguide.xml 2009-05-24 20:39:48 UTC (rev 2942) @@ -3182,16 +3182,68 @@ condor_requirements allows a requirements string to be specified when Condor is used as an LRM behind GRAM2. Example: <profile namespace="globus" key="condor_requirements">Arch == "X86_64" || Arch="INTEL"</profile> - coastersPerNode specifies -the number of coaster workers to be run on each node. This profile entry -is used by the coaster execution provider. + +When using coasters, this parameter +specifies the maximum number of jobs/blocks that the coaster scheduler will have running at any given time. +The default is 20. - coasterWorkerMaxwalltime -specifies the maxwalltime to be used when submitting coaster workers. This -profile entry is used by the coaster execution -provider. If this entry is not specified, the coaster provider -will compute a maxwalltime based on the maxwalltime of jobs submitted. (since Swift 0.9) + +This parameter determines how many coaster workers are +started one each compute node. The default value is 1. + +When allocating a coaster worker block, this parameter +restricts the number of nodes in a block to a multiple of this value. The total number of workers will +then be a multiple of workersPerNode * nodeGranularity. The default value is 1. + + +Each time the coaster block scheduler computes a schedule, it will attempt to allocate a +number of slots from the number of available slots (limited using the above slots profile). This +parameter specifies the maximum fraction of slots that are allocated in one schedule. Default is +0.1. + + +Overallocation is a function of the walltime of a job which determines how long (time-wise) a +worker job will be. For example, if a number of 10s jobs are submitted to the coaster service, +and the overallocation for 10s jobs is 10, the coaster scheduler will attempt to start worker +jobs that have a walltime of 100s. The overallocation is controlled by manipulating the end-points +of an overallocation function. The low endpoint, specified by this parameter, is the overallocation +for a 1s job. The high endpoint is the overallocation for a (theoretical) job of infinite length. +The overallocation for job sizes in the [1s, +inf) interval is determined using an exponential decay function: + +overallocation(walltime) = walltime * (lowOverallocation - highOverallocation) * exp(-walltime * overallocationDecayFactor) + highOverallocation + +The default value of lowOverallocation is 10. + + +The high overallocation endpoint (as described above). Default: 1 + + +The decay factor for the overallocation curve. Default 0.001 (1e-3). + + +When a large number of jobs is submitted to the a coaster service, the work is divided into blocks. This +parameter allows a rough control of the relative sizes of those blocks. A value of 0 indicates that all work +should be divided equally between the blocks (and blocks will therefore have equal sizes). A value of 1 +indicates the largest possible spread. The existence of the spread parameter is based on the assumption +that smaller overall jobs will generally spend less time in the queue than larger jobs. By submitting +blocks of different sizes, submitted jobs may be finished quicker by smaller blocks. Default: 0.9. + + +Reserve time is a time in the allocation of a worker that sits at the end of the worker time and +is useable only for critical operations. For example, a job will not be submitted to a worker if +it overlaps its reserve time, but a job that (due to inaccurate walltime specification) runs into +the reserve time will not be killed (note that once the worker exceeds its walltime, the queuing +system will kill the job anyway). Default 10 (s). + + +Determines the maximum number of nodes that can be allocated in one coaster block. Default: unlimited. + + +Indicates the maximum walltime that a coaster block can have. Default: unlimited. + +
env namespace From noreply at svn.ci.uchicago.edu Sun May 24 15:48:12 2009 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 24 May 2009 15:48:12 -0500 (CDT) Subject: [Swift-commit] r2943 - trunk/docs Message-ID: <20090524204812.C189A9CC7F@vm-125-59.ci.uchicago.edu> Author: hategan Date: 2009-05-24 15:48:12 -0500 (Sun, 24 May 2009) New Revision: 2943 Modified: trunk/docs/userguide.xml Log: updated userguide Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2009-05-24 20:39:48 UTC (rev 2942) +++ trunk/docs/userguide.xml 2009-05-24 20:48:12 UTC (rev 2943) @@ -3242,6 +3242,10 @@ Indicates the maximum walltime that a coaster block can have. Default: unlimited. + +If set to "true", the client side will get a Swing window showing, graphically, the state of the +coaster scheduler (blocks, jobs, etc.). Default: false +