From noreply at svn.ci.uchicago.edu Wed Dec 1 15:00:54 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Dec 2010 15:00:54 -0600 (CST) Subject: [Swift-commit] r3731 - trunk/bin Message-ID: <20101201210054.892DC9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-01 15:00:54 -0600 (Wed, 01 Dec 2010) New Revision: 3731 Modified: trunk/bin/swift Log: Return non-zero exit code if Swift is not installed correctly Modified: trunk/bin/swift =================================================================== --- trunk/bin/swift 2010-11-29 23:59:05 UTC (rev 3730) +++ trunk/bin/swift 2010-12-01 21:00:54 UTC (rev 3731) @@ -45,7 +45,7 @@ SWIFT_HOME=`dirname $MY_PATH`"/.." if [ ! -f "$SWIFT_HOME/lib/cog-swift-svn.jar" ] && [ ! -f "$SWIFT_HOME/lib/cog.jar" ] ; then echo "Error: SWIFT_HOME is not set, and all attempts at guessing it failed" - exit + exit 1 fi fi fi From noreply at svn.ci.uchicago.edu Wed Dec 1 21:37:14 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 1 Dec 2010 21:37:14 -0600 (CST) Subject: [Swift-commit] r3732 - provenancedb Message-ID: <20101202033714.718CB9CC7F@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-01 21:37:14 -0600 (Wed, 01 Dec 2010) New Revision: 3732 Modified: provenancedb/prov-init.sql provenancedb/prov-to-sql.sh Log: Minor corrections to import scripts. Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2010-12-01 21:00:54 UTC (rev 3731) +++ provenancedb/prov-init.sql 2010-12-02 03:37:14 UTC (rev 3732) @@ -231,8 +231,6 @@ PRIMARY KEY (array_id,from_id,to_id,step_id) ); - - -- this GRANT does not work for sqlite; you'll get a syntax error but -- ignore it, as it is not needed in sqlite GRANT ALL ON Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2010-12-01 21:00:54 UTC (rev 3731) +++ provenancedb/prov-to-sql.sh 2010-12-02 03:37:14 UTC (rev 3732) @@ -9,7 +9,7 @@ echo Generating SQL for $RUNID -rm -f tmp-ds.sql tmp-p.sql tmp-e.sql tmp-e2.sql tmp-dsu.sql tmp-dsc.sql tmp-f.sql tmp-v.sql tmp-import.sql import.sql +rm -f tmp-u.sql tmp-ds.sql tmp-p.sql tmp-e.sql tmp-e2.sql tmp-dsu.sql tmp-dsc.sql tmp-f.sql tmp-v.sql tmp-import.sql import.sql # this gives a distinction between the root process for a workflow and the # workflow itself. perhaps better to model the workflow as a process From noreply at svn.ci.uchicago.edu Fri Dec 3 17:11:51 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 3 Dec 2010 17:11:51 -0600 (CST) Subject: [Swift-commit] r3733 - provenancedb Message-ID: <20101203231151.85DF89CC9B@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-03 17:11:51 -0600 (Fri, 03 Dec 2010) New Revision: 3733 Modified: provenancedb/prov-to-sql.sh Log: Minor fixes. Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2010-12-02 03:37:14 UTC (rev 3732) +++ provenancedb/prov-to-sql.sh 2010-12-03 23:11:51 UTC (rev 3733) @@ -9,7 +9,7 @@ echo Generating SQL for $RUNID -rm -f tmp-u.sql tmp-ds.sql tmp-p.sql tmp-e.sql tmp-e2.sql tmp-dsu.sql tmp-dsc.sql tmp-f.sql tmp-v.sql tmp-import.sql import.sql +rm -f tmp-u.sql tmp-ds.sql tmp-p.sql tmp-e.sql tmp-e2.sql tmp-dsu.sql tmp-dsc.sql tmp-f.sql tmp-v.sql tmp-import.sql import.sql # this gives a distinction between the root process for a workflow and the # workflow itself. perhaps better to model the workflow as a process @@ -90,13 +90,17 @@ # TODO this could merge with other naming tables while read start duration thread final_state procname ; do - compoundid=$WFID$thread - echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$compoundid', 'compound', '$procname', '$WF');" >> tmp-p.sql + if [ "$duration" != "last-event-line" ]; then + compoundid=$WFID$thread + echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$compoundid', 'compound', '$procname', '$WF');" >> tmp-p.sql + fi done < compound.event while read start duration thread final_state procname ; do - fqid=$WFID$thread - echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$fqid', 'internal', '$procname', '$WF');" >> tmp-p.sql + if [ "$duration" != "last-event-line" ]; then + fqid=$WFID$thread + echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$fqid', 'internal', '$procname', '$WF');" >> tmp-p.sql + fi done < internalproc.event while read t ; do @@ -132,7 +136,6 @@ cat tmp-dsc.sql | sort | uniq >> import.sql cat tmp-import.sql | sort | uniq >> import.sql cat tmp-u.sql | sort | uniq >> import.sql - echo Sending SQL to DB $SQLCMD < import.sql From noreply at svn.ci.uchicago.edu Mon Dec 6 13:13:44 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 13:13:44 -0600 (CST) Subject: [Swift-commit] r3734 - trunk/libexec Message-ID: <20101206191344.892199CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 13:13:44 -0600 (Mon, 06 Dec 2010) New Revision: 3734 Modified: trunk/libexec/cdm_broadcast.sh Log: Whitespace cleanup Modified: trunk/libexec/cdm_broadcast.sh =================================================================== --- trunk/libexec/cdm_broadcast.sh 2010-12-03 23:11:51 UTC (rev 3733) +++ trunk/libexec/cdm_broadcast.sh 2010-12-06 19:13:44 UTC (rev 3734) @@ -10,12 +10,12 @@ while [[ ${*} != "" ]] do L=$1 # -l - shift + shift ARGS=$1 # Location shift while true do - if [[ $1 == "-l" || $1 == "" ]] + if [[ $1 == "-l" || $1 == "" ]] then break fi @@ -34,14 +34,14 @@ shift WORK=( ${*} ) - IP=$( listip ${LOCATION} ) + IP=$( listip ${LOCATION} ) - if [[ ${#WORK[@]} > 3 ]] + if [[ ${#WORK[@]} > 3 ]] then SCRIPT=$( mktemp ) { echo "#!/bin/sh" - while [[ ${*} != "" ]] + while [[ ${*} != "" ]] do FILE=$1 DEST=$2 @@ -52,7 +52,7 @@ scp ${SCRIPT} ${IP}:${SCRIPT} ssh ${IP} ${SCRIPT} else - while [[ ${*} != "" ]] + while [[ ${*} != "" ]] do FILE=$1 DEST=$2 @@ -65,12 +65,12 @@ # Repeat command N times until success ssh_until_success() { - N=$1 - shift + N=$1 + shift for (( i=0 ; i < N ; i++ )) do ssh -o PasswordAuthentication=no ${*} - if [[ $? == 0 ]]; + if [[ $? == 0 ]]; then break fi @@ -86,16 +86,16 @@ DEST=$3 cp -v ${FILE} ${DEST}/${FILE} } - + set -x { declare -p PWD LOG - + if [[ $( uname -p ) == "ppc64" ]] then bgp_broadcast ${*} - else + else bgp_local ${*} fi - + } >> /tmp/cdm_broadcast.log 2>&1 # ${LOG} 2>&1 From noreply at svn.ci.uchicago.edu Mon Dec 6 13:18:11 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 13:18:11 -0600 (CST) Subject: [Swift-commit] r3735 - trunk/libexec Message-ID: <20101206191811.EFAF59CC7F@svn.ci.uchicago.edu> Author: wilde Date: 2010-12-06 13:18:11 -0600 (Mon, 06 Dec 2010) New Revision: 3735 Modified: trunk/libexec/cdm_lib.sh Log: Correct the argument handling for CDM LOCAL file action. Modified: trunk/libexec/cdm_lib.sh =================================================================== --- trunk/libexec/cdm_lib.sh 2010-12-06 19:13:44 UTC (rev 3734) +++ trunk/libexec/cdm_lib.sh 2010-12-06 19:18:11 UTC (rev 3735) @@ -51,12 +51,16 @@ fi ;; LOCAL) - TOOL=${ARGS[0]} - REMOTE_DIR=${ARGS[1]} - FLAGS=${ARGS[3]} - log "CDM[LOCAL]: Copying $DIRECT_DIR/$FILE to $JOBDIR/$FILE" + #TOOL=${ARGS[0]} + #REMOTE_DIR=${ARGS[1]} + #FLAGS=${ARGS[3]} + TOOL=$1 + REMOTE_DIR=$2 + FLAGS=$3 + log "CDM[LOCAL]: TOOL=$TOOL FLAGS=$FLAGS REMOTE_DIR=$REMOTE_DIR ARGS=$ARGS" + log "CDM[LOCAL]: Copying $REMOTE_DIR/$FILE to $JOBDIR/$FILE" if [ $MODE == "INPUT" ]; then - [ -f "$DIRECT_DIR/$FILE" ] + [ -f "$REMOTE_DIR/$FILE" ] checkError 254 "CDM[LOCAL]: $REMOTE_DIR/$FILE does not exist!" $TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE checkError 254 "CDM[LOCAL]: Tool failed!" From noreply at svn.ci.uchicago.edu Mon Dec 6 13:36:57 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 13:36:57 -0600 (CST) Subject: [Swift-commit] r3736 - trunk/src/org/globus/swift/data Message-ID: <20101206193657.3E96C9CC95@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 13:36:57 -0600 (Mon, 06 Dec 2010) New Revision: 3736 Modified: trunk/src/org/globus/swift/data/Director.java Log: Fix warning Modified: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java 2010-12-06 19:18:11 UTC (rev 3735) +++ trunk/src/org/globus/swift/data/Director.java 2010-12-06 19:36:57 UTC (rev 3736) @@ -78,11 +78,11 @@ /** Read in the user-supplied CDM policy file. */ - public static void load(File policyFile) throws IOException { - logger.info("CDM file: " + policyFile); + public static void load(File file) throws IOException { + logger.info("CDM file: " + file); enabled = true; - Director.policyFile = policyFile; - List list = LineReader.read(policyFile); + Director.policyFile = file; + List list = LineReader.read(file); for (String s : list) addLine(s); } @@ -177,10 +177,10 @@ If the location is added twice, the second addition is considered to be an empty allocation with no CDM state. */ - public static synchronized void addAllocation(String allocation) { - logger.debug("addAllocation(): " + allocation); - allocations.add(allocation); - broadcasted.put(allocation, new HashSet()); + public static synchronized void addBlock(String blockId) { + logger.debug("addBlock(): " + blockId); + allocations.add(blockId); + broadcasted.put(blockId, new HashSet()); doBroadcast(); } From noreply at svn.ci.uchicago.edu Mon Dec 6 15:22:28 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 15:22:28 -0600 (CST) Subject: [Swift-commit] r3737 - in trunk: libexec src/org/globus/swift/data src/org/globus/swift/data/policy src/org/griphyn/vdl/karajan src/org/griphyn/vdl/util Message-ID: <20101206212228.AD0929CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 15:22:28 -0600 (Mon, 06 Dec 2010) New Revision: 3737 Modified: trunk/libexec/cdm_broadcast.sh trunk/src/org/globus/swift/data/Action.java trunk/src/org/globus/swift/data/Director.java trunk/src/org/globus/swift/data/policy/AllocationHook.java trunk/src/org/globus/swift/data/policy/Broadcast.java trunk/src/org/griphyn/vdl/karajan/Loader.java trunk/src/org/griphyn/vdl/util/VDL2Config.java Log: Cleanups and fixes to CDM BROADCAST Modified: trunk/libexec/cdm_broadcast.sh =================================================================== --- trunk/libexec/cdm_broadcast.sh 2010-12-06 19:36:57 UTC (rev 3736) +++ trunk/libexec/cdm_broadcast.sh 2010-12-06 21:22:28 UTC (rev 3737) @@ -1,34 +1,14 @@ #!/bin/sh +# Called by org.globus.swift.data.policy.Broadcast +# usage: cdm_broadcast.sh +# copies DIR/FILE to DEST via MODE with logging to LOG + SWIFT_HOME=$( dirname $( dirname $0 ) ) -LOG=${SWIFT_HOME}/etc/cdm_broadcast.log -# For each given location, broadcast the given files to it -# Input: bgp_broadcast [-l *]* -bgp_broadcast() -{ - while [[ ${*} != "" ]] - do - L=$1 # -l - shift - ARGS=$1 # Location - shift - while true - do - if [[ $1 == "-l" || $1 == "" ]] - then - break - fi - ARGS="${ARGS} $1" - shift - done - bgp_broadcast_perform ${ARGS} - done -} - # Broadcast the given files to the given location # Input: bgp_broadcast_perform * -bgp_broadcast_perform() +bgp_broadcast() { LOCATION=$1 shift @@ -81,21 +61,48 @@ local_broadcast() { - DIR=$1 + ALLOCATION=$1 # Ignored (LOCAL_FILE) FILE=$2 DEST=$3 cp -v ${FILE} ${DEST}/${FILE} } -set -x +MODE=$1 +LOG=$2 +shift 2 + +[[ ${LOG} != /dev/null ]] && set -x { declare -p PWD LOG - if [[ $( uname -p ) == "ppc64" ]] + if [[ ${MODE} == "f2cn" ]] then - bgp_broadcast ${*} + BROADCAST="bgp_broadcast" + elif [[ ${MODE} == "file" ]] + then + BROADCAST="local_broadcast" else - bgp_local ${*} + echo "Unknown broadcast mode!" + exit 1 fi -} >> /tmp/cdm_broadcast.log 2>&1 # ${LOG} 2>&1 + while [[ ${*} != "" ]] + do + L=$1 # -l + shift + ARGS=$1 # Location + shift + while true + do + if [[ $1 == "-l" || $1 == "" ]] + then + break + fi + ARGS="${ARGS} $1" + shift + done + ${BROADCAST} ${ARGS} + done +} >> ${LOG} 2>&1 + +exit 0 Modified: trunk/src/org/globus/swift/data/Action.java =================================================================== --- trunk/src/org/globus/swift/data/Action.java 2010-12-06 19:36:57 UTC (rev 3736) +++ trunk/src/org/globus/swift/data/Action.java 2010-12-06 21:22:28 UTC (rev 3737) @@ -1,7 +1,5 @@ package org.globus.swift.data; -import java.io.IOException; - import org.apache.log4j.Logger; import org.globus.cog.karajan.arguments.Arg; Modified: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java 2010-12-06 19:36:57 UTC (rev 3736) +++ trunk/src/org/globus/swift/data/Director.java 2010-12-06 21:22:28 UTC (rev 3737) @@ -19,9 +19,11 @@ import org.globus.swift.data.policy.Policy; import org.globus.swift.data.policy.Broadcast; import org.globus.swift.data.util.LineReader; +import org.griphyn.vdl.util.VDL2Config; /** * Manages CDM policies for files based on pattern matching. + * Initialized when given a CDM file to read. * @author wozniak * */ @@ -54,6 +56,7 @@ /** Remember the files we have broadcasted. Map from allocations to filenames. + The keys represent the list of known allocations. NOTE: must be accessed only using synchronized Director methods */ private static Map> broadcasted = @@ -65,28 +68,46 @@ */ private static Set broadcastWork = new LinkedHashSet(); - /** - Remember all known allocations - */ - private static List allocations = new ArrayList(); - public static boolean isEnabled() { return enabled; } + /** + How to broadcast: either "file" or "f2cn" + */ + public static String broadcastMode = null; + + /** + Convenience reference to the Swift logfile name + */ + public static String logfile = null; + /** Read in the user-supplied CDM policy file. */ public static void load(File file) throws IOException { - logger.info("CDM file: " + file); - enabled = true; + logger.debug("CDM file: " + file); Director.policyFile = file; List list = LineReader.read(file); for (String s : list) addLine(s); + init(); } + static void init() throws IOException + { + VDL2Config config = VDL2Config.getConfig(); + + broadcastMode = config.getProperty("cdm.broadcast.mode"); + logfile = config.getProperty("logfile"); + + if (broadcastMode.equals("file")) + broadcasted.put("LOCAL_FILE", new HashSet()); + + enabled = true; + } + /** A line is either a rule or a property. */ @@ -173,13 +194,12 @@ } /** - Add a location to the list of allocations. + Add a Coasters allocation to the list of allocations. If the location is added twice, the second addition is considered to be an empty allocation with no CDM state. */ - public static synchronized void addBlock(String blockId) { - logger.debug("addBlock(): " + blockId); - allocations.add(blockId); + public static synchronized void addAllocation(String blockId) { + logger.debug("addAllocation(): " + blockId); broadcasted.put(blockId, new HashSet()); doBroadcast(); } @@ -195,7 +215,7 @@ return; logger.debug("doBroadcast(): batch: " + batch); Broadcast.perform(batch); - markBroadcasts(batch); + noteBroadcasts(batch); logger.debug("marked: " + broadcasted); } @@ -207,15 +227,18 @@ */ private static Map> getBroadcastBatch() { logger.debug("getBroadcastBatch(): "); - Map> batch = new LinkedHashMap>(); + Map> batch = + new LinkedHashMap>(); for (String file : broadcastWork) { logger.debug("file: " + file); - logger.debug("allocations: " + allocations); - for (String allocation : allocations) { - Set files = broadcasted.get(allocation); + for (Map.Entry> entry : + broadcasted.entrySet()) { + String allocation = entry.getKey(); + Set files = entry.getValue(); logger.debug("files: " + files); if (! files.contains(file)) { - logger.debug("adding: " + file + " to: " + allocation); + logger.debug("adding: " + file + + " to: " + allocation); List work = batch.get(allocation); if (work == null) { work = new ArrayList(); @@ -229,18 +252,21 @@ } /** - Mark that the files in the given batch have been successfully broadcasted. + Note that the files in the given batch have been successfully + broadcasted by adding them to Set broadcasted. Should only be called by {@link doBroadcast}. */ - private static void markBroadcasts(Map> batch) { + private static void noteBroadcasts(Map> batch) { logger.debug("markBroadcasts: batch: " + batch); - for (Map.Entry> entry : batch.entrySet()) { + for (Map.Entry> entry : + batch.entrySet()) { String location = entry.getKey(); logger.debug("markBroadcasts: location: " + location); List files = entry.getValue(); for (String file : files) { Set contents = broadcasted.get(location); - assert (! contents.contains(file)); + assert(! contents.contains(file)); logger.debug("markBroadcasts: add: " + file); contents.add(file); } Modified: trunk/src/org/globus/swift/data/policy/AllocationHook.java =================================================================== --- trunk/src/org/globus/swift/data/policy/AllocationHook.java 2010-12-06 19:36:57 UTC (rev 3736) +++ trunk/src/org/globus/swift/data/policy/AllocationHook.java 2010-12-06 21:22:28 UTC (rev 3737) @@ -12,15 +12,12 @@ * */ public class AllocationHook extends Hook { - public void blockActive(StatusEvent e) + public void blockActive(StatusEvent e, String blockId) { if (!Director.isEnabled()) return; - System.out.println("blockActive: " + e.getStatus().getMessage()); - String msg = e.getStatus().getMessage(); - String[] tokens = msg.split("="); - String allocation = tokens[1]; - Director.addAllocation(allocation); + System.out.println("blockActive: " + blockId); + Director.addAllocation(blockId); } } Modified: trunk/src/org/globus/swift/data/policy/Broadcast.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Broadcast.java 2010-12-06 19:36:57 UTC (rev 3736) +++ trunk/src/org/globus/swift/data/policy/Broadcast.java 2010-12-06 21:22:28 UTC (rev 3737) @@ -4,10 +4,14 @@ import java.util.Arrays; import java.util.List; import java.util.Map; + +import org.apache.log4j.Logger; import org.globus.swift.data.Director; public class Broadcast extends Policy { + static Logger logger = Logger.getLogger(Broadcast.class); + String destination = null; @Override @@ -25,7 +29,7 @@ */ public static void perform(Map> batch) { String[] line = commandLine(batch); - System.out.println("Broadcast.perform(): " + Arrays.toString(line)); + logger.debug("arguments: " + Arrays.toString(line)); Process process = null; try { process = Runtime.getRuntime().exec(line); @@ -33,7 +37,8 @@ } catch (Exception e) { e.printStackTrace(); - throw new RuntimeException("Could not launch external broadcast"); + throw new RuntimeException + ("Could not launch external broadcast", e); } int code = process.exitValue(); if (code != 0) @@ -47,6 +52,11 @@ String home = System.getProperties().getProperty("swift.home"); List line = new ArrayList(); line.add(home+"/libexec/cdm_broadcast.sh"); + line.add(Director.broadcastMode); + if (logger.isDebugEnabled()) + line.add(Director.logfile); + else + line.add("/dev/null"); for (Map.Entry> entry : batch.entrySet()) { line.add("-l"); String location = entry.getKey(); @@ -54,7 +64,7 @@ line.add(location); for (String file : files) { line.add(file); - line.add(getDestination(file)+"/"+file); + line.add(getDestination(file)); } } String[] result = new String[line.size()]; Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-12-06 19:36:57 UTC (rev 3736) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-12-06 21:22:28 UTC (rev 3737) @@ -35,7 +35,6 @@ import org.globus.cog.karajan.workflow.PrintStreamChannel; import org.globus.cog.karajan.workflow.nodes.FlowElement; import org.globus.cog.karajan.workflow.nodes.grid.AbstractGridNode; -import org.globus.cog.karajan.workflow.service.channels.AbstractKarajanChannel; import org.globus.cog.util.ArgumentParser; import org.globus.cog.util.ArgumentParserException; import org.globus.swift.data.Director; @@ -93,9 +92,6 @@ if (ap.isPresent(ARG_MONITOR)) { new Monitor().start(); } - if (ap.isPresent(ARG_CDMFILE)) { - loadCDM(ap); - } if (!ap.hasValue(ArgumentParser.DEFAULT)) { error("No SwiftScript program specified"); } @@ -130,6 +126,10 @@ setupLogging(ap, projectName, runID); logger.debug("Max heap: " + Runtime.getRuntime().maxMemory()); + if (ap.isPresent(ARG_CDMFILE)) { + loadCDM(ap); + } + if (!(new File(project).exists())) { logger.error("Input file " + project + " does not exist."); System.exit(4); @@ -454,6 +454,9 @@ logfile = projectName + "-" + runID + ".log"; } + VDL2Config config = VDL2Config.getConfig(); + config.put("logfile", logfile); + File f = new File(logfile); FileAppender fa = (FileAppender) getAppender(FileAppender.class); Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2010-12-06 19:36:57 UTC (rev 3736) +++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2010-12-06 21:22:28 UTC (rev 3737) @@ -95,6 +95,8 @@ put("status.mode", "files"); put("wrapper.parameter.mode", "args"); put("wrapper.invocation.mode", "absolute"); + + put("cdm.broadcast.mode", "file"); } private VDL2Config(VDL2Config other) { From noreply at svn.ci.uchicago.edu Mon Dec 6 15:24:40 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 15:24:40 -0600 (CST) Subject: [Swift-commit] r3738 - trunk/tests/cdm Message-ID: <20101206212440.46FBE9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 15:24:40 -0600 (Mon, 06 Dec 2010) New Revision: 3738 Modified: trunk/tests/cdm/README.txt Log: Add GATHER note Modified: trunk/tests/cdm/README.txt =================================================================== --- trunk/tests/cdm/README.txt 2010-12-06 21:22:28 UTC (rev 3737) +++ trunk/tests/cdm/README.txt 2010-12-06 21:24:40 UTC (rev 3738) @@ -3,3 +3,4 @@ 200 series : DIRECT tests 300 series : BROADCAST tests 400 series : EXTERNAL tests +500 series : GATHER tests From noreply at svn.ci.uchicago.edu Mon Dec 6 15:25:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 15:25:25 -0600 (CST) Subject: [Swift-commit] r3739 - trunk/tests/cdm Message-ID: <20101206212525.D8B359CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 15:25:25 -0600 (Mon, 06 Dec 2010) New Revision: 3739 Added: trunk/tests/cdm/301-bcast.check.sh trunk/tests/cdm/301-bcast.clean.sh trunk/tests/cdm/301-bcast.setup.sh trunk/tests/cdm/301-bcast.swift Modified: trunk/tests/cdm/ trunk/tests/cdm/fs.template.data Log: New CDM BROADCAST test Property changes on: trunk/tests/cdm ___________________________________________________________________ Name: svn:ignore + fs.data Added: trunk/tests/cdm/301-bcast.check.sh =================================================================== --- trunk/tests/cdm/301-bcast.check.sh (rev 0) +++ trunk/tests/cdm/301-bcast.check.sh 2010-12-06 21:25:25 UTC (rev 3739) @@ -0,0 +1,8 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 301/301-input.txt || exit 1 +grep $( uname -m ) 301-output.txt || exit 1 + +exit 0 Property changes on: trunk/tests/cdm/301-bcast.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/301-bcast.clean.sh =================================================================== --- trunk/tests/cdm/301-bcast.clean.sh (rev 0) +++ trunk/tests/cdm/301-bcast.clean.sh 2010-12-06 21:25:25 UTC (rev 3739) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -rv 301-output.txt 301 + +exit 0 Property changes on: trunk/tests/cdm/301-bcast.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/301-bcast.setup.sh =================================================================== --- trunk/tests/cdm/301-bcast.setup.sh (rev 0) +++ trunk/tests/cdm/301-bcast.setup.sh 2010-12-06 21:25:25 UTC (rev 3739) @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +mkdir -pv 301 || exit 1 + +{ + uname -a + date +} > 301-input.txt + +exit 0 Property changes on: trunk/tests/cdm/301-bcast.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/301-bcast.swift =================================================================== --- trunk/tests/cdm/301-bcast.swift (rev 0) +++ trunk/tests/cdm/301-bcast.swift 2010-12-06 21:25:25 UTC (rev 3739) @@ -0,0 +1,14 @@ + +type file; + +app (file o) copy (file i) +{ + cp @i @o; +} + +// Will be transformed by CDM to 201/output.txt: +file f1<"301-input.txt">; + +file f2<"301-output.txt">; + +f2 = copy(f1); Modified: trunk/tests/cdm/fs.template.data =================================================================== --- trunk/tests/cdm/fs.template.data 2010-12-06 21:24:40 UTC (rev 3738) +++ trunk/tests/cdm/fs.template.data 2010-12-06 21:25:25 UTC (rev 3739) @@ -1,2 +1,9 @@ + +property GATHER_LIMIT 50000 # ~50KB +property GATHER_DIR /tmp/cdm_gather +property GATHER_TARGET _PWD_/501-output + rule .*201-output.txt DIRECT _PWD_/201 rule .*210-input.txt EXTERNAL _PWD_/external.sh +rule .*301-input.txt BROADCAST _PWD_/301 +rule .*501-output.*.txt GATHER From noreply at svn.ci.uchicago.edu Mon Dec 6 15:28:01 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 15:28:01 -0600 (CST) Subject: [Swift-commit] r3740 - trunk/tests/cdm Message-ID: <20101206212801.44FF29CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 15:28:00 -0600 (Mon, 06 Dec 2010) New Revision: 3740 Modified: trunk/tests/cdm/301-bcast.swift Log: Fix comment Modified: trunk/tests/cdm/301-bcast.swift =================================================================== --- trunk/tests/cdm/301-bcast.swift 2010-12-06 21:25:25 UTC (rev 3739) +++ trunk/tests/cdm/301-bcast.swift 2010-12-06 21:28:00 UTC (rev 3740) @@ -6,7 +6,7 @@ cp @i @o; } -// Will be transformed by CDM to 201/output.txt: +// Will be transformed by CDM to 301/301-input.txt: file f1<"301-input.txt">; file f2<"301-output.txt">; From noreply at svn.ci.uchicago.edu Mon Dec 6 15:52:18 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 15:52:18 -0600 (CST) Subject: [Swift-commit] r3741 - trunk/src/org/globus/swift/data/policy Message-ID: <20101206215218.1617A9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 15:52:17 -0600 (Mon, 06 Dec 2010) New Revision: 3741 Modified: trunk/src/org/globus/swift/data/policy/Local.java Log: Change the toString() name of CDM LOCAL from "PULL" to "LOCAL" Modified: trunk/src/org/globus/swift/data/policy/Local.java =================================================================== --- trunk/src/org/globus/swift/data/policy/Local.java 2010-12-06 21:28:00 UTC (rev 3740) +++ trunk/src/org/globus/swift/data/policy/Local.java 2010-12-06 21:52:17 UTC (rev 3741) @@ -5,7 +5,7 @@ public class Local extends Policy { public String toString() { - return "PULL"; + return "LOCAL"; } @Override From noreply at svn.ci.uchicago.edu Mon Dec 6 15:53:10 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 15:53:10 -0600 (CST) Subject: [Swift-commit] r3742 - trunk/tests/cdm Message-ID: <20101206215310.801B89CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 15:53:10 -0600 (Mon, 06 Dec 2010) New Revision: 3742 Added: trunk/tests/cdm/601-local.check.sh trunk/tests/cdm/601-local.clean.sh trunk/tests/cdm/601-local.setup.sh trunk/tests/cdm/601-local.swift Modified: trunk/tests/cdm/README.txt trunk/tests/cdm/fs.template.data Log: Add new CDM LOCAL test (cp) Added: trunk/tests/cdm/601-local.check.sh =================================================================== --- trunk/tests/cdm/601-local.check.sh (rev 0) +++ trunk/tests/cdm/601-local.check.sh 2010-12-06 21:53:10 UTC (rev 3742) @@ -0,0 +1,8 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 601/601-input.txt || exit 1 +grep $( uname -m ) 601-output.txt || exit 1 + +exit 0 Property changes on: trunk/tests/cdm/601-local.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/601-local.clean.sh =================================================================== --- trunk/tests/cdm/601-local.clean.sh (rev 0) +++ trunk/tests/cdm/601-local.clean.sh 2010-12-06 21:53:10 UTC (rev 3742) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -rv 601-output.txt 601 + +exit 0 Property changes on: trunk/tests/cdm/601-local.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/601-local.setup.sh =================================================================== --- trunk/tests/cdm/601-local.setup.sh (rev 0) +++ trunk/tests/cdm/601-local.setup.sh 2010-12-06 21:53:10 UTC (rev 3742) @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +mkdir -pv 601 || exit 1 + +{ + uname -a + date +} > 601/601-input.txt + +exit 0 Property changes on: trunk/tests/cdm/601-local.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/601-local.swift =================================================================== --- trunk/tests/cdm/601-local.swift (rev 0) +++ trunk/tests/cdm/601-local.swift 2010-12-06 21:53:10 UTC (rev 3742) @@ -0,0 +1,14 @@ + +type file; + +app (file o) copy (file i) +{ + cp @i @o; +} + +// Will be transformed by CDM to 601/601-input.txt: +file f1<"601-input.txt">; + +file f2<"601-output.txt">; + +f2 = copy(f1); Modified: trunk/tests/cdm/README.txt =================================================================== --- trunk/tests/cdm/README.txt 2010-12-06 21:52:17 UTC (rev 3741) +++ trunk/tests/cdm/README.txt 2010-12-06 21:53:10 UTC (rev 3742) @@ -4,3 +4,4 @@ 300 series : BROADCAST tests 400 series : EXTERNAL tests 500 series : GATHER tests +600 series : LOCAL tests Modified: trunk/tests/cdm/fs.template.data =================================================================== --- trunk/tests/cdm/fs.template.data 2010-12-06 21:52:17 UTC (rev 3741) +++ trunk/tests/cdm/fs.template.data 2010-12-06 21:53:10 UTC (rev 3742) @@ -7,3 +7,4 @@ rule .*210-input.txt EXTERNAL _PWD_/external.sh rule .*301-input.txt BROADCAST _PWD_/301 rule .*501-output.*.txt GATHER +rule .*601-input.*.txt LOCAL cp _PWD_/601 From noreply at svn.ci.uchicago.edu Mon Dec 6 16:02:58 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 16:02:58 -0600 (CST) Subject: [Swift-commit] r3743 - trunk/tests/cdm Message-ID: <20101206220258.37E2C9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 16:02:58 -0600 (Mon, 06 Dec 2010) New Revision: 3743 Added: trunk/tests/cdm/602-local.check.sh trunk/tests/cdm/602-local.clean.sh trunk/tests/cdm/602-local.setup.sh trunk/tests/cdm/602-local.swift Modified: trunk/tests/cdm/fs.template.data Log: New CDM LOCAL test with dd Added: trunk/tests/cdm/602-local.check.sh =================================================================== --- trunk/tests/cdm/602-local.check.sh (rev 0) +++ trunk/tests/cdm/602-local.check.sh 2010-12-06 22:02:58 UTC (rev 3743) @@ -0,0 +1,8 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 602/602-input.txt || exit 1 +grep $( uname -m ) 602-output.txt || exit 1 + +exit 0 Property changes on: trunk/tests/cdm/602-local.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/602-local.clean.sh =================================================================== --- trunk/tests/cdm/602-local.clean.sh (rev 0) +++ trunk/tests/cdm/602-local.clean.sh 2010-12-06 22:02:58 UTC (rev 3743) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -rv 602-output.txt 602 + +exit 0 Property changes on: trunk/tests/cdm/602-local.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/602-local.setup.sh =================================================================== --- trunk/tests/cdm/602-local.setup.sh (rev 0) +++ trunk/tests/cdm/602-local.setup.sh 2010-12-06 22:02:58 UTC (rev 3743) @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +mkdir -pv 602 || exit 1 + +{ + uname -a + date +} > 602/602-input.txt + +exit 0 Property changes on: trunk/tests/cdm/602-local.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/cdm/602-local.swift =================================================================== --- trunk/tests/cdm/602-local.swift (rev 0) +++ trunk/tests/cdm/602-local.swift 2010-12-06 22:02:58 UTC (rev 3743) @@ -0,0 +1,14 @@ + +type file; + +app (file o) copy (file i) +{ + cp @i @o; +} + +// Will be transformed by CDM to 602/602-input.txt: +file f1<"602-input.txt">; + +file f2<"602-output.txt">; + +f2 = copy(f1); Modified: trunk/tests/cdm/fs.template.data =================================================================== --- trunk/tests/cdm/fs.template.data 2010-12-06 21:53:10 UTC (rev 3742) +++ trunk/tests/cdm/fs.template.data 2010-12-06 22:02:58 UTC (rev 3743) @@ -8,3 +8,4 @@ rule .*301-input.txt BROADCAST _PWD_/301 rule .*501-output.*.txt GATHER rule .*601-input.*.txt LOCAL cp _PWD_/601 +rule .*602-input.*.txt LOCAL dd _PWD_/602 From noreply at svn.ci.uchicago.edu Mon Dec 6 16:04:56 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 6 Dec 2010 16:04:56 -0600 (CST) Subject: [Swift-commit] r3744 - trunk/tests/cdm Message-ID: <20101206220456.08F2D9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-06 16:04:55 -0600 (Mon, 06 Dec 2010) New Revision: 3744 Added: trunk/tests/cdm/601-local-cp.check.sh trunk/tests/cdm/601-local-cp.clean.sh trunk/tests/cdm/601-local-cp.setup.sh trunk/tests/cdm/601-local-cp.swift trunk/tests/cdm/602-local-dd.check.sh trunk/tests/cdm/602-local-dd.clean.sh trunk/tests/cdm/602-local-dd.setup.sh trunk/tests/cdm/602-local-dd.swift Removed: trunk/tests/cdm/601-local.check.sh trunk/tests/cdm/601-local.clean.sh trunk/tests/cdm/601-local.setup.sh trunk/tests/cdm/601-local.swift trunk/tests/cdm/602-local.check.sh trunk/tests/cdm/602-local.clean.sh trunk/tests/cdm/602-local.setup.sh trunk/tests/cdm/602-local.swift Log: Put cp/dd in the test name Copied: trunk/tests/cdm/601-local-cp.check.sh (from rev 3742, trunk/tests/cdm/601-local.check.sh) =================================================================== --- trunk/tests/cdm/601-local-cp.check.sh (rev 0) +++ trunk/tests/cdm/601-local-cp.check.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -0,0 +1,8 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 601/601-input.txt || exit 1 +grep $( uname -m ) 601-output.txt || exit 1 + +exit 0 Copied: trunk/tests/cdm/601-local-cp.clean.sh (from rev 3742, trunk/tests/cdm/601-local.clean.sh) =================================================================== --- trunk/tests/cdm/601-local-cp.clean.sh (rev 0) +++ trunk/tests/cdm/601-local-cp.clean.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -rv 601-output.txt 601 + +exit 0 Copied: trunk/tests/cdm/601-local-cp.setup.sh (from rev 3742, trunk/tests/cdm/601-local.setup.sh) =================================================================== --- trunk/tests/cdm/601-local-cp.setup.sh (rev 0) +++ trunk/tests/cdm/601-local-cp.setup.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +mkdir -pv 601 || exit 1 + +{ + uname -a + date +} > 601/601-input.txt + +exit 0 Copied: trunk/tests/cdm/601-local-cp.swift (from rev 3742, trunk/tests/cdm/601-local.swift) =================================================================== --- trunk/tests/cdm/601-local-cp.swift (rev 0) +++ trunk/tests/cdm/601-local-cp.swift 2010-12-06 22:04:55 UTC (rev 3744) @@ -0,0 +1,14 @@ + +type file; + +app (file o) copy (file i) +{ + cp @i @o; +} + +// Will be transformed by CDM to 601/601-input.txt: +file f1<"601-input.txt">; + +file f2<"601-output.txt">; + +f2 = copy(f1); Deleted: trunk/tests/cdm/601-local.check.sh =================================================================== --- trunk/tests/cdm/601-local.check.sh 2010-12-06 22:02:58 UTC (rev 3743) +++ trunk/tests/cdm/601-local.check.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -1,8 +0,0 @@ -#!/bin/sh - -set -x - -grep $( uname -m ) 601/601-input.txt || exit 1 -grep $( uname -m ) 601-output.txt || exit 1 - -exit 0 Deleted: trunk/tests/cdm/601-local.clean.sh =================================================================== --- trunk/tests/cdm/601-local.clean.sh 2010-12-06 22:02:58 UTC (rev 3743) +++ trunk/tests/cdm/601-local.clean.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -1,7 +0,0 @@ -#!/bin/sh - -set -x - -rm -rv 601-output.txt 601 - -exit 0 Deleted: trunk/tests/cdm/601-local.setup.sh =================================================================== --- trunk/tests/cdm/601-local.setup.sh 2010-12-06 22:02:58 UTC (rev 3743) +++ trunk/tests/cdm/601-local.setup.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -1,12 +0,0 @@ -#!/bin/sh - -set -x - -mkdir -pv 601 || exit 1 - -{ - uname -a - date -} > 601/601-input.txt - -exit 0 Deleted: trunk/tests/cdm/601-local.swift =================================================================== --- trunk/tests/cdm/601-local.swift 2010-12-06 22:02:58 UTC (rev 3743) +++ trunk/tests/cdm/601-local.swift 2010-12-06 22:04:55 UTC (rev 3744) @@ -1,14 +0,0 @@ - -type file; - -app (file o) copy (file i) -{ - cp @i @o; -} - -// Will be transformed by CDM to 601/601-input.txt: -file f1<"601-input.txt">; - -file f2<"601-output.txt">; - -f2 = copy(f1); Copied: trunk/tests/cdm/602-local-dd.check.sh (from rev 3743, trunk/tests/cdm/602-local.check.sh) =================================================================== --- trunk/tests/cdm/602-local-dd.check.sh (rev 0) +++ trunk/tests/cdm/602-local-dd.check.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -0,0 +1,8 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 602/602-input.txt || exit 1 +grep $( uname -m ) 602-output.txt || exit 1 + +exit 0 Copied: trunk/tests/cdm/602-local-dd.clean.sh (from rev 3743, trunk/tests/cdm/602-local.clean.sh) =================================================================== --- trunk/tests/cdm/602-local-dd.clean.sh (rev 0) +++ trunk/tests/cdm/602-local-dd.clean.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -rv 602-output.txt 602 + +exit 0 Copied: trunk/tests/cdm/602-local-dd.setup.sh (from rev 3743, trunk/tests/cdm/602-local.setup.sh) =================================================================== --- trunk/tests/cdm/602-local-dd.setup.sh (rev 0) +++ trunk/tests/cdm/602-local-dd.setup.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +mkdir -pv 602 || exit 1 + +{ + uname -a + date +} > 602/602-input.txt + +exit 0 Copied: trunk/tests/cdm/602-local-dd.swift (from rev 3743, trunk/tests/cdm/602-local.swift) =================================================================== --- trunk/tests/cdm/602-local-dd.swift (rev 0) +++ trunk/tests/cdm/602-local-dd.swift 2010-12-06 22:04:55 UTC (rev 3744) @@ -0,0 +1,14 @@ + +type file; + +app (file o) copy (file i) +{ + cp @i @o; +} + +// Will be transformed by CDM to 602/602-input.txt: +file f1<"602-input.txt">; + +file f2<"602-output.txt">; + +f2 = copy(f1); Deleted: trunk/tests/cdm/602-local.check.sh =================================================================== --- trunk/tests/cdm/602-local.check.sh 2010-12-06 22:02:58 UTC (rev 3743) +++ trunk/tests/cdm/602-local.check.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -1,8 +0,0 @@ -#!/bin/sh - -set -x - -grep $( uname -m ) 602/602-input.txt || exit 1 -grep $( uname -m ) 602-output.txt || exit 1 - -exit 0 Deleted: trunk/tests/cdm/602-local.clean.sh =================================================================== --- trunk/tests/cdm/602-local.clean.sh 2010-12-06 22:02:58 UTC (rev 3743) +++ trunk/tests/cdm/602-local.clean.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -1,7 +0,0 @@ -#!/bin/sh - -set -x - -rm -rv 602-output.txt 602 - -exit 0 Deleted: trunk/tests/cdm/602-local.setup.sh =================================================================== --- trunk/tests/cdm/602-local.setup.sh 2010-12-06 22:02:58 UTC (rev 3743) +++ trunk/tests/cdm/602-local.setup.sh 2010-12-06 22:04:55 UTC (rev 3744) @@ -1,12 +0,0 @@ -#!/bin/sh - -set -x - -mkdir -pv 602 || exit 1 - -{ - uname -a - date -} > 602/602-input.txt - -exit 0 Deleted: trunk/tests/cdm/602-local.swift =================================================================== --- trunk/tests/cdm/602-local.swift 2010-12-06 22:02:58 UTC (rev 3743) +++ trunk/tests/cdm/602-local.swift 2010-12-06 22:04:55 UTC (rev 3744) @@ -1,14 +0,0 @@ - -type file; - -app (file o) copy (file i) -{ - cp @i @o; -} - -// Will be transformed by CDM to 602/602-input.txt: -file f1<"602-input.txt">; - -file f2<"602-output.txt">; - -f2 = copy(f1); From noreply at svn.ci.uchicago.edu Tue Dec 7 19:21:35 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 7 Dec 2010 19:21:35 -0600 (CST) Subject: [Swift-commit] r3745 - text/parco10submission Message-ID: <20101208012135.E77DA9CC7F@svn.ci.uchicago.edu> Author: wilde Date: 2010-12-07 19:21:35 -0600 (Tue, 07 Dec 2010) New Revision: 3745 Added: text/parco10submission/ResponseToReviews.txt Modified: text/parco10submission/paper.tex Log: minor initial edits after PARCO review Added: text/parco10submission/ResponseToReviews.txt =================================================================== --- text/parco10submission/ResponseToReviews.txt (rev 0) +++ text/parco10submission/ResponseToReviews.txt 2010-12-08 01:21:35 UTC (rev 3745) @@ -0,0 +1,287 @@ + +---------- Forwarded message ---------- +Date: Thu, 14 Oct 2010 13:29:09 +From: Parallel Computing +To: wozniak at mcs.anl.gov +Cc: worleyph at ornl.gov, Rupak.Biswas at nasa.gov, Rajesh.Nishtala at gmail.com, + loliker at lbl.gov +Subject: Your Submission PARCO-D-10-00054 + +Ms. Ref. No.: PARCO-D-10-00054 +Title: Swift: A language for distributed parallel scripting +Parallel Computing + +Dear Justin, + +Reviewers have now commented on your paper. You will see that they are advising that you revise your manuscript. If you are prepared to undertake the work required, I would be pleased to reconsider my decision. + +For your guidance, reviewers' comments are appended below. + +If you decide to revise the work, please submit a list of changes or a rebuttal against each point which is being raised when you submit the revised manuscript. + +Your revision should be submitted before Nov 11, 2010. + +To submit a revision, please go to http://ees.elsevier.com/parco/ and login as an Author. On your Main Menu page is a folder entitled "Submissions Needing Revision". You will find your submission record there. + +When submitting your revised manuscript, please ensure that you upload the source files (e.g. (La)TeX or Word; (La)TeX files as 1 comprehensive file and each figure and table separately). Uploading a PDF file at this stage will create delays should your manuscript be finally accepted for publication. If your revised submission does not include the source files, we will contact you to request them. + +Yours sincerely, + +Patrick Haven Worley, Ph.D. +Associate Editor +Parallel Computing + +********************************************************** + +Reviewers' comments: + +Dear Authors, + Thank you for your submission to the Parallel Computing special issue on Emerging Programming Paradigms for Large-Scale Scientific Computing. As there were several concerns about the manuscript, we ask that you update your submission to address the reviewer feedback, and include a point by point response to the reviewer comments. + +With kind regards, +ParCo Guest Editors: +Rupak Biswas, Rajesh Nishtala, Lenny Oliker + + +**************************************************************************************** + +Reviewer #1: The paper describes Swift, a scripting language for distributed parallel applications. I have mixed feelings about this paper. I can see the usefulness of a scripting language like Swift. The problems Swift was designed to address are certainly interesting and important. + +On the other hand, I don't see much scientific merit in the paper. The paper reads more like a Swift user manual than a scientific paper. For the language design, the only thing that might be novel is the notion of mapped type, but I consider it to be quite minor. I also don't see any new ideas in the data-flow dependency based execution model. + +>>> Novelty and scientific merit: + +<<< + +For the distributed execution, one important missing piece is performance evaluation. Data locality is very important for data-intensive applications. As I understand it, data have to be moved in and out the clusters. So, understanding the cost of scheduling and data transfer is very important to validate the Swift design. Perhaps, it was +published somewhere else, but it would be nice to discuss it in this paper. Here are some more detailed comments: + +>>> Performance evaluation: + +<<< + +1. Swift uses restart log to reuse the results of successfully completed components. The paper mentioned "appropriate manual intervention". This seems to be something you can almost completely automate. Based on my experiences with large-scale and long running applications, this can be very useful. + +>>> Automation of restart + +<<< + +2. Swift reduces job submission cost using clustering. It is not clear to me if a subgraph can be batched together by your clustering technique. This obviously requires a little bit of analysis of the data-flow graph to do it properly. But it could be quite useful to achieve better data locality. + +>>> Clustering + +<<< + +3. In terms of programming models, modern systems such as Microsoft's DryadLINQ and Google's FlumeJava successfully integrate data-flow constructs into state of the art programming languages (C# and Java). This integration approach is quite nice and powerful. It would be nice if the authors can compare Swift with these two systems. + +>>> Comparison to Dryad and FlumeJava + +<<< + +Reviewer #2: The paper presents a powerful high-level scripting language, SwiftScript, for performing a massive number of tasks/jobs coupled with collections of file-based data. It describes details of the language syntax and semantics, which is based on data flow with support of arrays and procedures, and discusses the implementation and several use cases with the main focus on a grid. Although a similar work was published before, this paper gives elaborated summary of the technical details, which is useful for general audience. + +The current work is an attempt to overcome issues in dealing with a massive amount of simulations and data either on a grid or on a massive parallel system, such as large data files, various dependencies, fault tolerance. The framework simplifies the process in the scripting design by an "app" interface to the actual command line for program execution and a "mapper" to map data structures with actual files without the need of going into details of how the program actually gets executed and without explicitly specifying the types of the files involved. The paper also describes the implementation that is built on top of the CoG Karajan workflow engine with built-in fault tolerance and demonstrates the usefulness of the system with a number of examples. + +The primary focus of the work is for the grid applications, although the paper has indicated the applicability to other systems, such as more tightly coupled massive computing systems. + +For those who might not be familiar with the Karajan language, it would be useful to add a reference to the related work. + +>>> Reference to Karajan: <<< + +It would be helpful to include some discussion on the "auto-parallelization" capability (achieved via data flow analysis?). + +>>> auto-parallelization + +<<< + +Out of the four application examples presented, two of the cases (4.3 and 4.4) do not contain enough details to support the discussion; deleting the two examples should not affect the clarity of the paper. + +>>> Examples: clarify or delete + +<<< + +It would be helpful to elaborate more in example 4.2 on how each task/job gets scheduled onto Ranger nodes or how Swift interacts with the local batch job scheduler, which would in turn help audience understand better how SwiftScript could be used for a certain class of applications on a more tightly coupled massive computing environment (such as +parameter studies). + +>>> 4.2 - task scheduling on Ranger + +<<< + +There are a few minor typos in the manuscript. +For instance, on Page 7 section 2.2, in the code snippet: + + output[ix] = rotate(frames, 180); + +Should "frames" be "f" in this case? + +>>> typo + +<<< + +Reviewer #3: This is an interesting paper aimed at the practical problem of +managing a large ensemble of computations with possible dependencies +among the tasks. The general outline and structure of the paper is +fine. There are a number of small errors which should have been +caught by proofreading the manuscript. + +>>> typos + +<<< + +The most substantive comment I have concerns examples 4.3 and 4.4. I +believe that, as they stand, these are too sparsely annotated to add +significant value to the paper. So I suggest either documenting these +examples more carefully, so that a reader new to Swift can understand +them, or else omit these examples entirely. + +>>> examples 4.3 and 4.4: annotate or delete + +<<< + +Further comments: + +1. Sometimes "Swift" is used and sometimes "SwiftScript" is used. Is +there any reason to have two different terms? From the website it +appears that "SwiftScript" referred to an earlier version of the +language, so perhaps "SwiftScript" should be replaced by "Swift" +everywhere. + +>>> <<< + +2. It's a bit awkward that "single assignment" is used in section 2.1 +but not defined until section 2.3. + +>>> <<< + +3. The example on p. 5 appears erroneous, it seems the rotate +procedure should have an angle input in addition to the image input +(this is corrected on p.6). + +>>> <<< + +4. In section 2.2, should rotate be invoked as "rotate(f, 180)" instead +of "rotate(frames, 180)"? + +>>> <<< + +5. There are some acronyms in the paper that should, I believe, be +defined. In some cases the acronym can be omitted, for example I +believe RDBMS is only used once so there is really no need for the +acronym. Other acronyms appear in Figure 1: CoG, OSG, AWS; these +probably should be defined. Other acronyms: GRAM, fMRI (and FMRI, +cf. Fig. 2, which should probably be fMRI). + +>>> <<< + +6. All these appear: "stage in", "stage-in", "stagein". Please be +consistent (similarly for stage out). + +>>> <<< + +7. The mysterious numbers '12, 1000, 1000, "81 3 3"' in example 4.1 +might merit an explanation. + +>>> <<< + +8. The Figure numbering in the text of section 4.1 needs correction. + +>>> <<< + +9. In the text of section 4.1, I believe the variables or, iv, +direction, overwrite, and ov should be in a different font for clarity. +In the text of section 4.1 "n x n" should be in a different font and +"n2" is clearly wrong. + +>>> <<< + +10. Section 4.2, what is an "emblem experiment"? + +>>> <<< + +11. The margins in sections 4.2 and 4.4 need fixing as some lines +run completely off the page. + +>>> <<< + +12. "Karajan" is mentioned several times, there really should be short +definition of it and a reference to it in the bibliography. + +>>> <<< + +13. Many of the references look incomplete; journal references really +should have page numbers, some references are missing a year. +Reference 8 severely mangles "Bresnahan". + +>>> <<< + +14. It is somewhat disingenuous to refer to thousands and millions of +cores (section 6.2). All systems I know of are managed as nodes, +where each node might have 8 or 12 or 16 cores. This is no essential +simplification, I know, but why not be honest and refer to nodes +instead of cores? + +>>> <<< + +15. "Swift is implemented by compiling to a Karajan program". Why +would one want to _compile_ a scripting language? It seems more natural +(to this naive reader) to have an interpreter or a translator. + +>>> <<< + +16. The coaster idea looks quite interesting, could this be expanded, +or could an example with coasters be constructed? + +>>> <<< + +17. Table 1, 1st row, 3rd column: should it be f->data.txt instead of +f->file.txt? + +>>> <<< + +18. There are many (too many to list) typos, missing words, mistakes +such as "en queued" instead of "enqueued", subject/verb mismatches of +number and/or tense. A careful proofreading is sorely needed. + + +>>> <<< + +==== Other improvement notes + +mention futures in parco paper, show them visually to show fine grain + +mention habanero (c and java) and other fresh stack languages (x10) +compare to GEL - from SIngapore + +mention csp bsp sim and diff to mpi (IPO) + +Why a new model? + +examine determinism + +examine language vs library + +examine how it builds on karajan + + +--- + +Innov: fine grained parallelism; no need for flow analysis; +sep of concerns: how throttling and site mgmt are isolated + +How we can manage data locality + +How restart is more transparent than it sounds here + +Fine: how work takes off before a proc returns + +Add table of critical benchmarks on multi sys types + +How complex flows are easily composed + +How types and mappers encapsulate complexity + +2.3 order of exec: show more complex patterns here or later + +Second 2 is the part that reads like a LRM; make it more interesting + +2.5 don't say marker types Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-06 22:04:55 UTC (rev 3744) +++ text/parco10submission/paper.tex 2010-12-08 01:21:35 UTC (rev 3745) @@ -734,13 +734,13 @@ multiple sites, it is necessary to choose between the available sites. The Swift \emph{site selector} achieves this by maintaining a score for each site which determines the load that Swift will place on that site. -As a site is successful in executing jobs, this score will be increased -and as the site is uncsuccessful, this score will be decreased. In -addition to selecting between sites, this mechanism provides some +As a site succeeds in executing jobs, this score is increased +and as job executions at a site fail, this score is decreased. In +addition to selecting between sites, this mechanism provides a measure of dynamic rate limiting if sites fail due to overload~\cite{FTSH_2003}. This provides an empirically measured estimate of a site's ability to -bear load, distinct from more static information elsewhere published. +bear load, distinct from more static information published elsewhere. In part, this is due to unreliable and incomplete published information (for example, site policies restricting (for example) job counts are often not published in usable form); in addition, that site's ability @@ -774,8 +774,11 @@ will fail resulting ultimately in the entire script failing. In such a case, Swift provides a \emph{restart log} which encapsulates -which procedure invocations have been successfully completed. After -appropriate manual intervention, a subsequent Swift run may be started +which procedure invocations have been successfully completed. +%%%%%% What manual interv. and why??? +After +appropriate manual intervention, +a subsequent Swift run may be started with this restart log; this will suppress re-execution of already executed invocations but otherwise allow the script to continue. From noreply at svn.ci.uchicago.edu Wed Dec 8 12:15:33 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 12:15:33 -0600 (CST) Subject: [Swift-commit] r3746 - provenancedb Message-ID: <20101208181533.3247F9CC80@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-08 12:15:33 -0600 (Wed, 08 Dec 2010) New Revision: 3746 Modified: provenancedb/import-run-to-sql provenancedb/pql_functions.sql provenancedb/prepare-provenance-chart provenancedb/prov-to-sql.sh provenancedb/swift-prov-import-all-logs Log: Minor fixes. Modified: provenancedb/import-run-to-sql =================================================================== --- provenancedb/import-run-to-sql 2010-12-08 01:21:35 UTC (rev 3745) +++ provenancedb/import-run-to-sql 2010-12-08 18:15:33 UTC (rev 3746) @@ -9,5 +9,5 @@ # with kickstart records expected to be in the same directory as the # log file. -PROVIDPREFIX=$PROVIDPREFIX prov-to-sql.sh $1 +version=$version prov-to-sql.sh $1 Modified: provenancedb/pql_functions.sql =================================================================== --- provenancedb/pql_functions.sql 2010-12-08 01:21:35 UTC (rev 3745) +++ provenancedb/pql_functions.sql 2010-12-08 18:15:33 UTC (rev 3746) @@ -1,23 +1,32 @@ +-- Set of PosrgreSQL-specific SQL functions and PL/pgSQL procedures +-- to query provenance. + -- SQL Functions -CREATE OR REPLACE FUNCTION list_runs() RETURNS SETOF VARCHAR AS $$ - SELECT DISTINCT(log_filename) FROM workflow; +-- list_runs_* lists workflows recorded in the database by id or log_filename +DROP TYPE list_runs_type CASCADE; + +CREATE TYPE list_runs_type +AS (id VARCHAR, + log_filename VARCHAR, + swift_version VARCHAR, + start_time TIMESTAMP WITH TIME ZONE, + duration NUMERIC, + final_state VARCHAR); +CREATE OR REPLACE FUNCTION list_runs() +RETURNS SETOF list_runs_type AS $$ + SELECT id, log_filename, swift_version, TO_TIMESTAMP(start_time), + duration, import_status AS final_state + FROM workflow; $$ LANGUAGE SQL; --- lists distinct processes by name in a workflow -CREATE OR REPLACE FUNCTION process_names(wf_id VARCHAR) RETURNS SETOF VARCHAR AS $$ - SELECT DISTINCT(process.name) - FROM process - WHERE process.workflow_id=$1; -$$ LANGUAGE SQL; - -- lists variations in a parameter's value across workflows - DROP TYPE param_across_wf_type CASCADE; CREATE TYPE param_across_wf_type AS (workflow VARCHAR, parameter VARCHAR, value VARCHAR); -CREATE OR REPLACE FUNCTION param_across_wf(param_name VARCHAR) RETURNS SETOF param_across_wf_type AS $$ +CREATE OR REPLACE FUNCTION param_across_wf(param_name VARCHAR) +RETURNS SETOF param_across_wf_type AS $$ SELECT workflow.log_filename,ds_usage.param_name,variable.value FROM variable,ds_usage,process,workflow WHERE variable.id=ds_usage.dataset_id AND ds_usage.process_id=process.id AND @@ -25,6 +34,16 @@ GROUP BY workflow.log_filename,ds_usage.param_name,variable.value; $$ LANGUAGE SQL; +-- lists variations of the values of a set of parameters + +DROP TYPE param_across_wf_class_type CASCADE; +CREATE TYPE param_across_wf_class_type AS (workflow VARCHAR, parameter VARCHAR, value VARCHAR); + +CREATE OR REPLACE FUNCTION param_across_wf_class(param_name VARCHAR, wf_class VARACHAR) +RETURNS SETOF param_across_wf_type AS $$ + +$$ LANGUAGE plpgsql; + -- correlate a parameter with workflow runtime statistics DROP TYPE correlate_param_runtime_type CASCADE; CREATE TYPE correlate_param_runtime_type @@ -34,21 +53,41 @@ parameter VARCHAR, parameter_value VARCHAR); -CREATE OR REPLACE FUNCTION correlate_param_runtime(param_name VARCHAR) RETURNS SETOF correlate_param_runtime_type AS $$ +CREATE OR REPLACE FUNCTION correlate_param_runtime(param_name VARCHAR) +RETURNS SETOF correlate_param_runtime_type AS $$ SELECT A.workflow,to_timestamp(B.start_time),B.duration,A.parameter,A.value FROM param_across_wf($1) AS A, workflow AS B WHERE A.workflow=B.log_filename; $$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION correlate_param_runtime(param_name VARCHAR) +RETURNS SETOF correlate_param_runtime_type AS $$ + SELECT workflow.id,to_timestamp(workflow.start_time),workflow.duration,ds_usage.param_name,variable.value + FROM variable,ds_usage,process,workflow + WHERE variable.id=ds_usage.dataset_id AND ds_usage.process_id=process.id AND + process.workflow_id=workflow.id AND ds_usage.param_name=$1 +$$ LANGUAGE SQL; + +-- lists distinct processes by name in a workflow +CREATE OR REPLACE FUNCTION process_names(wf_id VARCHAR) +RETURNS SETOF VARCHAR AS $$ + SELECT DISTINCT(process.name) + FROM process + WHERE process.workflow_id=$1; +$$ LANGUAGE SQL; + +-- lists distinct processes in a + -- OOPS-specific functions + + CREATE OR REPLACE FUNCTION list_oops_runs() RETURNS SETOF VARCHAR AS $$ SELECT DISTINCT(value) FROM annot_wf_txt WHERE annot_wf_txt.name=('oops_run_id'); $$ LANGUAGE SQL; DROP TYPE oops_param_across_wf_type CASCADE; CREATE TYPE oops_param_across_wf_type AS (oops_run_id VARCHAR, param_name VARCHAR, variable VARCHAR); - CREATE OR REPLACE FUNCTION oops_param_across_wf(VARCHAR) RETURNS SETOF oops_param_across_wf_type AS $$ SELECT annot_wf_txt.value,ds_usage.param_name,variable.value FROM variable,ds_usage,process,annot_wf_txt @@ -63,7 +102,6 @@ DROP TYPE oops_summary CASCADE; CREATE TYPE oops_summary AS (oops_run_id VARCHAR, start_time TIMESTAMP WITH TIME ZONE, duration_sec NUMERIC, swift_version VARCHAR); - CREATE OR REPLACE FUNCTION oops_run_summary(varchar) RETURNS SETOF oops_summary AS $$ SELECT annot_wf_txt.value as oops_run_id, to_timestamp(workflow.start_time) as start_time, workflow.duration as duration_sec,workflow.swift_version as swift_version @@ -80,16 +118,15 @@ DROP TYPE oops_wf_param_summary CASCADE; CREATE TYPE oops_wf_param_summary AS (oops_run_id varchar, param_name varchar, value varchar); - CREATE OR REPLACE FUNCTION oops_variable_summary() RETURNS SETOF oops_wf_param_summary AS $$ SELECT annot_wf_txt.value,ds_usage.param_name,variable.value FROM variable,ds_usage,process,annot_wf_txt WHERE variable.id=ds_usage.dataset_id and ds_usage.process_id=process.id and process.workflow_id=annot_wf_txt.id; $$ LANGUAGE SQL; + DROP TYPE oops_param_summary CASCADE; CREATE TYPE oops_param_summary AS (param_name varchar, value varchar); - CREATE OR REPLACE FUNCTION oops_science_summary(varchar) RETURNS SETOF oops_param_summary AS $$ SELECT ds_usage.param_name,variable.value FROM variable,ds_usage,process,annot_wf_txt Modified: provenancedb/prepare-provenance-chart =================================================================== --- provenancedb/prepare-provenance-chart 2010-12-08 01:21:35 UTC (rev 3745) +++ provenancedb/prepare-provenance-chart 2010-12-08 18:15:33 UTC (rev 3746) @@ -8,8 +8,8 @@ export RUNID=$(basename $1 .log) -export WFID=$PROVIDPREFIX"execute:${RUNID}:" -export EXECUTE2PREFIX=$PROVIDPREFIX"execute2:${RUNID}:" +export WFID="execute:${RUNID}:" +export EXECUTE2PREFIX="execute2:${RUNID}:" # will output log information about datasets from a log file passed as $1 Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2010-12-08 01:21:35 UTC (rev 3745) +++ provenancedb/prov-to-sql.sh 2010-12-08 18:15:33 UTC (rev 3746) @@ -2,10 +2,10 @@ export RUNID=$(basename $1 .log) -export WFID=$PROVIDPREFIX"execute:${RUNID}:" +export WFID="execute:${RUNID}:" # TODO is there already a URI form for identifying workflows? -export WF=$PROVIDPREFIX"execute:${RUNID}:run" +export WF="${RUNID}" echo Generating SQL for $RUNID @@ -27,34 +27,48 @@ echo "INSERT INTO execute2 (id, execute_id, start_time, duration, final_state, site) VALUES ('$globalid', '$inv_id', $start_time, $duration, '$endstate', '$site');" >> tmp-e2.sql done < execute2.global.event -while read col1 col2 col3 col4 col5 threadst namest lhsst rhsst resultst; do - thread=`echo $threadst | awk 'BEGIN { FS = "=" }; {print $2}'` - name=`echo $namest | awk 'BEGIN { FS = "=" }; {print $2}'` - lhs=`echo $lhsst | awk 'BEGIN { FS = "=" }; {print $2}'` - rhs=`echo $rhsst | awk 'BEGIN { FS = "=" }; {print $2}'` - result=`echo $resultst | awk 'BEGIN { FS = "=" }; {print $2}'` +while read col1 col2 col3 col4 col5 thread name lhs rhs result; do + thread=$(echo $thread | awk 'BEGIN { FS = "=" }; {print $2}') + name=$(echo $name | awk 'BEGIN { FS = "=" }; {print $2}') + lhs=$(echo $lhs | awk 'BEGIN { FS = "=" }; {print $2}') + rhs=$(echo $rhs | awk 'BEGIN { FS = "=" }; {print $2}') + result=$(echo $result | awk 'BEGIN { FS = "=" }; {print $2}') operatorid="${WFID}operator:$thread" - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$lhs');" >> tmp-ds.sql - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$rhs');" >> tmp-ds.sql - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$result');" >> tmp-ds.sql + if [ $version -le 3726 ]; then + $lhs=$(echo $lhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + $rhs=$(echo $rhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + $result=$(echo $result | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + fi + + echo "INSERT INTO dataset (id) VALUES ('$lhs');" >> tmp-ds.sql + echo "INSERT INTO dataset (id) VALUES ('$rhs');" >> tmp-ds.sql + echo "INSERT INTO dataset (id) VALUES ('$result');" >> tmp-ds.sql echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$operatorid', 'operator', '$name', '$WF');" >> tmp-p.sql - echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$PROVIDPREFIX$lhs', 'lhs');" >> tmp-dsu.sql - echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$PROVIDPREFIX$rhs', 'rhs');" >> tmp-dsu.sql - echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'O', '$PROVIDPREFIX$result', 'result');" >> tmp-dsu.sql + echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$lhs', 'lhs');" >> tmp-dsu.sql + echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$rhs', 'rhs');" >> tmp-dsu.sql + echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'O', '$result', 'result');" >> tmp-dsu.sql done < operators.txt while read id name output; do - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$output');" >> tmp-ds.sql + if [ $version -le 3726 ]; then + $output=$(echo $output | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + fi + echo "INSERT INTO dataset (id) VALUES ('$output');" >> tmp-ds.sql echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$id', 'function', '$name', '$WF');" >> tmp-p.sql - echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'O', '$PROVIDPREFIX$output', 'result');" >> tmp-dsu.sql + echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'O', '$output', 'result');" >> tmp-dsu.sql done < functions.txt while read id value; do # TODO need ordering/naming - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$value');" >> tmp-ds.sql - echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'I', '$PROVIDPREFIX$value', 'undefined');" >> tmp-dsu.sql + + if [ $version -le 3726 ]; then + $value=$(echo $value | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + fi + + echo "INSERT INTO dataset (id) VALUES ('$value');" >> tmp-ds.sql + echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'I', '$value', 'undefined');" >> tmp-dsu.sql done < function-inputs.txt @@ -63,19 +77,35 @@ done < invocation-procedure-names.txt while read outer inner; do - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$outer');" >> tmp-ds.sql - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$inner');" >> tmp-ds.sql - echo "INSERT INTO ds_containment (out_id, in_id) VALUES ('$PROVIDPREFIX$outer', '$PROVIDPREFIX$inner');" >> tmp-dsc.sql + + if [ $version -le 3726 ]; then + $outer=$(echo $outer | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + $inner=$(echo $inner | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + fi + + echo "INSERT INTO dataset (id) VALUES ('$outer');" >> tmp-ds.sql + echo "INSERT INTO dataset (id) VALUES ('$inner');" >> tmp-ds.sql + echo "INSERT INTO ds_containment (out_id, in_id) VALUES ('$outer', '$inner');" >> tmp-dsc.sql done < tie-containers.txt while read dataset filename; do - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$dataset');" >> tmp-ds.sql - echo "INSERT INTO file (id, filename) VALUES ('$PROVIDPREFIX$dataset', '$filename');" >> tmp-f.sql + + if [ $version -le 3726 ]; then + $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + fi + + echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql + echo "INSERT INTO file (id, filename) VALUES ('$dataset', '$filename');" >> tmp-f.sql done < dataset-filenames.txt while read dataset value; do - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$dataset');" >> tmp-ds.sql - echo "INSERT INTO variable (id, value) VALUES ('$PROVIDPREFIX$dataset', '$value');" >> tmp-v.sql + + if [ $version -le 3726 ]; then + $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + fi + + echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql + echo "INSERT INTO variable (id, value) VALUES ('$dataset', '$value');" >> tmp-v.sql done < dataset-values.txt while read start duration wfid rest; do @@ -114,8 +144,13 @@ else dir=O fi - echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$dataset');" >> tmp-ds.sql - echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$thread', '$dir', '$PROVIDPREFIX$dataset', '$variable');" >> tmp-dsu.sql + + if [ $version -le 3726 ]; then + $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + fi + + echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql + echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$thread', '$dir', '$dataset', '$variable');" >> tmp-dsu.sql done < tie-data-invocs.txt while read id ; do Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2010-12-08 01:21:35 UTC (rev 3745) +++ provenancedb/swift-prov-import-all-logs 2010-12-08 18:15:33 UTC (rev 3746) @@ -50,22 +50,19 @@ fi export RUNID=$(basename $filename .log) - if [ $version -le 3726 ]; then - PROVIDPREFIX="tag:ci.uchicago.edu,2008:swiftlogs:" - fi - export WF=$PROVIDPREFIX"execute:${RUNID}:run" + export WF="${RUNID}" echo "INSERT INTO workflow (id, log_filename, swift_version, import_status) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD echo version $version in log file $filename echo ============= will import ============= - PROVIDPREFIX=$PROVIDPREFIX prepare-for-import $filename + prepare-for-import $filename if [ "$?" != "0" ]; then echo prepare-for-import failed exit 2 fi - PROVIDPREFIX=$PROVIDPREFIX import-run-to-sql $filename + version=$version import-run-to-sql $filename if [ "$?" != "0" ]; then echo import-run-to-sql failed exit 3 From noreply at svn.ci.uchicago.edu Wed Dec 8 12:19:28 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 12:19:28 -0600 (CST) Subject: [Swift-commit] r3747 - provenancedb/etc Message-ID: <20101208181928.80D679CC80@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-08 12:19:27 -0600 (Wed, 08 Dec 2010) New Revision: 3747 Modified: provenancedb/etc/provenance.config.ci Log: Minor update. Modified: provenancedb/etc/provenance.config.ci =================================================================== --- provenancedb/etc/provenance.config.ci 2010-12-08 18:15:33 UTC (rev 3746) +++ provenancedb/etc/provenance.config.ci 2010-12-08 18:19:27 UTC (rev 3747) @@ -2,7 +2,6 @@ # presently hardcoded # this is the path to log repo on benc's laptop -export LOGREPO=~/swift-logs/ +export LOGREPO=~/swift-logs +export SQLCMD="psql -U provdb -h db.ci.uchicago.edu provdb" -# export SQLCMD="psql -p 5435 -d provdb " - From noreply at svn.ci.uchicago.edu Wed Dec 8 13:28:18 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 13:28:18 -0600 (CST) Subject: [Swift-commit] r3748 - provenancedb Message-ID: <20101208192818.527E39CC80@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-08 13:28:18 -0600 (Wed, 08 Dec 2010) New Revision: 3748 Modified: provenancedb/prov-to-sql.sh Log: Minor fixes. Modified: provenancedb/prov-to-sql.sh =================================================================== --- provenancedb/prov-to-sql.sh 2010-12-08 18:19:27 UTC (rev 3747) +++ provenancedb/prov-to-sql.sh 2010-12-08 19:28:18 UTC (rev 3748) @@ -37,9 +37,9 @@ operatorid="${WFID}operator:$thread" if [ $version -le 3726 ]; then - $lhs=$(echo $lhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') - $rhs=$(echo $rhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') - $result=$(echo $result | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + lhs=$(echo $lhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + rhs=$(echo $rhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + result=$(echo $result | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') fi echo "INSERT INTO dataset (id) VALUES ('$lhs');" >> tmp-ds.sql @@ -53,7 +53,7 @@ while read id name output; do if [ $version -le 3726 ]; then - $output=$(echo $output | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + output=$(echo $output | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') fi echo "INSERT INTO dataset (id) VALUES ('$output');" >> tmp-ds.sql echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$id', 'function', '$name', '$WF');" >> tmp-p.sql @@ -64,7 +64,7 @@ # TODO need ordering/naming if [ $version -le 3726 ]; then - $value=$(echo $value | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + value=$(echo $value | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') fi echo "INSERT INTO dataset (id) VALUES ('$value');" >> tmp-ds.sql @@ -79,8 +79,8 @@ while read outer inner; do if [ $version -le 3726 ]; then - $outer=$(echo $outer | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') - $inner=$(echo $inner | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + outer=$(echo $outer | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + inner=$(echo $inner | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') fi echo "INSERT INTO dataset (id) VALUES ('$outer');" >> tmp-ds.sql @@ -91,7 +91,7 @@ while read dataset filename; do if [ $version -le 3726 ]; then - $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') fi echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql @@ -101,7 +101,7 @@ while read dataset value; do if [ $version -le 3726 ]; then - $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') fi echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql @@ -146,7 +146,7 @@ fi if [ $version -le 3726 ]; then - $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') + dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g') fi echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql From noreply at svn.ci.uchicago.edu Wed Dec 8 16:28:44 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 16:28:44 -0600 (CST) Subject: [Swift-commit] r3749 - in provenancedb: . apps/oops Message-ID: <20101208222844.8973A9CC80@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-08 16:28:44 -0600 (Wed, 08 Dec 2010) New Revision: 3749 Modified: provenancedb/apps/oops/oops_extractor.sh provenancedb/prov-init.sql provenancedb/swift-prov-import-all-logs Log: Minor fixes. Modified: provenancedb/apps/oops/oops_extractor.sh =================================================================== --- provenancedb/apps/oops/oops_extractor.sh 2010-12-08 19:28:18 UTC (rev 3748) +++ provenancedb/apps/oops/oops_extractor.sh 2010-12-08 22:28:44 UTC (rev 3749) @@ -6,7 +6,7 @@ # OOPS' Swift logs. PROVDB_HOME=~/provenancedb -PROTESTS_HOME=//CASP +PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP IMPORT_HOME=~/protests source $PROVDB_HOME/etc/provenance.config @@ -27,12 +27,14 @@ if ! grep --silent $i provdb_imported; then if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/psim.loops-*.log; then cd swift-logs - for j in `ls $PROTESTS_HOME/$k/$i | grep psim.loops-`; do - ln -s $PROTESTS_HOME/$k/$i/$j - done + #for j in `ls $PROTESTS_HOME/$k/$i | grep psim.loops-`; do + # ln -s $PROTESTS_HOME/$k/$i/$j + #done cd import # swift-prov-import-all-logs also controls what already has been # imported, so it does not repeat work + echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config + echo "export SQLCMD=\"psql -U provdb -h db.ci.uchicago.edu provdb\"" >> $PROVDB_HOME/etc/provenance.config $PROVDB_HOME/swift-prov-import-all-logs cd $IMPORT_HOME echo $i >> provdb_imported Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2010-12-08 19:28:18 UTC (rev 3748) +++ provenancedb/prov-init.sql 2010-12-08 22:28:44 UTC (rev 3749) @@ -252,7 +252,7 @@ annot_wf_num, annot_wf_txt, annot_wf_bool, - extrainfo, +-- extrainfo, createarray, createarray_member, array_range Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2010-12-08 19:28:18 UTC (rev 3748) +++ provenancedb/swift-prov-import-all-logs 2010-12-08 22:28:44 UTC (rev 3749) @@ -41,35 +41,38 @@ EXISTING=$($SQLCMD --tuples-only -c "select count(*) from workflow where log_filename='$filename';") if [ "$EXISTING" -eq "0" ]; then - echo IMPORTING - - if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then - wfstatus="SUCCESS" - else - wfstatus="FAIL" + PROV_ENABLED=$(grep provenance $filename | wc -l) + if [ "$PROV_ENABLED" -ge "0" ]; then + echo IMPORTING + + if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then + wfstatus="SUCCESS" + else + wfstatus="FAIL" + fi + + export RUNID=$(basename $filename .log) + + export WF="${RUNID}" + + echo "INSERT INTO workflow (id, log_filename, swift_version, import_status) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD + + echo version $version in log file $filename + echo ============= will import ============= + prepare-for-import $filename + if [ "$?" != "0" ]; then + echo prepare-for-import failed + exit 2 + fi + version=$version import-run-to-sql $filename + if [ "$?" != "0" ]; then + echo import-run-to-sql failed + exit 3 + fi + + # import-run-to-xml $filename + echo SKIP: provenance.log not true in swift.properties fi - - export RUNID=$(basename $filename .log) - - export WF="${RUNID}" - - echo "INSERT INTO workflow (id, log_filename, swift_version, import_status) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD - - echo version $version in log file $filename - echo ============= will import ============= - prepare-for-import $filename - if [ "$?" != "0" ]; then - echo prepare-for-import failed - exit 2 - fi - version=$version import-run-to-sql $filename - if [ "$?" != "0" ]; then - echo import-run-to-sql failed - exit 3 - fi - - # import-run-to-xml $filename - else echo SKIP: Already known in workflow fi From noreply at svn.ci.uchicago.edu Wed Dec 8 16:38:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 16:38:55 -0600 (CST) Subject: [Swift-commit] r3750 - provenancedb Message-ID: <20101208223855.182679CC80@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-08 16:38:54 -0600 (Wed, 08 Dec 2010) New Revision: 3750 Modified: provenancedb/swift-prov-import-all-logs Log: Minor fixes. Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2010-12-08 22:28:44 UTC (rev 3749) +++ provenancedb/swift-prov-import-all-logs 2010-12-08 22:38:54 UTC (rev 3750) @@ -71,6 +71,7 @@ fi # import-run-to-xml $filename + else echo SKIP: provenance.log not true in swift.properties fi else From noreply at svn.ci.uchicago.edu Wed Dec 8 17:05:00 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 17:05:00 -0600 (CST) Subject: [Swift-commit] r3751 - provenancedb Message-ID: <20101208230500.224479CC80@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-08 17:05:00 -0600 (Wed, 08 Dec 2010) New Revision: 3751 Modified: provenancedb/swift-prov-import-all-logs Log: Minor fixes. Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2010-12-08 22:38:54 UTC (rev 3750) +++ provenancedb/swift-prov-import-all-logs 2010-12-08 23:05:00 UTC (rev 3751) @@ -42,7 +42,7 @@ if [ "$EXISTING" -eq "0" ]; then PROV_ENABLED=$(grep provenance $filename | wc -l) - if [ "$PROV_ENABLED" -ge "0" ]; then + if [ $PROV_ENABLED -gt 0 ]; then echo IMPORTING if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then @@ -72,7 +72,7 @@ # import-run-to-xml $filename else - echo SKIP: provenance.log not true in swift.properties + echo SKIP: provenance.log not set to true in etc/swift.properties fi else echo SKIP: Already known in workflow From noreply at svn.ci.uchicago.edu Wed Dec 8 19:39:04 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 19:39:04 -0600 (CST) Subject: [Swift-commit] r3754 - text/parco10submission Message-ID: <20101209013904.5BC019CC80@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-08 19:39:04 -0600 (Wed, 08 Dec 2010) New Revision: 3754 Modified: text/parco10submission/ResponseToReviews.txt text/parco10submission/Wozniak.bib text/parco10submission/paper.tex Log: more small changes in Parco paper Modified: text/parco10submission/ResponseToReviews.txt =================================================================== --- text/parco10submission/ResponseToReviews.txt 2010-12-09 01:29:08 UTC (rev 3753) +++ text/parco10submission/ResponseToReviews.txt 2010-12-09 01:39:04 UTC (rev 3754) @@ -181,8 +181,13 @@ probably should be defined. Other acronyms: GRAM, fMRI (and FMRI, cf. Fig. 2, which should probably be fMRI). ->>> <<< +>>> +RDBMS removed. +FMRI fixed. + +<<< + 6. All these appear: "stage in", "stage-in", "stagein". Please be consistent (similarly for stage out). @@ -230,8 +235,12 @@ should have page numbers, some references are missing a year. Reference 8 severely mangles "Bresnahan". ->>> <<< +>>> +Ref 8 fixed. + +<<< + 14. It is somewhat disingenuous to refer to thousands and millions of cores (section 6.2). All systems I know of are managed as nodes, where each node might have 8 or 12 or 16 cores. This is no essential Modified: text/parco10submission/Wozniak.bib =================================================================== --- text/parco10submission/Wozniak.bib 2010-12-09 01:29:08 UTC (rev 3753) +++ text/parco10submission/Wozniak.bib 2010-12-09 01:39:04 UTC (rev 3754) @@ -288,7 +288,7 @@ } @INPROCEEDINGS{GridFTP_2005, - author = {William Allcock and John Bresnahan an Rajkumar Kettimuthu and Michael + author = {William Allcock and John Bresnahan and Rajkumar Kettimuthu and Michael Link and Catalin Dumitrescu and Ioan Raicu and Ian Foster}, title = {The {G}lobus Striped {GridFTP} Framework and Server}, booktitle = {Proc. Supercomputing}, Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-09 01:29:08 UTC (rev 3753) +++ text/parco10submission/paper.tex 2010-12-09 01:39:04 UTC (rev 3754) @@ -526,8 +526,8 @@ \end{verbatim} Sometimes data may be stored in a form that does not fit with Swift's -file-and-site model; \katznote{Swift's ``file-and-site model'' hasn't been introduced before. I'm not even sure what it is.} for example, data might be stored in an RDBMS on some -database server. In that case, a variable can be declared to have +file-and-site model; \katznote{Swift's ``file-and-site model'' hasn't been introduced before. I'm not even sure what it is.} for example, data might be stored in an database on a +server. In that case, a variable can be declared to have \verb|extern| type. This indicates that Swift should use the variable to determine execution dependency, but should not attempt other data management; for example, it will not perform any form of data stage-in or stage-out; @@ -967,7 +967,7 @@ \begin{figure}[htbp] \begin{center} \includegraphics[scale=1]{img/IMG_fmridataset} - \caption{FMRI application data structure\label{FMRI_app_image}} + \caption{fMRI application data structure\label{FMRI_app_image}} \end{center} \end{figure} @@ -1012,7 +1012,7 @@ } \end{verbatim} - \caption{FMRI application in SwiftScript\label{FMRI_app_script}} + \caption{fMRI application in SwiftScript\label{FMRI_app_script}} \end{figure} In this example, the logical structure of the fMRI dataset shown in From noreply at svn.ci.uchicago.edu Wed Dec 8 20:02:37 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 20:02:37 -0600 (CST) Subject: [Swift-commit] r3755 - text/parco10submission Message-ID: <20101209020237.C34029CC80@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-08 20:02:37 -0600 (Wed, 08 Dec 2010) New Revision: 3755 Modified: text/parco10submission/paper.tex Log: adding note to Parco paper Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-09 01:39:04 UTC (rev 3754) +++ text/parco10submission/paper.tex 2010-12-09 02:02:37 UTC (rev 3755) @@ -85,6 +85,9 @@ \katznote{in general, I would like to use application, job, task, program, etc. in some way that is consistent throughout the paper, perhaps defined in the introduction} +\katznote{are swift tasks idempotent? are swift scripts are deterministic (maybe?) deadlock-free? (no)} + + Swift is a scripting language designed for composing ordinary application programs (serial or parallel) into distributed, parallelized applications for execution on grids and supercomputers From noreply at svn.ci.uchicago.edu Wed Dec 8 22:10:37 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 22:10:37 -0600 (CST) Subject: [Swift-commit] r3756 - text/parco10submission Message-ID: <20101209041037.1FAFB9CC95@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-08 22:10:36 -0600 (Wed, 08 Dec 2010) New Revision: 3756 Modified: text/parco10submission/ResponseToReviews.txt text/parco10submission/paper.tex Log: more fixes based on reviewer comments Modified: text/parco10submission/ResponseToReviews.txt =================================================================== --- text/parco10submission/ResponseToReviews.txt 2010-12-09 02:02:37 UTC (rev 3755) +++ text/parco10submission/ResponseToReviews.txt 2010-12-09 04:10:36 UTC (rev 3756) @@ -148,13 +148,21 @@ language, so perhaps "SwiftScript" should be replaced by "Swift" everywhere. ->>> <<< +>>> +this has been done. + +<<< + 2. It's a bit awkward that "single assignment" is used in section 2.1 but not defined until section 2.3. ->>> <<< +>>> +fixed. + +<<< + 3. The example on p. 5 appears erroneous, it seems the rotate procedure should have an angle input in addition to the image input (this is corrected on p.6). @@ -185,6 +193,8 @@ RDBMS removed. FMRI fixed. +GRAM doesn't seem to need explanation, it's cited where first used. +still need to think about acronyms in Figure 1. <<< @@ -200,8 +210,13 @@ 7. The mysterious numbers '12, 1000, 1000, "81 3 3"' in example 4.1 might merit an explanation. ->>> <<< +>>> +It seems clear that these are parameters passed into the app program; +I don't think explanation is needed + +<<< + 8. The Figure numbering in the text of section 4.1 needs correction. >>> @@ -215,8 +230,12 @@ In the text of section 4.1 "n x n" should be in a different font and "n2" is clearly wrong. ->>> <<< +>>> +fixed. + +<<< + 10. Section 4.2, what is an "emblem experiment"? >>> <<< @@ -224,8 +243,12 @@ 11. The margins in sections 4.2 and 4.4 need fixing as some lines run completely off the page. ->>> <<< +>>> +fixed in 4.2 - not sure about the state of 4.4 yet + +<<< + 12. "Karajan" is mentioned several times, there really should be short definition of it and a reference to it in the bibliography. @@ -238,6 +261,7 @@ >>> Ref 8 fixed. +page numbers and years still need to be fixed. <<< Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-09 02:02:37 UTC (rev 3755) +++ text/parco10submission/paper.tex 2010-12-09 04:10:36 UTC (rev 3756) @@ -175,7 +175,7 @@ ongoing and future work in the Swift project, and we offer concluding remarks in Section~\ref{Conclusion}. -\section{The SwiftScript language} +\section{The Swift language} \label{Language} The Swift programming model is data-oriented: it encapsulates the @@ -238,8 +238,12 @@ procedures. Swift variables hold either primitive values, files, or collections of -files. All variables are \emph{single assignment}, which provides the -basis for Swift's model of procedure chaining. Procedures are +files. All variables are \emph{single-assignment} (meaning +that they must be assigned to exactly one value during execution), +which provides the +basis for Swift's model of procedure chaining. (Note that Swift arrays are not +strictly single-assignment, but each element of an array is, as discussed in +Section~\ref{ordering}.) Procedures are executed when their input parameters have all been set from existing data or prior procedure executions. Procedures are chained by specifying that an output variable of one procedure is passed as the @@ -314,15 +318,15 @@ The examples above have used the type \verb|image| without a definition of that type. We can declare it as a \emph{marker type}, -which has no structure exposed to SwiftScript: +which has no structure exposed to Swift: \begin{verbatim} type image; \end{verbatim} This does not indicate that the data is unstructured; it indicates -that the structure of the data is not exposed to SwiftScript. -SwiftScript will treat variables of this type as individual opaque +that the structure of the data is not exposed to Swift. +Swift will treat variables of this type as individual opaque files. With mechanisms to declare types, map variables to data files, and @@ -389,9 +393,10 @@ procedure, using each execution's output as the input to the next step. \subsection{Ordering of execution} +\label{ordering} -Non-array variables are \emph{single-assignment}, which means that -they must be assigned to exactly one value during execution. A +As previously stated, non-array variables are single-assignment, +which means thatthey must be assigned to exactly one value during execution. A procedure or expression will be executed when all of its input parameters have been assigned values. As a result of such execution, more variables may become assigned, possibly allowing further parts of @@ -413,7 +418,7 @@ z=q(y); \end{verbatim} -Arrays in SwiftScript are more generally \emph{monotonic}, \katznote{is this common use of monotonic? Are the arrays monotonic? Or is the assignment of elements in the array monotonic?} +Arrays in Swift are more generally \emph{monotonic}, \katznote{is this common use of monotonic? Are the arrays monotonic? Or is the assignment of elements in the array monotonic?} that is, knowledge about the content of an array increases during execution, but cannot otherwise change. Once a value for a particular element is @@ -446,9 +451,9 @@ \subsection{Compound procedures} As with many other programming languages, procedures consisting of -SwiftScript code can be defined. These differ from the previously +Swift code can be defined. These differ from the previously mentioned procedures declared with the \verb|app| keyword, as they -invoke other SwiftScript procedures rather than a component +invoke other Swift procedures rather than a component program. The basic structure of a composite procedure may be thought of as a graph of calls to other procedures. The following script will invoke two procedures, with an intermediate @@ -485,13 +490,13 @@ \subsection{More about types} \label{LanguageTypes} -Each variable and procedure parameter in SwiftScript is strongly typed. +Each variable and procedure parameter in Swift is strongly typed. Types are used to structure data, to aid in debugging and program correctness and to influence how Swift interacts with data. The \verb|image| type declared in previous examples is a \emph{marker type}. Marker types indicate that data for a variable is stored in a -single file with no further structure exposed at the SwiftScript level. +single file with no further structure exposed at the Swift level. Arrays have been mentioned above, in the arrays section. A code block may be applied to each element of an array using \verb|foreach|; or @@ -564,7 +569,7 @@ that database. The declaration of \verb|database| contains no mapping; and the procedures which use \verb|database| do not reference them in any way; the description of \verb|database| is entirely outside of the script. -The single assignment and execution ordering rules will still apply though; +The single-assignment and execution ordering rules will still apply though; populateDatabase will always be run before analyseDatabase. \subsection{Swift mappers} @@ -623,7 +628,7 @@ \hline \end{tabular} \end{center} - \caption{SwiftScript built-in mappers: conceptual syntax} + \caption{Swift built-in mappers: conceptual syntax} \label{mappertable} \end{table} @@ -635,7 +640,7 @@ \subsection{The execution environment for component programs} \label{LanguageEnvironment} -A SwiftScript \verb|app| declaration describes how a component program +A Swift \verb|app| declaration describes how a component program is invoked. In order to ensure the correctness of the Swift model, the environment in which programs are executed needs to be constrained. @@ -771,11 +776,11 @@ \subsection{Reliable execution} \label{ExecutingReliably} -The functional nature of SwiftScript provides a clearly defined +The functional nature of Swift provides a clearly defined interface to imperative components, which in addition to allowing Swift great flexibility in where and when it runs component programs, allows those imperative components to be treated as atomic components that can be -executed multiple times for any given SwiftScript procedure invocation. +executed multiple times for any given Swift procedure invocation. This facilitates three different reliability mechanisms that are implemented by the runtime and that need not be exposed at the language level: \emph{retries}, \emph{restarts} and \emph{replication}. @@ -783,7 +788,7 @@ In the simplest form of error handling in Swift, if a component program fails, Swift will make a subsequent attempt to rerun the program. In contrast to many other systems, retry here is at the level -of the SwiftScript procedure invocation, and includes completely +of the Swift procedure invocation, and includes completely reattempting site selection, stage-in, execution and stage-out. This provides a natural way to deal both with many transient errors, such as temporary network loss, and with many changes in site state. @@ -1015,11 +1020,11 @@ } \end{verbatim} - \caption{fMRI application in SwiftScript\label{FMRI_app_script}} + \caption{fMRI application in Swift\label{FMRI_app_script}} \end{figure} In this example, the logical structure of the fMRI dataset shown in -Figure~\ref{FMRI_app_image} can be represented by the SwiftScript type declarations in +Figure~\ref{FMRI_app_image} can be represented by the Swift type declarations in lines 1-6 in Figure~\ref{FMRI_app_script}. Here, Study is declared as containing an array of Group, which in turn contains an array of Subject, etc. Similarly, an fMRI Run is a series of brain scans called volumes, with a Volume @@ -1030,17 +1035,18 @@ which take typed data described by a mapper, perform computations on those data, and produce data to be stored in locations specified by a mapper. The -procedure reslice\_wf defines a compound procedure, which comprises +procedure {\tt reslice\_wf} defines a compound procedure, which comprises a series of procedure calls, using variables to establish data dependencies. -In the example, reslice\_wf defines a four-step pipeline computation, +In the example, {\tt reslice\_wf} defines a four-step pipeline computation, using variables to establish data dependencies. It applies reorientRun to a run first in the x axis and then in the y axis, and then aligns each image in the resulting run with the first image. The program -alignlinear \katznote{should this be alignlinearRun?} determines how to spatially adjust an image to match a +{\tt alignlinear} \katznote{should this be alignlinearRun?} determines how to spatially adjust an image to match a reference image, and produces an air parameter file. The actual -alignment is done by the program reslice. \katznote{or resliceRun?} Note that variable yR is +alignment is done by the program {\tt reslice}. +\katznote{or resliceRun?} Note that variable {\tt yR} is the output of the first step and the input of the second step, and as such, defines the data dependencies between the two steps. %The pipeline is %illustrated in the center of % Figure~\ref{FMRIFigure2}, @@ -1054,19 +1060,19 @@ The procedure reorientRun is also a compound procedure. The foreach statement defines an iteration over the input run -ir and applies the procedure reorient (which rotates a brain image +{\tt ir} and applies the procedure {\tt reorient} (which rotates a brain image along a certain axis) to each volume in the run to produces a reoriented run or. Because the multiple calls to reorient operate on independent data elements, they can proceed in parallel. The procedure reorient in this example is an atomic procedure. -This procedure has typed input parameters iv, direction and overwrite and -one output ov. The body specifies that it +This procedure has typed input parameters ({\tt iv}, {\tt direction}, and {\tt overwrite}) and +one output ({\tt ov}). The body specifies that it invokes a program (also called reorient) that will be dynamically mapped to a binary executable, which will execute at an execution site chosen by the Swift runtime system. The body also specifies how input parameters map to command line -arguments. The notation @filename is a built-in mapping function that +arguments. The notation {\tt @filename} is a built-in mapping function that maps a logical data structure to a physical file name. In this case, it extracts the file name of input header and output header, which are then put in the command line to invoke the reorient program. @@ -1110,7 +1116,7 @@ self-contained, structural equation models---we generated 65,535 R objects representing all models with 1 to 16 connections of varying weights between 4 pre-selected regions of interest. -A 4x4 matrix represents connections between the four regions, +A 4~x~4 matrix represents connections between the four regions, with 16 possible connections (connections between the same two regions but in different directions are tested separately). \katznote{should refer to figure~\ref{omxFigure} here? If so, the residual variances @@ -1138,8 +1144,8 @@ A model generator was developed for the OpenMx package and is designed explicitly to enable parallel execution of exhaustive or partially -pruned sets of model objects. Given an n x n covariance matrix it can -generate the entire set of possible models with anywhere from 0 to n2 +pruned sets of model objects. Given an $n$~x~$n$ covariance matrix it can +generate the entire set of possible models with anywhere from 0 to $n^2$ connections; however, it can also take as input a single index from that set and it will generate and run a single model. What this means in the context of workflow design is that the generator can be @@ -1167,7 +1173,7 @@ initweight, speech); 12. } \end{verbatim} -\caption{Swiftscript for 4-region exhaustive SEM for a single experimental condition\label{omxScript1}} +\caption{Swift script for 4-region exhaustive SEM for a single experimental condition\label{omxScript1}} \end{figure} First, a covariance matrix containing activation data for 4 brain regions, @@ -1218,7 +1224,7 @@ initweight, cond); 9. } \end{verbatim} -\caption{Swiftscript for 4-region exhaustive SEM for 2 experimental conditions\label{omxScript2}} +\caption{Swift script for 4-region exhaustive SEM for 2 experimental conditions\label{omxScript2}} \end{figure} @@ -1255,7 +1261,7 @@ p = readdata("paramslist.txt"); runDocks(p); \end{verbatim} -\caption{Swiftscript for running Dock\label{DockScript}} +\caption{Swift script for running Dock\label{DockScript}} \end{figure} \katznote{no text in this subsection -- no reference to Figure~\ref{DockScript}} @@ -1266,7 +1272,7 @@ data from a large dataset of files that categorize the Earth's surface, from the MODIS sensor instruments that orbit Earth on two NASA satellites of the Earth Observing System. -The SwiftScript analyzes the dataset to find the files with the ten +The Swift script analyzes the dataset to find the files with the ten largest total urban area and then produces a new dataset with viewable color images of those top-ten urban data ``tiles''. From noreply at svn.ci.uchicago.edu Wed Dec 8 22:45:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 22:45:24 -0600 (CST) Subject: [Swift-commit] r3757 - text/parco10submission Message-ID: <20101209044524.5F6239CC80@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-08 22:45:24 -0600 (Wed, 08 Dec 2010) New Revision: 3757 Modified: text/parco10submission/Wozniak.bib Log: fixing references - done up through #8 Modified: text/parco10submission/Wozniak.bib =================================================================== --- text/parco10submission/Wozniak.bib 2010-12-09 04:10:36 UTC (rev 3756) +++ text/parco10submission/Wozniak.bib 2010-12-09 04:45:24 UTC (rev 3757) @@ -1514,16 +1514,23 @@ month = nov } - at ARTICLE{GRAM_1998, - author = {Karl Czajkowski and Ian Foster and Nick Karonis and Carl Kesselman - and Stuart Martin and Warren Smith and Steven Tuecke}, - title = {A Resource Management Architecture for Metacomputing Systems}, - journal = {Lecture Notes in Computer Science}, - year = {1998}, - volume = {1459}, + at incollection {GRAM_1998, + author = {Czajkowski, Karl and Foster, Ian and Karonis, Nick and Kesselman, Carl and Martin, Stuart and Smith, Warren and Tuecke, Steven}, + affiliation = {University of Southern California Information Sciences Institute Marina del Rey 90292-6695 CA}, + title = {A resource management architecture for metacomputing systems}, + booktitle = {Job Scheduling Strategies for Parallel Processing}, + series = {Lecture Notes in Computer Science}, + editor = {Feitelson, Dror and Rudolph, Larry}, + publisher = {Springer Berlin / Heidelberg}, + isbn = {}, + pages = {62-82}, + volume = {1459}, + url = {http://dx.doi.org/10.1007/BFb0053981}, + note = {10.1007/BFb0053981}, + year = {1998} } -url = {citeseer.ist.psu.edu/czajkowski97resource.html} + @INPROCEEDINGS{Coallocation_1999, author = {Karl Czajkowski and Ian Foster and Carl Kesselman}, title = {Resource Co-Allocation in Computational Grids}, @@ -1954,7 +1961,8 @@ journal = {J. Supercomputer Applications}, year = {1997}, volume = {11}, - issue = {2} + issue = {2}, + pages = {115--128} } @ARTICLE{OGSA_2002, @@ -2614,17 +2622,24 @@ url = {citeseer.ist.psu.edu/huston95partially.html} } - at ARTICLE{BGP_2008, - author = {{IBM Blue Gene team}}, - title = {Overview of the {IBM} {Blue} {Gene/P} project}, - journal = {IBM J. Research and Development}, - year = {2008}, - volume = {52}, - number = {1/2}, - owner = {jwozniak}, - timestamp = {2009-01-26} -} + at article{BGP_2008, + author = {{IBM Blue Gene team}}, + title = {{Overview of the IBM Blue Gene/P project}}, + journal = {IBM J. Res. Dev.}, + volume = {52}, + issue = {1/2}, + month = {January}, + year = {2008}, + issn = {0018-8646}, + pages = {199--220}, + numpages = {22}, + url = {http://portal.acm.org/citation.cfm?id=1375990.1376008}, + acmid = {1376008}, + publisher = {IBM Corp.}, + address = {Riverton, NJ, USA}, +} + @UNPUBLISHED{IzHM0x, author = {Jes{\'u}s A.\ Izaguirre and Scott Hampton and Thierry Matthey}, title = {Shadow {H}ybrid {M}onte {C}arlo: An Improved Method for Conformational @@ -4414,14 +4429,21 @@ timestamp = {2007-08-16} } - at INPROCEEDINGS{NFS_1985, - author = {Russel Sandberg and David Goldberg and Steve Kleiman and Dan Walsh - and Bob Lyon}, - title = {Design and implementation of the {Sun Network File System}}, - booktitle = USENIX, - year = {1985} -} + at incollection{NFS_1985, + author = {Sandberg, R. and Golgberg, D. and Kleiman, S. and Walsh, D. and Lyon, B.}, + chapter = {Design and implementation of the Sun network filesystem}, + title = {Innovations in Internetworking}, + year = {1988}, + isbn = {0-89006-337-0}, + pages = {379--390}, + numpages = {12}, + url = {http://portal.acm.org/citation.cfm?id=59309.59338}, + acmid = {59338}, + publisher = {Artech House, Inc.}, + address = {Norwood, MA, USA}, +} + @INPROCEEDINGS{wetherall:duppkt:USENIX, author = {J. Santos and D. Wetherall}, title = {Increasing Effective Link Bandwidth by Suppressing Replicated Data}, @@ -4506,8 +4528,9 @@ @INPROCEEDINGS{GPFS_2002, author = {Frank Schmuck and Roger Haskin}, title = {{GPFS}: {A} shared-disk file system for large computing clusters}, - booktitle = FAST, + booktitle = {Proceedings of the Conference on File and Storage Technologies (FAST?02)}, year = {2002}, + pages={231--244}, owner = {wozniak}, timestamp = {2009.09.19} } @@ -5716,15 +5739,17 @@ } @INPROCEEDINGS{Swift_2007, - author = {Yong Zhao and Mihael Hategan and Ben Clifford and Ian Foster and - Gregor von Laszewski and Ioan Raicu and Tiberiu Stef-Praun and Mike - Wilde}, - title = {Swift: {Fast}, Reliable, Loosely Coupled Parallel Computation}, - booktitle = {Proc. Workshop on Scientific Workflows}, - year = {2007}, - owner = {wozniak}, - timestamp = {2009.02.05} -} +author={Yong Zhao and Hategan, M. and Clifford, B. and Foster, I. and von Laszewski, G. and Nefedova, V. and Raicu, I. and Stef-Praun, T. and Wilde, M.}, +booktitle={2007 IEEE Congress on Services}, +title={{Swift: Fast, Reliable, Loosely Coupled Parallel Computation}}, +year={2007}, +month={July}, +volume={}, +number={}, +pages={199 -206}, +keywords={GriPhyN virtual data system;Swift system;SwiftScript language;complex file system structures;runtime system;scripting language;formal specification;grid computing;software reliability;}, +doi={10.1109/SERVICES.2007.63}, +ISSN={},} @INPROCEEDINGS{VDG_Data_2004, author = {Y. Zhao and M. Wilde and I. Foster and J. Voeckler and J. Dobson @@ -6207,11 +6232,11 @@ pages = {18}, @INPROCEEDINGS{SunConstellation_2008, - title = {Performance and Scalability Study of + title = {{Performance and Scalability Study of Sun Constellation Cluster 'Ranger' - using Application-Based Benchmarks}, + using Application-Based Benchmarks}}, author = {Byoung-Do Kim and John E. Cazes}, - booktitle = {Proc. TeraGrid}, + booktitle = {Proc. TeraGrid'2008}, year = 2008 } From noreply at svn.ci.uchicago.edu Wed Dec 8 23:32:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 8 Dec 2010 23:32:32 -0600 (CST) Subject: [Swift-commit] r3758 - text/parco10submission Message-ID: <20101209053232.957439CC80@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-08 23:32:32 -0600 (Wed, 08 Dec 2010) New Revision: 3758 Modified: text/parco10submission/ResponseToReviews.txt text/parco10submission/Wozniak.bib Log: finished fixing references Modified: text/parco10submission/ResponseToReviews.txt =================================================================== --- text/parco10submission/ResponseToReviews.txt 2010-12-09 04:45:24 UTC (rev 3757) +++ text/parco10submission/ResponseToReviews.txt 2010-12-09 05:32:32 UTC (rev 3758) @@ -260,8 +260,7 @@ >>> -Ref 8 fixed. -page numbers and years still need to be fixed. +all Refs fixed. <<< Modified: text/parco10submission/Wozniak.bib =================================================================== --- text/parco10submission/Wozniak.bib 2010-12-09 04:45:24 UTC (rev 3757) +++ text/parco10submission/Wozniak.bib 2010-12-09 05:32:32 UTC (rev 3758) @@ -39,7 +39,7 @@ @STRING{PDSW = {Proc. Petascale Data Storage Workshop}} - at STRING{PTRS_A = {Philosophical Transactions of the Royal Society}} + at STRING{PTRS_A = {Philosophical Transactions of the Royal Society A}} @STRING{SC = {Proc. Supercomputing}} @@ -287,14 +287,20 @@ pages = {161-163} } - at INPROCEEDINGS{GridFTP_2005, - author = {William Allcock and John Bresnahan and Rajkumar Kettimuthu and Michael - Link and Catalin Dumitrescu and Ioan Raicu and Ian Foster}, - title = {The {G}lobus Striped {GridFTP} Framework and Server}, - booktitle = {Proc. Supercomputing}, - year = {2005}, - timestamp = {2007.02.01} -} + at inproceedings{GridFTP_2005, + author = {Allcock, William and Bresnahan, John and Kettimuthu, Rajkumar and Link, Michael and Dumitrescu, Catalin and Raicu, Ioan and Foster, Ian}, + title = {{The Globus Striped GridFTP Framework and Server}}, + booktitle = {Proceedings of the 2005 ACM/IEEE Conference on Supercomputing}, + series = {SC '05}, + year = {2005}, + isbn = {1-59593-061-2}, + pages = {54--}, + url = {http://dx.doi.org/10.1109/SC.2005.72}, + doi = {http://dx.doi.org/10.1109/SC.2005.72}, + acmid = {1105819}, + publisher = {IEEE Computer Society}, + address = {Washington, DC, USA}, +} @INPROCEEDINGS{almeroth00:mcastmonitor, author = {K. Almeroth}, @@ -1514,20 +1520,20 @@ month = nov } - at incollection {GRAM_1998, - author = {Czajkowski, Karl and Foster, Ian and Karonis, Nick and Kesselman, Carl and Martin, Stuart and Smith, Warren and Tuecke, Steven}, - affiliation = {University of Southern California Information Sciences Institute Marina del Rey 90292-6695 CA}, - title = {A resource management architecture for metacomputing systems}, - booktitle = {Job Scheduling Strategies for Parallel Processing}, - series = {Lecture Notes in Computer Science}, - editor = {Feitelson, Dror and Rudolph, Larry}, - publisher = {Springer Berlin / Heidelberg}, - isbn = {}, - pages = {62-82}, - volume = {1459}, - url = {http://dx.doi.org/10.1007/BFb0053981}, - note = {10.1007/BFb0053981}, - year = {1998} + at incollection {GRAM_1998, + author = {Czajkowski, Karl and Foster, Ian and Karonis, Nick and Kesselman, Carl and Martin, Stuart and Smith, Warren and Tuecke, Steven}, + affiliation = {University of Southern California Information Sciences Institute Marina del Rey 90292-6695 CA}, + title = {A resource management architecture for metacomputing systems}, + booktitle = {Job Scheduling Strategies for Parallel Processing}, + series = {Lecture Notes in Computer Science}, + editor = {Feitelson, Dror and Rudolph, Larry}, + publisher = {Springer Berlin / Heidelberg}, + isbn = {}, + pages = {62-82}, + volume = {1459}, + url = {http://dx.doi.org/10.1007/BFb0053981}, + note = {10.1007/BFb0053981}, + year = {1998} } @@ -1593,15 +1599,25 @@ timestamp = {2010.01.25} } - at INPROCEEDINGS{MapReduce_2004, - author = {Jeffrey Dean and Sanjay Ghemawat}, - title = {{MapReduce}: {Simplified} Data Processing on Large Clusters}, - booktitle = OSDI, - year = {2004}, - owner = {wozniak}, - timestamp = {2008-09-16} -} + at article{MapReduce_2004, + author = {Dean, Jeffrey and Ghemawat, Sanjay}, + title = {MapReduce: simplified data processing on large clusters}, + journal = {Commun. ACM}, + volume = {51}, + issue = {1}, + month = {January}, + year = {2008}, + issn = {0001-0782}, + pages = {107--113}, + numpages = {7}, + url = {http://doi.acm.org/10.1145/1327452.1327492}, + doi = {http://doi.acm.org/10.1145/1327452.1327492}, + acmid = {1327492}, + publisher = {ACM}, + address = {New York, NY, USA}, +} + @ARTICLE{OOPS_2009, author = {Joe DeBartolo and Andres Colubri and Ahbishek K. Jha and James Fitzgerald and Karl F. Freed, Tobin R. Sosnick}, @@ -1633,7 +1649,9 @@ distributed systems}, journal = {Scientific Programming}, year = {2005}, + pages={219--237}, volume = {13}, + issue={3}, timestamp = {2007.10.21} } @@ -2040,15 +2058,19 @@ url = {citeseer.ist.psu.edu/french90scientific.html} } - at ARTICLE{Condor-G_2002, - author = {James Frey and Todd Tannenbaum and Ian Foster and Miron Livny and - Steven Tuecke}, - title = {{Condor-G:} {A} Computation Management Agent for Multi-Institutional - Grids}, - journal = {Cluster Computing}, - year = {2002}, - volume = {5}, - number = {3} + at article {Condor-G_2002, + author = {Frey, James and Tannenbaum, Todd and Livny, Miron and Foster, Ian and Tuecke, Steven}, + title = {{Condor-G: A Computation Management Agent for Multi-Institutional Grids}}, + journal = {Cluster Computing}, + publisher = {Springer Netherlands}, + issn = {1386-7857}, + keyword = {Computer Science}, + pages = {237-246}, + volume = {5}, + issue = {3}, + url = {http://dx.doi.org/10.1023/A:1015617019423}, + note = {10.1023/A:1015617019423}, + year = {2002} } @INPROCEEDINGS{RT-Clusters_2006, @@ -3838,14 +3860,18 @@ } @ARTICLE{Scripting_1998, - author = {John Ousterhout}, - title = {Scripting: {Higher-level} programming for the 21st century}, - journal = {{IEEE} Computer}, - year = {1998}, - month = Mar, - timestamp = {2009.09.01} -} +author={Ousterhout, J.K.}, +journal={Computer}, title={Scripting: higher level programming for the 21st Century}, +year={1998}, +month=mar, +volume={31}, +number={3}, +pages={23--30}, +keywords={21st Century;C;C++;Perl;Tcl;data structures;higher level programming;scripting;scripting languages;software components;system programming languages;authoring languages;programming;}, +doi={10.1109/2.660187}, +ISSN={0018-9162},} + @ARTICLE{Sprite_1988, author = {John K. Ousterhout and Andrew R. Cherenson and Frederick Douglis and Michael N. Nelson and Brent B. Welch}, @@ -4099,15 +4125,23 @@ timestamp = {2009.09.02} } - at INPROCEEDINGS{Falkon_2008, - author = {Ioan Raicu and Zhao Zhang and Mike Wilde and Ian Foster and Pete - Beckman and Kamil Iskra and Ben Clifford}, - title = {Towards Loosely-Coupled Programming on Petascale Systems}, - booktitle = SC08, - year = {2008}, - owner = {wozniak}, - timestamp = {2009.02.05} -} + at inproceedings{Falkon_2008, + author = {Raicu, Ioan and Zhang, Zhao and Wilde, Mike and Foster, Ian and Beckman, Pete and Iskra, Kamil and Clifford, Ben}, + title = {Toward loosely coupled programming on petascale systems}, + booktitle = {Proceedings of the 2008 ACM/IEEE Conference on Supercomputing}, + series = {SC '08}, + year = {2008}, + isbn = {978-1-4244-2835-9}, + location = {Austin, Texas}, + pages = {22:1--22:12}, + articleno = {22}, + numpages = {12}, + url = {http://portal.acm.org/citation.cfm?id=1413370.1413393}, + acmid = {1413393}, + publisher = {IEEE Press}, + address = {Piscataway, NJ, USA}, + keywords = {Blue Gene, Falkon, Swift, high throughput computing, loosely coupled applications, many task computing, petascale}, +} @INPROCEEDINGS{Falkon_2007, author = {Ioan Raicu and Yong Zhao and Catalin Dumitrescu and Ian Foster and @@ -4431,8 +4465,8 @@ @incollection{NFS_1985, author = {Sandberg, R. and Golgberg, D. and Kleiman, S. and Walsh, D. and Lyon, B.}, - chapter = {Design and implementation of the Sun network filesystem}, - title = {Innovations in Internetworking}, + title = {Design and implementation of the Sun network filesystem}, + booktitle = {Innovations in Internetworking}, year = {1988}, isbn = {0-89006-337-0}, pages = {379--390}, @@ -5140,6 +5174,8 @@ year = {2005}, volume = {17}, number = {2-4}, + pages={323--356}, + doi={10.1002/cpe.938}, timestamp = {2007.07.05} } @@ -5931,17 +5967,23 @@ timestamp = {2007-10-25} } - at INPROCEEDINGS{CDM_2009, - author = {Justin M. Wozniak and Michael Wilde}, - title = {Case Studies in Storage Access by Loosely Coupled Petascale Applications}, - booktitle = {Proc. Petascale Data Storage Workshop at SC'09}, - year = {2009}, - owner = {wozniak}, - timestamp = {2010/04/02} -} + at inproceedings{CDM_2009, + author = {Wozniak, Justin M. and Wilde, Michael}, + title = {Case studies in storage access by loosely coupled petascale applications}, + booktitle = {Proceedings of the 4th Annual Workshop on Petascale Data Storage}, + series = {PDSW '09}, + year = {2009}, + isbn = {978-1-60558-883-4}, + location = {Portland, Oregon}, + pages = {16--20}, + numpages = {5}, + url = {http://doi.acm.org/10.1145/1713072.1713078}, + doi = {http://doi.acm.org/10.1145/1713072.1713078}, + acmid = {1713078}, + publisher = {ACM}, + address = {New York, NY, USA}, +} -{jabref-meta: selector_publisher:1;} - @INBOOK{P2P-Grids_2003, author = {Geoffrey Fox and Dennis Gannon and Sung-Hoon Ko and Sangmi-Lee and Shrideep Pallickara and Marlon Pierce and @@ -6068,30 +6110,32 @@ } {Pages: 541--551} - at ARTICLE{OSG_2007, - author = {Ruth Pordes and Don Petravick and Bill Kramer and - Doug Olson and Miron Livny and Alain Roy and - Paul Avery and Kent Blackburn and Torre Wenaus and - Frank Wurthwein and Ian Foster and Rob Gardner and - Mike Wilde and Alan Blatecky and John McGee and Rob Quick}, - title = {The {O}pen {S}cience {G}rid}, - journal = {Journal of Physics: Conference Series}, - volume = {78}, - number = {1}, - year = {2007}, - abstract = {The Open Science Grid (OSG) provides a distributed facility where the Consortium members provide guaranteed and opportunistic access to shared computing and storage resources. OSG provides support for and evolution of the infrastructure through activities that cover operations, security, software, troubleshooting, addition of new capabilities, and support for existing and engagement with new communities. The OSG SciDAC-2 project provides specific activities to manage and evolve the distributed infrastructure and support it's use. The innovative aspects of the project are the maintenance and performance of a collaborative (shared & common) petascale national facility over tens of autonomous computing sites, for many hundreds of users, transferring terabytes of data a day, executing tens of thousands of jobs a day, and providing robust and usable resources for scientific groups of all types and sizes. More information can be found at the OSG web site: www.opensci encegrid.org.} + at article{OSG_2007, + author={Ruth Pordes and Don Petravick and Bill Kramer and Doug Olson and Miron Livny and Alain Roy and Paul Avery and Kent +Blackburn and Torre Wenaus and Frank W?rthwein and Ian Foster and Rob Gardner and Mike Wilde and Alan Blatecky and John +McGee and Rob Quick}, + title={{The Open Science Grid}}, + journal={Journal of Physics: Conference Series}, + volume={78}, + number={1}, + pages={012057}, + url={http://stacks.iop.org/1742-6596/78/i=1/a=012057}, + year={2007}, + abstract={The Open Science Grid (OSG) provides a distributed facility where the Consortium members provide guaranteed and opportunistic access to shared computing and storage resources. OSG provides support for and evolution of the infrastructure through activities that cover operations, security, software, troubleshooting, addition of new capabilities, and support for existing and engagement with new communities. The OSG SciDAC-2 project provides specific activities to manage and evolve the distributed infrastructure and support it's use. The innovative aspects of the project are the maintenance and performance of a collaborative (shared & common) petascale national facility over tens of autonomous computing sites, for many hundreds of users, transferring terabytes of data a day, executing tens of thousands of jobs a day, and providing robust and usable resources for scientific groups of all types and sizes. More information can be found at the OSG web site: www.openscien cegrid.org.} } - + @ARTICLE{TeraGrid_2005, title = {Building the {TeraGrid}}, author = {Peter H Beckman}, journal = PTRS_A, volume = 363, number = 1833, - year = 2005 -} -{pages 1715-1728} + year = 2005, + doi={10.1098/rsta.2005.1602}, + pages={1715--1728} + } + @MISC{RFC:4918_WebDAV_2007, author = {{IETF} {Network Working Group}}, title = {{HTTP} Extensions for Web Distributed @@ -6150,13 +6194,21 @@ year = 2010 } - at INPROCEEDINGS{FTSH_2003, - title = {The ethernet approach to grid computing}, - author = {Thain, Douglas and Livny, Miron}, - booktitle = HPDC, - year = 2003, -} + at inproceedings{FTSH_2003, + author = {Thain, Douglas and Livny, Miron}, + title = {The Ethernet Approach to Grid Computing}, + booktitle = {Proceedings of the 12th IEEE International Symposium on High Performance Distributed Computing}, + series = {HPDC '03}, + year = {2003}, + isbn = {0-7695-1965-2}, + pages = {138--}, + url = {http://portal.acm.org/citation.cfm?id=822087.823417}, + acmid = {823417}, + publisher = {IEEE Computer Society}, + address = {Washington, DC, USA}, +} + @INPROCEEDINGS{MatlabCondor_2003, title = {Implementation of a Grid Computation Toolkit for Design Optimisation with {M}atlab and {C}ondor}, @@ -6189,18 +6241,26 @@ title = {{Strand}: {A} practical parallel programming language}, author = {Foster, I. and Taylor, S.}, booktitle = {Proc. North American Conference on Logic Programming}, - year = 1989 + year = 1989, + pages={497--512} } - at ARTICLE{PCN_1993, - title = {Productive parallel programming: {T}he {PCN} approach}, - author = {Foster, I. and Olson, R. and Tuecke, S.}, - journal = {Scientific Programming}, - volume = 1, - number = 1, - year = 1992, -} -{pages = {51--66}, publisher={IOS Press}} + at article{PCN_1993, + author = {Foster, Ian and Olson, Robert and Tuecke, Steven}, + title = {Productive Parallel Programming: The PCN Approach}, + journal = {Sci. Program.}, + volume = {1}, + issue = {1}, + month = {January}, + year = {1992}, + issn = {1058-9244}, + pages = {51--66}, + numpages = {16}, + url = {http://portal.acm.org/citation.cfm?id=1402583.1402587}, + acmid = {1402587}, + publisher = {IOS Press}, + address = {Amsterdam, The Netherlands, The Netherlands}, +} @article{Sawzall_2005, title = {Interpreting the data: {P}arallel analysis with {Sawzall}}, @@ -6209,27 +6269,32 @@ volume = {13}, number = {4}, year = 2005, + pages = {277--298}, + publisher={IOS Press} } -{pages = {277--298}, publisher={IOS Press}} @BOOK{BPEL_2006, author = {Juric, Matjaz B.}, title = {Business Process Execution Language for Web Services}, year = 2006, + publisher={Packt Publishing} } isbn = 1904811817, publisher = {Packt Publishing} - at INBOOK{Sedna_2007, - chapter = {{S}edna: {A} {BPEL}-Based Environment for - Visual Scientific Workflow Modeling}, - title = {Workflows for e-{S}cience}, - publisher = {Springer}, - year = 2007, - author = {Bruno Wassermann and Wolfgang Emmerich and Ben - Butchart and Nick Cameron and Liang Chen and Jignesh - Patel} + at incollection {Sedna_2007, + author = {Wassermann, Bruno and Emmerich, Wolfgang and Butchart, Ben and Cameron, Nick and Chen, Liang and Patel, Jignesh}, + affiliation = {University College London Software Systems Engineering Group, Department of Computer Science Gower Street London WC1E 6BT UK}, + title = {Sedna: A BPEL-Based Environment for Visual Scientific Workflow Modeling}, + booktitle = {Workflows for e-Science}, + editor = {Taylor, Ian J. and Deelman, Ewa and Gannon, Dennis B. and Shields, Matthew}, + publisher = {Springer London}, + isbn = {978-1-84628-757-2}, + keyword = {Computer Science}, + pages = {428--449}, + url = {http://dx.doi.org/10.1007/978-1-84628-757-2\_26}, + note = {10.1007/978-1-84628-757-2\_26}, + year = {2007} } -pages = {18}, @INPROCEEDINGS{SunConstellation_2008, title = {{Performance and Scalability Study of @@ -6237,15 +6302,22 @@ using Application-Based Benchmarks}}, author = {Byoung-Do Kim and John E. Cazes}, booktitle = {Proc. TeraGrid'2008}, + pages={}, year = 2008 } - at INPROCEEDINGS{ReSS_2007, - author = {G. Garzoglio and T. Levshina and P. Mhashilkar and S. Timm}, - title = {{ReSS}: {A} Resource Selection Service for the - {O}pen {S}cience {G}rid}, - booktitle = {Proc. International Symposium of Grid Computing}, - year = 2007 + at incollection {ReSS_2007, + author = {Garzoglio, Gabriele and Levshina, Tanya and Mhashilkar, Parag and Timm, Steve}, + title = {{ReSS: A Resource Selection Service for the Open Science Grid}}, + booktitle = {Grid Computing}, + editor = {Lin, Simon C. and Yen, Eric}, + publisher = {Springer US}, + isbn = {978-0-387-78417-5}, + keyword = {Computer Science}, + pages = {89--98}, + url = {http://dx.doi.org/10.1007/978-0-387-78417-5\_8}, + note = {10.1007/978-0-387-78417-5\_8}, + year = {2009} } @ARTICLE{CNARI_2007, From noreply at svn.ci.uchicago.edu Thu Dec 9 16:28:16 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 9 Dec 2010 16:28:16 -0600 (CST) Subject: [Swift-commit] r3759 - in provenancedb: . apps/oops Message-ID: <20101209222816.A59579CC7F@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-09 16:28:16 -0600 (Thu, 09 Dec 2010) New Revision: 3759 Modified: provenancedb/apps/oops/oops_extractor.sh provenancedb/swift-prov-import-all-logs Log: Minor updates to provenancedb. Modified: provenancedb/apps/oops/oops_extractor.sh =================================================================== --- provenancedb/apps/oops/oops_extractor.sh 2010-12-09 05:32:32 UTC (rev 3758) +++ provenancedb/apps/oops/oops_extractor.sh 2010-12-09 22:28:16 UTC (rev 3759) @@ -18,18 +18,15 @@ fi cd $PROTESTS_HOME -for k in `ls -1`; +for k in $(ls -1); do cd $PROTESTS_HOME/$k - for i in `ls | grep run.loops`; + for i in $(ls | grep run.loops); do cd $IMPORT_HOME if ! grep --silent $i provdb_imported; then if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/psim.loops-*.log; then cd swift-logs - #for j in `ls $PROTESTS_HOME/$k/$i | grep psim.loops-`; do - # ln -s $PROTESTS_HOME/$k/$i/$j - #done cd import # swift-prov-import-all-logs also controls what already has been # imported, so it does not repeat work @@ -39,45 +36,41 @@ cd $IMPORT_HOME echo $i >> provdb_imported cd swift-logs - # annotate workflows with their oops runid - OOPS_RUN_ID=`echo $i | awk -F . '{print $3}'` + + # annotate workflows with their oops runid + OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}') cd $PROTESTS_HOME/$k/$i - LOG_FILENAME=`ls | grep psim.loops- | grep "\."log$` - WORKFLOW_ID=`echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}'` + LOG_FILENAME=$(ls | grep psim.loops- | grep "\."log$) + WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}') cd $IMPORT_HOME/swift-logs echo "insert into annot_wf_txt (id, name, value) values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD - echo "select id,filename from file where filename like '%params%' and id in (select in_id from ds_containment where out_id in (select ds_usage.dataset_id from ds_usage,process,execute where ds_usage.process_id=process.id and process.id=execute.id and execute.procedure_name='loopPrepare' and ds_usage.direction='I' and process.workflow_id like '%$WORKFLOW_ID%'));" > query.sql + # + echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql; - #query for the previous database schema - #echo "select dataset_id,filename from dataset_filenames where filename like '%params%' and dataset_id in (select inner_dataset_id from dataset_containment where outer_dataset_id in (select dataset_usage.dataset_id from invocation_procedure_names,dataset_usage,processes_in_workflows where invocation_procedure_names.execute_id=dataset_usage.process_id and dataset_usage.process_id=processes_in_workflows.process_id and invocation_procedure_names.procedure_name='loopPrepare' and dataset_usage.direction='I' and processes_in_workflows.workflow_id like '%$WORKFLOW_ID%'));" > query.sql - $SQLCMD -t -A -F " " -f query.sql -o result.txt - #DATASET_ID=`awk '{print $1}' result.txt` - DATASET_ID=`awk '{if (NR==1) print $1}' result.txt` + DATASET_ID=$(awk '{print $1}' result.txt) + FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g') - #FILENAME=`awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g'` - FILENAME=`awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g'` - cd $PROTESTS_HOME/$k/run.loops.$OOPS_RUN_ID while read line; do - NAME=`echo $line | awk 'BEGIN { FS = "=" }; {print $1}'` + NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}') if [ "$NAME" = "SAMPLE RANGE" ]; then - VALUE1=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}'` - VALUE2=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}'` + VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}') + VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}') echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD fi if [ "$NAME" = "RESTRAIN DISTANCE" ]; then - VALUE1=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}'` - VALUE2=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}'` + VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}') + VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}') echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD fi if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then - VALUE=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}'` + VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}') echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD fi done < $FILENAME Modified: provenancedb/swift-prov-import-all-logs =================================================================== --- provenancedb/swift-prov-import-all-logs 2010-12-09 05:32:32 UTC (rev 3758) +++ provenancedb/swift-prov-import-all-logs 2010-12-09 22:28:16 UTC (rev 3759) @@ -41,7 +41,7 @@ EXISTING=$($SQLCMD --tuples-only -c "select count(*) from workflow where log_filename='$filename';") if [ "$EXISTING" -eq "0" ]; then - PROV_ENABLED=$(grep provenance $filename | wc -l) + PROV_ENABLED=$(grep provenanceid $filename | wc -l) if [ $PROV_ENABLED -gt 0 ]; then echo IMPORTING From noreply at svn.ci.uchicago.edu Fri Dec 10 12:12:34 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 10 Dec 2010 12:12:34 -0600 (CST) Subject: [Swift-commit] r3760 - provenancedb/apps/oops Message-ID: <20101210181234.C8CD49D908@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-10 12:12:34 -0600 (Fri, 10 Dec 2010) New Revision: 3760 Added: provenancedb/apps/oops/raptor_extractor.sh Modified: provenancedb/apps/oops/oops_extractor.sh Log: RaptorLoops annotation extractor for provenancedb. Modified: provenancedb/apps/oops/oops_extractor.sh =================================================================== --- provenancedb/apps/oops/oops_extractor.sh 2010-12-09 22:28:16 UTC (rev 3759) +++ provenancedb/apps/oops/oops_extractor.sh 2010-12-10 18:12:34 UTC (rev 3760) @@ -2,8 +2,6 @@ # Annotation extractor for the OOPS application # -# The directory $PROTESTS/swift-logs contains symbolic links to -# OOPS' Swift logs. PROVDB_HOME=~/provenancedb PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP Added: provenancedb/apps/oops/raptor_extractor.sh =================================================================== --- provenancedb/apps/oops/raptor_extractor.sh (rev 0) +++ provenancedb/apps/oops/raptor_extractor.sh 2010-12-10 18:12:34 UTC (rev 3760) @@ -0,0 +1,75 @@ +#!/bin/bash + +# Annotation extractor for the OOPS application +# +# The directory $PROTESTS/swift-logs contains symbolic links to +# OOPS' Swift logs. +PROVDB_HOME=~/provenancedb +PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP +IMPORT_HOME=~/protests + +source $PROVDB_HOME/etc/provenance.config + +# provdb_imported records runs already imported to the provenance database +cd $IMPORT_HOME +if [ ! -a provdb_imported_raptor ]; then + touch provdb_imported_raptor +fi + +cd $PROTESTS_HOME +for k in $(ls -1); +do + cd $PROTESTS_HOME/$k + for i in $(ls | grep run.raptorloops); + do + cd $IMPORT_HOME + if ! grep --silent $i provdb_imported_raptor; then + if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/RaptorLoops-*.log; then + # swift-prov-import-all-logs also controls what already has been + # imported, so it does not repeat work + echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config + echo "export SQLCMD=\"psql -U provdb -h db.ci.uchicago.edu provdb\"" >> $PROVDB_HOME/etc/provenance.config + $PROVDB_HOME/swift-prov-import-all-logs + cd $IMPORT_HOME + echo $i >> provdb_imported_raptor + cd swift-logs + # annotate workflows with their oops runid + OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}') + cd $PROTESTS_HOME/$k/$i + LOG_FILENAME=$(ls | grep RaptorLoops- | grep "\."log$) + WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}') + cd $IMPORT_HOME/swift-logs + echo "insert into annot_wf_txt values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD + + echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql; + + $SQLCMD -t -A -F " " -f query.sql -o result.txt + + DATASET_ID=$(awk '{print $1}' result.txt) + FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g') + + cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID + + while read line; do + NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}') + if [ "$NAME" = "SAMPLE RANGE" ]; then + VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}') + VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}') + echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD + echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD + fi + if [ "$NAME" = "RESTRAIN DISTANCE" ]; then + VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}') + VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}') + echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD + echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD + fi + if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then + VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}') + echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD + fi + done < $FILENAME + fi + fi + done +done Property changes on: provenancedb/apps/oops/raptor_extractor.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Fri Dec 10 12:40:14 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 10 Dec 2010 12:40:14 -0600 (CST) Subject: [Swift-commit] r3761 - provenancedb/apps/oops Message-ID: <20101210184014.65C829D908@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-10 12:40:14 -0600 (Fri, 10 Dec 2010) New Revision: 3761 Modified: provenancedb/apps/oops/raptor_extractor.sh Log: Minor fixes to RaptorLoops provenance extractor. Modified: provenancedb/apps/oops/raptor_extractor.sh =================================================================== --- provenancedb/apps/oops/raptor_extractor.sh 2010-12-10 18:12:34 UTC (rev 3760) +++ provenancedb/apps/oops/raptor_extractor.sh 2010-12-10 18:40:14 UTC (rev 3761) @@ -45,8 +45,8 @@ $SQLCMD -t -A -F " " -f query.sql -o result.txt - DATASET_ID=$(awk '{print $1}' result.txt) - FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g') + DATASET_ID=`awk '{if (NR==1) print $1}' result.txt` + FILENAME=`awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g'` cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID From noreply at svn.ci.uchicago.edu Fri Dec 10 15:32:41 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 10 Dec 2010 15:32:41 -0600 (CST) Subject: [Swift-commit] r3762 - trunk/tests Message-ID: <20101210213241.BB6789D908@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-10 15:32:41 -0600 (Fri, 10 Dec 2010) New Revision: 3762 Added: trunk/tests/site/ Log: Site-specific test groups From noreply at svn.ci.uchicago.edu Fri Dec 10 15:33:04 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 10 Dec 2010 15:33:04 -0600 (CST) Subject: [Swift-commit] r3763 - trunk/tests/site Message-ID: <20101210213304.8C2049D908@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-10 15:33:04 -0600 (Fri, 10 Dec 2010) New Revision: 3763 Added: trunk/tests/site/intrepid/ Log: Site-specific tests for Intrepid From noreply at svn.ci.uchicago.edu Sat Dec 11 16:24:50 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 11 Dec 2010 16:24:50 -0600 (CST) Subject: [Swift-commit] r3764 - SwiftApps/SwiftR/Swift/man Message-ID: <20101211222450.95F369CC80@svn.ci.uchicago.edu> Author: wilde Date: 2010-12-11 16:24:49 -0600 (Sat, 11 Dec 2010) New Revision: 3764 Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd Log: Minor doc edits. Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd =================================================================== --- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-12-10 21:33:04 UTC (rev 3763) +++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-12-11 22:24:49 UTC (rev 3764) @@ -123,13 +123,13 @@ Download the appropriate Java Runtime (JRE) for Linux at:\verb{ http://www.java.com/en/download/manual.jsp} -Typically either 32 bit with this link:\verb{ - Linux (self-extracting file) filesize: 19.9} +Typically either 32 bit with the link named:\verb{ + "Linux (self-extracting file) filesize: 19.9"} -Or 64 bit with this link\verb{ - Linux x64 * filesize: 19.3 MB } +...or 64 bit with the link named: \verb{ + "Linux x64 (self-extracting file) filesize: 19.3 MB" } -Its better to install the JDK (~80MB) from: +Its better to install the full development kit (JDK, ~80MB) from: http://www.oracle.com/technetwork/java/javase/downloads/jdk6-jsp-136632.html (JDKs) @@ -141,11 +141,12 @@ Verify that you have Sun Java installed and in your PATH correctly by doing: +\verb{ $ java -version java version "1.6.0_21" Java(TM) SE Runtime Environment (build 1.6.0_21-b06) Java HotSpot(TM) 64-Bit Server VM (build 17.0-b16, mixed mode) -$ +$ } 2) R v2.11 or higher in your PATH (on client and server machines) @@ -173,13 +174,14 @@ \section{INSTALLATION}{ +\preformatted{ mkdir ~/RPackages ~/RLibrary # if not already created cd ~/RPackages wget http://www.ci.uchicago.edu/~wilde/Swift_0.1.tar.gz R CMS INSTALL -l ~/RLibrary Swift_0.1.tar.gz export R_LIBS=~/RLibrary -export GLOBUS_HOSTNAME=10.0.0.200 # Eg for Merlot: internal address of the login node +export GLOBUS_HOSTNAME=10.0.0.200 # Eg for Merlot: internal address of the login node} } \section{QUICK_START}{ @@ -211,7 +213,7 @@ its best to set this to /scratch/local, as /tmp is very limited in space. Seldom needed. -GLOBUS_HOSTNAME should be set to the IP address of the login host if +For the pbsf server type only: GLOBUS_HOSTNAME should be set to the IP address of the login host if it contains multiple network interfaces (see /sbin/ifconfig) and if the worker nodes can only reach the login host on a subset of these interfaces. From noreply at svn.ci.uchicago.edu Sat Dec 11 16:46:59 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sat, 11 Dec 2010 16:46:59 -0600 (CST) Subject: [Swift-commit] r3765 - SwiftApps/SwiftR/Swift/exec Message-ID: <20101211224659.4D0559CC7F@svn.ci.uchicago.edu> Author: wilde Date: 2010-12-11 16:46:59 -0600 (Sat, 11 Dec 2010) New Revision: 3765 Added: SwiftApps/SwiftR/Swift/exec/configure-server-pbs Log: Add config script for local pbs clusters Added: SwiftApps/SwiftR/Swift/exec/configure-server-pbs =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-pbs (rev 0) +++ SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2010-12-11 22:46:59 UTC (rev 3765) @@ -0,0 +1,82 @@ +#! /bin/bash + +# configuration for PBS with manually-started Swift workers (passive coasters) + +cores=$1 + +throttlePBS=5.0 + +cat >tc <sites.xml < + + + + 0.15 + 10000 + + $(pwd)/swiftwork + + + + + passive + $cores + $throttlePBS + 10000 + + $HOME/swiftwork + + + +END + +cat >cf <00:00:01 +# fast + + + + fast + 00:59:00 + $throttlePBS + 10000 + + $HOME/swiftwork + + + + + short + 12000 + 32 + 1 + 1 + 1 + 2.55 + 10000 + + $HOME/swiftwork + + +END Property changes on: SwiftApps/SwiftR/Swift/exec/configure-server-pbs ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Sun Dec 12 17:29:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 12 Dec 2010 17:29:55 -0600 (CST) Subject: [Swift-commit] r3766 - in provenancedb: . apps/oops Message-ID: <20101212232955.784239CC80@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-12 17:29:54 -0600 (Sun, 12 Dec 2010) New Revision: 3766 Modified: provenancedb/apps/oops/oops_extractor.sh provenancedb/apps/oops/raptor_extractor.sh provenancedb/prov-init.sql Log: Minor changes to OOPS provenance extractor. Modified: provenancedb/apps/oops/oops_extractor.sh =================================================================== --- provenancedb/apps/oops/oops_extractor.sh 2010-12-11 22:46:59 UTC (rev 3765) +++ provenancedb/apps/oops/oops_extractor.sh 2010-12-12 23:29:54 UTC (rev 3766) @@ -19,13 +19,11 @@ for k in $(ls -1); do cd $PROTESTS_HOME/$k - for i in $(ls | grep run.loops); + for i in $(ls | grep run.raptorloops; ls | grep run.loops); do cd $IMPORT_HOME if ! grep --silent $i provdb_imported; then if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/psim.loops-*.log; then - cd swift-logs - cd import # swift-prov-import-all-logs also controls what already has been # imported, so it does not repeat work echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config @@ -38,40 +36,85 @@ # annotate workflows with their oops runid OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}') cd $PROTESTS_HOME/$k/$i - LOG_FILENAME=$(ls | grep psim.loops- | grep "\."log$) + LOG_FILENAME=$(ls *-*-*-*.log) WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}') cd $IMPORT_HOME/swift-logs echo "insert into annot_wf_txt (id, name, value) values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD - # - echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql; - + #extracts scientific parameters given as input to the workflow in file *.params. + echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID' and ds_usage.direction='O';" > query.sql; + $SQLCMD -t -A -F " " -f query.sql -o result.txt - DATASET_ID=$(awk '{print $1}' result.txt) - FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g') + DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt) + FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g') - cd $PROTESTS_HOME/$k/run.loops.$OOPS_RUN_ID + cd $PROTESTS_HOME/$k/$i while read line; do NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}') + RIGHT=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}') if [ "$NAME" = "SAMPLE RANGE" ]; then - VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}') - VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}') - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD + echo "insert into annot_ds_txt values ('$DATASET_ID', 'sample_range', '$RIGHT');" | $SQLCMD fi if [ "$NAME" = "RESTRAIN DISTANCE" ]; then - VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}') + VALUE1=$(echo $RIGHT | awk 'BEGIN { FS = "," }; {print $1}') VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}') - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD + echo "insert into annot_ds_num values ('$DATASET_ID', 'restrain_distance_1', $VALUE1);" | $SQLCMD + echo "insert into annot_ds_num values ('$DATASET_ID', 'restrain_distance_2', $VALUE2);" | $SQLCMD fi if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then - VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}') - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD + echo "insert into annot_ds_num values ('$DATASET_ID', 'maximum_number_of_steps', $RIGHT);" | $SQLCMD fi done < $FILENAME + + # extracts scientific parameters given as input to the workflow in file *.params. + # relevant lines: + # zone2 (Initial Energy: -21352.116911) + # Total Function Evaluations: 20000 + # Accepted transitions: 7410 + # Increasing transitions: 4525 + # Decreasing transitions: 2885 + # Rejected transitions: 12590 + # Final Energy: -27152.264775 + # Final Temp: 79.778142 + # Total Running Time: 18006 + + echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.log' and process.name='LoopModel' and process.workflow_id='$WORKFLOW_ID' and ds_usage.direction='O';" > query.sql; + + $SQLCMD -t -A -F " " -f query.sql -o result.txt + + while read dataset filename; do + DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt) + FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g') + while read token1 token2 token3 token4; do + if [ "$token2" = "(Initial Energy:" ]; then + initialenergy=$(echo $token3 | awk 'BEGIN { FS = "\)" }; {print $1}') + echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'initial_energy', $initialenergy);" | $SQLCMD + fi + if [ "$token1" = "Total" && "$token2" = "Function" && "$token3" = "Evaluations:" ]; then + echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'total_function_evaluations', $token4);" | $SQLCMD + fi + if [ "$token1" = "Increasing" && "$token2" = "transitions:" ]; then + echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'accepted_increasing_transitions', $token3);" | $SQLCMD + fi + if [ "$token1" = "Decreasing" && "$token2" = "transitions:" ]; then + echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'accepted_decreasing_transitions', $token3);" | $SQLCMD + fi + if [ "$token1" = "Rejected" && "$token2" = "transitions:" ]; then + echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'rejected_transitions', $token3);" | $SQLCMD + fi + if [ "$token1" = "Final" && "$token2" = "Energy:" ]; then + echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'final_energy', $token3);" | $SQLCMD + fi + if [ "$token1" = "Final" && "$token2" = "Temp:" ]; then + echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'final_temp', $token3);" | $SQLCMD + fi + if [ "$token1" = "Total" && "$token2" = "Running" && "$token3" = "Time:" ]; then + echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'total_running_time', $token4);" | $SQLCMD + fi + done < result.txt + done < $FILENAME fi fi done Modified: provenancedb/apps/oops/raptor_extractor.sh =================================================================== --- provenancedb/apps/oops/raptor_extractor.sh 2010-12-11 22:46:59 UTC (rev 3765) +++ provenancedb/apps/oops/raptor_extractor.sh 2010-12-12 23:29:54 UTC (rev 3766) @@ -20,7 +20,7 @@ for k in $(ls -1); do cd $PROTESTS_HOME/$k - for i in $(ls | grep run.raptorloops); + for i in $(ls | grep run.raptorloops; ls | grep run.loops); do cd $IMPORT_HOME if ! grep --silent $i provdb_imported_raptor; then @@ -33,7 +33,7 @@ cd $IMPORT_HOME echo $i >> provdb_imported_raptor cd swift-logs - # annotate workflows with their oops runid + # annotate workflows with their oops runid OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}') cd $PROTESTS_HOME/$k/$i LOG_FILENAME=$(ls | grep RaptorLoops- | grep "\."log$) @@ -45,8 +45,8 @@ $SQLCMD -t -A -F " " -f query.sql -o result.txt - DATASET_ID=`awk '{if (NR==1) print $1}' result.txt` - FILENAME=`awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g'` + DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt) + FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g') cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID Modified: provenancedb/prov-init.sql =================================================================== --- provenancedb/prov-init.sql 2010-12-11 22:46:59 UTC (rev 3765) +++ provenancedb/prov-init.sql 2010-12-12 23:29:54 UTC (rev 3766) @@ -1,41 +1,41 @@ -- this is the schema definition used for the main relational provenance -- implementation (in both sqlite3 and postgres) -DROP TABLE dataset CASCADE; -DROP TABLE file CASCADE; -DROP TABLE variable CASCADE; -DROP TABLE ds_containment CASCADE; -DROP TABLE process CASCADE; -DROP TABLE execute CASCADE; -DROP TABLE execute2 CASCADE; -DROP TABLE workflow CASCADE; -DROP TABLE ds_usage CASCADE; -DROP TABLE annot_ds_num CASCADE; -DROP TABLE annot_ds_txt CASCADE; -DROP TABLE annot_ds_bool CASCADE; -DROP TABLE annot_p_num CASCADE; -DROP TABLE annot_p_txt CASCADE; -DROP TABLE annot_p_bool CASCADE; -DROP TABLE annot_wf_num CASCADE; -DROP TABLE annot_wf_txt CASCADE; -DROP TABLE annot_wf_bool CASCADE; --- DROP TABLE extrainfo CASCADE; -DROP TABLE createarray CASCADE; -DROP TABLE createarray_member CASCADE; -DROP TABLE array_range CASCADE; +drop table dataset cascade; +drop table file cascade; +drop table variable cascade; +drop table ds_containment cascade; +drop table process cascade; +drop table execute cascade; +drop table execute2 cascade; +drop table workflow cascade; +drop table ds_usage cascade; +drop table annot_ds_num cascade; +drop table annot_ds_txt cascade; +drop table annot_ds_bool cascade; +drop table annot_p_num cascade; +drop table annot_p_txt cascade; +drop table annot_p_bool cascade; +drop table annot_wf_num cascade; +drop table annot_wf_txt cascade; +drop table annot_wf_bool cascade; +-- drop table extrainfo cascade; +drop table createarray cascade; +drop table createarray_member cascade; +drop table array_range cascade; -- workflow stores some information about each workflow log that has -- been seen by the importer: the log filename, swift version and import -- status. -- Might be interesting to store xml translation of the swiftscript code -- here for prospective provenance/versioning -CREATE TABLE workflow - (id VARCHAR(256) PRIMARY KEY, - log_filename VARCHAR(2048), - swift_version VARCHAR(16), - import_status VARCHAR(16), - start_time NUMERIC, - duration NUMERIC +create table workflow + (id varchar(256) primary key, + log_filename varchar(2048), + swift_version varchar(16), + import_status varchar(16), + start_time numeric, + duration numeric ); -- workflow_run stores the start time and duration for each workflow @@ -47,14 +47,14 @@ -- ); -- dataset stores all dataset identifiers. -CREATE TABLE dataset - (id VARCHAR(256) PRIMARY KEY +create table dataset + (id varchar(256) primary key ); -- file stores the filename mapped to each dataset. -CREATE TABLE file - ( id VARCHAR(256) PRIMARY KEY REFERENCES dataset (id) ON DELETE CASCADE, - filename VARCHAR(2048) +create table file + ( id varchar(256) primary key references dataset (id) on delete cascade, + filename varchar(2048) ); -- dataset_values stores the value for each dataset which is known to have @@ -62,9 +62,9 @@ -- to expose that value as an SQL type other than a string, and so (for -- example) SQL numerical operations should not be expected to work, even -- though the user knows that a particular dataset stores a numeric value. -CREATE TABLE variable - ( id VARCHAR(256) PRIMARY KEY REFERENCES dataset (id) ON DELETE CASCADE, - value VARCHAR(2048) +create table variable + ( id varchar(256) primary key references dataset (id) on delete cascade, + value varchar(2048) ); -- dataset_containment stores the containment hierarchy between @@ -74,10 +74,10 @@ -- constructors and accessors, rather than, or in addition to, -- a containment hierarchy. The relationship (such as array index or -- structure member name) should also be stored in this table. -CREATE TABLE ds_containment - ( out_id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE, - in_id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE, - PRIMARY KEY (out_id,in_id) +create table ds_containment + ( out_id varchar(256) references dataset (id) on delete cascade, + in_id varchar(256) references dataset (id) on delete cascade, + primary key (out_id,in_id) ); -- process gives information about each process (in the OPM sense) @@ -88,114 +88,114 @@ -- Having this type here seems poor normalisation, though? -- process types: internal, rootthread, execute, function, compound, scope, operator -- maybe create a table for each type? -CREATE TABLE process - (id VARCHAR(256) PRIMARY KEY, - type VARCHAR(16), - name VARCHAR(256), -- in the case of an execute this refers to the transformation name in tc.data - workflow_id VARCHAR(256) REFERENCES workflow (id) ON DELETE CASCADE -- normalize: workflow_id of sub-procedure determined +create table process + (id varchar(256) primary key, + type varchar(16), + name varchar(256), -- in the case of an execute this refers to the transformation name in tc.data + workflow_id varchar(256) references workflow (id) on delete cascade -- normalize: workflow_id of sub-procedure determined -- by compound procedure ); -- this gives information about each execute. -- each execute is identified by a unique URI. other information from -- swift logs is also stored here. an execute is an OPM process. -CREATE TABLE execute - (id VARCHAR(256) PRIMARY KEY REFERENCES process (id) ON DELETE CASCADE, - procedure_name VARCHAR(256), -- name of the app procedure that invokes the transformation - start_time NUMERIC, - duration NUMERIC, - final_state VARCHAR(16), - scratch VARCHAR(2048) +create table execute + (id varchar(256) primary key references process (id) on delete cascade, + procedure_name varchar(256), -- name of the app procedure that invokes the transformation + start_time numeric, + duration numeric, + final_state varchar(16), + scratch varchar(2048) ); -- this gives information about each execute2, which is an attempt to -- perform an execution. the execute2 id is tied to per-execution-attempt -- information such as wrapper logs -CREATE TABLE execute2 - (id VARCHAR(256) PRIMARY KEY, - execute_id VARCHAR(256) REFERENCES execute (id) ON DELETE CASCADE, - start_time NUMERIC, - duration NUMERIC, - final_state VARCHAR(16), - site VARCHAR(256) +create table execute2 + (id varchar(256) primary key, + execute_id varchar(256) references execute (id) on delete cascade, + start_time numeric, + duration numeric, + final_state varchar(16), + site varchar(256) ); -- dataset_usage records usage relationships between processes and datasets; -- in SwiftScript terms, the input and output parameters for each -- application procedure invocation; in OPM terms, the artificts which are -- input to and output from each process that is a Swift execution -CREATE TABLE ds_usage - (process_id VARCHAR(256) REFERENCES process(id) ON DELETE CASCADE, - direction CHAR(1), -- I or O for input or output - dataset_id VARCHAR(256) REFERENCES dataset(id) ON DELETE CASCADE, - param_name VARCHAR(256), -- the name of the parameter in this execute that +create table ds_usage + (process_id varchar(256) references process(id) on delete cascade, + direction char(1), -- I or O for input or output + dataset_id varchar(256) references dataset(id) on delete cascade, + param_name varchar(256), -- the name of the parameter in this execute that -- this dataset was bound to. sometimes this must -- be contrived (for example, in positional varargs) - PRIMARY KEY (process_id,direction,dataset_id,param_name) + primary key (process_id,direction,dataset_id,param_name) ); -- annotations -CREATE TABLE annot_ds_num - ( id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE, - name VARCHAR(256), - value NUMERIC, - PRIMARY KEY (id, name) +create table annot_ds_num + ( id varchar(256) references dataset (id) on delete cascade, + name varchar(256), + value numeric, + primary key (id, name) ); -CREATE TABLE annot_ds_txt - ( id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE, - name VARCHAR(256), - value VARCHAR(2048), - PRIMARY KEY (id, name) +create table annot_ds_txt + ( id varchar(256) references dataset (id) on delete cascade, + name varchar(256), + value varchar(2048), + primary key (id, name) ); -CREATE TABLE annot_ds_bool - ( id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE, - name VARCHAR(256), - value BOOLEAN, - PRIMARY KEY (id, name) +create table annot_ds_bool + ( id varchar(256) references dataset (id) on delete cascade, + name varchar(256), + value boolean, + primary key (id, name) ); -CREATE TABLE annot_p_num - ( id VARCHAR(256) REFERENCES process (id) ON DELETE CASCADE, - name VARCHAR(256), - value NUMERIC, - PRIMARY KEY (id, name) +create table annot_p_num + ( id varchar(256) references process (id) on delete cascade, + name varchar(256), + value numeric, + primary key (id, name) ); -CREATE TABLE annot_p_txt - ( id VARCHAR(256) REFERENCES process (id) ON DELETE CASCADE, - name VARCHAR(256), - value VARCHAR(2048), - PRIMARY KEY (id, name) +create table annot_p_txt + ( id varchar(256) references process (id) on delete cascade, + name varchar(256), + value varchar(2048), + primary key (id, name) ); -CREATE TABLE annot_p_bool - ( id VARCHAR(256) REFERENCES process (id) ON DELETE CASCADE, - name VARCHAR(256), - value BOOLEAN, - PRIMARY KEY (id, name) +create table annot_p_bool + ( id varchar(256) references process (id) on delete cascade, + name varchar(256), + value boolean, + primary key (id, name) ); -CREATE TABLE annot_wf_num - ( id VARCHAR(256) REFERENCES workflow (id) ON DELETE CASCADE, - name VARCHAR(256), - value NUMERIC, - PRIMARY KEY (id, name) +create table annot_wf_num + ( id varchar(256) references workflow (id) on delete cascade, + name varchar(256), + value numeric, + primary key (id, name) ); -CREATE TABLE annot_wf_txt - ( id VARCHAR(256) REFERENCES workflow (id) ON DELETE CASCADE, - name VARCHAR(256), - value VARCHAR(2048), - PRIMARY KEY (id, name) +create table annot_wf_txt + ( id varchar(256) references workflow (id) on delete cascade, + name varchar(256), + value varchar(2048), + primary key (id, name) ); -CREATE TABLE annot_wf_bool - ( id VARCHAR(256) REFERENCES workflow (id) ON DELETE CASCADE, - name VARCHAR(2048), - value BOOLEAN, - PRIMARY KEY (id, name) +create table annot_wf_bool + ( id varchar(256) references workflow (id) on delete cascade, + name varchar(2048), + value boolean, + primary key (id, name) ); -- extrainfo stores lines generated by the SWIFT_EXTRA_INFO feature @@ -211,29 +211,29 @@ -- terms of accessors and constructors. -- It is unclear which is the better representation. -CREATE TABLE createarray - ( id VARCHAR(256) PRIMARY KEY +create table createarray + ( id varchar(256) primary key ); -CREATE TABLE createarray_member - ( array_id VARCHAR(256) REFERENCES createarray (id) ON DELETE CASCADE, - ix VARCHAR(256), - member_id VARCHAR(256), - PRIMARY KEY (array_id, ix) +create table createarray_member + ( array_id varchar(256) references createarray (id) on delete cascade, + ix varchar(256), + member_id varchar(256), + primary key (array_id, ix) ); -- TODO step -CREATE TABLE array_range - ( array_id VARCHAR(256) REFERENCES createarray (id) ON DELETE CASCADE, - from_id VARCHAR(256), - to_id VARCHAR(256), - step_id VARCHAR(256), -- nullable, if step is unspecified - PRIMARY KEY (array_id,from_id,to_id,step_id) +create table array_range + ( array_id varchar(256) references createarray (id) on delete cascade, + from_id varchar(256), + to_id varchar(256), + step_id varchar(256), -- nullable, if step is unspecified + primary key (array_id,from_id,to_id,step_id) ); -- this GRANT does not work for sqlite; you'll get a syntax error but -- ignore it, as it is not needed in sqlite -GRANT ALL ON +grant all on dataset, file, variable, @@ -256,4 +256,4 @@ createarray, createarray_member, array_range -TO public, operators; +to public, operators; From noreply at svn.ci.uchicago.edu Sun Dec 12 17:47:03 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 12 Dec 2010 17:47:03 -0600 (CST) Subject: [Swift-commit] r3767 - provenancedb/apps/oops Message-ID: <20101212234703.BE89C9CC82@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-12 17:47:03 -0600 (Sun, 12 Dec 2010) New Revision: 3767 Removed: provenancedb/apps/oops/raptor_extractor.sh Modified: provenancedb/apps/oops/oops_extractor.sh Log: Minor fixes. Modified: provenancedb/apps/oops/oops_extractor.sh =================================================================== --- provenancedb/apps/oops/oops_extractor.sh 2010-12-12 23:29:54 UTC (rev 3766) +++ provenancedb/apps/oops/oops_extractor.sh 2010-12-12 23:47:03 UTC (rev 3767) @@ -85,8 +85,7 @@ $SQLCMD -t -A -F " " -f query.sql -o result.txt while read dataset filename; do - DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt) - FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g') + FILENAME=$(echo $filename | sed 's/file:\/\/localhost\///g') while read token1 token2 token3 token4; do if [ "$token2" = "(Initial Energy:" ]; then initialenergy=$(echo $token3 | awk 'BEGIN { FS = "\)" }; {print $1}') Deleted: provenancedb/apps/oops/raptor_extractor.sh =================================================================== --- provenancedb/apps/oops/raptor_extractor.sh 2010-12-12 23:29:54 UTC (rev 3766) +++ provenancedb/apps/oops/raptor_extractor.sh 2010-12-12 23:47:03 UTC (rev 3767) @@ -1,75 +0,0 @@ -#!/bin/bash - -# Annotation extractor for the OOPS application -# -# The directory $PROTESTS/swift-logs contains symbolic links to -# OOPS' Swift logs. -PROVDB_HOME=~/provenancedb -PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP -IMPORT_HOME=~/protests - -source $PROVDB_HOME/etc/provenance.config - -# provdb_imported records runs already imported to the provenance database -cd $IMPORT_HOME -if [ ! -a provdb_imported_raptor ]; then - touch provdb_imported_raptor -fi - -cd $PROTESTS_HOME -for k in $(ls -1); -do - cd $PROTESTS_HOME/$k - for i in $(ls | grep run.raptorloops; ls | grep run.loops); - do - cd $IMPORT_HOME - if ! grep --silent $i provdb_imported_raptor; then - if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/RaptorLoops-*.log; then - # swift-prov-import-all-logs also controls what already has been - # imported, so it does not repeat work - echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config - echo "export SQLCMD=\"psql -U provdb -h db.ci.uchicago.edu provdb\"" >> $PROVDB_HOME/etc/provenance.config - $PROVDB_HOME/swift-prov-import-all-logs - cd $IMPORT_HOME - echo $i >> provdb_imported_raptor - cd swift-logs - # annotate workflows with their oops runid - OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}') - cd $PROTESTS_HOME/$k/$i - LOG_FILENAME=$(ls | grep RaptorLoops- | grep "\."log$) - WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}') - cd $IMPORT_HOME/swift-logs - echo "insert into annot_wf_txt values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD - - echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql; - - $SQLCMD -t -A -F " " -f query.sql -o result.txt - - DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt) - FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g') - - cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID - - while read line; do - NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}') - if [ "$NAME" = "SAMPLE RANGE" ]; then - VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}') - VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}') - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD - fi - if [ "$NAME" = "RESTRAIN DISTANCE" ]; then - VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}') - VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}') - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD - fi - if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then - VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}') - echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD - fi - done < $FILENAME - fi - fi - done -done From noreply at svn.ci.uchicago.edu Sun Dec 12 22:45:17 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 12 Dec 2010 22:45:17 -0600 (CST) Subject: [Swift-commit] r3768 - in SwiftApps/SwiftR: . Swift/exec Message-ID: <20101213044517.AE69C9CC82@svn.ci.uchicago.edu> Author: wilde Date: 2010-12-12 22:45:17 -0600 (Sun, 12 Dec 2010) New Revision: 3768 Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbs SwiftApps/SwiftR/Swift/exec/start-swift SwiftApps/SwiftR/install.sh Log: initial changes to handle SGE. May need to re-adjust PBS changes. Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbs =================================================================== --- SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2010-12-12 23:47:03 UTC (rev 3767) +++ SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2010-12-13 04:45:17 UTC (rev 3768) @@ -50,7 +50,7 @@ END -true to save <00:00:01 # fast Modified: SwiftApps/SwiftR/Swift/exec/start-swift =================================================================== --- SwiftApps/SwiftR/Swift/exec/start-swift 2010-12-12 23:47:03 UTC (rev 3767) +++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-12-13 04:45:17 UTC (rev 3768) @@ -1,5 +1,7 @@ #! /bin/bash +set -x + export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap # Define internal functions @@ -54,6 +56,8 @@ echo $sshpids > $sshpidfile } +# FIXME: does PBS need same workers-per-node logic as SGE? + make-pbs-submit-file() { if [ $queue != default ]; then @@ -61,7 +65,7 @@ else queueDirective="" fi -cat >pbs.sub <batch.sub <pbs.sub <batch.sub <batch.sub <$pbsjobidfile + qsub batch.sub >$jobidfile - echo Started workers from PBS job $(cat $pbsjobidfile) + echo Started workers from batch job $(cat $jobidfile) } usage() @@ -216,6 +284,7 @@ throttle=10 hosts=no-hosts-specified queue=default +project=default while [ $# -gt 0 ] do @@ -225,7 +294,8 @@ -n) nodes=$2; verify-is-numeric nodes $nodes; shift ;; -p) throttle=$2; verify-is-numeric throttle $throttle; shift ;; -q) queue=$2; verify-not-null queue $queue; shift ;; - -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf; shift ;; + -A) project=$2; verify-not-null project $project; shift ;; + -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge; shift ;; -t) time=$2; verify-not-null time $time; shift ;; *) usage; exit 1 ;; esac @@ -300,31 +370,31 @@ wait-and-start-ssh-workers & starterpid=$! -elif [ \( $server = pbs \) -o \( $server = pbsf \) ]; then +elif [ \( $server = pbs \) -o \( $server = pbsf \) -o \( $server = sge \) ]; then - source $SWIFTRBIN/configure-server-pbs $cores + source $SWIFTRBIN/configure-server-${server} $cores - pbsjobidfile=${out/stdouterr/pbsjobid} + jobidfile=${out/stdouterr/jobid} TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap function onexit { coasterservicepid="" # null: saved in case we go back to using coaster servers trap - $TRAPEVENTS - pbsjobid=$(cat $pbsjobidfile) - echo Terminating worker processes starter $starterpid and PBS job $pbsjobid + jobid=$(cat $jobidfile) + echo Terminating worker processes starter $starterpid and batch job $jobid if [ "_$starterpid != _ ]; then kill $starterpid fi - if [ "_$pbsjobid != _ ]; then - qdel $pbsjobid + if [ "_$jobid != _ ]; then + qdel $jobid fi kill 0 # Kill all procs in current process group # FIXME: what was this for???? } trap onexit $TRAPEVENTS - wait-and-start-pbs-workers & + wait-and-start-batch-workers & starterpid=$! fi Modified: SwiftApps/SwiftR/install.sh =================================================================== --- SwiftApps/SwiftR/install.sh 2010-12-12 23:47:03 UTC (rev 3767) +++ SwiftApps/SwiftR/install.sh 2010-12-13 04:45:17 UTC (rev 3768) @@ -1,4 +1,8 @@ ver=0.1 +rm -rf Swift/inst/swift/* +mkdir -p Swift/inst/swift +SWIFTREL=$(cd $(dirname $(which swift))/..; pwd) +cp -pr $SWIFTREL/* Swift/inst/swift R CMD build Swift R CMD INSTALL Swift_${ver}.tar.gz cp Swift_${ver}.tar.gz ~/public_html From noreply at svn.ci.uchicago.edu Mon Dec 13 11:46:18 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 11:46:18 -0600 (CST) Subject: [Swift-commit] r3769 - trunk/tests/site/intrepid Message-ID: <20101213174618.648029CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 11:46:18 -0600 (Mon, 13 Dec 2010) New Revision: 3769 Added: trunk/tests/site/intrepid/100-cp.check.sh trunk/tests/site/intrepid/100-cp.clean.sh trunk/tests/site/intrepid/100-cp.setup.sh trunk/tests/site/intrepid/100-cp.swift trunk/tests/site/intrepid/100-cp.timeout trunk/tests/site/intrepid/sites.template.xml trunk/tests/site/intrepid/tc.template.data trunk/tests/site/intrepid/title.txt Log: Intrepid test Added: trunk/tests/site/intrepid/100-cp.check.sh =================================================================== --- trunk/tests/site/intrepid/100-cp.check.sh (rev 0) +++ trunk/tests/site/intrepid/100-cp.check.sh 2010-12-13 17:46:18 UTC (rev 3769) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +grep $( uname -m ) 100-cp-output.txt || exit 1 + +exit 0 Property changes on: trunk/tests/site/intrepid/100-cp.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/site/intrepid/100-cp.clean.sh =================================================================== --- trunk/tests/site/intrepid/100-cp.clean.sh (rev 0) +++ trunk/tests/site/intrepid/100-cp.clean.sh 2010-12-13 17:46:18 UTC (rev 3769) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +rm -v 100-cp-input.txt 100-cp-output.txt || exit 1 + +exit 0 Property changes on: trunk/tests/site/intrepid/100-cp.clean.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/site/intrepid/100-cp.setup.sh =================================================================== --- trunk/tests/site/intrepid/100-cp.setup.sh (rev 0) +++ trunk/tests/site/intrepid/100-cp.setup.sh 2010-12-13 17:46:18 UTC (rev 3769) @@ -0,0 +1,7 @@ +#!/bin/sh + +set -x + +uname -a > 100-cp-input.txt || exit 1 + +exit 0 Property changes on: trunk/tests/site/intrepid/100-cp.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/site/intrepid/100-cp.swift =================================================================== --- trunk/tests/site/intrepid/100-cp.swift (rev 0) +++ trunk/tests/site/intrepid/100-cp.swift 2010-12-13 17:46:18 UTC (rev 3769) @@ -0,0 +1,13 @@ + +type file; + +app (file o) cp(file i) +{ + cp @i @o; +} + +file input<"100-cp-input.txt">; +file output<"100-cp-output.txt">; + +output = cp(input); + Added: trunk/tests/site/intrepid/100-cp.timeout =================================================================== --- trunk/tests/site/intrepid/100-cp.timeout (rev 0) +++ trunk/tests/site/intrepid/100-cp.timeout 2010-12-13 17:46:18 UTC (rev 3769) @@ -0,0 +1 @@ +3000 Added: trunk/tests/site/intrepid/sites.template.xml =================================================================== --- trunk/tests/site/intrepid/sites.template.xml (rev 0) +++ trunk/tests/site/intrepid/sites.template.xml 2010-12-13 17:46:18 UTC (rev 3769) @@ -0,0 +1,32 @@ + + + + + + /scratch/wozniak/work + + 0.04 + file + + + + + + + _HOST_ + _PROJECT_ + _QUEUE_ + zeptoos + true + 21 + 10000 + 1 + DEBUG + 1 + 900 + 64 + 64 + _WORK_ + + + Added: trunk/tests/site/intrepid/tc.template.data =================================================================== --- trunk/tests/site/intrepid/tc.template.data (rev 0) +++ trunk/tests/site/intrepid/tc.template.data 2010-12-13 17:46:18 UTC (rev 3769) @@ -0,0 +1 @@ +coasters_alcfbgp cp /bin/cp INSTALLED INTEL32::LINUX null Added: trunk/tests/site/intrepid/title.txt =================================================================== --- trunk/tests/site/intrepid/title.txt (rev 0) +++ trunk/tests/site/intrepid/title.txt 2010-12-13 17:46:18 UTC (rev 3769) @@ -0,0 +1 @@ +Site Test: BG/P: Intrepid From noreply at svn.ci.uchicago.edu Mon Dec 13 11:54:26 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 11:54:26 -0600 (CST) Subject: [Swift-commit] r3770 - trunk/tests Message-ID: <20101213175426.AB2139CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 11:54:26 -0600 (Mon, 13 Dec 2010) New Revision: 3770 Modified: trunk/tests/nightly.sh Log: Minor improvements to run on Intrepid Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-12-13 17:46:18 UTC (rev 3769) +++ trunk/tests/nightly.sh 2010-12-13 17:54:26 UTC (rev 3770) @@ -15,6 +15,8 @@ # Everything for a Swift test is written in its RUNDIR # The temporary output always goes to OUTPUT (TOPDIR/exec.out) +# Note that some schedulers restrict your choice of RUNDIR + # Each *.swift test may be accompanied by a # *.setup.sh, *.check.sh, and/or *.clean.sh script # and a *.timeout specifier @@ -44,6 +46,9 @@ # PID TREE: # Background processes are used so that hung Swift jobs can be killed # These are the background processes (PIDs are tracked) +# Note that PID management has not yet been perfected. Check ps +# in error cases. +# # nightly.sh # +-monitor() # +-sleep @@ -172,6 +177,18 @@ fi } +# Ensure all given variables are set +checkvars() { + while (( ${#*} )) + do + VAR=$1 + V=$( eval "echo \${${VAR}+1}" ) + [[ $V == 1 ]] || crash "Not set: $VAR" + shift + done + return 0 +} + crash() { MSG=$1 echo $MSG @@ -568,7 +585,21 @@ test_exec() { banner "$TEST (part $SEQ)" echo "Executing $TEST (part $SEQ)" - process_exec "$@" + pwd + printf "\nExecuting: $@" >>$LOG + + rm -f $OUTPUT + "$@" > $OUTPUT 2>&1 + EXITCODE=$? + + if [ "$EXITCODE" == "127" ]; then + echo "Command not found: $@" > $OUTPUT + fi + + if [ -f $OUTPUT ]; then + cat $OUTPUT >> $LOG + fi + RESULT=$( result ) test_log out test $SEQ "$LASTCMD" $RESULT $TEST_LOG @@ -576,6 +607,7 @@ check_bailout let "SEQ=$SEQ+1" + return $EXITCODE } # Background process monitoring function @@ -761,6 +793,18 @@ out package "swift-$DATE.tar.gz" } +# Environment must contain PROJECT, QUEUE, and WORK +make_sites_sed() { + checkvars WORK QUEUE PROJECT + { + echo "s at _WORK_@$WORK@" + echo "s at _HOST_@$GLOBUS_HOSTNAME@" + echo "s at _PROJECT_@$PROJECT@" + echo "s at _QUEUE_@$QUEUE@" + } > $RUNDIR/sites.sed + return 0 +} + if which ifconfig > /dev/null; then IFCONFIG=ifconfig else @@ -770,10 +814,11 @@ GLOBUS_HOSTNAME=$( $IFCONFIG | grep inet | head -1 | cut -d ':' -f 2 | \ awk '{print $1}' ) [ $? != 0 ] && crash "Could not obtain GLOBUS_HOSTNAME!" + group_sites_xml() { TEMPLATE=$GROUP/sites.template.xml if [ -f $TEMPLATE ]; then - sed "s at _WORK_@$PWD/work@;s at _HOST_@$GLOBUS_HOSTNAME@" < $TEMPLATE > sites.xml + sed -f $RUNDIR/sites.sed < $TEMPLATE > sites.xml [ $? != 0 ] && crash "Could not create sites.xml!" echo "Using: $GROUP/sites.template.xml" else @@ -851,6 +896,8 @@ date > $LOG +make_sites_sed + header start_test_results cd $TOPDIR @@ -914,6 +961,7 @@ $TESTDIR/cdm/ps \ $TESTDIR/cdm/star $TESTDIR/cdm/ps/pinned + # $TESTDIR/site/intrepid ) GROUPCOUNT=1 From noreply at svn.ci.uchicago.edu Mon Dec 13 12:06:06 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 12:06:06 -0600 (CST) Subject: [Swift-commit] r3771 - trunk/tests Message-ID: <20101213180606.9AF289CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 12:06:06 -0600 (Mon, 13 Dec 2010) New Revision: 3771 Modified: trunk/tests/nightly.sh Log: More usage notes Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-12-13 17:54:26 UTC (rev 3770) +++ trunk/tests/nightly.sh 2010-12-13 18:06:06 UTC (rev 3771) @@ -15,8 +15,6 @@ # Everything for a Swift test is written in its RUNDIR # The temporary output always goes to OUTPUT (TOPDIR/exec.out) -# Note that some schedulers restrict your choice of RUNDIR - # Each *.swift test may be accompanied by a # *.setup.sh, *.check.sh, and/or *.clean.sh script # and a *.timeout specifier @@ -30,12 +28,15 @@ # Tests are GROUPed into directories # Each GROUP directory has: # 1) a list of *.swift tests (plus *.sh scripts) -# 2) optionally a tc.template.data -# 3) optionally a fs.template.data -# 4) optionally a swift.properties -# 5) optionally a title.txt -# 6) preferably a README.txt +# 2) optionally a sites.template.xml +# 3) optionally a tc.template.data +# 4) optionally a fs.template.data +# 5) optionally a swift.properties +# 6) optionally a title.txt +# 7) preferably a README.txt # Edit GROUPLIST at the end of this script before running +# template files are lightly processed by sed before use +# Missing files will be pulled from swift/etc # OUTPUT is the stdout of the current test # stdout.txt retains stdout from the previous test (for *.clean.sh) @@ -62,6 +63,12 @@ # The response of nightly.sh to the exit code of these Swift # executions is reversed. +# SCHEDULERS +# Environment must contain PROJECT, QUEUE, and WORK +# These variables will be incorporated into the sites.xml +# via make_sites_sed() -> group_sites_xml() +# Note that some schedulers restrict your choice of RUNDIR + printhelp() { echo "nightly.sh " echo "" @@ -392,6 +399,7 @@ echo "<$TYPE>$1|$2|$3|$4|$5|$6|$7|$8|$9|" } +# Create HTML output out() { # echo $@ TYPE=$1 @@ -534,6 +542,7 @@ fi } +# Execute process in the background process_exec() { printf "\nExecuting: $@" >>$LOG rm -f $OUTPUT @@ -793,7 +802,7 @@ out package "swift-$DATE.tar.gz" } -# Environment must contain PROJECT, QUEUE, and WORK +# Generate the sites.sed file make_sites_sed() { checkvars WORK QUEUE PROJECT { @@ -805,6 +814,7 @@ return 0 } +# Setup coasters variables if which ifconfig > /dev/null; then IFCONFIG=ifconfig else @@ -815,6 +825,7 @@ awk '{print $1}' ) [ $? != 0 ] && crash "Could not obtain GLOBUS_HOSTNAME!" +# Generate sites.xml group_sites_xml() { TEMPLATE=$GROUP/sites.template.xml if [ -f $TEMPLATE ]; then @@ -828,6 +839,7 @@ fi } +# Generate tc.data group_tc_data() { if [ -f $GROUP/tc.template.data ]; then sed "s at _DIR_@$GROUP@" < $GROUP/tc.template.data > tc.data @@ -839,6 +851,7 @@ fi } +# Generate the CDM file, fs.data group_fs_data() { if [ -f $GROUP/fs.template.data ]; then sed "s at _PWD_@$PWD@" < $GROUP/fs.template.data > fs.data @@ -849,6 +862,7 @@ fi } +# Generate swift.properties group_swift_properties() { if [ -f $GROUP/swift.properties ]; then cp -v $GROUP/swift.properties . @@ -859,6 +873,7 @@ fi } +# Obtain the group title group_title() { if [ -r $GROUP/title.txt ]; then cat $GROUP/title.txt @@ -867,6 +882,7 @@ fi } +# Execute all tests in current GROUP test_group() { group_sites_xml From noreply at svn.ci.uchicago.edu Mon Dec 13 12:49:14 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 12:49:14 -0600 (CST) Subject: [Swift-commit] r3772 - www Message-ID: <20101213184914.1DDAB9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 12:49:13 -0600 (Mon, 13 Dec 2010) New Revision: 3772 Added: www/update.sh www/updatenodocs.sh Log: Original version of update scripts: from /ci/www/projects/swift Added: www/update.sh =================================================================== --- www/update.sh (rev 0) +++ www/update.sh 2010-12-13 18:49:13 UTC (rev 3772) @@ -0,0 +1,17 @@ +#!/bin/sh + +umask 002 +echo "--------- Updating www... ----------" +svn update +chmod -R g+w .svn 2>/dev/null +echo "--------- Updating docs... ----------" +cd guides +svn update +chmod -R g+w .svn 2>/dev/null +chmod -R g+w formatting/.svn 2>/dev/null +./buildguides.sh +chmod g+w *.php +chmod g+w *.pdf +chmod -R g+w userguide +echo "--------- All done ----------" + Property changes on: www/update.sh ___________________________________________________________________ Name: svn:executable + * Added: www/updatenodocs.sh =================================================================== --- www/updatenodocs.sh (rev 0) +++ www/updatenodocs.sh 2010-12-13 18:49:13 UTC (rev 3772) @@ -0,0 +1,8 @@ +#!/bin/sh + +umask 002 +echo "--------- Updating www... ----------" +svn update +chmod -R g+w .svn 2>/dev/null +echo "--------- All done ----------" + Property changes on: www/updatenodocs.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Mon Dec 13 15:06:48 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:06:48 -0600 (CST) Subject: [Swift-commit] r3773 - trunk/docs Message-ID: <20101213210648.04EF09CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:06:47 -0600 (Mon, 13 Dec 2010) New Revision: 3773 Modified: trunk/docs/README Log: Start notes on how to run Swift web site on workstation Modified: trunk/docs/README =================================================================== --- trunk/docs/README 2010-12-13 18:49:13 UTC (rev 3772) +++ trunk/docs/README 2010-12-13 21:06:47 UTC (rev 3773) @@ -37,7 +37,23 @@ $ cd formatting $ ln -s /opt/local/share/xsl/docbook-xsl/ docbook +C) in general: + 1) Install Apache + + 2) Install PHP (cf. http://dan.drydog.com/apache2php.html) + + 3) Add these lines to Apache's httpd.conf: + + AddHandler php5-script php + AddType text/html php + + 4) Update httpd.conf, adding index.php: + + DirectoryIndex index.html index.php + + 5) Make sure perms are correct + Once the links are set up, the buildguides.sh script will build all guides as php. Run it with no parameters, like this: From noreply at svn.ci.uchicago.edu Mon Dec 13 15:08:04 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:08:04 -0600 (CST) Subject: [Swift-commit] r3774 - trunk/docs/formatting Message-ID: <20101213210804.972049CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:08:04 -0600 (Mon, 13 Dec 2010) New Revision: 3774 Modified: trunk/docs/formatting/swiftsh_html_chunked.xsl Log: Clean whitespace Modified: trunk/docs/formatting/swiftsh_html_chunked.xsl =================================================================== --- trunk/docs/formatting/swiftsh_html_chunked.xsl 2010-12-13 21:06:47 UTC (rev 3773) +++ trunk/docs/formatting/swiftsh_html_chunked.xsl 2010-12-13 21:08:04 UTC (rev 3774) @@ -10,29 +10,29 @@ 1 - + 1 /docbook-images/ @@ -42,10 +42,10 @@ 0 0 @@ -54,7 +54,7 @@ 2 4 1 - + @@ -99,49 +99,49 @@ yes - + - + @@ -165,7 +165,7 @@ - + @@ -186,9 +186,9 @@ - + - + @@ -213,7 +213,7 @@ @@ -231,14 +231,14 @@ - + - +
@@ -256,13 +256,13 @@ ]]> - + - +
From noreply at svn.ci.uchicago.edu Mon Dec 13 15:08:30 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:08:30 -0600 (CST) Subject: [Swift-commit] r3775 - trunk/docs/formatting Message-ID: <20101213210830.837949CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:08:30 -0600 (Mon, 13 Dec 2010) New Revision: 3775 Modified: trunk/docs/formatting/swiftsh_html_chunked.xsl Log: Set include paths for new directory structure Modified: trunk/docs/formatting/swiftsh_html_chunked.xsl =================================================================== --- trunk/docs/formatting/swiftsh_html_chunked.xsl 2010-12-13 21:08:04 UTC (rev 3774) +++ trunk/docs/formatting/swiftsh_html_chunked.xsl 2010-12-13 21:08:30 UTC (rev 3775) @@ -243,12 +243,14 @@
@@ -262,7 +264,7 @@
- + From noreply at svn.ci.uchicago.edu Mon Dec 13 15:09:57 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:09:57 -0600 (CST) Subject: [Swift-commit] r3777 - trunk/docs Message-ID: <20101213210957.C9E8C9CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:09:57 -0600 (Mon, 13 Dec 2010) New Revision: 3777 Modified: trunk/docs/userguide.xml Log: Fix a link Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2010-12-13 21:09:36 UTC (rev 3776) +++ trunk/docs/userguide.xml 2010-12-13 21:09:57 UTC (rev 3777) @@ -13,7 +13,7 @@ This manual provides reference material for Swift: the SwiftScript language and the Swift runtime system. For introductory material, consult -the Swift +the Swift tutorial. From noreply at svn.ci.uchicago.edu Mon Dec 13 15:10:34 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:10:34 -0600 (CST) Subject: [Swift-commit] r3778 - trunk/docs Message-ID: <20101213211034.EBEA29CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:10:34 -0600 (Mon, 13 Dec 2010) New Revision: 3778 Modified: trunk/docs/Makefile Log: Adding targets to Makefile... Modified: trunk/docs/Makefile =================================================================== --- trunk/docs/Makefile 2010-12-13 21:09:57 UTC (rev 3777) +++ trunk/docs/Makefile 2010-12-13 21:10:34 UTC (rev 3778) @@ -1,5 +1,7 @@ -#!/bin/sh +# FOP = fop +FOP = fop/fop.sh + all: phps pdfs phps: userguide.php tutorial.php tutorial-live.php quickstartguide.php reallyquickstartguide.php provenance.php historical/languagespec.php historical/languagespec-0.6.php log-processing.php plot-tour.php @@ -8,15 +10,26 @@ pdfs: userguide.pdf tutorial.pdf tutorial-live.pdf quickstartguide.pdf reallyquickstartguide.pdf provenance.pdf historical/languagespec.pdf historical/languagespec-0.6.pdf log-processing.pdf +GUIDE_PHP=$(shell find userguide -name "*.php" ) +GUIDE_HTML=$(patsubst %.php,%.html,$(GUIDE_PHP)) + +guide_html: $(GUIDE_HTML) + chunked-userguide: userguide.xml ./build-chunked-userguide.sh %.php: %.xml formatting/swiftsh_html.xsl xsltproc --nonet formatting/swiftsh_html.xsl $< sed -e "s/index.html#/#/g" index.html >$@ + chmod a+rx $@ %.pdf: %.xml formatting/vdl2_fo.xsl - fop/fop.sh -xsl formatting/vdl2_fo.xsl -xml $< -pdf $@ + $(FOP) -xsl formatting/vdl2_fo.xsl -xml $< -pdf $@ + chmod a+rx $@ %.html: %.php cp $< $@ + chmod a+rx $@ + +clean: + rm -fv *.php userguide/*.php From noreply at svn.ci.uchicago.edu Mon Dec 13 15:13:42 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:13:42 -0600 (CST) Subject: [Swift-commit] r3779 - www Message-ID: <20101213211342.DA6EA9CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:13:42 -0600 (Mon, 13 Dec 2010) New Revision: 3779 Added: www/README.txt Log: Notes Added: www/README.txt =================================================================== --- www/README.txt (rev 0) +++ www/README.txt 2010-12-13 21:13:42 UTC (rev 3779) @@ -0,0 +1,16 @@ + +Formerly, + +/guides was trunk/docs + +Now, + +/guides/trunk is SVN trunk/docs +/guides/release-0.91 is SVN branches/1.0/docs +/guides/current is SVN branches/1.0/docs + +/docs/index.php will be updated to have multiple versions for QS +Guide, Tutorial, User Guide and Log Processing Tools. The current +versions will be labeled "current" and "trunk". Other versions could +easily be exposed in the future. + From noreply at svn.ci.uchicago.edu Mon Dec 13 15:15:52 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:15:52 -0600 (CST) Subject: [Swift-commit] r3780 - trunk/docs Message-ID: <20101213211552.34C419CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:15:52 -0600 (Mon, 13 Dec 2010) New Revision: 3780 Added: trunk/docs/README.txt Removed: trunk/docs/README Log: Rename README Deleted: trunk/docs/README =================================================================== --- trunk/docs/README 2010-12-13 21:13:42 UTC (rev 3779) +++ trunk/docs/README 2010-12-13 21:15:52 UTC (rev 3780) @@ -1,65 +0,0 @@ -Swift Documentation -=================== - -General principles: - - - sections and subsections are ordered from , , or using - arbitrary depth
tags - - code samples are given inside tags, which will cause - syntax highlighting to happen automatically - - user interactions / screen output are given inside tags. - - be careful to escape in-text "<" to "<" - - there are some conventions for using id attributes at various - places in the documents - for example, some tutorial sections use - 'tutorial.'; profile entries in the user guide use - 'profile..'. Try to keep id attributes unique across - the entire document set. - -The first time guides are built in a particular checkout, it is necessary -to place the docbook formatting stylesheets under the formatting/docbook/ -directory. This can be done with a symlink if docbook is installed elsewhere. - -For example: - -A) On the CI network, /home/hategan/docbook contains a docbook installation that -can be linked like this: - -$ cd formatting -$ ln -s /home/hategan/docbook/ docbook - - -B) on benc's os x machine: - -# install docbook from DarwinPorts -$ sudo port install docbook-xsl - -# setup links -$ cd formatting -$ ln -s /opt/local/share/xsl/docbook-xsl/ docbook - -C) in general: - - 1) Install Apache - - 2) Install PHP (cf. http://dan.drydog.com/apache2php.html) - - 3) Add these lines to Apache's httpd.conf: - - AddHandler php5-script php - AddType text/html php - - 4) Update httpd.conf, adding index.php: - - DirectoryIndex index.html index.php - - 5) Make sure perms are correct - -Once the links are set up, the buildguides.sh script will build all guides -as php. Run it with no parameters, like this: - -$ ./buildguides.sh - -or use make to get HTML documents like this: - -$ make userguide.html - Copied: trunk/docs/README.txt (from rev 3779, trunk/docs/README) =================================================================== --- trunk/docs/README.txt (rev 0) +++ trunk/docs/README.txt 2010-12-13 21:15:52 UTC (rev 3780) @@ -0,0 +1,65 @@ +Swift Documentation +=================== + +General principles: + + - sections and subsections are ordered from , , or using + arbitrary depth
tags + - code samples are given inside tags, which will cause + syntax highlighting to happen automatically + - user interactions / screen output are given inside tags. + - be careful to escape in-text "<" to "<" + - there are some conventions for using id attributes at various + places in the documents - for example, some tutorial sections use + 'tutorial.'; profile entries in the user guide use + 'profile..'. Try to keep id attributes unique across + the entire document set. + +The first time guides are built in a particular checkout, it is necessary +to place the docbook formatting stylesheets under the formatting/docbook/ +directory. This can be done with a symlink if docbook is installed elsewhere. + +For example: + +A) On the CI network, /home/hategan/docbook contains a docbook installation that +can be linked like this: + +$ cd formatting +$ ln -s /home/hategan/docbook/ docbook + + +B) on benc's os x machine: + +# install docbook from DarwinPorts +$ sudo port install docbook-xsl + +# setup links +$ cd formatting +$ ln -s /opt/local/share/xsl/docbook-xsl/ docbook + +C) in general: + + 1) Install Apache + + 2) Install PHP (cf. http://dan.drydog.com/apache2php.html) + + 3) Add these lines to Apache's httpd.conf: + + AddHandler php5-script php + AddType text/html php + + 4) Update httpd.conf, adding index.php: + + DirectoryIndex index.html index.php + + 5) Make sure perms are correct + +Once the links are set up, the buildguides.sh script will build all guides +as php. Run it with no parameters, like this: + +$ ./buildguides.sh + +or use make to get HTML documents like this: + +$ make userguide.html + From noreply at svn.ci.uchicago.edu Mon Dec 13 15:22:38 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:22:38 -0600 (CST) Subject: [Swift-commit] r3781 - trunk/docs Message-ID: <20101213212238.CBECB9CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:22:38 -0600 (Mon, 13 Dec 2010) New Revision: 3781 Modified: trunk/docs/build-chunked-userguide.sh Log: Add chmod Modified: trunk/docs/build-chunked-userguide.sh =================================================================== --- trunk/docs/build-chunked-userguide.sh 2010-12-13 21:15:52 UTC (rev 3780) +++ trunk/docs/build-chunked-userguide.sh 2010-12-13 21:22:38 UTC (rev 3781) @@ -7,4 +7,5 @@ cp ../*.jpeg . xsltproc --nonet ../formatting/swiftsh_html_chunked.xsl ../userguide.xml +chmod a+r *.php From noreply at svn.ci.uchicago.edu Mon Dec 13 15:25:36 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:25:36 -0600 (CST) Subject: [Swift-commit] r3782 - www/docs Message-ID: <20101213212536.4CE899CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:25:36 -0600 (Mon, 13 Dec 2010) New Revision: 3782 Modified: www/docs/index.php Log: Fix whitespace Modified: www/docs/index.php =================================================================== --- www/docs/index.php 2010-12-13 21:22:38 UTC (rev 3781) +++ www/docs/index.php 2010-12-13 21:25:36 UTC (rev 3782) @@ -23,7 +23,7 @@
- +

DOCUMENTATION

Swift Quickstart Guide

@@ -33,7 +33,7 @@

Use the Quickstart Guide to help you install and configure swift and run a - simple 'Hello World' example. + simple 'Hello World' example.

Swift Tutorials

@@ -53,7 +53,7 @@

- This tutorial (part of the Open Science Grid teaching lab) introduces + This tutorial (part of the Open Science Grid teaching lab) introduces you to the Swift environment and the SwiftScript language in more depth using a provided training account where installation of the software has already been performed. @@ -68,7 +68,7 @@

The User Guide provides more detailed reference documentation and background information on swift. It is assumed that the reader is already familiar with - the material in the Quickstart and Tutorial documents. + the material in the Quickstart and Tutorial documents.

Log Processing tools

@@ -106,7 +106,7 @@
  • SwiftScript Language Reference v 0.5 [doc]

    • - +
    @@ -118,7 +118,7 @@
    - + From noreply at svn.ci.uchicago.edu Mon Dec 13 15:26:07 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:26:07 -0600 (CST) Subject: [Swift-commit] r3783 - www/docs Message-ID: <20101213212607.CE9DC9CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:26:07 -0600 (Mon, 13 Dec 2010) New Revision: 3783 Modified: www/docs/index.php Log: Point default userguide to trunk version (for now) Modified: www/docs/index.php =================================================================== --- www/docs/index.php 2010-12-13 21:25:36 UTC (rev 3782) +++ www/docs/index.php 2010-12-13 21:26:07 UTC (rev 3783) @@ -62,7 +62,7 @@

    Swift User Guide

    From noreply at svn.ci.uchicago.edu Mon Dec 13 15:31:48 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:31:48 -0600 (CST) Subject: [Swift-commit] r3784 - trunk/docs Message-ID: <20101213213148.668679CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:31:48 -0600 (Mon, 13 Dec 2010) New Revision: 3784 Modified: trunk/docs/README.txt Log: More notes Modified: trunk/docs/README.txt =================================================================== --- trunk/docs/README.txt 2010-12-13 21:26:07 UTC (rev 3783) +++ trunk/docs/README.txt 2010-12-13 21:31:48 UTC (rev 3784) @@ -54,6 +54,10 @@ 5) Make sure perms are correct + 6) Create formatting/docbook link (see above) + + 7) Create fop link + Once the links are set up, the buildguides.sh script will build all guides as php. Run it with no parameters, like this: From noreply at svn.ci.uchicago.edu Mon Dec 13 15:33:57 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:33:57 -0600 (CST) Subject: [Swift-commit] r3785 - www Message-ID: <20101213213357.CC6D59CC80@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:33:57 -0600 (Mon, 13 Dec 2010) New Revision: 3785 Modified: www/update.sh Log: Use update.sh on guides/trunk Modified: www/update.sh =================================================================== --- www/update.sh 2010-12-13 21:31:48 UTC (rev 3784) +++ www/update.sh 2010-12-13 21:33:57 UTC (rev 3785) @@ -4,14 +4,21 @@ echo "--------- Updating www... ----------" svn update chmod -R g+w .svn 2>/dev/null +chmod a+r docs/index.php echo "--------- Updating docs... ----------" -cd guides +echo "--------- Updating docs: trunk... ----------" +pushd guides/trunk svn update chmod -R g+w .svn 2>/dev/null chmod -R g+w formatting/.svn 2>/dev/null +chmod a+rx . userguide +pushd userguide +chmod a+r *.php *.jpeg *.png +popd ./buildguides.sh chmod g+w *.php chmod g+w *.pdf chmod -R g+w userguide +popd echo "--------- All done ----------" From noreply at svn.ci.uchicago.edu Mon Dec 13 15:39:51 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:39:51 -0600 (CST) Subject: [Swift-commit] r3786 - www/docs Message-ID: <20101213213951.A73709CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:39:51 -0600 (Mon, 13 Dec 2010) New Revision: 3786 Modified: www/docs/index.php Log: Version-specific userguide link Modified: www/docs/index.php =================================================================== --- www/docs/index.php 2010-12-13 21:33:57 UTC (rev 3785) +++ www/docs/index.php 2010-12-13 21:39:51 UTC (rev 3786) @@ -62,15 +62,20 @@

    Swift User Guide

    -

    The User Guide provides more detailed reference documentation and background information on swift. It is assumed that the reader is already familiar with the material in the Quickstart and Tutorial documents.

    +
  • trunk + [multi-page html] + [single-page html] + [pdf] +
    • + +

    Log Processing tools

    • Swift log processing tools [html] [pdf]
      • From noreply at svn.ci.uchicago.edu Mon Dec 13 15:53:52 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:53:52 -0600 (CST) Subject: [Swift-commit] r3787 - in branches/1.0/docs: . formatting Message-ID: <20101213215352.98EDC9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:53:52 -0600 (Mon, 13 Dec 2010) New Revision: 3787 Modified: branches/1.0/docs/Makefile branches/1.0/docs/formatting/swiftsh_html_chunked.xsl Log: Fix up for use in new www setup Modified: branches/1.0/docs/Makefile =================================================================== --- branches/1.0/docs/Makefile 2010-12-13 21:39:51 UTC (rev 3786) +++ branches/1.0/docs/Makefile 2010-12-13 21:53:52 UTC (rev 3787) @@ -1,5 +1,7 @@ -#!/bin/sh +FOP = fop +# FOP = fop/fop.sh + all: phps pdfs phps: userguide.php tutorial.php tutorial-live.php quickstartguide.php reallyquickstartguide.php provenance.php historical/languagespec.php historical/languagespec-0.6.php log-processing.php plot-tour.php @@ -8,15 +10,26 @@ pdfs: userguide.pdf tutorial.pdf tutorial-live.pdf quickstartguide.pdf reallyquickstartguide.pdf provenance.pdf historical/languagespec.pdf historical/languagespec-0.6.pdf log-processing.pdf +GUIDE_PHP=$(shell find userguide -name "*.php" ) +GUIDE_HTML=$(patsubst %.php,%.html,$(GUIDE_PHP)) + +guide_html: $(GUIDE_HTML) + chunked-userguide: userguide.xml ./build-chunked-userguide.sh %.php: %.xml formatting/swiftsh_html.xsl xsltproc --nonet formatting/swiftsh_html.xsl $< sed -e "s/index.html#/#/g" index.html >$@ + chmod a+rx $@ %.pdf: %.xml formatting/vdl2_fo.xsl - fop/fop.sh -xsl formatting/vdl2_fo.xsl -xml $< -pdf $@ + $(FOP) -xsl formatting/vdl2_fo.xsl -xml $< -pdf $@ + chmod a+rx $@ %.html: %.php cp $< $@ + chmod a+rx $@ + +clean: + rm -fv *.php userguide/*.php Modified: branches/1.0/docs/formatting/swiftsh_html_chunked.xsl =================================================================== --- branches/1.0/docs/formatting/swiftsh_html_chunked.xsl 2010-12-13 21:39:51 UTC (rev 3786) +++ branches/1.0/docs/formatting/swiftsh_html_chunked.xsl 2010-12-13 21:53:52 UTC (rev 3787) @@ -10,29 +10,29 @@ 1 - + 1 /docbook-images/ @@ -42,10 +42,10 @@ 0 0 @@ -54,7 +54,7 @@ 2 4 1 - + @@ -99,49 +99,49 @@ yes - + - + @@ -165,7 +165,7 @@ - + @@ -186,9 +186,9 @@ - + - + @@ -213,7 +213,7 @@ @@ -231,24 +231,26 @@ - + - +
      @@ -256,13 +258,13 @@ ]]> - + - +
      From noreply at svn.ci.uchicago.edu Mon Dec 13 15:56:02 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 15:56:02 -0600 (CST) Subject: [Swift-commit] r3788 - in www: . docs Message-ID: <20101213215602.464DE9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 15:56:02 -0600 (Mon, 13 Dec 2010) New Revision: 3788 Modified: www/docs/index.php www/update.sh Log: Enable second userguide links Modified: www/docs/index.php =================================================================== --- www/docs/index.php 2010-12-13 21:53:52 UTC (rev 3787) +++ www/docs/index.php 2010-12-13 21:56:02 UTC (rev 3788) @@ -67,6 +67,12 @@ information on swift. It is assumed that the reader is already familiar with the material in the Quickstart and Tutorial documents.

      +
    • current + [multi-page html] + [single-page html] + [pdf] +
    • trunk [multi-page html] [single-page html] Modified: www/update.sh =================================================================== --- www/update.sh 2010-12-13 21:53:52 UTC (rev 3787) +++ www/update.sh 2010-12-13 21:56:02 UTC (rev 3788) @@ -1,24 +1,36 @@ #!/bin/sh +update_docs() +{ + DIR=$1 + echo "--------- Updating docs: ${DIR} ----------" + pushd ${DIR} + svn update + chmod -R g+w .svn 2>/dev/null + chmod -R g+w formatting/.svn 2>/dev/null + chmod a+rx . userguide + pushd userguide + chmod a+r *.php *.jpeg *.png + popd + ./buildguides.sh + chmod g+w *.php + chmod g+w *.pdf + chmod -R g+w userguide + popd + return 0 +} + umask 002 echo "--------- Updating www... ----------" svn update chmod -R g+w .svn 2>/dev/null chmod a+r docs/index.php echo "--------- Updating docs... ----------" -echo "--------- Updating docs: trunk... ----------" -pushd guides/trunk -svn update -chmod -R g+w .svn 2>/dev/null -chmod -R g+w formatting/.svn 2>/dev/null -chmod a+rx . userguide -pushd userguide -chmod a+r *.php *.jpeg *.png -popd -./buildguides.sh -chmod g+w *.php -chmod g+w *.pdf -chmod -R g+w userguide -popd + +for SET in guides/trunk guides/release-0.91 +do + update_docs ${SET} +done + echo "--------- All done ----------" From noreply at svn.ci.uchicago.edu Mon Dec 13 16:15:02 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 16:15:02 -0600 (CST) Subject: [Swift-commit] r3789 - branches/1.0/docs Message-ID: <20101213221502.525F59CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 16:15:02 -0600 (Mon, 13 Dec 2010) New Revision: 3789 Modified: branches/1.0/docs/Makefile Log: Revert to previous fop name Modified: branches/1.0/docs/Makefile =================================================================== --- branches/1.0/docs/Makefile 2010-12-13 21:56:02 UTC (rev 3788) +++ branches/1.0/docs/Makefile 2010-12-13 22:15:02 UTC (rev 3789) @@ -1,6 +1,6 @@ -FOP = fop -# FOP = fop/fop.sh +# FOP = fop +FOP = fop/fop.sh all: phps pdfs From noreply at svn.ci.uchicago.edu Mon Dec 13 16:15:46 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 16:15:46 -0600 (CST) Subject: [Swift-commit] r3790 - www Message-ID: <20101213221546.980669CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 16:15:46 -0600 (Mon, 13 Dec 2010) New Revision: 3790 Modified: www/update.sh Log: Fix perms, automate docbook and fop links, improve naming Modified: www/update.sh =================================================================== --- www/update.sh 2010-12-13 22:15:02 UTC (rev 3789) +++ www/update.sh 2010-12-13 22:15:46 UTC (rev 3790) @@ -1,21 +1,31 @@ #!/bin/sh -update_docs() +crash() { + echo $1 + exit 1 +} + +update_guide() +{ DIR=$1 - echo "--------- Updating docs: ${DIR} ----------" - pushd ${DIR} + echo "--------- Updating guide: ${DIR} ----------" + pushd ${DIR} || crash "Could not enter: ${DIR}" + [[ ! -d formatting/docbook ]] && \ + ln -s ../formatting/docbook formatting/docbook + [[ ! -d formatting/fop ]]&& \ + ln -s ../../fop formatting/fop svn update chmod -R g+w .svn 2>/dev/null chmod -R g+w formatting/.svn 2>/dev/null - chmod a+rx . userguide + ./buildguides.sh pushd userguide chmod a+r *.php *.jpeg *.png - popd - ./buildguides.sh chmod g+w *.php chmod g+w *.pdf + popd chmod -R g+w userguide + chmod a+rx userguide popd return 0 } @@ -29,7 +39,7 @@ for SET in guides/trunk guides/release-0.91 do - update_docs ${SET} + update_guide ${SET} done echo "--------- All done ----------" From noreply at svn.ci.uchicago.edu Mon Dec 13 16:22:39 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 16:22:39 -0600 (CST) Subject: [Swift-commit] r3791 - www/inc Message-ID: <20101213222239.7968B9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 16:22:39 -0600 (Mon, 13 Dec 2010) New Revision: 3791 Modified: www/inc/footer.php Log: Use host-relative links in footer Modified: www/inc/footer.php =================================================================== --- www/inc/footer.php 2010-12-13 22:15:46 UTC (rev 3790) +++ www/inc/footer.php 2010-12-13 22:22:39 UTC (rev 3791) @@ -1,7 +1,7 @@ -HOME - : DOWNLOADS - : SUPPORT - : DOCUMENTATION - : PAPERS - +HOME + : DOWNLOADS + : SUPPORT + : DOCUMENTATION + : PAPERS + From noreply at svn.ci.uchicago.edu Mon Dec 13 16:24:39 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 16:24:39 -0600 (CST) Subject: [Swift-commit] r3792 - www/inc Message-ID: <20101213222439.607BB9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 16:24:39 -0600 (Mon, 13 Dec 2010) New Revision: 3792 Modified: www/inc/nav.php Log: Use host-relative links in nav bar Modified: www/inc/nav.php =================================================================== --- www/inc/nav.php 2010-12-13 22:22:39 UTC (rev 3791) +++ www/inc/nav.php 2010-12-13 22:24:39 UTC (rev 3792) @@ -1,9 +1,9 @@ From noreply at svn.ci.uchicago.edu Mon Dec 13 16:29:11 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 16:29:11 -0600 (CST) Subject: [Swift-commit] r3793 - www/inc Message-ID: <20101213222911.BC5269CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 16:29:11 -0600 (Mon, 13 Dec 2010) New Revision: 3793 Modified: www/inc/footer.php www/inc/nav.php Log: Fix paths Modified: www/inc/footer.php =================================================================== --- www/inc/footer.php 2010-12-13 22:24:39 UTC (rev 3792) +++ www/inc/footer.php 2010-12-13 22:29:11 UTC (rev 3793) @@ -1,7 +1,7 @@ HOME - : DOWNLOADS - : SUPPORT - : DOCUMENTATION - : PAPERS + : DOWNLOADS + : SUPPORT + : DOCUMENTATION + : PAPERS Modified: www/inc/nav.php =================================================================== --- www/inc/nav.php 2010-12-13 22:24:39 UTC (rev 3792) +++ www/inc/nav.php 2010-12-13 22:29:11 UTC (rev 3793) @@ -1,9 +1,9 @@ From noreply at svn.ci.uchicago.edu Mon Dec 13 16:30:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 16:30:25 -0600 (CST) Subject: [Swift-commit] r3794 - www Message-ID: <20101213223025.179449CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-13 16:30:24 -0600 (Mon, 13 Dec 2010) New Revision: 3794 Modified: www/update.sh Log: Add chmods Modified: www/update.sh =================================================================== --- www/update.sh 2010-12-13 22:29:11 UTC (rev 3793) +++ www/update.sh 2010-12-13 22:30:24 UTC (rev 3794) @@ -35,6 +35,8 @@ svn update chmod -R g+w .svn 2>/dev/null chmod a+r docs/index.php +chmod a+r downloads/index.php +chmod a+r inc/nav.php inc/footer.php echo "--------- Updating docs... ----------" for SET in guides/trunk guides/release-0.91 From noreply at svn.ci.uchicago.edu Mon Dec 13 19:16:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 19:16:32 -0600 (CST) Subject: [Swift-commit] r3795 - branches Message-ID: <20101214011632.7A5EA9CC7F@svn.ci.uchicago.edu> Author: skenny Date: 2010-12-13 19:16:32 -0600 (Mon, 13 Dec 2010) New Revision: 3795 Added: branches/release-0.91/ Log: formerly 1.0 Copied: branches/release-0.91 (from rev 3794, branches/1.0) From noreply at svn.ci.uchicago.edu Mon Dec 13 19:53:43 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 19:53:43 -0600 (CST) Subject: [Swift-commit] r3796 - trunk/docs Message-ID: <20101214015343.289C99CC80@svn.ci.uchicago.edu> Author: skenny Date: 2010-12-13 19:53:43 -0600 (Mon, 13 Dec 2010) New Revision: 3796 Modified: trunk/docs/userguide.xml Log: describes mapping files based on URI Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2010-12-14 01:16:32 UTC (rev 3795) +++ trunk/docs/userguide.xml 2010-12-14 01:53:43 UTC (rev 3796) @@ -1385,7 +1385,24 @@
    +
    mapping URIs + +The above mappers may be used to map files based on a URI which can be specified in the filename. This is useful for mapping files on remote machines. + + + + Example: + + + student st <single_file_mapper;file="gsiftp://communicado.ci.uchicago.edu//tmp/student.txt">; + + + + +
    + +
    Commands The commands detailed in this section are available in the From noreply at svn.ci.uchicago.edu Mon Dec 13 22:00:44 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 13 Dec 2010 22:00:44 -0600 (CST) Subject: [Swift-commit] r3797 - trunk/docs Message-ID: <20101214040044.D6C029CC7F@svn.ci.uchicago.edu> Author: skenny Date: 2010-12-13 22:00:44 -0600 (Mon, 13 Dec 2010) New Revision: 3797 Modified: trunk/docs/userguide.xml Log: documented using shell wrapper instead of individual entries in tc.data Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2010-12-14 01:53:43 UTC (rev 3796) +++ trunk/docs/userguide.xml 2010-12-14 04:00:44 UTC (rev 3797) @@ -3611,12 +3611,15 @@ Some example entries: localhost echo /bin/echo INSTALLED INTEL32::LINUX null +localhost touch /bin/touch INSTALLED INTEL32::LINUX null + TGUC touch /usr/bin/touch INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="0:1" +TGUC R /usr/bin/R INSTALLED INTEL32::LINUX env::R_LIBS=/home/skenny/R_libs The fields are: site, transformation name, executable path, installation -status, platform, and profile entrys. +status, platform, and profile entries. The site field should correspond to a site name listed in the sites @@ -3637,7 +3640,29 @@ The profiles field should be set to null if no profile entries are to be specified, or should contain the profile entries separated by semicolons. + +
    shell invocation + +Because the above implementation requires an entry for each executable on a given site, it is often preferable to simply have a single element for each site in the transformation catalog representing a wrapper that sets the environment and then invokes the shell to call a given application. This wrapper is installed on the site and can be used to set the PATH and other environment variables prior to invoking the shell so that each executable need not be entered in the tc.data file. + + for example, the entries for TGUC and localhost can now each be collapsed into a single line: + +localhost shell /usr/local/bin/swiftshell INSTALLED INTEL32::LINUX null +TGUC shell /usr/local/bin/swiftshell INSTALLED INTEL32::LINUX null + +where swiftshell sets up the users's environment so that all the installed applications are added to the PATH before the application is invoked. + + +touch would now be called in the SwiftScript like this: + +app (file tout) shelltest(){ + shell "touch" @filename(tout); +} + + +
    +
    Build options See the From noreply at svn.ci.uchicago.edu Tue Dec 14 10:28:49 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 10:28:49 -0600 (CST) Subject: [Swift-commit] r3798 - www Message-ID: <20101214162849.70B479CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 10:28:48 -0600 (Tue, 14 Dec 2010) New Revision: 3798 Modified: www/update.sh Log: More chmods Modified: www/update.sh =================================================================== --- www/update.sh 2010-12-14 04:00:44 UTC (rev 3797) +++ www/update.sh 2010-12-14 16:28:48 UTC (rev 3798) @@ -1,5 +1,7 @@ #!/bin/sh +# Run this script from /ci/www/projects/swift + crash() { echo $1 @@ -30,6 +32,10 @@ return 0 } +# Ensure tools are group-usable: +chmod g+rw $0 +chmod g+rw updatenodocs.sh + umask 002 echo "--------- Updating www... ----------" svn update From noreply at svn.ci.uchicago.edu Tue Dec 14 10:36:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 10:36:55 -0600 (CST) Subject: [Swift-commit] r3799 - www Message-ID: <20101214163655.F1C829CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 10:36:55 -0600 (Tue, 14 Dec 2010) New Revision: 3799 Modified: www/update.sh Log: Ensure *.php is g+rw Modified: www/update.sh =================================================================== --- www/update.sh 2010-12-14 16:28:48 UTC (rev 3798) +++ www/update.sh 2010-12-14 16:36:55 UTC (rev 3799) @@ -40,9 +40,14 @@ echo "--------- Updating www... ----------" svn update chmod -R g+w .svn 2>/dev/null -chmod a+r docs/index.php + +find . -name .svn -prune -o -name "*.php" | xargs -n 100 chmod -f g+rw,a+r + +chmod a+r,g+w docs/index.php chmod a+r downloads/index.php chmod a+r inc/nav.php inc/footer.php + + echo "--------- Updating docs... ----------" for SET in guides/trunk guides/release-0.91 From noreply at svn.ci.uchicago.edu Tue Dec 14 10:38:20 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 10:38:20 -0600 (CST) Subject: [Swift-commit] r3800 - www Message-ID: <20101214163820.160C79CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 10:38:19 -0600 (Tue, 14 Dec 2010) New Revision: 3800 Modified: www/update.sh Log: Cut unnecessary lines Modified: www/update.sh =================================================================== --- www/update.sh 2010-12-14 16:36:55 UTC (rev 3799) +++ www/update.sh 2010-12-14 16:38:19 UTC (rev 3800) @@ -43,11 +43,6 @@ find . -name .svn -prune -o -name "*.php" | xargs -n 100 chmod -f g+rw,a+r -chmod a+r,g+w docs/index.php -chmod a+r downloads/index.php -chmod a+r inc/nav.php inc/footer.php - - echo "--------- Updating docs... ----------" for SET in guides/trunk guides/release-0.91 From noreply at svn.ci.uchicago.edu Tue Dec 14 15:57:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 15:57:25 -0600 (CST) Subject: [Swift-commit] r3801 - trunk/tests Message-ID: <20101214215725.2725C9CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 15:57:24 -0600 (Tue, 14 Dec 2010) New Revision: 3801 Added: trunk/tests/groups/ Log: Location for GROUPLIST definition files From noreply at svn.ci.uchicago.edu Tue Dec 14 15:59:08 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 15:59:08 -0600 (CST) Subject: [Swift-commit] r3802 - trunk/tests/groups Message-ID: <20101214215908.1F7B29CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 15:59:07 -0600 (Tue, 14 Dec 2010) New Revision: 3802 Added: trunk/tests/groups/group-all-local.sh trunk/tests/groups/group-intrepid.sh Log: First example GROUPLIST defn files Added: trunk/tests/groups/group-all-local.sh =================================================================== --- trunk/tests/groups/group-all-local.sh (rev 0) +++ trunk/tests/groups/group-all-local.sh 2010-12-14 21:59:07 UTC (rev 3802) @@ -0,0 +1,13 @@ + +# GROUPLIST definition to run all local tests + +GROUPLIST=( $TESTDIR/language-behaviour \ + $TESTDIR/language/working \ + $TESTDIR/local \ + $TESTDIR/language/should-not-work \ + $TESTDIR/cdm \ + $TESTDIR/cdm/ps \ + $TESTDIR/cdm/star + $TESTDIR/cdm/ps/pinned + # $TESTDIR/site/intrepid + ) Added: trunk/tests/groups/group-intrepid.sh =================================================================== --- trunk/tests/groups/group-intrepid.sh (rev 0) +++ trunk/tests/groups/group-intrepid.sh 2010-12-14 21:59:07 UTC (rev 3802) @@ -0,0 +1,6 @@ + +# GROUPLIST definition to run on Intrepid + +GROUPLIST=( $TESTDIR/local \ + $TESTDIR/site/intrepid + ) From noreply at svn.ci.uchicago.edu Tue Dec 14 16:15:08 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 16:15:08 -0600 (CST) Subject: [Swift-commit] r3803 - trunk/tests Message-ID: <20101214221508.BFA0F9CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 16:15:08 -0600 (Tue, 14 Dec 2010) New Revision: 3803 Modified: trunk/tests/nightly.sh Log: Define GROUPLIST in external file Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-12-14 21:59:07 UTC (rev 3802) +++ trunk/tests/nightly.sh 2010-12-14 22:15:08 UTC (rev 3803) @@ -34,6 +34,7 @@ # 5) optionally a swift.properties # 6) optionally a title.txt # 7) preferably a README.txt +# 7) optionally a *.timeout # Edit GROUPLIST at the end of this script before running # template files are lightly processed by sed before use # Missing files will be pulled from swift/etc @@ -47,9 +48,9 @@ # PID TREE: # Background processes are used so that hung Swift jobs can be killed # These are the background processes (PIDs are tracked) -# Note that PID management has not yet been perfected. Check ps -# in error cases. -# +# Note that PID management has not yet been perfected. Check ps +# in error cases. +# # nightly.sh # +-monitor() # +-sleep @@ -65,7 +66,7 @@ # SCHEDULERS # Environment must contain PROJECT, QUEUE, and WORK -# These variables will be incorporated into the sites.xml +# These variables will be incorporated into the sites.xml # via make_sites_sed() -> group_sites_xml() # Note that some schedulers restrict your choice of RUNDIR @@ -73,17 +74,18 @@ echo "nightly.sh " echo "" echo "usage:" - printf "\t -a Do not run ant dist \n" - printf "\t -c Do not remove dist (clean) \n" - printf "\t -g Do not run grid tests \n" - printf "\t -h This message \n" - printf "\t -k Skip first N tests \n" - printf "\t -n Run N tests and quit \n" - printf "\t -p Do not build the package \n" - printf "\t -s Do not do a fresh svn checkout \n" - printf "\t -x Do not continue after a failure \n" - printf "\t -v Verbose (set -x, HTML comments) \n" - printf "\t output Location for output (TOPDIR) \n" + printf "\t -a Do not run ant dist \n" + printf "\t -c Do not remove dist (clean) \n" + printf "\t -g Do not run grid tests \n" + printf "\t -h This message \n" + printf "\t -k Skip first N tests \n" + printf "\t -n Run N tests and quit \n" + printf "\t -p Do not build the package \n" + printf "\t -s Do not do a fresh svn checkout \n" + printf "\t -x Do not continue after a failure \n" + printf "\t -v Verbose (set -x, HTML comments) \n" + printf "\t -o output Location for output (TOPDIR) \n" + printf "\t GROUPLIST definition file \n" } # Defaults: @@ -120,6 +122,9 @@ -n) NUMBER_OF_TESTS=$2 shift 2;; + -o) + TOPDIR=$2 + shift 2;; -p) BUILD_PACKAGE=0 shift;; @@ -133,7 +138,7 @@ VERBOSE=1 shift;; *) - TOPDIR=$1 + GROUPLISTFILE=$1 shift;; esac done @@ -170,6 +175,12 @@ echo "RUNNING_IN: $RUNDIR" echo "HTML_OUTPUT: $HTML" +TESTDIR=$TOPDIR/cog/modules/swift/tests + +checkvars GROUPLISTFILE +echo "GROUPLISTFILE: $GROUPLISTFILE" +source $GROUPLISTFILE + cd $TOPDIR mkdir -p $RUNDIR [ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1 @@ -598,7 +609,7 @@ printf "\nExecuting: $@" >>$LOG rm -f $OUTPUT - "$@" > $OUTPUT 2>&1 + "$@" > $OUTPUT 2>&1 EXITCODE=$? if [ "$EXITCODE" == "127" ]; then @@ -805,7 +816,7 @@ # Generate the sites.sed file make_sites_sed() { checkvars WORK QUEUE PROJECT - { + { echo "s at _WORK_@$WORK@" echo "s at _HOST_@$GLOBUS_HOSTNAME@" echo "s at _PROJECT_@$PROJECT@" @@ -965,21 +976,8 @@ EXITONFAILURE=false fi -TESTDIR=$TOPDIR/cog/modules/swift/tests - SKIP_COUNTER=0 -GROUPLIST=( $TESTDIR/language-behaviour \ - $TESTDIR/language/working \ - $TESTDIR/local \ - $TESTDIR/language/should-not-work \ - $TESTDIR/cdm \ - $TESTDIR/cdm/ps \ - $TESTDIR/cdm/star - $TESTDIR/cdm/ps/pinned - # $TESTDIR/site/intrepid - ) - GROUPCOUNT=1 for G in ${GROUPLIST[@]}; do export GROUP=$G From noreply at svn.ci.uchicago.edu Tue Dec 14 16:19:04 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 16:19:04 -0600 (CST) Subject: [Swift-commit] r3804 - trunk/tests Message-ID: <20101214221904.A090B9CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 16:19:04 -0600 (Tue, 14 Dec 2010) New Revision: 3804 Modified: trunk/tests/nightly.sh Log: Usage notes Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-12-14 22:15:08 UTC (rev 3803) +++ trunk/tests/nightly.sh 2010-12-14 22:19:04 UTC (rev 3804) @@ -35,10 +35,14 @@ # 6) optionally a title.txt # 7) preferably a README.txt # 7) optionally a *.timeout -# Edit GROUPLIST at the end of this script before running # template files are lightly processed by sed before use # Missing files will be pulled from swift/etc +# The GROUPLIST is obtained from an external script in the +# groups/ subdirectory by the name of GROUPLISTFILE. +# The GROUPLISTFILE: +# 1) sets the array + # OUTPUT is the stdout of the current test # stdout.txt retains stdout from the previous test (for *.clean.sh) # output_*.txt is the HTML-linked permanent output from a test From noreply at svn.ci.uchicago.edu Tue Dec 14 16:23:08 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 16:23:08 -0600 (CST) Subject: [Swift-commit] r3805 - trunk/tests Message-ID: <20101214222308.270BF9CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 16:23:08 -0600 (Tue, 14 Dec 2010) New Revision: 3805 Modified: trunk/tests/nightly.sh Log: Move site-specific environment check into GROUPLIST defn file Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-12-14 22:19:04 UTC (rev 3804) +++ trunk/tests/nightly.sh 2010-12-14 22:23:08 UTC (rev 3805) @@ -42,6 +42,7 @@ # groups/ subdirectory by the name of GROUPLISTFILE. # The GROUPLISTFILE: # 1) sets the array +# 2) checks any variables needed by make_sites_sed() # OUTPUT is the stdout of the current test # stdout.txt retains stdout from the previous test (for *.clean.sh) @@ -181,24 +182,6 @@ TESTDIR=$TOPDIR/cog/modules/swift/tests -checkvars GROUPLISTFILE -echo "GROUPLISTFILE: $GROUPLISTFILE" -source $GROUPLISTFILE - -cd $TOPDIR -mkdir -p $RUNDIR -[ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1 - -checkfail() { - ERR=$? - shift - MSG=${*} - if [[ $ERR != 0 ]]; then - echo "FAILED($ERR): $MSG" - exit $ERR - fi -} - # Ensure all given variables are set checkvars() { while (( ${#*} )) @@ -211,6 +194,16 @@ return 0 } +checkfail() { + ERR=$? + shift + MSG=${*} + if [[ $ERR != 0 ]]; then + echo "FAILED($ERR): $MSG" + exit $ERR + fi +} + crash() { MSG=$1 echo $MSG @@ -819,7 +812,6 @@ # Generate the sites.sed file make_sites_sed() { - checkvars WORK QUEUE PROJECT { echo "s at _WORK_@$WORK@" echo "s at _HOST_@$GLOBUS_HOSTNAME@" @@ -925,6 +917,14 @@ done } +checkvars GROUPLISTFILE +echo "GROUPLISTFILE: $GROUPLISTFILE" +source $GROUPLISTFILE || exit 1 + +cd $TOPDIR +mkdir -p $RUNDIR +[ $? != 0 ] && echo "Could not mkdir: $RUNDIR" && exit 1 + date > $LOG make_sites_sed From noreply at svn.ci.uchicago.edu Tue Dec 14 16:24:37 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 16:24:37 -0600 (CST) Subject: [Swift-commit] r3806 - trunk/tests/groups Message-ID: <20101214222437.762739CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 16:24:37 -0600 (Tue, 14 Dec 2010) New Revision: 3806 Modified: trunk/tests/groups/group-intrepid.sh Log: Cobalt/Intrepid needs WORK, QUEUE, and PROJECT Modified: trunk/tests/groups/group-intrepid.sh =================================================================== --- trunk/tests/groups/group-intrepid.sh 2010-12-14 22:23:08 UTC (rev 3805) +++ trunk/tests/groups/group-intrepid.sh 2010-12-14 22:24:37 UTC (rev 3806) @@ -4,3 +4,5 @@ GROUPLIST=( $TESTDIR/local \ $TESTDIR/site/intrepid ) + +checkvars WORK QUEUE PROJECT From noreply at svn.ci.uchicago.edu Tue Dec 14 16:35:53 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 16:35:53 -0600 (CST) Subject: [Swift-commit] r3807 - trunk/tests Message-ID: <20101214223553.86CC99CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 16:35:53 -0600 (Tue, 14 Dec 2010) New Revision: 3807 Added: trunk/tests/meta.sh trunk/tests/run-nightly.sh Log: Sketches of scripts to support multi-site testing Added: trunk/tests/meta.sh =================================================================== --- trunk/tests/meta.sh (rev 0) +++ trunk/tests/meta.sh 2010-12-14 22:35:53 UTC (rev 3807) @@ -0,0 +1,10 @@ +#!/bin/bash + +# Sketch of meta script + +# Runs nightly.sh on various sites + +DIR=$1 # E.g., /home/wozniak/nightly-tests + +ssh intrepid.alcf.anl.gov $DIR/run-nightly.sh groups/group-intrepid.sh + Property changes on: trunk/tests/meta.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/run-nightly.sh =================================================================== --- trunk/tests/run-nightly.sh (rev 0) +++ trunk/tests/run-nightly.sh 2010-12-14 22:35:53 UTC (rev 3807) @@ -0,0 +1,15 @@ +#!/bin/bash + +# Sketch of script that is called on remote test site + +GROUPLISTFILE=$1 # E.g., groups/group-all-local.sh + +# Enter test directory +cd $( dirname $0 ) || exit 1 + +svn up || exit 1 + +./nightly.sh -g -o /scratch/wozniak/nightly $GROUPLISTFILE +[ $? != 0 ] && exit 1 + +exit 0 Property changes on: trunk/tests/run-nightly.sh ___________________________________________________________________ Name: svn:executable + * From noreply at svn.ci.uchicago.edu Tue Dec 14 16:38:55 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 14 Dec 2010 16:38:55 -0600 (CST) Subject: [Swift-commit] r3808 - trunk/tests Message-ID: <20101214223855.80B839CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-14 16:38:55 -0600 (Tue, 14 Dec 2010) New Revision: 3808 Modified: trunk/tests/run-nightly.sh Log: Better use of TOPDIR Modified: trunk/tests/run-nightly.sh =================================================================== --- trunk/tests/run-nightly.sh 2010-12-14 22:35:53 UTC (rev 3807) +++ trunk/tests/run-nightly.sh 2010-12-14 22:38:55 UTC (rev 3808) @@ -9,7 +9,9 @@ svn up || exit 1 -./nightly.sh -g -o /scratch/wozniak/nightly $GROUPLISTFILE +# Work within, e.g., /home/wozniak/nightly/topdir +rm -rf topdir +./nightly.sh -g -o topdir $GROUPLISTFILE [ $? != 0 ] && exit 1 exit 0 From noreply at svn.ci.uchicago.edu Thu Dec 16 18:03:40 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 16 Dec 2010 18:03:40 -0600 (CST) Subject: [Swift-commit] r3809 - trunk/docs Message-ID: <20101217000340.9780B9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-16 18:03:40 -0600 (Thu, 16 Dec 2010) New Revision: 3809 Modified: trunk/docs/userguide.xml Log: Add CDM section Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2010-12-14 22:38:55 UTC (rev 3808) +++ trunk/docs/userguide.xml 2010-12-17 00:03:40 UTC (rev 3809) @@ -1389,7 +1389,7 @@ The above mappers may be used to map files based on a URI which can be specified in the filename. This is useful for mapping files on remote machines. - + Example: @@ -3643,14 +3643,14 @@
    shell invocation -Because the above implementation requires an entry for each executable on a given site, it is often preferable to simply have a single element for each site in the transformation catalog representing a wrapper that sets the environment and then invokes the shell to call a given application. This wrapper is installed on the site and can be used to set the PATH and other environment variables prior to invoking the shell so that each executable need not be entered in the tc.data file. +Because the above implementation requires an entry for each executable on a given site, it is often preferable to simply have a single element for each site in the transformation catalog representing a wrapper that sets the environment and then invokes the shell to call a given application. This wrapper is installed on the site and can be used to set the PATH and other environment variables prior to invoking the shell so that each executable need not be entered in the tc.data file. for example, the entries for TGUC and localhost can now each be collapsed into a single line: localhost shell /usr/local/bin/swiftshell INSTALLED INTEL32::LINUX null TGUC shell /usr/local/bin/swiftshell INSTALLED INTEL32::LINUX null -where swiftshell sets up the users's environment so that all the installed applications are added to the PATH before the application is invoked. +where swiftshell sets up the users's environment so that all the installed applications are added to the PATH before the application is invoked. touch would now be called in the SwiftScript like this: @@ -4124,6 +4124,10 @@
    +
    + Collective Data Management + +
    From noreply at svn.ci.uchicago.edu Thu Dec 16 18:18:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 16 Dec 2010 18:18:25 -0600 (CST) Subject: [Swift-commit] r3810 - trunk/docs Message-ID: <20101217001825.ED1909CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-16 18:18:25 -0600 (Thu, 16 Dec 2010) New Revision: 3810 Modified: trunk/docs/userguide.xml Log: Paste in CDM features doc Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2010-12-17 00:03:40 UTC (rev 3809) +++ trunk/docs/userguide.xml 2010-12-17 00:18:25 UTC (rev 3810) @@ -4127,7 +4127,211 @@
    Collective Data Management + +CDM Features + +Overview: + + 1. The user specifies a CDM policy in a file, customarily fs.data. + 2. fs.data is given to Swift on the command line. + 3. The Swift data module (org.globus.swift.data) is informed of the CDM policy. + 4. At job launch time, the VDL Karajan code queries the CDM policy, + 1. altering the file staging phase, and + 2. sending fs.data to the compute site. + 5. At job run time, the Swift wrapper script + 1. consults a Perl script to obtain policy, and + 2. uses wrapper extensions to modify data movement. + 6. Similarly, stage out can be changed. + + +Command line: + + * swift -sites.file sites.xml -tc.file tc.data -cdm.file fs.data stream.swift + + +CDM policy file format: + +Example: + +# Describe CDM for my job +property GATHER_LIMIT 1 +rule .*input.txt DIRECT /gpfs/homes/wozniak/data +rule .*xfile*.data BROADCAST /dev/shm +rule .* DEFAULT + +The lines contain: + + 1. A directive, either rule or property + 2. A rule has: + 1. A regular expression + 2. A policy token + 3. Additional policy-specific arguments + 3. A property has + 1. A policy property token + 2. The token value + 4. Comments with # . + 5. Blank lines are ignored. + + +Notes: + + 1. The policy file is used as a lookup database by Swift and Perl methods. + 2. For example, a lookup with the database above given the argument input.txt would result in the Direct policy. + 3. If the lookup does not succeed, the result is DEFAULT. + 4. Policies are listed as subclasses of org.globus.swift.data.policy.Policy . + + +Policy Descriptions: + +Default: + + * Just use file staging as provided by Swift/CoG. Identical to behavior if no CDM file is given. + + +Broadcast: + + rule .*xfile*.data BROADCAST /dev/shm + + * The input files matching the pattern will be stored in the given directory, an LFS location, with links in the job directory. + * On the BG/P, this will make use of the f2cn tool. + * On machines not implementing an efficient broadcast, we will just ensure correctness. For example, on a workstation, the location could be in a /tmp RAM FS. + + +Direct: + + rule .*input.txt DIRECT /gpfs/scratch/wozniak/ + + * Allows for direct I/O to the parallel FS without staging. + * The input files matching the pattern must already exist in the given directory, a GFS location. Links will be placed in the job directory. + * The output files matching the pattern will be stored in the given directory, with links in the job directory. + * Example: In the rule above, the Swift-generated file name ./data/input.txt would be accessed by the user job in /gpfs/homes/wozniak/data/input.txt . + + +Local: + + rule .*input.txt LOCAL dd /gpfs/homes/wozniak/data obs=64K + + * Allows for client-directed input copy to the compute node. + * The user may specify cp or dd as the input transfer program. + + * The input files matching the pattern must already exist in the given directory, a GFS location. Copies will be placed in the job directory. + * Argument list: * + + +Gather: + + property GATHER_LIMIT 500000000 # 500 MB + property GATHER_DIR /dev/shm/gather + property GATHER_TARGET /gpfs/wozniak/data/gather_target + rule .*.output.txt GATHER + + * The output files matching the pattern will be present to tasks in the job directory as usual but noted in a _swiftwrap shell array GATHER_OUTPUT. + * The GATHER_OUTPUT files will be cached in the GATHER_DIR, an LFS location. + * The cache will be flushed when a job ends if a du on GATHER_DIR exceeds GATHER_LIMIT. + * As the cache fills or on stage out, the files will be bundled into randomly named tarballs in GATHER_TARGET, a GFS location. + * If the compute node is an SMP, GATHER_DIR is a shared resource. It is protected by the link file GATHER_DIR/.cdm.lock . + * Unpacking the tarballs in GATHER_TARGET will produce the user-specified filenames. + + Summary: + + 1. Files created by application + 2. Acquire lock + 3. Move files to cache + 4. Check cache size + 5. If limit exceeded, move all cache files to outbox + 6. Release lock + 7. If limit was exceeded, stream outbox as tarball to target + + + Notes: + + * Gather required quite a bit of shell functionality to manage the lock, etc. This is placed in cdm_lib.sh . + * vdl_int.k needed an additional task submission (cdm_cleanup.sh) to perform the final flush at workflow completion time . This task also uses cdm_lib.sh . + + +VDL/Karajan processing: + + 1. CDM functions are available in Karajan via the cdm namespace. + 2. These functions are defined in org.globus.swift.data.Query . + 3. If CDM is enabled, VDL skips file staging for files unless the policy is DEFAULT. + + +Swift wrapper CDM routines: + + 1. The cdm.pl script is shipped to the compute node if CDM is enabled. + 2. When linking in inputs, CDM is consulted by _swiftwrap:cdm_lookup(). + 3. The cdm_action() shell function handles CDM methods, typically just producing a link. + + +Test case: + +(See About.txt for more information.) + + 1. Simple test cases are in: + https://svn.mcs.anl.gov/repos/wozniak/collab/cdm/scripts/cdm-direct and + https://svn.mcs.anl.gov/repos/wozniak/collab/cdm/scripts/all-pairs + 2. Do a: + mkdir cdm + cd cdm + svn co https://svn.mcs.anl.gov/repos/wozniak/collab/cdm/scripts + 3. In cdm-direct, run: + source ./setup.sh local local local + 4. Run workflow: + swift -sites.file sites.xml -tc.file tc.data -cdm.file fs.data stream.swift + 5. Note that staging is skipped for input.txt + policy: file://localhost/input.txt : DIRECT + FILE_STAGE_IN_START file=input.txt ... + FILE_STAGE_IN_SKIP file=input.txt policy=DIRECT + FILE_STAGE_IN_END file=input.txt ... + 6. In the wrapper output, the input file is handled by CDM functionality: + Progress 2010-01-21 13:50:32.466572727-0600 LINK_INPUTS + CDM_POLICY: DIRECT /homes/wozniak/cdm/scripts/cdm-direct + CDM: jobs/t/cp_sh-tkul4nmj input.txt DIRECT /homes/wozniak/cdm/scripts/cdm-direct + CDM[DIRECT]: Linking jobs/t/cp_sh-tkul4nmj/input.txt to /homes/wozniak/cdm/scripts/cdm-direct/input.txt + Progress 2010-01-21 13:50:32.486016708-0600 EXECUTE + 7. all-pairs is quite similar but uses more policies. + + +PTMap case: + + 1. Start with vanilla PTMap: + cd cdm + mkdir apps + cd apps + https://svn.mcs.anl.gov/repos/wozniak/collab/cdm/apps/ptmap + 2. Source setup.sh + 3. Use start.sh, which + 1. applies CDM policy from fs.local.data + + +CDM site-aware policy file format: + +Example: + +# Describe CDM for my job +# Use DIRECT and BROADCAST if on cluster1, else use DEFAULT behavior +property GATHER_LIMIT 1 +rule cluster1 .*input.txt DIRECT /gpfs/homes/wozniak/data +rule cluster1 .*xfile*.data BROADCAST /dev/shm +rule ANYWHERE .* DEFAULT + +The lines contain: + + 1. A directive, either rule or property + 2. A rule has: + 1. A regular expression for site matchin + 2. A regular expression for filename matching + 3. A policy token + 4. Additional policy-specific arguments + 3. A property has + 1. A policy property token + 2. The token value + 4. Comments with # . + 5. Blank lines are ignored. + + +
    From noreply at svn.ci.uchicago.edu Thu Dec 16 18:23:05 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 16 Dec 2010 18:23:05 -0600 (CST) Subject: [Swift-commit] r3811 - trunk/docs Message-ID: <20101217002305.6E53A9CC7F@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-16 18:23:05 -0600 (Thu, 16 Dec 2010) New Revision: 3811 Modified: trunk/docs/userguide.xml Log: XML correction Modified: trunk/docs/userguide.xml =================================================================== --- trunk/docs/userguide.xml 2010-12-17 00:18:25 UTC (rev 3810) +++ trunk/docs/userguide.xml 2010-12-17 00:23:05 UTC (rev 3811) @@ -4216,7 +4216,7 @@ * The user may specify cp or dd as the input transfer program. * The input files matching the pattern must already exist in the given directory, a GFS location. Copies will be placed in the job directory. - * Argument list: * + * Argument list: [tool] [GFS directory] [tool arguments]* Gather: From noreply at svn.ci.uchicago.edu Sun Dec 19 06:11:42 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Sun, 19 Dec 2010 06:11:42 -0600 (CST) Subject: [Swift-commit] r3812 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20101219121142.3D36C9CC82@svn.ci.uchicago.edu> Author: skenny Date: 2010-12-19 06:11:38 -0600 (Sun, 19 Dec 2010) New Revision: 3812 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: print swift version to commandline Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-12-17 00:23:05 UTC (rev 3811) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-12-19 12:11:38 UTC (rev 3812) @@ -56,6 +56,7 @@ private static final Logger logger = Logger.getLogger(Loader.class); public static final String ARG_HELP = "help"; + public static final String ARG_VERSION = "version"; public static final String ARG_MONITOR = "monitor"; public static final String ARG_RESUME = "resume"; public static final String ARG_INSTANCE_CONFIG = "config"; @@ -89,6 +90,10 @@ ap.usage(); System.exit(0); } + if (ap.isPresent(ARG_VERSION)){ + ap.version(); + System.exit(0); + } if (ap.isPresent(ARG_MONITOR)) { new Monitor().start(); } @@ -371,6 +376,7 @@ ArgumentParser.OPTIONAL); ap.addFlag(ARG_HELP, "Display usage information"); + ap.addFlag(ARG_VERSION, "Version:"); ap.addAlias(ARG_HELP, "h"); ap.addFlag(ARG_RECOMPILE, From noreply at svn.ci.uchicago.edu Tue Dec 21 15:13:02 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Dec 2010 15:13:02 -0600 (CST) Subject: [Swift-commit] r3813 - text/parco10submission Message-ID: <20101221211302.D6B8D9CC8D@svn.ci.uchicago.edu> Author: hategan Date: 2010-12-21 15:13:02 -0600 (Tue, 21 Dec 2010) New Revision: 3813 Modified: text/parco10submission/paper.tex Log: an alternate take on the swift rationale Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-19 12:11:38 UTC (rev 3812) +++ text/parco10submission/paper.tex 2010-12-21 21:13:02 UTC (rev 3813) @@ -3,9 +3,13 @@ \usepackage{graphicx} -\usepackage{color} +\usepackage{xcolor} +\usepackage{framed} \newcommand{\katznote}[1]{ {\textcolor{cyan} { ***Dan: #1 }}} +\definecolor{shadecolor}{RGB}{240,255,200} +\newenvironment{msection}{\begin{shaded}}{\end{shaded}} + \journal{Parallel Computing} \makeatletter \g at addto@macro\@verbatim\small @@ -87,7 +91,6 @@ \katznote{are swift tasks idempotent? are swift scripts are deterministic (maybe?) deadlock-free? (no)} - Swift is a scripting language designed for composing ordinary application programs (serial or parallel) into distributed, parallelized applications for execution on grids and supercomputers @@ -131,6 +134,93 @@ \subsection*{Rationale} +\begin{msection} +The main goal of Swift is to allow the composition coarse grained +processes. Such a composition will be termed \emph{Swift program}. A +secondary goal is to parallelize and manage the execution of a Swift +program on shared collections of resources (i.e. Grids), while at the +same time maintaining data determinism. + +The exact number of processing units available on such shared resources +varies with time. In order to take advantage of as many processing units +as possible during the execution of a Swift program, it is necessary to +be flexible in the way the execution of individual processes is +parallelized. Having the results be independent of the way the processes +are parallelized implies that the processes must, for the same input, +produce the same output, irrespective of the time, order or location in +which they are "executed". This characteristic is reminiscent of +referential transparency, and one may readily extend the concept to +encompass arbitrary processes without difficulty. + +The referential transparency requirement suggests that an appropriate +course of action in building a language allowing process composition is +to define a correspondence between the familiar concept of a function +and a process. We do so in Swift by representing input data and process +parameters as function parameters and output data as return values, +while processes are represented by functions. + +This duality allows the formal specification of process behavior. When +writing as part of a program: +\begin{verbatim} +rotatedImage = rotate(image, angle); +\end{verbatim} +the semantics are defined in terms of the specification of the function +"rotate" when supplied with specific parameter types, and whether the +implementation can be described as a "library call" or a "program +invocation" changes nothing with respect to what the piece of program +fundamentally does: produce a rotated version of the original. + +Indeed, there is no strict requirement in the specification of the Swift +language dictating that functions be implemented as command-line +applications. They can equally consist of library calls or functions +written in Swift itself, as long as they are side-effect free. A soft +restriction arises from the desire to distribute the execution of +functions across a collection of heterogeneous resources, which, with +the advent of projects such as TeraGrid, suggests an implementation in +which functions are applications readily executable on them through the +careful employment of Grid middleware. + +Having established the constraint that Swift functions must in general +be referentially transparent, and in order to preserve referential +transparency at different levels of abstractions within the language, it +follows that the appropriate form for the Swift language is functional. +We choose to make the Swift language purely functional (i.e. disallow +side-effects in the language) in order to prevent the difficulties that +arise from having to track side-effects to ensure determinism in complex +concurrency scenarios. + +It is already known that functional programming allows consistent +implementations of evaluation strategies different from the widespread +eager evaluation, a fact readily apparent in lazily evaluated languages +such as Haskell and Miranda. In order to achieve automatic +parallelization in Swift, instead of thunking (i.e. suspended +computations) which yields lazy evaluation, we employ futures, which +results in eager parallelism. In this process, we trade the ability to +efficiently deal with infinite structures for the ability to minimize +computation time. It must, however, be noted that a middle ground +exists: lazy futures (futures whose computation is delayed until a value +is first needed). + +A number of issues may be noted at this point. First, there exist a +certain class of processes which may break referential transparency, +while still producing "valid" results. An example would be Monte Carlo +processes. Under the assumption that eager evaluation of compositions of +Monte Carlo processes also produces valid results, an eager Swift +implementation (which is the case with the current implementation) +readily accommodates Monte Carlo processes. However, further discussion +is necessary if optimizations (such as memoization) are employed. +Second, referential transparency and lack of side-effects are not +equivalent concepts. For example, it is possible for a referentially +transparent function to employ side-effects as part of its +implementation. While this is not currently possible within Swift +itself, it is also not readily obvious that such a feature would +contradict the main goals of Swift, and therefore is not ruled out as a +possible future development. + + +\end{msection} + + The emergence of large-scale production computing infrastructure such as clusters, grids and high-performance computing (HPC), and the inherent complexity of programming on these systems, necessitates a From noreply at svn.ci.uchicago.edu Tue Dec 21 15:13:44 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Dec 2010 15:13:44 -0600 (CST) Subject: [Swift-commit] r3814 - text/parco10submission Message-ID: <20101221211344.C2A1A9CC8D@svn.ci.uchicago.edu> Author: hategan Date: 2010-12-21 15:13:44 -0600 (Tue, 21 Dec 2010) New Revision: 3814 Modified: text/parco10submission/paper.tex Log: missing word Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-21 21:13:02 UTC (rev 3813) +++ text/parco10submission/paper.tex 2010-12-21 21:13:44 UTC (rev 3814) @@ -135,7 +135,7 @@ \subsection*{Rationale} \begin{msection} -The main goal of Swift is to allow the composition coarse grained +The main goal of Swift is to allow the composition of coarse grained processes. Such a composition will be termed \emph{Swift program}. A secondary goal is to parallelize and manage the execution of a Swift program on shared collections of resources (i.e. Grids), while at the From noreply at svn.ci.uchicago.edu Tue Dec 21 15:35:32 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Dec 2010 15:35:32 -0600 (CST) Subject: [Swift-commit] r3815 - trunk/tests Message-ID: <20101221213532.A763E9CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-21 15:35:32 -0600 (Tue, 21 Dec 2010) New Revision: 3815 Modified: trunk/tests/meta.sh trunk/tests/nightly.sh Log: Additional notes Modified: trunk/tests/meta.sh =================================================================== --- trunk/tests/meta.sh 2010-12-21 21:13:44 UTC (rev 3814) +++ trunk/tests/meta.sh 2010-12-21 21:35:32 UTC (rev 3815) @@ -8,3 +8,5 @@ ssh intrepid.alcf.anl.gov $DIR/run-nightly.sh groups/group-intrepid.sh +# Retrieve results +# scp ... Modified: trunk/tests/nightly.sh =================================================================== --- trunk/tests/nightly.sh 2010-12-21 21:13:44 UTC (rev 3814) +++ trunk/tests/nightly.sh 2010-12-21 21:35:32 UTC (rev 3815) @@ -6,6 +6,8 @@ # subdirectory called run-DATE, and generate useful HTML output and # tests.log +# Usage: nightly.sh * + # Run nightly.sh -h for quick help # When something goes wrong, find and check tests.log or use -v # Code is checked out into TOPDIR (PWD by default) (PATH is not used) @@ -54,7 +56,7 @@ # Background processes are used so that hung Swift jobs can be killed # These are the background processes (PIDs are tracked) # Note that PID management has not yet been perfected. Check ps -# in error cases. +# in error cases, especially if you kill nightly.sh . # # nightly.sh # +-monitor() From noreply at svn.ci.uchicago.edu Tue Dec 21 17:57:20 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Dec 2010 17:57:20 -0600 (CST) Subject: [Swift-commit] r3816 - in trunk/tests: . providers providers/local providers/local-pbs providers/local-pbs-coasters providers/sge-local Message-ID: <20101221235720.929B69CC8D@svn.ci.uchicago.edu> Author: davidk Date: 2010-12-21 17:57:20 -0600 (Tue, 21 Dec 2010) New Revision: 3816 Added: trunk/tests/providers/ trunk/tests/providers/local-pbs-coasters/ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.swift trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.timeout trunk/tests/providers/local-pbs-coasters/catsn.0001.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0002.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0003.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0004.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0005.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0006.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0007.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0008.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0009.out.expected trunk/tests/providers/local-pbs-coasters/catsn.0010.out.expected trunk/tests/providers/local-pbs-coasters/data.txt trunk/tests/providers/local-pbs-coasters/sites.template.xml trunk/tests/providers/local-pbs-coasters/tc.template.data trunk/tests/providers/local-pbs-coasters/title.txt trunk/tests/providers/local-pbs/ trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh trunk/tests/providers/local-pbs/001-catsn-localpbs.swift trunk/tests/providers/local-pbs/001-catsn-localpbs.timeout trunk/tests/providers/local-pbs/catsn.0001.out.expected trunk/tests/providers/local-pbs/catsn.0002.out.expected trunk/tests/providers/local-pbs/catsn.0003.out.expected trunk/tests/providers/local-pbs/catsn.0004.out.expected trunk/tests/providers/local-pbs/catsn.0005.out.expected trunk/tests/providers/local-pbs/catsn.0006.out.expected trunk/tests/providers/local-pbs/catsn.0007.out.expected trunk/tests/providers/local-pbs/catsn.0008.out.expected trunk/tests/providers/local-pbs/catsn.0009.out.expected trunk/tests/providers/local-pbs/catsn.0010.out.expected trunk/tests/providers/local-pbs/data.txt trunk/tests/providers/local-pbs/sites.template.xml trunk/tests/providers/local-pbs/tc.template.data trunk/tests/providers/local-pbs/title.txt trunk/tests/providers/local/ trunk/tests/providers/local/001-catsn-local.check.sh trunk/tests/providers/local/001-catsn-local.setup.sh trunk/tests/providers/local/001-catsn-local.swift trunk/tests/providers/local/001-catsn-local.timeout trunk/tests/providers/local/catsn.0001.out.expected trunk/tests/providers/local/catsn.0002.out.expected trunk/tests/providers/local/catsn.0003.out.expected trunk/tests/providers/local/catsn.0004.out.expected trunk/tests/providers/local/catsn.0005.out.expected trunk/tests/providers/local/catsn.0006.out.expected trunk/tests/providers/local/catsn.0007.out.expected trunk/tests/providers/local/catsn.0008.out.expected trunk/tests/providers/local/catsn.0009.out.expected trunk/tests/providers/local/catsn.0010.out.expected trunk/tests/providers/local/data.txt trunk/tests/providers/local/sites.template.xml trunk/tests/providers/local/tc.template.data trunk/tests/providers/local/title.txt trunk/tests/providers/sge-local/ trunk/tests/providers/sge-local/001-catsn-local.check.sh trunk/tests/providers/sge-local/001-catsn-local.setup.sh trunk/tests/providers/sge-local/001-catsn-local.swift trunk/tests/providers/sge-local/001-catsn-local.timeout trunk/tests/providers/sge-local/catsn.0001.out.expected trunk/tests/providers/sge-local/catsn.0002.out.expected trunk/tests/providers/sge-local/catsn.0003.out.expected trunk/tests/providers/sge-local/catsn.0004.out.expected trunk/tests/providers/sge-local/catsn.0005.out.expected trunk/tests/providers/sge-local/catsn.0006.out.expected trunk/tests/providers/sge-local/catsn.0007.out.expected trunk/tests/providers/sge-local/catsn.0008.out.expected trunk/tests/providers/sge-local/catsn.0009.out.expected trunk/tests/providers/sge-local/catsn.0010.out.expected trunk/tests/providers/sge-local/data.txt trunk/tests/providers/sge-local/sites.template.xml trunk/tests/providers/sge-local/tc.template.data trunk/tests/providers/sge-local/title.txt Removed: trunk/tests/site/ Log: Updates for provider tests Added: trunk/tests/providers/local/001-catsn-local.check.sh =================================================================== --- trunk/tests/providers/local/001-catsn-local.check.sh (rev 0) +++ trunk/tests/providers/local/001-catsn-local.check.sh 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +for count in `seq --format "%04.f" 1 1 10` +do + [ -f catsn.$count.out ] || exit 1 + CONTENTS1=$( cat catsn.$count.out.expected ) + CONTENTS2=$( cat catsn.$count.out ) + [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1 +done +exit 0 Property changes on: trunk/tests/providers/local/001-catsn-local.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/local/001-catsn-local.setup.sh =================================================================== --- trunk/tests/providers/local/001-catsn-local.setup.sh (rev 0) +++ trunk/tests/providers/local/001-catsn-local.setup.sh 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,4 @@ +#!/bin/bash + +cp -v $GROUP/data.txt . || exit 1 +cp -v $GROUP/*expected . || exit 1 Property changes on: trunk/tests/providers/local/001-catsn-local.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/local/001-catsn-local.swift =================================================================== --- trunk/tests/providers/local/001-catsn-local.swift (rev 0) +++ trunk/tests/providers/local/001-catsn-local.swift 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,15 @@ +type file; + +app (file o) cat (file i) +{ + cat @i stdout=@o; +} + +string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +string char[] = @strsplit(t, ""); + +file out[]; +foreach j in [1:@toint(@arg("n","10"))] { + file data<"data.txt">; + out[j] = cat(data); +} Added: trunk/tests/providers/local/001-catsn-local.timeout =================================================================== --- trunk/tests/providers/local/001-catsn-local.timeout (rev 0) +++ trunk/tests/providers/local/001-catsn-local.timeout 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +30 Added: trunk/tests/providers/local/catsn.0001.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0001.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0001.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0002.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0002.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0002.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0003.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0003.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0003.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0004.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0004.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0004.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0005.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0005.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0005.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0006.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0006.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0006.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0007.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0007.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0007.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0008.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0008.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0008.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0009.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0009.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0009.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/catsn.0010.out.expected =================================================================== --- trunk/tests/providers/local/catsn.0010.out.expected (rev 0) +++ trunk/tests/providers/local/catsn.0010.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/data.txt =================================================================== --- trunk/tests/providers/local/data.txt (rev 0) +++ trunk/tests/providers/local/data.txt 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local/sites.template.xml =================================================================== --- trunk/tests/providers/local/sites.template.xml (rev 0) +++ trunk/tests/providers/local/sites.template.xml 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,9 @@ + + + + + 10000 + .31 + _WORK_ + + Added: trunk/tests/providers/local/tc.template.data =================================================================== --- trunk/tests/providers/local/tc.template.data (rev 0) +++ trunk/tests/providers/local/tc.template.data 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,8 @@ +local echo /bin/echo INSTALLED INTEL32::LINUX +local cat /bin/cat INSTALLED INTEL32::LINUX +local ls /bin/ls INSTALLED INTEL32::LINUX +local grep /bin/grep INSTALLED INTEL32::LINUX +local sort /bin/sort INSTALLED INTEL32::LINUX +local paste /bin/paste INSTALLED INTEL32::LINUX +local wc /usr/bin/wc INSTALLED INTEL32::LINUX + Added: trunk/tests/providers/local/title.txt =================================================================== --- trunk/tests/providers/local/title.txt (rev 0) +++ trunk/tests/providers/local/title.txt 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Local Configuration Test Added: trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh =================================================================== --- trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh (rev 0) +++ trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +for count in `seq --format "%04.f" 1 1 10` +do + [ -f catsn.$count.out ] || exit 1 + CONTENTS1=$( cat catsn.$count.out.expected ) + CONTENTS2=$( cat catsn.$count.out ) + [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1 +done +exit 0 Property changes on: trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh =================================================================== --- trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh (rev 0) +++ trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,4 @@ +#!/bin/bash + +cp -v $GROUP/data.txt . || exit 1 +cp -v $GROUP/*expected . || exit 1 Property changes on: trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/local-pbs/001-catsn-localpbs.swift =================================================================== --- trunk/tests/providers/local-pbs/001-catsn-localpbs.swift (rev 0) +++ trunk/tests/providers/local-pbs/001-catsn-localpbs.swift 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,15 @@ +type file; + +app (file o) cat (file i) +{ + cat @i stdout=@o; +} + +string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +string char[] = @strsplit(t, ""); + +file out[]; +foreach j in [1:@toint(@arg("n","10"))] { + file data<"data.txt">; + out[j] = cat(data); +} Added: trunk/tests/providers/local-pbs/001-catsn-localpbs.timeout =================================================================== --- trunk/tests/providers/local-pbs/001-catsn-localpbs.timeout (rev 0) +++ trunk/tests/providers/local-pbs/001-catsn-localpbs.timeout 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +180 Added: trunk/tests/providers/local-pbs/catsn.0001.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0001.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0001.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0002.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0002.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0002.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0003.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0003.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0003.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0004.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0004.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0004.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0005.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0005.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0005.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0006.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0006.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0006.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0007.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0007.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0007.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0008.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0008.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0008.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0009.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0009.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0009.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/catsn.0010.out.expected =================================================================== --- trunk/tests/providers/local-pbs/catsn.0010.out.expected (rev 0) +++ trunk/tests/providers/local-pbs/catsn.0010.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/data.txt =================================================================== --- trunk/tests/providers/local-pbs/data.txt (rev 0) +++ trunk/tests/providers/local-pbs/data.txt 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs/sites.template.xml =================================================================== --- trunk/tests/providers/local-pbs/sites.template.xml (rev 0) +++ trunk/tests/providers/local-pbs/sites.template.xml 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,8 @@ + + + + + 0 + _WORK_ + + Added: trunk/tests/providers/local-pbs/tc.template.data =================================================================== --- trunk/tests/providers/local-pbs/tc.template.data (rev 0) +++ trunk/tests/providers/local-pbs/tc.template.data 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,8 @@ +local-pbs echo /bin/echo INSTALLED INTEL32::LINUX +local-pbs cat /bin/cat INSTALLED INTEL32::LINUX +local-pbs ls /bin/ls INSTALLED INTEL32::LINUX +local-pbs grep /bin/grep INSTALLED INTEL32::LINUX +local-pbs sort /bin/sort INSTALLED INTEL32::LINUX +local-pbs paste /bin/paste INSTALLED INTEL32::LINUX +local-pbs wc /usr/bin/wc INSTALLED INTEL32::LINUX + Added: trunk/tests/providers/local-pbs/title.txt =================================================================== --- trunk/tests/providers/local-pbs/title.txt (rev 0) +++ trunk/tests/providers/local-pbs/title.txt 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Local PBS Configuration Test Added: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh =================================================================== --- trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh (rev 0) +++ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +for count in `seq --format "%04.f" 1 1 10` +do + [ -f catsn.$count.out ] || exit 1 + CONTENTS1=$( cat catsn.$count.out.expected ) + CONTENTS2=$( cat catsn.$count.out ) + [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1 +done +exit 0 Property changes on: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh =================================================================== --- trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh (rev 0) +++ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,4 @@ +#!/bin/bash + +cp -v $GROUP/data.txt . || exit 1 +cp -v $GROUP/*expected . || exit 1 Property changes on: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.swift =================================================================== --- trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.swift (rev 0) +++ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.swift 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,15 @@ +type file; + +app (file o) cat (file i) +{ + cat @i stdout=@o; +} + +string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +string char[] = @strsplit(t, ""); + +file out[]; +foreach j in [1:@toint(@arg("n","10"))] { + file data<"data.txt">; + out[j] = cat(data); +} Added: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.timeout =================================================================== --- trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.timeout (rev 0) +++ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.timeout 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +180 Added: trunk/tests/providers/local-pbs-coasters/catsn.0001.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0001.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0001.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0002.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0002.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0002.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0003.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0003.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0003.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0004.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0004.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0004.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0005.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0005.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0005.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0006.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0006.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0006.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0007.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0007.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0007.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0008.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0008.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0008.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0009.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0009.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0009.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/catsn.0010.out.expected =================================================================== --- trunk/tests/providers/local-pbs-coasters/catsn.0010.out.expected (rev 0) +++ trunk/tests/providers/local-pbs-coasters/catsn.0010.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/data.txt =================================================================== --- trunk/tests/providers/local-pbs-coasters/data.txt (rev 0) +++ trunk/tests/providers/local-pbs-coasters/data.txt 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/local-pbs-coasters/sites.template.xml =================================================================== --- trunk/tests/providers/local-pbs-coasters/sites.template.xml (rev 0) +++ trunk/tests/providers/local-pbs-coasters/sites.template.xml 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,15 @@ + + + + + 3000 + 1 + 1 + 1 + 1 + fast + 5.99 + 10000 + _WORK_ + + Added: trunk/tests/providers/local-pbs-coasters/tc.template.data =================================================================== --- trunk/tests/providers/local-pbs-coasters/tc.template.data (rev 0) +++ trunk/tests/providers/local-pbs-coasters/tc.template.data 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,8 @@ +local-pbs-coasters echo /bin/echo INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00" +local-pbs-coasters cat /bin/cat INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00" +local-pbs-coasters ls /bin/ls INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00" +local-pbs-coasters grep /bin/grep INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00" +local-pbs-coasters sort /bin/sort INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00" +local-pbs-coasters paste /bin/paste INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00" +local-pbs-coasters wc /usr/bin/wc INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00" + Added: trunk/tests/providers/local-pbs-coasters/title.txt =================================================================== --- trunk/tests/providers/local-pbs-coasters/title.txt (rev 0) +++ trunk/tests/providers/local-pbs-coasters/title.txt 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Local PBS with Coasters Configuration Test Added: trunk/tests/providers/sge-local/001-catsn-local.check.sh =================================================================== --- trunk/tests/providers/sge-local/001-catsn-local.check.sh (rev 0) +++ trunk/tests/providers/sge-local/001-catsn-local.check.sh 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +for count in `seq --format "%04.f" 1 1 10` +do + [ -f catsn.$count.out ] || exit 1 + CONTENTS1=$( cat catsn.$count.out.expected ) + CONTENTS2=$( cat catsn.$count.out ) + [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1 +done +exit 0 Property changes on: trunk/tests/providers/sge-local/001-catsn-local.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/sge-local/001-catsn-local.setup.sh =================================================================== --- trunk/tests/providers/sge-local/001-catsn-local.setup.sh (rev 0) +++ trunk/tests/providers/sge-local/001-catsn-local.setup.sh 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,4 @@ +#!/bin/bash + +cp -v $GROUP/data.txt . || exit 1 +cp -v $GROUP/*expected . || exit 1 Property changes on: trunk/tests/providers/sge-local/001-catsn-local.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/sge-local/001-catsn-local.swift =================================================================== --- trunk/tests/providers/sge-local/001-catsn-local.swift (rev 0) +++ trunk/tests/providers/sge-local/001-catsn-local.swift 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,15 @@ +type file; + +app (file o) cat (file i) +{ + cat @i stdout=@o; +} + +string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +string char[] = @strsplit(t, ""); + +file out[]; +foreach j in [1:@toint(@arg("n","10"))] { + file data<"data.txt">; + out[j] = cat(data); +} Added: trunk/tests/providers/sge-local/001-catsn-local.timeout =================================================================== --- trunk/tests/providers/sge-local/001-catsn-local.timeout (rev 0) +++ trunk/tests/providers/sge-local/001-catsn-local.timeout 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +30 Added: trunk/tests/providers/sge-local/catsn.0001.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0001.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0001.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0002.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0002.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0002.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0003.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0003.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0003.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0004.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0004.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0004.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0005.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0005.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0005.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0006.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0006.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0006.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0007.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0007.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0007.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0008.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0008.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0008.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0009.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0009.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0009.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/catsn.0010.out.expected =================================================================== --- trunk/tests/providers/sge-local/catsn.0010.out.expected (rev 0) +++ trunk/tests/providers/sge-local/catsn.0010.out.expected 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/data.txt =================================================================== --- trunk/tests/providers/sge-local/data.txt (rev 0) +++ trunk/tests/providers/sge-local/data.txt 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/sge-local/sites.template.xml =================================================================== --- trunk/tests/providers/sge-local/sites.template.xml (rev 0) +++ trunk/tests/providers/sge-local/sites.template.xml 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,10 @@ + + + + threaded + .49 + 10000 + + _WORK_ + + Added: trunk/tests/providers/sge-local/tc.template.data =================================================================== --- trunk/tests/providers/sge-local/tc.template.data (rev 0) +++ trunk/tests/providers/sge-local/tc.template.data 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1,8 @@ +sge-local echo /bin/echo INSTALLED INTEL32::LINUX +sge-local cat /bin/cat INSTALLED INTEL32::LINUX +sge-local ls /bin/ls INSTALLED INTEL32::LINUX +sge-local grep /bin/grep INSTALLED INTEL32::LINUX +sge-local sort /bin/sort INSTALLED INTEL32::LINUX +sge-local paste /bin/paste INSTALLED INTEL32::LINUX +sge-local wc /usr/bin/wc INSTALLED INTEL32::LINUX + Added: trunk/tests/providers/sge-local/title.txt =================================================================== --- trunk/tests/providers/sge-local/title.txt (rev 0) +++ trunk/tests/providers/sge-local/title.txt 2010-12-21 23:57:20 UTC (rev 3816) @@ -0,0 +1 @@ +SGE Local Configuration Test From noreply at svn.ci.uchicago.edu Tue Dec 21 20:08:21 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Tue, 21 Dec 2010 20:08:21 -0600 (CST) Subject: [Swift-commit] r3817 - trunk/tests/groups Message-ID: <20101222020821.BC08E9CC8D@svn.ci.uchicago.edu> Author: davidk Date: 2010-12-21 20:08:21 -0600 (Tue, 21 Dec 2010) New Revision: 3817 Added: trunk/tests/groups/group-pbs.sh trunk/tests/groups/group-sge.sh Log: Group definitions for pbs and sge tests Added: trunk/tests/groups/group-pbs.sh =================================================================== --- trunk/tests/groups/group-pbs.sh (rev 0) +++ trunk/tests/groups/group-pbs.sh 2010-12-22 02:08:21 UTC (rev 3817) @@ -0,0 +1,7 @@ + +# GROUPLIST definition to run pbs tests + +GROUPLIST=( + $TESTDIR/providers/local-pbs \ + $TESTDIR/providers/local-pbs-coasters \ +) Added: trunk/tests/groups/group-sge.sh =================================================================== --- trunk/tests/groups/group-sge.sh (rev 0) +++ trunk/tests/groups/group-sge.sh 2010-12-22 02:08:21 UTC (rev 3817) @@ -0,0 +1,6 @@ + +# GROUPLIST definition to run sge tests + +GROUPLIST=( + $TESTDIR/providers/sge-local \ +) From noreply at svn.ci.uchicago.edu Wed Dec 22 09:02:25 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 22 Dec 2010 09:02:25 -0600 (CST) Subject: [Swift-commit] r3818 - text/parco10submission Message-ID: <20101222150225.032E89CC7F@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-22 09:02:23 -0600 (Wed, 22 Dec 2010) New Revision: 3818 Modified: text/parco10submission/paper.tex Log: minor changes to new rationale section Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-22 02:08:21 UTC (rev 3817) +++ text/parco10submission/paper.tex 2010-12-22 15:02:23 UTC (rev 3818) @@ -123,22 +123,18 @@ component programs on grids and other parallel platforms, providing automated site selection, data management, and reliability. -This paper goes into greater depth than prior -publications~\cite{Swift_2007} in describing the Swift language, how -its implementation handles large-scale and distributed execution -environments, and its contribution to distributed and parallel computing. -\katznote{Alternative: Swift has been described previously~\cite{Swift_2007} ; +Swift has been described previously~\cite{Swift_2007}; this paper goes into greater depth in describing the Swift language, how its implementation handles large-scale and distributed execution -environments, and its contribution to distributed and parallel computing.} +environments, and its contribution to distributed and parallel computing. \subsection*{Rationale} \begin{msection} The main goal of Swift is to allow the composition of coarse grained -processes. Such a composition will be termed \emph{Swift program}. A +processes. Such a composition is termed a \emph{Swift program}. A secondary goal is to parallelize and manage the execution of a Swift -program on shared collections of resources (i.e. Grids), while at the +program on shared collections of resources (e.g., grids), while at the same time maintaining data determinism. The exact number of processing units available on such shared resources @@ -148,7 +144,7 @@ parallelized. Having the results be independent of the way the processes are parallelized implies that the processes must, for the same input, produce the same output, irrespective of the time, order or location in -which they are "executed". This characteristic is reminiscent of +which they are ``executed''. This characteristic is reminiscent of referential transparency, and one may readily extend the concept to encompass arbitrary processes without difficulty. @@ -165,9 +161,9 @@ rotatedImage = rotate(image, angle); \end{verbatim} the semantics are defined in terms of the specification of the function -"rotate" when supplied with specific parameter types, and whether the -implementation can be described as a "library call" or a "program -invocation" changes nothing with respect to what the piece of program +``rotate'' when supplied with specific parameter types, and whether the +implementation can be described as a ``library call'' or a ``program +invocation'' changes nothing with respect to what the piece of program fundamentally does: produce a rotated version of the original. Indeed, there is no strict requirement in the specification of the Swift @@ -178,32 +174,32 @@ functions across a collection of heterogeneous resources, which, with the advent of projects such as TeraGrid, suggests an implementation in which functions are applications readily executable on them through the -careful employment of Grid middleware. +careful employment of grid middleware. Having established the constraint that Swift functions must in general be referentially transparent, and in order to preserve referential transparency at different levels of abstractions within the language, it follows that the appropriate form for the Swift language is functional. -We choose to make the Swift language purely functional (i.e. disallow -side-effects in the language) in order to prevent the difficulties that -arise from having to track side-effects to ensure determinism in complex +We choose to make the Swift language purely functional (i.e., we disallow +side effects in the language) in order to prevent the difficulties that +arise from having to track side effects to ensure determinism in complex concurrency scenarios. It is already known that functional programming allows consistent implementations of evaluation strategies different from the widespread eager evaluation, a fact readily apparent in lazily evaluated languages such as Haskell and Miranda. In order to achieve automatic -parallelization in Swift, instead of thunking (i.e. suspended -computations) which yields lazy evaluation, we employ futures, which -results in eager parallelism. In this process, we trade the ability to +parallelization in Swift, instead of using thunks (i.e., suspended +computations), which yield lazy evaluation, we employ futures, which +result in eager parallelism. In this process, we trade the ability to efficiently deal with infinite structures for the ability to minimize computation time. It must, however, be noted that a middle ground exists: lazy futures (futures whose computation is delayed until a value is first needed). A number of issues may be noted at this point. First, there exist a -certain class of processes which may break referential transparency, -while still producing "valid" results. An example would be Monte Carlo +certain class of processes that may break referential transparency, +while still producing ``valid'' results. An example would be Monte Carlo processes. Under the assumption that eager evaluation of compositions of Monte Carlo processes also produces valid results, an eager Swift implementation (which is the case with the current implementation) From noreply at svn.ci.uchicago.edu Wed Dec 22 10:03:10 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 22 Dec 2010 10:03:10 -0600 (CST) Subject: [Swift-commit] r3819 - text/parco10submission Message-ID: <20101222160310.1B5079CC82@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-22 10:03:09 -0600 (Wed, 22 Dec 2010) New Revision: 3819 Modified: text/parco10submission/paper.tex Log: 2 more small changes Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-22 15:02:23 UTC (rev 3818) +++ text/parco10submission/paper.tex 2010-12-22 16:03:09 UTC (rev 3819) @@ -106,7 +106,7 @@ Swift's contribution and primary value is that it provides a simple, minimal set of language constructs to specify how applications are glued together at large scale in a simple compact form, while keeping -the language simple and elegant, and minimizing any overlap with the +the language simple \katznote{second ``simple'' in this sentence...} and elegant, and minimizing any overlap with the tasks that existing scripting languages do well. Swift regularizes and abstracts notions of external data and processes for distributed parallel execution of application programs. Applications expressed in @@ -964,8 +964,8 @@ Grid (OSG)~\cite{OSG_2007}, where the set of sites that may be used is large and changing. It is impractical to maintain a site catalog by hand in this situation. In collaboration with the OSG Engagement -group, Swift has been interfaced to ReSS~\cite{ReSS_2007} \katznote{say what ReSS is} -so that the site +group, Swift has been interfaced to ReSS~\cite{ReSS_2007}, +an OSG resource selection service, so that the site catalog is generated from that information system. This provides a very straightforward way to generate a large catalog of sites. From noreply at svn.ci.uchicago.edu Wed Dec 22 16:44:04 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Wed, 22 Dec 2010 16:44:04 -0600 (CST) Subject: [Swift-commit] r3820 - text/parco10submission Message-ID: <20101222224404.E1A6E9CC82@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-22 16:44:04 -0600 (Wed, 22 Dec 2010) New Revision: 3820 Modified: text/parco10submission/paper.tex Log: adding Mihael's suggestion re 'function' vs 'procedure' as a note Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-22 16:03:09 UTC (rev 3819) +++ text/parco10submission/paper.tex 2010-12-22 22:44:04 UTC (rev 3820) @@ -89,6 +89,10 @@ \katznote{in general, I would like to use application, job, task, program, etc. in some way that is consistent throughout the paper, perhaps defined in the introduction} +\katznote{Mihael's suggestion: ``Should we abandon the +"procedure" term that we use for functions? It alludes to imperative +programming which Swift is pretty far from.'' Dan agrees with this.} + \katznote{are swift tasks idempotent? are swift scripts are deterministic (maybe?) deadlock-free? (no)} Swift is a scripting language designed for composing ordinary From noreply at svn.ci.uchicago.edu Thu Dec 23 11:37:14 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Dec 2010 11:37:14 -0600 (CST) Subject: [Swift-commit] r3822 - trunk/libexec Message-ID: <20101223173714.6A68C9CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-23 11:37:14 -0600 (Thu, 23 Dec 2010) New Revision: 3822 Modified: trunk/libexec/_swiftwrap Log: DEBUG=0 by default, debug MPI_RANK Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2010-12-23 17:35:52 UTC (rev 3821) +++ trunk/libexec/_swiftwrap 2010-12-23 17:37:14 UTC (rev 3822) @@ -3,7 +3,7 @@ # note that this script modifies $IFS # Toggle debugging output from debug() -DEBUG=1 +DEBUG=0 infosection() { echo >& "$INFO" @@ -52,7 +52,7 @@ } debug() { - [ $DEBUG == 1 ] && echo "$@" >& "$INFO" + [[ $DEBUG == 1 ]] && echo "$@" >& "$INFO" } fail() { @@ -352,9 +352,9 @@ log "OUTF=$OUTF" log "KICKSTART=$KICKSTART" log "CDM_FILE=$CDM_FILE" +[ -z $MPI_RANK ] && log "MPI_RANK=$MPI_RANK" log "ARGS=$@" log "ARGC=$#" - IFS="|" logstate "CREATE_JOBDIR" @@ -426,6 +426,8 @@ fi logstate "EXECUTE" +debug "Command line: $EXEC ${CMDARGS[@]}" + if [ ! -f "$EXEC" ]; then fail 254 "The executable $EXEC does not exist" fi From noreply at svn.ci.uchicago.edu Thu Dec 23 12:31:47 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Dec 2010 12:31:47 -0600 (CST) Subject: [Swift-commit] r3823 - trunk/libexec Message-ID: <20101223183147.6A7879CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-23 12:31:47 -0600 (Thu, 23 Dec 2010) New Revision: 3823 Modified: trunk/libexec/cdm_lib.sh Log: Fix CDM LOCAL cp/dd options Modified: trunk/libexec/cdm_lib.sh =================================================================== --- trunk/libexec/cdm_lib.sh 2010-12-23 17:37:14 UTC (rev 3822) +++ trunk/libexec/cdm_lib.sh 2010-12-23 18:31:47 UTC (rev 3823) @@ -51,9 +51,7 @@ fi ;; LOCAL) - #TOOL=${ARGS[0]} - #REMOTE_DIR=${ARGS[1]} - #FLAGS=${ARGS[3]} + # TODO: Can/should we use this as a cache? TOOL=$1 REMOTE_DIR=$2 FLAGS=$3 @@ -62,10 +60,17 @@ if [ $MODE == "INPUT" ]; then [ -f "$REMOTE_DIR/$FILE" ] checkError 254 "CDM[LOCAL]: $REMOTE_DIR/$FILE does not exist!" - $TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE - checkError 254 "CDM[LOCAL]: Tool failed!" + if [ $TOOL == "cp" ]; then + $TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE + checkError 254 "CDM[LOCAL]: cp failed!" + elif [ $TOOL == "dd" ]; then + $TOOL $FLAGS if=$REMOTE_DIR/$FILE of=$JOBDIR/$FILE + checkError 254 "CDM[LOCAL]: dd failed!" + else + fail 254 "CDM[LOCAL]: Unknown TOOL: $TOOL" + fi elif [ $MODE == "OUTPUT" ]; then - log "CDM[LOCAL]..." + log "CDM[LOCAL]..." # This should probably be an error else fail 254 "Unknown MODE: $MODE" fi From noreply at svn.ci.uchicago.edu Thu Dec 23 12:34:57 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Dec 2010 12:34:57 -0600 (CST) Subject: [Swift-commit] r3824 - trunk/libexec Message-ID: <20101223183457.2F1769CC8D@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-23 12:34:56 -0600 (Thu, 23 Dec 2010) New Revision: 3824 Modified: trunk/libexec/vdl-int.k Log: Add emacs buffer variables Modified: trunk/libexec/vdl-int.k =================================================================== --- trunk/libexec/vdl-int.k 2010-12-23 18:31:47 UTC (rev 3823) +++ trunk/libexec/vdl-int.k 2010-12-23 18:34:56 UTC (rev 3824) @@ -402,7 +402,7 @@ replicationGroup, replicationChannel] stagein := list(unique(each(stagein))) stageout := list(unique(each(stageout))) - + allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein) ddir := initDDir() @@ -617,3 +617,9 @@ ) ) ) + +// Local variables: +// mode: scheme +// tab-width: 4 +// indent-tabs-mode: t +// End: From noreply at svn.ci.uchicago.edu Thu Dec 23 12:36:52 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Dec 2010 12:36:52 -0600 (CST) Subject: [Swift-commit] r3825 - trunk/tests/mpi Message-ID: <20101223183652.6CBCA9CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-23 12:36:52 -0600 (Thu, 23 Dec 2010) New Revision: 3825 Added: trunk/tests/mpi/title.txt Modified: trunk/tests/mpi/100-mci.setup.sh trunk/tests/mpi/100-mpi-cp.c trunk/tests/mpi/tc.template.data Log: MPI test works again Modified: trunk/tests/mpi/100-mci.setup.sh =================================================================== --- trunk/tests/mpi/100-mci.setup.sh 2010-12-23 18:34:56 UTC (rev 3824) +++ trunk/tests/mpi/100-mci.setup.sh 2010-12-23 18:36:52 UTC (rev 3825) @@ -4,7 +4,7 @@ which mpicc || exit 1 -mpicc -std=gnu99 $GROUP/100-mpi-cp.c -o mpi-cp || exit 1 +mpicc -std=gnu99 $GROUP/100-mpi-cp.c -o $GROUP/mpi-cp || exit 1 cp -v $GROUP/100-input.txt . || exit 1 Modified: trunk/tests/mpi/100-mpi-cp.c =================================================================== --- trunk/tests/mpi/100-mpi-cp.c 2010-12-23 18:34:56 UTC (rev 3824) +++ trunk/tests/mpi/100-mpi-cp.c 2010-12-23 18:36:52 UTC (rev 3825) @@ -1,5 +1,11 @@ -/* + +/** * Simple Hydra test + * + * usage: mpi-cp + * + * Rank 0 reads the input file and sends it to rank 1. + * Rank 1 recvs the data and writes the output. */ #include Modified: trunk/tests/mpi/tc.template.data =================================================================== --- trunk/tests/mpi/tc.template.data 2010-12-23 18:34:56 UTC (rev 3824) +++ trunk/tests/mpi/tc.template.data 2010-12-23 18:36:52 UTC (rev 3825) @@ -19,6 +19,4 @@ # hydra-tests-2 -coasterslocal mpi_cp _PWD_/mpi-cp INSTALLED INTEL32::LINUX globus::hostCount=2 - -# coasterslocal mpi_cp /bin/false INSTALLED INTEL32::LINUX globus::hostCount=2 +coasterslocal mpi_cp _DIR_/mpi-cp INSTALLED INTEL32::LINUX globus::hostCount=2 Added: trunk/tests/mpi/title.txt =================================================================== --- trunk/tests/mpi/title.txt (rev 0) +++ trunk/tests/mpi/title.txt 2010-12-23 18:36:52 UTC (rev 3825) @@ -0,0 +1,2 @@ +MPI Tests + From noreply at svn.ci.uchicago.edu Thu Dec 23 12:45:16 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Dec 2010 12:45:16 -0600 (CST) Subject: [Swift-commit] r3826 - usertools/mpich Message-ID: <20101223184516.490529CC82@svn.ci.uchicago.edu> Author: wozniak Date: 2010-12-23 12:45:16 -0600 (Thu, 23 Dec 2010) New Revision: 3826 Added: usertools/mpich/mpich2-r7562.diff Modified: usertools/mpich/README.txt Log: New working MPICH patch for MPICH/Coasters Modified: usertools/mpich/README.txt =================================================================== --- usertools/mpich/README.txt 2010-12-23 18:36:52 UTC (rev 3825) +++ usertools/mpich/README.txt 2010-12-23 18:45:16 UTC (rev 3826) @@ -1,5 +1,10 @@ + +See: http://www.ci.uchicago.edu/wiki/bin/view/SWFT/CoastersMpi + Contains the following patches: -mpich2-1.3.diff +mpich2-r7562.diff: works with MPICH2, revision 7562 -See: http://www.ci.uchicago.edu/wiki/bin/view/SWFT/CoastersMpi +mpich2-1.3.diff: obsolete due to MPICH changes + + Added: usertools/mpich/mpich2-r7562.diff =================================================================== --- usertools/mpich/mpich2-r7562.diff (rev 0) +++ usertools/mpich/mpich2-r7562.diff 2010-12-23 18:45:16 UTC (rev 3826) @@ -0,0 +1,144 @@ +Index: src/pm/hydra/tools/bootstrap/external/external_init.c +=================================================================== +--- src/pm/hydra/tools/bootstrap/external/external_init.c (revision 7562) ++++ src/pm/hydra/tools/bootstrap/external/external_init.c (working copy) +@@ -52,6 +52,9 @@ + if (!strcmp(HYDT_bsci_info.launcher, "ll")) + HYDT_bsci_fns.launch_procs = HYDT_bscd_ll_launch_procs; + ++ if (!strcmp(HYDT_bsci_info.launcher, "none")) ++ HYDT_bsci_fns.launch_procs = HYDT_bscd_none_launch_procs; ++ + /* for everything else, set default values */ + if (HYDT_bsci_fns.launch_procs == NULL) + HYDT_bsci_fns.launch_procs = HYDT_bscd_external_launch_procs; +@@ -102,6 +105,11 @@ + return external_launcher_init(); + } + ++HYD_status HYDT_bsci_launcher_none_init(void) ++{ ++ return external_launcher_init(); ++} ++ + HYD_status HYDT_bsci_rmk_lsf_init(void) + { + return external_rmk_init(); +@@ -131,3 +139,4 @@ + { + return external_rmk_init(); + } ++ +Index: src/pm/hydra/tools/bootstrap/external/external_launch.c +=================================================================== +--- src/pm/hydra/tools/bootstrap/external/external_launch.c (revision 7562) ++++ src/pm/hydra/tools/bootstrap/external/external_launch.c (working copy) +@@ -4,6 +4,8 @@ + * See COPYRIGHT in top-level directory. + */ + ++#include "assert.h" ++ + #include "hydra_utils.h" + #include "bsci.h" + #include "bscu.h" +@@ -313,3 +315,28 @@ + fn_fail: + goto fn_exit; + } ++ ++HYD_status HYDT_bscd_none_launch_procs(char **args, struct HYD_node *node_list, ++ int *control_fd, int enable_stdin) ++{ ++ int i, proxy_id_index; ++ struct HYD_node *node; ++ char *proxy_id_string; ++ ++ proxy_id_string = HYDU_malloc(16); ++ ++ for (i = 0; args[i] != NULL; ) { ++ i++; ++ } ++ proxy_id_index = i; ++ args[proxy_id_index+1] = NULL; ++ for (i = 0, node = node_list; node; node = node->next, i++) { ++ assert(!strcmp(HYDT_bsci_info.launcher, "none")); ++ printf("HYDRA_NONE_LINE: "); ++ sprintf(proxy_id_string, "%d", i); ++ args[proxy_id_index] = proxy_id_string; ++ HYDU_print_strlist(args); ++ } ++ printf("HYDRA_NONE_END:\n"); ++ fflush(stdout); ++} +Index: src/pm/hydra/tools/bootstrap/external/external.h +=================================================================== +--- src/pm/hydra/tools/bootstrap/external/external.h (revision 7562) ++++ src/pm/hydra/tools/bootstrap/external/external.h (working copy) +@@ -20,4 +20,9 @@ + HYD_status HYDT_bscd_external_query_env_inherit(const char *env_name, int *ret); + HYD_status HYDT_bscd_external_query_native_int(int *ret); + ++HYD_status HYDT_bscd_none_launch_procs(char **args, ++ struct HYD_node *node_list, ++ int *control_fd, ++ int enable_stdin); ++ + #endif /* EXTERNAL_H_INCLUDED */ +Index: src/pm/hydra/utils/args/args.c +=================================================================== +--- src/pm/hydra/utils/args/args.c (revision 7562) ++++ src/pm/hydra/utils/args/args.c (working copy) +@@ -112,7 +112,8 @@ + } + + status = m->handler_fn(arg, argv_p); +- HYDU_ERR_POP(status, "match handler returned error\n"); ++ HYDU_ERR_POP(status, "match handler returned error (for %s)\n", ++ arg); + break; + } + m++; +Index: src/pm/hydra/configure.in +=================================================================== +--- src/pm/hydra/configure.in (revision 7562) ++++ src/pm/hydra/configure.in (working copy) +@@ -211,6 +211,7 @@ + hydra_bss_none=true + available_bss=`echo $available_bss none` + available_rmks=`echo $available_rmks none` ++ available_launchers=`echo $available_launchers none` + ;; + *) + ;; +@@ -357,7 +358,7 @@ + have_plpa=yes + AC_DEFINE(HAVE_PLPA,1,[Define if plpa is available]) + fi +- ;; ++ ;; + hwloc) + if test "$with_hwloc_prefix" = "embedded" ; then + PAC_CONFIG_SUBDIR([tools/bind/hwloc/hwloc],[have_hwloc=yes],[have_hwloc=no]) +@@ -428,7 +429,7 @@ + AC_DEFINE(HAVE_BLCR,1,[Define if blcr is available]) + available_ckpointlibs=`echo $available_ckpointlibs blcr` + fi +- ;; ++ ;; + *) + ;; + esac +Index: src/pm/hydra/pm/pmiserv/pmip_cb.c +=================================================================== +--- src/pm/hydra/pm/pmiserv/pmip_cb.c (revision 7562) ++++ src/pm/hydra/pm/pmiserv/pmip_cb.c (working copy) +@@ -715,7 +715,7 @@ + cpuset); + HYDU_ERR_POP(status, "create process returned error\n"); + +- if (HYD_pmcd_pmip.system_global.enable_stdin) { ++ if (HYD_pmcd_pmip.system_global.enable_stdin && 0) { + stdin_fd = STDIN_FILENO; + status = HYDT_dmx_register_fd(1, &stdin_fd, HYD_POLLIN, + (void *) (size_t) STDIN_FILENO, stdio_cb); From noreply at svn.ci.uchicago.edu Thu Dec 23 13:07:40 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Dec 2010 13:07:40 -0600 (CST) Subject: [Swift-commit] r3827 - www/downloads Message-ID: <20101223190740.46ACF9CC82@svn.ci.uchicago.edu> Author: skenny Date: 2010-12-23 13:07:40 -0600 (Thu, 23 Dec 2010) New Revision: 3827 Modified: www/downloads/index.php Log: binary dist available for .91 release Modified: www/downloads/index.php =================================================================== --- www/downloads/index.php 2010-12-23 18:45:16 UTC (rev 3826) +++ www/downloads/index.php 2010-12-23 19:07:40 UTC (rev 3827) @@ -27,89 +27,46 @@

    DOWNLOADS

    Latest Release

    -

    Swift 0.9 - 2009/04/27

    -

    Swift v0.9 is a development release intended to release functionality -and fixes that have gone in to trunk since v0.8. -[Swift 0.91 - 2010/12/23 +

    Precompiled binary distribution +[swift-0.9.tar.gz] -[swift-0.9-stripped.tar.gz - for installation on systems with existing grid stacks] -[release-notes-0.9.txt] -

    - +

    +

    Source Code

    -

    Nightly Builds and Tests

    -

    - We run nightly integration builds and tests for Swift. The latest build & tests page links to the older - ones. In addition, Swift is built and tested in Metronome, the NMI - build and test system. -

    - -

    Development Distribution

    -

    Source repository access

    - -

    The code is browsable online, or accessible through Subversion.

    -

    Building Swift

    -

    - Note: The development code has the highest chance of containing buggy and - untested code. If you need stability please use one of the releases or the stable - branch below. -

    - -

    You need Apache Ant - for the actual build, and a Java JDK ≥ 1.4. + for the actual build, and a Java JDK ≥ 1.5.
    So here are the steps:

    + +

    Latest Release

    +

    Checkout CoG:

    svn co \ -https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog
    +https://cogkit.svn.sourceforge.net/svnroot/cogkit/branches/4.1.7/src/cog

    Checkout Swift:

    cd cog/modules
    -
    svn co https://svn.ci.uchicago.edu/svn/vdl2/trunk swift
    +
    svn co https://svn.ci.uchicago.edu/svn/vdl2/branches/release-0.91 swift

    @@ -129,19 +86,24 @@ The dist directory will contain the complete build.

    -

    Stable branch

    +

    Development Version

    + Note: The development code has the highest chance of containing buggy and + untested code. If you need stability please use the latest release. +

    + +

    Checkout CoG:

    svn co \ -https://cogkit.svn.sourceforge.net/svnroot/cogkit/branches/4.1.7/src/cog
    +https://cogkit.svn.sourceforge.net/svnroot/cogkit/trunk/current/src/cog

    Checkout Swift:

    cd cog/modules
    -
    svn co https://svn.ci.uchicago.edu/svn/vdl2/branches/1.0 swift
    +
    svn co https://svn.ci.uchicago.edu/svn/vdl2/trunk swift

    @@ -163,6 +125,12 @@

    Historical releases

    +

    Swift 0.9 - 2009/04/27

    +

    [swift-0.9.tar.gz]

    + +

    Swift 0.8 - 2009/01/30

    Swift v0.8 is a development release intended to release functionality and fixes that have gone in to trunk since v0.7. From noreply at svn.ci.uchicago.edu Thu Dec 23 13:12:30 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 23 Dec 2010 13:12:30 -0600 (CST) Subject: [Swift-commit] r3828 - www/downloads Message-ID: <20101223191230.0A9E69CC82@svn.ci.uchicago.edu> Author: skenny Date: 2010-12-23 13:12:29 -0600 (Thu, 23 Dec 2010) New Revision: 3828 Modified: www/downloads/index.php Log: typo Modified: www/downloads/index.php =================================================================== --- www/downloads/index.php 2010-12-23 19:07:40 UTC (rev 3827) +++ www/downloads/index.php 2010-12-23 19:12:29 UTC (rev 3828) @@ -31,7 +31,7 @@

    Precompiled binary distribution [swift-0.9.tar.gz] +>swift-0.91.tar.gz] Author: wozniak Date: 2010-12-23 16:48:52 -0600 (Thu, 23 Dec 2010) New Revision: 3829 Added: trunk/src/org/globus/swift/catalog/TCEntry.java Log: Rename of TransformationCatalogEntry Added: trunk/src/org/globus/swift/catalog/TCEntry.java =================================================================== --- trunk/src/org/globus/swift/catalog/TCEntry.java (rev 0) +++ trunk/src/org/globus/swift/catalog/TCEntry.java 2010-12-23 22:48:52 UTC (rev 3829) @@ -0,0 +1,513 @@ +/* + * This file or a portion of this file is licensed under the terms of + * the Globus Toolkit Public License, found in file GTPL, or at + * http://www.globus.org/toolkit/download/license.html. This notice must + * appear in redistributions of this file, with or without modification. + * + * Redistributions of this Software, with or without modification, must + * reproduce the GTPL in: (1) the Software, or (2) the Documentation or + * some other similar material which is provided with the Software (if + * any). + * + * Copyright 1999-2004 University of Chicago and The University of + * Southern California. All rights reserved. + */ + +package org.globus.swift.catalog; + +/** + * An object of this class corresponds to a + * tuple in the Transformation Catalog. + * @author Gaurang Mehta + * @$Revision: 1.9 $ + * + * @see org.griphyn.common.classes.SysInfo + * @see org.griphyn.common.classes.TCType + */ + +import org.globus.swift.catalog.types.SysInfo; +import org.globus.swift.catalog.types.TCType; +import org.globus.swift.catalog.util.Profile; +import org.globus.swift.catalog.util.ProfileParser; +import org.globus.swift.catalog.util.Separator; + +import java.util.ArrayList; +import java.util.List; + +public class TCEntry + implements CatalogEntry { + + /** + * The logical namespace of the transformation + */ + private String namespace; + + /** + * The version of the transformation. + */ + private String version; + + /** + * The logical name of the transformation. + */ + private String name; + + /** + * The Id of the resource on which the transformation + * is installed. + */ + private String resourceid; + + /** + * The physical path on the resource for a particual arch, os and type. + */ + private String physicalname; + + /** + * The profiles associated with the transformation; + */ + private List profiles; + + /** + * The System Info for the transformation. + */ + private SysInfo sysinfo; + + /** + * The type of transformation. Takes one of the predefined enumerated type TCType. + */ + private TCType type = TCType.INSTALLED; + + /** + * The basic constructor + */ + public TCEntry() { + namespace = null; + name = null; + version = null; + resourceid = null; + physicalname = null; + profiles = null; + sysinfo = null; + } + + /** + * Optimized Constructor + * + * @param namespace String + * @param name String + * @param version String + */ + public TCEntry( String namespace, + String name, + String version){ + this.namespace = namespace; + this.version = version; + this.name = name; + } + /** + * Optimized Constructor + * @param namespace String + * @param name String + * @param version String + * @param resourceid String + * @param physicalname String + * @param type TCType + * @param profiles List + * @param sysinfo SysInfo + */ + public TCEntry( String namespace, String name, + String version, + String resourceid, String physicalname, TCType type, + List profiles, + SysInfo sysinfo ) { + this.namespace = namespace; + this.version = version; + this.name = name; + this.resourceid = resourceid; + this.physicalname = physicalname; + this.profiles = profiles; + this.sysinfo = sysinfo; + this.type = type; + + } + + /** + * creates a new instance of this object and returns + * you it. A shallow clone. + * TO DO : Gaurang correct the clone method. + * + * @return Object + */ + public Object clone() { + return new TCEntry( namespace, name, version, + resourceid, physicalname, + type, profiles, sysinfo ); + } + + /** + * gets the String version of the + * data class + * @return String + */ + public String toString() { + String st = "\n " + + "\n Logical Namespace : " + this.namespace + + "\n Logical Name : " + this.name + + "\n Version : " + this.version + + "\n Resource Id : " + this.resourceid + + "\n Physical Name : " + this.physicalname + + "\n SysInfo : " + ((this.sysinfo == null) ? "" : this.sysinfo.toString()) + + "\n TYPE : " + ((this.type == null) ? "" : type.toString()); + if(profiles != null){ + for (Profile p : profiles) + st = st + "\n Profile : " + p.toString(); + } + return st; + } + + /** + * Prints out a TC file format String. + * @return String + */ + public String toTCString() { + String st = this.getResourceId() + "\t" + + this.getLogicalTransformation() + "\t" + + this.getPhysicalTransformation() + "\t" + + this.getType() + "\t" + + this.getSysInfo() + "\t"; + if ( profiles != null ) { + st += ProfileParser.combine( profiles ); + } else { + st += "NULL"; + } + return st; + } + + /** + * Returns an xml output of the contents of the data class. + * @return String + */ + public String toXML() { + String xml = "\t\t( 5 ); + } + this.profiles.add( profile ); + } + } + + /** + * Allows you to add multiple profiles to the transformation. + * @param profiles List of Profile objects containing the profile information. + */ + public void setProfiles( List profiles ) { + if ( profiles != null ) { + if ( this.profiles == null ) { + this.profiles = + new ArrayList( profiles.size() ); + } + this.profiles.addAll( profiles ); + } + } + + /** + * Gets the Fully Qualified Transformation name in the format NS::Name:Ver. + * @return String + */ + public String getLogicalTransformation() { + return joinLFN( namespace, name, version ); + } + + /** + * Returns the Namespace associated with the logical transformation. + * @return String Returns null if no namespace associated with the transformation. + */ + public String getLogicalNamespace() { + return this.namespace; + } + + /** + * Returns the Name of the logical transformation. + * @return String + */ + public String getLogicalName() { + return this.name; + } + + /** + * Returns the version of the logical transformation. + * @return String Returns null if no version assocaited with the transformation. + */ + public String getLogicalVersion() { + return this.version; + } + + /** + * Returns the resource where the transformation is located. + * @return String + */ + public String getResourceId() { + return this.resourceid; + } + + /** + * Returns the type of the transformation. + * @return TCType + */ + public TCType getType() { + return this.type; + } + + /** + * Returns the physical location of the transformation. + * @return String + */ + public String getPhysicalTransformation() { + return this.physicalname; + } + + /** + * Returns the system information associated with the transformation in the formation ARCH::OS:OSver:GlibVer + * @return SysInfo + */ + public SysInfo getSysInfo() { + return this.sysinfo; + } + + /** + * Returns the list of profiles associated with the transformation. + * @return List Returns null if no profiles associated. + */ + public List getProfiles() { + return this.profiles; + } + + /** + * Returns the profiles for a particular Namespace. + * @param namespace String The namespace of the profile + * @return List List of Profile objects. returns null if none are found. + */ + @SuppressWarnings("hiding") + public List getProfiles( String namespace ) { + List results = null; + if ( profiles != null ) { + for ( Profile p : profiles ) { + if ( p.getProfileNamespace().equalsIgnoreCase( namespace ) ) { + if ( results == null ) { + results = new ArrayList(); + } + results.add( p ); + } + } + return results; + } else { + return results; + } + } + + /** + * Joins the 3 components into a fully qualified logical name of the format NS::NAME:VER + * @param namespace String + * @param name String + * @param version String + * @return String + */ + private static String joinLFN( String namespace, String name, + String version ) { + return Separator.combine( namespace, name, version ); + } + + /** + * Splits the full qualified logical transformation into its components. + * @param logicaltransformation String + * @return String[] + */ + private static String[] splitLFN( String logicaltransformation ) { + return Separator.split( logicaltransformation ); + } + + /** + * Converts the file profile string to a list of Profiles. + * @param profiles String The profile string. + * @return List Returns a list of profile objects + */ + /* public List stringToProfiles( String profiles ) { + if ( profiles == null ) { + return null; + } + List resultprofiles = new ArrayList(); + String[] namespaces = profiles.split( ";" ); + for ( int i = 0; i < namespaces.length; i++ ) { + String[] nsprofiles = namespaces[ i ].split( "::", 2 ); + if ( nsprofiles.length == 2 ) { + String ns = nsprofiles[ 0 ].trim(); + String[] keyvalues = nsprofiles[ 1 ].trim().split( "," ); + for ( int j = 0; j < keyvalues.length; j++ ) { + String[] keyvalue = keyvalues[ j ].trim().split( "=", 2 ); + String key = null; + String value = null; + if ( keyvalue.length == 2 ) { + key = keyvalue[ 0 ].trim(); + value = keyvalue[ 1 ].trim(); + } + if ( key != null && !key.equals( "" ) && value != null ) { + Profile p = new Profile( ns, key, value ); + resultprofiles.add( p ); + } + } + } + } + return resultprofiles; + } + */ + + /* + * Generates a file type profiles String. + * @param listprofiles List + * @return String + */ + /* public String profilesToString( List listprofiles ) { + String lprofiles = null; +// String temp = null; + if ( listprofiles != null ) { + lprofiles=ProfileParser.combine( listprofiles ); + } + String currentns = ""; + for ( Iterator i = listprofiles.iterator(); i.hasNext(); ) { + Profile p = ( Profile ) i.next(); + if ( !currentns.equalsIgnoreCase( p.getProfileNamespace() ) ) { + currentns = p.getProfileNamespace(); + if ( lprofiles != null ) { + lprofiles = lprofiles + temp + ";" + currentns + "::"; + temp = null; + } else { + lprofiles = currentns + "::"; + } + } + if ( temp != null ) { + temp = temp + "," + p.getProfileKey() + "=" + + p.getProfileValue(); + } else { + temp = p.getProfileKey() + "=" + p.getProfileValue(); + } + } + lprofiles += temp; + } + return lprofiles; + } + */ + + /** + * Compares two catalog entries for equality. + * + * @param entry is the entry to compare with + * @return true if the entries match, false otherwise + */ + public boolean equals( TCEntry entry ) { + return this.toTCString().equalsIgnoreCase( entry.toTCString() ); + } +} From noreply at svn.ci.uchicago.edu Mon Dec 27 17:45:24 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Dec 2010 17:45:24 -0600 (CST) Subject: [Swift-commit] r3830 - in trunk/tests: groups providers providers/ssh providers/ssh-pbs-coasters Message-ID: <20101227234524.4C3669CC7F@svn.ci.uchicago.edu> Author: davidk Date: 2010-12-27 17:45:22 -0600 (Mon, 27 Dec 2010) New Revision: 3830 Added: trunk/tests/groups/group-ssh.sh trunk/tests/providers/ssh-pbs-coasters/ trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.check.sh trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.setup.sh trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.swift trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.timeout trunk/tests/providers/ssh-pbs-coasters/README trunk/tests/providers/ssh-pbs-coasters/catsn.0001.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0002.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0003.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0004.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0005.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0006.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0007.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0008.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0009.out.expected trunk/tests/providers/ssh-pbs-coasters/catsn.0010.out.expected trunk/tests/providers/ssh-pbs-coasters/data.txt trunk/tests/providers/ssh-pbs-coasters/sites.template.xml trunk/tests/providers/ssh-pbs-coasters/tc.template.data trunk/tests/providers/ssh-pbs-coasters/title.txt trunk/tests/providers/ssh/ trunk/tests/providers/ssh/001-catsn-ssh.check.sh trunk/tests/providers/ssh/001-catsn-ssh.setup.sh trunk/tests/providers/ssh/001-catsn-ssh.swift trunk/tests/providers/ssh/001-catsn-ssh.timeout trunk/tests/providers/ssh/README trunk/tests/providers/ssh/catsn.0001.out.expected trunk/tests/providers/ssh/catsn.0002.out.expected trunk/tests/providers/ssh/catsn.0003.out.expected trunk/tests/providers/ssh/catsn.0004.out.expected trunk/tests/providers/ssh/catsn.0005.out.expected trunk/tests/providers/ssh/catsn.0006.out.expected trunk/tests/providers/ssh/catsn.0007.out.expected trunk/tests/providers/ssh/catsn.0008.out.expected trunk/tests/providers/ssh/catsn.0009.out.expected trunk/tests/providers/ssh/catsn.0010.out.expected trunk/tests/providers/ssh/data.txt trunk/tests/providers/ssh/sites.template.xml trunk/tests/providers/ssh/tc.template.data trunk/tests/providers/ssh/title.txt Log: Provider tests for ssh and ssh:pbs with coasters Added: trunk/tests/groups/group-ssh.sh =================================================================== --- trunk/tests/groups/group-ssh.sh (rev 0) +++ trunk/tests/groups/group-ssh.sh 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,7 @@ + +# GROUPLIST definition to run sge tests + +GROUPLIST=( + $TESTDIR/providers/ssh \ + $TESTDIR/providers/ssh-pbs-coasters \ +) Added: trunk/tests/providers/ssh/001-catsn-ssh.check.sh =================================================================== --- trunk/tests/providers/ssh/001-catsn-ssh.check.sh (rev 0) +++ trunk/tests/providers/ssh/001-catsn-ssh.check.sh 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +for count in `seq --format "%04.f" 1 1 10` +do + [ -f catsn.$count.out ] || exit 1 + CONTENTS1=$( cat catsn.$count.out.expected ) + CONTENTS2=$( cat catsn.$count.out ) + [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1 +done +exit 0 Property changes on: trunk/tests/providers/ssh/001-catsn-ssh.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/ssh/001-catsn-ssh.setup.sh =================================================================== --- trunk/tests/providers/ssh/001-catsn-ssh.setup.sh (rev 0) +++ trunk/tests/providers/ssh/001-catsn-ssh.setup.sh 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,4 @@ +#!/bin/bash + +cp -v $GROUP/data.txt . || exit 1 +cp -v $GROUP/*expected . || exit 1 Property changes on: trunk/tests/providers/ssh/001-catsn-ssh.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/ssh/001-catsn-ssh.swift =================================================================== --- trunk/tests/providers/ssh/001-catsn-ssh.swift (rev 0) +++ trunk/tests/providers/ssh/001-catsn-ssh.swift 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,15 @@ +type file; + +app (file o) cat (file i) +{ + cat @i stdout=@o; +} + +string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +string char[] = @strsplit(t, ""); + +file out[]; +foreach j in [1:@toint(@arg("n","10"))] { + file data<"data.txt">; + out[j] = cat(data); +} Added: trunk/tests/providers/ssh/001-catsn-ssh.timeout =================================================================== --- trunk/tests/providers/ssh/001-catsn-ssh.timeout (rev 0) +++ trunk/tests/providers/ssh/001-catsn-ssh.timeout 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +500 Added: trunk/tests/providers/ssh/README =================================================================== --- trunk/tests/providers/ssh/README (rev 0) +++ trunk/tests/providers/ssh/README 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,2 @@ +May need to edit sites.template.xml file if work directory contained in $WORK differs on local and remote machines +Requires auth.defaults to be set up correctly in ~/.ssh Added: trunk/tests/providers/ssh/catsn.0001.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0001.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0001.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0002.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0002.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0002.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0003.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0003.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0003.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0004.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0004.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0004.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0005.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0005.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0005.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0006.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0006.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0006.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0007.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0007.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0007.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0008.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0008.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0008.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0009.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0009.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0009.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/catsn.0010.out.expected =================================================================== --- trunk/tests/providers/ssh/catsn.0010.out.expected (rev 0) +++ trunk/tests/providers/ssh/catsn.0010.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/data.txt =================================================================== --- trunk/tests/providers/ssh/data.txt (rev 0) +++ trunk/tests/providers/ssh/data.txt 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh/sites.template.xml =================================================================== --- trunk/tests/providers/ssh/sites.template.xml (rev 0) +++ trunk/tests/providers/ssh/sites.template.xml 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,8 @@ + + + + + 0 + _WORK_ + + Added: trunk/tests/providers/ssh/tc.template.data =================================================================== --- trunk/tests/providers/ssh/tc.template.data (rev 0) +++ trunk/tests/providers/ssh/tc.template.data 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,8 @@ +ssh echo /bin/echo INSTALLED INTEL32::LINUX +ssh cat /bin/cat INSTALLED INTEL32::LINUX +ssh ls /bin/ls INSTALLED INTEL32::LINUX +ssh grep /bin/grep INSTALLED INTEL32::LINUX +ssh sort /bin/sort INSTALLED INTEL32::LINUX +ssh paste /bin/paste INSTALLED INTEL32::LINUX +ssh wc /usr/bin/wc INSTALLED INTEL32::LINUX + Added: trunk/tests/providers/ssh/title.txt =================================================================== --- trunk/tests/providers/ssh/title.txt (rev 0) +++ trunk/tests/providers/ssh/title.txt 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +SSH Configuration Test Added: trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.check.sh =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.check.sh (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.check.sh 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,12 @@ +#!/bin/bash + +set -x + +for count in `seq --format "%04.f" 1 1 10` +do + [ -f catsn.$count.out ] || exit 1 + CONTENTS1=$( cat catsn.$count.out.expected ) + CONTENTS2=$( cat catsn.$count.out ) + [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1 +done +exit 0 Property changes on: trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.check.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.setup.sh =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.setup.sh (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.setup.sh 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,4 @@ +#!/bin/bash + +cp -v $GROUP/data.txt . || exit 1 +cp -v $GROUP/*expected . || exit 1 Property changes on: trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.setup.sh ___________________________________________________________________ Name: svn:executable + * Added: trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.swift =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.swift (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.swift 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,15 @@ +type file; + +app (file o) cat (file i) +{ + cat @i stdout=@o; +} + +string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; +string char[] = @strsplit(t, ""); + +file out[]; +foreach j in [1:@toint(@arg("n","10"))] { + file data<"data.txt">; + out[j] = cat(data); +} Added: trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.timeout =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.timeout (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/001-catsn-ssh-pbs-coasters.timeout 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +500 Added: trunk/tests/providers/ssh-pbs-coasters/README =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/README (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/README 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,3 @@ +May need to edit sites.template.xml file if work directory contained in $WORK differs on local and remote machines +Requires auth.defaults to be set up correctly in ~/.ssh +Requires a valid proxy - run grid-proxy-init before running this test if you haven't already Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0001.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0001.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0001.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0002.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0002.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0002.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0003.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0003.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0003.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0004.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0004.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0004.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0005.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0005.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0005.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0006.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0006.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0006.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0007.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0007.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0007.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0008.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0008.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0008.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0009.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0009.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0009.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/catsn.0010.out.expected =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/catsn.0010.out.expected (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/catsn.0010.out.expected 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/data.txt =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/data.txt (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/data.txt 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +Hello world Added: trunk/tests/providers/ssh-pbs-coasters/sites.template.xml =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/sites.template.xml (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/sites.template.xml 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,15 @@ + + + + + 3000 + 8 + 1 + 1 + 10 + short + 0.5 + 10000 + _WORK_ + + Added: trunk/tests/providers/ssh-pbs-coasters/tc.template.data =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/tc.template.data (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/tc.template.data 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1,8 @@ +ssh-pbs-coasters echo /bin/echo INSTALLED INTEL32::LINUX +ssh-pbs-coasters cat /bin/cat INSTALLED INTEL32::LINUX +ssh-pbs-coasters ls /bin/ls INSTALLED INTEL32::LINUX +ssh-pbs-coasters grep /bin/grep INSTALLED INTEL32::LINUX +ssh-pbs-coasters sort /bin/sort INSTALLED INTEL32::LINUX +ssh-pbs-coasters paste /bin/paste INSTALLED INTEL32::LINUX +ssh-pbs-coasters wc /usr/bin/wc INSTALLED INTEL32::LINUX + Added: trunk/tests/providers/ssh-pbs-coasters/title.txt =================================================================== --- trunk/tests/providers/ssh-pbs-coasters/title.txt (rev 0) +++ trunk/tests/providers/ssh-pbs-coasters/title.txt 2010-12-27 23:45:22 UTC (rev 3830) @@ -0,0 +1 @@ +SSH with PBS and Coasters Configuration Test From noreply at svn.ci.uchicago.edu Mon Dec 27 18:16:36 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Dec 2010 18:16:36 -0600 (CST) Subject: [Swift-commit] r3831 - text/parco10submission Message-ID: <20101228001636.9A25D9CC7F@svn.ci.uchicago.edu> Author: hategan Date: 2010-12-27 18:16:36 -0600 (Mon, 27 Dec 2010) New Revision: 3831 Modified: text/parco10submission/paper.bib Log: added go web site Modified: text/parco10submission/paper.bib =================================================================== --- text/parco10submission/paper.bib 2010-12-27 23:45:22 UTC (rev 3830) +++ text/parco10submission/paper.bib 2010-12-28 00:16:36 UTC (rev 3831) @@ -232,3 +232,8 @@ year = 2005, booktitle = {Second International Workshop on Collaborative and Learning Applications of Grid Technology and Grid Education} } + + at MISC{GOLANG, + title = {The Go Programming Language}, + url = {http://golang.org}, +} From noreply at svn.ci.uchicago.edu Mon Dec 27 18:17:14 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Dec 2010 18:17:14 -0600 (CST) Subject: [Swift-commit] r3832 - text/parco10submission Message-ID: <20101228001714.B88399CC7F@svn.ci.uchicago.edu> Author: hategan Date: 2010-12-27 18:17:14 -0600 (Mon, 27 Dec 2010) New Revision: 3832 Modified: text/parco10submission/paper.tex Log: alternate language section Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-28 00:16:36 UTC (rev 3831) +++ text/parco10submission/paper.tex 2010-12-28 00:17:14 UTC (rev 3832) @@ -201,6 +201,10 @@ exists: lazy futures (futures whose computation is delayed until a value is first needed). +A useful side effect of using a futures based evaluation strategy is that +automatic parallelization is achieved without the need to resort to +dependency analysis, which is known to be a non-trivial engineering task. + A number of issues may be noted at this point. First, there exist a certain class of processes that may break referential transparency, while still producing ``valid'' results. An example would be Monte Carlo @@ -268,6 +272,112 @@ \section{The Swift language} \label{Language} +\begin{msection} + +\subsection{Language facilities} + +At the core of the Swift language are function definitions, of which +two types exist: +\begin{description} +\item[External functions] (also called ``atomic'') are functions whose +implementations are not written in Swift. Currently external functions +are implemented as command-line applications. +\item[Internal functions] (also called ``compound'') are functions +implemented in Swift. +\end{description} + +In addition to functions, the Swift language provides conditional +execution through the \emph{if} and \emph{switch} statements as well as +a \emph{foreach} construct used for iterating over arrays of data. +{\color{red} Note: I'm skipping \emph{iterate} on purpose. We should +deprecate it since it's hard to understand and everything that can be +done with it can also be done with \emph{foreach}} + +The \emph{if} and \emph{switch} statements are rather standard, but +\emph{foreach} merits more discussion. Similar to \emph{Go} +(\ref{GOLANG}) and\emph{Python}, its control ``variables'' can be both +an index and a value. The syntax is as follows: + +\begin{verbatim} +foreach v[, k] in array { + ... +} +\end{verbatim} + +This is necessary because Swift does not allow the use of mutable state +(i.e. variables are single-assignment), therefore one would not be able +to write statements such as \verb|i = i + 1|. + +We provide a few examples of some standard functions seen in functional +languages and the corresponding Swift implementation using +\emph{foreach}: + +\begin{description} +\item[b = map(f, a)] +\begin{verbatim} + +foreach v, k in a { + b[k] = f(v); +} +\end{verbatim} + +\item[c = zipWith(f, a, b)] +\begin{verbatim} + + foreach v, k in a { + c[k] = f(v, b[k]); + } +\end{verbatim} + +\item[r = foldr(f, v0, a)] +\begin{verbatim} + + r[0] = v0; + foreach v, k in a { + r[k + 1] = f(r[k], v); + } +\end{verbatim} +\end{description} + +\subsection{Data model} + +Swift provides two basic classes of data types: +\begin{description} +\item[Primitive types] (e.g. \emph{integer}, \emph{string}) are types +provided by the Swift runtime. Standard operators are defined for +primitive types, such as addition, multiplication, concatenation, etc. + +\item[Mapped types] are types of data for which some external +implementation exists. Swift provides a mechanism to describe +isomorphisms between instances of Swift data structures and subsets in +the external implementation. This mechanism is called ``mapping'' and +specific instances of isomorphisms are called ``mappers''. Currently the +only external implementation is a POSIX-like filesystem. However the +``external'' data type can be used to accommodate any external data that +Swift cannot and should not directly handle. Swift mapped types can be +seen as generalizations of reference types in traditional languages in +that reference types are language representations of data stored in +internal memory, in contrast with primitive (value) types for which no +explicit storage is generally specified. +\end{description} + + +There is no syntactic distinction between primitive types and reference +types. In addition, the semantic differences between the two classes of +types is kept to a minimum. + +Data can be organized in so called ``composite types''. Swift provides +facilities to aggregate data into \emph{arrays} and \emph{structures}. +This can be done recursively in that arrays of arrays, structures +containing structures, arrays of structures as well as structures +containing arrays are possible. Any types that have no internal +structure (i.e. non-composite types) are called ``atomic types''. + +Atomic mapped types do not specify any information about the structure of +the data. It is up to the user to assign a ``proper'' type to external +data. Consequently Swift must and does implement nominal type equivalence. +\end{msection} + The Swift programming model is data-oriented: it encapsulates the invocation of ``ordinary programs''---technically, POSIX {\tt exec()} operations---in a manner that explicitly specifies the files and other @@ -315,6 +425,7 @@ \subsection{Language basics} + A Swift script describes data, application components, invocations of applications components, and the inter-relations (data flow) between those invocations, using a C-like syntax. From noreply at svn.ci.uchicago.edu Mon Dec 27 19:51:41 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Mon, 27 Dec 2010 19:51:41 -0600 (CST) Subject: [Swift-commit] r3833 - in provenancedb: . apps/oops Message-ID: <20101228015141.2AEBE9CC7F@svn.ci.uchicago.edu> Author: lgadelha Date: 2010-12-27 19:51:41 -0600 (Mon, 27 Dec 2010) New Revision: 3833 Modified: provenancedb/apps/oops/oops_extractor.sh provenancedb/pql_functions.sql Log: Minor fixes to provenancedb. Modified: provenancedb/apps/oops/oops_extractor.sh =================================================================== --- provenancedb/apps/oops/oops_extractor.sh 2010-12-28 00:17:14 UTC (rev 3832) +++ provenancedb/apps/oops/oops_extractor.sh 2010-12-28 01:51:41 UTC (rev 3833) @@ -23,7 +23,7 @@ do cd $IMPORT_HOME if ! grep --silent $i provdb_imported; then - if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/psim.loops-*.log; then + if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/*-*-*-*.log; then # swift-prov-import-all-logs also controls what already has been # imported, so it does not repeat work echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config @@ -68,7 +68,23 @@ fi done < $FILENAME - # extracts scientific parameters given as input to the workflow in file *.params. + #extracts length of the fasta sequence given as input to the workflow in file *.fasta. + cd $IMPORT_HOME/swift-logs + + echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.fasta' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID' and ds_usage.direction='O';" > query.sql; + + $SQLCMD -t -A -F " " -f query.sql -o result.txt + + DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt) + FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g') + + cd $PROTESTS_HOME/$k/$i + + if [ -n "$FILENAME" ]; then + SEQLENGTH=$(awk '{if (NR==2) print $1}' $FILENAME | wc -c) + echo "insert into annot_ds_txt values ('$DATASET_ID', 'fasta_sequence_length', $SEQLENGTH);" | $SQLCMD + fi + # extracts scientific parameters given as output by the workflow in *.log. # relevant lines: # zone2 (Initial Energy: -21352.116911) # Total Function Evaluations: 20000 @@ -79,10 +95,14 @@ # Final Energy: -27152.264775 # Final Temp: 79.778142 # Total Running Time: 18006 + + cd $IMPORT_HOME/swift-logs echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.log' and process.name='LoopModel' and process.workflow_id='$WORKFLOW_ID' and ds_usage.direction='O';" > query.sql; $SQLCMD -t -A -F " " -f query.sql -o result.txt + + cd $PROTESTS_HOME/$k/$i while read dataset filename; do FILENAME=$(echo $filename | sed 's/file:\/\/localhost\///g') @@ -91,29 +111,29 @@ initialenergy=$(echo $token3 | awk 'BEGIN { FS = "\)" }; {print $1}') echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'initial_energy', $initialenergy);" | $SQLCMD fi - if [ "$token1" = "Total" && "$token2" = "Function" && "$token3" = "Evaluations:" ]; then + if [ "$token1" = "Total" ] && [ "$token2" = "Function" ] && [ "$token3" = "Evaluations:" ]; then echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'total_function_evaluations', $token4);" | $SQLCMD fi - if [ "$token1" = "Increasing" && "$token2" = "transitions:" ]; then + if [ "$token1" = "Increasing" ] && [ "$token2" = "transitions:" ]; then echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'accepted_increasing_transitions', $token3);" | $SQLCMD fi - if [ "$token1" = "Decreasing" && "$token2" = "transitions:" ]; then + if [ "$token1" = "Decreasing" ] && [ "$token2" = "transitions:" ]; then echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'accepted_decreasing_transitions', $token3);" | $SQLCMD fi - if [ "$token1" = "Rejected" && "$token2" = "transitions:" ]; then + if [ "$token1" = "Rejected" ] && [ "$token2" = "transitions:" ]; then echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'rejected_transitions', $token3);" | $SQLCMD fi - if [ "$token1" = "Final" && "$token2" = "Energy:" ]; then + if [ "$token1" = "Final" ] && [ "$token2" = "Energy:" ]; then echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'final_energy', $token3);" | $SQLCMD fi - if [ "$token1" = "Final" && "$token2" = "Temp:" ]; then + if [ "$token1" = "Final" ] && [ "$token2" = "Temp:" ]; then echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'final_temp', $token3);" | $SQLCMD fi - if [ "$token1" = "Total" && "$token2" = "Running" && "$token3" = "Time:" ]; then + if [ "$token1" = "Total" ] && [ "$token2" = "Running" ] && [ "$token3" = "Time:" ]; then echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'total_running_time', $token4);" | $SQLCMD fi - done < result.txt - done < $FILENAME + done < $FILENAME + done < $IMPORT_HOME/swift-logs/result.txt fi fi done Modified: provenancedb/pql_functions.sql =================================================================== --- provenancedb/pql_functions.sql 2010-12-28 00:17:14 UTC (rev 3832) +++ provenancedb/pql_functions.sql 2010-12-28 01:51:41 UTC (rev 3833) @@ -34,6 +34,20 @@ GROUP BY workflow.log_filename,ds_usage.param_name,variable.value; $$ LANGUAGE SQL; +drop type annot_across_wf_type cascade; +create type annot_across_wf_type as (workflow varchar, process varchar, name varchar, value numeric); + +create or replace function annot_across_wf(name varchar) +returns setof annot_across_wf_type as $$ + select process.workflow_id,process.id,annot_ds_num.name,annot_ds_num.value + from process,ds_usage,ds_containment,annot_ds_num + where process.id=ds_usage.process_id and ds_usage.dataset_id=ds_containment.out_id and + ds_containment.in_id=annot_ds_num.id and annot_ds_num.name=$1 + order by process.workflow_id,process.id; +$$ language sql; + + + -- lists variations of the values of a set of parameters DROP TYPE param_across_wf_class_type CASCADE; @@ -41,7 +55,7 @@ CREATE OR REPLACE FUNCTION param_across_wf_class(param_name VARCHAR, wf_class VARACHAR) RETURNS SETOF param_across_wf_type AS $$ - + $$ LANGUAGE plpgsql; -- correlate a parameter with workflow runtime statistics From noreply at svn.ci.uchicago.edu Thu Dec 30 15:53:09 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Thu, 30 Dec 2010 15:53:09 -0600 (CST) Subject: [Swift-commit] r3834 - text/parco10submission Message-ID: <20101230215309.3377A9CC7F@svn.ci.uchicago.edu> Author: dsk Date: 2010-12-30 15:53:08 -0600 (Thu, 30 Dec 2010) New Revision: 3834 Modified: text/parco10submission/paper.tex Log: some very small changes Modified: text/parco10submission/paper.tex =================================================================== --- text/parco10submission/paper.tex 2010-12-28 01:51:41 UTC (rev 3833) +++ text/parco10submission/paper.tex 2010-12-30 21:53:08 UTC (rev 3834) @@ -295,7 +295,7 @@ The \emph{if} and \emph{switch} statements are rather standard, but \emph{foreach} merits more discussion. Similar to \emph{Go} -(\ref{GOLANG}) and\emph{Python}, its control ``variables'' can be both +(\ref{GOLANG}) and \emph{Python}, its control ``variables'' can be both an index and a value. The syntax is as follows: \begin{verbatim} @@ -305,7 +305,7 @@ \end{verbatim} This is necessary because Swift does not allow the use of mutable state -(i.e. variables are single-assignment), therefore one would not be able +(i.e., variables are single-assignment), therefore one would not be able to write statements such as \verb|i = i + 1|. We provide a few examples of some standard functions seen in functional @@ -343,7 +343,7 @@ Swift provides two basic classes of data types: \begin{description} -\item[Primitive types] (e.g. \emph{integer}, \emph{string}) are types +\item[Primitive types] (e.g., \emph{integer}, \emph{string}) are types provided by the Swift runtime. Standard operators are defined for primitive types, such as addition, multiplication, concatenation, etc. From noreply at svn.ci.uchicago.edu Fri Dec 31 17:39:13 2010 From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu) Date: Fri, 31 Dec 2010 17:39:13 -0600 (CST) Subject: [Swift-commit] r3835 - trunk/src/org/griphyn/vdl/karajan/functions Message-ID: <20101231233913.4C1CF9CC94@svn.ci.uchicago.edu> Author: skenny Date: 2010-12-31 17:39:12 -0600 (Fri, 31 Dec 2010) New Revision: 3835 Modified: trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java Log: don't show java exceptions, only the swift error Modified: trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java 2010-12-30 21:53:08 UTC (rev 3834) +++ trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java 2010-12-31 23:39:12 UTC (rev 3835) @@ -90,31 +90,39 @@ } public static String getMessageChain(Throwable e) { + Throwable orig = e; StringBuffer sb = new StringBuffer(); String prev = null; - boolean first = true; + String lastmsg = null; + boolean first = true; while (e != null) { String msg; if (e instanceof NullPointerException || e instanceof ClassCastException) { CharArrayWriter caw = new CharArrayWriter(); e.printStackTrace(new PrintWriter(caw)); msg = caw.toString(); + } else { msg = e.getMessage(); + if(msg != null){ + lastmsg = msg; + } + } if (msg != null && (prev == null || prev.indexOf(msg) == -1)) { - if (!first) { - sb.append("\nCaused by:\n\t"); - } - else { - first = false; - } - sb.append(msg); - prev = msg; + if (!first) { + sb.append("\nCaused by:\n\t"); + } + else { + first = false; + } + sb.append(msg); + lastmsg = msg; + prev = msg; } - e = e.getCause(); + e = e.getCause(); } - return sb.toString(); + return lastmsg; } }