From noreply at svn.ci.uchicago.edu Wed Dec 1 15:00:54 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 1 Dec 2010 15:00:54 -0600 (CST)
Subject: [Swift-commit] r3731 - trunk/bin
Message-ID: <20101201210054.892DC9CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-01 15:00:54 -0600 (Wed, 01 Dec 2010)
New Revision: 3731
Modified:
trunk/bin/swift
Log:
Return non-zero exit code if Swift is not installed correctly
Modified: trunk/bin/swift
===================================================================
--- trunk/bin/swift 2010-11-29 23:59:05 UTC (rev 3730)
+++ trunk/bin/swift 2010-12-01 21:00:54 UTC (rev 3731)
@@ -45,7 +45,7 @@
SWIFT_HOME=`dirname $MY_PATH`"/.."
if [ ! -f "$SWIFT_HOME/lib/cog-swift-svn.jar" ] && [ ! -f "$SWIFT_HOME/lib/cog.jar" ] ; then
echo "Error: SWIFT_HOME is not set, and all attempts at guessing it failed"
- exit
+ exit 1
fi
fi
fi
From noreply at svn.ci.uchicago.edu Wed Dec 1 21:37:14 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 1 Dec 2010 21:37:14 -0600 (CST)
Subject: [Swift-commit] r3732 - provenancedb
Message-ID: <20101202033714.718CB9CC7F@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-01 21:37:14 -0600 (Wed, 01 Dec 2010)
New Revision: 3732
Modified:
provenancedb/prov-init.sql
provenancedb/prov-to-sql.sh
Log:
Minor corrections to import scripts.
Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql 2010-12-01 21:00:54 UTC (rev 3731)
+++ provenancedb/prov-init.sql 2010-12-02 03:37:14 UTC (rev 3732)
@@ -231,8 +231,6 @@
PRIMARY KEY (array_id,from_id,to_id,step_id)
);
-
-
-- this GRANT does not work for sqlite; you'll get a syntax error but
-- ignore it, as it is not needed in sqlite
GRANT ALL ON
Modified: provenancedb/prov-to-sql.sh
===================================================================
--- provenancedb/prov-to-sql.sh 2010-12-01 21:00:54 UTC (rev 3731)
+++ provenancedb/prov-to-sql.sh 2010-12-02 03:37:14 UTC (rev 3732)
@@ -9,7 +9,7 @@
echo Generating SQL for $RUNID
-rm -f tmp-ds.sql tmp-p.sql tmp-e.sql tmp-e2.sql tmp-dsu.sql tmp-dsc.sql tmp-f.sql tmp-v.sql tmp-import.sql import.sql
+rm -f tmp-u.sql tmp-ds.sql tmp-p.sql tmp-e.sql tmp-e2.sql tmp-dsu.sql tmp-dsc.sql tmp-f.sql tmp-v.sql tmp-import.sql import.sql
# this gives a distinction between the root process for a workflow and the
# workflow itself. perhaps better to model the workflow as a process
From noreply at svn.ci.uchicago.edu Fri Dec 3 17:11:51 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 3 Dec 2010 17:11:51 -0600 (CST)
Subject: [Swift-commit] r3733 - provenancedb
Message-ID: <20101203231151.85DF89CC9B@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-03 17:11:51 -0600 (Fri, 03 Dec 2010)
New Revision: 3733
Modified:
provenancedb/prov-to-sql.sh
Log:
Minor fixes.
Modified: provenancedb/prov-to-sql.sh
===================================================================
--- provenancedb/prov-to-sql.sh 2010-12-02 03:37:14 UTC (rev 3732)
+++ provenancedb/prov-to-sql.sh 2010-12-03 23:11:51 UTC (rev 3733)
@@ -9,7 +9,7 @@
echo Generating SQL for $RUNID
-rm -f tmp-u.sql tmp-ds.sql tmp-p.sql tmp-e.sql tmp-e2.sql tmp-dsu.sql tmp-dsc.sql tmp-f.sql tmp-v.sql tmp-import.sql import.sql
+rm -f tmp-u.sql tmp-ds.sql tmp-p.sql tmp-e.sql tmp-e2.sql tmp-dsu.sql tmp-dsc.sql tmp-f.sql tmp-v.sql tmp-import.sql import.sql
# this gives a distinction between the root process for a workflow and the
# workflow itself. perhaps better to model the workflow as a process
@@ -90,13 +90,17 @@
# TODO this could merge with other naming tables
while read start duration thread final_state procname ; do
- compoundid=$WFID$thread
- echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$compoundid', 'compound', '$procname', '$WF');" >> tmp-p.sql
+ if [ "$duration" != "last-event-line" ]; then
+ compoundid=$WFID$thread
+ echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$compoundid', 'compound', '$procname', '$WF');" >> tmp-p.sql
+ fi
done < compound.event
while read start duration thread final_state procname ; do
- fqid=$WFID$thread
- echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$fqid', 'internal', '$procname', '$WF');" >> tmp-p.sql
+ if [ "$duration" != "last-event-line" ]; then
+ fqid=$WFID$thread
+ echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$fqid', 'internal', '$procname', '$WF');" >> tmp-p.sql
+ fi
done < internalproc.event
while read t ; do
@@ -132,7 +136,6 @@
cat tmp-dsc.sql | sort | uniq >> import.sql
cat tmp-import.sql | sort | uniq >> import.sql
cat tmp-u.sql | sort | uniq >> import.sql
-
echo Sending SQL to DB
$SQLCMD < import.sql
From noreply at svn.ci.uchicago.edu Mon Dec 6 13:13:44 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 13:13:44 -0600 (CST)
Subject: [Swift-commit] r3734 - trunk/libexec
Message-ID: <20101206191344.892199CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 13:13:44 -0600 (Mon, 06 Dec 2010)
New Revision: 3734
Modified:
trunk/libexec/cdm_broadcast.sh
Log:
Whitespace cleanup
Modified: trunk/libexec/cdm_broadcast.sh
===================================================================
--- trunk/libexec/cdm_broadcast.sh 2010-12-03 23:11:51 UTC (rev 3733)
+++ trunk/libexec/cdm_broadcast.sh 2010-12-06 19:13:44 UTC (rev 3734)
@@ -10,12 +10,12 @@
while [[ ${*} != "" ]]
do
L=$1 # -l
- shift
+ shift
ARGS=$1 # Location
shift
while true
do
- if [[ $1 == "-l" || $1 == "" ]]
+ if [[ $1 == "-l" || $1 == "" ]]
then
break
fi
@@ -34,14 +34,14 @@
shift
WORK=( ${*} )
- IP=$( listip ${LOCATION} )
+ IP=$( listip ${LOCATION} )
- if [[ ${#WORK[@]} > 3 ]]
+ if [[ ${#WORK[@]} > 3 ]]
then
SCRIPT=$( mktemp )
{
echo "#!/bin/sh"
- while [[ ${*} != "" ]]
+ while [[ ${*} != "" ]]
do
FILE=$1
DEST=$2
@@ -52,7 +52,7 @@
scp ${SCRIPT} ${IP}:${SCRIPT}
ssh ${IP} ${SCRIPT}
else
- while [[ ${*} != "" ]]
+ while [[ ${*} != "" ]]
do
FILE=$1
DEST=$2
@@ -65,12 +65,12 @@
# Repeat command N times until success
ssh_until_success()
{
- N=$1
- shift
+ N=$1
+ shift
for (( i=0 ; i < N ; i++ ))
do
ssh -o PasswordAuthentication=no ${*}
- if [[ $? == 0 ]];
+ if [[ $? == 0 ]];
then
break
fi
@@ -86,16 +86,16 @@
DEST=$3
cp -v ${FILE} ${DEST}/${FILE}
}
-
+
set -x
{
declare -p PWD LOG
-
+
if [[ $( uname -p ) == "ppc64" ]]
then
bgp_broadcast ${*}
- else
+ else
bgp_local ${*}
fi
-
+
} >> /tmp/cdm_broadcast.log 2>&1 # ${LOG} 2>&1
From noreply at svn.ci.uchicago.edu Mon Dec 6 13:18:11 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 13:18:11 -0600 (CST)
Subject: [Swift-commit] r3735 - trunk/libexec
Message-ID: <20101206191811.EFAF59CC7F@svn.ci.uchicago.edu>
Author: wilde
Date: 2010-12-06 13:18:11 -0600 (Mon, 06 Dec 2010)
New Revision: 3735
Modified:
trunk/libexec/cdm_lib.sh
Log:
Correct the argument handling for CDM LOCAL file action.
Modified: trunk/libexec/cdm_lib.sh
===================================================================
--- trunk/libexec/cdm_lib.sh 2010-12-06 19:13:44 UTC (rev 3734)
+++ trunk/libexec/cdm_lib.sh 2010-12-06 19:18:11 UTC (rev 3735)
@@ -51,12 +51,16 @@
fi
;;
LOCAL)
- TOOL=${ARGS[0]}
- REMOTE_DIR=${ARGS[1]}
- FLAGS=${ARGS[3]}
- log "CDM[LOCAL]: Copying $DIRECT_DIR/$FILE to $JOBDIR/$FILE"
+ #TOOL=${ARGS[0]}
+ #REMOTE_DIR=${ARGS[1]}
+ #FLAGS=${ARGS[3]}
+ TOOL=$1
+ REMOTE_DIR=$2
+ FLAGS=$3
+ log "CDM[LOCAL]: TOOL=$TOOL FLAGS=$FLAGS REMOTE_DIR=$REMOTE_DIR ARGS=$ARGS"
+ log "CDM[LOCAL]: Copying $REMOTE_DIR/$FILE to $JOBDIR/$FILE"
if [ $MODE == "INPUT" ]; then
- [ -f "$DIRECT_DIR/$FILE" ]
+ [ -f "$REMOTE_DIR/$FILE" ]
checkError 254 "CDM[LOCAL]: $REMOTE_DIR/$FILE does not exist!"
$TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE
checkError 254 "CDM[LOCAL]: Tool failed!"
From noreply at svn.ci.uchicago.edu Mon Dec 6 13:36:57 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 13:36:57 -0600 (CST)
Subject: [Swift-commit] r3736 - trunk/src/org/globus/swift/data
Message-ID: <20101206193657.3E96C9CC95@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 13:36:57 -0600 (Mon, 06 Dec 2010)
New Revision: 3736
Modified:
trunk/src/org/globus/swift/data/Director.java
Log:
Fix warning
Modified: trunk/src/org/globus/swift/data/Director.java
===================================================================
--- trunk/src/org/globus/swift/data/Director.java 2010-12-06 19:18:11 UTC (rev 3735)
+++ trunk/src/org/globus/swift/data/Director.java 2010-12-06 19:36:57 UTC (rev 3736)
@@ -78,11 +78,11 @@
/**
Read in the user-supplied CDM policy file.
*/
- public static void load(File policyFile) throws IOException {
- logger.info("CDM file: " + policyFile);
+ public static void load(File file) throws IOException {
+ logger.info("CDM file: " + file);
enabled = true;
- Director.policyFile = policyFile;
- List list = LineReader.read(policyFile);
+ Director.policyFile = file;
+ List list = LineReader.read(file);
for (String s : list)
addLine(s);
}
@@ -177,10 +177,10 @@
If the location is added twice, the second addition
is considered to be an empty allocation with no CDM state.
*/
- public static synchronized void addAllocation(String allocation) {
- logger.debug("addAllocation(): " + allocation);
- allocations.add(allocation);
- broadcasted.put(allocation, new HashSet());
+ public static synchronized void addBlock(String blockId) {
+ logger.debug("addBlock(): " + blockId);
+ allocations.add(blockId);
+ broadcasted.put(blockId, new HashSet());
doBroadcast();
}
From noreply at svn.ci.uchicago.edu Mon Dec 6 15:22:28 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 15:22:28 -0600 (CST)
Subject: [Swift-commit] r3737 - in trunk: libexec src/org/globus/swift/data
src/org/globus/swift/data/policy src/org/griphyn/vdl/karajan
src/org/griphyn/vdl/util
Message-ID: <20101206212228.AD0929CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 15:22:28 -0600 (Mon, 06 Dec 2010)
New Revision: 3737
Modified:
trunk/libexec/cdm_broadcast.sh
trunk/src/org/globus/swift/data/Action.java
trunk/src/org/globus/swift/data/Director.java
trunk/src/org/globus/swift/data/policy/AllocationHook.java
trunk/src/org/globus/swift/data/policy/Broadcast.java
trunk/src/org/griphyn/vdl/karajan/Loader.java
trunk/src/org/griphyn/vdl/util/VDL2Config.java
Log:
Cleanups and fixes to CDM BROADCAST
Modified: trunk/libexec/cdm_broadcast.sh
===================================================================
--- trunk/libexec/cdm_broadcast.sh 2010-12-06 19:36:57 UTC (rev 3736)
+++ trunk/libexec/cdm_broadcast.sh 2010-12-06 21:22:28 UTC (rev 3737)
@@ -1,34 +1,14 @@
#!/bin/sh
+# Called by org.globus.swift.data.policy.Broadcast
+# usage: cdm_broadcast.sh
+# copies DIR/FILE to DEST via MODE with logging to LOG
+
SWIFT_HOME=$( dirname $( dirname $0 ) )
-LOG=${SWIFT_HOME}/etc/cdm_broadcast.log
-# For each given location, broadcast the given files to it
-# Input: bgp_broadcast [-l *]*
-bgp_broadcast()
-{
- while [[ ${*} != "" ]]
- do
- L=$1 # -l
- shift
- ARGS=$1 # Location
- shift
- while true
- do
- if [[ $1 == "-l" || $1 == "" ]]
- then
- break
- fi
- ARGS="${ARGS} $1"
- shift
- done
- bgp_broadcast_perform ${ARGS}
- done
-}
-
# Broadcast the given files to the given location
# Input: bgp_broadcast_perform *
-bgp_broadcast_perform()
+bgp_broadcast()
{
LOCATION=$1
shift
@@ -81,21 +61,48 @@
local_broadcast()
{
- DIR=$1
+ ALLOCATION=$1 # Ignored (LOCAL_FILE)
FILE=$2
DEST=$3
cp -v ${FILE} ${DEST}/${FILE}
}
-set -x
+MODE=$1
+LOG=$2
+shift 2
+
+[[ ${LOG} != /dev/null ]] && set -x
{
declare -p PWD LOG
- if [[ $( uname -p ) == "ppc64" ]]
+ if [[ ${MODE} == "f2cn" ]]
then
- bgp_broadcast ${*}
+ BROADCAST="bgp_broadcast"
+ elif [[ ${MODE} == "file" ]]
+ then
+ BROADCAST="local_broadcast"
else
- bgp_local ${*}
+ echo "Unknown broadcast mode!"
+ exit 1
fi
-} >> /tmp/cdm_broadcast.log 2>&1 # ${LOG} 2>&1
+ while [[ ${*} != "" ]]
+ do
+ L=$1 # -l
+ shift
+ ARGS=$1 # Location
+ shift
+ while true
+ do
+ if [[ $1 == "-l" || $1 == "" ]]
+ then
+ break
+ fi
+ ARGS="${ARGS} $1"
+ shift
+ done
+ ${BROADCAST} ${ARGS}
+ done
+} >> ${LOG} 2>&1
+
+exit 0
Modified: trunk/src/org/globus/swift/data/Action.java
===================================================================
--- trunk/src/org/globus/swift/data/Action.java 2010-12-06 19:36:57 UTC (rev 3736)
+++ trunk/src/org/globus/swift/data/Action.java 2010-12-06 21:22:28 UTC (rev 3737)
@@ -1,7 +1,5 @@
package org.globus.swift.data;
-import java.io.IOException;
-
import org.apache.log4j.Logger;
import org.globus.cog.karajan.arguments.Arg;
Modified: trunk/src/org/globus/swift/data/Director.java
===================================================================
--- trunk/src/org/globus/swift/data/Director.java 2010-12-06 19:36:57 UTC (rev 3736)
+++ trunk/src/org/globus/swift/data/Director.java 2010-12-06 21:22:28 UTC (rev 3737)
@@ -19,9 +19,11 @@
import org.globus.swift.data.policy.Policy;
import org.globus.swift.data.policy.Broadcast;
import org.globus.swift.data.util.LineReader;
+import org.griphyn.vdl.util.VDL2Config;
/**
* Manages CDM policies for files based on pattern matching.
+ * Initialized when given a CDM file to read.
* @author wozniak
* */
@@ -54,6 +56,7 @@
/**
Remember the files we have broadcasted.
Map from allocations to filenames.
+ The keys represent the list of known allocations.
NOTE: must be accessed only using synchronized Director methods
*/
private static Map> broadcasted =
@@ -65,28 +68,46 @@
*/
private static Set broadcastWork = new LinkedHashSet();
- /**
- Remember all known allocations
- */
- private static List allocations = new ArrayList();
-
public static boolean isEnabled()
{
return enabled;
}
+ /**
+ How to broadcast: either "file" or "f2cn"
+ */
+ public static String broadcastMode = null;
+
+ /**
+ Convenience reference to the Swift logfile name
+ */
+ public static String logfile = null;
+
/**
Read in the user-supplied CDM policy file.
*/
public static void load(File file) throws IOException {
- logger.info("CDM file: " + file);
- enabled = true;
+ logger.debug("CDM file: " + file);
Director.policyFile = file;
List list = LineReader.read(file);
for (String s : list)
addLine(s);
+ init();
}
+ static void init() throws IOException
+ {
+ VDL2Config config = VDL2Config.getConfig();
+
+ broadcastMode = config.getProperty("cdm.broadcast.mode");
+ logfile = config.getProperty("logfile");
+
+ if (broadcastMode.equals("file"))
+ broadcasted.put("LOCAL_FILE", new HashSet());
+
+ enabled = true;
+ }
+
/**
A line is either a rule or a property.
*/
@@ -173,13 +194,12 @@
}
/**
- Add a location to the list of allocations.
+ Add a Coasters allocation to the list of allocations.
If the location is added twice, the second addition
is considered to be an empty allocation with no CDM state.
*/
- public static synchronized void addBlock(String blockId) {
- logger.debug("addBlock(): " + blockId);
- allocations.add(blockId);
+ public static synchronized void addAllocation(String blockId) {
+ logger.debug("addAllocation(): " + blockId);
broadcasted.put(blockId, new HashSet());
doBroadcast();
}
@@ -195,7 +215,7 @@
return;
logger.debug("doBroadcast(): batch: " + batch);
Broadcast.perform(batch);
- markBroadcasts(batch);
+ noteBroadcasts(batch);
logger.debug("marked: " + broadcasted);
}
@@ -207,15 +227,18 @@
*/
private static Map> getBroadcastBatch() {
logger.debug("getBroadcastBatch(): ");
- Map> batch = new LinkedHashMap>();
+ Map> batch =
+ new LinkedHashMap>();
for (String file : broadcastWork) {
logger.debug("file: " + file);
- logger.debug("allocations: " + allocations);
- for (String allocation : allocations) {
- Set files = broadcasted.get(allocation);
+ for (Map.Entry> entry :
+ broadcasted.entrySet()) {
+ String allocation = entry.getKey();
+ Set files = entry.getValue();
logger.debug("files: " + files);
if (! files.contains(file)) {
- logger.debug("adding: " + file + " to: " + allocation);
+ logger.debug("adding: " + file +
+ " to: " + allocation);
List work = batch.get(allocation);
if (work == null) {
work = new ArrayList();
@@ -229,18 +252,21 @@
}
/**
- Mark that the files in the given batch have been successfully broadcasted.
+ Note that the files in the given batch have been successfully
+ broadcasted by adding them to Set broadcasted.
Should only be called by {@link doBroadcast}.
*/
- private static void markBroadcasts(Map> batch) {
+ private static void noteBroadcasts(Map> batch) {
logger.debug("markBroadcasts: batch: " + batch);
- for (Map.Entry> entry : batch.entrySet()) {
+ for (Map.Entry> entry :
+ batch.entrySet()) {
String location = entry.getKey();
logger.debug("markBroadcasts: location: " + location);
List files = entry.getValue();
for (String file : files) {
Set contents = broadcasted.get(location);
- assert (! contents.contains(file));
+ assert(! contents.contains(file));
logger.debug("markBroadcasts: add: " + file);
contents.add(file);
}
Modified: trunk/src/org/globus/swift/data/policy/AllocationHook.java
===================================================================
--- trunk/src/org/globus/swift/data/policy/AllocationHook.java 2010-12-06 19:36:57 UTC (rev 3736)
+++ trunk/src/org/globus/swift/data/policy/AllocationHook.java 2010-12-06 21:22:28 UTC (rev 3737)
@@ -12,15 +12,12 @@
* */
public class AllocationHook extends Hook
{
- public void blockActive(StatusEvent e)
+ public void blockActive(StatusEvent e, String blockId)
{
if (!Director.isEnabled())
return;
- System.out.println("blockActive: " + e.getStatus().getMessage());
- String msg = e.getStatus().getMessage();
- String[] tokens = msg.split("=");
- String allocation = tokens[1];
- Director.addAllocation(allocation);
+ System.out.println("blockActive: " + blockId);
+ Director.addAllocation(blockId);
}
}
Modified: trunk/src/org/globus/swift/data/policy/Broadcast.java
===================================================================
--- trunk/src/org/globus/swift/data/policy/Broadcast.java 2010-12-06 19:36:57 UTC (rev 3736)
+++ trunk/src/org/globus/swift/data/policy/Broadcast.java 2010-12-06 21:22:28 UTC (rev 3737)
@@ -4,10 +4,14 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+
+import org.apache.log4j.Logger;
import org.globus.swift.data.Director;
public class Broadcast extends Policy {
+ static Logger logger = Logger.getLogger(Broadcast.class);
+
String destination = null;
@Override
@@ -25,7 +29,7 @@
*/
public static void perform(Map> batch) {
String[] line = commandLine(batch);
- System.out.println("Broadcast.perform(): " + Arrays.toString(line));
+ logger.debug("arguments: " + Arrays.toString(line));
Process process = null;
try {
process = Runtime.getRuntime().exec(line);
@@ -33,7 +37,8 @@
}
catch (Exception e) {
e.printStackTrace();
- throw new RuntimeException("Could not launch external broadcast");
+ throw new RuntimeException
+ ("Could not launch external broadcast", e);
}
int code = process.exitValue();
if (code != 0)
@@ -47,6 +52,11 @@
String home = System.getProperties().getProperty("swift.home");
List line = new ArrayList();
line.add(home+"/libexec/cdm_broadcast.sh");
+ line.add(Director.broadcastMode);
+ if (logger.isDebugEnabled())
+ line.add(Director.logfile);
+ else
+ line.add("/dev/null");
for (Map.Entry> entry : batch.entrySet()) {
line.add("-l");
String location = entry.getKey();
@@ -54,7 +64,7 @@
line.add(location);
for (String file : files) {
line.add(file);
- line.add(getDestination(file)+"/"+file);
+ line.add(getDestination(file));
}
}
String[] result = new String[line.size()];
Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-12-06 19:36:57 UTC (rev 3736)
+++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-12-06 21:22:28 UTC (rev 3737)
@@ -35,7 +35,6 @@
import org.globus.cog.karajan.workflow.PrintStreamChannel;
import org.globus.cog.karajan.workflow.nodes.FlowElement;
import org.globus.cog.karajan.workflow.nodes.grid.AbstractGridNode;
-import org.globus.cog.karajan.workflow.service.channels.AbstractKarajanChannel;
import org.globus.cog.util.ArgumentParser;
import org.globus.cog.util.ArgumentParserException;
import org.globus.swift.data.Director;
@@ -93,9 +92,6 @@
if (ap.isPresent(ARG_MONITOR)) {
new Monitor().start();
}
- if (ap.isPresent(ARG_CDMFILE)) {
- loadCDM(ap);
- }
if (!ap.hasValue(ArgumentParser.DEFAULT)) {
error("No SwiftScript program specified");
}
@@ -130,6 +126,10 @@
setupLogging(ap, projectName, runID);
logger.debug("Max heap: " + Runtime.getRuntime().maxMemory());
+ if (ap.isPresent(ARG_CDMFILE)) {
+ loadCDM(ap);
+ }
+
if (!(new File(project).exists())) {
logger.error("Input file " + project + " does not exist.");
System.exit(4);
@@ -454,6 +454,9 @@
logfile = projectName + "-" + runID + ".log";
}
+ VDL2Config config = VDL2Config.getConfig();
+ config.put("logfile", logfile);
+
File f = new File(logfile);
FileAppender fa = (FileAppender) getAppender(FileAppender.class);
Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2010-12-06 19:36:57 UTC (rev 3736)
+++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2010-12-06 21:22:28 UTC (rev 3737)
@@ -95,6 +95,8 @@
put("status.mode", "files");
put("wrapper.parameter.mode", "args");
put("wrapper.invocation.mode", "absolute");
+
+ put("cdm.broadcast.mode", "file");
}
private VDL2Config(VDL2Config other) {
From noreply at svn.ci.uchicago.edu Mon Dec 6 15:24:40 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 15:24:40 -0600 (CST)
Subject: [Swift-commit] r3738 - trunk/tests/cdm
Message-ID: <20101206212440.46FBE9CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 15:24:40 -0600 (Mon, 06 Dec 2010)
New Revision: 3738
Modified:
trunk/tests/cdm/README.txt
Log:
Add GATHER note
Modified: trunk/tests/cdm/README.txt
===================================================================
--- trunk/tests/cdm/README.txt 2010-12-06 21:22:28 UTC (rev 3737)
+++ trunk/tests/cdm/README.txt 2010-12-06 21:24:40 UTC (rev 3738)
@@ -3,3 +3,4 @@
200 series : DIRECT tests
300 series : BROADCAST tests
400 series : EXTERNAL tests
+500 series : GATHER tests
From noreply at svn.ci.uchicago.edu Mon Dec 6 15:25:25 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 15:25:25 -0600 (CST)
Subject: [Swift-commit] r3739 - trunk/tests/cdm
Message-ID: <20101206212525.D8B359CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 15:25:25 -0600 (Mon, 06 Dec 2010)
New Revision: 3739
Added:
trunk/tests/cdm/301-bcast.check.sh
trunk/tests/cdm/301-bcast.clean.sh
trunk/tests/cdm/301-bcast.setup.sh
trunk/tests/cdm/301-bcast.swift
Modified:
trunk/tests/cdm/
trunk/tests/cdm/fs.template.data
Log:
New CDM BROADCAST test
Property changes on: trunk/tests/cdm
___________________________________________________________________
Name: svn:ignore
+ fs.data
Added: trunk/tests/cdm/301-bcast.check.sh
===================================================================
--- trunk/tests/cdm/301-bcast.check.sh (rev 0)
+++ trunk/tests/cdm/301-bcast.check.sh 2010-12-06 21:25:25 UTC (rev 3739)
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -x
+
+grep $( uname -m ) 301/301-input.txt || exit 1
+grep $( uname -m ) 301-output.txt || exit 1
+
+exit 0
Property changes on: trunk/tests/cdm/301-bcast.check.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/301-bcast.clean.sh
===================================================================
--- trunk/tests/cdm/301-bcast.clean.sh (rev 0)
+++ trunk/tests/cdm/301-bcast.clean.sh 2010-12-06 21:25:25 UTC (rev 3739)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -x
+
+rm -rv 301-output.txt 301
+
+exit 0
Property changes on: trunk/tests/cdm/301-bcast.clean.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/301-bcast.setup.sh
===================================================================
--- trunk/tests/cdm/301-bcast.setup.sh (rev 0)
+++ trunk/tests/cdm/301-bcast.setup.sh 2010-12-06 21:25:25 UTC (rev 3739)
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -x
+
+mkdir -pv 301 || exit 1
+
+{
+ uname -a
+ date
+} > 301-input.txt
+
+exit 0
Property changes on: trunk/tests/cdm/301-bcast.setup.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/301-bcast.swift
===================================================================
--- trunk/tests/cdm/301-bcast.swift (rev 0)
+++ trunk/tests/cdm/301-bcast.swift 2010-12-06 21:25:25 UTC (rev 3739)
@@ -0,0 +1,14 @@
+
+type file;
+
+app (file o) copy (file i)
+{
+ cp @i @o;
+}
+
+// Will be transformed by CDM to 201/output.txt:
+file f1<"301-input.txt">;
+
+file f2<"301-output.txt">;
+
+f2 = copy(f1);
Modified: trunk/tests/cdm/fs.template.data
===================================================================
--- trunk/tests/cdm/fs.template.data 2010-12-06 21:24:40 UTC (rev 3738)
+++ trunk/tests/cdm/fs.template.data 2010-12-06 21:25:25 UTC (rev 3739)
@@ -1,2 +1,9 @@
+
+property GATHER_LIMIT 50000 # ~50KB
+property GATHER_DIR /tmp/cdm_gather
+property GATHER_TARGET _PWD_/501-output
+
rule .*201-output.txt DIRECT _PWD_/201
rule .*210-input.txt EXTERNAL _PWD_/external.sh
+rule .*301-input.txt BROADCAST _PWD_/301
+rule .*501-output.*.txt GATHER
From noreply at svn.ci.uchicago.edu Mon Dec 6 15:28:01 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 15:28:01 -0600 (CST)
Subject: [Swift-commit] r3740 - trunk/tests/cdm
Message-ID: <20101206212801.44FF29CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 15:28:00 -0600 (Mon, 06 Dec 2010)
New Revision: 3740
Modified:
trunk/tests/cdm/301-bcast.swift
Log:
Fix comment
Modified: trunk/tests/cdm/301-bcast.swift
===================================================================
--- trunk/tests/cdm/301-bcast.swift 2010-12-06 21:25:25 UTC (rev 3739)
+++ trunk/tests/cdm/301-bcast.swift 2010-12-06 21:28:00 UTC (rev 3740)
@@ -6,7 +6,7 @@
cp @i @o;
}
-// Will be transformed by CDM to 201/output.txt:
+// Will be transformed by CDM to 301/301-input.txt:
file f1<"301-input.txt">;
file f2<"301-output.txt">;
From noreply at svn.ci.uchicago.edu Mon Dec 6 15:52:18 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 15:52:18 -0600 (CST)
Subject: [Swift-commit] r3741 - trunk/src/org/globus/swift/data/policy
Message-ID: <20101206215218.1617A9CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 15:52:17 -0600 (Mon, 06 Dec 2010)
New Revision: 3741
Modified:
trunk/src/org/globus/swift/data/policy/Local.java
Log:
Change the toString() name of CDM LOCAL from "PULL" to "LOCAL"
Modified: trunk/src/org/globus/swift/data/policy/Local.java
===================================================================
--- trunk/src/org/globus/swift/data/policy/Local.java 2010-12-06 21:28:00 UTC (rev 3740)
+++ trunk/src/org/globus/swift/data/policy/Local.java 2010-12-06 21:52:17 UTC (rev 3741)
@@ -5,7 +5,7 @@
public class Local extends Policy {
public String toString() {
- return "PULL";
+ return "LOCAL";
}
@Override
From noreply at svn.ci.uchicago.edu Mon Dec 6 15:53:10 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 15:53:10 -0600 (CST)
Subject: [Swift-commit] r3742 - trunk/tests/cdm
Message-ID: <20101206215310.801B89CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 15:53:10 -0600 (Mon, 06 Dec 2010)
New Revision: 3742
Added:
trunk/tests/cdm/601-local.check.sh
trunk/tests/cdm/601-local.clean.sh
trunk/tests/cdm/601-local.setup.sh
trunk/tests/cdm/601-local.swift
Modified:
trunk/tests/cdm/README.txt
trunk/tests/cdm/fs.template.data
Log:
Add new CDM LOCAL test (cp)
Added: trunk/tests/cdm/601-local.check.sh
===================================================================
--- trunk/tests/cdm/601-local.check.sh (rev 0)
+++ trunk/tests/cdm/601-local.check.sh 2010-12-06 21:53:10 UTC (rev 3742)
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -x
+
+grep $( uname -m ) 601/601-input.txt || exit 1
+grep $( uname -m ) 601-output.txt || exit 1
+
+exit 0
Property changes on: trunk/tests/cdm/601-local.check.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/601-local.clean.sh
===================================================================
--- trunk/tests/cdm/601-local.clean.sh (rev 0)
+++ trunk/tests/cdm/601-local.clean.sh 2010-12-06 21:53:10 UTC (rev 3742)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -x
+
+rm -rv 601-output.txt 601
+
+exit 0
Property changes on: trunk/tests/cdm/601-local.clean.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/601-local.setup.sh
===================================================================
--- trunk/tests/cdm/601-local.setup.sh (rev 0)
+++ trunk/tests/cdm/601-local.setup.sh 2010-12-06 21:53:10 UTC (rev 3742)
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -x
+
+mkdir -pv 601 || exit 1
+
+{
+ uname -a
+ date
+} > 601/601-input.txt
+
+exit 0
Property changes on: trunk/tests/cdm/601-local.setup.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/601-local.swift
===================================================================
--- trunk/tests/cdm/601-local.swift (rev 0)
+++ trunk/tests/cdm/601-local.swift 2010-12-06 21:53:10 UTC (rev 3742)
@@ -0,0 +1,14 @@
+
+type file;
+
+app (file o) copy (file i)
+{
+ cp @i @o;
+}
+
+// Will be transformed by CDM to 601/601-input.txt:
+file f1<"601-input.txt">;
+
+file f2<"601-output.txt">;
+
+f2 = copy(f1);
Modified: trunk/tests/cdm/README.txt
===================================================================
--- trunk/tests/cdm/README.txt 2010-12-06 21:52:17 UTC (rev 3741)
+++ trunk/tests/cdm/README.txt 2010-12-06 21:53:10 UTC (rev 3742)
@@ -4,3 +4,4 @@
300 series : BROADCAST tests
400 series : EXTERNAL tests
500 series : GATHER tests
+600 series : LOCAL tests
Modified: trunk/tests/cdm/fs.template.data
===================================================================
--- trunk/tests/cdm/fs.template.data 2010-12-06 21:52:17 UTC (rev 3741)
+++ trunk/tests/cdm/fs.template.data 2010-12-06 21:53:10 UTC (rev 3742)
@@ -7,3 +7,4 @@
rule .*210-input.txt EXTERNAL _PWD_/external.sh
rule .*301-input.txt BROADCAST _PWD_/301
rule .*501-output.*.txt GATHER
+rule .*601-input.*.txt LOCAL cp _PWD_/601
From noreply at svn.ci.uchicago.edu Mon Dec 6 16:02:58 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 16:02:58 -0600 (CST)
Subject: [Swift-commit] r3743 - trunk/tests/cdm
Message-ID: <20101206220258.37E2C9CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 16:02:58 -0600 (Mon, 06 Dec 2010)
New Revision: 3743
Added:
trunk/tests/cdm/602-local.check.sh
trunk/tests/cdm/602-local.clean.sh
trunk/tests/cdm/602-local.setup.sh
trunk/tests/cdm/602-local.swift
Modified:
trunk/tests/cdm/fs.template.data
Log:
New CDM LOCAL test with dd
Added: trunk/tests/cdm/602-local.check.sh
===================================================================
--- trunk/tests/cdm/602-local.check.sh (rev 0)
+++ trunk/tests/cdm/602-local.check.sh 2010-12-06 22:02:58 UTC (rev 3743)
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -x
+
+grep $( uname -m ) 602/602-input.txt || exit 1
+grep $( uname -m ) 602-output.txt || exit 1
+
+exit 0
Property changes on: trunk/tests/cdm/602-local.check.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/602-local.clean.sh
===================================================================
--- trunk/tests/cdm/602-local.clean.sh (rev 0)
+++ trunk/tests/cdm/602-local.clean.sh 2010-12-06 22:02:58 UTC (rev 3743)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -x
+
+rm -rv 602-output.txt 602
+
+exit 0
Property changes on: trunk/tests/cdm/602-local.clean.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/602-local.setup.sh
===================================================================
--- trunk/tests/cdm/602-local.setup.sh (rev 0)
+++ trunk/tests/cdm/602-local.setup.sh 2010-12-06 22:02:58 UTC (rev 3743)
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -x
+
+mkdir -pv 602 || exit 1
+
+{
+ uname -a
+ date
+} > 602/602-input.txt
+
+exit 0
Property changes on: trunk/tests/cdm/602-local.setup.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/cdm/602-local.swift
===================================================================
--- trunk/tests/cdm/602-local.swift (rev 0)
+++ trunk/tests/cdm/602-local.swift 2010-12-06 22:02:58 UTC (rev 3743)
@@ -0,0 +1,14 @@
+
+type file;
+
+app (file o) copy (file i)
+{
+ cp @i @o;
+}
+
+// Will be transformed by CDM to 602/602-input.txt:
+file f1<"602-input.txt">;
+
+file f2<"602-output.txt">;
+
+f2 = copy(f1);
Modified: trunk/tests/cdm/fs.template.data
===================================================================
--- trunk/tests/cdm/fs.template.data 2010-12-06 21:53:10 UTC (rev 3742)
+++ trunk/tests/cdm/fs.template.data 2010-12-06 22:02:58 UTC (rev 3743)
@@ -8,3 +8,4 @@
rule .*301-input.txt BROADCAST _PWD_/301
rule .*501-output.*.txt GATHER
rule .*601-input.*.txt LOCAL cp _PWD_/601
+rule .*602-input.*.txt LOCAL dd _PWD_/602
From noreply at svn.ci.uchicago.edu Mon Dec 6 16:04:56 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Dec 2010 16:04:56 -0600 (CST)
Subject: [Swift-commit] r3744 - trunk/tests/cdm
Message-ID: <20101206220456.08F2D9CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-06 16:04:55 -0600 (Mon, 06 Dec 2010)
New Revision: 3744
Added:
trunk/tests/cdm/601-local-cp.check.sh
trunk/tests/cdm/601-local-cp.clean.sh
trunk/tests/cdm/601-local-cp.setup.sh
trunk/tests/cdm/601-local-cp.swift
trunk/tests/cdm/602-local-dd.check.sh
trunk/tests/cdm/602-local-dd.clean.sh
trunk/tests/cdm/602-local-dd.setup.sh
trunk/tests/cdm/602-local-dd.swift
Removed:
trunk/tests/cdm/601-local.check.sh
trunk/tests/cdm/601-local.clean.sh
trunk/tests/cdm/601-local.setup.sh
trunk/tests/cdm/601-local.swift
trunk/tests/cdm/602-local.check.sh
trunk/tests/cdm/602-local.clean.sh
trunk/tests/cdm/602-local.setup.sh
trunk/tests/cdm/602-local.swift
Log:
Put cp/dd in the test name
Copied: trunk/tests/cdm/601-local-cp.check.sh (from rev 3742, trunk/tests/cdm/601-local.check.sh)
===================================================================
--- trunk/tests/cdm/601-local-cp.check.sh (rev 0)
+++ trunk/tests/cdm/601-local-cp.check.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -x
+
+grep $( uname -m ) 601/601-input.txt || exit 1
+grep $( uname -m ) 601-output.txt || exit 1
+
+exit 0
Copied: trunk/tests/cdm/601-local-cp.clean.sh (from rev 3742, trunk/tests/cdm/601-local.clean.sh)
===================================================================
--- trunk/tests/cdm/601-local-cp.clean.sh (rev 0)
+++ trunk/tests/cdm/601-local-cp.clean.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -x
+
+rm -rv 601-output.txt 601
+
+exit 0
Copied: trunk/tests/cdm/601-local-cp.setup.sh (from rev 3742, trunk/tests/cdm/601-local.setup.sh)
===================================================================
--- trunk/tests/cdm/601-local-cp.setup.sh (rev 0)
+++ trunk/tests/cdm/601-local-cp.setup.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -x
+
+mkdir -pv 601 || exit 1
+
+{
+ uname -a
+ date
+} > 601/601-input.txt
+
+exit 0
Copied: trunk/tests/cdm/601-local-cp.swift (from rev 3742, trunk/tests/cdm/601-local.swift)
===================================================================
--- trunk/tests/cdm/601-local-cp.swift (rev 0)
+++ trunk/tests/cdm/601-local-cp.swift 2010-12-06 22:04:55 UTC (rev 3744)
@@ -0,0 +1,14 @@
+
+type file;
+
+app (file o) copy (file i)
+{
+ cp @i @o;
+}
+
+// Will be transformed by CDM to 601/601-input.txt:
+file f1<"601-input.txt">;
+
+file f2<"601-output.txt">;
+
+f2 = copy(f1);
Deleted: trunk/tests/cdm/601-local.check.sh
===================================================================
--- trunk/tests/cdm/601-local.check.sh 2010-12-06 22:02:58 UTC (rev 3743)
+++ trunk/tests/cdm/601-local.check.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-set -x
-
-grep $( uname -m ) 601/601-input.txt || exit 1
-grep $( uname -m ) 601-output.txt || exit 1
-
-exit 0
Deleted: trunk/tests/cdm/601-local.clean.sh
===================================================================
--- trunk/tests/cdm/601-local.clean.sh 2010-12-06 22:02:58 UTC (rev 3743)
+++ trunk/tests/cdm/601-local.clean.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-set -x
-
-rm -rv 601-output.txt 601
-
-exit 0
Deleted: trunk/tests/cdm/601-local.setup.sh
===================================================================
--- trunk/tests/cdm/601-local.setup.sh 2010-12-06 22:02:58 UTC (rev 3743)
+++ trunk/tests/cdm/601-local.setup.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -1,12 +0,0 @@
-#!/bin/sh
-
-set -x
-
-mkdir -pv 601 || exit 1
-
-{
- uname -a
- date
-} > 601/601-input.txt
-
-exit 0
Deleted: trunk/tests/cdm/601-local.swift
===================================================================
--- trunk/tests/cdm/601-local.swift 2010-12-06 22:02:58 UTC (rev 3743)
+++ trunk/tests/cdm/601-local.swift 2010-12-06 22:04:55 UTC (rev 3744)
@@ -1,14 +0,0 @@
-
-type file;
-
-app (file o) copy (file i)
-{
- cp @i @o;
-}
-
-// Will be transformed by CDM to 601/601-input.txt:
-file f1<"601-input.txt">;
-
-file f2<"601-output.txt">;
-
-f2 = copy(f1);
Copied: trunk/tests/cdm/602-local-dd.check.sh (from rev 3743, trunk/tests/cdm/602-local.check.sh)
===================================================================
--- trunk/tests/cdm/602-local-dd.check.sh (rev 0)
+++ trunk/tests/cdm/602-local-dd.check.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -x
+
+grep $( uname -m ) 602/602-input.txt || exit 1
+grep $( uname -m ) 602-output.txt || exit 1
+
+exit 0
Copied: trunk/tests/cdm/602-local-dd.clean.sh (from rev 3743, trunk/tests/cdm/602-local.clean.sh)
===================================================================
--- trunk/tests/cdm/602-local-dd.clean.sh (rev 0)
+++ trunk/tests/cdm/602-local-dd.clean.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -x
+
+rm -rv 602-output.txt 602
+
+exit 0
Copied: trunk/tests/cdm/602-local-dd.setup.sh (from rev 3743, trunk/tests/cdm/602-local.setup.sh)
===================================================================
--- trunk/tests/cdm/602-local-dd.setup.sh (rev 0)
+++ trunk/tests/cdm/602-local-dd.setup.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -x
+
+mkdir -pv 602 || exit 1
+
+{
+ uname -a
+ date
+} > 602/602-input.txt
+
+exit 0
Copied: trunk/tests/cdm/602-local-dd.swift (from rev 3743, trunk/tests/cdm/602-local.swift)
===================================================================
--- trunk/tests/cdm/602-local-dd.swift (rev 0)
+++ trunk/tests/cdm/602-local-dd.swift 2010-12-06 22:04:55 UTC (rev 3744)
@@ -0,0 +1,14 @@
+
+type file;
+
+app (file o) copy (file i)
+{
+ cp @i @o;
+}
+
+// Will be transformed by CDM to 602/602-input.txt:
+file f1<"602-input.txt">;
+
+file f2<"602-output.txt">;
+
+f2 = copy(f1);
Deleted: trunk/tests/cdm/602-local.check.sh
===================================================================
--- trunk/tests/cdm/602-local.check.sh 2010-12-06 22:02:58 UTC (rev 3743)
+++ trunk/tests/cdm/602-local.check.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-set -x
-
-grep $( uname -m ) 602/602-input.txt || exit 1
-grep $( uname -m ) 602-output.txt || exit 1
-
-exit 0
Deleted: trunk/tests/cdm/602-local.clean.sh
===================================================================
--- trunk/tests/cdm/602-local.clean.sh 2010-12-06 22:02:58 UTC (rev 3743)
+++ trunk/tests/cdm/602-local.clean.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-set -x
-
-rm -rv 602-output.txt 602
-
-exit 0
Deleted: trunk/tests/cdm/602-local.setup.sh
===================================================================
--- trunk/tests/cdm/602-local.setup.sh 2010-12-06 22:02:58 UTC (rev 3743)
+++ trunk/tests/cdm/602-local.setup.sh 2010-12-06 22:04:55 UTC (rev 3744)
@@ -1,12 +0,0 @@
-#!/bin/sh
-
-set -x
-
-mkdir -pv 602 || exit 1
-
-{
- uname -a
- date
-} > 602/602-input.txt
-
-exit 0
Deleted: trunk/tests/cdm/602-local.swift
===================================================================
--- trunk/tests/cdm/602-local.swift 2010-12-06 22:02:58 UTC (rev 3743)
+++ trunk/tests/cdm/602-local.swift 2010-12-06 22:04:55 UTC (rev 3744)
@@ -1,14 +0,0 @@
-
-type file;
-
-app (file o) copy (file i)
-{
- cp @i @o;
-}
-
-// Will be transformed by CDM to 602/602-input.txt:
-file f1<"602-input.txt">;
-
-file f2<"602-output.txt">;
-
-f2 = copy(f1);
From noreply at svn.ci.uchicago.edu Tue Dec 7 19:21:35 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 7 Dec 2010 19:21:35 -0600 (CST)
Subject: [Swift-commit] r3745 - text/parco10submission
Message-ID: <20101208012135.E77DA9CC7F@svn.ci.uchicago.edu>
Author: wilde
Date: 2010-12-07 19:21:35 -0600 (Tue, 07 Dec 2010)
New Revision: 3745
Added:
text/parco10submission/ResponseToReviews.txt
Modified:
text/parco10submission/paper.tex
Log:
minor initial edits after PARCO review
Added: text/parco10submission/ResponseToReviews.txt
===================================================================
--- text/parco10submission/ResponseToReviews.txt (rev 0)
+++ text/parco10submission/ResponseToReviews.txt 2010-12-08 01:21:35 UTC (rev 3745)
@@ -0,0 +1,287 @@
+
+---------- Forwarded message ----------
+Date: Thu, 14 Oct 2010 13:29:09
+From: Parallel Computing
+To: wozniak at mcs.anl.gov
+Cc: worleyph at ornl.gov, Rupak.Biswas at nasa.gov, Rajesh.Nishtala at gmail.com,
+ loliker at lbl.gov
+Subject: Your Submission PARCO-D-10-00054
+
+Ms. Ref. No.: PARCO-D-10-00054
+Title: Swift: A language for distributed parallel scripting
+Parallel Computing
+
+Dear Justin,
+
+Reviewers have now commented on your paper. You will see that they are advising that you revise your manuscript. If you are prepared to undertake the work required, I would be pleased to reconsider my decision.
+
+For your guidance, reviewers' comments are appended below.
+
+If you decide to revise the work, please submit a list of changes or a rebuttal against each point which is being raised when you submit the revised manuscript.
+
+Your revision should be submitted before Nov 11, 2010.
+
+To submit a revision, please go to http://ees.elsevier.com/parco/ and login as an Author. On your Main Menu page is a folder entitled "Submissions Needing Revision". You will find your submission record there.
+
+When submitting your revised manuscript, please ensure that you upload the source files (e.g. (La)TeX or Word; (La)TeX files as 1 comprehensive file and each figure and table separately). Uploading a PDF file at this stage will create delays should your manuscript be finally accepted for publication. If your revised submission does not include the source files, we will contact you to request them.
+
+Yours sincerely,
+
+Patrick Haven Worley, Ph.D.
+Associate Editor
+Parallel Computing
+
+**********************************************************
+
+Reviewers' comments:
+
+Dear Authors,
+ Thank you for your submission to the Parallel Computing special issue on Emerging Programming Paradigms for Large-Scale Scientific Computing. As there were several concerns about the manuscript, we ask that you update your submission to address the reviewer feedback, and include a point by point response to the reviewer comments.
+
+With kind regards,
+ParCo Guest Editors:
+Rupak Biswas, Rajesh Nishtala, Lenny Oliker
+
+
+****************************************************************************************
+
+Reviewer #1: The paper describes Swift, a scripting language for distributed parallel applications. I have mixed feelings about this paper. I can see the usefulness of a scripting language like Swift. The problems Swift was designed to address are certainly interesting and important.
+
+On the other hand, I don't see much scientific merit in the paper. The paper reads more like a Swift user manual than a scientific paper. For the language design, the only thing that might be novel is the notion of mapped type, but I consider it to be quite minor. I also don't see any new ideas in the data-flow dependency based execution model.
+
+>>> Novelty and scientific merit:
+
+<<<
+
+For the distributed execution, one important missing piece is performance evaluation. Data locality is very important for data-intensive applications. As I understand it, data have to be moved in and out the clusters. So, understanding the cost of scheduling and data transfer is very important to validate the Swift design. Perhaps, it was
+published somewhere else, but it would be nice to discuss it in this paper. Here are some more detailed comments:
+
+>>> Performance evaluation:
+
+<<<
+
+1. Swift uses restart log to reuse the results of successfully completed components. The paper mentioned "appropriate manual intervention". This seems to be something you can almost completely automate. Based on my experiences with large-scale and long running applications, this can be very useful.
+
+>>> Automation of restart
+
+<<<
+
+2. Swift reduces job submission cost using clustering. It is not clear to me if a subgraph can be batched together by your clustering technique. This obviously requires a little bit of analysis of the data-flow graph to do it properly. But it could be quite useful to achieve better data locality.
+
+>>> Clustering
+
+<<<
+
+3. In terms of programming models, modern systems such as Microsoft's DryadLINQ and Google's FlumeJava successfully integrate data-flow constructs into state of the art programming languages (C# and Java). This integration approach is quite nice and powerful. It would be nice if the authors can compare Swift with these two systems.
+
+>>> Comparison to Dryad and FlumeJava
+
+<<<
+
+Reviewer #2: The paper presents a powerful high-level scripting language, SwiftScript, for performing a massive number of tasks/jobs coupled with collections of file-based data. It describes details of the language syntax and semantics, which is based on data flow with support of arrays and procedures, and discusses the implementation and several use cases with the main focus on a grid. Although a similar work was published before, this paper gives elaborated summary of the technical details, which is useful for general audience.
+
+The current work is an attempt to overcome issues in dealing with a massive amount of simulations and data either on a grid or on a massive parallel system, such as large data files, various dependencies, fault tolerance. The framework simplifies the process in the scripting design by an "app" interface to the actual command line for program execution and a "mapper" to map data structures with actual files without the need of going into details of how the program actually gets executed and without explicitly specifying the types of the files involved. The paper also describes the implementation that is built on top of the CoG Karajan workflow engine with built-in fault tolerance and demonstrates the usefulness of the system with a number of examples.
+
+The primary focus of the work is for the grid applications, although the paper has indicated the applicability to other systems, such as more tightly coupled massive computing systems.
+
+For those who might not be familiar with the Karajan language, it would be useful to add a reference to the related work.
+
+>>> Reference to Karajan: <<<
+
+It would be helpful to include some discussion on the "auto-parallelization" capability (achieved via data flow analysis?).
+
+>>> auto-parallelization
+
+<<<
+
+Out of the four application examples presented, two of the cases (4.3 and 4.4) do not contain enough details to support the discussion; deleting the two examples should not affect the clarity of the paper.
+
+>>> Examples: clarify or delete
+
+<<<
+
+It would be helpful to elaborate more in example 4.2 on how each task/job gets scheduled onto Ranger nodes or how Swift interacts with the local batch job scheduler, which would in turn help audience understand better how SwiftScript could be used for a certain class of applications on a more tightly coupled massive computing environment (such as
+parameter studies).
+
+>>> 4.2 - task scheduling on Ranger
+
+<<<
+
+There are a few minor typos in the manuscript.
+For instance, on Page 7 section 2.2, in the code snippet:
+
+ output[ix] = rotate(frames, 180);
+
+Should "frames" be "f" in this case?
+
+>>> typo
+
+<<<
+
+Reviewer #3: This is an interesting paper aimed at the practical problem of
+managing a large ensemble of computations with possible dependencies
+among the tasks. The general outline and structure of the paper is
+fine. There are a number of small errors which should have been
+caught by proofreading the manuscript.
+
+>>> typos
+
+<<<
+
+The most substantive comment I have concerns examples 4.3 and 4.4. I
+believe that, as they stand, these are too sparsely annotated to add
+significant value to the paper. So I suggest either documenting these
+examples more carefully, so that a reader new to Swift can understand
+them, or else omit these examples entirely.
+
+>>> examples 4.3 and 4.4: annotate or delete
+
+<<<
+
+Further comments:
+
+1. Sometimes "Swift" is used and sometimes "SwiftScript" is used. Is
+there any reason to have two different terms? From the website it
+appears that "SwiftScript" referred to an earlier version of the
+language, so perhaps "SwiftScript" should be replaced by "Swift"
+everywhere.
+
+>>> <<<
+
+2. It's a bit awkward that "single assignment" is used in section 2.1
+but not defined until section 2.3.
+
+>>> <<<
+
+3. The example on p. 5 appears erroneous, it seems the rotate
+procedure should have an angle input in addition to the image input
+(this is corrected on p.6).
+
+>>> <<<
+
+4. In section 2.2, should rotate be invoked as "rotate(f, 180)" instead
+of "rotate(frames, 180)"?
+
+>>> <<<
+
+5. There are some acronyms in the paper that should, I believe, be
+defined. In some cases the acronym can be omitted, for example I
+believe RDBMS is only used once so there is really no need for the
+acronym. Other acronyms appear in Figure 1: CoG, OSG, AWS; these
+probably should be defined. Other acronyms: GRAM, fMRI (and FMRI,
+cf. Fig. 2, which should probably be fMRI).
+
+>>> <<<
+
+6. All these appear: "stage in", "stage-in", "stagein". Please be
+consistent (similarly for stage out).
+
+>>> <<<
+
+7. The mysterious numbers '12, 1000, 1000, "81 3 3"' in example 4.1
+might merit an explanation.
+
+>>> <<<
+
+8. The Figure numbering in the text of section 4.1 needs correction.
+
+>>> <<<
+
+9. In the text of section 4.1, I believe the variables or, iv,
+direction, overwrite, and ov should be in a different font for clarity.
+In the text of section 4.1 "n x n" should be in a different font and
+"n2" is clearly wrong.
+
+>>> <<<
+
+10. Section 4.2, what is an "emblem experiment"?
+
+>>> <<<
+
+11. The margins in sections 4.2 and 4.4 need fixing as some lines
+run completely off the page.
+
+>>> <<<
+
+12. "Karajan" is mentioned several times, there really should be short
+definition of it and a reference to it in the bibliography.
+
+>>> <<<
+
+13. Many of the references look incomplete; journal references really
+should have page numbers, some references are missing a year.
+Reference 8 severely mangles "Bresnahan".
+
+>>> <<<
+
+14. It is somewhat disingenuous to refer to thousands and millions of
+cores (section 6.2). All systems I know of are managed as nodes,
+where each node might have 8 or 12 or 16 cores. This is no essential
+simplification, I know, but why not be honest and refer to nodes
+instead of cores?
+
+>>> <<<
+
+15. "Swift is implemented by compiling to a Karajan program". Why
+would one want to _compile_ a scripting language? It seems more natural
+(to this naive reader) to have an interpreter or a translator.
+
+>>> <<<
+
+16. The coaster idea looks quite interesting, could this be expanded,
+or could an example with coasters be constructed?
+
+>>> <<<
+
+17. Table 1, 1st row, 3rd column: should it be f->data.txt instead of
+f->file.txt?
+
+>>> <<<
+
+18. There are many (too many to list) typos, missing words, mistakes
+such as "en queued" instead of "enqueued", subject/verb mismatches of
+number and/or tense. A careful proofreading is sorely needed.
+
+
+>>> <<<
+
+==== Other improvement notes
+
+mention futures in parco paper, show them visually to show fine grain
+
+mention habanero (c and java) and other fresh stack languages (x10)
+compare to GEL - from SIngapore
+
+mention csp bsp sim and diff to mpi (IPO)
+
+Why a new model?
+
+examine determinism
+
+examine language vs library
+
+examine how it builds on karajan
+
+
+---
+
+Innov: fine grained parallelism; no need for flow analysis;
+sep of concerns: how throttling and site mgmt are isolated
+
+How we can manage data locality
+
+How restart is more transparent than it sounds here
+
+Fine: how work takes off before a proc returns
+
+Add table of critical benchmarks on multi sys types
+
+How complex flows are easily composed
+
+How types and mappers encapsulate complexity
+
+2.3 order of exec: show more complex patterns here or later
+
+Second 2 is the part that reads like a LRM; make it more interesting
+
+2.5 don't say marker types
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-06 22:04:55 UTC (rev 3744)
+++ text/parco10submission/paper.tex 2010-12-08 01:21:35 UTC (rev 3745)
@@ -734,13 +734,13 @@
multiple sites, it is necessary to choose between the available sites.
The Swift \emph{site selector} achieves this by maintaining a score for
each site which determines the load that Swift will place on that site.
-As a site is successful in executing jobs, this score will be increased
-and as the site is uncsuccessful, this score will be decreased. In
-addition to selecting between sites, this mechanism provides some
+As a site succeeds in executing jobs, this score is increased
+and as job executions at a site fail, this score is decreased. In
+addition to selecting between sites, this mechanism provides a measure of
dynamic rate limiting if sites fail due to overload~\cite{FTSH_2003}.
This provides an empirically measured estimate of a site's ability to
-bear load, distinct from more static information elsewhere published.
+bear load, distinct from more static information published elsewhere.
In part, this is due to unreliable and incomplete published information
(for example, site policies restricting (for example) job counts are
often not published in usable form); in addition, that site's ability
@@ -774,8 +774,11 @@
will fail resulting ultimately in the entire script failing.
In such a case, Swift provides a \emph{restart log} which encapsulates
-which procedure invocations have been successfully completed. After
-appropriate manual intervention, a subsequent Swift run may be started
+which procedure invocations have been successfully completed.
+%%%%%% What manual interv. and why???
+After
+appropriate manual intervention,
+a subsequent Swift run may be started
with this restart log; this will suppress re-execution of already
executed invocations but otherwise allow the script to continue.
From noreply at svn.ci.uchicago.edu Wed Dec 8 12:15:33 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 12:15:33 -0600 (CST)
Subject: [Swift-commit] r3746 - provenancedb
Message-ID: <20101208181533.3247F9CC80@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-08 12:15:33 -0600 (Wed, 08 Dec 2010)
New Revision: 3746
Modified:
provenancedb/import-run-to-sql
provenancedb/pql_functions.sql
provenancedb/prepare-provenance-chart
provenancedb/prov-to-sql.sh
provenancedb/swift-prov-import-all-logs
Log:
Minor fixes.
Modified: provenancedb/import-run-to-sql
===================================================================
--- provenancedb/import-run-to-sql 2010-12-08 01:21:35 UTC (rev 3745)
+++ provenancedb/import-run-to-sql 2010-12-08 18:15:33 UTC (rev 3746)
@@ -9,5 +9,5 @@
# with kickstart records expected to be in the same directory as the
# log file.
-PROVIDPREFIX=$PROVIDPREFIX prov-to-sql.sh $1
+version=$version prov-to-sql.sh $1
Modified: provenancedb/pql_functions.sql
===================================================================
--- provenancedb/pql_functions.sql 2010-12-08 01:21:35 UTC (rev 3745)
+++ provenancedb/pql_functions.sql 2010-12-08 18:15:33 UTC (rev 3746)
@@ -1,23 +1,32 @@
+-- Set of PosrgreSQL-specific SQL functions and PL/pgSQL procedures
+-- to query provenance.
+
-- SQL Functions
-CREATE OR REPLACE FUNCTION list_runs() RETURNS SETOF VARCHAR AS $$
- SELECT DISTINCT(log_filename) FROM workflow;
+-- list_runs_* lists workflows recorded in the database by id or log_filename
+DROP TYPE list_runs_type CASCADE;
+
+CREATE TYPE list_runs_type
+AS (id VARCHAR,
+ log_filename VARCHAR,
+ swift_version VARCHAR,
+ start_time TIMESTAMP WITH TIME ZONE,
+ duration NUMERIC,
+ final_state VARCHAR);
+CREATE OR REPLACE FUNCTION list_runs()
+RETURNS SETOF list_runs_type AS $$
+ SELECT id, log_filename, swift_version, TO_TIMESTAMP(start_time),
+ duration, import_status AS final_state
+ FROM workflow;
$$ LANGUAGE SQL;
--- lists distinct processes by name in a workflow
-CREATE OR REPLACE FUNCTION process_names(wf_id VARCHAR) RETURNS SETOF VARCHAR AS $$
- SELECT DISTINCT(process.name)
- FROM process
- WHERE process.workflow_id=$1;
-$$ LANGUAGE SQL;
-
-- lists variations in a parameter's value across workflows
-
DROP TYPE param_across_wf_type CASCADE;
CREATE TYPE param_across_wf_type AS (workflow VARCHAR, parameter VARCHAR, value VARCHAR);
-CREATE OR REPLACE FUNCTION param_across_wf(param_name VARCHAR) RETURNS SETOF param_across_wf_type AS $$
+CREATE OR REPLACE FUNCTION param_across_wf(param_name VARCHAR)
+RETURNS SETOF param_across_wf_type AS $$
SELECT workflow.log_filename,ds_usage.param_name,variable.value
FROM variable,ds_usage,process,workflow
WHERE variable.id=ds_usage.dataset_id AND ds_usage.process_id=process.id AND
@@ -25,6 +34,16 @@
GROUP BY workflow.log_filename,ds_usage.param_name,variable.value;
$$ LANGUAGE SQL;
+-- lists variations of the values of a set of parameters
+
+DROP TYPE param_across_wf_class_type CASCADE;
+CREATE TYPE param_across_wf_class_type AS (workflow VARCHAR, parameter VARCHAR, value VARCHAR);
+
+CREATE OR REPLACE FUNCTION param_across_wf_class(param_name VARCHAR, wf_class VARACHAR)
+RETURNS SETOF param_across_wf_type AS $$
+
+$$ LANGUAGE plpgsql;
+
-- correlate a parameter with workflow runtime statistics
DROP TYPE correlate_param_runtime_type CASCADE;
CREATE TYPE correlate_param_runtime_type
@@ -34,21 +53,41 @@
parameter VARCHAR,
parameter_value VARCHAR);
-CREATE OR REPLACE FUNCTION correlate_param_runtime(param_name VARCHAR) RETURNS SETOF correlate_param_runtime_type AS $$
+CREATE OR REPLACE FUNCTION correlate_param_runtime(param_name VARCHAR)
+RETURNS SETOF correlate_param_runtime_type AS $$
SELECT A.workflow,to_timestamp(B.start_time),B.duration,A.parameter,A.value
FROM param_across_wf($1) AS A, workflow AS B
WHERE A.workflow=B.log_filename;
$$ LANGUAGE SQL;
+CREATE OR REPLACE FUNCTION correlate_param_runtime(param_name VARCHAR)
+RETURNS SETOF correlate_param_runtime_type AS $$
+ SELECT workflow.id,to_timestamp(workflow.start_time),workflow.duration,ds_usage.param_name,variable.value
+ FROM variable,ds_usage,process,workflow
+ WHERE variable.id=ds_usage.dataset_id AND ds_usage.process_id=process.id AND
+ process.workflow_id=workflow.id AND ds_usage.param_name=$1
+$$ LANGUAGE SQL;
+
+-- lists distinct processes by name in a workflow
+CREATE OR REPLACE FUNCTION process_names(wf_id VARCHAR)
+RETURNS SETOF VARCHAR AS $$
+ SELECT DISTINCT(process.name)
+ FROM process
+ WHERE process.workflow_id=$1;
+$$ LANGUAGE SQL;
+
+-- lists distinct processes in a
+
-- OOPS-specific functions
+
+
CREATE OR REPLACE FUNCTION list_oops_runs() RETURNS SETOF VARCHAR AS $$
SELECT DISTINCT(value) FROM annot_wf_txt WHERE annot_wf_txt.name=('oops_run_id');
$$ LANGUAGE SQL;
DROP TYPE oops_param_across_wf_type CASCADE;
CREATE TYPE oops_param_across_wf_type AS (oops_run_id VARCHAR, param_name VARCHAR, variable VARCHAR);
-
CREATE OR REPLACE FUNCTION oops_param_across_wf(VARCHAR) RETURNS SETOF oops_param_across_wf_type AS $$
SELECT annot_wf_txt.value,ds_usage.param_name,variable.value
FROM variable,ds_usage,process,annot_wf_txt
@@ -63,7 +102,6 @@
DROP TYPE oops_summary CASCADE;
CREATE TYPE oops_summary AS (oops_run_id VARCHAR, start_time TIMESTAMP WITH TIME ZONE, duration_sec NUMERIC, swift_version VARCHAR);
-
CREATE OR REPLACE FUNCTION oops_run_summary(varchar) RETURNS SETOF oops_summary AS $$
SELECT annot_wf_txt.value as oops_run_id, to_timestamp(workflow.start_time) as start_time,
workflow.duration as duration_sec,workflow.swift_version as swift_version
@@ -80,16 +118,15 @@
DROP TYPE oops_wf_param_summary CASCADE;
CREATE TYPE oops_wf_param_summary AS (oops_run_id varchar, param_name varchar, value varchar);
-
CREATE OR REPLACE FUNCTION oops_variable_summary() RETURNS SETOF oops_wf_param_summary AS $$
SELECT annot_wf_txt.value,ds_usage.param_name,variable.value
FROM variable,ds_usage,process,annot_wf_txt
WHERE variable.id=ds_usage.dataset_id and ds_usage.process_id=process.id and process.workflow_id=annot_wf_txt.id;
$$ LANGUAGE SQL;
+
DROP TYPE oops_param_summary CASCADE;
CREATE TYPE oops_param_summary AS (param_name varchar, value varchar);
-
CREATE OR REPLACE FUNCTION oops_science_summary(varchar) RETURNS SETOF oops_param_summary AS $$
SELECT ds_usage.param_name,variable.value
FROM variable,ds_usage,process,annot_wf_txt
Modified: provenancedb/prepare-provenance-chart
===================================================================
--- provenancedb/prepare-provenance-chart 2010-12-08 01:21:35 UTC (rev 3745)
+++ provenancedb/prepare-provenance-chart 2010-12-08 18:15:33 UTC (rev 3746)
@@ -8,8 +8,8 @@
export RUNID=$(basename $1 .log)
-export WFID=$PROVIDPREFIX"execute:${RUNID}:"
-export EXECUTE2PREFIX=$PROVIDPREFIX"execute2:${RUNID}:"
+export WFID="execute:${RUNID}:"
+export EXECUTE2PREFIX="execute2:${RUNID}:"
# will output log information about datasets from a log file passed as $1
Modified: provenancedb/prov-to-sql.sh
===================================================================
--- provenancedb/prov-to-sql.sh 2010-12-08 01:21:35 UTC (rev 3745)
+++ provenancedb/prov-to-sql.sh 2010-12-08 18:15:33 UTC (rev 3746)
@@ -2,10 +2,10 @@
export RUNID=$(basename $1 .log)
-export WFID=$PROVIDPREFIX"execute:${RUNID}:"
+export WFID="execute:${RUNID}:"
# TODO is there already a URI form for identifying workflows?
-export WF=$PROVIDPREFIX"execute:${RUNID}:run"
+export WF="${RUNID}"
echo Generating SQL for $RUNID
@@ -27,34 +27,48 @@
echo "INSERT INTO execute2 (id, execute_id, start_time, duration, final_state, site) VALUES ('$globalid', '$inv_id', $start_time, $duration, '$endstate', '$site');" >> tmp-e2.sql
done < execute2.global.event
-while read col1 col2 col3 col4 col5 threadst namest lhsst rhsst resultst; do
- thread=`echo $threadst | awk 'BEGIN { FS = "=" }; {print $2}'`
- name=`echo $namest | awk 'BEGIN { FS = "=" }; {print $2}'`
- lhs=`echo $lhsst | awk 'BEGIN { FS = "=" }; {print $2}'`
- rhs=`echo $rhsst | awk 'BEGIN { FS = "=" }; {print $2}'`
- result=`echo $resultst | awk 'BEGIN { FS = "=" }; {print $2}'`
+while read col1 col2 col3 col4 col5 thread name lhs rhs result; do
+ thread=$(echo $thread | awk 'BEGIN { FS = "=" }; {print $2}')
+ name=$(echo $name | awk 'BEGIN { FS = "=" }; {print $2}')
+ lhs=$(echo $lhs | awk 'BEGIN { FS = "=" }; {print $2}')
+ rhs=$(echo $rhs | awk 'BEGIN { FS = "=" }; {print $2}')
+ result=$(echo $result | awk 'BEGIN { FS = "=" }; {print $2}')
operatorid="${WFID}operator:$thread"
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$lhs');" >> tmp-ds.sql
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$rhs');" >> tmp-ds.sql
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$result');" >> tmp-ds.sql
+ if [ $version -le 3726 ]; then
+ $lhs=$(echo $lhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ $rhs=$(echo $rhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ $result=$(echo $result | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ fi
+
+ echo "INSERT INTO dataset (id) VALUES ('$lhs');" >> tmp-ds.sql
+ echo "INSERT INTO dataset (id) VALUES ('$rhs');" >> tmp-ds.sql
+ echo "INSERT INTO dataset (id) VALUES ('$result');" >> tmp-ds.sql
echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$operatorid', 'operator', '$name', '$WF');" >> tmp-p.sql
- echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$PROVIDPREFIX$lhs', 'lhs');" >> tmp-dsu.sql
- echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$PROVIDPREFIX$rhs', 'rhs');" >> tmp-dsu.sql
- echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'O', '$PROVIDPREFIX$result', 'result');" >> tmp-dsu.sql
+ echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$lhs', 'lhs');" >> tmp-dsu.sql
+ echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'I', '$rhs', 'rhs');" >> tmp-dsu.sql
+ echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$operatorid', 'O', '$result', 'result');" >> tmp-dsu.sql
done < operators.txt
while read id name output; do
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$output');" >> tmp-ds.sql
+ if [ $version -le 3726 ]; then
+ $output=$(echo $output | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ fi
+ echo "INSERT INTO dataset (id) VALUES ('$output');" >> tmp-ds.sql
echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$id', 'function', '$name', '$WF');" >> tmp-p.sql
- echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'O', '$PROVIDPREFIX$output', 'result');" >> tmp-dsu.sql
+ echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'O', '$output', 'result');" >> tmp-dsu.sql
done < functions.txt
while read id value; do
# TODO need ordering/naming
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$value');" >> tmp-ds.sql
- echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'I', '$PROVIDPREFIX$value', 'undefined');" >> tmp-dsu.sql
+
+ if [ $version -le 3726 ]; then
+ $value=$(echo $value | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ fi
+
+ echo "INSERT INTO dataset (id) VALUES ('$value');" >> tmp-ds.sql
+ echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$id', 'I', '$value', 'undefined');" >> tmp-dsu.sql
done < function-inputs.txt
@@ -63,19 +77,35 @@
done < invocation-procedure-names.txt
while read outer inner; do
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$outer');" >> tmp-ds.sql
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$inner');" >> tmp-ds.sql
- echo "INSERT INTO ds_containment (out_id, in_id) VALUES ('$PROVIDPREFIX$outer', '$PROVIDPREFIX$inner');" >> tmp-dsc.sql
+
+ if [ $version -le 3726 ]; then
+ $outer=$(echo $outer | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ $inner=$(echo $inner | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ fi
+
+ echo "INSERT INTO dataset (id) VALUES ('$outer');" >> tmp-ds.sql
+ echo "INSERT INTO dataset (id) VALUES ('$inner');" >> tmp-ds.sql
+ echo "INSERT INTO ds_containment (out_id, in_id) VALUES ('$outer', '$inner');" >> tmp-dsc.sql
done < tie-containers.txt
while read dataset filename; do
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$dataset');" >> tmp-ds.sql
- echo "INSERT INTO file (id, filename) VALUES ('$PROVIDPREFIX$dataset', '$filename');" >> tmp-f.sql
+
+ if [ $version -le 3726 ]; then
+ $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ fi
+
+ echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql
+ echo "INSERT INTO file (id, filename) VALUES ('$dataset', '$filename');" >> tmp-f.sql
done < dataset-filenames.txt
while read dataset value; do
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$dataset');" >> tmp-ds.sql
- echo "INSERT INTO variable (id, value) VALUES ('$PROVIDPREFIX$dataset', '$value');" >> tmp-v.sql
+
+ if [ $version -le 3726 ]; then
+ $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ fi
+
+ echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql
+ echo "INSERT INTO variable (id, value) VALUES ('$dataset', '$value');" >> tmp-v.sql
done < dataset-values.txt
while read start duration wfid rest; do
@@ -114,8 +144,13 @@
else
dir=O
fi
- echo "INSERT INTO dataset (id) VALUES ('$PROVIDPREFIX$dataset');" >> tmp-ds.sql
- echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$thread', '$dir', '$PROVIDPREFIX$dataset', '$variable');" >> tmp-dsu.sql
+
+ if [ $version -le 3726 ]; then
+ $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ fi
+
+ echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql
+ echo "INSERT INTO ds_usage (process_id, direction, dataset_id, param_name) VALUES ('$thread', '$dir', '$dataset', '$variable');" >> tmp-dsu.sql
done < tie-data-invocs.txt
while read id ; do
Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs 2010-12-08 01:21:35 UTC (rev 3745)
+++ provenancedb/swift-prov-import-all-logs 2010-12-08 18:15:33 UTC (rev 3746)
@@ -50,22 +50,19 @@
fi
export RUNID=$(basename $filename .log)
- if [ $version -le 3726 ]; then
- PROVIDPREFIX="tag:ci.uchicago.edu,2008:swiftlogs:"
- fi
- export WF=$PROVIDPREFIX"execute:${RUNID}:run"
+ export WF="${RUNID}"
echo "INSERT INTO workflow (id, log_filename, swift_version, import_status) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD
echo version $version in log file $filename
echo ============= will import =============
- PROVIDPREFIX=$PROVIDPREFIX prepare-for-import $filename
+ prepare-for-import $filename
if [ "$?" != "0" ]; then
echo prepare-for-import failed
exit 2
fi
- PROVIDPREFIX=$PROVIDPREFIX import-run-to-sql $filename
+ version=$version import-run-to-sql $filename
if [ "$?" != "0" ]; then
echo import-run-to-sql failed
exit 3
From noreply at svn.ci.uchicago.edu Wed Dec 8 12:19:28 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 12:19:28 -0600 (CST)
Subject: [Swift-commit] r3747 - provenancedb/etc
Message-ID: <20101208181928.80D679CC80@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-08 12:19:27 -0600 (Wed, 08 Dec 2010)
New Revision: 3747
Modified:
provenancedb/etc/provenance.config.ci
Log:
Minor update.
Modified: provenancedb/etc/provenance.config.ci
===================================================================
--- provenancedb/etc/provenance.config.ci 2010-12-08 18:15:33 UTC (rev 3746)
+++ provenancedb/etc/provenance.config.ci 2010-12-08 18:19:27 UTC (rev 3747)
@@ -2,7 +2,6 @@
# presently hardcoded
# this is the path to log repo on benc's laptop
-export LOGREPO=~/swift-logs/
+export LOGREPO=~/swift-logs
+export SQLCMD="psql -U provdb -h db.ci.uchicago.edu provdb"
-# export SQLCMD="psql -p 5435 -d provdb "
-
From noreply at svn.ci.uchicago.edu Wed Dec 8 13:28:18 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 13:28:18 -0600 (CST)
Subject: [Swift-commit] r3748 - provenancedb
Message-ID: <20101208192818.527E39CC80@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-08 13:28:18 -0600 (Wed, 08 Dec 2010)
New Revision: 3748
Modified:
provenancedb/prov-to-sql.sh
Log:
Minor fixes.
Modified: provenancedb/prov-to-sql.sh
===================================================================
--- provenancedb/prov-to-sql.sh 2010-12-08 18:19:27 UTC (rev 3747)
+++ provenancedb/prov-to-sql.sh 2010-12-08 19:28:18 UTC (rev 3748)
@@ -37,9 +37,9 @@
operatorid="${WFID}operator:$thread"
if [ $version -le 3726 ]; then
- $lhs=$(echo $lhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
- $rhs=$(echo $rhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
- $result=$(echo $result | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ lhs=$(echo $lhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ rhs=$(echo $rhs | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ result=$(echo $result | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
fi
echo "INSERT INTO dataset (id) VALUES ('$lhs');" >> tmp-ds.sql
@@ -53,7 +53,7 @@
while read id name output; do
if [ $version -le 3726 ]; then
- $output=$(echo $output | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ output=$(echo $output | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
fi
echo "INSERT INTO dataset (id) VALUES ('$output');" >> tmp-ds.sql
echo "INSERT INTO process (id, type, name, workflow_id) VALUES ('$id', 'function', '$name', '$WF');" >> tmp-p.sql
@@ -64,7 +64,7 @@
# TODO need ordering/naming
if [ $version -le 3726 ]; then
- $value=$(echo $value | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ value=$(echo $value | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
fi
echo "INSERT INTO dataset (id) VALUES ('$value');" >> tmp-ds.sql
@@ -79,8 +79,8 @@
while read outer inner; do
if [ $version -le 3726 ]; then
- $outer=$(echo $outer | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
- $inner=$(echo $inner | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ outer=$(echo $outer | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ inner=$(echo $inner | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
fi
echo "INSERT INTO dataset (id) VALUES ('$outer');" >> tmp-ds.sql
@@ -91,7 +91,7 @@
while read dataset filename; do
if [ $version -le 3726 ]; then
- $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
fi
echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql
@@ -101,7 +101,7 @@
while read dataset value; do
if [ $version -le 3726 ]; then
- $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
fi
echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql
@@ -146,7 +146,7 @@
fi
if [ $version -le 3726 ]; then
- $dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
+ dataset=$(echo $dataset | sed -e 's/tag:benc at ci.uchicago.edu,2008:swift://g')
fi
echo "INSERT INTO dataset (id) VALUES ('$dataset');" >> tmp-ds.sql
From noreply at svn.ci.uchicago.edu Wed Dec 8 16:28:44 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 16:28:44 -0600 (CST)
Subject: [Swift-commit] r3749 - in provenancedb: . apps/oops
Message-ID: <20101208222844.8973A9CC80@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-08 16:28:44 -0600 (Wed, 08 Dec 2010)
New Revision: 3749
Modified:
provenancedb/apps/oops/oops_extractor.sh
provenancedb/prov-init.sql
provenancedb/swift-prov-import-all-logs
Log:
Minor fixes.
Modified: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh 2010-12-08 19:28:18 UTC (rev 3748)
+++ provenancedb/apps/oops/oops_extractor.sh 2010-12-08 22:28:44 UTC (rev 3749)
@@ -6,7 +6,7 @@
# OOPS' Swift logs.
PROVDB_HOME=~/provenancedb
-PROTESTS_HOME=//CASP
+PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP
IMPORT_HOME=~/protests
source $PROVDB_HOME/etc/provenance.config
@@ -27,12 +27,14 @@
if ! grep --silent $i provdb_imported; then
if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/psim.loops-*.log; then
cd swift-logs
- for j in `ls $PROTESTS_HOME/$k/$i | grep psim.loops-`; do
- ln -s $PROTESTS_HOME/$k/$i/$j
- done
+ #for j in `ls $PROTESTS_HOME/$k/$i | grep psim.loops-`; do
+ # ln -s $PROTESTS_HOME/$k/$i/$j
+ #done
cd import
# swift-prov-import-all-logs also controls what already has been
# imported, so it does not repeat work
+ echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config
+ echo "export SQLCMD=\"psql -U provdb -h db.ci.uchicago.edu provdb\"" >> $PROVDB_HOME/etc/provenance.config
$PROVDB_HOME/swift-prov-import-all-logs
cd $IMPORT_HOME
echo $i >> provdb_imported
Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql 2010-12-08 19:28:18 UTC (rev 3748)
+++ provenancedb/prov-init.sql 2010-12-08 22:28:44 UTC (rev 3749)
@@ -252,7 +252,7 @@
annot_wf_num,
annot_wf_txt,
annot_wf_bool,
- extrainfo,
+-- extrainfo,
createarray,
createarray_member,
array_range
Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs 2010-12-08 19:28:18 UTC (rev 3748)
+++ provenancedb/swift-prov-import-all-logs 2010-12-08 22:28:44 UTC (rev 3749)
@@ -41,35 +41,38 @@
EXISTING=$($SQLCMD --tuples-only -c "select count(*) from workflow where log_filename='$filename';")
if [ "$EXISTING" -eq "0" ]; then
- echo IMPORTING
-
- if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then
- wfstatus="SUCCESS"
- else
- wfstatus="FAIL"
+ PROV_ENABLED=$(grep provenance $filename | wc -l)
+ if [ "$PROV_ENABLED" -ge "0" ]; then
+ echo IMPORTING
+
+ if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then
+ wfstatus="SUCCESS"
+ else
+ wfstatus="FAIL"
+ fi
+
+ export RUNID=$(basename $filename .log)
+
+ export WF="${RUNID}"
+
+ echo "INSERT INTO workflow (id, log_filename, swift_version, import_status) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD
+
+ echo version $version in log file $filename
+ echo ============= will import =============
+ prepare-for-import $filename
+ if [ "$?" != "0" ]; then
+ echo prepare-for-import failed
+ exit 2
+ fi
+ version=$version import-run-to-sql $filename
+ if [ "$?" != "0" ]; then
+ echo import-run-to-sql failed
+ exit 3
+ fi
+
+ # import-run-to-xml $filename
+ echo SKIP: provenance.log not true in swift.properties
fi
-
- export RUNID=$(basename $filename .log)
-
- export WF="${RUNID}"
-
- echo "INSERT INTO workflow (id, log_filename, swift_version, import_status) VALUES ('$WF','$filename','$version','$wfstatus');" | $SQLCMD
-
- echo version $version in log file $filename
- echo ============= will import =============
- prepare-for-import $filename
- if [ "$?" != "0" ]; then
- echo prepare-for-import failed
- exit 2
- fi
- version=$version import-run-to-sql $filename
- if [ "$?" != "0" ]; then
- echo import-run-to-sql failed
- exit 3
- fi
-
- # import-run-to-xml $filename
-
else
echo SKIP: Already known in workflow
fi
From noreply at svn.ci.uchicago.edu Wed Dec 8 16:38:55 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 16:38:55 -0600 (CST)
Subject: [Swift-commit] r3750 - provenancedb
Message-ID: <20101208223855.182679CC80@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-08 16:38:54 -0600 (Wed, 08 Dec 2010)
New Revision: 3750
Modified:
provenancedb/swift-prov-import-all-logs
Log:
Minor fixes.
Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs 2010-12-08 22:28:44 UTC (rev 3749)
+++ provenancedb/swift-prov-import-all-logs 2010-12-08 22:38:54 UTC (rev 3750)
@@ -71,6 +71,7 @@
fi
# import-run-to-xml $filename
+ else
echo SKIP: provenance.log not true in swift.properties
fi
else
From noreply at svn.ci.uchicago.edu Wed Dec 8 17:05:00 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 17:05:00 -0600 (CST)
Subject: [Swift-commit] r3751 - provenancedb
Message-ID: <20101208230500.224479CC80@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-08 17:05:00 -0600 (Wed, 08 Dec 2010)
New Revision: 3751
Modified:
provenancedb/swift-prov-import-all-logs
Log:
Minor fixes.
Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs 2010-12-08 22:38:54 UTC (rev 3750)
+++ provenancedb/swift-prov-import-all-logs 2010-12-08 23:05:00 UTC (rev 3751)
@@ -42,7 +42,7 @@
if [ "$EXISTING" -eq "0" ]; then
PROV_ENABLED=$(grep provenance $filename | wc -l)
- if [ "$PROV_ENABLED" -ge "0" ]; then
+ if [ $PROV_ENABLED -gt 0 ]; then
echo IMPORTING
if grep --silent "DEBUG Loader Swift finished with no errors" $filename; then
@@ -72,7 +72,7 @@
# import-run-to-xml $filename
else
- echo SKIP: provenance.log not true in swift.properties
+ echo SKIP: provenance.log not set to true in etc/swift.properties
fi
else
echo SKIP: Already known in workflow
From noreply at svn.ci.uchicago.edu Wed Dec 8 19:39:04 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 19:39:04 -0600 (CST)
Subject: [Swift-commit] r3754 - text/parco10submission
Message-ID: <20101209013904.5BC019CC80@svn.ci.uchicago.edu>
Author: dsk
Date: 2010-12-08 19:39:04 -0600 (Wed, 08 Dec 2010)
New Revision: 3754
Modified:
text/parco10submission/ResponseToReviews.txt
text/parco10submission/Wozniak.bib
text/parco10submission/paper.tex
Log:
more small changes in Parco paper
Modified: text/parco10submission/ResponseToReviews.txt
===================================================================
--- text/parco10submission/ResponseToReviews.txt 2010-12-09 01:29:08 UTC (rev 3753)
+++ text/parco10submission/ResponseToReviews.txt 2010-12-09 01:39:04 UTC (rev 3754)
@@ -181,8 +181,13 @@
probably should be defined. Other acronyms: GRAM, fMRI (and FMRI,
cf. Fig. 2, which should probably be fMRI).
->>> <<<
+>>>
+RDBMS removed.
+FMRI fixed.
+
+<<<
+
6. All these appear: "stage in", "stage-in", "stagein". Please be
consistent (similarly for stage out).
@@ -230,8 +235,12 @@
should have page numbers, some references are missing a year.
Reference 8 severely mangles "Bresnahan".
->>> <<<
+>>>
+Ref 8 fixed.
+
+<<<
+
14. It is somewhat disingenuous to refer to thousands and millions of
cores (section 6.2). All systems I know of are managed as nodes,
where each node might have 8 or 12 or 16 cores. This is no essential
Modified: text/parco10submission/Wozniak.bib
===================================================================
--- text/parco10submission/Wozniak.bib 2010-12-09 01:29:08 UTC (rev 3753)
+++ text/parco10submission/Wozniak.bib 2010-12-09 01:39:04 UTC (rev 3754)
@@ -288,7 +288,7 @@
}
@INPROCEEDINGS{GridFTP_2005,
- author = {William Allcock and John Bresnahan an Rajkumar Kettimuthu and Michael
+ author = {William Allcock and John Bresnahan and Rajkumar Kettimuthu and Michael
Link and Catalin Dumitrescu and Ioan Raicu and Ian Foster},
title = {The {G}lobus Striped {GridFTP} Framework and Server},
booktitle = {Proc. Supercomputing},
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-09 01:29:08 UTC (rev 3753)
+++ text/parco10submission/paper.tex 2010-12-09 01:39:04 UTC (rev 3754)
@@ -526,8 +526,8 @@
\end{verbatim}
Sometimes data may be stored in a form that does not fit with Swift's
-file-and-site model; \katznote{Swift's ``file-and-site model'' hasn't been introduced before. I'm not even sure what it is.} for example, data might be stored in an RDBMS on some
-database server. In that case, a variable can be declared to have
+file-and-site model; \katznote{Swift's ``file-and-site model'' hasn't been introduced before. I'm not even sure what it is.} for example, data might be stored in an database on a
+server. In that case, a variable can be declared to have
\verb|extern| type. This indicates that Swift should use the variable to
determine execution dependency, but should not attempt other data management;
for example, it will not perform any form of data stage-in or stage-out;
@@ -967,7 +967,7 @@
\begin{figure}[htbp]
\begin{center}
\includegraphics[scale=1]{img/IMG_fmridataset}
- \caption{FMRI application data structure\label{FMRI_app_image}}
+ \caption{fMRI application data structure\label{FMRI_app_image}}
\end{center}
\end{figure}
@@ -1012,7 +1012,7 @@
}
\end{verbatim}
- \caption{FMRI application in SwiftScript\label{FMRI_app_script}}
+ \caption{fMRI application in SwiftScript\label{FMRI_app_script}}
\end{figure}
In this example, the logical structure of the fMRI dataset shown in
From noreply at svn.ci.uchicago.edu Wed Dec 8 20:02:37 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 20:02:37 -0600 (CST)
Subject: [Swift-commit] r3755 - text/parco10submission
Message-ID: <20101209020237.C34029CC80@svn.ci.uchicago.edu>
Author: dsk
Date: 2010-12-08 20:02:37 -0600 (Wed, 08 Dec 2010)
New Revision: 3755
Modified:
text/parco10submission/paper.tex
Log:
adding note to Parco paper
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-09 01:39:04 UTC (rev 3754)
+++ text/parco10submission/paper.tex 2010-12-09 02:02:37 UTC (rev 3755)
@@ -85,6 +85,9 @@
\katznote{in general, I would like to use application, job, task, program, etc. in some way that is consistent throughout the paper, perhaps defined in the introduction}
+\katznote{are swift tasks idempotent? are swift scripts are deterministic (maybe?) deadlock-free? (no)}
+
+
Swift is a scripting language designed for composing ordinary
application programs (serial or parallel) into distributed,
parallelized applications for execution on grids and supercomputers
From noreply at svn.ci.uchicago.edu Wed Dec 8 22:10:37 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 22:10:37 -0600 (CST)
Subject: [Swift-commit] r3756 - text/parco10submission
Message-ID: <20101209041037.1FAFB9CC95@svn.ci.uchicago.edu>
Author: dsk
Date: 2010-12-08 22:10:36 -0600 (Wed, 08 Dec 2010)
New Revision: 3756
Modified:
text/parco10submission/ResponseToReviews.txt
text/parco10submission/paper.tex
Log:
more fixes based on reviewer comments
Modified: text/parco10submission/ResponseToReviews.txt
===================================================================
--- text/parco10submission/ResponseToReviews.txt 2010-12-09 02:02:37 UTC (rev 3755)
+++ text/parco10submission/ResponseToReviews.txt 2010-12-09 04:10:36 UTC (rev 3756)
@@ -148,13 +148,21 @@
language, so perhaps "SwiftScript" should be replaced by "Swift"
everywhere.
->>> <<<
+>>>
+this has been done.
+
+<<<
+
2. It's a bit awkward that "single assignment" is used in section 2.1
but not defined until section 2.3.
->>> <<<
+>>>
+fixed.
+
+<<<
+
3. The example on p. 5 appears erroneous, it seems the rotate
procedure should have an angle input in addition to the image input
(this is corrected on p.6).
@@ -185,6 +193,8 @@
RDBMS removed.
FMRI fixed.
+GRAM doesn't seem to need explanation, it's cited where first used.
+still need to think about acronyms in Figure 1.
<<<
@@ -200,8 +210,13 @@
7. The mysterious numbers '12, 1000, 1000, "81 3 3"' in example 4.1
might merit an explanation.
->>> <<<
+>>>
+It seems clear that these are parameters passed into the app program;
+I don't think explanation is needed
+
+<<<
+
8. The Figure numbering in the text of section 4.1 needs correction.
>>>
@@ -215,8 +230,12 @@
In the text of section 4.1 "n x n" should be in a different font and
"n2" is clearly wrong.
->>> <<<
+>>>
+fixed.
+
+<<<
+
10. Section 4.2, what is an "emblem experiment"?
>>> <<<
@@ -224,8 +243,12 @@
11. The margins in sections 4.2 and 4.4 need fixing as some lines
run completely off the page.
->>> <<<
+>>>
+fixed in 4.2 - not sure about the state of 4.4 yet
+
+<<<
+
12. "Karajan" is mentioned several times, there really should be short
definition of it and a reference to it in the bibliography.
@@ -238,6 +261,7 @@
>>>
Ref 8 fixed.
+page numbers and years still need to be fixed.
<<<
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-09 02:02:37 UTC (rev 3755)
+++ text/parco10submission/paper.tex 2010-12-09 04:10:36 UTC (rev 3756)
@@ -175,7 +175,7 @@
ongoing and future work in the Swift project, and we offer concluding
remarks in Section~\ref{Conclusion}.
-\section{The SwiftScript language}
+\section{The Swift language}
\label{Language}
The Swift programming model is data-oriented: it encapsulates the
@@ -238,8 +238,12 @@
procedures.
Swift variables hold either primitive values, files, or collections of
-files. All variables are \emph{single assignment}, which provides the
-basis for Swift's model of procedure chaining. Procedures are
+files. All variables are \emph{single-assignment} (meaning
+that they must be assigned to exactly one value during execution),
+which provides the
+basis for Swift's model of procedure chaining. (Note that Swift arrays are not
+strictly single-assignment, but each element of an array is, as discussed in
+Section~\ref{ordering}.) Procedures are
executed when their input parameters have all been set from existing
data or prior procedure executions. Procedures are chained by
specifying that an output variable of one procedure is passed as the
@@ -314,15 +318,15 @@
The examples above have used the type \verb|image| without a
definition of that type. We can declare it as a \emph{marker type},
-which has no structure exposed to SwiftScript:
+which has no structure exposed to Swift:
\begin{verbatim}
type image;
\end{verbatim}
This does not indicate that the data is unstructured; it indicates
-that the structure of the data is not exposed to SwiftScript.
-SwiftScript will treat variables of this type as individual opaque
+that the structure of the data is not exposed to Swift.
+Swift will treat variables of this type as individual opaque
files.
With mechanisms to declare types, map variables to data files, and
@@ -389,9 +393,10 @@
procedure, using each execution's output as the input to the next step.
\subsection{Ordering of execution}
+\label{ordering}
-Non-array variables are \emph{single-assignment}, which means that
-they must be assigned to exactly one value during execution. A
+As previously stated, non-array variables are single-assignment,
+which means thatthey must be assigned to exactly one value during execution. A
procedure or expression will be executed when all of its input
parameters have been assigned values. As a result of such execution,
more variables may become assigned, possibly allowing further parts of
@@ -413,7 +418,7 @@
z=q(y);
\end{verbatim}
-Arrays in SwiftScript are more generally \emph{monotonic}, \katznote{is this common use of monotonic? Are the arrays monotonic? Or is the assignment of elements in the array monotonic?}
+Arrays in Swift are more generally \emph{monotonic}, \katznote{is this common use of monotonic? Are the arrays monotonic? Or is the assignment of elements in the array monotonic?}
that is,
knowledge about the content of an array increases during execution,
but cannot otherwise change. Once a value for a particular element is
@@ -446,9 +451,9 @@
\subsection{Compound procedures}
As with many other programming languages, procedures consisting of
-SwiftScript code can be defined. These differ from the previously
+Swift code can be defined. These differ from the previously
mentioned procedures declared with the \verb|app| keyword, as they
-invoke other SwiftScript procedures rather than a component
+invoke other Swift procedures rather than a component
program. The basic structure of a composite procedure may be thought
of as a graph of calls to other procedures.
The following script will invoke two procedures, with an intermediate
@@ -485,13 +490,13 @@
\subsection{More about types}
\label{LanguageTypes}
-Each variable and procedure parameter in SwiftScript is strongly typed.
+Each variable and procedure parameter in Swift is strongly typed.
Types are used to structure data, to aid in debugging and program
correctness and to influence how Swift interacts with data.
The \verb|image| type declared in previous examples is a \emph{marker
type}. Marker types indicate that data for a variable is stored in a
-single file with no further structure exposed at the SwiftScript level.
+single file with no further structure exposed at the Swift level.
Arrays have been mentioned above, in the arrays section. A code block
may be applied to each element of an array using \verb|foreach|; or
@@ -564,7 +569,7 @@
that database. The declaration of \verb|database| contains no mapping; and
the procedures which use \verb|database| do not reference them in any
way; the description of \verb|database| is entirely outside of the script.
-The single assignment and execution ordering rules will still apply though;
+The single-assignment and execution ordering rules will still apply though;
populateDatabase will always be run before analyseDatabase.
\subsection{Swift mappers}
@@ -623,7 +628,7 @@
\hline
\end{tabular}
\end{center}
- \caption{SwiftScript built-in mappers: conceptual syntax}
+ \caption{Swift built-in mappers: conceptual syntax}
\label{mappertable}
\end{table}
@@ -635,7 +640,7 @@
\subsection{The execution environment for component programs}
\label{LanguageEnvironment}
-A SwiftScript \verb|app| declaration describes how a component program
+A Swift \verb|app| declaration describes how a component program
is invoked. In order to ensure the correctness of the Swift model, the
environment in which programs are executed needs to be constrained.
@@ -771,11 +776,11 @@
\subsection{Reliable execution}
\label{ExecutingReliably}
-The functional nature of SwiftScript provides a clearly defined
+The functional nature of Swift provides a clearly defined
interface to imperative components, which in addition to allowing Swift great
flexibility in where and when it runs component programs, allows those
imperative components to be treated as atomic components that can be
-executed multiple times for any given SwiftScript procedure invocation.
+executed multiple times for any given Swift procedure invocation.
This facilitates three different reliability mechanisms that are implemented by
the runtime and that need not be exposed at the language level: \emph{retries},
\emph{restarts} and \emph{replication}.
@@ -783,7 +788,7 @@
In the simplest form of error handling in Swift, if a component
program fails, Swift will make a subsequent attempt to rerun the
program. In contrast to many other systems, retry here is at the level
-of the SwiftScript procedure invocation, and includes completely
+of the Swift procedure invocation, and includes completely
reattempting site selection, stage-in, execution and stage-out. This
provides a natural way to deal both with many transient errors, such as
temporary network loss, and with many changes in site state.
@@ -1015,11 +1020,11 @@
}
\end{verbatim}
- \caption{fMRI application in SwiftScript\label{FMRI_app_script}}
+ \caption{fMRI application in Swift\label{FMRI_app_script}}
\end{figure}
In this example, the logical structure of the fMRI dataset shown in
-Figure~\ref{FMRI_app_image} can be represented by the SwiftScript type declarations in
+Figure~\ref{FMRI_app_image} can be represented by the Swift type declarations in
lines 1-6 in Figure~\ref{FMRI_app_script}. Here, Study is declared as containing an array
of Group, which in turn contains an array of Subject, etc. Similarly,
an fMRI Run is a series of brain scans called volumes, with a Volume
@@ -1030,17 +1035,18 @@
which take typed data described by a mapper, perform computations
on those data, and produce data to be stored in locations specified
by a mapper. The
-procedure reslice\_wf defines a compound procedure, which comprises
+procedure {\tt reslice\_wf} defines a compound procedure, which comprises
a series of procedure calls, using variables to establish
data dependencies.
-In the example, reslice\_wf defines a four-step pipeline computation,
+In the example, {\tt reslice\_wf} defines a four-step pipeline computation,
using variables to establish data dependencies. It applies reorientRun
to a run first in the x axis and then in the y axis, and then aligns
each image in the resulting run with the first image. The program
-alignlinear \katznote{should this be alignlinearRun?} determines how to spatially adjust an image to match a
+{\tt alignlinear} \katznote{should this be alignlinearRun?} determines how to spatially adjust an image to match a
reference image, and produces an air parameter file. The actual
-alignment is done by the program reslice. \katznote{or resliceRun?} Note that variable yR is
+alignment is done by the program {\tt reslice}.
+\katznote{or resliceRun?} Note that variable {\tt yR} is
the output of the first step and the input of the second step, and as such, defines
the data dependencies between the two steps. %The pipeline is
%illustrated in the center of % Figure~\ref{FMRIFigure2},
@@ -1054,19 +1060,19 @@
The procedure reorientRun
is also a compound procedure.
The foreach statement defines an iteration over the input run
-ir and applies the procedure reorient (which rotates a brain image
+{\tt ir} and applies the procedure {\tt reorient} (which rotates a brain image
along a certain axis) to each volume in the run to produces a
reoriented run or. Because the multiple calls to reorient operate on
independent data elements, they can proceed in parallel.
The procedure reorient in this example is an atomic procedure.
-This procedure has typed input parameters iv, direction and overwrite and
-one output ov. The body specifies that it
+This procedure has typed input parameters ({\tt iv}, {\tt direction}, and {\tt overwrite}) and
+one output ({\tt ov}). The body specifies that it
invokes a program (also called reorient) that will be
dynamically mapped to a binary executable, which will
execute at an execution site chosen by the Swift runtime system. The
body also specifies how input parameters map to command line
-arguments. The notation @filename is a built-in mapping function that
+arguments. The notation {\tt @filename} is a built-in mapping function that
maps a logical data structure to a physical file name. In this case,
it extracts the file name of input header and output header, which are
then put in the command line to invoke the reorient program.
@@ -1110,7 +1116,7 @@
self-contained, structural equation models---we generated 65,535 R
objects representing all models with 1 to 16 connections
of varying weights between 4 pre-selected regions of interest.
-A 4x4 matrix represents connections between the four regions,
+A 4~x~4 matrix represents connections between the four regions,
with 16 possible connections (connections between the same two
regions but in different directions are tested separately).
\katznote{should refer to figure~\ref{omxFigure} here? If so, the residual variances
@@ -1138,8 +1144,8 @@
A model generator was developed for the OpenMx package and is designed
explicitly to enable parallel execution of exhaustive or partially
-pruned sets of model objects. Given an n x n covariance matrix it can
-generate the entire set of possible models with anywhere from 0 to n2
+pruned sets of model objects. Given an $n$~x~$n$ covariance matrix it can
+generate the entire set of possible models with anywhere from 0 to $n^2$
connections; however, it can also take as input a single index from
that set and it will generate and run a single model. What this means
in the context of workflow design is that the generator can be
@@ -1167,7 +1173,7 @@
initweight, speech);
12. }
\end{verbatim}
-\caption{Swiftscript for 4-region exhaustive SEM for a single experimental condition\label{omxScript1}}
+\caption{Swift script for 4-region exhaustive SEM for a single experimental condition\label{omxScript1}}
\end{figure}
First, a covariance matrix containing activation data for 4 brain regions,
@@ -1218,7 +1224,7 @@
initweight, cond);
9. }
\end{verbatim}
-\caption{Swiftscript for 4-region exhaustive SEM for 2 experimental conditions\label{omxScript2}}
+\caption{Swift script for 4-region exhaustive SEM for 2 experimental conditions\label{omxScript2}}
\end{figure}
@@ -1255,7 +1261,7 @@
p = readdata("paramslist.txt");
runDocks(p);
\end{verbatim}
-\caption{Swiftscript for running Dock\label{DockScript}}
+\caption{Swift script for running Dock\label{DockScript}}
\end{figure}
\katznote{no text in this subsection -- no reference to Figure~\ref{DockScript}}
@@ -1266,7 +1272,7 @@
data from a large dataset of files that categorize the Earth's surface,
from the MODIS sensor instruments that orbit Earth on two NASA
satellites of the Earth Observing System.
-The SwiftScript analyzes the dataset to find the files with the ten
+The Swift script analyzes the dataset to find the files with the ten
largest total urban area and then produces a new dataset with viewable
color images of those top-ten urban data ``tiles''.
From noreply at svn.ci.uchicago.edu Wed Dec 8 22:45:24 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 22:45:24 -0600 (CST)
Subject: [Swift-commit] r3757 - text/parco10submission
Message-ID: <20101209044524.5F6239CC80@svn.ci.uchicago.edu>
Author: dsk
Date: 2010-12-08 22:45:24 -0600 (Wed, 08 Dec 2010)
New Revision: 3757
Modified:
text/parco10submission/Wozniak.bib
Log:
fixing references - done up through #8
Modified: text/parco10submission/Wozniak.bib
===================================================================
--- text/parco10submission/Wozniak.bib 2010-12-09 04:10:36 UTC (rev 3756)
+++ text/parco10submission/Wozniak.bib 2010-12-09 04:45:24 UTC (rev 3757)
@@ -1514,16 +1514,23 @@
month = nov
}
- at ARTICLE{GRAM_1998,
- author = {Karl Czajkowski and Ian Foster and Nick Karonis and Carl Kesselman
- and Stuart Martin and Warren Smith and Steven Tuecke},
- title = {A Resource Management Architecture for Metacomputing Systems},
- journal = {Lecture Notes in Computer Science},
- year = {1998},
- volume = {1459},
+ at incollection {GRAM_1998,
+ author = {Czajkowski, Karl and Foster, Ian and Karonis, Nick and Kesselman, Carl and Martin, Stuart and Smith, Warren and Tuecke, Steven},
+ affiliation = {University of Southern California Information Sciences Institute Marina del Rey 90292-6695 CA},
+ title = {A resource management architecture for metacomputing systems},
+ booktitle = {Job Scheduling Strategies for Parallel Processing},
+ series = {Lecture Notes in Computer Science},
+ editor = {Feitelson, Dror and Rudolph, Larry},
+ publisher = {Springer Berlin / Heidelberg},
+ isbn = {},
+ pages = {62-82},
+ volume = {1459},
+ url = {http://dx.doi.org/10.1007/BFb0053981},
+ note = {10.1007/BFb0053981},
+ year = {1998}
}
-url = {citeseer.ist.psu.edu/czajkowski97resource.html}
+
@INPROCEEDINGS{Coallocation_1999,
author = {Karl Czajkowski and Ian Foster and Carl Kesselman},
title = {Resource Co-Allocation in Computational Grids},
@@ -1954,7 +1961,8 @@
journal = {J. Supercomputer Applications},
year = {1997},
volume = {11},
- issue = {2}
+ issue = {2},
+ pages = {115--128}
}
@ARTICLE{OGSA_2002,
@@ -2614,17 +2622,24 @@
url = {citeseer.ist.psu.edu/huston95partially.html}
}
- at ARTICLE{BGP_2008,
- author = {{IBM Blue Gene team}},
- title = {Overview of the {IBM} {Blue} {Gene/P} project},
- journal = {IBM J. Research and Development},
- year = {2008},
- volume = {52},
- number = {1/2},
- owner = {jwozniak},
- timestamp = {2009-01-26}
-}
+ at article{BGP_2008,
+ author = {{IBM Blue Gene team}},
+ title = {{Overview of the IBM Blue Gene/P project}},
+ journal = {IBM J. Res. Dev.},
+ volume = {52},
+ issue = {1/2},
+ month = {January},
+ year = {2008},
+ issn = {0018-8646},
+ pages = {199--220},
+ numpages = {22},
+ url = {http://portal.acm.org/citation.cfm?id=1375990.1376008},
+ acmid = {1376008},
+ publisher = {IBM Corp.},
+ address = {Riverton, NJ, USA},
+}
+
@UNPUBLISHED{IzHM0x,
author = {Jes{\'u}s A.\ Izaguirre and Scott Hampton and Thierry Matthey},
title = {Shadow {H}ybrid {M}onte {C}arlo: An Improved Method for Conformational
@@ -4414,14 +4429,21 @@
timestamp = {2007-08-16}
}
- at INPROCEEDINGS{NFS_1985,
- author = {Russel Sandberg and David Goldberg and Steve Kleiman and Dan Walsh
- and Bob Lyon},
- title = {Design and implementation of the {Sun Network File System}},
- booktitle = USENIX,
- year = {1985}
-}
+ at incollection{NFS_1985,
+ author = {Sandberg, R. and Golgberg, D. and Kleiman, S. and Walsh, D. and Lyon, B.},
+ chapter = {Design and implementation of the Sun network filesystem},
+ title = {Innovations in Internetworking},
+ year = {1988},
+ isbn = {0-89006-337-0},
+ pages = {379--390},
+ numpages = {12},
+ url = {http://portal.acm.org/citation.cfm?id=59309.59338},
+ acmid = {59338},
+ publisher = {Artech House, Inc.},
+ address = {Norwood, MA, USA},
+}
+
@INPROCEEDINGS{wetherall:duppkt:USENIX,
author = {J. Santos and D. Wetherall},
title = {Increasing Effective Link Bandwidth by Suppressing Replicated Data},
@@ -4506,8 +4528,9 @@
@INPROCEEDINGS{GPFS_2002,
author = {Frank Schmuck and Roger Haskin},
title = {{GPFS}: {A} shared-disk file system for large computing clusters},
- booktitle = FAST,
+ booktitle = {Proceedings of the Conference on File and Storage Technologies (FAST?02)},
year = {2002},
+ pages={231--244},
owner = {wozniak},
timestamp = {2009.09.19}
}
@@ -5716,15 +5739,17 @@
}
@INPROCEEDINGS{Swift_2007,
- author = {Yong Zhao and Mihael Hategan and Ben Clifford and Ian Foster and
- Gregor von Laszewski and Ioan Raicu and Tiberiu Stef-Praun and Mike
- Wilde},
- title = {Swift: {Fast}, Reliable, Loosely Coupled Parallel Computation},
- booktitle = {Proc. Workshop on Scientific Workflows},
- year = {2007},
- owner = {wozniak},
- timestamp = {2009.02.05}
-}
+author={Yong Zhao and Hategan, M. and Clifford, B. and Foster, I. and von Laszewski, G. and Nefedova, V. and Raicu, I. and Stef-Praun, T. and Wilde, M.},
+booktitle={2007 IEEE Congress on Services},
+title={{Swift: Fast, Reliable, Loosely Coupled Parallel Computation}},
+year={2007},
+month={July},
+volume={},
+number={},
+pages={199 -206},
+keywords={GriPhyN virtual data system;Swift system;SwiftScript language;complex file system structures;runtime system;scripting language;formal specification;grid computing;software reliability;},
+doi={10.1109/SERVICES.2007.63},
+ISSN={},}
@INPROCEEDINGS{VDG_Data_2004,
author = {Y. Zhao and M. Wilde and I. Foster and J. Voeckler and J. Dobson
@@ -6207,11 +6232,11 @@
pages = {18},
@INPROCEEDINGS{SunConstellation_2008,
- title = {Performance and Scalability Study of
+ title = {{Performance and Scalability Study of
Sun Constellation Cluster 'Ranger'
- using Application-Based Benchmarks},
+ using Application-Based Benchmarks}},
author = {Byoung-Do Kim and John E. Cazes},
- booktitle = {Proc. TeraGrid},
+ booktitle = {Proc. TeraGrid'2008},
year = 2008
}
From noreply at svn.ci.uchicago.edu Wed Dec 8 23:32:32 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Dec 2010 23:32:32 -0600 (CST)
Subject: [Swift-commit] r3758 - text/parco10submission
Message-ID: <20101209053232.957439CC80@svn.ci.uchicago.edu>
Author: dsk
Date: 2010-12-08 23:32:32 -0600 (Wed, 08 Dec 2010)
New Revision: 3758
Modified:
text/parco10submission/ResponseToReviews.txt
text/parco10submission/Wozniak.bib
Log:
finished fixing references
Modified: text/parco10submission/ResponseToReviews.txt
===================================================================
--- text/parco10submission/ResponseToReviews.txt 2010-12-09 04:45:24 UTC (rev 3757)
+++ text/parco10submission/ResponseToReviews.txt 2010-12-09 05:32:32 UTC (rev 3758)
@@ -260,8 +260,7 @@
>>>
-Ref 8 fixed.
-page numbers and years still need to be fixed.
+all Refs fixed.
<<<
Modified: text/parco10submission/Wozniak.bib
===================================================================
--- text/parco10submission/Wozniak.bib 2010-12-09 04:45:24 UTC (rev 3757)
+++ text/parco10submission/Wozniak.bib 2010-12-09 05:32:32 UTC (rev 3758)
@@ -39,7 +39,7 @@
@STRING{PDSW = {Proc. Petascale Data Storage Workshop}}
- at STRING{PTRS_A = {Philosophical Transactions of the Royal Society}}
+ at STRING{PTRS_A = {Philosophical Transactions of the Royal Society A}}
@STRING{SC = {Proc. Supercomputing}}
@@ -287,14 +287,20 @@
pages = {161-163}
}
- at INPROCEEDINGS{GridFTP_2005,
- author = {William Allcock and John Bresnahan and Rajkumar Kettimuthu and Michael
- Link and Catalin Dumitrescu and Ioan Raicu and Ian Foster},
- title = {The {G}lobus Striped {GridFTP} Framework and Server},
- booktitle = {Proc. Supercomputing},
- year = {2005},
- timestamp = {2007.02.01}
-}
+ at inproceedings{GridFTP_2005,
+ author = {Allcock, William and Bresnahan, John and Kettimuthu, Rajkumar and Link, Michael and Dumitrescu, Catalin and Raicu, Ioan and Foster, Ian},
+ title = {{The Globus Striped GridFTP Framework and Server}},
+ booktitle = {Proceedings of the 2005 ACM/IEEE Conference on Supercomputing},
+ series = {SC '05},
+ year = {2005},
+ isbn = {1-59593-061-2},
+ pages = {54--},
+ url = {http://dx.doi.org/10.1109/SC.2005.72},
+ doi = {http://dx.doi.org/10.1109/SC.2005.72},
+ acmid = {1105819},
+ publisher = {IEEE Computer Society},
+ address = {Washington, DC, USA},
+}
@INPROCEEDINGS{almeroth00:mcastmonitor,
author = {K. Almeroth},
@@ -1514,20 +1520,20 @@
month = nov
}
- at incollection {GRAM_1998,
- author = {Czajkowski, Karl and Foster, Ian and Karonis, Nick and Kesselman, Carl and Martin, Stuart and Smith, Warren and Tuecke, Steven},
- affiliation = {University of Southern California Information Sciences Institute Marina del Rey 90292-6695 CA},
- title = {A resource management architecture for metacomputing systems},
- booktitle = {Job Scheduling Strategies for Parallel Processing},
- series = {Lecture Notes in Computer Science},
- editor = {Feitelson, Dror and Rudolph, Larry},
- publisher = {Springer Berlin / Heidelberg},
- isbn = {},
- pages = {62-82},
- volume = {1459},
- url = {http://dx.doi.org/10.1007/BFb0053981},
- note = {10.1007/BFb0053981},
- year = {1998}
+ at incollection {GRAM_1998,
+ author = {Czajkowski, Karl and Foster, Ian and Karonis, Nick and Kesselman, Carl and Martin, Stuart and Smith, Warren and Tuecke, Steven},
+ affiliation = {University of Southern California Information Sciences Institute Marina del Rey 90292-6695 CA},
+ title = {A resource management architecture for metacomputing systems},
+ booktitle = {Job Scheduling Strategies for Parallel Processing},
+ series = {Lecture Notes in Computer Science},
+ editor = {Feitelson, Dror and Rudolph, Larry},
+ publisher = {Springer Berlin / Heidelberg},
+ isbn = {},
+ pages = {62-82},
+ volume = {1459},
+ url = {http://dx.doi.org/10.1007/BFb0053981},
+ note = {10.1007/BFb0053981},
+ year = {1998}
}
@@ -1593,15 +1599,25 @@
timestamp = {2010.01.25}
}
- at INPROCEEDINGS{MapReduce_2004,
- author = {Jeffrey Dean and Sanjay Ghemawat},
- title = {{MapReduce}: {Simplified} Data Processing on Large Clusters},
- booktitle = OSDI,
- year = {2004},
- owner = {wozniak},
- timestamp = {2008-09-16}
-}
+ at article{MapReduce_2004,
+ author = {Dean, Jeffrey and Ghemawat, Sanjay},
+ title = {MapReduce: simplified data processing on large clusters},
+ journal = {Commun. ACM},
+ volume = {51},
+ issue = {1},
+ month = {January},
+ year = {2008},
+ issn = {0001-0782},
+ pages = {107--113},
+ numpages = {7},
+ url = {http://doi.acm.org/10.1145/1327452.1327492},
+ doi = {http://doi.acm.org/10.1145/1327452.1327492},
+ acmid = {1327492},
+ publisher = {ACM},
+ address = {New York, NY, USA},
+}
+
@ARTICLE{OOPS_2009,
author = {Joe DeBartolo and Andres Colubri and Ahbishek K. Jha and James Fitzgerald
and Karl F. Freed, Tobin R. Sosnick},
@@ -1633,7 +1649,9 @@
distributed systems},
journal = {Scientific Programming},
year = {2005},
+ pages={219--237},
volume = {13},
+ issue={3},
timestamp = {2007.10.21}
}
@@ -2040,15 +2058,19 @@
url = {citeseer.ist.psu.edu/french90scientific.html}
}
- at ARTICLE{Condor-G_2002,
- author = {James Frey and Todd Tannenbaum and Ian Foster and Miron Livny and
- Steven Tuecke},
- title = {{Condor-G:} {A} Computation Management Agent for Multi-Institutional
- Grids},
- journal = {Cluster Computing},
- year = {2002},
- volume = {5},
- number = {3}
+ at article {Condor-G_2002,
+ author = {Frey, James and Tannenbaum, Todd and Livny, Miron and Foster, Ian and Tuecke, Steven},
+ title = {{Condor-G: A Computation Management Agent for Multi-Institutional Grids}},
+ journal = {Cluster Computing},
+ publisher = {Springer Netherlands},
+ issn = {1386-7857},
+ keyword = {Computer Science},
+ pages = {237-246},
+ volume = {5},
+ issue = {3},
+ url = {http://dx.doi.org/10.1023/A:1015617019423},
+ note = {10.1023/A:1015617019423},
+ year = {2002}
}
@INPROCEEDINGS{RT-Clusters_2006,
@@ -3838,14 +3860,18 @@
}
@ARTICLE{Scripting_1998,
- author = {John Ousterhout},
- title = {Scripting: {Higher-level} programming for the 21st century},
- journal = {{IEEE} Computer},
- year = {1998},
- month = Mar,
- timestamp = {2009.09.01}
-}
+author={Ousterhout, J.K.},
+journal={Computer}, title={Scripting: higher level programming for the 21st Century},
+year={1998},
+month=mar,
+volume={31},
+number={3},
+pages={23--30},
+keywords={21st Century;C;C++;Perl;Tcl;data structures;higher level programming;scripting;scripting languages;software components;system programming languages;authoring languages;programming;},
+doi={10.1109/2.660187},
+ISSN={0018-9162},}
+
@ARTICLE{Sprite_1988,
author = {John K. Ousterhout and Andrew R. Cherenson and Frederick Douglis
and Michael N. Nelson and Brent B. Welch},
@@ -4099,15 +4125,23 @@
timestamp = {2009.09.02}
}
- at INPROCEEDINGS{Falkon_2008,
- author = {Ioan Raicu and Zhao Zhang and Mike Wilde and Ian Foster and Pete
- Beckman and Kamil Iskra and Ben Clifford},
- title = {Towards Loosely-Coupled Programming on Petascale Systems},
- booktitle = SC08,
- year = {2008},
- owner = {wozniak},
- timestamp = {2009.02.05}
-}
+ at inproceedings{Falkon_2008,
+ author = {Raicu, Ioan and Zhang, Zhao and Wilde, Mike and Foster, Ian and Beckman, Pete and Iskra, Kamil and Clifford, Ben},
+ title = {Toward loosely coupled programming on petascale systems},
+ booktitle = {Proceedings of the 2008 ACM/IEEE Conference on Supercomputing},
+ series = {SC '08},
+ year = {2008},
+ isbn = {978-1-4244-2835-9},
+ location = {Austin, Texas},
+ pages = {22:1--22:12},
+ articleno = {22},
+ numpages = {12},
+ url = {http://portal.acm.org/citation.cfm?id=1413370.1413393},
+ acmid = {1413393},
+ publisher = {IEEE Press},
+ address = {Piscataway, NJ, USA},
+ keywords = {Blue Gene, Falkon, Swift, high throughput computing, loosely coupled applications, many task computing, petascale},
+}
@INPROCEEDINGS{Falkon_2007,
author = {Ioan Raicu and Yong Zhao and Catalin Dumitrescu and Ian Foster and
@@ -4431,8 +4465,8 @@
@incollection{NFS_1985,
author = {Sandberg, R. and Golgberg, D. and Kleiman, S. and Walsh, D. and Lyon, B.},
- chapter = {Design and implementation of the Sun network filesystem},
- title = {Innovations in Internetworking},
+ title = {Design and implementation of the Sun network filesystem},
+ booktitle = {Innovations in Internetworking},
year = {1988},
isbn = {0-89006-337-0},
pages = {379--390},
@@ -5140,6 +5174,8 @@
year = {2005},
volume = {17},
number = {2-4},
+ pages={323--356},
+ doi={10.1002/cpe.938},
timestamp = {2007.07.05}
}
@@ -5931,17 +5967,23 @@
timestamp = {2007-10-25}
}
- at INPROCEEDINGS{CDM_2009,
- author = {Justin M. Wozniak and Michael Wilde},
- title = {Case Studies in Storage Access by Loosely Coupled Petascale Applications},
- booktitle = {Proc. Petascale Data Storage Workshop at SC'09},
- year = {2009},
- owner = {wozniak},
- timestamp = {2010/04/02}
-}
+ at inproceedings{CDM_2009,
+ author = {Wozniak, Justin M. and Wilde, Michael},
+ title = {Case studies in storage access by loosely coupled petascale applications},
+ booktitle = {Proceedings of the 4th Annual Workshop on Petascale Data Storage},
+ series = {PDSW '09},
+ year = {2009},
+ isbn = {978-1-60558-883-4},
+ location = {Portland, Oregon},
+ pages = {16--20},
+ numpages = {5},
+ url = {http://doi.acm.org/10.1145/1713072.1713078},
+ doi = {http://doi.acm.org/10.1145/1713072.1713078},
+ acmid = {1713078},
+ publisher = {ACM},
+ address = {New York, NY, USA},
+}
-{jabref-meta: selector_publisher:1;}
-
@INBOOK{P2P-Grids_2003,
author = {Geoffrey Fox and Dennis Gannon and Sung-Hoon Ko and
Sangmi-Lee and Shrideep Pallickara and Marlon Pierce and
@@ -6068,30 +6110,32 @@
}
{Pages: 541--551}
- at ARTICLE{OSG_2007,
- author = {Ruth Pordes and Don Petravick and Bill Kramer and
- Doug Olson and Miron Livny and Alain Roy and
- Paul Avery and Kent Blackburn and Torre Wenaus and
- Frank Wurthwein and Ian Foster and Rob Gardner and
- Mike Wilde and Alan Blatecky and John McGee and Rob Quick},
- title = {The {O}pen {S}cience {G}rid},
- journal = {Journal of Physics: Conference Series},
- volume = {78},
- number = {1},
- year = {2007},
- abstract = {The Open Science Grid (OSG) provides a distributed facility where the Consortium members provide guaranteed and opportunistic access to shared computing and storage resources. OSG provides support for and evolution of the infrastructure through activities that cover operations, security, software, troubleshooting, addition of new capabilities, and support for existing and engagement with new communities. The OSG SciDAC-2 project provides specific activities to manage and evolve the distributed infrastructure and support it's use. The innovative aspects of the project are the maintenance and performance of a collaborative (shared & common) petascale national facility over tens of autonomous computing sites, for many hundreds of users, transferring terabytes of data a day, executing tens of thousands of jobs a day, and providing robust and usable resources for scientific groups of all types and sizes. More information can be found at the OSG web site: www.opensci
encegrid.org.}
+ at article{OSG_2007,
+ author={Ruth Pordes and Don Petravick and Bill Kramer and Doug Olson and Miron Livny and Alain Roy and Paul Avery and Kent
+Blackburn and Torre Wenaus and Frank W?rthwein and Ian Foster and Rob Gardner and Mike Wilde and Alan Blatecky and John
+McGee and Rob Quick},
+ title={{The Open Science Grid}},
+ journal={Journal of Physics: Conference Series},
+ volume={78},
+ number={1},
+ pages={012057},
+ url={http://stacks.iop.org/1742-6596/78/i=1/a=012057},
+ year={2007},
+ abstract={The Open Science Grid (OSG) provides a distributed facility where the Consortium members provide guaranteed and opportunistic access to shared computing and storage resources. OSG provides support for and evolution of the infrastructure through activities that cover operations, security, software, troubleshooting, addition of new capabilities, and support for existing and engagement with new communities. The OSG SciDAC-2 project provides specific activities to manage and evolve the distributed infrastructure and support it's use. The innovative aspects of the project are the maintenance and performance of a collaborative (shared & common) petascale national facility over tens of autonomous computing sites, for many hundreds of users, transferring terabytes of data a day, executing tens of thousands of jobs a day, and providing robust and usable resources for scientific groups of all types and sizes. More information can be found at the OSG web site: www.openscien
cegrid.org.}
}
-
+
@ARTICLE{TeraGrid_2005,
title = {Building the {TeraGrid}},
author = {Peter H Beckman},
journal = PTRS_A,
volume = 363,
number = 1833,
- year = 2005
-}
-{pages 1715-1728}
+ year = 2005,
+ doi={10.1098/rsta.2005.1602},
+ pages={1715--1728}
+ }
+
@MISC{RFC:4918_WebDAV_2007,
author = {{IETF} {Network Working Group}},
title = {{HTTP} Extensions for Web Distributed
@@ -6150,13 +6194,21 @@
year = 2010
}
- at INPROCEEDINGS{FTSH_2003,
- title = {The ethernet approach to grid computing},
- author = {Thain, Douglas and Livny, Miron},
- booktitle = HPDC,
- year = 2003,
-}
+ at inproceedings{FTSH_2003,
+ author = {Thain, Douglas and Livny, Miron},
+ title = {The Ethernet Approach to Grid Computing},
+ booktitle = {Proceedings of the 12th IEEE International Symposium on High Performance Distributed Computing},
+ series = {HPDC '03},
+ year = {2003},
+ isbn = {0-7695-1965-2},
+ pages = {138--},
+ url = {http://portal.acm.org/citation.cfm?id=822087.823417},
+ acmid = {823417},
+ publisher = {IEEE Computer Society},
+ address = {Washington, DC, USA},
+}
+
@INPROCEEDINGS{MatlabCondor_2003,
title = {Implementation of a Grid Computation Toolkit for
Design Optimisation with {M}atlab and {C}ondor},
@@ -6189,18 +6241,26 @@
title = {{Strand}: {A} practical parallel programming language},
author = {Foster, I. and Taylor, S.},
booktitle = {Proc. North American Conference on Logic Programming},
- year = 1989
+ year = 1989,
+ pages={497--512}
}
- at ARTICLE{PCN_1993,
- title = {Productive parallel programming: {T}he {PCN} approach},
- author = {Foster, I. and Olson, R. and Tuecke, S.},
- journal = {Scientific Programming},
- volume = 1,
- number = 1,
- year = 1992,
-}
-{pages = {51--66}, publisher={IOS Press}}
+ at article{PCN_1993,
+ author = {Foster, Ian and Olson, Robert and Tuecke, Steven},
+ title = {Productive Parallel Programming: The PCN Approach},
+ journal = {Sci. Program.},
+ volume = {1},
+ issue = {1},
+ month = {January},
+ year = {1992},
+ issn = {1058-9244},
+ pages = {51--66},
+ numpages = {16},
+ url = {http://portal.acm.org/citation.cfm?id=1402583.1402587},
+ acmid = {1402587},
+ publisher = {IOS Press},
+ address = {Amsterdam, The Netherlands, The Netherlands},
+}
@article{Sawzall_2005,
title = {Interpreting the data: {P}arallel analysis with {Sawzall}},
@@ -6209,27 +6269,32 @@
volume = {13},
number = {4},
year = 2005,
+ pages = {277--298},
+ publisher={IOS Press}
}
-{pages = {277--298}, publisher={IOS Press}}
@BOOK{BPEL_2006,
author = {Juric, Matjaz B.},
title = {Business Process Execution Language for Web Services},
year = 2006,
+ publisher={Packt Publishing}
}
isbn = 1904811817, publisher = {Packt Publishing}
- at INBOOK{Sedna_2007,
- chapter = {{S}edna: {A} {BPEL}-Based Environment for
- Visual Scientific Workflow Modeling},
- title = {Workflows for e-{S}cience},
- publisher = {Springer},
- year = 2007,
- author = {Bruno Wassermann and Wolfgang Emmerich and Ben
- Butchart and Nick Cameron and Liang Chen and Jignesh
- Patel}
+ at incollection {Sedna_2007,
+ author = {Wassermann, Bruno and Emmerich, Wolfgang and Butchart, Ben and Cameron, Nick and Chen, Liang and Patel, Jignesh},
+ affiliation = {University College London Software Systems Engineering Group, Department of Computer Science Gower Street London WC1E 6BT UK},
+ title = {Sedna: A BPEL-Based Environment for Visual Scientific Workflow Modeling},
+ booktitle = {Workflows for e-Science},
+ editor = {Taylor, Ian J. and Deelman, Ewa and Gannon, Dennis B. and Shields, Matthew},
+ publisher = {Springer London},
+ isbn = {978-1-84628-757-2},
+ keyword = {Computer Science},
+ pages = {428--449},
+ url = {http://dx.doi.org/10.1007/978-1-84628-757-2\_26},
+ note = {10.1007/978-1-84628-757-2\_26},
+ year = {2007}
}
-pages = {18},
@INPROCEEDINGS{SunConstellation_2008,
title = {{Performance and Scalability Study of
@@ -6237,15 +6302,22 @@
using Application-Based Benchmarks}},
author = {Byoung-Do Kim and John E. Cazes},
booktitle = {Proc. TeraGrid'2008},
+ pages={},
year = 2008
}
- at INPROCEEDINGS{ReSS_2007,
- author = {G. Garzoglio and T. Levshina and P. Mhashilkar and S. Timm},
- title = {{ReSS}: {A} Resource Selection Service for the
- {O}pen {S}cience {G}rid},
- booktitle = {Proc. International Symposium of Grid Computing},
- year = 2007
+ at incollection {ReSS_2007,
+ author = {Garzoglio, Gabriele and Levshina, Tanya and Mhashilkar, Parag and Timm, Steve},
+ title = {{ReSS: A Resource Selection Service for the Open Science Grid}},
+ booktitle = {Grid Computing},
+ editor = {Lin, Simon C. and Yen, Eric},
+ publisher = {Springer US},
+ isbn = {978-0-387-78417-5},
+ keyword = {Computer Science},
+ pages = {89--98},
+ url = {http://dx.doi.org/10.1007/978-0-387-78417-5\_8},
+ note = {10.1007/978-0-387-78417-5\_8},
+ year = {2009}
}
@ARTICLE{CNARI_2007,
From noreply at svn.ci.uchicago.edu Thu Dec 9 16:28:16 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 9 Dec 2010 16:28:16 -0600 (CST)
Subject: [Swift-commit] r3759 - in provenancedb: . apps/oops
Message-ID: <20101209222816.A59579CC7F@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-09 16:28:16 -0600 (Thu, 09 Dec 2010)
New Revision: 3759
Modified:
provenancedb/apps/oops/oops_extractor.sh
provenancedb/swift-prov-import-all-logs
Log:
Minor updates to provenancedb.
Modified: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh 2010-12-09 05:32:32 UTC (rev 3758)
+++ provenancedb/apps/oops/oops_extractor.sh 2010-12-09 22:28:16 UTC (rev 3759)
@@ -18,18 +18,15 @@
fi
cd $PROTESTS_HOME
-for k in `ls -1`;
+for k in $(ls -1);
do
cd $PROTESTS_HOME/$k
- for i in `ls | grep run.loops`;
+ for i in $(ls | grep run.loops);
do
cd $IMPORT_HOME
if ! grep --silent $i provdb_imported; then
if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/psim.loops-*.log; then
cd swift-logs
- #for j in `ls $PROTESTS_HOME/$k/$i | grep psim.loops-`; do
- # ln -s $PROTESTS_HOME/$k/$i/$j
- #done
cd import
# swift-prov-import-all-logs also controls what already has been
# imported, so it does not repeat work
@@ -39,45 +36,41 @@
cd $IMPORT_HOME
echo $i >> provdb_imported
cd swift-logs
- # annotate workflows with their oops runid
- OOPS_RUN_ID=`echo $i | awk -F . '{print $3}'`
+
+ # annotate workflows with their oops runid
+ OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}')
cd $PROTESTS_HOME/$k/$i
- LOG_FILENAME=`ls | grep psim.loops- | grep "\."log$`
- WORKFLOW_ID=`echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}'`
+ LOG_FILENAME=$(ls | grep psim.loops- | grep "\."log$)
+ WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}')
cd $IMPORT_HOME/swift-logs
echo "insert into annot_wf_txt (id, name, value) values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD
- echo "select id,filename from file where filename like '%params%' and id in (select in_id from ds_containment where out_id in (select ds_usage.dataset_id from ds_usage,process,execute where ds_usage.process_id=process.id and process.id=execute.id and execute.procedure_name='loopPrepare' and ds_usage.direction='I' and process.workflow_id like '%$WORKFLOW_ID%'));" > query.sql
+ #
+ echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql;
- #query for the previous database schema
- #echo "select dataset_id,filename from dataset_filenames where filename like '%params%' and dataset_id in (select inner_dataset_id from dataset_containment where outer_dataset_id in (select dataset_usage.dataset_id from invocation_procedure_names,dataset_usage,processes_in_workflows where invocation_procedure_names.execute_id=dataset_usage.process_id and dataset_usage.process_id=processes_in_workflows.process_id and invocation_procedure_names.procedure_name='loopPrepare' and dataset_usage.direction='I' and processes_in_workflows.workflow_id like '%$WORKFLOW_ID%'));" > query.sql
-
$SQLCMD -t -A -F " " -f query.sql -o result.txt
- #DATASET_ID=`awk '{print $1}' result.txt`
- DATASET_ID=`awk '{if (NR==1) print $1}' result.txt`
+ DATASET_ID=$(awk '{print $1}' result.txt)
+ FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g')
- #FILENAME=`awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g'`
- FILENAME=`awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g'`
-
cd $PROTESTS_HOME/$k/run.loops.$OOPS_RUN_ID
while read line; do
- NAME=`echo $line | awk 'BEGIN { FS = "=" }; {print $1}'`
+ NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}')
if [ "$NAME" = "SAMPLE RANGE" ]; then
- VALUE1=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}'`
- VALUE2=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}'`
+ VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}')
+ VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}')
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD
fi
if [ "$NAME" = "RESTRAIN DISTANCE" ]; then
- VALUE1=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}'`
- VALUE2=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}'`
+ VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}')
+ VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}')
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD
fi
if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then
- VALUE=`echo $line | awk 'BEGIN { FS = "=" }; {print $2}'`
+ VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}')
echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD
fi
done < $FILENAME
Modified: provenancedb/swift-prov-import-all-logs
===================================================================
--- provenancedb/swift-prov-import-all-logs 2010-12-09 05:32:32 UTC (rev 3758)
+++ provenancedb/swift-prov-import-all-logs 2010-12-09 22:28:16 UTC (rev 3759)
@@ -41,7 +41,7 @@
EXISTING=$($SQLCMD --tuples-only -c "select count(*) from workflow where log_filename='$filename';")
if [ "$EXISTING" -eq "0" ]; then
- PROV_ENABLED=$(grep provenance $filename | wc -l)
+ PROV_ENABLED=$(grep provenanceid $filename | wc -l)
if [ $PROV_ENABLED -gt 0 ]; then
echo IMPORTING
From noreply at svn.ci.uchicago.edu Fri Dec 10 12:12:34 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 10 Dec 2010 12:12:34 -0600 (CST)
Subject: [Swift-commit] r3760 - provenancedb/apps/oops
Message-ID: <20101210181234.C8CD49D908@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-10 12:12:34 -0600 (Fri, 10 Dec 2010)
New Revision: 3760
Added:
provenancedb/apps/oops/raptor_extractor.sh
Modified:
provenancedb/apps/oops/oops_extractor.sh
Log:
RaptorLoops annotation extractor for provenancedb.
Modified: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh 2010-12-09 22:28:16 UTC (rev 3759)
+++ provenancedb/apps/oops/oops_extractor.sh 2010-12-10 18:12:34 UTC (rev 3760)
@@ -2,8 +2,6 @@
# Annotation extractor for the OOPS application
#
-# The directory $PROTESTS/swift-logs contains symbolic links to
-# OOPS' Swift logs.
PROVDB_HOME=~/provenancedb
PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP
Added: provenancedb/apps/oops/raptor_extractor.sh
===================================================================
--- provenancedb/apps/oops/raptor_extractor.sh (rev 0)
+++ provenancedb/apps/oops/raptor_extractor.sh 2010-12-10 18:12:34 UTC (rev 3760)
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Annotation extractor for the OOPS application
+#
+# The directory $PROTESTS/swift-logs contains symbolic links to
+# OOPS' Swift logs.
+PROVDB_HOME=~/provenancedb
+PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP
+IMPORT_HOME=~/protests
+
+source $PROVDB_HOME/etc/provenance.config
+
+# provdb_imported records runs already imported to the provenance database
+cd $IMPORT_HOME
+if [ ! -a provdb_imported_raptor ]; then
+ touch provdb_imported_raptor
+fi
+
+cd $PROTESTS_HOME
+for k in $(ls -1);
+do
+ cd $PROTESTS_HOME/$k
+ for i in $(ls | grep run.raptorloops);
+ do
+ cd $IMPORT_HOME
+ if ! grep --silent $i provdb_imported_raptor; then
+ if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/RaptorLoops-*.log; then
+ # swift-prov-import-all-logs also controls what already has been
+ # imported, so it does not repeat work
+ echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config
+ echo "export SQLCMD=\"psql -U provdb -h db.ci.uchicago.edu provdb\"" >> $PROVDB_HOME/etc/provenance.config
+ $PROVDB_HOME/swift-prov-import-all-logs
+ cd $IMPORT_HOME
+ echo $i >> provdb_imported_raptor
+ cd swift-logs
+ # annotate workflows with their oops runid
+ OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}')
+ cd $PROTESTS_HOME/$k/$i
+ LOG_FILENAME=$(ls | grep RaptorLoops- | grep "\."log$)
+ WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}')
+ cd $IMPORT_HOME/swift-logs
+ echo "insert into annot_wf_txt values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD
+
+ echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql;
+
+ $SQLCMD -t -A -F " " -f query.sql -o result.txt
+
+ DATASET_ID=$(awk '{print $1}' result.txt)
+ FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g')
+
+ cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID
+
+ while read line; do
+ NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}')
+ if [ "$NAME" = "SAMPLE RANGE" ]; then
+ VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}')
+ VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}')
+ echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD
+ echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD
+ fi
+ if [ "$NAME" = "RESTRAIN DISTANCE" ]; then
+ VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}')
+ VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}')
+ echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD
+ echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD
+ fi
+ if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then
+ VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}')
+ echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD
+ fi
+ done < $FILENAME
+ fi
+ fi
+ done
+done
Property changes on: provenancedb/apps/oops/raptor_extractor.sh
___________________________________________________________________
Name: svn:executable
+ *
From noreply at svn.ci.uchicago.edu Fri Dec 10 12:40:14 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 10 Dec 2010 12:40:14 -0600 (CST)
Subject: [Swift-commit] r3761 - provenancedb/apps/oops
Message-ID: <20101210184014.65C829D908@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-10 12:40:14 -0600 (Fri, 10 Dec 2010)
New Revision: 3761
Modified:
provenancedb/apps/oops/raptor_extractor.sh
Log:
Minor fixes to RaptorLoops provenance extractor.
Modified: provenancedb/apps/oops/raptor_extractor.sh
===================================================================
--- provenancedb/apps/oops/raptor_extractor.sh 2010-12-10 18:12:34 UTC (rev 3760)
+++ provenancedb/apps/oops/raptor_extractor.sh 2010-12-10 18:40:14 UTC (rev 3761)
@@ -45,8 +45,8 @@
$SQLCMD -t -A -F " " -f query.sql -o result.txt
- DATASET_ID=$(awk '{print $1}' result.txt)
- FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g')
+ DATASET_ID=`awk '{if (NR==1) print $1}' result.txt`
+ FILENAME=`awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g'`
cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID
From noreply at svn.ci.uchicago.edu Fri Dec 10 15:32:41 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 10 Dec 2010 15:32:41 -0600 (CST)
Subject: [Swift-commit] r3762 - trunk/tests
Message-ID: <20101210213241.BB6789D908@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-10 15:32:41 -0600 (Fri, 10 Dec 2010)
New Revision: 3762
Added:
trunk/tests/site/
Log:
Site-specific test groups
From noreply at svn.ci.uchicago.edu Fri Dec 10 15:33:04 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 10 Dec 2010 15:33:04 -0600 (CST)
Subject: [Swift-commit] r3763 - trunk/tests/site
Message-ID: <20101210213304.8C2049D908@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-10 15:33:04 -0600 (Fri, 10 Dec 2010)
New Revision: 3763
Added:
trunk/tests/site/intrepid/
Log:
Site-specific tests for Intrepid
From noreply at svn.ci.uchicago.edu Sat Dec 11 16:24:50 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sat, 11 Dec 2010 16:24:50 -0600 (CST)
Subject: [Swift-commit] r3764 - SwiftApps/SwiftR/Swift/man
Message-ID: <20101211222450.95F369CC80@svn.ci.uchicago.edu>
Author: wilde
Date: 2010-12-11 16:24:49 -0600 (Sat, 11 Dec 2010)
New Revision: 3764
Modified:
SwiftApps/SwiftR/Swift/man/Swift-package.Rd
Log:
Minor doc edits.
Modified: SwiftApps/SwiftR/Swift/man/Swift-package.Rd
===================================================================
--- SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-12-10 21:33:04 UTC (rev 3763)
+++ SwiftApps/SwiftR/Swift/man/Swift-package.Rd 2010-12-11 22:24:49 UTC (rev 3764)
@@ -123,13 +123,13 @@
Download the appropriate Java Runtime (JRE) for Linux at:\verb{
http://www.java.com/en/download/manual.jsp}
-Typically either 32 bit with this link:\verb{
- Linux (self-extracting file) filesize: 19.9}
+Typically either 32 bit with the link named:\verb{
+ "Linux (self-extracting file) filesize: 19.9"}
-Or 64 bit with this link\verb{
- Linux x64 * filesize: 19.3 MB }
+...or 64 bit with the link named: \verb{
+ "Linux x64 (self-extracting file) filesize: 19.3 MB" }
-Its better to install the JDK (~80MB) from:
+Its better to install the full development kit (JDK, ~80MB) from:
http://www.oracle.com/technetwork/java/javase/downloads/jdk6-jsp-136632.html (JDKs)
@@ -141,11 +141,12 @@
Verify that you have Sun Java installed and in your PATH correctly by doing:
+\verb{
$ java -version
java version "1.6.0_21"
Java(TM) SE Runtime Environment (build 1.6.0_21-b06)
Java HotSpot(TM) 64-Bit Server VM (build 17.0-b16, mixed mode)
-$
+$ }
2) R v2.11 or higher in your PATH (on client and server machines)
@@ -173,13 +174,14 @@
\section{INSTALLATION}{
+\preformatted{
mkdir ~/RPackages ~/RLibrary # if not already created
cd ~/RPackages
wget http://www.ci.uchicago.edu/~wilde/Swift_0.1.tar.gz
R CMS INSTALL -l ~/RLibrary Swift_0.1.tar.gz
export R_LIBS=~/RLibrary
-export GLOBUS_HOSTNAME=10.0.0.200 # Eg for Merlot: internal address of the login node
+export GLOBUS_HOSTNAME=10.0.0.200 # Eg for Merlot: internal address of the login node}
}
\section{QUICK_START}{
@@ -211,7 +213,7 @@
its best to set this to /scratch/local, as /tmp is very limited in
space. Seldom needed.
-GLOBUS_HOSTNAME should be set to the IP address of the login host if
+For the pbsf server type only: GLOBUS_HOSTNAME should be set to the IP address of the login host if
it contains multiple network interfaces (see /sbin/ifconfig) and if
the worker nodes can only reach the login host on a subset of these
interfaces.
From noreply at svn.ci.uchicago.edu Sat Dec 11 16:46:59 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sat, 11 Dec 2010 16:46:59 -0600 (CST)
Subject: [Swift-commit] r3765 - SwiftApps/SwiftR/Swift/exec
Message-ID: <20101211224659.4D0559CC7F@svn.ci.uchicago.edu>
Author: wilde
Date: 2010-12-11 16:46:59 -0600 (Sat, 11 Dec 2010)
New Revision: 3765
Added:
SwiftApps/SwiftR/Swift/exec/configure-server-pbs
Log:
Add config script for local pbs clusters
Added: SwiftApps/SwiftR/Swift/exec/configure-server-pbs
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-pbs (rev 0)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2010-12-11 22:46:59 UTC (rev 3765)
@@ -0,0 +1,82 @@
+#! /bin/bash
+
+# configuration for PBS with manually-started Swift workers (passive coasters)
+
+cores=$1
+
+throttlePBS=5.0
+
+cat >tc <sites.xml <
+
+
+
+ 0.15
+ 10000
+
+ $(pwd)/swiftwork
+
+
+
+
+ passive
+ $cores
+ $throttlePBS
+ 10000
+
+ $HOME/swiftwork
+
+
+
+END
+
+cat >cf <00:00:01
+# fast
+
+
+
+ fast
+ 00:59:00
+ $throttlePBS
+ 10000
+
+ $HOME/swiftwork
+
+
+
+
+ short
+ 12000
+ 32
+ 1
+ 1
+ 1
+ 2.55
+ 10000
+
+ $HOME/swiftwork
+
+
+END
Property changes on: SwiftApps/SwiftR/Swift/exec/configure-server-pbs
___________________________________________________________________
Name: svn:executable
+ *
From noreply at svn.ci.uchicago.edu Sun Dec 12 17:29:55 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 12 Dec 2010 17:29:55 -0600 (CST)
Subject: [Swift-commit] r3766 - in provenancedb: . apps/oops
Message-ID: <20101212232955.784239CC80@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-12 17:29:54 -0600 (Sun, 12 Dec 2010)
New Revision: 3766
Modified:
provenancedb/apps/oops/oops_extractor.sh
provenancedb/apps/oops/raptor_extractor.sh
provenancedb/prov-init.sql
Log:
Minor changes to OOPS provenance extractor.
Modified: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh 2010-12-11 22:46:59 UTC (rev 3765)
+++ provenancedb/apps/oops/oops_extractor.sh 2010-12-12 23:29:54 UTC (rev 3766)
@@ -19,13 +19,11 @@
for k in $(ls -1);
do
cd $PROTESTS_HOME/$k
- for i in $(ls | grep run.loops);
+ for i in $(ls | grep run.raptorloops; ls | grep run.loops);
do
cd $IMPORT_HOME
if ! grep --silent $i provdb_imported; then
if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/psim.loops-*.log; then
- cd swift-logs
- cd import
# swift-prov-import-all-logs also controls what already has been
# imported, so it does not repeat work
echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config
@@ -38,40 +36,85 @@
# annotate workflows with their oops runid
OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}')
cd $PROTESTS_HOME/$k/$i
- LOG_FILENAME=$(ls | grep psim.loops- | grep "\."log$)
+ LOG_FILENAME=$(ls *-*-*-*.log)
WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}')
cd $IMPORT_HOME/swift-logs
echo "insert into annot_wf_txt (id, name, value) values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD
- #
- echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql;
-
+ #extracts scientific parameters given as input to the workflow in file *.params.
+ echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID' and ds_usage.direction='O';" > query.sql;
+
$SQLCMD -t -A -F " " -f query.sql -o result.txt
- DATASET_ID=$(awk '{print $1}' result.txt)
- FILENAME=$(awk '{print $2}' result.txt | sed 's/file:\/\/localhost\///g')
+ DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt)
+ FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g')
- cd $PROTESTS_HOME/$k/run.loops.$OOPS_RUN_ID
+ cd $PROTESTS_HOME/$k/$i
while read line; do
NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}')
+ RIGHT=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}')
if [ "$NAME" = "SAMPLE RANGE" ]; then
- VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}')
- VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}')
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD
+ echo "insert into annot_ds_txt values ('$DATASET_ID', 'sample_range', '$RIGHT');" | $SQLCMD
fi
if [ "$NAME" = "RESTRAIN DISTANCE" ]; then
- VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}')
+ VALUE1=$(echo $RIGHT | awk 'BEGIN { FS = "," }; {print $1}')
VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}')
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD
+ echo "insert into annot_ds_num values ('$DATASET_ID', 'restrain_distance_1', $VALUE1);" | $SQLCMD
+ echo "insert into annot_ds_num values ('$DATASET_ID', 'restrain_distance_2', $VALUE2);" | $SQLCMD
fi
if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then
- VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}')
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD
+ echo "insert into annot_ds_num values ('$DATASET_ID', 'maximum_number_of_steps', $RIGHT);" | $SQLCMD
fi
done < $FILENAME
+
+ # extracts scientific parameters given as input to the workflow in file *.params.
+ # relevant lines:
+ # zone2 (Initial Energy: -21352.116911)
+ # Total Function Evaluations: 20000
+ # Accepted transitions: 7410
+ # Increasing transitions: 4525
+ # Decreasing transitions: 2885
+ # Rejected transitions: 12590
+ # Final Energy: -27152.264775
+ # Final Temp: 79.778142
+ # Total Running Time: 18006
+
+ echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.log' and process.name='LoopModel' and process.workflow_id='$WORKFLOW_ID' and ds_usage.direction='O';" > query.sql;
+
+ $SQLCMD -t -A -F " " -f query.sql -o result.txt
+
+ while read dataset filename; do
+ DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt)
+ FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g')
+ while read token1 token2 token3 token4; do
+ if [ "$token2" = "(Initial Energy:" ]; then
+ initialenergy=$(echo $token3 | awk 'BEGIN { FS = "\)" }; {print $1}')
+ echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'initial_energy', $initialenergy);" | $SQLCMD
+ fi
+ if [ "$token1" = "Total" && "$token2" = "Function" && "$token3" = "Evaluations:" ]; then
+ echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'total_function_evaluations', $token4);" | $SQLCMD
+ fi
+ if [ "$token1" = "Increasing" && "$token2" = "transitions:" ]; then
+ echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'accepted_increasing_transitions', $token3);" | $SQLCMD
+ fi
+ if [ "$token1" = "Decreasing" && "$token2" = "transitions:" ]; then
+ echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'accepted_decreasing_transitions', $token3);" | $SQLCMD
+ fi
+ if [ "$token1" = "Rejected" && "$token2" = "transitions:" ]; then
+ echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'rejected_transitions', $token3);" | $SQLCMD
+ fi
+ if [ "$token1" = "Final" && "$token2" = "Energy:" ]; then
+ echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'final_energy', $token3);" | $SQLCMD
+ fi
+ if [ "$token1" = "Final" && "$token2" = "Temp:" ]; then
+ echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'final_temp', $token3);" | $SQLCMD
+ fi
+ if [ "$token1" = "Total" && "$token2" = "Running" && "$token3" = "Time:" ]; then
+ echo "insert into annot_ds_num (id, name, value) values ('$dataset', 'total_running_time', $token4);" | $SQLCMD
+ fi
+ done < result.txt
+ done < $FILENAME
fi
fi
done
Modified: provenancedb/apps/oops/raptor_extractor.sh
===================================================================
--- provenancedb/apps/oops/raptor_extractor.sh 2010-12-11 22:46:59 UTC (rev 3765)
+++ provenancedb/apps/oops/raptor_extractor.sh 2010-12-12 23:29:54 UTC (rev 3766)
@@ -20,7 +20,7 @@
for k in $(ls -1);
do
cd $PROTESTS_HOME/$k
- for i in $(ls | grep run.raptorloops);
+ for i in $(ls | grep run.raptorloops; ls | grep run.loops);
do
cd $IMPORT_HOME
if ! grep --silent $i provdb_imported_raptor; then
@@ -33,7 +33,7 @@
cd $IMPORT_HOME
echo $i >> provdb_imported_raptor
cd swift-logs
- # annotate workflows with their oops runid
+ # annotate workflows with their oops runid
OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}')
cd $PROTESTS_HOME/$k/$i
LOG_FILENAME=$(ls | grep RaptorLoops- | grep "\."log$)
@@ -45,8 +45,8 @@
$SQLCMD -t -A -F " " -f query.sql -o result.txt
- DATASET_ID=`awk '{if (NR==1) print $1}' result.txt`
- FILENAME=`awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g'`
+ DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt)
+ FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g')
cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID
Modified: provenancedb/prov-init.sql
===================================================================
--- provenancedb/prov-init.sql 2010-12-11 22:46:59 UTC (rev 3765)
+++ provenancedb/prov-init.sql 2010-12-12 23:29:54 UTC (rev 3766)
@@ -1,41 +1,41 @@
-- this is the schema definition used for the main relational provenance
-- implementation (in both sqlite3 and postgres)
-DROP TABLE dataset CASCADE;
-DROP TABLE file CASCADE;
-DROP TABLE variable CASCADE;
-DROP TABLE ds_containment CASCADE;
-DROP TABLE process CASCADE;
-DROP TABLE execute CASCADE;
-DROP TABLE execute2 CASCADE;
-DROP TABLE workflow CASCADE;
-DROP TABLE ds_usage CASCADE;
-DROP TABLE annot_ds_num CASCADE;
-DROP TABLE annot_ds_txt CASCADE;
-DROP TABLE annot_ds_bool CASCADE;
-DROP TABLE annot_p_num CASCADE;
-DROP TABLE annot_p_txt CASCADE;
-DROP TABLE annot_p_bool CASCADE;
-DROP TABLE annot_wf_num CASCADE;
-DROP TABLE annot_wf_txt CASCADE;
-DROP TABLE annot_wf_bool CASCADE;
--- DROP TABLE extrainfo CASCADE;
-DROP TABLE createarray CASCADE;
-DROP TABLE createarray_member CASCADE;
-DROP TABLE array_range CASCADE;
+drop table dataset cascade;
+drop table file cascade;
+drop table variable cascade;
+drop table ds_containment cascade;
+drop table process cascade;
+drop table execute cascade;
+drop table execute2 cascade;
+drop table workflow cascade;
+drop table ds_usage cascade;
+drop table annot_ds_num cascade;
+drop table annot_ds_txt cascade;
+drop table annot_ds_bool cascade;
+drop table annot_p_num cascade;
+drop table annot_p_txt cascade;
+drop table annot_p_bool cascade;
+drop table annot_wf_num cascade;
+drop table annot_wf_txt cascade;
+drop table annot_wf_bool cascade;
+-- drop table extrainfo cascade;
+drop table createarray cascade;
+drop table createarray_member cascade;
+drop table array_range cascade;
-- workflow stores some information about each workflow log that has
-- been seen by the importer: the log filename, swift version and import
-- status.
-- Might be interesting to store xml translation of the swiftscript code
-- here for prospective provenance/versioning
-CREATE TABLE workflow
- (id VARCHAR(256) PRIMARY KEY,
- log_filename VARCHAR(2048),
- swift_version VARCHAR(16),
- import_status VARCHAR(16),
- start_time NUMERIC,
- duration NUMERIC
+create table workflow
+ (id varchar(256) primary key,
+ log_filename varchar(2048),
+ swift_version varchar(16),
+ import_status varchar(16),
+ start_time numeric,
+ duration numeric
);
-- workflow_run stores the start time and duration for each workflow
@@ -47,14 +47,14 @@
-- );
-- dataset stores all dataset identifiers.
-CREATE TABLE dataset
- (id VARCHAR(256) PRIMARY KEY
+create table dataset
+ (id varchar(256) primary key
);
-- file stores the filename mapped to each dataset.
-CREATE TABLE file
- ( id VARCHAR(256) PRIMARY KEY REFERENCES dataset (id) ON DELETE CASCADE,
- filename VARCHAR(2048)
+create table file
+ ( id varchar(256) primary key references dataset (id) on delete cascade,
+ filename varchar(2048)
);
-- dataset_values stores the value for each dataset which is known to have
@@ -62,9 +62,9 @@
-- to expose that value as an SQL type other than a string, and so (for
-- example) SQL numerical operations should not be expected to work, even
-- though the user knows that a particular dataset stores a numeric value.
-CREATE TABLE variable
- ( id VARCHAR(256) PRIMARY KEY REFERENCES dataset (id) ON DELETE CASCADE,
- value VARCHAR(2048)
+create table variable
+ ( id varchar(256) primary key references dataset (id) on delete cascade,
+ value varchar(2048)
);
-- dataset_containment stores the containment hierarchy between
@@ -74,10 +74,10 @@
-- constructors and accessors, rather than, or in addition to,
-- a containment hierarchy. The relationship (such as array index or
-- structure member name) should also be stored in this table.
-CREATE TABLE ds_containment
- ( out_id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE,
- in_id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE,
- PRIMARY KEY (out_id,in_id)
+create table ds_containment
+ ( out_id varchar(256) references dataset (id) on delete cascade,
+ in_id varchar(256) references dataset (id) on delete cascade,
+ primary key (out_id,in_id)
);
-- process gives information about each process (in the OPM sense)
@@ -88,114 +88,114 @@
-- Having this type here seems poor normalisation, though?
-- process types: internal, rootthread, execute, function, compound, scope, operator
-- maybe create a table for each type?
-CREATE TABLE process
- (id VARCHAR(256) PRIMARY KEY,
- type VARCHAR(16),
- name VARCHAR(256), -- in the case of an execute this refers to the transformation name in tc.data
- workflow_id VARCHAR(256) REFERENCES workflow (id) ON DELETE CASCADE -- normalize: workflow_id of sub-procedure determined
+create table process
+ (id varchar(256) primary key,
+ type varchar(16),
+ name varchar(256), -- in the case of an execute this refers to the transformation name in tc.data
+ workflow_id varchar(256) references workflow (id) on delete cascade -- normalize: workflow_id of sub-procedure determined
-- by compound procedure
);
-- this gives information about each execute.
-- each execute is identified by a unique URI. other information from
-- swift logs is also stored here. an execute is an OPM process.
-CREATE TABLE execute
- (id VARCHAR(256) PRIMARY KEY REFERENCES process (id) ON DELETE CASCADE,
- procedure_name VARCHAR(256), -- name of the app procedure that invokes the transformation
- start_time NUMERIC,
- duration NUMERIC,
- final_state VARCHAR(16),
- scratch VARCHAR(2048)
+create table execute
+ (id varchar(256) primary key references process (id) on delete cascade,
+ procedure_name varchar(256), -- name of the app procedure that invokes the transformation
+ start_time numeric,
+ duration numeric,
+ final_state varchar(16),
+ scratch varchar(2048)
);
-- this gives information about each execute2, which is an attempt to
-- perform an execution. the execute2 id is tied to per-execution-attempt
-- information such as wrapper logs
-CREATE TABLE execute2
- (id VARCHAR(256) PRIMARY KEY,
- execute_id VARCHAR(256) REFERENCES execute (id) ON DELETE CASCADE,
- start_time NUMERIC,
- duration NUMERIC,
- final_state VARCHAR(16),
- site VARCHAR(256)
+create table execute2
+ (id varchar(256) primary key,
+ execute_id varchar(256) references execute (id) on delete cascade,
+ start_time numeric,
+ duration numeric,
+ final_state varchar(16),
+ site varchar(256)
);
-- dataset_usage records usage relationships between processes and datasets;
-- in SwiftScript terms, the input and output parameters for each
-- application procedure invocation; in OPM terms, the artificts which are
-- input to and output from each process that is a Swift execution
-CREATE TABLE ds_usage
- (process_id VARCHAR(256) REFERENCES process(id) ON DELETE CASCADE,
- direction CHAR(1), -- I or O for input or output
- dataset_id VARCHAR(256) REFERENCES dataset(id) ON DELETE CASCADE,
- param_name VARCHAR(256), -- the name of the parameter in this execute that
+create table ds_usage
+ (process_id varchar(256) references process(id) on delete cascade,
+ direction char(1), -- I or O for input or output
+ dataset_id varchar(256) references dataset(id) on delete cascade,
+ param_name varchar(256), -- the name of the parameter in this execute that
-- this dataset was bound to. sometimes this must
-- be contrived (for example, in positional varargs)
- PRIMARY KEY (process_id,direction,dataset_id,param_name)
+ primary key (process_id,direction,dataset_id,param_name)
);
-- annotations
-CREATE TABLE annot_ds_num
- ( id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE,
- name VARCHAR(256),
- value NUMERIC,
- PRIMARY KEY (id, name)
+create table annot_ds_num
+ ( id varchar(256) references dataset (id) on delete cascade,
+ name varchar(256),
+ value numeric,
+ primary key (id, name)
);
-CREATE TABLE annot_ds_txt
- ( id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE,
- name VARCHAR(256),
- value VARCHAR(2048),
- PRIMARY KEY (id, name)
+create table annot_ds_txt
+ ( id varchar(256) references dataset (id) on delete cascade,
+ name varchar(256),
+ value varchar(2048),
+ primary key (id, name)
);
-CREATE TABLE annot_ds_bool
- ( id VARCHAR(256) REFERENCES dataset (id) ON DELETE CASCADE,
- name VARCHAR(256),
- value BOOLEAN,
- PRIMARY KEY (id, name)
+create table annot_ds_bool
+ ( id varchar(256) references dataset (id) on delete cascade,
+ name varchar(256),
+ value boolean,
+ primary key (id, name)
);
-CREATE TABLE annot_p_num
- ( id VARCHAR(256) REFERENCES process (id) ON DELETE CASCADE,
- name VARCHAR(256),
- value NUMERIC,
- PRIMARY KEY (id, name)
+create table annot_p_num
+ ( id varchar(256) references process (id) on delete cascade,
+ name varchar(256),
+ value numeric,
+ primary key (id, name)
);
-CREATE TABLE annot_p_txt
- ( id VARCHAR(256) REFERENCES process (id) ON DELETE CASCADE,
- name VARCHAR(256),
- value VARCHAR(2048),
- PRIMARY KEY (id, name)
+create table annot_p_txt
+ ( id varchar(256) references process (id) on delete cascade,
+ name varchar(256),
+ value varchar(2048),
+ primary key (id, name)
);
-CREATE TABLE annot_p_bool
- ( id VARCHAR(256) REFERENCES process (id) ON DELETE CASCADE,
- name VARCHAR(256),
- value BOOLEAN,
- PRIMARY KEY (id, name)
+create table annot_p_bool
+ ( id varchar(256) references process (id) on delete cascade,
+ name varchar(256),
+ value boolean,
+ primary key (id, name)
);
-CREATE TABLE annot_wf_num
- ( id VARCHAR(256) REFERENCES workflow (id) ON DELETE CASCADE,
- name VARCHAR(256),
- value NUMERIC,
- PRIMARY KEY (id, name)
+create table annot_wf_num
+ ( id varchar(256) references workflow (id) on delete cascade,
+ name varchar(256),
+ value numeric,
+ primary key (id, name)
);
-CREATE TABLE annot_wf_txt
- ( id VARCHAR(256) REFERENCES workflow (id) ON DELETE CASCADE,
- name VARCHAR(256),
- value VARCHAR(2048),
- PRIMARY KEY (id, name)
+create table annot_wf_txt
+ ( id varchar(256) references workflow (id) on delete cascade,
+ name varchar(256),
+ value varchar(2048),
+ primary key (id, name)
);
-CREATE TABLE annot_wf_bool
- ( id VARCHAR(256) REFERENCES workflow (id) ON DELETE CASCADE,
- name VARCHAR(2048),
- value BOOLEAN,
- PRIMARY KEY (id, name)
+create table annot_wf_bool
+ ( id varchar(256) references workflow (id) on delete cascade,
+ name varchar(2048),
+ value boolean,
+ primary key (id, name)
);
-- extrainfo stores lines generated by the SWIFT_EXTRA_INFO feature
@@ -211,29 +211,29 @@
-- terms of accessors and constructors.
-- It is unclear which is the better representation.
-CREATE TABLE createarray
- ( id VARCHAR(256) PRIMARY KEY
+create table createarray
+ ( id varchar(256) primary key
);
-CREATE TABLE createarray_member
- ( array_id VARCHAR(256) REFERENCES createarray (id) ON DELETE CASCADE,
- ix VARCHAR(256),
- member_id VARCHAR(256),
- PRIMARY KEY (array_id, ix)
+create table createarray_member
+ ( array_id varchar(256) references createarray (id) on delete cascade,
+ ix varchar(256),
+ member_id varchar(256),
+ primary key (array_id, ix)
);
-- TODO step
-CREATE TABLE array_range
- ( array_id VARCHAR(256) REFERENCES createarray (id) ON DELETE CASCADE,
- from_id VARCHAR(256),
- to_id VARCHAR(256),
- step_id VARCHAR(256), -- nullable, if step is unspecified
- PRIMARY KEY (array_id,from_id,to_id,step_id)
+create table array_range
+ ( array_id varchar(256) references createarray (id) on delete cascade,
+ from_id varchar(256),
+ to_id varchar(256),
+ step_id varchar(256), -- nullable, if step is unspecified
+ primary key (array_id,from_id,to_id,step_id)
);
-- this GRANT does not work for sqlite; you'll get a syntax error but
-- ignore it, as it is not needed in sqlite
-GRANT ALL ON
+grant all on
dataset,
file,
variable,
@@ -256,4 +256,4 @@
createarray,
createarray_member,
array_range
-TO public, operators;
+to public, operators;
From noreply at svn.ci.uchicago.edu Sun Dec 12 17:47:03 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 12 Dec 2010 17:47:03 -0600 (CST)
Subject: [Swift-commit] r3767 - provenancedb/apps/oops
Message-ID: <20101212234703.BE89C9CC82@svn.ci.uchicago.edu>
Author: lgadelha
Date: 2010-12-12 17:47:03 -0600 (Sun, 12 Dec 2010)
New Revision: 3767
Removed:
provenancedb/apps/oops/raptor_extractor.sh
Modified:
provenancedb/apps/oops/oops_extractor.sh
Log:
Minor fixes.
Modified: provenancedb/apps/oops/oops_extractor.sh
===================================================================
--- provenancedb/apps/oops/oops_extractor.sh 2010-12-12 23:29:54 UTC (rev 3766)
+++ provenancedb/apps/oops/oops_extractor.sh 2010-12-12 23:47:03 UTC (rev 3767)
@@ -85,8 +85,7 @@
$SQLCMD -t -A -F " " -f query.sql -o result.txt
while read dataset filename; do
- DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt)
- FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g')
+ FILENAME=$(echo $filename | sed 's/file:\/\/localhost\///g')
while read token1 token2 token3 token4; do
if [ "$token2" = "(Initial Energy:" ]; then
initialenergy=$(echo $token3 | awk 'BEGIN { FS = "\)" }; {print $1}')
Deleted: provenancedb/apps/oops/raptor_extractor.sh
===================================================================
--- provenancedb/apps/oops/raptor_extractor.sh 2010-12-12 23:29:54 UTC (rev 3766)
+++ provenancedb/apps/oops/raptor_extractor.sh 2010-12-12 23:47:03 UTC (rev 3767)
@@ -1,75 +0,0 @@
-#!/bin/bash
-
-# Annotation extractor for the OOPS application
-#
-# The directory $PROTESTS/swift-logs contains symbolic links to
-# OOPS' Swift logs.
-PROVDB_HOME=~/provenancedb
-PROTESTS_HOME=/gpfs/pads/oops/aashish/CASP
-IMPORT_HOME=~/protests
-
-source $PROVDB_HOME/etc/provenance.config
-
-# provdb_imported records runs already imported to the provenance database
-cd $IMPORT_HOME
-if [ ! -a provdb_imported_raptor ]; then
- touch provdb_imported_raptor
-fi
-
-cd $PROTESTS_HOME
-for k in $(ls -1);
-do
- cd $PROTESTS_HOME/$k
- for i in $(ls | grep run.raptorloops; ls | grep run.loops);
- do
- cd $IMPORT_HOME
- if ! grep --silent $i provdb_imported_raptor; then
- if grep --silent "Swift finished with no errors" $PROTESTS_HOME/$k/$i/RaptorLoops-*.log; then
- # swift-prov-import-all-logs also controls what already has been
- # imported, so it does not repeat work
- echo "export LOGREPO=$PROTESTS_HOME/$k/$i" > $PROVDB_HOME/etc/provenance.config
- echo "export SQLCMD=\"psql -U provdb -h db.ci.uchicago.edu provdb\"" >> $PROVDB_HOME/etc/provenance.config
- $PROVDB_HOME/swift-prov-import-all-logs
- cd $IMPORT_HOME
- echo $i >> provdb_imported_raptor
- cd swift-logs
- # annotate workflows with their oops runid
- OOPS_RUN_ID=$(echo $i | awk -F . '{print $3}')
- cd $PROTESTS_HOME/$k/$i
- LOG_FILENAME=$(ls | grep RaptorLoops- | grep "\."log$)
- WORKFLOW_ID=$(echo "select id from workflow where log_filename like '%$LOG_FILENAME%'" | $SQLCMD -t | awk '{print $1}')
- cd $IMPORT_HOME/swift-logs
- echo "insert into annot_wf_txt values ('$WORKFLOW_ID','oops_run_id','$OOPS_RUN_ID');" | $SQLCMD
-
- echo "select file.id,file.filename from process, ds_usage, ds_containment, file where process.id=ds_usage.process_id and ds_usage.dataset_id=out_id and file.id=ds_containment.in_id and filename like '%.params' and process.name='PrepLoop' and process.workflow_id='$WORKFLOW_ID';" > query.sql;
-
- $SQLCMD -t -A -F " " -f query.sql -o result.txt
-
- DATASET_ID=$(awk '{if (NR==1) print $1}' result.txt)
- FILENAME=$(awk '{if (NR==1) print $2}' result.txt | sed 's/file:\/\/localhost\///g')
-
- cd $PROTESTS_HOME/$k/run.raptorloops.$OOPS_RUN_ID
-
- while read line; do
- NAME=$(echo $line | awk 'BEGIN { FS = "=" }; {print $1}')
- if [ "$NAME" = "SAMPLE RANGE" ]; then
- VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $1}')
- VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "-" }; {print $2}')
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME BEGIN', $VALUE1);" | $SQLCMD
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME END', $VALUE2);" | $SQLCMD
- fi
- if [ "$NAME" = "RESTRAIN DISTANCE" ]; then
- VALUE1=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $1}')
- VALUE2=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}' | awk 'BEGIN { FS = "," }; {print $2}')
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 1', $VALUE1);" | $SQLCMD
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME 2', $VALUE2);" | $SQLCMD
- fi
- if [ "$NAME" = "MAXIMUM NUMBER OF STEPS" ]; then
- VALUE=$(echo $line | awk 'BEGIN { FS = "=" }; {print $2}')
- echo "insert into annot_ds_num values ('$DATASET_ID', '$NAME', $VALUE);" | $SQLCMD
- fi
- done < $FILENAME
- fi
- fi
- done
-done
From noreply at svn.ci.uchicago.edu Sun Dec 12 22:45:17 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 12 Dec 2010 22:45:17 -0600 (CST)
Subject: [Swift-commit] r3768 - in SwiftApps/SwiftR: . Swift/exec
Message-ID: <20101213044517.AE69C9CC82@svn.ci.uchicago.edu>
Author: wilde
Date: 2010-12-12 22:45:17 -0600 (Sun, 12 Dec 2010)
New Revision: 3768
Modified:
SwiftApps/SwiftR/Swift/exec/configure-server-pbs
SwiftApps/SwiftR/Swift/exec/start-swift
SwiftApps/SwiftR/install.sh
Log:
initial changes to handle SGE. May need to re-adjust PBS changes.
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbs
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2010-12-12 23:47:03 UTC (rev 3767)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2010-12-13 04:45:17 UTC (rev 3768)
@@ -50,7 +50,7 @@
END
-true to save <00:00:01
# fast
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2010-12-12 23:47:03 UTC (rev 3767)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2010-12-13 04:45:17 UTC (rev 3768)
@@ -1,5 +1,7 @@
#! /bin/bash
+set -x
+
export TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap
# Define internal functions
@@ -54,6 +56,8 @@
echo $sshpids > $sshpidfile
}
+# FIXME: does PBS need same workers-per-node logic as SGE?
+
make-pbs-submit-file()
{
if [ $queue != default ]; then
@@ -61,7 +65,7 @@
else
queueDirective=""
fi
-cat >pbs.sub <batch.sub <pbs.sub <batch.sub <batch.sub <$pbsjobidfile
+ qsub batch.sub >$jobidfile
- echo Started workers from PBS job $(cat $pbsjobidfile)
+ echo Started workers from batch job $(cat $jobidfile)
}
usage()
@@ -216,6 +284,7 @@
throttle=10
hosts=no-hosts-specified
queue=default
+project=default
while [ $# -gt 0 ]
do
@@ -225,7 +294,8 @@
-n) nodes=$2; verify-is-numeric nodes $nodes; shift ;;
-p) throttle=$2; verify-is-numeric throttle $throttle; shift ;;
-q) queue=$2; verify-not-null queue $queue; shift ;;
- -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf; shift ;;
+ -A) project=$2; verify-not-null project $project; shift ;;
+ -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge; shift ;;
-t) time=$2; verify-not-null time $time; shift ;;
*) usage; exit 1 ;;
esac
@@ -300,31 +370,31 @@
wait-and-start-ssh-workers &
starterpid=$!
-elif [ \( $server = pbs \) -o \( $server = pbsf \) ]; then
+elif [ \( $server = pbs \) -o \( $server = pbsf \) -o \( $server = sge \) ]; then
- source $SWIFTRBIN/configure-server-pbs $cores
+ source $SWIFTRBIN/configure-server-${server} $cores
- pbsjobidfile=${out/stdouterr/pbsjobid}
+ jobidfile=${out/stdouterr/jobid}
TRAPEVENTS="EXIT 1 2 3 15" # Signals and conditions to trap
function onexit {
coasterservicepid="" # null: saved in case we go back to using coaster servers
trap - $TRAPEVENTS
- pbsjobid=$(cat $pbsjobidfile)
- echo Terminating worker processes starter $starterpid and PBS job $pbsjobid
+ jobid=$(cat $jobidfile)
+ echo Terminating worker processes starter $starterpid and batch job $jobid
if [ "_$starterpid != _ ]; then
kill $starterpid
fi
- if [ "_$pbsjobid != _ ]; then
- qdel $pbsjobid
+ if [ "_$jobid != _ ]; then
+ qdel $jobid
fi
kill 0 # Kill all procs in current process group # FIXME: what was this for????
}
trap onexit $TRAPEVENTS
- wait-and-start-pbs-workers &
+ wait-and-start-batch-workers &
starterpid=$!
fi
Modified: SwiftApps/SwiftR/install.sh
===================================================================
--- SwiftApps/SwiftR/install.sh 2010-12-12 23:47:03 UTC (rev 3767)
+++ SwiftApps/SwiftR/install.sh 2010-12-13 04:45:17 UTC (rev 3768)
@@ -1,4 +1,8 @@
ver=0.1
+rm -rf Swift/inst/swift/*
+mkdir -p Swift/inst/swift
+SWIFTREL=$(cd $(dirname $(which swift))/..; pwd)
+cp -pr $SWIFTREL/* Swift/inst/swift
R CMD build Swift
R CMD INSTALL Swift_${ver}.tar.gz
cp Swift_${ver}.tar.gz ~/public_html
From noreply at svn.ci.uchicago.edu Mon Dec 13 11:46:18 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 13 Dec 2010 11:46:18 -0600 (CST)
Subject: [Swift-commit] r3769 - trunk/tests/site/intrepid
Message-ID: <20101213174618.648029CC80@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-13 11:46:18 -0600 (Mon, 13 Dec 2010)
New Revision: 3769
Added:
trunk/tests/site/intrepid/100-cp.check.sh
trunk/tests/site/intrepid/100-cp.clean.sh
trunk/tests/site/intrepid/100-cp.setup.sh
trunk/tests/site/intrepid/100-cp.swift
trunk/tests/site/intrepid/100-cp.timeout
trunk/tests/site/intrepid/sites.template.xml
trunk/tests/site/intrepid/tc.template.data
trunk/tests/site/intrepid/title.txt
Log:
Intrepid test
Added: trunk/tests/site/intrepid/100-cp.check.sh
===================================================================
--- trunk/tests/site/intrepid/100-cp.check.sh (rev 0)
+++ trunk/tests/site/intrepid/100-cp.check.sh 2010-12-13 17:46:18 UTC (rev 3769)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -x
+
+grep $( uname -m ) 100-cp-output.txt || exit 1
+
+exit 0
Property changes on: trunk/tests/site/intrepid/100-cp.check.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/site/intrepid/100-cp.clean.sh
===================================================================
--- trunk/tests/site/intrepid/100-cp.clean.sh (rev 0)
+++ trunk/tests/site/intrepid/100-cp.clean.sh 2010-12-13 17:46:18 UTC (rev 3769)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -x
+
+rm -v 100-cp-input.txt 100-cp-output.txt || exit 1
+
+exit 0
Property changes on: trunk/tests/site/intrepid/100-cp.clean.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/site/intrepid/100-cp.setup.sh
===================================================================
--- trunk/tests/site/intrepid/100-cp.setup.sh (rev 0)
+++ trunk/tests/site/intrepid/100-cp.setup.sh 2010-12-13 17:46:18 UTC (rev 3769)
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -x
+
+uname -a > 100-cp-input.txt || exit 1
+
+exit 0
Property changes on: trunk/tests/site/intrepid/100-cp.setup.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/site/intrepid/100-cp.swift
===================================================================
--- trunk/tests/site/intrepid/100-cp.swift (rev 0)
+++ trunk/tests/site/intrepid/100-cp.swift 2010-12-13 17:46:18 UTC (rev 3769)
@@ -0,0 +1,13 @@
+
+type file;
+
+app (file o) cp(file i)
+{
+ cp @i @o;
+}
+
+file input<"100-cp-input.txt">;
+file output<"100-cp-output.txt">;
+
+output = cp(input);
+
Added: trunk/tests/site/intrepid/100-cp.timeout
===================================================================
--- trunk/tests/site/intrepid/100-cp.timeout (rev 0)
+++ trunk/tests/site/intrepid/100-cp.timeout 2010-12-13 17:46:18 UTC (rev 3769)
@@ -0,0 +1 @@
+3000
Added: trunk/tests/site/intrepid/sites.template.xml
===================================================================
--- trunk/tests/site/intrepid/sites.template.xml (rev 0)
+++ trunk/tests/site/intrepid/sites.template.xml 2010-12-13 17:46:18 UTC (rev 3769)
@@ -0,0 +1,32 @@
+
+
+
+
+
+ /scratch/wozniak/work
+
+ 0.04
+ file
+
+
+
+
+
+
+ _HOST_
+ _PROJECT_
+ _QUEUE_
+ zeptoos
+ true
+ 21
+ 10000
+ 1
+ DEBUG
+ 1
+ 900
+ 64
+ 64
+ _WORK_
+
+
+
Added: trunk/tests/site/intrepid/tc.template.data
===================================================================
--- trunk/tests/site/intrepid/tc.template.data (rev 0)
+++ trunk/tests/site/intrepid/tc.template.data 2010-12-13 17:46:18 UTC (rev 3769)
@@ -0,0 +1 @@
+coasters_alcfbgp cp /bin/cp INSTALLED INTEL32::LINUX null
Added: trunk/tests/site/intrepid/title.txt
===================================================================
--- trunk/tests/site/intrepid/title.txt (rev 0)
+++ trunk/tests/site/intrepid/title.txt 2010-12-13 17:46:18 UTC (rev 3769)
@@ -0,0 +1 @@
+Site Test: BG/P: Intrepid
From noreply at svn.ci.uchicago.edu Mon Dec 13 11:54:26 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 13 Dec 2010 11:54:26 -0600 (CST)
Subject: [Swift-commit] r3770 - trunk/tests
Message-ID: <20101213175426.AB2139CC80@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-13 11:54:26 -0600 (Mon, 13 Dec 2010)
New Revision: 3770
Modified:
trunk/tests/nightly.sh
Log:
Minor improvements to run on Intrepid
Modified: trunk/tests/nightly.sh
===================================================================
--- trunk/tests/nightly.sh 2010-12-13 17:46:18 UTC (rev 3769)
+++ trunk/tests/nightly.sh 2010-12-13 17:54:26 UTC (rev 3770)
@@ -15,6 +15,8 @@
# Everything for a Swift test is written in its RUNDIR
# The temporary output always goes to OUTPUT (TOPDIR/exec.out)
+# Note that some schedulers restrict your choice of RUNDIR
+
# Each *.swift test may be accompanied by a
# *.setup.sh, *.check.sh, and/or *.clean.sh script
# and a *.timeout specifier
@@ -44,6 +46,9 @@
# PID TREE:
# Background processes are used so that hung Swift jobs can be killed
# These are the background processes (PIDs are tracked)
+# Note that PID management has not yet been perfected. Check ps
+# in error cases.
+#
# nightly.sh
# +-monitor()
# +-sleep
@@ -172,6 +177,18 @@
fi
}
+# Ensure all given variables are set
+checkvars() {
+ while (( ${#*} ))
+ do
+ VAR=$1
+ V=$( eval "echo \${${VAR}+1}" )
+ [[ $V == 1 ]] || crash "Not set: $VAR"
+ shift
+ done
+ return 0
+}
+
crash() {
MSG=$1
echo $MSG
@@ -568,7 +585,21 @@
test_exec() {
banner "$TEST (part $SEQ)"
echo "Executing $TEST (part $SEQ)"
- process_exec "$@"
+ pwd
+ printf "\nExecuting: $@" >>$LOG
+
+ rm -f $OUTPUT
+ "$@" > $OUTPUT 2>&1
+ EXITCODE=$?
+
+ if [ "$EXITCODE" == "127" ]; then
+ echo "Command not found: $@" > $OUTPUT
+ fi
+
+ if [ -f $OUTPUT ]; then
+ cat $OUTPUT >> $LOG
+ fi
+
RESULT=$( result )
test_log
out test $SEQ "$LASTCMD" $RESULT $TEST_LOG
@@ -576,6 +607,7 @@
check_bailout
let "SEQ=$SEQ+1"
+ return $EXITCODE
}
# Background process monitoring function
@@ -761,6 +793,18 @@
out package "swift-$DATE.tar.gz"
}
+# Environment must contain PROJECT, QUEUE, and WORK
+make_sites_sed() {
+ checkvars WORK QUEUE PROJECT
+ {
+ echo "s at _WORK_@$WORK@"
+ echo "s at _HOST_@$GLOBUS_HOSTNAME@"
+ echo "s at _PROJECT_@$PROJECT@"
+ echo "s at _QUEUE_@$QUEUE@"
+ } > $RUNDIR/sites.sed
+ return 0
+}
+
if which ifconfig > /dev/null; then
IFCONFIG=ifconfig
else
@@ -770,10 +814,11 @@
GLOBUS_HOSTNAME=$( $IFCONFIG | grep inet | head -1 | cut -d ':' -f 2 | \
awk '{print $1}' )
[ $? != 0 ] && crash "Could not obtain GLOBUS_HOSTNAME!"
+
group_sites_xml() {
TEMPLATE=$GROUP/sites.template.xml
if [ -f $TEMPLATE ]; then
- sed "s at _WORK_@$PWD/work@;s at _HOST_@$GLOBUS_HOSTNAME@" < $TEMPLATE > sites.xml
+ sed -f $RUNDIR/sites.sed < $TEMPLATE > sites.xml
[ $? != 0 ] && crash "Could not create sites.xml!"
echo "Using: $GROUP/sites.template.xml"
else
@@ -851,6 +896,8 @@
date > $LOG
+make_sites_sed
+
header
start_test_results
cd $TOPDIR
@@ -914,6 +961,7 @@
$TESTDIR/cdm/ps \
$TESTDIR/cdm/star
$TESTDIR/cdm/ps/pinned
+ # $TESTDIR/site/intrepid
)
GROUPCOUNT=1
From noreply at svn.ci.uchicago.edu Mon Dec 13 12:06:06 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 13 Dec 2010 12:06:06 -0600 (CST)
Subject: [Swift-commit] r3771 - trunk/tests
Message-ID: <20101213180606.9AF289CC80@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-13 12:06:06 -0600 (Mon, 13 Dec 2010)
New Revision: 3771
Modified:
trunk/tests/nightly.sh
Log:
More usage notes
Modified: trunk/tests/nightly.sh
===================================================================
--- trunk/tests/nightly.sh 2010-12-13 17:54:26 UTC (rev 3770)
+++ trunk/tests/nightly.sh 2010-12-13 18:06:06 UTC (rev 3771)
@@ -15,8 +15,6 @@
# Everything for a Swift test is written in its RUNDIR
# The temporary output always goes to OUTPUT (TOPDIR/exec.out)
-# Note that some schedulers restrict your choice of RUNDIR
-
# Each *.swift test may be accompanied by a
# *.setup.sh, *.check.sh, and/or *.clean.sh script
# and a *.timeout specifier
@@ -30,12 +28,15 @@
# Tests are GROUPed into directories
# Each GROUP directory has:
# 1) a list of *.swift tests (plus *.sh scripts)
-# 2) optionally a tc.template.data
-# 3) optionally a fs.template.data
-# 4) optionally a swift.properties
-# 5) optionally a title.txt
-# 6) preferably a README.txt
+# 2) optionally a sites.template.xml
+# 3) optionally a tc.template.data
+# 4) optionally a fs.template.data
+# 5) optionally a swift.properties
+# 6) optionally a title.txt
+# 7) preferably a README.txt
# Edit GROUPLIST at the end of this script before running
+# template files are lightly processed by sed before use
+# Missing files will be pulled from swift/etc
# OUTPUT is the stdout of the current test
# stdout.txt retains stdout from the previous test (for *.clean.sh)
@@ -62,6 +63,12 @@
# The response of nightly.sh to the exit code of these Swift
# executions is reversed.
+# SCHEDULERS
+# Environment must contain PROJECT, QUEUE, and WORK
+# These variables will be incorporated into the sites.xml
+# via make_sites_sed() -> group_sites_xml()
+# Note that some schedulers restrict your choice of RUNDIR
+
printhelp() {
echo "nightly.sh
From noreply at svn.ci.uchicago.edu Mon Dec 13 16:30:25 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 13 Dec 2010 16:30:25 -0600 (CST)
Subject: [Swift-commit] r3794 - www
Message-ID: <20101213223025.179449CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-13 16:30:24 -0600 (Mon, 13 Dec 2010)
New Revision: 3794
Modified:
www/update.sh
Log:
Add chmods
Modified: www/update.sh
===================================================================
--- www/update.sh 2010-12-13 22:29:11 UTC (rev 3793)
+++ www/update.sh 2010-12-13 22:30:24 UTC (rev 3794)
@@ -35,6 +35,8 @@
svn update
chmod -R g+w .svn 2>/dev/null
chmod a+r docs/index.php
+chmod a+r downloads/index.php
+chmod a+r inc/nav.php inc/footer.php
echo "--------- Updating docs... ----------"
for SET in guides/trunk guides/release-0.91
From noreply at svn.ci.uchicago.edu Mon Dec 13 19:16:32 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 13 Dec 2010 19:16:32 -0600 (CST)
Subject: [Swift-commit] r3795 - branches
Message-ID: <20101214011632.7A5EA9CC7F@svn.ci.uchicago.edu>
Author: skenny
Date: 2010-12-13 19:16:32 -0600 (Mon, 13 Dec 2010)
New Revision: 3795
Added:
branches/release-0.91/
Log:
formerly 1.0
Copied: branches/release-0.91 (from rev 3794, branches/1.0)
From noreply at svn.ci.uchicago.edu Mon Dec 13 19:53:43 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 13 Dec 2010 19:53:43 -0600 (CST)
Subject: [Swift-commit] r3796 - trunk/docs
Message-ID: <20101214015343.289C99CC80@svn.ci.uchicago.edu>
Author: skenny
Date: 2010-12-13 19:53:43 -0600 (Mon, 13 Dec 2010)
New Revision: 3796
Modified:
trunk/docs/userguide.xml
Log:
describes mapping files based on URI
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2010-12-14 01:16:32 UTC (rev 3795)
+++ trunk/docs/userguide.xml 2010-12-14 01:53:43 UTC (rev 3796)
@@ -1385,7 +1385,24 @@
+ mapping URIs
+
+The above mappers may be used to map files based on a URI which can be specified in the filename. This is useful for mapping files on remote machines.
+
+
+
+ Example:
+
+
+ student st <single_file_mapper;file="gsiftp://communicado.ci.uchicago.edu//tmp/student.txt">;
+
+
+
+
+
+
+
Commands
The commands detailed in this section are available in the
From noreply at svn.ci.uchicago.edu Mon Dec 13 22:00:44 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 13 Dec 2010 22:00:44 -0600 (CST)
Subject: [Swift-commit] r3797 - trunk/docs
Message-ID: <20101214040044.D6C029CC7F@svn.ci.uchicago.edu>
Author: skenny
Date: 2010-12-13 22:00:44 -0600 (Mon, 13 Dec 2010)
New Revision: 3797
Modified:
trunk/docs/userguide.xml
Log:
documented using shell wrapper instead of individual entries in tc.data
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2010-12-14 01:53:43 UTC (rev 3796)
+++ trunk/docs/userguide.xml 2010-12-14 04:00:44 UTC (rev 3797)
@@ -3611,12 +3611,15 @@
Some example entries:
localhost echo /bin/echo INSTALLED INTEL32::LINUX null
+localhost touch /bin/touch INSTALLED INTEL32::LINUX null
+
TGUC touch /usr/bin/touch INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="0:1"
+TGUC R /usr/bin/R INSTALLED INTEL32::LINUX env::R_LIBS=/home/skenny/R_libs
The fields are: site, transformation name, executable path, installation
-status, platform, and profile entrys.
+status, platform, and profile entries.
The site field should correspond to a site name listed in the sites
@@ -3637,7 +3640,29 @@
The profiles field should be set to null if no profile entries are to be
specified, or should contain the profile entries separated by semicolons.
+
+ shell invocation
+
+Because the above implementation requires an entry for each executable on a given site, it is often preferable to simply have a single element for each site in the transformation catalog representing a wrapper that sets the environment and then invokes the shell to call a given application. This wrapper is installed on the site and can be used to set the PATH and other environment variables prior to invoking the shell so that each executable need not be entered in the tc.data file.
+
+ for example, the entries for TGUC and localhost can now each be collapsed into a single line:
+
+localhost shell /usr/local/bin/swiftshell INSTALLED INTEL32::LINUX null
+TGUC shell /usr/local/bin/swiftshell INSTALLED INTEL32::LINUX null
+
+where swiftshell sets up the users's environment so that all the installed applications are added to the PATH before the application is invoked.
+
+
+touch would now be called in the SwiftScript like this:
+
+app (file tout) shelltest(){
+ shell "touch" @filename(tout);
+}
+
+
+
+
Build options
See the
From noreply at svn.ci.uchicago.edu Tue Dec 14 10:28:49 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Dec 2010 10:28:49 -0600 (CST)
Subject: [Swift-commit] r3798 - www
Message-ID: <20101214162849.70B479CC82@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-14 10:28:48 -0600 (Tue, 14 Dec 2010)
New Revision: 3798
Modified:
www/update.sh
Log:
More chmods
Modified: www/update.sh
===================================================================
--- www/update.sh 2010-12-14 04:00:44 UTC (rev 3797)
+++ www/update.sh 2010-12-14 16:28:48 UTC (rev 3798)
@@ -1,5 +1,7 @@
#!/bin/sh
+# Run this script from /ci/www/projects/swift
+
crash()
{
echo $1
@@ -30,6 +32,10 @@
return 0
}
+# Ensure tools are group-usable:
+chmod g+rw $0
+chmod g+rw updatenodocs.sh
+
umask 002
echo "--------- Updating www... ----------"
svn update
From noreply at svn.ci.uchicago.edu Tue Dec 14 10:36:55 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Dec 2010 10:36:55 -0600 (CST)
Subject: [Swift-commit] r3799 - www
Message-ID: <20101214163655.F1C829CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-14 10:36:55 -0600 (Tue, 14 Dec 2010)
New Revision: 3799
Modified:
www/update.sh
Log:
Ensure *.php is g+rw
Modified: www/update.sh
===================================================================
--- www/update.sh 2010-12-14 16:28:48 UTC (rev 3798)
+++ www/update.sh 2010-12-14 16:36:55 UTC (rev 3799)
@@ -40,9 +40,14 @@
echo "--------- Updating www... ----------"
svn update
chmod -R g+w .svn 2>/dev/null
-chmod a+r docs/index.php
+
+find . -name .svn -prune -o -name "*.php" | xargs -n 100 chmod -f g+rw,a+r
+
+chmod a+r,g+w docs/index.php
chmod a+r downloads/index.php
chmod a+r inc/nav.php inc/footer.php
+
+
echo "--------- Updating docs... ----------"
for SET in guides/trunk guides/release-0.91
From noreply at svn.ci.uchicago.edu Tue Dec 14 10:38:20 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Dec 2010 10:38:20 -0600 (CST)
Subject: [Swift-commit] r3800 - www
Message-ID: <20101214163820.160C79CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-14 10:38:19 -0600 (Tue, 14 Dec 2010)
New Revision: 3800
Modified:
www/update.sh
Log:
Cut unnecessary lines
Modified: www/update.sh
===================================================================
--- www/update.sh 2010-12-14 16:36:55 UTC (rev 3799)
+++ www/update.sh 2010-12-14 16:38:19 UTC (rev 3800)
@@ -43,11 +43,6 @@
find . -name .svn -prune -o -name "*.php" | xargs -n 100 chmod -f g+rw,a+r
-chmod a+r,g+w docs/index.php
-chmod a+r downloads/index.php
-chmod a+r inc/nav.php inc/footer.php
-
-
echo "--------- Updating docs... ----------"
for SET in guides/trunk guides/release-0.91
From noreply at svn.ci.uchicago.edu Tue Dec 14 15:57:25 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Dec 2010 15:57:25 -0600 (CST)
Subject: [Swift-commit] r3801 - trunk/tests
Message-ID: <20101214215725.2725C9CC82@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-14 15:57:24 -0600 (Tue, 14 Dec 2010)
New Revision: 3801
Added:
trunk/tests/groups/
Log:
Location for GROUPLIST definition files
From noreply at svn.ci.uchicago.edu Tue Dec 14 15:59:08 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Dec 2010 15:59:08 -0600 (CST)
Subject: [Swift-commit] r3802 - trunk/tests/groups
Message-ID: <20101214215908.1F7B29CC82@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-14 15:59:07 -0600 (Tue, 14 Dec 2010)
New Revision: 3802
Added:
trunk/tests/groups/group-all-local.sh
trunk/tests/groups/group-intrepid.sh
Log:
First example GROUPLIST defn files
Added: trunk/tests/groups/group-all-local.sh
===================================================================
--- trunk/tests/groups/group-all-local.sh (rev 0)
+++ trunk/tests/groups/group-all-local.sh 2010-12-14 21:59:07 UTC (rev 3802)
@@ -0,0 +1,13 @@
+
+# GROUPLIST definition to run all local tests
+
+GROUPLIST=( $TESTDIR/language-behaviour \
+ $TESTDIR/language/working \
+ $TESTDIR/local \
+ $TESTDIR/language/should-not-work \
+ $TESTDIR/cdm \
+ $TESTDIR/cdm/ps \
+ $TESTDIR/cdm/star
+ $TESTDIR/cdm/ps/pinned
+ # $TESTDIR/site/intrepid
+ )
Added: trunk/tests/groups/group-intrepid.sh
===================================================================
--- trunk/tests/groups/group-intrepid.sh (rev 0)
+++ trunk/tests/groups/group-intrepid.sh 2010-12-14 21:59:07 UTC (rev 3802)
@@ -0,0 +1,6 @@
+
+# GROUPLIST definition to run on Intrepid
+
+GROUPLIST=( $TESTDIR/local \
+ $TESTDIR/site/intrepid
+ )
From noreply at svn.ci.uchicago.edu Tue Dec 14 16:15:08 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Dec 2010 16:15:08 -0600 (CST)
Subject: [Swift-commit] r3803 - trunk/tests
Message-ID: <20101214221508.BFA0F9CC82@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-14 16:15:08 -0600 (Tue, 14 Dec 2010)
New Revision: 3803
Modified:
trunk/tests/nightly.sh
Log:
Define GROUPLIST in external file
Modified: trunk/tests/nightly.sh
===================================================================
--- trunk/tests/nightly.sh 2010-12-14 21:59:07 UTC (rev 3802)
+++ trunk/tests/nightly.sh 2010-12-14 22:15:08 UTC (rev 3803)
@@ -34,6 +34,7 @@
# 5) optionally a swift.properties
# 6) optionally a title.txt
# 7) preferably a README.txt
+# 7) optionally a *.timeout
# Edit GROUPLIST at the end of this script before running
# template files are lightly processed by sed before use
# Missing files will be pulled from swift/etc
@@ -47,9 +48,9 @@
# PID TREE:
# Background processes are used so that hung Swift jobs can be killed
# These are the background processes (PIDs are tracked)
-# Note that PID management has not yet been perfected. Check ps
-# in error cases.
-#
+# Note that PID management has not yet been perfected. Check ps
+# in error cases.
+#
# nightly.sh
# +-monitor()
# +-sleep
@@ -65,7 +66,7 @@
# SCHEDULERS
# Environment must contain PROJECT, QUEUE, and WORK
-# These variables will be incorporated into the sites.xml
+# These variables will be incorporated into the sites.xml
# via make_sites_sed() -> group_sites_xml()
# Note that some schedulers restrict your choice of RUNDIR
@@ -73,17 +74,18 @@
echo "nightly.sh
+
+ Collective Data Management
+
+
From noreply at svn.ci.uchicago.edu Thu Dec 16 18:18:25 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 16 Dec 2010 18:18:25 -0600 (CST)
Subject: [Swift-commit] r3810 - trunk/docs
Message-ID: <20101217001825.ED1909CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-16 18:18:25 -0600 (Thu, 16 Dec 2010)
New Revision: 3810
Modified:
trunk/docs/userguide.xml
Log:
Paste in CDM features doc
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2010-12-17 00:03:40 UTC (rev 3809)
+++ trunk/docs/userguide.xml 2010-12-17 00:18:25 UTC (rev 3810)
@@ -4127,7 +4127,211 @@
Collective Data Management
+
+CDM Features
+
+Overview:
+
+ 1. The user specifies a CDM policy in a file, customarily fs.data.
+ 2. fs.data is given to Swift on the command line.
+ 3. The Swift data module (org.globus.swift.data) is informed of the CDM policy.
+ 4. At job launch time, the VDL Karajan code queries the CDM policy,
+ 1. altering the file staging phase, and
+ 2. sending fs.data to the compute site.
+ 5. At job run time, the Swift wrapper script
+ 1. consults a Perl script to obtain policy, and
+ 2. uses wrapper extensions to modify data movement.
+ 6. Similarly, stage out can be changed.
+
+
+Command line:
+
+ * swift -sites.file sites.xml -tc.file tc.data -cdm.file fs.data stream.swift
+
+
+CDM policy file format:
+
+Example:
+
+# Describe CDM for my job
+property GATHER_LIMIT 1
+rule .*input.txt DIRECT /gpfs/homes/wozniak/data
+rule .*xfile*.data BROADCAST /dev/shm
+rule .* DEFAULT
+
+The lines contain:
+
+ 1. A directive, either rule or property
+ 2. A rule has:
+ 1. A regular expression
+ 2. A policy token
+ 3. Additional policy-specific arguments
+ 3. A property has
+ 1. A policy property token
+ 2. The token value
+ 4. Comments with # .
+ 5. Blank lines are ignored.
+
+
+Notes:
+
+ 1. The policy file is used as a lookup database by Swift and Perl methods.
+ 2. For example, a lookup with the database above given the argument input.txt would result in the Direct policy.
+ 3. If the lookup does not succeed, the result is DEFAULT.
+ 4. Policies are listed as subclasses of org.globus.swift.data.policy.Policy .
+
+
+Policy Descriptions:
+
+Default:
+
+ * Just use file staging as provided by Swift/CoG. Identical to behavior if no CDM file is given.
+
+
+Broadcast:
+
+ rule .*xfile*.data BROADCAST /dev/shm
+
+ * The input files matching the pattern will be stored in the given directory, an LFS location, with links in the job directory.
+ * On the BG/P, this will make use of the f2cn tool.
+ * On machines not implementing an efficient broadcast, we will just ensure correctness. For example, on a workstation, the location could be in a /tmp RAM FS.
+
+
+Direct:
+
+ rule .*input.txt DIRECT /gpfs/scratch/wozniak/
+
+ * Allows for direct I/O to the parallel FS without staging.
+ * The input files matching the pattern must already exist in the given directory, a GFS location. Links will be placed in the job directory.
+ * The output files matching the pattern will be stored in the given directory, with links in the job directory.
+ * Example: In the rule above, the Swift-generated file name ./data/input.txt would be accessed by the user job in /gpfs/homes/wozniak/data/input.txt .
+
+
+Local:
+
+ rule .*input.txt LOCAL dd /gpfs/homes/wozniak/data obs=64K
+
+ * Allows for client-directed input copy to the compute node.
+ * The user may specify cp or dd as the input transfer program.
+
+ * The input files matching the pattern must already exist in the given directory, a GFS location. Copies will be placed in the job directory.
+ * Argument list: *
+
+
+Gather:
+
+ property GATHER_LIMIT 500000000 # 500 MB
+ property GATHER_DIR /dev/shm/gather
+ property GATHER_TARGET /gpfs/wozniak/data/gather_target
+ rule .*.output.txt GATHER
+
+ * The output files matching the pattern will be present to tasks in the job directory as usual but noted in a _swiftwrap shell array GATHER_OUTPUT.
+ * The GATHER_OUTPUT files will be cached in the GATHER_DIR, an LFS location.
+ * The cache will be flushed when a job ends if a du on GATHER_DIR exceeds GATHER_LIMIT.
+ * As the cache fills or on stage out, the files will be bundled into randomly named tarballs in GATHER_TARGET, a GFS location.
+ * If the compute node is an SMP, GATHER_DIR is a shared resource. It is protected by the link file GATHER_DIR/.cdm.lock .
+ * Unpacking the tarballs in GATHER_TARGET will produce the user-specified filenames.
+
+ Summary:
+
+ 1. Files created by application
+ 2. Acquire lock
+ 3. Move files to cache
+ 4. Check cache size
+ 5. If limit exceeded, move all cache files to outbox
+ 6. Release lock
+ 7. If limit was exceeded, stream outbox as tarball to target
+
+
+ Notes:
+
+ * Gather required quite a bit of shell functionality to manage the lock, etc. This is placed in cdm_lib.sh .
+ * vdl_int.k needed an additional task submission (cdm_cleanup.sh) to perform the final flush at workflow completion time . This task also uses cdm_lib.sh .
+
+
+VDL/Karajan processing:
+
+ 1. CDM functions are available in Karajan via the cdm namespace.
+ 2. These functions are defined in org.globus.swift.data.Query .
+ 3. If CDM is enabled, VDL skips file staging for files unless the policy is DEFAULT.
+
+
+Swift wrapper CDM routines:
+
+ 1. The cdm.pl script is shipped to the compute node if CDM is enabled.
+ 2. When linking in inputs, CDM is consulted by _swiftwrap:cdm_lookup().
+ 3. The cdm_action() shell function handles CDM methods, typically just producing a link.
+
+
+Test case:
+
+(See About.txt for more information.)
+
+ 1. Simple test cases are in:
+ https://svn.mcs.anl.gov/repos/wozniak/collab/cdm/scripts/cdm-direct and
+ https://svn.mcs.anl.gov/repos/wozniak/collab/cdm/scripts/all-pairs
+ 2. Do a:
+ mkdir cdm
+ cd cdm
+ svn co https://svn.mcs.anl.gov/repos/wozniak/collab/cdm/scripts
+ 3. In cdm-direct, run:
+ source ./setup.sh local local local
+ 4. Run workflow:
+ swift -sites.file sites.xml -tc.file tc.data -cdm.file fs.data stream.swift
+ 5. Note that staging is skipped for input.txt
+ policy: file://localhost/input.txt : DIRECT
+ FILE_STAGE_IN_START file=input.txt ...
+ FILE_STAGE_IN_SKIP file=input.txt policy=DIRECT
+ FILE_STAGE_IN_END file=input.txt ...
+ 6. In the wrapper output, the input file is handled by CDM functionality:
+ Progress 2010-01-21 13:50:32.466572727-0600 LINK_INPUTS
+ CDM_POLICY: DIRECT /homes/wozniak/cdm/scripts/cdm-direct
+ CDM: jobs/t/cp_sh-tkul4nmj input.txt DIRECT /homes/wozniak/cdm/scripts/cdm-direct
+ CDM[DIRECT]: Linking jobs/t/cp_sh-tkul4nmj/input.txt to /homes/wozniak/cdm/scripts/cdm-direct/input.txt
+ Progress 2010-01-21 13:50:32.486016708-0600 EXECUTE
+ 7. all-pairs is quite similar but uses more policies.
+
+
+PTMap case:
+
+ 1. Start with vanilla PTMap:
+ cd cdm
+ mkdir apps
+ cd apps
+ https://svn.mcs.anl.gov/repos/wozniak/collab/cdm/apps/ptmap
+ 2. Source setup.sh
+ 3. Use start.sh, which
+ 1. applies CDM policy from fs.local.data
+
+
+CDM site-aware policy file format:
+
+Example:
+
+# Describe CDM for my job
+# Use DIRECT and BROADCAST if on cluster1, else use DEFAULT behavior
+property GATHER_LIMIT 1
+rule cluster1 .*input.txt DIRECT /gpfs/homes/wozniak/data
+rule cluster1 .*xfile*.data BROADCAST /dev/shm
+rule ANYWHERE .* DEFAULT
+
+The lines contain:
+
+ 1. A directive, either rule or property
+ 2. A rule has:
+ 1. A regular expression for site matchin
+ 2. A regular expression for filename matching
+ 3. A policy token
+ 4. Additional policy-specific arguments
+ 3. A property has
+ 1. A policy property token
+ 2. The token value
+ 4. Comments with # .
+ 5. Blank lines are ignored.
+
+
+
From noreply at svn.ci.uchicago.edu Thu Dec 16 18:23:05 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 16 Dec 2010 18:23:05 -0600 (CST)
Subject: [Swift-commit] r3811 - trunk/docs
Message-ID: <20101217002305.6E53A9CC7F@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-16 18:23:05 -0600 (Thu, 16 Dec 2010)
New Revision: 3811
Modified:
trunk/docs/userguide.xml
Log:
XML correction
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2010-12-17 00:18:25 UTC (rev 3810)
+++ trunk/docs/userguide.xml 2010-12-17 00:23:05 UTC (rev 3811)
@@ -4216,7 +4216,7 @@
* The user may specify cp or dd as the input transfer program.
* The input files matching the pattern must already exist in the given directory, a GFS location. Copies will be placed in the job directory.
- * Argument list: *
+ * Argument list: [tool] [GFS directory] [tool arguments]*
Gather:
From noreply at svn.ci.uchicago.edu Sun Dec 19 06:11:42 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 19 Dec 2010 06:11:42 -0600 (CST)
Subject: [Swift-commit] r3812 - trunk/src/org/griphyn/vdl/karajan
Message-ID: <20101219121142.3D36C9CC82@svn.ci.uchicago.edu>
Author: skenny
Date: 2010-12-19 06:11:38 -0600 (Sun, 19 Dec 2010)
New Revision: 3812
Modified:
trunk/src/org/griphyn/vdl/karajan/Loader.java
Log:
print swift version to commandline
Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-12-17 00:23:05 UTC (rev 3811)
+++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2010-12-19 12:11:38 UTC (rev 3812)
@@ -56,6 +56,7 @@
private static final Logger logger = Logger.getLogger(Loader.class);
public static final String ARG_HELP = "help";
+ public static final String ARG_VERSION = "version";
public static final String ARG_MONITOR = "monitor";
public static final String ARG_RESUME = "resume";
public static final String ARG_INSTANCE_CONFIG = "config";
@@ -89,6 +90,10 @@
ap.usage();
System.exit(0);
}
+ if (ap.isPresent(ARG_VERSION)){
+ ap.version();
+ System.exit(0);
+ }
if (ap.isPresent(ARG_MONITOR)) {
new Monitor().start();
}
@@ -371,6 +376,7 @@
ArgumentParser.OPTIONAL);
ap.addFlag(ARG_HELP, "Display usage information");
+ ap.addFlag(ARG_VERSION, "Version:");
ap.addAlias(ARG_HELP, "h");
ap.addFlag(ARG_RECOMPILE,
From noreply at svn.ci.uchicago.edu Tue Dec 21 15:13:02 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 21 Dec 2010 15:13:02 -0600 (CST)
Subject: [Swift-commit] r3813 - text/parco10submission
Message-ID: <20101221211302.D6B8D9CC8D@svn.ci.uchicago.edu>
Author: hategan
Date: 2010-12-21 15:13:02 -0600 (Tue, 21 Dec 2010)
New Revision: 3813
Modified:
text/parco10submission/paper.tex
Log:
an alternate take on the swift rationale
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-19 12:11:38 UTC (rev 3812)
+++ text/parco10submission/paper.tex 2010-12-21 21:13:02 UTC (rev 3813)
@@ -3,9 +3,13 @@
\usepackage{graphicx}
-\usepackage{color}
+\usepackage{xcolor}
+\usepackage{framed}
\newcommand{\katznote}[1]{ {\textcolor{cyan} { ***Dan: #1 }}}
+\definecolor{shadecolor}{RGB}{240,255,200}
+\newenvironment{msection}{\begin{shaded}}{\end{shaded}}
+
\journal{Parallel Computing}
\makeatletter
\g at addto@macro\@verbatim\small
@@ -87,7 +91,6 @@
\katznote{are swift tasks idempotent? are swift scripts are deterministic (maybe?) deadlock-free? (no)}
-
Swift is a scripting language designed for composing ordinary
application programs (serial or parallel) into distributed,
parallelized applications for execution on grids and supercomputers
@@ -131,6 +134,93 @@
\subsection*{Rationale}
+\begin{msection}
+The main goal of Swift is to allow the composition coarse grained
+processes. Such a composition will be termed \emph{Swift program}. A
+secondary goal is to parallelize and manage the execution of a Swift
+program on shared collections of resources (i.e. Grids), while at the
+same time maintaining data determinism.
+
+The exact number of processing units available on such shared resources
+varies with time. In order to take advantage of as many processing units
+as possible during the execution of a Swift program, it is necessary to
+be flexible in the way the execution of individual processes is
+parallelized. Having the results be independent of the way the processes
+are parallelized implies that the processes must, for the same input,
+produce the same output, irrespective of the time, order or location in
+which they are "executed". This characteristic is reminiscent of
+referential transparency, and one may readily extend the concept to
+encompass arbitrary processes without difficulty.
+
+The referential transparency requirement suggests that an appropriate
+course of action in building a language allowing process composition is
+to define a correspondence between the familiar concept of a function
+and a process. We do so in Swift by representing input data and process
+parameters as function parameters and output data as return values,
+while processes are represented by functions.
+
+This duality allows the formal specification of process behavior. When
+writing as part of a program:
+\begin{verbatim}
+rotatedImage = rotate(image, angle);
+\end{verbatim}
+the semantics are defined in terms of the specification of the function
+"rotate" when supplied with specific parameter types, and whether the
+implementation can be described as a "library call" or a "program
+invocation" changes nothing with respect to what the piece of program
+fundamentally does: produce a rotated version of the original.
+
+Indeed, there is no strict requirement in the specification of the Swift
+language dictating that functions be implemented as command-line
+applications. They can equally consist of library calls or functions
+written in Swift itself, as long as they are side-effect free. A soft
+restriction arises from the desire to distribute the execution of
+functions across a collection of heterogeneous resources, which, with
+the advent of projects such as TeraGrid, suggests an implementation in
+which functions are applications readily executable on them through the
+careful employment of Grid middleware.
+
+Having established the constraint that Swift functions must in general
+be referentially transparent, and in order to preserve referential
+transparency at different levels of abstractions within the language, it
+follows that the appropriate form for the Swift language is functional.
+We choose to make the Swift language purely functional (i.e. disallow
+side-effects in the language) in order to prevent the difficulties that
+arise from having to track side-effects to ensure determinism in complex
+concurrency scenarios.
+
+It is already known that functional programming allows consistent
+implementations of evaluation strategies different from the widespread
+eager evaluation, a fact readily apparent in lazily evaluated languages
+such as Haskell and Miranda. In order to achieve automatic
+parallelization in Swift, instead of thunking (i.e. suspended
+computations) which yields lazy evaluation, we employ futures, which
+results in eager parallelism. In this process, we trade the ability to
+efficiently deal with infinite structures for the ability to minimize
+computation time. It must, however, be noted that a middle ground
+exists: lazy futures (futures whose computation is delayed until a value
+is first needed).
+
+A number of issues may be noted at this point. First, there exist a
+certain class of processes which may break referential transparency,
+while still producing "valid" results. An example would be Monte Carlo
+processes. Under the assumption that eager evaluation of compositions of
+Monte Carlo processes also produces valid results, an eager Swift
+implementation (which is the case with the current implementation)
+readily accommodates Monte Carlo processes. However, further discussion
+is necessary if optimizations (such as memoization) are employed.
+Second, referential transparency and lack of side-effects are not
+equivalent concepts. For example, it is possible for a referentially
+transparent function to employ side-effects as part of its
+implementation. While this is not currently possible within Swift
+itself, it is also not readily obvious that such a feature would
+contradict the main goals of Swift, and therefore is not ruled out as a
+possible future development.
+
+
+\end{msection}
+
+
The emergence of large-scale production computing infrastructure such
as clusters, grids and high-performance computing (HPC), and the
inherent complexity of programming on these systems, necessitates a
From noreply at svn.ci.uchicago.edu Tue Dec 21 15:13:44 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 21 Dec 2010 15:13:44 -0600 (CST)
Subject: [Swift-commit] r3814 - text/parco10submission
Message-ID: <20101221211344.C2A1A9CC8D@svn.ci.uchicago.edu>
Author: hategan
Date: 2010-12-21 15:13:44 -0600 (Tue, 21 Dec 2010)
New Revision: 3814
Modified:
text/parco10submission/paper.tex
Log:
missing word
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-21 21:13:02 UTC (rev 3813)
+++ text/parco10submission/paper.tex 2010-12-21 21:13:44 UTC (rev 3814)
@@ -135,7 +135,7 @@
\subsection*{Rationale}
\begin{msection}
-The main goal of Swift is to allow the composition coarse grained
+The main goal of Swift is to allow the composition of coarse grained
processes. Such a composition will be termed \emph{Swift program}. A
secondary goal is to parallelize and manage the execution of a Swift
program on shared collections of resources (i.e. Grids), while at the
From noreply at svn.ci.uchicago.edu Tue Dec 21 15:35:32 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 21 Dec 2010 15:35:32 -0600 (CST)
Subject: [Swift-commit] r3815 - trunk/tests
Message-ID: <20101221213532.A763E9CC8D@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-21 15:35:32 -0600 (Tue, 21 Dec 2010)
New Revision: 3815
Modified:
trunk/tests/meta.sh
trunk/tests/nightly.sh
Log:
Additional notes
Modified: trunk/tests/meta.sh
===================================================================
--- trunk/tests/meta.sh 2010-12-21 21:13:44 UTC (rev 3814)
+++ trunk/tests/meta.sh 2010-12-21 21:35:32 UTC (rev 3815)
@@ -8,3 +8,5 @@
ssh intrepid.alcf.anl.gov $DIR/run-nightly.sh groups/group-intrepid.sh
+# Retrieve results
+# scp ...
Modified: trunk/tests/nightly.sh
===================================================================
--- trunk/tests/nightly.sh 2010-12-21 21:13:44 UTC (rev 3814)
+++ trunk/tests/nightly.sh 2010-12-21 21:35:32 UTC (rev 3815)
@@ -6,6 +6,8 @@
# subdirectory called run-DATE, and generate useful HTML output and
# tests.log
+# Usage: nightly.sh *
+
# Run nightly.sh -h for quick help
# When something goes wrong, find and check tests.log or use -v
# Code is checked out into TOPDIR (PWD by default) (PATH is not used)
@@ -54,7 +56,7 @@
# Background processes are used so that hung Swift jobs can be killed
# These are the background processes (PIDs are tracked)
# Note that PID management has not yet been perfected. Check ps
-# in error cases.
+# in error cases, especially if you kill nightly.sh .
#
# nightly.sh
# +-monitor()
From noreply at svn.ci.uchicago.edu Tue Dec 21 17:57:20 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 21 Dec 2010 17:57:20 -0600 (CST)
Subject: [Swift-commit] r3816 - in trunk/tests: . providers providers/local
providers/local-pbs providers/local-pbs-coasters providers/sge-local
Message-ID: <20101221235720.929B69CC8D@svn.ci.uchicago.edu>
Author: davidk
Date: 2010-12-21 17:57:20 -0600 (Tue, 21 Dec 2010)
New Revision: 3816
Added:
trunk/tests/providers/
trunk/tests/providers/local-pbs-coasters/
trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh
trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh
trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.swift
trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.timeout
trunk/tests/providers/local-pbs-coasters/catsn.0001.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0002.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0003.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0004.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0005.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0006.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0007.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0008.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0009.out.expected
trunk/tests/providers/local-pbs-coasters/catsn.0010.out.expected
trunk/tests/providers/local-pbs-coasters/data.txt
trunk/tests/providers/local-pbs-coasters/sites.template.xml
trunk/tests/providers/local-pbs-coasters/tc.template.data
trunk/tests/providers/local-pbs-coasters/title.txt
trunk/tests/providers/local-pbs/
trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh
trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh
trunk/tests/providers/local-pbs/001-catsn-localpbs.swift
trunk/tests/providers/local-pbs/001-catsn-localpbs.timeout
trunk/tests/providers/local-pbs/catsn.0001.out.expected
trunk/tests/providers/local-pbs/catsn.0002.out.expected
trunk/tests/providers/local-pbs/catsn.0003.out.expected
trunk/tests/providers/local-pbs/catsn.0004.out.expected
trunk/tests/providers/local-pbs/catsn.0005.out.expected
trunk/tests/providers/local-pbs/catsn.0006.out.expected
trunk/tests/providers/local-pbs/catsn.0007.out.expected
trunk/tests/providers/local-pbs/catsn.0008.out.expected
trunk/tests/providers/local-pbs/catsn.0009.out.expected
trunk/tests/providers/local-pbs/catsn.0010.out.expected
trunk/tests/providers/local-pbs/data.txt
trunk/tests/providers/local-pbs/sites.template.xml
trunk/tests/providers/local-pbs/tc.template.data
trunk/tests/providers/local-pbs/title.txt
trunk/tests/providers/local/
trunk/tests/providers/local/001-catsn-local.check.sh
trunk/tests/providers/local/001-catsn-local.setup.sh
trunk/tests/providers/local/001-catsn-local.swift
trunk/tests/providers/local/001-catsn-local.timeout
trunk/tests/providers/local/catsn.0001.out.expected
trunk/tests/providers/local/catsn.0002.out.expected
trunk/tests/providers/local/catsn.0003.out.expected
trunk/tests/providers/local/catsn.0004.out.expected
trunk/tests/providers/local/catsn.0005.out.expected
trunk/tests/providers/local/catsn.0006.out.expected
trunk/tests/providers/local/catsn.0007.out.expected
trunk/tests/providers/local/catsn.0008.out.expected
trunk/tests/providers/local/catsn.0009.out.expected
trunk/tests/providers/local/catsn.0010.out.expected
trunk/tests/providers/local/data.txt
trunk/tests/providers/local/sites.template.xml
trunk/tests/providers/local/tc.template.data
trunk/tests/providers/local/title.txt
trunk/tests/providers/sge-local/
trunk/tests/providers/sge-local/001-catsn-local.check.sh
trunk/tests/providers/sge-local/001-catsn-local.setup.sh
trunk/tests/providers/sge-local/001-catsn-local.swift
trunk/tests/providers/sge-local/001-catsn-local.timeout
trunk/tests/providers/sge-local/catsn.0001.out.expected
trunk/tests/providers/sge-local/catsn.0002.out.expected
trunk/tests/providers/sge-local/catsn.0003.out.expected
trunk/tests/providers/sge-local/catsn.0004.out.expected
trunk/tests/providers/sge-local/catsn.0005.out.expected
trunk/tests/providers/sge-local/catsn.0006.out.expected
trunk/tests/providers/sge-local/catsn.0007.out.expected
trunk/tests/providers/sge-local/catsn.0008.out.expected
trunk/tests/providers/sge-local/catsn.0009.out.expected
trunk/tests/providers/sge-local/catsn.0010.out.expected
trunk/tests/providers/sge-local/data.txt
trunk/tests/providers/sge-local/sites.template.xml
trunk/tests/providers/sge-local/tc.template.data
trunk/tests/providers/sge-local/title.txt
Removed:
trunk/tests/site/
Log:
Updates for provider tests
Added: trunk/tests/providers/local/001-catsn-local.check.sh
===================================================================
--- trunk/tests/providers/local/001-catsn-local.check.sh (rev 0)
+++ trunk/tests/providers/local/001-catsn-local.check.sh 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -x
+
+for count in `seq --format "%04.f" 1 1 10`
+do
+ [ -f catsn.$count.out ] || exit 1
+ CONTENTS1=$( cat catsn.$count.out.expected )
+ CONTENTS2=$( cat catsn.$count.out )
+ [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1
+done
+exit 0
Property changes on: trunk/tests/providers/local/001-catsn-local.check.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/providers/local/001-catsn-local.setup.sh
===================================================================
--- trunk/tests/providers/local/001-catsn-local.setup.sh (rev 0)
+++ trunk/tests/providers/local/001-catsn-local.setup.sh 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cp -v $GROUP/data.txt . || exit 1
+cp -v $GROUP/*expected . || exit 1
Property changes on: trunk/tests/providers/local/001-catsn-local.setup.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/providers/local/001-catsn-local.swift
===================================================================
--- trunk/tests/providers/local/001-catsn-local.swift (rev 0)
+++ trunk/tests/providers/local/001-catsn-local.swift 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,15 @@
+type file;
+
+app (file o) cat (file i)
+{
+ cat @i stdout=@o;
+}
+
+string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+string char[] = @strsplit(t, "");
+
+file out[];
+foreach j in [1:@toint(@arg("n","10"))] {
+ file data<"data.txt">;
+ out[j] = cat(data);
+}
Added: trunk/tests/providers/local/001-catsn-local.timeout
===================================================================
--- trunk/tests/providers/local/001-catsn-local.timeout (rev 0)
+++ trunk/tests/providers/local/001-catsn-local.timeout 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+30
Added: trunk/tests/providers/local/catsn.0001.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0001.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0001.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0002.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0002.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0002.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0003.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0003.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0003.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0004.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0004.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0004.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0005.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0005.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0005.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0006.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0006.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0006.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0007.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0007.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0007.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0008.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0008.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0008.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0009.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0009.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0009.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/catsn.0010.out.expected
===================================================================
--- trunk/tests/providers/local/catsn.0010.out.expected (rev 0)
+++ trunk/tests/providers/local/catsn.0010.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/data.txt
===================================================================
--- trunk/tests/providers/local/data.txt (rev 0)
+++ trunk/tests/providers/local/data.txt 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local/sites.template.xml
===================================================================
--- trunk/tests/providers/local/sites.template.xml (rev 0)
+++ trunk/tests/providers/local/sites.template.xml 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,9 @@
+
+
+
+
+ 10000
+ .31
+ _WORK_
+
+
Added: trunk/tests/providers/local/tc.template.data
===================================================================
--- trunk/tests/providers/local/tc.template.data (rev 0)
+++ trunk/tests/providers/local/tc.template.data 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,8 @@
+local echo /bin/echo INSTALLED INTEL32::LINUX
+local cat /bin/cat INSTALLED INTEL32::LINUX
+local ls /bin/ls INSTALLED INTEL32::LINUX
+local grep /bin/grep INSTALLED INTEL32::LINUX
+local sort /bin/sort INSTALLED INTEL32::LINUX
+local paste /bin/paste INSTALLED INTEL32::LINUX
+local wc /usr/bin/wc INSTALLED INTEL32::LINUX
+
Added: trunk/tests/providers/local/title.txt
===================================================================
--- trunk/tests/providers/local/title.txt (rev 0)
+++ trunk/tests/providers/local/title.txt 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Local Configuration Test
Added: trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh
===================================================================
--- trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh (rev 0)
+++ trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -x
+
+for count in `seq --format "%04.f" 1 1 10`
+do
+ [ -f catsn.$count.out ] || exit 1
+ CONTENTS1=$( cat catsn.$count.out.expected )
+ CONTENTS2=$( cat catsn.$count.out )
+ [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1
+done
+exit 0
Property changes on: trunk/tests/providers/local-pbs/001-catsn-localpbs.check.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh
===================================================================
--- trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh (rev 0)
+++ trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cp -v $GROUP/data.txt . || exit 1
+cp -v $GROUP/*expected . || exit 1
Property changes on: trunk/tests/providers/local-pbs/001-catsn-localpbs.setup.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/providers/local-pbs/001-catsn-localpbs.swift
===================================================================
--- trunk/tests/providers/local-pbs/001-catsn-localpbs.swift (rev 0)
+++ trunk/tests/providers/local-pbs/001-catsn-localpbs.swift 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,15 @@
+type file;
+
+app (file o) cat (file i)
+{
+ cat @i stdout=@o;
+}
+
+string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+string char[] = @strsplit(t, "");
+
+file out[];
+foreach j in [1:@toint(@arg("n","10"))] {
+ file data<"data.txt">;
+ out[j] = cat(data);
+}
Added: trunk/tests/providers/local-pbs/001-catsn-localpbs.timeout
===================================================================
--- trunk/tests/providers/local-pbs/001-catsn-localpbs.timeout (rev 0)
+++ trunk/tests/providers/local-pbs/001-catsn-localpbs.timeout 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+180
Added: trunk/tests/providers/local-pbs/catsn.0001.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0001.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0001.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0002.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0002.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0002.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0003.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0003.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0003.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0004.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0004.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0004.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0005.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0005.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0005.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0006.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0006.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0006.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0007.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0007.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0007.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0008.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0008.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0008.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0009.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0009.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0009.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/catsn.0010.out.expected
===================================================================
--- trunk/tests/providers/local-pbs/catsn.0010.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs/catsn.0010.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/data.txt
===================================================================
--- trunk/tests/providers/local-pbs/data.txt (rev 0)
+++ trunk/tests/providers/local-pbs/data.txt 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs/sites.template.xml
===================================================================
--- trunk/tests/providers/local-pbs/sites.template.xml (rev 0)
+++ trunk/tests/providers/local-pbs/sites.template.xml 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,8 @@
+
+
+
+
+ 0
+ _WORK_
+
+
Added: trunk/tests/providers/local-pbs/tc.template.data
===================================================================
--- trunk/tests/providers/local-pbs/tc.template.data (rev 0)
+++ trunk/tests/providers/local-pbs/tc.template.data 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,8 @@
+local-pbs echo /bin/echo INSTALLED INTEL32::LINUX
+local-pbs cat /bin/cat INSTALLED INTEL32::LINUX
+local-pbs ls /bin/ls INSTALLED INTEL32::LINUX
+local-pbs grep /bin/grep INSTALLED INTEL32::LINUX
+local-pbs sort /bin/sort INSTALLED INTEL32::LINUX
+local-pbs paste /bin/paste INSTALLED INTEL32::LINUX
+local-pbs wc /usr/bin/wc INSTALLED INTEL32::LINUX
+
Added: trunk/tests/providers/local-pbs/title.txt
===================================================================
--- trunk/tests/providers/local-pbs/title.txt (rev 0)
+++ trunk/tests/providers/local-pbs/title.txt 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Local PBS Configuration Test
Added: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh
===================================================================
--- trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -x
+
+for count in `seq --format "%04.f" 1 1 10`
+do
+ [ -f catsn.$count.out ] || exit 1
+ CONTENTS1=$( cat catsn.$count.out.expected )
+ CONTENTS2=$( cat catsn.$count.out )
+ [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1
+done
+exit 0
Property changes on: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.check.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh
===================================================================
--- trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cp -v $GROUP/data.txt . || exit 1
+cp -v $GROUP/*expected . || exit 1
Property changes on: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.setup.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.swift
===================================================================
--- trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.swift (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.swift 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,15 @@
+type file;
+
+app (file o) cat (file i)
+{
+ cat @i stdout=@o;
+}
+
+string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+string char[] = @strsplit(t, "");
+
+file out[];
+foreach j in [1:@toint(@arg("n","10"))] {
+ file data<"data.txt">;
+ out[j] = cat(data);
+}
Added: trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.timeout
===================================================================
--- trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.timeout (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/001-catsn-pads-local-pbs-coasters.timeout 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+180
Added: trunk/tests/providers/local-pbs-coasters/catsn.0001.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0001.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0001.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0002.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0002.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0002.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0003.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0003.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0003.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0004.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0004.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0004.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0005.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0005.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0005.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0006.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0006.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0006.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0007.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0007.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0007.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0008.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0008.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0008.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0009.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0009.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0009.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/catsn.0010.out.expected
===================================================================
--- trunk/tests/providers/local-pbs-coasters/catsn.0010.out.expected (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/catsn.0010.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/data.txt
===================================================================
--- trunk/tests/providers/local-pbs-coasters/data.txt (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/data.txt 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/local-pbs-coasters/sites.template.xml
===================================================================
--- trunk/tests/providers/local-pbs-coasters/sites.template.xml (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/sites.template.xml 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,15 @@
+
+
+
+
+ 3000
+ 1
+ 1
+ 1
+ 1
+ fast
+ 5.99
+ 10000
+ _WORK_
+
+
Added: trunk/tests/providers/local-pbs-coasters/tc.template.data
===================================================================
--- trunk/tests/providers/local-pbs-coasters/tc.template.data (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/tc.template.data 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,8 @@
+local-pbs-coasters echo /bin/echo INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00"
+local-pbs-coasters cat /bin/cat INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00"
+local-pbs-coasters ls /bin/ls INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00"
+local-pbs-coasters grep /bin/grep INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00"
+local-pbs-coasters sort /bin/sort INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00"
+local-pbs-coasters paste /bin/paste INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00"
+local-pbs-coasters wc /usr/bin/wc INSTALLED INTEL32::LINUX GLOBUS::maxwalltime="00:05:00"
+
Added: trunk/tests/providers/local-pbs-coasters/title.txt
===================================================================
--- trunk/tests/providers/local-pbs-coasters/title.txt (rev 0)
+++ trunk/tests/providers/local-pbs-coasters/title.txt 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Local PBS with Coasters Configuration Test
Added: trunk/tests/providers/sge-local/001-catsn-local.check.sh
===================================================================
--- trunk/tests/providers/sge-local/001-catsn-local.check.sh (rev 0)
+++ trunk/tests/providers/sge-local/001-catsn-local.check.sh 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -x
+
+for count in `seq --format "%04.f" 1 1 10`
+do
+ [ -f catsn.$count.out ] || exit 1
+ CONTENTS1=$( cat catsn.$count.out.expected )
+ CONTENTS2=$( cat catsn.$count.out )
+ [[ $CONTENTS1 == $CONTENTS2 ]] || exit 1
+done
+exit 0
Property changes on: trunk/tests/providers/sge-local/001-catsn-local.check.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/providers/sge-local/001-catsn-local.setup.sh
===================================================================
--- trunk/tests/providers/sge-local/001-catsn-local.setup.sh (rev 0)
+++ trunk/tests/providers/sge-local/001-catsn-local.setup.sh 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cp -v $GROUP/data.txt . || exit 1
+cp -v $GROUP/*expected . || exit 1
Property changes on: trunk/tests/providers/sge-local/001-catsn-local.setup.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/providers/sge-local/001-catsn-local.swift
===================================================================
--- trunk/tests/providers/sge-local/001-catsn-local.swift (rev 0)
+++ trunk/tests/providers/sge-local/001-catsn-local.swift 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,15 @@
+type file;
+
+app (file o) cat (file i)
+{
+ cat @i stdout=@o;
+}
+
+string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+string char[] = @strsplit(t, "");
+
+file out[];
+foreach j in [1:@toint(@arg("n","10"))] {
+ file data<"data.txt">;
+ out[j] = cat(data);
+}
Added: trunk/tests/providers/sge-local/001-catsn-local.timeout
===================================================================
--- trunk/tests/providers/sge-local/001-catsn-local.timeout (rev 0)
+++ trunk/tests/providers/sge-local/001-catsn-local.timeout 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+30
Added: trunk/tests/providers/sge-local/catsn.0001.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0001.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0001.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0002.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0002.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0002.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0003.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0003.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0003.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0004.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0004.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0004.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0005.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0005.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0005.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0006.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0006.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0006.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0007.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0007.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0007.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0008.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0008.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0008.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0009.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0009.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0009.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/catsn.0010.out.expected
===================================================================
--- trunk/tests/providers/sge-local/catsn.0010.out.expected (rev 0)
+++ trunk/tests/providers/sge-local/catsn.0010.out.expected 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/data.txt
===================================================================
--- trunk/tests/providers/sge-local/data.txt (rev 0)
+++ trunk/tests/providers/sge-local/data.txt 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+Hello world
Added: trunk/tests/providers/sge-local/sites.template.xml
===================================================================
--- trunk/tests/providers/sge-local/sites.template.xml (rev 0)
+++ trunk/tests/providers/sge-local/sites.template.xml 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,10 @@
+
+
+
+ threaded
+ .49
+ 10000
+
+ _WORK_
+
+
Added: trunk/tests/providers/sge-local/tc.template.data
===================================================================
--- trunk/tests/providers/sge-local/tc.template.data (rev 0)
+++ trunk/tests/providers/sge-local/tc.template.data 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1,8 @@
+sge-local echo /bin/echo INSTALLED INTEL32::LINUX
+sge-local cat /bin/cat INSTALLED INTEL32::LINUX
+sge-local ls /bin/ls INSTALLED INTEL32::LINUX
+sge-local grep /bin/grep INSTALLED INTEL32::LINUX
+sge-local sort /bin/sort INSTALLED INTEL32::LINUX
+sge-local paste /bin/paste INSTALLED INTEL32::LINUX
+sge-local wc /usr/bin/wc INSTALLED INTEL32::LINUX
+
Added: trunk/tests/providers/sge-local/title.txt
===================================================================
--- trunk/tests/providers/sge-local/title.txt (rev 0)
+++ trunk/tests/providers/sge-local/title.txt 2010-12-21 23:57:20 UTC (rev 3816)
@@ -0,0 +1 @@
+SGE Local Configuration Test
From noreply at svn.ci.uchicago.edu Tue Dec 21 20:08:21 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 21 Dec 2010 20:08:21 -0600 (CST)
Subject: [Swift-commit] r3817 - trunk/tests/groups
Message-ID: <20101222020821.BC08E9CC8D@svn.ci.uchicago.edu>
Author: davidk
Date: 2010-12-21 20:08:21 -0600 (Tue, 21 Dec 2010)
New Revision: 3817
Added:
trunk/tests/groups/group-pbs.sh
trunk/tests/groups/group-sge.sh
Log:
Group definitions for pbs and sge tests
Added: trunk/tests/groups/group-pbs.sh
===================================================================
--- trunk/tests/groups/group-pbs.sh (rev 0)
+++ trunk/tests/groups/group-pbs.sh 2010-12-22 02:08:21 UTC (rev 3817)
@@ -0,0 +1,7 @@
+
+# GROUPLIST definition to run pbs tests
+
+GROUPLIST=(
+ $TESTDIR/providers/local-pbs \
+ $TESTDIR/providers/local-pbs-coasters \
+)
Added: trunk/tests/groups/group-sge.sh
===================================================================
--- trunk/tests/groups/group-sge.sh (rev 0)
+++ trunk/tests/groups/group-sge.sh 2010-12-22 02:08:21 UTC (rev 3817)
@@ -0,0 +1,6 @@
+
+# GROUPLIST definition to run sge tests
+
+GROUPLIST=(
+ $TESTDIR/providers/sge-local \
+)
From noreply at svn.ci.uchicago.edu Wed Dec 22 09:02:25 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 22 Dec 2010 09:02:25 -0600 (CST)
Subject: [Swift-commit] r3818 - text/parco10submission
Message-ID: <20101222150225.032E89CC7F@svn.ci.uchicago.edu>
Author: dsk
Date: 2010-12-22 09:02:23 -0600 (Wed, 22 Dec 2010)
New Revision: 3818
Modified:
text/parco10submission/paper.tex
Log:
minor changes to new rationale section
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-22 02:08:21 UTC (rev 3817)
+++ text/parco10submission/paper.tex 2010-12-22 15:02:23 UTC (rev 3818)
@@ -123,22 +123,18 @@
component programs on grids and other parallel platforms, providing
automated site selection, data management, and reliability.
-This paper goes into greater depth than prior
-publications~\cite{Swift_2007} in describing the Swift language, how
-its implementation handles large-scale and distributed execution
-environments, and its contribution to distributed and parallel computing.
-\katznote{Alternative: Swift has been described previously~\cite{Swift_2007} ;
+Swift has been described previously~\cite{Swift_2007};
this paper goes into greater depth in describing the Swift language, how
its implementation handles large-scale and distributed execution
-environments, and its contribution to distributed and parallel computing.}
+environments, and its contribution to distributed and parallel computing.
\subsection*{Rationale}
\begin{msection}
The main goal of Swift is to allow the composition of coarse grained
-processes. Such a composition will be termed \emph{Swift program}. A
+processes. Such a composition is termed a \emph{Swift program}. A
secondary goal is to parallelize and manage the execution of a Swift
-program on shared collections of resources (i.e. Grids), while at the
+program on shared collections of resources (e.g., grids), while at the
same time maintaining data determinism.
The exact number of processing units available on such shared resources
@@ -148,7 +144,7 @@
parallelized. Having the results be independent of the way the processes
are parallelized implies that the processes must, for the same input,
produce the same output, irrespective of the time, order or location in
-which they are "executed". This characteristic is reminiscent of
+which they are ``executed''. This characteristic is reminiscent of
referential transparency, and one may readily extend the concept to
encompass arbitrary processes without difficulty.
@@ -165,9 +161,9 @@
rotatedImage = rotate(image, angle);
\end{verbatim}
the semantics are defined in terms of the specification of the function
-"rotate" when supplied with specific parameter types, and whether the
-implementation can be described as a "library call" or a "program
-invocation" changes nothing with respect to what the piece of program
+``rotate'' when supplied with specific parameter types, and whether the
+implementation can be described as a ``library call'' or a ``program
+invocation'' changes nothing with respect to what the piece of program
fundamentally does: produce a rotated version of the original.
Indeed, there is no strict requirement in the specification of the Swift
@@ -178,32 +174,32 @@
functions across a collection of heterogeneous resources, which, with
the advent of projects such as TeraGrid, suggests an implementation in
which functions are applications readily executable on them through the
-careful employment of Grid middleware.
+careful employment of grid middleware.
Having established the constraint that Swift functions must in general
be referentially transparent, and in order to preserve referential
transparency at different levels of abstractions within the language, it
follows that the appropriate form for the Swift language is functional.
-We choose to make the Swift language purely functional (i.e. disallow
-side-effects in the language) in order to prevent the difficulties that
-arise from having to track side-effects to ensure determinism in complex
+We choose to make the Swift language purely functional (i.e., we disallow
+side effects in the language) in order to prevent the difficulties that
+arise from having to track side effects to ensure determinism in complex
concurrency scenarios.
It is already known that functional programming allows consistent
implementations of evaluation strategies different from the widespread
eager evaluation, a fact readily apparent in lazily evaluated languages
such as Haskell and Miranda. In order to achieve automatic
-parallelization in Swift, instead of thunking (i.e. suspended
-computations) which yields lazy evaluation, we employ futures, which
-results in eager parallelism. In this process, we trade the ability to
+parallelization in Swift, instead of using thunks (i.e., suspended
+computations), which yield lazy evaluation, we employ futures, which
+result in eager parallelism. In this process, we trade the ability to
efficiently deal with infinite structures for the ability to minimize
computation time. It must, however, be noted that a middle ground
exists: lazy futures (futures whose computation is delayed until a value
is first needed).
A number of issues may be noted at this point. First, there exist a
-certain class of processes which may break referential transparency,
-while still producing "valid" results. An example would be Monte Carlo
+certain class of processes that may break referential transparency,
+while still producing ``valid'' results. An example would be Monte Carlo
processes. Under the assumption that eager evaluation of compositions of
Monte Carlo processes also produces valid results, an eager Swift
implementation (which is the case with the current implementation)
From noreply at svn.ci.uchicago.edu Wed Dec 22 10:03:10 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 22 Dec 2010 10:03:10 -0600 (CST)
Subject: [Swift-commit] r3819 - text/parco10submission
Message-ID: <20101222160310.1B5079CC82@svn.ci.uchicago.edu>
Author: dsk
Date: 2010-12-22 10:03:09 -0600 (Wed, 22 Dec 2010)
New Revision: 3819
Modified:
text/parco10submission/paper.tex
Log:
2 more small changes
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-22 15:02:23 UTC (rev 3818)
+++ text/parco10submission/paper.tex 2010-12-22 16:03:09 UTC (rev 3819)
@@ -106,7 +106,7 @@
Swift's contribution and primary value is that it provides a simple,
minimal set of language constructs to specify how applications are
glued together at large scale in a simple compact form, while keeping
-the language simple and elegant, and minimizing any overlap with the
+the language simple \katznote{second ``simple'' in this sentence...} and elegant, and minimizing any overlap with the
tasks that existing scripting languages do well. Swift regularizes and
abstracts notions of external data and processes for distributed
parallel execution of application programs. Applications expressed in
@@ -964,8 +964,8 @@
Grid (OSG)~\cite{OSG_2007}, where the set of sites that may be used is
large and changing. It is impractical to maintain a site catalog by
hand in this situation. In collaboration with the OSG Engagement
-group, Swift has been interfaced to ReSS~\cite{ReSS_2007} \katznote{say what ReSS is}
-so that the site
+group, Swift has been interfaced to ReSS~\cite{ReSS_2007},
+an OSG resource selection service, so that the site
catalog is generated from that information system. This provides a
very straightforward way to generate a large catalog of sites.
From noreply at svn.ci.uchicago.edu Wed Dec 22 16:44:04 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 22 Dec 2010 16:44:04 -0600 (CST)
Subject: [Swift-commit] r3820 - text/parco10submission
Message-ID: <20101222224404.E1A6E9CC82@svn.ci.uchicago.edu>
Author: dsk
Date: 2010-12-22 16:44:04 -0600 (Wed, 22 Dec 2010)
New Revision: 3820
Modified:
text/parco10submission/paper.tex
Log:
adding Mihael's suggestion re 'function' vs 'procedure' as a note
Modified: text/parco10submission/paper.tex
===================================================================
--- text/parco10submission/paper.tex 2010-12-22 16:03:09 UTC (rev 3819)
+++ text/parco10submission/paper.tex 2010-12-22 22:44:04 UTC (rev 3820)
@@ -89,6 +89,10 @@
\katznote{in general, I would like to use application, job, task, program, etc. in some way that is consistent throughout the paper, perhaps defined in the introduction}
+\katznote{Mihael's suggestion: ``Should we abandon the
+"procedure" term that we use for functions? It alludes to imperative
+programming which Swift is pretty far from.'' Dan agrees with this.}
+
\katznote{are swift tasks idempotent? are swift scripts are deterministic (maybe?) deadlock-free? (no)}
Swift is a scripting language designed for composing ordinary
From noreply at svn.ci.uchicago.edu Thu Dec 23 11:37:14 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 23 Dec 2010 11:37:14 -0600 (CST)
Subject: [Swift-commit] r3822 - trunk/libexec
Message-ID: <20101223173714.6A68C9CC8D@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-23 11:37:14 -0600 (Thu, 23 Dec 2010)
New Revision: 3822
Modified:
trunk/libexec/_swiftwrap
Log:
DEBUG=0 by default, debug MPI_RANK
Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap 2010-12-23 17:35:52 UTC (rev 3821)
+++ trunk/libexec/_swiftwrap 2010-12-23 17:37:14 UTC (rev 3822)
@@ -3,7 +3,7 @@
# note that this script modifies $IFS
# Toggle debugging output from debug()
-DEBUG=1
+DEBUG=0
infosection() {
echo >& "$INFO"
@@ -52,7 +52,7 @@
}
debug() {
- [ $DEBUG == 1 ] && echo "$@" >& "$INFO"
+ [[ $DEBUG == 1 ]] && echo "$@" >& "$INFO"
}
fail() {
@@ -352,9 +352,9 @@
log "OUTF=$OUTF"
log "KICKSTART=$KICKSTART"
log "CDM_FILE=$CDM_FILE"
+[ -z $MPI_RANK ] && log "MPI_RANK=$MPI_RANK"
log "ARGS=$@"
log "ARGC=$#"
-
IFS="|"
logstate "CREATE_JOBDIR"
@@ -426,6 +426,8 @@
fi
logstate "EXECUTE"
+debug "Command line: $EXEC ${CMDARGS[@]}"
+
if [ ! -f "$EXEC" ]; then
fail 254 "The executable $EXEC does not exist"
fi
From noreply at svn.ci.uchicago.edu Thu Dec 23 12:31:47 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 23 Dec 2010 12:31:47 -0600 (CST)
Subject: [Swift-commit] r3823 - trunk/libexec
Message-ID: <20101223183147.6A7879CC8D@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-23 12:31:47 -0600 (Thu, 23 Dec 2010)
New Revision: 3823
Modified:
trunk/libexec/cdm_lib.sh
Log:
Fix CDM LOCAL cp/dd options
Modified: trunk/libexec/cdm_lib.sh
===================================================================
--- trunk/libexec/cdm_lib.sh 2010-12-23 17:37:14 UTC (rev 3822)
+++ trunk/libexec/cdm_lib.sh 2010-12-23 18:31:47 UTC (rev 3823)
@@ -51,9 +51,7 @@
fi
;;
LOCAL)
- #TOOL=${ARGS[0]}
- #REMOTE_DIR=${ARGS[1]}
- #FLAGS=${ARGS[3]}
+ # TODO: Can/should we use this as a cache?
TOOL=$1
REMOTE_DIR=$2
FLAGS=$3
@@ -62,10 +60,17 @@
if [ $MODE == "INPUT" ]; then
[ -f "$REMOTE_DIR/$FILE" ]
checkError 254 "CDM[LOCAL]: $REMOTE_DIR/$FILE does not exist!"
- $TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE
- checkError 254 "CDM[LOCAL]: Tool failed!"
+ if [ $TOOL == "cp" ]; then
+ $TOOL $FLAGS $REMOTE_DIR/$FILE $JOBDIR/$FILE
+ checkError 254 "CDM[LOCAL]: cp failed!"
+ elif [ $TOOL == "dd" ]; then
+ $TOOL $FLAGS if=$REMOTE_DIR/$FILE of=$JOBDIR/$FILE
+ checkError 254 "CDM[LOCAL]: dd failed!"
+ else
+ fail 254 "CDM[LOCAL]: Unknown TOOL: $TOOL"
+ fi
elif [ $MODE == "OUTPUT" ]; then
- log "CDM[LOCAL]..."
+ log "CDM[LOCAL]..." # This should probably be an error
else
fail 254 "Unknown MODE: $MODE"
fi
From noreply at svn.ci.uchicago.edu Thu Dec 23 12:34:57 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 23 Dec 2010 12:34:57 -0600 (CST)
Subject: [Swift-commit] r3824 - trunk/libexec
Message-ID: <20101223183457.2F1769CC8D@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-23 12:34:56 -0600 (Thu, 23 Dec 2010)
New Revision: 3824
Modified:
trunk/libexec/vdl-int.k
Log:
Add emacs buffer variables
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2010-12-23 18:31:47 UTC (rev 3823)
+++ trunk/libexec/vdl-int.k 2010-12-23 18:34:56 UTC (rev 3824)
@@ -402,7 +402,7 @@
replicationGroup, replicationChannel]
stagein := list(unique(each(stagein)))
stageout := list(unique(each(stageout)))
-
+
allocateHost(rhost, constraints=vdl:jobConstraints(tr, stagein=stagein)
ddir := initDDir()
@@ -617,3 +617,9 @@
)
)
)
+
+// Local variables:
+// mode: scheme
+// tab-width: 4
+// indent-tabs-mode: t
+// End:
From noreply at svn.ci.uchicago.edu Thu Dec 23 12:36:52 2010
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 23 Dec 2010 12:36:52 -0600 (CST)
Subject: [Swift-commit] r3825 - trunk/tests/mpi
Message-ID: <20101223183652.6CBCA9CC82@svn.ci.uchicago.edu>
Author: wozniak
Date: 2010-12-23 12:36:52 -0600 (Thu, 23 Dec 2010)
New Revision: 3825
Added:
trunk/tests/mpi/title.txt
Modified:
trunk/tests/mpi/100-mci.setup.sh
trunk/tests/mpi/100-mpi-cp.c
trunk/tests/mpi/tc.template.data
Log:
MPI test works again
Modified: trunk/tests/mpi/100-mci.setup.sh
===================================================================
--- trunk/tests/mpi/100-mci.setup.sh 2010-12-23 18:34:56 UTC (rev 3824)
+++ trunk/tests/mpi/100-mci.setup.sh 2010-12-23 18:36:52 UTC (rev 3825)
@@ -4,7 +4,7 @@
which mpicc || exit 1
-mpicc -std=gnu99 $GROUP/100-mpi-cp.c -o mpi-cp || exit 1
+mpicc -std=gnu99 $GROUP/100-mpi-cp.c -o $GROUP/mpi-cp || exit 1
cp -v $GROUP/100-input.txt . || exit 1
Modified: trunk/tests/mpi/100-mpi-cp.c
===================================================================
--- trunk/tests/mpi/100-mpi-cp.c 2010-12-23 18:34:56 UTC (rev 3824)
+++ trunk/tests/mpi/100-mpi-cp.c 2010-12-23 18:36:52 UTC (rev 3825)
@@ -1,5 +1,11 @@
-/*
+
+/**
* Simple Hydra test
+ *
+ * usage: mpi-cp