;
+
+outfile = greeting(n);
+
From noreply at svn.ci.uchicago.edu Tue Aug 5 09:14:06 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 5 Aug 2008 09:14:06 -0500 (CDT)
Subject: [Swift-commit] r2176 - trunk/tests/language-behaviour
Message-ID: <20080805141406.AAAF61360015@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-05 09:14:06 -0500 (Tue, 05 Aug 2008)
New Revision: 2176
Added:
trunk/tests/language-behaviour/readData.float.out.expected
Modified:
trunk/tests/language-behaviour/readData.swift
Log:
test that readData can cope with floats
Added: trunk/tests/language-behaviour/readData.float.out.expected
===================================================================
--- trunk/tests/language-behaviour/readData.float.out.expected (rev 0)
+++ trunk/tests/language-behaviour/readData.float.out.expected 2008-08-05 14:14:06 UTC (rev 2176)
@@ -0,0 +1 @@
+3.1400001
Modified: trunk/tests/language-behaviour/readData.swift
===================================================================
--- trunk/tests/language-behaviour/readData.swift 2008-08-05 11:27:08 UTC (rev 2175)
+++ trunk/tests/language-behaviour/readData.swift 2008-08-05 14:14:06 UTC (rev 2176)
@@ -13,6 +13,13 @@
}
}
+(file f) createFloat() {
+ app{
+ echo "3.1400001" stdout=@filename(f);
+ }
+}
+
+
(file f) write(string data) {
app{
echo data stdout=@filename(f);
@@ -20,6 +27,7 @@
}
int i;
+float fl;
int ia[];
@@ -35,9 +43,14 @@
//input file. assumed on disk
file g <"readData.intArray.in">;
+// will be generated
+file h <"readData.float.in">;
+
f = createInt();
+h = createFloat();
i = readData(f);
+fl = readData(h);
file o1 <"readData.int.out">;
o1 = write(@strcat(i));
@@ -61,3 +74,7 @@
file o5 <"readData.circleArray.out">;
o5 = write(@strcat(ca[0].x, " ", ca[1].y, " ", ca[0].r, " ", ca[1].name));
+
+file o6 <"readData.float.out">;
+o6 = write(@strcat(fl));
+
From noreply at svn.ci.uchicago.edu Wed Aug 6 03:57:32 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 6 Aug 2008 03:57:32 -0500 (CDT)
Subject: [Swift-commit] r2177 - trunk/src/org/griphyn/vdl/karajan
Message-ID: <20080806085732.D8D999B4001@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-06 03:57:27 -0500 (Wed, 06 Aug 2008)
New Revision: 2177
Removed:
trunk/src/org/griphyn/vdl/karajan/VDSScheduler.java
Log:
remove unused old scheduler - this has dependencies on VDS precompiled java classes that I am trying to eliminate.
Deleted: trunk/src/org/griphyn/vdl/karajan/VDSScheduler.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/VDSScheduler.java 2008-08-05 14:14:06 UTC (rev 2176)
+++ trunk/src/org/griphyn/vdl/karajan/VDSScheduler.java 2008-08-06 08:57:27 UTC (rev 2177)
@@ -1,83 +0,0 @@
-/*
- * Created on Jun 12, 2006
- */
-package org.griphyn.vdl.karajan;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.globus.cog.karajan.scheduler.LateBindingScheduler;
-import org.globus.cog.karajan.scheduler.NoFreeResourceException;
-import org.globus.cog.karajan.scheduler.TaskConstraints;
-import org.globus.cog.karajan.util.BoundContact;
-import org.globus.cog.karajan.util.ContactSet;
-import org.griphyn.cPlanner.classes.SubInfo;
-import org.griphyn.cPlanner.selector.SiteSelector;
-import org.griphyn.cPlanner.selector.site.RoundRobin;
-import org.griphyn.common.catalog.transformation.Mapper;
-import org.griphyn.common.catalog.transformation.mapper.Installed;
-
-public class VDSScheduler extends LateBindingScheduler {
- private SiteSelector siteSelector;
- private List pools;
- private Mapper tcmapper;
-
- public VDSScheduler() {
- this.siteSelector = new RoundRobin();
- this.siteSelector.setTCMapper(tcmapper = new Installed());
- this.addTaskTransformer(new VDSTaskTransformer(tcmapper));
- }
-
- public void setResources(ContactSet grid) {
- super.setResources(grid);
- pools = new ArrayList();
- for (int i = 0; i < grid.size(); i++) {
- pools.add(grid.get(i).getHost());
- }
- }
-
- protected BoundContact getNextContact(TaskConstraints t) throws NoFreeResourceException {
- String host = siteSelector.mapJob2ExecPool(toSubInfo(t), pools);
- if (host.endsWith(":null")) {
- host = host.substring(0, host.length() - 5);
- }
- BoundContact bc = getResources().getContact(host);
- if (bc == null) {
- throw new RuntimeException("Could not find a valid site for transformation '"
- + t.getConstraint("tr") + "'");
- }
- else {
- return bc;
- }
- }
-
- private SubInfo toSubInfo(TaskConstraints t) {
- SubInfo si = new SubInfo();
- if (isPresent("tr", t)) {
- String tr = (String) t.getConstraint("tr");
- si.setTransformation(null, tr, null);
- si.setJobClass(SubInfo.STAGED_COMPUTE_JOB);
- }
- return si;
- }
-
- private boolean isPresent(String constraint, TaskConstraints t) {
- if (t == null) {
- return false;
- }
- if (t.getConstraint(constraint) == null) {
- return false;
- }
- return true;
- }
-
- public void setProperty(String name, Object value) {
- if ("siteSelectorClassName".equals(name)) {
- this.siteSelector = SiteSelector.loadSiteSelector((String) value, null);
- this.siteSelector.setTCMapper(new Installed());
- }
- else {
- super.setProperty(name, value);
- }
- }
-}
From noreply at svn.ci.uchicago.edu Wed Aug 6 12:00:46 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 6 Aug 2008 12:00:46 -0500 (CDT)
Subject: [Swift-commit] r2178 - in trunk/tests: misc sites/wonky
Message-ID: <20080806170046.0DED81AB0193@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-06 12:00:45 -0500 (Wed, 06 Aug 2008)
New Revision: 2178
Added:
trunk/tests/misc/wonky-replication.sh
trunk/tests/sites/wonky/slow-queue-fast-queue.xml
Log:
test for replication, making use of provider-wonky. this test hangs on
my laptop at the moment.
Added: trunk/tests/misc/wonky-replication.sh
===================================================================
--- trunk/tests/misc/wonky-replication.sh (rev 0)
+++ trunk/tests/misc/wonky-replication.sh 2008-08-06 17:00:45 UTC (rev 2178)
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# run tests with replication enabled and one site set to queue for a long
+# time
+
+export CF=swift.properties.enable-replication
+
+cd ../language-behaviour
+
+./generate-tc.data
+
+cat tc.data | grep -e '^localhost' > tmp.tc.data.wonky-twosite
+
+cat tc.data ../sites/tc.data > tmp.tc.data.sites
+
+cat tmp.tc.data.wonky-twosite | sed 's/localhost/wonkyA/' >> tmp.tc.data.sites
+cat tmp.tc.data.wonky-twosite | sed 's/localhost/wonkyB/' >> tmp.tc.data.sites
+
+cat $(dirname $(which swift))/../etc/swift.properties | grep --invert-match -E '^replication.enabled=' > $CF
+echo replication.enabled=true >> $CF
+
+export SWIFT_TEST_PARAMS="-config $CF -tc.file tmp.tc.data.sites -sites.file ../sites/wonky/slow-queue-fast-queue.xml -debug"
+
+./run 066-many.swift
+
+
Property changes on: trunk/tests/misc/wonky-replication.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/sites/wonky/slow-queue-fast-queue.xml
===================================================================
--- trunk/tests/sites/wonky/slow-queue-fast-queue.xml (rev 0)
+++ trunk/tests/sites/wonky/slow-queue-fast-queue.xml 2008-08-06 17:00:45 UTC (rev 2178)
@@ -0,0 +1,18 @@
+
+
+
+
+
+ /var/tmp
+ 4
+
+
+
+
+
+ /var/tmp
+ 4
+
+
+
+
From noreply at svn.ci.uchicago.edu Wed Aug 6 12:02:54 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 6 Aug 2008 12:02:54 -0500 (CDT)
Subject: [Swift-commit] r2179 - trunk/tests/misc
Message-ID: <20080806170254.641C61AB0193@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-06 12:02:54 -0500 (Wed, 06 Aug 2008)
New Revision: 2179
Added:
trunk/tests/misc/wonky-slow-queue-fast-queue.sh
Log:
another test using provider-wonky that seems to hang on my laptop
Added: trunk/tests/misc/wonky-slow-queue-fast-queue.sh
===================================================================
--- trunk/tests/misc/wonky-slow-queue-fast-queue.sh (rev 0)
+++ trunk/tests/misc/wonky-slow-queue-fast-queue.sh 2008-08-06 17:02:54 UTC (rev 2179)
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# run tests with replication enabled and one site set to queue for a long
+# time
+
+cd ../language-behaviour
+
+./generate-tc.data
+
+cat tc.data | grep -e '^localhost' > tmp.tc.data.wonky-twosite
+
+cat tc.data ../sites/tc.data > tmp.tc.data.sites
+
+cat tmp.tc.data.wonky-twosite | sed 's/localhost/wonkyA/' >> tmp.tc.data.sites
+cat tmp.tc.data.wonky-twosite | sed 's/localhost/wonkyB/' >> tmp.tc.data.sites
+
+export SWIFT_TEST_PARAMS="-tc.file tmp.tc.data.sites -sites.file ../sites/wonky/slow-queue-fast-queue.xml -debug"
+
+./run 066-many.swift
+
+
Property changes on: trunk/tests/misc/wonky-slow-queue-fast-queue.sh
___________________________________________________________________
Name: svn:executable
+ *
From noreply at svn.ci.uchicago.edu Thu Aug 7 08:31:48 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 7 Aug 2008 08:31:48 -0500 (CDT)
Subject: [Swift-commit] r2180 - trunk/tests/sites/wonky
Message-ID: <20080807133148.518D71360015@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-07 08:31:47 -0500 (Thu, 07 Aug 2008)
New Revision: 2180
Modified:
trunk/tests/sites/wonky/slow-queue-fast-queue.xml
Log:
slow-queue fast-queue site config was causing hangs; use different delay parameter to make it work
Modified: trunk/tests/sites/wonky/slow-queue-fast-queue.xml
===================================================================
--- trunk/tests/sites/wonky/slow-queue-fast-queue.xml 2008-08-06 17:02:54 UTC (rev 2179)
+++ trunk/tests/sites/wonky/slow-queue-fast-queue.xml 2008-08-07 13:31:47 UTC (rev 2180)
@@ -9,7 +9,7 @@
-
+
/var/tmp
4
From noreply at svn.ci.uchicago.edu Thu Aug 7 08:34:45 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 7 Aug 2008 08:34:45 -0500 (CDT)
Subject: [Swift-commit] r2181 - trunk/src/org/griphyn/vdl/util
Message-ID: <20080807133445.7769B1360015@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-07 08:34:45 -0500 (Thu, 07 Aug 2008)
New Revision: 2181
Modified:
trunk/src/org/griphyn/vdl/util/VDL2Config.java
Log:
-config option to override config file was ignored in some cases; most immediately it meant that replication parameters were read from the default config, not a per-run overridden config file.
Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2008-08-07 13:31:47 UTC (rev 2180)
+++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2008-08-07 13:34:45 UTC (rev 2181)
@@ -24,8 +24,10 @@
private static VDL2Config config;
public static VDL2Config getConfig() throws IOException {
- VDL2Config conf = getDefaultConfig();
- return conf.check();
+ if(config == null) {
+ config = getDefaultConfig();
+ }
+ return config.check();
}
private static synchronized VDL2Config getDefaultConfig() throws IOException {
@@ -42,14 +44,15 @@
public static VDL2Config getConfig(String file) throws IOException {
VDL2Config c;
try {
- VDL2Config config = getConfig();
- c = new VDL2Config(config);
+ VDL2Config d = getConfig();
+ c = new VDL2Config(d);
}
catch (Exception e) {
c = new VDL2Config();
}
c.load(file);
- return c.check();
+ config = c;
+ return config.check();
}
private List files, tried;
From noreply at svn.ci.uchicago.edu Thu Aug 7 14:33:30 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 7 Aug 2008 14:33:30 -0500 (CDT)
Subject: [Swift-commit] r2182 -
trunk/src/org/griphyn/vdl/karajan/lib/replication
Message-ID: <20080807193330.5379D1360015@www.ci.uchicago.edu>
Author: hategan
Date: 2008-08-07 14:33:29 -0500 (Thu, 07 Aug 2008)
New Revision: 2182
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java
Log:
added initial queuing time estimate to prevent the no-replication-of-first-batch-of-jobs problem
Modified: trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java 2008-08-07 13:34:45 UTC (rev 2181)
+++ trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java 2008-08-07 19:33:29 UTC (rev 2182)
@@ -17,6 +17,8 @@
public static final Logger logger = Logger.getLogger(ReplicationManager.class);
public static final int STATUS_NEEDS_REPLICATION = 100;
+
+ public static final int INITIAL_QUEUE_TIME_ESTIMATE = 30; //seconds
private int n;
private long s;
@@ -83,17 +85,24 @@
}
public synchronized double getMean() {
- return s / n;
+ if (n == 0) {
+ return INITIAL_QUEUE_TIME_ESTIMATE;
+ }
+ else {
+ return s / n;
+ }
}
public synchronized double getStandardDeviation() {
- return Math.sqrt((s2 - s * s / n) / n);
+ if (n == 0) {
+ return 0;
+ }
+ else {
+ return Math.sqrt((s2 - s * s / n) / n);
+ }
}
public void checkTasks() {
- if (n == 0) {
- return;
- }
Map m;
synchronized (queued) {
m = new HashMap(queued);
@@ -115,7 +124,7 @@
return false;
}
long inTheQueue = (System.currentTimeMillis() - d.getTime()) / 1000;
- if (n > 0 && inTheQueue > minQueueTime && inTheQueue > 3 * getMean()
+ if (inTheQueue > minQueueTime && inTheQueue > 3 * getMean()
&& replicationGroups.getRequestedCount(t) < limit) {
return true;
}
From noreply at svn.ci.uchicago.edu Fri Aug 8 03:25:43 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 8 Aug 2008 03:25:43 -0500 (CDT)
Subject: [Swift-commit] r2183 - in trunk: libexec
src/org/griphyn/vdl/karajan/lib
Message-ID: <20080808082543.44BDC1360015@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-08 03:25:42 -0500 (Fri, 08 Aug 2008)
New Revision: 2183
Modified:
trunk/libexec/vdl-int.k
trunk/src/org/griphyn/vdl/karajan/lib/Execute.java
trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java
Log:
more states for the progress ticker
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2008-08-07 19:33:29 UTC (rev 2182)
+++ trunk/libexec/vdl-int.k 2008-08-08 08:25:42 UTC (rev 2183)
@@ -391,7 +391,7 @@
log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", maybe(" arguments=", arguments), " tmpdir={tmpdir} host={rhost}")
- vdl:setprogress("Executing")
+ vdl:setprogress("Submitting")
vdl:execute("/bin/bash",
list("shared/wrapper.sh", jobid,
"-jobdir", jobdir,
Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2008-08-07 19:33:29 UTC (rev 2182)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2008-08-08 08:25:42 UTC (rev 2183)
@@ -81,13 +81,16 @@
if (stack != null) {
int c = e.getStatus().getStatusCode();
if (c == Status.SUBMITTED) {
+ RuntimeStats.setProgress(stack, "Submitted");
getReplicationManager(stack).submitted(task, e.getStatus().getTime());
}
else if (c == Status.ACTIVE) {
+ RuntimeStats.setProgress(stack, "Active");
getReplicationManager(stack).active(task, e.getStatus().getTime());
((FutureVariableArguments) A_REPLICATION_CHANNEL.getValue(stack)).close();
}
else if (c == ReplicationManager.STATUS_NEEDS_REPLICATION) {
+ RuntimeStats.setProgress(stack, "Replicating");
((FutureVariableArguments) A_REPLICATION_CHANNEL.getValue(stack)).append(Boolean.TRUE);
}
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2008-08-07 19:33:29 UTC (rev 2182)
+++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2008-08-08 08:25:42 UTC (rev 2183)
@@ -25,9 +25,12 @@
"Initializing",
"Selecting site",
"Stage in",
- "Executing",
+ "Submitting",
+ "Submitted",
+ "Active",
"Stage out",
"Failed",
+ "Replicating",
"Finished successfully"
};
@@ -48,11 +51,15 @@
public Object vdl_setprogress(VariableStack stack) throws ExecutionException {
+ setProgress(stack, TypeUtil.toString(PA_STATE.getValue(stack)));
+ return null;
+ }
+
+ static public void setProgress(VariableStack stack, String newState) throws ExecutionException {
RuntimeProgress rp = (RuntimeProgress)stack.getVar("#swift-runtime-progress");
- rp.status = TypeUtil.toString(PA_STATE.getValue(stack));
+ rp.status = newState;
ProgressTicker p = (ProgressTicker)stack.getVar("#swift-runtime-progress-ticker");
p.dumpState();
- return null;
}
public Object vdl_initprogressstate(VariableStack stack) throws ExecutionException {
From noreply at svn.ci.uchicago.edu Fri Aug 8 10:04:16 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 8 Aug 2008 10:04:16 -0500 (CDT)
Subject: [Swift-commit] r2184 - trunk/src/org/griphyn/vdl/util
Message-ID: <20080808150416.8B87A1360015@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-08 10:04:15 -0500 (Fri, 08 Aug 2008)
New Revision: 2184
Modified:
trunk/src/org/griphyn/vdl/util/VDL2Config.java
Log:
remove check for old config file that was deprecated some time ago. This removes the error message that previous appeared, but does not otherwise change config file behaviour.
Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2008-08-08 08:25:42 UTC (rev 2183)
+++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2008-08-08 15:04:15 UTC (rev 2184)
@@ -31,7 +31,6 @@
}
private static synchronized VDL2Config getDefaultConfig() throws IOException {
- checkDeprecatedConfigFile();
if (config == null) {
config = new VDL2Config();
for (int i = 0; i < CONFIG_FILE_SEARCH_PATH.length; i++) {
@@ -205,13 +204,4 @@
return conf;
}
- // TODO this can be removed after 0.5 is released
- static public void checkDeprecatedConfigFile() {
- String fn = System.getProperty("user.home") + File.separator + ".vdl2" + File.separator + "vdl2.properties";
- File f = new File(fn);
- if(f.exists()) {
- System.err.println("The .vdl2 directory is deprecated. Swift has detected the presence of a now-unsupported configuration file, "+fn+". Configuration information will not be loaded from that file. Remove that file to suppress this message.");
- }
- }
-
}
From noreply at svn.ci.uchicago.edu Fri Aug 8 10:08:01 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 8 Aug 2008 10:08:01 -0500 (CDT)
Subject: [Swift-commit] r2185 - trunk/src/org/griphyn/vdl/karajan
Message-ID: <20080808150801.342F01360015@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-08 10:08:00 -0500 (Fri, 08 Aug 2008)
New Revision: 2185
Modified:
trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java
Log:
remove constructor and associated methods that were not used and were dependent on VDS precompiled code
Modified: trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2008-08-08 15:04:15 UTC (rev 2184)
+++ trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2008-08-08 15:08:00 UTC (rev 2185)
@@ -21,7 +21,6 @@
import org.globus.cog.karajan.util.Contact;
import org.globus.cog.karajan.workflow.KarajanRuntimeException;
import org.griphyn.common.catalog.TransformationCatalogEntry;
-import org.griphyn.common.catalog.transformation.Mapper;
import org.griphyn.common.classes.TCType;
import org.griphyn.vdl.util.FQN;
@@ -32,10 +31,6 @@
private Scheduler scheduler;
private Set checkedTRs;
- public VDSTaskTransformer(Mapper tcmapper) {
- this.impl = new MapperTransformer(tcmapper);
- }
-
public VDSTaskTransformer(TCCache tc) {
this.impl = new TCTransformer(tc);
}
@@ -130,37 +125,6 @@
protected abstract void applyTCEntry(Task task, Contact[] contacts);
}
- public static class MapperTransformer extends AbstractTransformer {
- private Mapper tcmapper;
-
- public MapperTransformer(Mapper tcmapper) {
- this.tcmapper = tcmapper;
- }
-
- protected void applyTCEntry(Task task, Contact[] contacts) {
- JobSpecification spec = (JobSpecification) task.getSpecification();
- BoundContact bc = (BoundContact) contacts[0];
- List l = new LinkedList();
- l.add(bc.getHost());
- TransformationCatalogEntry tce;
- Map siteMap = tcmapper.getSiteMap(null, spec.getExecutable(), null, l);
- if (siteMap == null) {
- return;
- }
- Object e = siteMap.get(bc.getHost());
- if (e instanceof List) {
- tce = (TransformationCatalogEntry) ((List) e).get(0);
- }
- else if (e instanceof TransformationCatalogEntry) {
- tce = (TransformationCatalogEntry) e;
- }
- else {
- throw new KarajanRuntimeException("Invalid object in transformation catalog: " + e);
- }
- spec.setExecutable(tce.getPhysicalTransformation());
- }
- }
-
public static class TCTransformer extends AbstractTransformer {
private TCCache tc;
private Set warnset = new HashSet();
From noreply at svn.ci.uchicago.edu Mon Aug 18 12:34:27 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 18 Aug 2008 12:34:27 -0500 (CDT)
Subject: [Swift-commit] r2186 - SwiftApps/SIDGrid/config
Message-ID: <20080818173427.5ED1F1AB0193@www.ci.uchicago.edu>
Author: skenny
Date: 2008-08-18 12:34:26 -0500 (Mon, 18 Aug 2008)
New Revision: 2186
Added:
SwiftApps/SIDGrid/config/sites_bigred.xml
Log:
include project number so that loadleveler will accept the submission
Added: SwiftApps/SIDGrid/config/sites_bigred.xml
===================================================================
--- SwiftApps/SIDGrid/config/sites_bigred.xml (rev 0)
+++ SwiftApps/SIDGrid/config/sites_bigred.xml 2008-08-18 17:34:26 UTC (rev 2186)
@@ -0,0 +1,17 @@
+
+
+
+
+
+
+
+
+
+ 0
+ 4
+ TG-DBS080004N
+
+
+ /N/u/tg-scommun/BigRed/sidgrid/sidgrid_out/{username}
+
+
From noreply at svn.ci.uchicago.edu Fri Aug 22 15:25:40 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 22 Aug 2008 15:25:40 -0500 (CDT)
Subject: [Swift-commit] r2187 - SwiftApps/SIDGrid/config
Message-ID: <20080822202540.678D71360037@www.ci.uchicago.edu>
Author: skenny
Date: 2008-08-22 15:25:39 -0500 (Fri, 22 Aug 2008)
New Revision: 2187
Modified:
SwiftApps/SIDGrid/config/sites_bsd.xml
Log:
updated gatekeeper for bsd cluster
Modified: SwiftApps/SIDGrid/config/sites_bsd.xml
===================================================================
--- SwiftApps/SIDGrid/config/sites_bsd.xml 2008-08-18 17:34:26 UTC (rev 2186)
+++ SwiftApps/SIDGrid/config/sites_bsd.xml 2008-08-22 20:25:39 UTC (rev 2187)
@@ -12,8 +12,8 @@
2000
Arch == "X86_64" || Arch == "INTEL"
-
-
+
+
/disks/gpfs/fmri/cnari/swift/work/{username}
From noreply at svn.ci.uchicago.edu Fri Aug 22 17:18:57 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 22 Aug 2008 17:18:57 -0500 (CDT)
Subject: [Swift-commit] r2188 - SwiftApps/SIDGrid/config
Message-ID: <20080822221857.B97CB1360037@www.ci.uchicago.edu>
Author: skenny
Date: 2008-08-22 17:18:56 -0500 (Fri, 22 Aug 2008)
New Revision: 2188
Modified:
SwiftApps/SIDGrid/config/sites_tacc.xml
Log:
default to gt2
Modified: SwiftApps/SIDGrid/config/sites_tacc.xml
===================================================================
--- SwiftApps/SIDGrid/config/sites_tacc.xml 2008-08-22 20:25:39 UTC (rev 2187)
+++ SwiftApps/SIDGrid/config/sites_tacc.xml 2008-08-22 22:18:56 UTC (rev 2188)
@@ -7,8 +7,8 @@
-
-
+
+
/work/00926/tg459516/sidgrid_out/{username}
From noreply at svn.ci.uchicago.edu Mon Aug 25 03:26:14 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 25 Aug 2008 03:26:14 -0500 (CDT)
Subject: [Swift-commit] r2191 - www/downloads
Message-ID: <20080825082614.495171360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-25 03:26:13 -0500 (Mon, 25 Aug 2008)
New Revision: 2191
Added:
www/downloads/release-notes-0.6.txt
Modified:
www/downloads/index.php
Log:
0.6 release info
Modified: www/downloads/index.php
===================================================================
--- www/downloads/index.php 2008-08-22 22:32:57 UTC (rev 2190)
+++ www/downloads/index.php 2008-08-25 08:26:13 UTC (rev 2191)
@@ -27,12 +27,13 @@
DOWNLOADS
Latest Release
-Swift 0.5 - 2008/04/16
-Swift v0.5 is a development release intended to release functionality
-and fixes that have gone in to trunk since v0.4.
-[vdsk-0.5.tar.gz]
-[release-notes-0.5.txt]
+
Swift 0.6 - 2008/08/15
+Swift v0.6 is a development release intended to release functionality
+and fixes that have gone in to trunk since v0.5.
+[vdsk-0.6.tar.gz]
+[release-notes-0.6.txt]
+
Nightly Builds and Tests
@@ -93,6 +94,12 @@
Historical releases
+Swift 0.5 - 2008/04/16
+Swift v0.5 is a development release intended to release functionality
+and fixes that have gone in to trunk since v0.4.
+[vdsk-0.5.tar.gz]
+[release-notes-0.5.txt]
+
Swift 0.4 - 2008/03/18
Swift v0.4 is a development release intended to release functionality
and fixes that have gone in to trunk since v0.3. More details are contained
Added: www/downloads/release-notes-0.6.txt
===================================================================
--- www/downloads/release-notes-0.6.txt (rev 0)
+++ www/downloads/release-notes-0.6.txt 2008-08-25 08:26:13 UTC (rev 2191)
@@ -0,0 +1,40 @@
+These are release notes for the Swift 0.6 release. Items
+listed below are changes since Swift 0.5 - both enhancements
+and known issues.
+
+Swift 0.6 is built from Swift SVN r2177 and CoG SVN r2127.
+(0.5 was swift r1783 cog r1962)
+
+A more detailed change log can be found in CHANGES.txt in the distribution
+tarball.
+
+This release contains the following new experimental features:
+
+* job replication
+
+When a job has been queued on a site for too long, Swift can resubmit the
+job (possibly to a different site). Replication is controlled by
+replication.* parameters in the swift.properties configuration file. Those
+parameters are documented inline in that configuration file.
+
+* cog coasters
+
+Swift can use cog coasters which allow a pilot job to be submitted to
+a site which will pull in actual Swift jobs for execution. This has
+several advantages:
+ . when a large number of jobs need to be submitted, using coasters
+ reduces the job submission overhead in (eg) GRAM to one job per
+ worker node.
+ . once a node is allocated to a coaster, it can be used to run
+ many swift jobs, avoiding the need for each job to be queued
+ separately.
+
+* rate limiting for bad sites
+
+Previous versions of Swift would try to ensure that each site had at
+least two jobs queued or running at once, no matter how poorly performing.
+Swift v0.6 will substantially reduce the number of jobs sent to poorly
+performing sites; when a site score goes below 1 (job at once) then
+Swift will delay sending more jobs to that site, with the delay increasing
+exponentially as the score lowers.
+
From noreply at svn.ci.uchicago.edu Mon Aug 25 03:30:42 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 25 Aug 2008 03:30:42 -0500 (CDT)
Subject: [Swift-commit] r2192 - www/inc
Message-ID: <20080825083042.E5EF51360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-25 03:30:42 -0500 (Mon, 25 Aug 2008)
New Revision: 2192
Modified:
www/inc/home_sidebar.php
Log:
'What's new' points to 0.6 release now
Modified: www/inc/home_sidebar.php
===================================================================
--- www/inc/home_sidebar.php 2008-08-25 08:26:13 UTC (rev 2191)
+++ www/inc/home_sidebar.php 2008-08-25 08:30:42 UTC (rev 2192)
@@ -1,7 +1,7 @@
What's New?
-SWIFT 0.4 RELEASE - 18 MAR 2008
+SWIFT 0.6 RELEASE - 25 MAR 2008
-The latest release of Swift, v0.4, is available from the
+The latest release of Swift, v0.6, is available from the
downloads page.
From noreply at svn.ci.uchicago.edu Mon Aug 25 03:53:02 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 25 Aug 2008 03:53:02 -0500 (CDT)
Subject: [Swift-commit] r2193 - in www: downloads inc
Message-ID: <20080825085302.2047E1360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-25 03:53:01 -0500 (Mon, 25 Aug 2008)
New Revision: 2193
Modified:
www/downloads/index.php
www/downloads/release-notes-0.6.txt
www/inc/home_sidebar.php
Log:
fix 0.6 dates
Modified: www/downloads/index.php
===================================================================
--- www/downloads/index.php 2008-08-25 08:30:42 UTC (rev 2192)
+++ www/downloads/index.php 2008-08-25 08:53:01 UTC (rev 2193)
@@ -27,7 +27,7 @@
DOWNLOADS
Latest Release
-Swift 0.6 - 2008/08/15
+Swift 0.6 - 2008/08/25
Swift v0.6 is a development release intended to release functionality
and fixes that have gone in to trunk since v0.5.
[vdsk-0.6.tar.gz]
Modified: www/downloads/release-notes-0.6.txt
===================================================================
--- www/downloads/release-notes-0.6.txt 2008-08-25 08:30:42 UTC (rev 2192)
+++ www/downloads/release-notes-0.6.txt 2008-08-25 08:53:01 UTC (rev 2193)
@@ -8,16 +8,16 @@
A more detailed change log can be found in CHANGES.txt in the distribution
tarball.
-This release contains the following new experimental features:
+This release contains the following new features:
-* job replication
+* job replication (experimental)
When a job has been queued on a site for too long, Swift can resubmit the
job (possibly to a different site). Replication is controlled by
replication.* parameters in the swift.properties configuration file. Those
parameters are documented inline in that configuration file.
-* cog coasters
+* cog coasters (experimental)
Swift can use cog coasters which allow a pilot job to be submitted to
a site which will pull in actual Swift jobs for execution. This has
@@ -38,3 +38,9 @@
Swift will delay sending more jobs to that site, with the delay increasing
exponentially as the score lowers.
+* more compile time type checking
+
+More type checking is done at compile time. This will catch some errors at the
+start of execution which would previously have been detected later on in a run,
+and will give more useful error messages.
+
Modified: www/inc/home_sidebar.php
===================================================================
--- www/inc/home_sidebar.php 2008-08-25 08:30:42 UTC (rev 2192)
+++ www/inc/home_sidebar.php 2008-08-25 08:53:01 UTC (rev 2193)
@@ -1,5 +1,5 @@
What's New?
-SWIFT 0.6 RELEASE - 25 MAR 2008
+SWIFT 0.6 RELEASE - 25 AUG 2008
The latest release of Swift, v0.6, is available from the
downloads page.
From noreply at svn.ci.uchicago.edu Tue Aug 26 09:41:23 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 26 Aug 2008 09:41:23 -0500 (CDT)
Subject: [Swift-commit] r2194 - log-processing
Message-ID: <20080826144123.2C3B11360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-26 09:41:21 -0500 (Tue, 26 Aug 2008)
New Revision: 2194
Modified:
log-processing/log-to-karatasks-transitions
Log:
in the case of unknown statuses, put the status number into the log. this is useful when replication is happening because there is a replication status with value 100 that is not known by the task status prettyprinter
Modified: log-processing/log-to-karatasks-transitions
===================================================================
--- log-processing/log-to-karatasks-transitions 2008-08-25 08:53:01 UTC (rev 2193)
+++ log-processing/log-to-karatasks-transitions 2008-08-26 14:41:21 UTC (rev 2194)
@@ -13,7 +13,8 @@
# then filter out unchanged lines because they're irrelevant
grep ' TaskImpl ' | ./iso-to-secs | \
-sed 's/^\([^ ]*\) [^ ]* TaskImpl Task(type=\([^,]*\), identity=urn:\([^)]*\)) setting status to \([^ ]*\)\(.*\)$/\1 \3 \4 \2 \5/'
+sed 's/^\([^ ]*\) [^ ]* TaskImpl Task(type=\([^,]*\), identity=urn:\([^)]*\)) setting status to \([^ ]*\) *\(.*\)$/\1 \3 \4 \2 \5/' | \
+sed 's/\([^ ]*\) \([^ ]*\) Unknown \([^ ]*\) *(\(.*\))/\1 \2 Unknown-\4 \3/'
exit 0
From noreply at svn.ci.uchicago.edu Tue Aug 26 11:31:50 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 26 Aug 2008 11:31:50 -0500 (CDT)
Subject: [Swift-commit] r2195 - log-processing
Message-ID: <20080826163150.EE5081360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-26 11:31:50 -0500 (Tue, 26 Aug 2008)
New Revision: 2195
Modified:
log-processing/log-to-execute2-transitions
Log:
put replication group in execute2.transitions
Modified: log-processing/log-to-execute2-transitions
===================================================================
--- log-processing/log-to-execute2-transitions 2008-08-26 14:41:21 UTC (rev 2194)
+++ log-processing/log-to-execute2-transitions 2008-08-26 16:31:50 UTC (rev 2195)
@@ -14,7 +14,7 @@
grep ' vdl:execute2 ' | ./iso-to-secs | \
grep -E '^[^ ]+ +[^ ]+ +vdl:execute2 ' | \
-sed 's/^\(.*\) DEBUG vdl:execute2 THREAD_ASSOCIATION jobid=\([^ ]*\) thread=\([0-9\-]*\) host=\([^ ]*\).*$/\1 \2 ASSOCIATED \3 \4/' | \
+sed 's/^\(.*\) DEBUG vdl:execute2 THREAD_ASSOCIATION jobid=\([^ ]*\) thread=\([0-9\-]*\) host=\([^ ]*\) replicationGroup=\([^ ]*\).*$/\1 \2 ASSOCIATED \3 \4 \5/' | \
\
sed 's/^\([^ ]*\) DEBUG vdl:execute2 \([^ ]*\) jobid=\([^ ]*\).*/\1 \3 \2/'
From noreply at svn.ci.uchicago.edu Tue Aug 26 12:19:36 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 26 Aug 2008 12:19:36 -0500 (CDT)
Subject: [Swift-commit] r2197 - SwiftApps/SIDGrid/uchicago_tools
Message-ID: <20080826171936.CE6271360037@www.ci.uchicago.edu>
Author: skenny
Date: 2008-08-26 12:19:36 -0500 (Tue, 26 Aug 2008)
New Revision: 2197
Modified:
SwiftApps/SIDGrid/uchicago_tools/SetUpSwift.sh
Log:
no longer setting x509 cert dir
Modified: SwiftApps/SIDGrid/uchicago_tools/SetUpSwift.sh
===================================================================
--- SwiftApps/SIDGrid/uchicago_tools/SetUpSwift.sh 2008-08-26 17:16:53 UTC (rev 2196)
+++ SwiftApps/SIDGrid/uchicago_tools/SetUpSwift.sh 2008-08-26 17:19:36 UTC (rev 2197)
@@ -1,9 +1,9 @@
#!/usr/bin/bash
export CNARI=/disks/gpfs/fmri/cnari
-export X509_CERT_DIR=$CNARI/swift/ca
-export JAVA_HOME=/home/skenny/java_1_4_2_06-b03
-export GLOBUS_HOSTNAME=$HOSTNAME
+#export X509_CERT_DIR=$CNARI/swift/ca
+#export JAVA_HOME=/home/skenny/java_1_4_2_06-b03
+#export GLOBUS_HOSTNAME=$HOSTNAME
export PATH=$CNARI/swift/swift/bin:$PATH
From noreply at svn.ci.uchicago.edu Tue Aug 26 12:20:45 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 26 Aug 2008 12:20:45 -0500 (CDT)
Subject: [Swift-commit] r2198 - SwiftApps/SIDGrid/uchicago_tools
Message-ID: <20080826172045.E15651360037@www.ci.uchicago.edu>
Author: skenny
Date: 2008-08-26 12:20:45 -0500 (Tue, 26 Aug 2008)
New Revision: 2198
Modified:
SwiftApps/SIDGrid/uchicago_tools/SetUpSwift.csh
Log:
no longer setting ca dir or java home
Modified: SwiftApps/SIDGrid/uchicago_tools/SetUpSwift.csh
===================================================================
--- SwiftApps/SIDGrid/uchicago_tools/SetUpSwift.csh 2008-08-26 17:19:36 UTC (rev 2197)
+++ SwiftApps/SIDGrid/uchicago_tools/SetUpSwift.csh 2008-08-26 17:20:45 UTC (rev 2198)
@@ -1,9 +1,9 @@
#!/usr/bin/tcsh
setenv CNARI /disks/gpfs/fmri/cnari
-setenv X509_CERT_DIR ${CNARI}/swift/ca
-setenv JAVA_HOME /home/skenny/java_1_4_2_06-b03
-setenv GLOBUS_HOSTNAME ${HOSTNAME}
+#setenv X509_CERT_DIR ${CNARI}/swift/ca
+#setenv JAVA_HOME /home/skenny/java_1_4_2_06-b03
+#setenv GLOBUS_HOSTNAME ${HOSTNAME}
setenv PATH ${CNARI}/swift/swift/bin:${PATH}
From noreply at svn.ci.uchicago.edu Tue Aug 26 15:24:20 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 26 Aug 2008 15:24:20 -0500 (CDT)
Subject: [Swift-commit] r2199 - in trunk: libexec
src/org/griphyn/vdl/karajan/lib
Message-ID: <20080826202420.A11F41360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-26 15:24:19 -0500 (Tue, 26 Aug 2008)
New Revision: 2199
Modified:
trunk/libexec/vdl-int.k
trunk/src/org/griphyn/vdl/karajan/lib/Execute.java
Log:
jobid to karajan task ID binding
Modified: trunk/libexec/vdl-int.k
===================================================================
--- trunk/libexec/vdl-int.k 2008-08-26 17:20:45 UTC (rev 2198)
+++ trunk/libexec/vdl-int.k 2008-08-26 20:24:19 UTC (rev 2199)
@@ -410,6 +410,7 @@
vdl:tcprofile(tr, rhost) //this gets various app params from the tc, such as environment, walltime, etc
replicationGroup=replicationGroup
replicationChannel=replicationChannel
+ jobid=jobid
)
checkJobStatus(rhost, wfdir, jobid, tr, jobdir)
Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2008-08-26 17:20:45 UTC (rev 2198)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2008-08-26 20:24:19 UTC (rev 2199)
@@ -25,6 +25,7 @@
public static final Arg A_REPLICATION_GROUP = new Arg.Optional("replicationGroup");
public static final Arg A_REPLICATION_CHANNEL = new Arg.Optional("replicationChannel");
+ public static final Arg A_JOBID = new Arg.Optional("jobid");
static {
setArguments(Execute.class, new Arg[] { A_EXECUTABLE, A_ARGS, A_ARGUMENTS, A_HOST,
@@ -33,7 +34,7 @@
A_ENVIRONMENT, A_QUEUE, A_PROJECT, A_MINMEMORY, A_MAXMEMORY, A_REDIRECT,
A_SECURITY_CONTEXT, A_DIRECTORY, A_NATIVESPEC, A_DELEGATION, A_ATTRIBUTES,
C_ENVIRONMENT, A_FAIL_ON_JOB_ERROR, A_BATCH, A_REPLICATION_GROUP,
- A_REPLICATION_CHANNEL });
+ A_REPLICATION_CHANNEL, A_JOBID });
}
public Execute() {
@@ -47,6 +48,8 @@
logger.debug(task);
logger.debug("Submitting task " + task);
}
+ String jobid = (String)A_JOBID.getValue(stack,null);
+ logger.info("jobid="+jobid+" task=" + task);
scheduler.addJobStatusListener(this, task);
synchronized (tasks) {
tasks.put(task, stack);
From noreply at svn.ci.uchicago.edu Tue Aug 26 16:10:12 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 26 Aug 2008 16:10:12 -0500 (CDT)
Subject: [Swift-commit] r2200 - log-processing
Message-ID: <20080826211012.A27941360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-26 16:10:11 -0500 (Tue, 26 Aug 2008)
New Revision: 2200
Added:
log-processing/extract-jobid-karajanid-bindings
Modified:
log-processing/makefile
Log:
extract jobid / karajan task ID bindings introduced in swift r2199
Added: log-processing/extract-jobid-karajanid-bindings
===================================================================
--- log-processing/extract-jobid-karajanid-bindings (rev 0)
+++ log-processing/extract-jobid-karajanid-bindings 2008-08-26 21:10:11 UTC (rev 2200)
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+grep ' Execute ' | ./iso-to-secs | sed 's/^[^ ]* INFO Execute jobid=\([^ ]*\) task=Task(type=JOB_SUBMISSION, identity=urn:\([^ )]*\).*$/\1 \2/'
Property changes on: log-processing/extract-jobid-karajanid-bindings
___________________________________________________________________
Name: svn:executable
+ *
Modified: log-processing/makefile
===================================================================
--- log-processing/makefile 2008-08-26 20:24:19 UTC (rev 2199)
+++ log-processing/makefile 2008-08-26 21:10:11 UTC (rev 2200)
@@ -149,3 +149,6 @@
cluster-jobcount-histogram.png: clusterstats
./plot-cluster-jobcount-histogram
+jobid-karajanid-bindings: $(LOG)
+ ./extract-jobid-karajanid-bindings <$(LOG) >jobid-karajanid-bindings
+
From noreply at svn.ci.uchicago.edu Tue Aug 26 16:43:59 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 26 Aug 2008 16:43:59 -0500 (CDT)
Subject: [Swift-commit] r2201 - trunk/etc
Message-ID: <20080826214359.2130B1360037@www.ci.uchicago.edu>
Author: hategan
Date: 2008-08-26 16:43:58 -0500 (Tue, 26 Aug 2008)
New Revision: 2201
Modified:
trunk/etc/log4j.properties
Log:
removed annoying warning when using gt4
Modified: trunk/etc/log4j.properties
===================================================================
--- trunk/etc/log4j.properties 2008-08-26 21:10:11 UTC (rev 2200)
+++ trunk/etc/log4j.properties 2008-08-26 21:43:58 UTC (rev 2201)
@@ -13,6 +13,8 @@
log4j.logger.swift=DEBUG
+log4j.logger.org.apache.axis.utils=ERROR
+
log4j.logger.org.globus.swift.trace=INFO
log4j.logger.org.griphyn.vdl.karajan.Loader=DEBUG
From noreply at svn.ci.uchicago.edu Wed Aug 27 02:52:56 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 27 Aug 2008 02:52:56 -0500 (CDT)
Subject: [Swift-commit] r2202 - log-processing
Message-ID: <20080827075256.E31A31360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-27 02:52:55 -0500 (Wed, 27 Aug 2008)
New Revision: 2202
Added:
log-processing/get-jobid-for-karajanid
Log:
script to return an execute2 id given a karajan task id
Added: log-processing/get-jobid-for-karajanid
===================================================================
--- log-processing/get-jobid-for-karajanid (rev 0)
+++ log-processing/get-jobid-for-karajanid 2008-08-27 07:52:55 UTC (rev 2202)
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# given a karajan task ID as $1, return the execute2 jobid (if any)
+# associated with the task
+
+# needs karajan->jobid bindings file, jobid-karajanid-bindings, to
+# exist already
+
+grep -e "^[^ ]* $1.*\$" jobid-karajanid-bindings | cut -d ' ' -f 1
Property changes on: log-processing/get-jobid-for-karajanid
___________________________________________________________________
Name: svn:executable
+ *
From noreply at svn.ci.uchicago.edu Wed Aug 27 03:11:56 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 27 Aug 2008 03:11:56 -0500 (CDT)
Subject: [Swift-commit] r2203 - log-processing
Message-ID: <20080827081156.46FD71360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-27 03:11:55 -0500 (Wed, 27 Aug 2008)
New Revision: 2203
Added:
log-processing/annotate-karatasks-with-execute2-id
Modified:
log-processing/makefile
Log:
karajan transitions file annotated with execute2 IDs
Added: log-processing/annotate-karatasks-with-execute2-id
===================================================================
--- log-processing/annotate-karatasks-with-execute2-id (rev 0)
+++ log-processing/annotate-karatasks-with-execute2-id 2008-08-27 08:11:55 UTC (rev 2203)
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# feed in karatasks.JOB_SUBMISSION.transitions and column 5 will be
+# made to be the execute2 id associated with that karajan task,
+# if there is such an association (not all execution tasks will be
+# associated with an execute2)
+
+# 1219781848.384 0-1-1219781848210 Active JOB_SUBMISSION
+
+while read time karaid state type rest; do
+ exec2id=$(./get-jobid-for-karajanid $karaid)
+ echo $time $karaid $state $type $exec2id $rest
+done
Property changes on: log-processing/annotate-karatasks-with-execute2-id
___________________________________________________________________
Name: svn:executable
+ *
Modified: log-processing/makefile
===================================================================
--- log-processing/makefile 2008-08-27 07:52:55 UTC (rev 2202)
+++ log-processing/makefile 2008-08-27 08:11:55 UTC (rev 2203)
@@ -152,3 +152,5 @@
jobid-karajanid-bindings: $(LOG)
./extract-jobid-karajanid-bindings <$(LOG) >jobid-karajanid-bindings
+karatasks.JOB_SUBMISSION.annotated-execute2.transitions: karatasks.JOB_SUBMISSION.transitions
+ ./annotate-karatasks-with-execute2-id < karatasks.JOB_SUBMISSION.transitions > karatasks.JOB_SUBMISSION.annotated-execute2.transitions
From noreply at svn.ci.uchicago.edu Wed Aug 27 04:05:38 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 27 Aug 2008 04:05:38 -0500 (CDT)
Subject: [Swift-commit] r2204 - log-processing
Message-ID: <20080827090539.0C62B1360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-27 04:05:38 -0500 (Wed, 27 Aug 2008)
New Revision: 2204
Added:
log-processing/get-replicationid-for-execute2id
Modified:
log-processing/annotate-karatasks-with-execute2-id
log-processing/makefile
Log:
annotate karatasks with replication id
Modified: log-processing/annotate-karatasks-with-execute2-id
===================================================================
--- log-processing/annotate-karatasks-with-execute2-id 2008-08-27 08:11:55 UTC (rev 2203)
+++ log-processing/annotate-karatasks-with-execute2-id 2008-08-27 09:05:38 UTC (rev 2204)
@@ -1,13 +1,15 @@
#!/bin/bash
# feed in karatasks.JOB_SUBMISSION.transitions and column 5 will be
-# made to be the execute2 id associated with that karajan task,
-# if there is such an association (not all execution tasks will be
+# made to be the execute2 id associated with that karajan task, column 6
+# with the replication group id.
+# (if there is such an association - not all execution tasks will be
# associated with an execute2)
# 1219781848.384 0-1-1219781848210 Active JOB_SUBMISSION
while read time karaid state type rest; do
exec2id=$(./get-jobid-for-karajanid $karaid)
- echo $time $karaid $state $type $exec2id $rest
+ replid=$(./get-replicationid-for-execute2id $exec2id)
+ echo $time $karaid $state $type $exec2id $replid $rest
done
Added: log-processing/get-replicationid-for-execute2id
===================================================================
--- log-processing/get-replicationid-for-execute2id (rev 0)
+++ log-processing/get-replicationid-for-execute2id 2008-08-27 09:05:38 UTC (rev 2204)
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# given an execute2 ID as $1, return the replication id (if any)
+
+# needs execute2.event to exist
+
+#1219825024.716 91.3469998836517 sleep-nkcz2kyi JOB_END 0-6-1 wonkyA ckcz2kyi
+
+grep -e "^[^ ]* [^ ]* $1 .*\$" execute2.event | cut -d ' ' -f 7
Property changes on: log-processing/get-replicationid-for-execute2id
___________________________________________________________________
Name: svn:executable
+ *
Modified: log-processing/makefile
===================================================================
--- log-processing/makefile 2008-08-27 08:11:55 UTC (rev 2203)
+++ log-processing/makefile 2008-08-27 09:05:38 UTC (rev 2204)
@@ -152,5 +152,5 @@
jobid-karajanid-bindings: $(LOG)
./extract-jobid-karajanid-bindings <$(LOG) >jobid-karajanid-bindings
-karatasks.JOB_SUBMISSION.annotated-execute2.transitions: karatasks.JOB_SUBMISSION.transitions
+karatasks.JOB_SUBMISSION.annotated-execute2.transitions: karatasks.JOB_SUBMISSION.transitions execute2.event
./annotate-karatasks-with-execute2-id < karatasks.JOB_SUBMISSION.transitions > karatasks.JOB_SUBMISSION.annotated-execute2.transitions
From noreply at svn.ci.uchicago.edu Wed Aug 27 04:48:35 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 27 Aug 2008 04:48:35 -0500 (CDT)
Subject: [Swift-commit] r2205 - trunk/resources
Message-ID: <20080827094835.595CA1360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-27 04:48:34 -0500 (Wed, 27 Aug 2008)
New Revision: 2205
Modified:
trunk/resources/swiftscript.g
Log:
disallow empty statements
Modified: trunk/resources/swiftscript.g
===================================================================
--- trunk/resources/swiftscript.g 2008-08-27 09:05:38 UTC (rev 2204)
+++ trunk/resources/swiftscript.g 2008-08-27 09:48:34 UTC (rev 2205)
@@ -458,7 +458,6 @@
| code=foreachStat
| code=switchStat
| code=iterateStat
- | SEMI {code=template("blank");}
;
ifStat returns [StringTemplate code=template("if")]
From noreply at svn.ci.uchicago.edu Wed Aug 27 05:04:22 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 27 Aug 2008 05:04:22 -0500 (CDT)
Subject: [Swift-commit] r2206 - trunk/tests/language/working
Message-ID: <20080827100422.4D6DA1360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-27 05:04:21 -0500 (Wed, 27 Aug 2008)
New Revision: 2206
Removed:
trunk/tests/language/working/041-only-semi-r356.swift
Log:
fix test broken by r2205
Deleted: trunk/tests/language/working/041-only-semi-r356.swift
===================================================================
--- trunk/tests/language/working/041-only-semi-r356.swift 2008-08-27 09:48:34 UTC (rev 2205)
+++ trunk/tests/language/working/041-only-semi-r356.swift 2008-08-27 10:04:21 UTC (rev 2206)
@@ -1 +0,0 @@
-;
From noreply at svn.ci.uchicago.edu Wed Aug 27 06:24:28 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 27 Aug 2008 06:24:28 -0500 (CDT)
Subject: [Swift-commit] r2207 - trunk
Message-ID: <20080827112428.3A2E61360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-27 06:24:27 -0500 (Wed, 27 Aug 2008)
New Revision: 2207
Modified:
trunk/build.xml
Log:
option to make distribution without command grid commands such as grid-proxy-init, to facilitate installation inside a VDT install
Modified: trunk/build.xml
===================================================================
--- trunk/build.xml 2008-08-27 10:04:21 UTC (rev 2206)
+++ trunk/build.xml 2008-08-27 11:24:27 UTC (rev 2207)
@@ -92,6 +92,7 @@
+
@@ -276,6 +277,13 @@
+
+
+
+
+
+
From noreply at svn.ci.uchicago.edu Wed Aug 27 06:25:37 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 27 Aug 2008 06:25:37 -0500 (CDT)
Subject: [Swift-commit] r2208 - trunk/docs
Message-ID: <20080827112537.2D5081360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-27 06:25:36 -0500 (Wed, 27 Aug 2008)
New Revision: 2208
Modified:
trunk/docs/userguide.xml
Log:
details of build options
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2008-08-27 11:24:27 UTC (rev 2207)
+++ trunk/docs/userguide.xml 2008-08-27 11:25:36 UTC (rev 2208)
@@ -2666,6 +2666,59 @@
site-shared filesystem directly.
+ Build options
+
+See the
+Swift download page for instructions on downloading and
+building Swift from source. When building, various build options can
+be supplied on the ant commandline. These are summarised here:
+
+
+with-provider-condor - build with CoG condor provider
+
+
+with-provider-coaster - build with CoG coaster provider
+
+
+with-provider-deef - build with Falkon provider deef. In order for this
+option to work, it is necessary to check out the provider-deef code in
+the cog/modules directory alongside vdsk:
+
+$ cd cog/modules
+$ svn co https://svn.ci.uchicago.edu/svn/vdl2/provider-deef
+$ cd ../vdsk
+$ ant -Dwith-provider-deef=true redist
+
+
+
+
+with-provider-wonky - build with provider-wonky, an execution provider
+that provides delays and unreliability for the purposes of testing Swift's
+fault tolerance mechanisms. In order for this option to work, it is
+necessary to check out the provider-wonky code in the cog/modules
+directory alongside vdsk:
+
+
+$ cd cog/modules
+$ svn co https://svn.ci.uchicago.edu/svn/vdl2/provider-wonky
+$ cd ../vdsk
+$ ant -Dwith-provider-wonky=true redist
+
+
+
+no-supporting - produces a distribution without supporting commands such
+as grid-proxy-init. This is intended for when the Swift distribution will be
+used in an environment where those commands are already provided by other
+packages, where the Swift package should be providing only Swift
+commands, and where the presence of commands such as grid-proxy-init from
+the Swift distribution in the path will mask the presence of those
+commands from their true distribution package such as a Globus Toolkit
+package.
+
+$ ant -Dno-supporting=true redist
+
+
+
From noreply at svn.ci.uchicago.edu Fri Aug 29 02:54:02 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 29 Aug 2008 02:54:02 -0500 (CDT)
Subject: [Swift-commit] r2209 - www/downloads
Message-ID: <20080829075402.A6EE71360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-29 02:54:01 -0500 (Fri, 29 Aug 2008)
New Revision: 2209
Modified:
www/downloads/index.php
Log:
notes on downloading using pacman
Modified: www/downloads/index.php
===================================================================
--- www/downloads/index.php 2008-08-27 11:25:36 UTC (rev 2208)
+++ www/downloads/index.php 2008-08-29 07:54:01 UTC (rev 2209)
@@ -33,6 +33,26 @@
[vdsk-0.6.tar.gz]
[release-notes-0.6.txt]
+
+As an alternative to the above traditional Swift packaging, Swift can be
+downloaded and installed using
+pacman, a package
+manager commonly used on the Open Science Grid.
+
+
+There are two installation targets:
+The first will install Swift from the above tarball:
+
+pacman -get http://www.ci.uchicago.edu/~benc/pacman:swift-0.6
+
+The second will install Swift from the above tarball, as well as a number
+of supporting packages from the VDT software release to support use of the
+DOE CA and use of VOMS. (These packages are also available in a regular OSG or
+VDT installation):
+
+pacman -get http://www.ci.uchicago.edu/~benc/pacman:swift-tools
+
+
Nightly Builds and Tests
From noreply at svn.ci.uchicago.edu Fri Aug 29 03:02:02 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 29 Aug 2008 03:02:02 -0500 (CDT)
Subject: [Swift-commit] r2210 - www/downloads
Message-ID: <20080829080202.98B081360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-29 03:02:01 -0500 (Fri, 29 Aug 2008)
New Revision: 2210
Modified:
www/downloads/index.php
Log:
fix weird styling
Modified: www/downloads/index.php
===================================================================
--- www/downloads/index.php 2008-08-29 07:54:01 UTC (rev 2209)
+++ www/downloads/index.php 2008-08-29 08:02:01 UTC (rev 2210)
@@ -44,7 +44,8 @@
The first will install Swift from the above tarball:
pacman -get http://www.ci.uchicago.edu/~benc/pacman:swift-0.6
-
+
+
The second will install Swift from the above tarball, as well as a number
of supporting packages from the VDT software release to support use of the
DOE CA and use of VOMS. (These packages are also available in a regular OSG or
From noreply at svn.ci.uchicago.edu Fri Aug 29 07:53:46 2008
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 29 Aug 2008 07:53:46 -0500 (CDT)
Subject: [Swift-commit] r2211 - trunk/tests/sites
Message-ID: <20080829125346.C8AC21360037@www.ci.uchicago.edu>
Author: benc
Date: 2008-08-29 07:53:45 -0500 (Fri, 29 Aug 2008)
New Revision: 2211
Modified:
trunk/tests/sites/tc.data
Log:
maybe this will make coasters work on nmi build test
Modified: trunk/tests/sites/tc.data
===================================================================
--- trunk/tests/sites/tc.data 2008-08-29 08:02:01 UTC (rev 2210)
+++ trunk/tests/sites/tc.data 2008-08-29 12:53:45 UTC (rev 2211)
@@ -65,3 +65,9 @@
tgpurdue sort /bin/sort INSTALLED INTEL32::LINUX null
tgpurdue paste /bin/paste INSTALLED INTEL32::LINUX null
tgpurdue touch /bin/touch INSTALLED INTEL32::LINUX null
+localhost echo /bin/echo INSTALLED INTEL32::LINUX null
+localhost cat /bin/cat INSTALLED INTEL32::LINUX null
+localhost ls /bin/ls INSTALLED INTEL32::LINUX null
+localhost grep /bin/grep INSTALLED INTEL32::LINUX null
+localhost sort /bin/sort INSTALLED INTEL32::LINUX null
+localhost paste /bin/paste INSTALLED INTEL32::LINUX null