From jonmon at ci.uchicago.edu Wed Jun 1 09:20:11 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 1 Jun 2011 09:20:11 -0500 (CDT)
Subject: [Swift-commit] r4536 - in SwiftApps/Montage: exec templates
Message-ID: <20110601142011.157C19CC9C@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-01 09:20:10 -0500 (Wed, 01 Jun 2011)
New Revision: 4536
Added:
SwiftApps/Montage/templates/local.xml
SwiftApps/Montage/templates/pads-diff-coasters.xml
SwiftApps/Montage/templates/pads-fit-coasters.xml
SwiftApps/Montage/templates/pads-proj-coasters.xml
Removed:
SwiftApps/Montage/templates/pads-coasters.xml
Modified:
SwiftApps/Montage/exec/montage.sh
Log:
o SwiftApps/Montage/exec/montage.sh
-- Added functionality to specify a different number of workers for each stage
o SwiftApps/Montage/templates/pads-proj-coasters.xml
o SwiftApps/Montage/templates/pads-diff-coasters.xml
o SwiftApps/Montage/templates/pads-fit-coasters.xml
-- Added template files for each stage in the workflow
o SwiftApps/Montage/templates/local.xml
-- Added template for the local operations
o SwiftApps/Montage/templates/pads-coasters.xml
-- Removed template. Now split up into seperate stages
Modified: SwiftApps/Montage/exec/montage.sh
===================================================================
--- SwiftApps/Montage/exec/montage.sh 2011-05-31 20:14:17 UTC (rev 4535)
+++ SwiftApps/Montage/exec/montage.sh 2011-06-01 14:20:10 UTC (rev 4536)
@@ -15,16 +15,29 @@
mkdir ${WORK_DIRECTORY}
if [ $HOSTNAME == "login2.pads.ci.uchicago.edu" ]; then
- echo "#site pads-coasters.xml internalhostname=192.5.86.6" >> sites.properties
+ echo "#site pads-proj-coasters.xml internalhostname=192.5.86.6" >> sites.properties
+ echo "#site pads-diff-coasters.xml internalhostname=192.5.86.6" >> sites.properties
+ echo "#site pads-fit-coasters.xml internalhostname=192.5.86.6" >> sites.properties
elif [ $HOSTNAME == "login1.pads.ci.uchicago.edu" ]; then
- echo "#site pads-coasters.xml internalhostname=192.5.86.5" >> sites.properties
+ echo "#site pads-proj-coasters.xml internalhostname=192.5.86.5" >> sites.properties
+ echo "#site pads-diff-coasters.xml internalhostname=192.5.86.5" >> sites.properties
+ echo "#site pads-fit-coasters.xml internalhostname=192.5.86.5" >> sites.properties
fi
-echo "#site pads-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
-echo "#site pads-coasters.xml slots=$1" >> sites.properties
+echo "#site pads-proj-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
+echo "#site pads-proj-coasters.xml slots=$1" >> sites.properties
-gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-coasters.xml > sites.xml
+echo "#site pads-diff-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
+echo "#site pads-diff-coasters.xml slots=$1" >> sites.properties
+echo "#site pads-fit-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
+echo "#site pads-fit-coasters.xml slots=$1" >> sites.properties
+
+gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-proj-coasters.xml > sites.xml
+gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-diff-coasters.xml >> sites.xml
+gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-fit-coasters.xml >> sites.xml
+gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties local.xml >> sites.xml
+
rm -f sites.properties
mv sites.xml ${WORK_DIRECTORY}
Added: SwiftApps/Montage/templates/local.xml
===================================================================
--- SwiftApps/Montage/templates/local.xml (rev 0)
+++ SwiftApps/Montage/templates/local.xml 2011-06-01 14:20:10 UTC (rev 4536)
@@ -0,0 +1,7 @@
+
+
+
+ .05
+ 1
+ _WORK_
+
Deleted: SwiftApps/Montage/templates/pads-coasters.xml
===================================================================
--- SwiftApps/Montage/templates/pads-coasters.xml 2011-05-31 20:14:17 UTC (rev 4535)
+++ SwiftApps/Montage/templates/pads-coasters.xml 2011-06-01 14:20:10 UTC (rev 4536)
@@ -1,15 +0,0 @@
-
-
-
- 3600
- _INTERNALHOSTNAME_
- 1
- _SLOTS_
- 1
- 1
- fast
- 1
- 10000
- 1
- _WORK_
-
Added: SwiftApps/Montage/templates/pads-diff-coasters.xml
===================================================================
--- SwiftApps/Montage/templates/pads-diff-coasters.xml (rev 0)
+++ SwiftApps/Montage/templates/pads-diff-coasters.xml 2011-06-01 14:20:10 UTC (rev 4536)
@@ -0,0 +1,15 @@
+
+
+
+ 3600
+ _INTERNALHOSTNAME_
+ 1
+ _SLOTS_
+ 1
+ 1
+ fast
+ 2
+ 100
+ 1
+ _WORK_
+
Added: SwiftApps/Montage/templates/pads-fit-coasters.xml
===================================================================
--- SwiftApps/Montage/templates/pads-fit-coasters.xml (rev 0)
+++ SwiftApps/Montage/templates/pads-fit-coasters.xml 2011-06-01 14:20:10 UTC (rev 4536)
@@ -0,0 +1,15 @@
+
+
+
+ 3600
+ _INTERNALHOSTNAME_
+ 1
+ _SLOTS_
+ 1
+ 1
+ fast
+ 2
+ 100
+ 1
+ _WORK_
+
Added: SwiftApps/Montage/templates/pads-proj-coasters.xml
===================================================================
--- SwiftApps/Montage/templates/pads-proj-coasters.xml (rev 0)
+++ SwiftApps/Montage/templates/pads-proj-coasters.xml 2011-06-01 14:20:10 UTC (rev 4536)
@@ -0,0 +1,15 @@
+
+
+
+ 3600
+ _INTERNALHOSTNAME_
+ 1
+ _SLOTS_
+ 1
+ 1
+ fast
+ 2
+ 100
+ 1
+ _WORK_
+
From jonmon at ci.uchicago.edu Wed Jun 1 09:52:55 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 1 Jun 2011 09:52:55 -0500 (CDT)
Subject: [Swift-commit] r4537 - SwiftApps/Montage/exec
Message-ID: <20110601145255.531BE9CC9C@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-01 09:52:55 -0500 (Wed, 01 Jun 2011)
New Revision: 4537
Modified:
SwiftApps/Montage/exec/montage.sh
Log:
o SwiftApps/Montage/exec/montage.sh
-- Modified run script to accept a different number of workers at each stage
Modified: SwiftApps/Montage/exec/montage.sh
===================================================================
--- SwiftApps/Montage/exec/montage.sh 2011-06-01 14:20:10 UTC (rev 4536)
+++ SwiftApps/Montage/exec/montage.sh 2011-06-01 14:52:55 UTC (rev 4537)
@@ -1,6 +1,6 @@
#!/bin/bash
-EXPECTED_ARGS=2
+EXPECTED_ARGS=4
if [ $# -ne $EXPECTED_ARGS ]
then
@@ -28,21 +28,25 @@
echo "#site pads-proj-coasters.xml slots=$1" >> sites.properties
echo "#site pads-diff-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
-echo "#site pads-diff-coasters.xml slots=$1" >> sites.properties
+echo "#site pads-diff-coasters.xml slots=$2" >> sites.properties
echo "#site pads-fit-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
-echo "#site pads-fit-coasters.xml slots=$1" >> sites.properties
+echo "#site pads-fit-coasters.xml slots=$3" >> sites.properties
-gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-proj-coasters.xml > sites.xml
+echo "#site local.xml work=/gpfs/pads/jonmon/Swift/work/local" >> sites.properties
+
+echo "" > sites.xml
+gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-proj-coasters.xml >> sites.xml
gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-diff-coasters.xml >> sites.xml
gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-fit-coasters.xml >> sites.xml
gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties local.xml >> sites.xml
+echo "" >> sites.xml
rm -f sites.properties
mv sites.xml ${WORK_DIRECTORY}
-mv tc.data ${WORK_DIRECTORY}
-cp $2 ${WORK_DIRECTORY}
+cp tc.data ${WORK_DIRECTORY}
+cp $4 ${WORK_DIRECTORY}
cd ${WORK_DIRECTORY}
#swift -tc.file tc.data -sites.file sites.xml -config ${SWIFT_MONTAGE}/etc swift_montage.properties
From jonmon at ci.uchicago.edu Wed Jun 1 11:19:18 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 1 Jun 2011 11:19:18 -0500 (CDT)
Subject: [Swift-commit] r4538 - SwiftApps/Montage/exec
Message-ID: <20110601161918.C2AF99CC9C@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-01 11:19:18 -0500 (Wed, 01 Jun 2011)
New Revision: 4538
Modified:
SwiftApps/Montage/exec/montage.sh
Log:
o SwiftApps/Montage/exec/montage.sh
-- Changed error message is invalid number of parameters was given
-- Fixed how the run scipt generated the run directories
Modified: SwiftApps/Montage/exec/montage.sh
===================================================================
--- SwiftApps/Montage/exec/montage.sh 2011-06-01 14:52:55 UTC (rev 4537)
+++ SwiftApps/Montage/exec/montage.sh 2011-06-01 16:19:18 UTC (rev 4538)
@@ -4,13 +4,13 @@
if [ $# -ne $EXPECTED_ARGS ]
then
- echo "Did not specify how many workers are to be run"
+ echo "Did not specify valid parameters"
exit 1
fi
-RUN_DIRECTORY=(`ls | grep run`)
-len=${#RUN_DIRECTORY[*]}
-WORK_DIRECTORY=(`printf "run.%04d" $len`)
+RUN_DIRECTORY=(`ls | grep run | tail -n 1`)
+NUM=`echo $RUN_DIRECTORY | awk -F . '{print $NF}'`
+WORK_DIRECTORY=(`printf "run.%04d" $((NUM+1))`)
mkdir ${WORK_DIRECTORY}
From jonmon at ci.uchicago.edu Wed Jun 1 11:41:33 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 1 Jun 2011 11:41:33 -0500 (CDT)
Subject: [Swift-commit] r4539 - SwiftApps/Montage/exec
Message-ID: <20110601164133.B38029CC9C@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-01 11:41:33 -0500 (Wed, 01 Jun 2011)
New Revision: 4539
Modified:
SwiftApps/Montage/exec/montage.sh
Log:
o SwiftApps/Montage/exec/montage.sh
-- The run script now checks to make sure that the SWIFT_MONTAGE environment variable is set
Modified: SwiftApps/Montage/exec/montage.sh
===================================================================
--- SwiftApps/Montage/exec/montage.sh 2011-06-01 16:19:18 UTC (rev 4538)
+++ SwiftApps/Montage/exec/montage.sh 2011-06-01 16:41:33 UTC (rev 4539)
@@ -8,6 +8,12 @@
exit 1
fi
+# Fix this. Not sure how to set this variable if it not set or even what the error message should be
+if [ "X$SWIFT_MONTAGE" == "X" ]; then
+ echo "Please set the SWIFT_MONTAGE environment variable to where the Swift_Montage wrappers home directory is."
+ exit 1
+fi
+
RUN_DIRECTORY=(`ls | grep run | tail -n 1`)
NUM=`echo $RUN_DIRECTORY | awk -F . '{print $NF}'`
WORK_DIRECTORY=(`printf "run.%04d" $((NUM+1))`)
@@ -36,10 +42,20 @@
echo "#site local.xml work=/gpfs/pads/jonmon/Swift/work/local" >> sites.properties
echo "" > sites.xml
+echo "" >> sites.xml
+
gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-proj-coasters.xml >> sites.xml
+echo "" >> sites.xml
+
gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-diff-coasters.xml >> sites.xml
+echo "" >> sites.xml
+
gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties pads-fit-coasters.xml >> sites.xml
+echo "" >> sites.xml
+
gensites -L ${SWIFT_MONTAGE}/templates -p sites.properties local.xml >> sites.xml
+echo "" >> sites.xml
+
echo "" >> sites.xml
rm -f sites.properties
From jonmon at ci.uchicago.edu Wed Jun 1 14:36:46 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 1 Jun 2011 14:36:46 -0500 (CDT)
Subject: [Swift-commit] r4540 - SwiftApps/Montage/scripts
Message-ID: <20110601193646.610249CCC8@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-01 14:36:46 -0500 (Wed, 01 Jun 2011)
New Revision: 4540
Modified:
SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
Log:
o SwiftAps/Montage/scripts/Swift_Montage_Apps.swift
-- Added app procedure for mShrink to shrink large images.
Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-01 16:41:33 UTC (rev 4539)
+++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-01 19:36:46 UTC (rev 4540)
@@ -71,8 +71,11 @@
mProject "-X" @raw_img @proj_img @hdr;
}
+app ( Image shrunk ) mShrink( Image original, int factor )
+{
+ mShrink @orginial @shrunk factor;
+}
-
/*----------------------------------------------------------------------------------*/
/* */
/* Util Scripts */
From tga at ci.uchicago.edu Wed Jun 1 14:56:02 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Wed, 1 Jun 2011 14:56:02 -0500 (CDT)
Subject: [Swift-commit] r4541 - SwiftApps/SwiftR/Swift/exec
Message-ID: <20110601195602.82A2F9CCC8@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-01 14:56:02 -0500 (Wed, 01 Jun 2011)
New Revision: 4541
Modified:
SwiftApps/SwiftR/Swift/exec/rserver.swift
SwiftApps/SwiftR/Swift/exec/start-swift
Log:
Added option to R server to choose whether "warmup" job is executed or not.
Modified: SwiftApps/SwiftR/Swift/exec/rserver.swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-06-01 19:36:46 UTC (rev 4540)
+++ SwiftApps/SwiftR/Swift/exec/rserver.swift 2011-06-01 19:56:02 UTC (rev 4541)
@@ -5,7 +5,6 @@
# needs to select between EvalRPersistent and simple RunR
# enable persistent for local mode (not sure how)
# condense stdout/err to 1 file
-# replace ack with ftracef to result pipe
# condense shellscript and rserverscript to one?
app (external e, RData result, file stout, file sterr) runR (file shellscript, file RServerScript, RData rcall, RData exports[])
@@ -46,7 +45,9 @@
}
}
-passivate();
+if (@arg("warmup", "false") == "true") {
+ passivate();
+}
string pipedir = @arg("pipedir");
global string requestPipeName = @strcat(pipedir,"/requestpipe");
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-01 19:36:46 UTC (rev 4540)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-01 19:56:02 UTC (rev 4541)
@@ -754,7 +754,10 @@
fi
-$SWIFTRBIN/../swift/bin/swift $swiftLoggingFlag -config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) >& $out & $out
Author: tga
Date: 2011-06-01 15:01:00 -0500 (Wed, 01 Jun 2011)
New Revision: 4542
Modified:
SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
SwiftApps/SwiftR/Swift/exec/configure-server-crayxt
SwiftApps/SwiftR/Swift/exec/configure-server-local
SwiftApps/SwiftR/Swift/exec/configure-server-pbs
SwiftApps/SwiftR/Swift/exec/configure-server-sge
SwiftApps/SwiftR/Swift/exec/configure-server-ssh
Log:
Removed obselete cruft from configuration files
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-06-01 19:56:02 UTC (rev 4541)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-06-01 20:01:00 UTC (rev 4542)
@@ -6,22 +6,12 @@
throttle=$2
cat >tc <sites.xml <
-
-
-
- 0.15
- 10000
-
- $(pwd)/swiftwork
-
-
passive
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-crayxt
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-crayxt 2011-06-01 19:56:02 UTC (rev 4541)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-crayxt 2011-06-01 20:01:00 UTC (rev 4542)
@@ -6,7 +6,6 @@
throttlePBS=$2
cat >tc <sites.xml <
-
-
-
- 0.15
- 10000
-
- $(pwd)/swiftwork
-
-
passive
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-local
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-local 2011-06-01 19:56:02 UTC (rev 4541)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-local 2011-06-01 20:01:00 UTC (rev 4542)
@@ -55,8 +55,6 @@
done
-# 1
-
echo '' >>sites.xml
cat >cf <tc <sites.xml <
-
-
-
- 0.15
- 10000
-
- $R_LIBS_USER
- $(pwd)/swiftwork
-
-
passive
@@ -52,34 +41,3 @@
#throttle.host.submit=1
END
-
-: SAVE FOR REFERENCE <00:00:01
-# fast
-
-
-
- fast
- 00:59:00
- $throttlePBS
- 10000
-
- $HOME/swiftwork
-
-
-
-
- short
- 12000
- 32
- 1
- 1
- 1
- 2.55
- 10000
-
- $HOME/swiftwork
-
-
-END
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-sge
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-sge 2011-06-01 19:56:02 UTC (rev 4541)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-sge 2011-06-01 20:01:00 UTC (rev 4542)
@@ -6,7 +6,6 @@
throttle=$2
cat >tc <sites.xml <
-
-
- 0.15
- 10000
-
- $R_LIBS_USER
- $(pwd)/swiftwork
- file
-
-
passive
@@ -54,32 +43,3 @@
#throttle.host.submit=1
END
-: SAVE FOR REFERENCE <
-
- 300000
- 4
- 1
- 1
- 0.15
- 10000
-
- $(pwd)/swiftwork
- proxy
-
-
-
-
-
-
- 8
- 1
- 1
- .15
- 10000
- proxy
- /scratch/local/wilde/pstest/swiftwork
-
-
-END
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-ssh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-ssh 2011-06-01 19:56:02 UTC (rev 4541)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-ssh 2011-06-01 20:01:00 UTC (rev 4542)
@@ -6,21 +6,11 @@
cat >tc <sites.xml <
-
-
- 0.15
- 10000
- $R_LIBS_USER
-
- $(pwd)/swiftwork
-
-
passive
@@ -46,19 +36,3 @@
#throttle.host.submit=1
END
-true to save <00:00:01
-# fast
-
-
-
-
- passive
- 4
- .03
- 10000
-
- $HOME/swiftwork
-
-END
From tga at ci.uchicago.edu Wed Jun 1 17:15:27 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Wed, 1 Jun 2011 17:15:27 -0500 (CDT)
Subject: [Swift-commit] r4543 - SwiftApps/SwiftR/Swift/exec
Message-ID: <20110601221527.61FE39CC9C@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-01 17:15:27 -0500 (Wed, 01 Jun 2011)
New Revision: 4543
Modified:
SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
SwiftApps/SwiftR/Swift/exec/configure-server-crayxt
SwiftApps/SwiftR/Swift/exec/configure-server-local
SwiftApps/SwiftR/Swift/exec/configure-server-pbs
SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto
SwiftApps/SwiftR/Swift/exec/configure-server-sge
SwiftApps/SwiftR/Swift/exec/configure-server-ssh
SwiftApps/SwiftR/Swift/exec/start-swift
Log:
cleaning up parameter passing to configure scripts.
Getting pbsauto working, so that automatic pbs job submission is available from R without custom config files. These changes ahve not been thoroughly tested yet.
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-06-01 20:01:00 UTC (rev 4542)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-06-01 22:15:27 UTC (rev 4543)
@@ -2,9 +2,6 @@
# configuration for cobalt with manually-started Swift workers (passive coasters)
-cores=$1
-throttle=$2
-
cat >tc <tc <1
-->
- $throttlePBS
+ $throttle10000$LUSTRE_TMP
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-local
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-local 2011-06-01 20:01:00 UTC (rev 4542)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-local 2011-06-01 22:15:27 UTC (rev 4543)
@@ -2,8 +2,6 @@
throttleOneCore="0.00" # FIXME: test if new swft fix makes zero OK rather than -0.001
-cores=$1
-
if [ -r /proc/cpuinfo ]; then
localcores=$(grep '^processor' /proc/cpuinfo | wc -l)
else
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbs
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2011-06-01 20:01:00 UTC (rev 4542)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-pbs 2011-06-01 22:15:27 UTC (rev 4543)
@@ -2,10 +2,6 @@
# configuration for PBS with manually-started Swift workers (passive coasters)
-cores=$1
-
-throttlePBS=$2
-
cat >tc <passive$cores
- $throttlePBS
+ $throttle10000$R_LIBS_USER
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto 2011-06-01 20:01:00 UTC (rev 4542)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto 2011-06-01 22:15:27 UTC (rev 4543)
@@ -1,63 +1,74 @@
#! /usr/bin/env bash
-throttlePBS=.31 # FIXME: parameterize thsi and several other variables, below.
-
+#TODO: make configurable
+maxwalltime="00:10:00"
cat >tc <sites.xml <
-
-
- 0.15
- 10000
-
- $R_LIBS_USER
- $(pwd)/swiftwork
-
-
-
- fast
- 00:59:00
- $throttlePBS
- 10000
-
- $R_LIBS_USER
- $HOME/swiftwork
-
- short
- 12000
- 32
- 1
- 1
- 1
- 2.55
+
+ $R_LIBS_USER
+ $(pwd)/swiftwork
+ $(pwd)/swiftscratch
+ $throttle10000
-
- $HOME/swiftwork
+
+ $nodes
+ $cores
+
+
+ 1
+ 1
+
+
+END
+if [ "$queue" != NONE ]; then
+ #TODO: error handling
+ # assume time in H:M:S format
+ t
+ cat >> sites.xml <$queue
+END
+fi
+
+if [ "$time" != NONE ]; then
+ cat >> sites.xml <$time_mins
+END
+fi
+
+cat >> sites.xml <
END
-# 00:00:01
-# fast
-
cat >cf <
+#
+# fast
+# 00:59:00
+# $throttlePBS
+# 10000
+#
+# $R_LIBS_USER
+# $HOME/swiftwork
+#
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-sge
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-sge 2011-06-01 20:01:00 UTC (rev 4542)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-sge 2011-06-01 22:15:27 UTC (rev 4543)
@@ -2,9 +2,6 @@
# Generate Swift configuration files for SGE with manually-started Swift workers (passive coasters)
-cores=$1
-throttle=$2
-
cat >tc <tc <passive$cores
- 2.55
+ $throttle10000$(pwd)/swiftwork
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-01 20:01:00 UTC (rev 4542)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-01 22:15:27 UTC (rev 4543)
@@ -425,6 +425,34 @@
fi
}
+verify-is-time()
+{
+ argname=$1; shift
+ #check if positive integer
+ if [ "$1" -gt 0 ] 2> /dev/null; then
+ time_mins=$time
+ # echo time_mins: $time_mins
+ return 0
+ fi
+
+ d=${1%%:*}
+ hm=${1#*:}
+ h=${hm%%:*}
+ m=${hm#*:}
+ # check than they are integers >= 0
+ if [ \( "$h" -ge 0 \) -a \( "$m" -ge 0 \) -a \( "$d" -ge 0 \) \
+ -a \( \( "$h" -ge 0 \) -o \( "$m" -ge 0 \) -o \( "$d" -ge 0 \) \) ]; then
+ time_mins=$(($m + 60 * ($h + $d * 24)))
+# echo time_mins: $time_mins
+ return 0
+ else
+ echo $0: "value for $argname was neither valid d:m:s time, or positive
+ integer number of minutes"
+ usage
+ exit 1
+ fi
+}
+
verify-not-null()
{
argname=$1; shift
@@ -459,7 +487,7 @@
-r rcmd ssh site specific, SGE only, typically ssh.
qrsh for siraf cluster
-s server local local, pbs, sge, ssh, pbsf (for firewalled workers)
- ,cobalt,crayxt,custom
+ ,cobalt,crayxt,custom, pbsauto
-t time 00:30:00 hh:mm:ss, for PBS, Cobalt and SGE only
-w wkloglvl NONE NONE, ERROR, WARN, INFO, DEBUG, TRACE
-k keepdir No argument, if flag is set, will keep working
@@ -485,6 +513,7 @@
server=local
time="00:30:00"
+time_mins=30
nodes=1
queue=short
cores=0
@@ -504,6 +533,7 @@
sites_file=
tc_file=
cf_file=
+warmupjob=true
rcmd=ssh # rcmd: ssh (typical) or qrsh (eg for siraf with node login restrictions)
workmode=slot # slot: start one worker on each slot; node: start one worker for all slots on a node
@@ -521,8 +551,8 @@
-p) throttle=$2; verify-is-numeric throttle $throttle; shift ;;
-q) queue=$2; verify-not-null queue $queue; shift ;;
-r) rcmd=$2; verify-is-one-of rcmd $rcmd ssh qrsh; shift ;;
- -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge cobalt crayxt custom; shift ;;
- -t) time=$2; verify-not-null time $time; shift ;;
+ -s) server=$2; verify-is-one-of server $server local ssh pbs pbsf sge cobalt crayxt custom pbsauto; shift ;;
+ -t) time=$2; verify-is-time time $time; shift ;;
-w) workerLogging=$2; verify-is-one-of workerLoggingLevel $workerLogging NONE ERROR WARN INFO DEBUG TRACE; shift ;;
-L) swiftLoggingFlag="" ;; # swift default is lots of logging
-k) keepdir=TRUE ;;
@@ -626,6 +656,7 @@
exitcmd=""
if [ $server = custom ]; then
+ warmupjob=false
# have already set up tc.data and sites.xml files, just set
#onexit
function onexit {
@@ -640,13 +671,13 @@
echo ok > ackfifo
fi
elif [ $server = local ]; then
-
+ warmupjob=true
if [ $cores -eq 0 ]; then
cores=$defaultLocalCores
fi
- echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle
+ echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server
- source $SWIFTRBIN/configure-server-local $cores
+ source $SWIFTRBIN/configure-server-local
function onexit {
stdcleanup_start
# Find and terminate R workers: they should register their PiD
@@ -671,13 +702,13 @@
echo ok > ackfifo
fi
elif [ $server = ssh ]; then
-
+ warmupjob=true
if [ $cores -eq 0 ]; then
cores=$defaultSshCores
fi
- echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle
+ echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server
- source $SWIFTRBIN/configure-server-ssh $cores $throttle $time
+ source $SWIFTRBIN/configure-server-ssh
sshpidfile=${out/stdouterr/workerpids}
@@ -711,18 +742,44 @@
wait-and-start-ssh-workers &
starterpid=$!
+elif [ \( $server = pbsauto \) ]; then
+ warmupjob=false
+ # Systems where Swift manages workers
+ if [ $cores -le 0 ]; then
+ cores=$defaultClusterCores
+ fi
+ echo server=$server project=$project cores=$cores nodes=$nodes queue=$queue
+ source $SWIFTRBIN/configure-server-pbsauto
+
+ function onexit {
+ stdcleanup_start
+ trap - $TRAPEVENTS
+ # exit cleanly
+ stdcleanup_end
+ exit 0
+}
+
+ trap onexit $TRAPEVENTS
+ exitcmd=onexit
+
+
+ if [ "$doack" = TRUE ]; then
+ echo ok > ackfifo
+ fi
+
elif [ \( $server = pbs \) -o \( $server = pbsf \) -o \( $server = sge \) \
-o \( $server = cobalt \) -o \( $server = crayxt \) ]; then
-
- if [ $cores -eq 0 ]; then
+ warmupjob=true
+ # Batch systems where we need to launch workers
+ if [ $cores -le 0 ]; then
cores=$defaultClusterCores
fi
- echo project=$project cores=$cores nodes=$nodes queue=$queue server=$server throttle=$throttle
+ echo server=$server project=$project cores=$cores nodes=$nodes queue=$queue
if [ $server = pbsf ]
then
- source $SWIFTRBIN/configure-server-pbs $cores $throttle
+ source $SWIFTRBIN/configure-server-pbs
else
- source $SWIFTRBIN/configure-server-${server} $cores $throttle
+ source $SWIFTRBIN/configure-server-${server}
fi
jobidfile=${out/stdouterr/jobid}
@@ -756,7 +813,7 @@
$SWIFTRBIN/../swift/bin/swift $swiftLoggingFlag \
-config cf -tc.file tc -sites.file sites.xml $script -pipedir=$(pwd) \
- -warmup=true \
+ -warmup=$warmupjob \
>& $out
Author: davidk
Date: 2011-06-02 16:27:03 -0500 (Thu, 02 Jun 2011)
New Revision: 4544
Added:
trunk/docs/newuser-guide/
trunk/docs/newuser-guide/pads-quickstart.txt
trunk/examples/catsn.swift
Removed:
trunk/etc/sites/coaster-service.conf
Modified:
trunk/tests/providers/local-pbs/pads/sites.template.xml
Log:
Updated PADS pbs-local template and copied to etc/sites
Added catsn.swift to examples directory
Added start of new user guide
Added: trunk/docs/newuser-guide/pads-quickstart.txt
===================================================================
--- trunk/docs/newuser-guide/pads-quickstart.txt (rev 0)
+++ trunk/docs/newuser-guide/pads-quickstart.txt 2011-06-02 21:27:03 UTC (rev 4544)
@@ -0,0 +1,188 @@
+PADS Quickstart Guide
+=====================
+
+Requesting Access
+-----------------
+If you do not already have a Computation Institute account, you can request
+one at https://www.ci.uchicago.edu/accounts/. This page will give you a list
+of resources you can request access to. Be sure that PADS is selected. If
+you already have an existing CI account, but do not have access to PADS,
+send an email to support at ci.uchicago.edu to request access.
+
+SSH Keys
+--------
+Before accessing PADS, be sure to have your SSH keys configured correctly.
+There is some basic information about SSH and how to generate your key at
+http://www.ci.uchicago.edu/wiki/bin/view/Resources/SshKeys. Once you have
+followed those instructions, you can add your key at
+https://www.ci.uchicago.edu/support/sshkeys/.
+
+Connecting to a login node
+--------------------------
+Once your keys are configured, you should be able to access a PADS login
+node with the following command:
+
+-----
+ssh yourusername at login.pads.ci.uchicago.edu
+-----
+
+Configuring softenv
+-------------------
+Softenv is a system used for managing applications. In order to run Swift,
+the softenv environment will have to be modified slightly. Softenv is
+configured by a file in your home directory called .soft. Edit this file
+to look like this:
+-----
++java-sun
++maui
++torque
+ at default
+-----
+
+Downloading and building swift
+------------------------------
+The most recent versions of Swift can be found at
+http://www.ci.uchicago.edu/swift/downloads/index.php. Follow the instructions
+provided on that site to download and build Swift.
+
+Adding Swift to your PATH
+-------------------------
+Once you have installed Swift, add the Swift binary to your PATH so you can
+easily run it from any directory.
+
+In your home directory, edit the file ".bashrc".
+
+If you have installed Swift via a source repository, add the following line
+at the bottom of .bashrc.
+
+-----
+export PATH=$PATH:$HOME/cog/modules/swift/dist/swift-svn/bin
+-----
+
+If you have installed Swift via a binary package, add this line:
+
+-----
+export PATH=$PATH:$HOME/swift-/bin
+-----
+
+Replace with the actual name of the swift directory in the example above.
+
+What You Need To Know Before Running Swift
+------------------------------------------
+Before you can create a Swift configuration file, there are some things
+you will need to know.
+
+Swift Work Directory
+~~~~~~~~~~~~~~~~~~~~
+The Swift work directory is a directory which Swift uses for processing work.
+This directory needs to be writable. Common options for this are:
+
+-----
+/home/username/swiftwork
+/home/username/work
+/tmp
+-----
+
+Which project(s) are you a member of?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+PADS requires that you are a member of a project. You can determine this by
+running the following command:
+
+-----
+$ projects --available
+
+The following projects are available for your use
+
+Project PI Title
+
+CI-CCR000013 Michael Wilde The Swift Parallel Scripting System
+-----
+
+If you are not a member of a project, you must first request access
+to a project at http://www.ci.uchicago.edu/hpc/projects.
+
+You should make sure that you have a project set as default. Run
+the projects command with no arguments to determine if you have a default.
+
+------
+$ projects
+You have no default project set.
+-----
+
+To set your default project, use projects --set
+------
+$ projects --set CI-CCR000013 --all
+Your default project for all CI clusters has been set to CI-CCR000013.
+-----
+
+Determine your Queue
+~~~~~~~~~~~~~~~~~~~~
+
+PADS has several different queues you can submit jobs to depending on
+the type of work you will be doing. The command "qstat -q" will print
+the most up to date list of this information.
+
+.PADS Queues
+[options="header"]
+|=========================================================
+|Queue |Memory|CPU Time|Walltime|Node|Run|Que|Lm |State
+|route |-- |-- |-- |-- | 0| 0|-- | E R
+|short |-- |-- |04:00:00|-- | 64| 0|-- | E R
+|extended|-- |-- |-- |-- | 0| 0|-- | E R
+|fast |-- |-- |01:00:00|1 | 0|152|-- | E R
+|long |-- |-- |24:00:00|-- |232|130|-- | E R
+|=========================================================
+
+Generating Configuration Files
+------------------------------
+Now that you know what queue to use, your project, and your work directory, it is time to
+set up Swift. Swift uses a configuration file called sites.xml to determine how it should run.
+There are two methods you can use for creating this file. You can manually edit
+the configuration file, or generate it with a utility called gensites.
+
+Manually Editing sites.xml
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+Below is the template that is used by Swift's test suite for running on PADS.
+
+-----
+include::../../tests/providers/local-pbs/pads/sites.template.xml[]
+-----
+
+The values to note here are the ones that are listed between underscores. In the example above, they are _QUEUE_, and _WORK_. Queue is the PADS queue to use and WORK is the swift work directory. These are placeholder values you will need to modify to fit your needs. Copy and paste this template, replace the values, and call it sites.xml.
+
+Manually Editing tc.data
+~~~~~~~~~~~~~~~~~~~~~~~~
+Below is the tc.data file used by Swift's test suite for running on PADS.
+
+-----
+include::../../tests/providers/local-pbs/pads/tc.template.data[]
+------
+
+Copy these commands and save it as tc.data.
+
+Catsn.swift
+-----------
+The swift script we will run is called catsn.swift. It simply cats a file and saves the result. This is a nice simple test to ensure jobs are running correctly. Create a file called data.txt which contains some simple input - a "hello world" will do the trick.
+
+-----
+include::../../examples/catsn.swift[]
+-----
+
+Running Swift
+-------------
+Now that everything is in place, run Swift with the following command:
+
+-----
+swift -sites.file sites.xml -tc.file tc.data catsn.swift -n=10
+-----
+
+You should see several new files being created, called catsn.0001.out, catsn.0002.out, etc. Each of these
+files should contain the contents of what you placed into data.txt. If this happens, your job has run
+successfully on PADS!
+
+More Help
+---------
+The best place for additional help is the Swift user mailing list. You can subscribe to this list at
+http://mail.ci.uchicago.edu/mailman/listinfo/swift-user. When submitting information, please send your sites.xml file, your tc.data, and any Swift log files that were created during your attempt.
+
+
Deleted: trunk/etc/sites/coaster-service.conf
===================================================================
--- trunk/etc/sites/coaster-service.conf 2011-06-01 22:15:27 UTC (rev 4543)
+++ trunk/etc/sites/coaster-service.conf 2011-06-02 21:27:03 UTC (rev 4544)
@@ -1,31 +0,0 @@
-# Keep all interesting settings in one place
-# User should modify this to fit environment
-
-# Location of SWIFT. If empty, PATH is referenced
-export SWIFT=
-
-# Where to place/launch worker.pl on the remote machine for sites.xml
-export WORKER_WORK=/home/${USER}/work
-
-# How to launch workers- local or ssh
-export WORKER_MODE=ssh
-
-# Worker logging setting passed to worker.pl for sites.xml
-export WORKER_LOGGING=INFO
-
-# Worker host names for ssh
-export WORKERS_HOSTS="crush thwomp stomp crush crank steamroller grind churn trounce thrash vanquish octagon octopus"
-
-# Directory to keep log files, relative to working directory when launching start-coaster-service
-export LOG_DIR=logs
-
-# Manually define ports. If not specified, ports will be automatically generated
-export LOCAL_PORT=
-export SERVICE_PORT=
-
-# Below are various settings to give information about how to create sites.xml
-# These must remain as comments in order for gensites to recognize them
-#site work=$HOME/work
-#site queue=prod-devel
-#site maxtime=20
-#site nodes=64
Added: trunk/examples/catsn.swift
===================================================================
--- trunk/examples/catsn.swift (rev 0)
+++ trunk/examples/catsn.swift 2011-06-02 21:27:03 UTC (rev 4544)
@@ -0,0 +1,15 @@
+type file;
+
+app (file o) cat (file i)
+{
+ cat @i stdout=@o;
+}
+
+string t = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+string char[] = @strsplit(t, "");
+
+file out[];
+foreach j in [1:@toint(@arg("n","10"))] {
+ file data<"data.txt">;
+ out[j] = cat(data);
+}
Modified: trunk/tests/providers/local-pbs/pads/sites.template.xml
===================================================================
--- trunk/tests/providers/local-pbs/pads/sites.template.xml 2011-06-01 22:15:27 UTC (rev 4543)
+++ trunk/tests/providers/local-pbs/pads/sites.template.xml 2011-06-02 21:27:03 UTC (rev 4544)
@@ -3,6 +3,7 @@
0
+ _QUEUE__WORK_
From tga at ci.uchicago.edu Thu Jun 2 17:25:02 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Thu, 2 Jun 2011 17:25:02 -0500 (CDT)
Subject: [Swift-commit] r4545 - in SwiftApps/SwiftR: . Swift/exec
Message-ID: <20110602222502.EC7999CCA5@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-02 17:25:02 -0500 (Thu, 02 Jun 2011)
New Revision: 4545
Modified:
SwiftApps/SwiftR/IMMEDIATE-TODO
SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto
SwiftApps/SwiftR/Swift/exec/start-swift
Log:
Fixes to get pbsauto working nicely on pads: now passes test suite and handles time, nodes and cores parameters correctly.
Modified: SwiftApps/SwiftR/IMMEDIATE-TODO
===================================================================
--- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-06-02 21:27:03 UTC (rev 4544)
+++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-06-02 22:25:02 UTC (rev 4545)
@@ -30,7 +30,7 @@
- E.g. PBS with Dynamic submission of jobs of various lengths
- other batch schedulers
- This informs specifying
+ This involves specifying
- job walltimes
- queue
- staging mechanism
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto 2011-06-02 21:27:03 UTC (rev 4544)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto 2011-06-02 22:25:02 UTC (rev 4545)
@@ -22,6 +22,7 @@
$nodes$cores
+ $cores
@@ -40,8 +41,10 @@
fi
if [ "$time" != NONE ]; then
+ # Hack: Add 60 seconds to time to convince to request blocks for
+ # full time.
cat >> sites.xml <$time_mins
+ $((time_secs + 60))
END
fi
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-02 21:27:03 UTC (rev 4544)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-02 22:25:02 UTC (rev 4545)
@@ -430,20 +430,21 @@
argname=$1; shift
#check if positive integer
if [ "$1" -gt 0 ] 2> /dev/null; then
- time_mins=$time
- # echo time_mins: $time_mins
+ time_secs=$time
+ # echo time_secs: $time_secs
return 0
fi
- d=${1%%:*}
- hm=${1#*:}
- h=${hm%%:*}
- m=${hm#*:}
- # check than they are integers >= 0
- if [ \( "$h" -ge 0 \) -a \( "$m" -ge 0 \) -a \( "$d" -ge 0 \) \
- -a \( \( "$h" -ge 0 \) -o \( "$m" -ge 0 \) -o \( "$d" -ge 0 \) \) ]; then
- time_mins=$(($m + 60 * ($h + $d * 24)))
-# echo time_mins: $time_mins
+ h=${1%%:*}
+ ms=${1#*:}
+ m=${ms%%:*}
+ s=${ms#*:}
+ # check than they are integers >= 1
+ if [ \( "$h" -ge 0 \) -a \( "$m" -ge 0 \) -a \( "$s" -ge 0 \) \
+ -a \( \( "$h" -ge 0 \) -o \( "$m" -ge 0 \) -o \( "$s" -ge 0 \) \) \
+ ] 2> /dev/null ; then
+ time_secs=$(($s + 60 * ($m + $h * 60)))
+# echo time_secs: $time_secs
return 0
else
echo $0: "value for $argname was neither valid d:m:s time, or positive
@@ -513,7 +514,7 @@
server=local
time="00:30:00"
-time_mins=30
+time_secs=30
nodes=1
queue=short
cores=0
From davidk at ci.uchicago.edu Fri Jun 3 09:53:59 2011
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 09:53:59 -0500 (CDT)
Subject: [Swift-commit] r4546 - www/docs
Message-ID: <20110603145359.33AB89CCC8@svn.ci.uchicago.edu>
Author: davidk
Date: 2011-06-03 09:53:59 -0500 (Fri, 03 Jun 2011)
New Revision: 4546
Modified:
www/docs/index.php
Log:
Asciidoc tutorial added for trunk
Modified: www/docs/index.php
===================================================================
--- www/docs/index.php 2011-06-02 22:25:02 UTC (rev 4545)
+++ www/docs/index.php 2011-06-03 14:53:59 UTC (rev 4546)
@@ -43,13 +43,13 @@
[html] [pdf]
@@ -100,8 +100,6 @@
document gives some usage information for those tools.
-
-
Elsewhere
From tga at ci.uchicago.edu Fri Jun 3 11:12:34 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 11:12:34 -0500 (CDT)
Subject: [Swift-commit] r4548 - SwiftApps/SwiftR/Swift/exec
Message-ID: <20110603161234.9F39F9CC9C@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-03 11:12:34 -0500 (Fri, 03 Jun 2011)
New Revision: 4548
Modified:
SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto
SwiftApps/SwiftR/Swift/exec/start-swift
Log:
Tweaked time settings for pbsauto so that time parameter is better respected.
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto 2011-06-03 15:12:39 UTC (rev 4547)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto 2011-06-03 16:12:34 UTC (rev 4548)
@@ -1,7 +1,21 @@
#! /usr/bin/env bash
-#TODO: make configurable
-maxwalltime="00:10:00"
+#TODO: is this best way to do this.
+
+# we assume that $time_secs specifies the lengths of jobs
+# that should be submitted to pbs.
+#We don't really know the expected duration of R jobs, so we will come up
+# with a sensible value based on $time-secs
+#
+#Swift kills jobs that run for double maxwalltime, so maxwalltime should be
+# > 0.5 $time_secs
+# Ideally we want to run multiple jobs in each batch allocation, so we don't
+# want coasters to mistakenly decide that it can't schedule a job when we
+# have most of the batch allocation left. So we really want
+# maxwalltime to be just over 0.5 $time_secs, once we round both to minutes
+maxwalltime=$(( ($time_secs + 120) / 120 ))
+
+
cat >tc <11
+ 2
+ 2
-
END
if [ "$queue" != NONE ]; then
#TODO: error handling
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-03 15:12:39 UTC (rev 4547)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-03 16:12:34 UTC (rev 4548)
@@ -427,6 +427,10 @@
verify-is-time()
{
+ # Three valid formats, following maxWallTime globus param:
+ # minutes (single int)
+ # h:m
+ # h:m:s
argname=$1; shift
#check if positive integer
if [ "$1" -gt 0 ] 2> /dev/null; then
@@ -439,9 +443,14 @@
ms=${1#*:}
m=${ms%%:*}
s=${ms#*:}
- # check than they are integers >= 1
+
+ # if in h:m format, then #*: won't have deleted anything from m
+ if [ "$ms" = "$s" ]; then
+ s=0
+ fi
+ # check than they are integers >= 0 and at least one is > 0
if [ \( "$h" -ge 0 \) -a \( "$m" -ge 0 \) -a \( "$s" -ge 0 \) \
- -a \( \( "$h" -ge 0 \) -o \( "$m" -ge 0 \) -o \( "$s" -ge 0 \) \) \
+ -a \( \( "$h" -gt 0 \) -o \( "$m" -gt 0 \) -o \( "$s" -gt 0 \) \) \
] 2> /dev/null ; then
time_secs=$(($s + 60 * ($m + $h * 60)))
# echo time_secs: $time_secs
From tga at ci.uchicago.edu Fri Jun 3 11:40:15 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 11:40:15 -0500 (CDT)
Subject: [Swift-commit] r4549 - in SwiftApps/SwiftR: . Swift/exec
Message-ID: <20110603164015.E24689CC9C@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-03 11:40:15 -0500 (Fri, 03 Jun 2011)
New Revision: 4549
Modified:
SwiftApps/SwiftR/IMMEDIATE-TODO
SwiftApps/SwiftR/Swift/exec/start-swift
Log:
Fixed default time bug.
Modified: SwiftApps/SwiftR/IMMEDIATE-TODO
===================================================================
--- SwiftApps/SwiftR/IMMEDIATE-TODO 2011-06-03 16:12:34 UTC (rev 4548)
+++ SwiftApps/SwiftR/IMMEDIATE-TODO 2011-06-03 16:40:15 UTC (rev 4549)
@@ -17,7 +17,7 @@
HIGH:
-- sometimes the server crashes after servicing first request(
- only observer when running locally and on sge)
+ only observed when running locally and on sge)
HIGH:
-- Benchmark ideas
@@ -27,7 +27,10 @@
MED:
-- Have a more user-friendly way to specify some of the more common
custom setups:
- - E.g. PBS with Dynamic submission of jobs of various lengths
+ - PBS with Dynamic submission of jobs of various lengths
+ * Done: pbsauto
+ * This needs to be tested a bit more on other setups
+
- other batch schedulers
This involves specifying
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-03 16:12:34 UTC (rev 4548)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-03 16:40:15 UTC (rev 4549)
@@ -523,7 +523,7 @@
server=local
time="00:30:00"
-time_secs=30
+time_secs=1800
nodes=1
queue=short
cores=0
From wozniak at ci.uchicago.edu Fri Jun 3 11:43:21 2011
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 11:43:21 -0500 (CDT)
Subject: [Swift-commit] r4550 - trunk/tests/functions
Message-ID: <20110603164321.2150F9CC9C@svn.ci.uchicago.edu>
Author: wozniak
Date: 2011-06-03 11:43:20 -0500 (Fri, 03 Jun 2011)
New Revision: 4550
Added:
trunk/tests/functions/tracef/
Log:
tracef() tests
From wozniak at ci.uchicago.edu Fri Jun 3 11:44:44 2011
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 11:44:44 -0500 (CDT)
Subject: [Swift-commit] r4551 - trunk/tests/functions
Message-ID: <20110603164444.F0EA99CC9C@svn.ci.uchicago.edu>
Author: wozniak
Date: 2011-06-03 11:44:44 -0500 (Fri, 03 Jun 2011)
New Revision: 4551
Removed:
trunk/tests/functions/tracef/
Log:
This wasn't necessary
From wozniak at ci.uchicago.edu Fri Jun 3 11:46:02 2011
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 11:46:02 -0500 (CDT)
Subject: [Swift-commit] r4552 - in trunk/tests: functions language-behaviour
Message-ID: <20110603164602.AFFAF9CC9C@svn.ci.uchicago.edu>
Author: wozniak
Date: 2011-06-03 11:46:02 -0500 (Fri, 03 Jun 2011)
New Revision: 4552
Added:
trunk/tests/functions/tracef.swift
Removed:
trunk/tests/language-behaviour/tracef.swift
Log:
Move tracef tests
Copied: trunk/tests/functions/tracef.swift (from rev 4548, trunk/tests/language-behaviour/tracef.swift)
===================================================================
--- trunk/tests/functions/tracef.swift (rev 0)
+++ trunk/tests/functions/tracef.swift 2011-06-03 16:46:02 UTC (rev 4552)
@@ -0,0 +1,4 @@
+
+tracef("3:%i\n", 3);
+tracef("4:%s\n", "4");
+tracef("fraction:%f\n", 22.0/7.0);
Deleted: trunk/tests/language-behaviour/tracef.swift
===================================================================
--- trunk/tests/language-behaviour/tracef.swift 2011-06-03 16:44:44 UTC (rev 4551)
+++ trunk/tests/language-behaviour/tracef.swift 2011-06-03 16:46:02 UTC (rev 4552)
@@ -1,4 +0,0 @@
-
-tracef("3:%i\n", 3);
-tracef("4:%s\n", "4");
-tracef("fraction:%f\n", 22.0/7.0);
From wozniak at ci.uchicago.edu Fri Jun 3 11:49:01 2011
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 11:49:01 -0500 (CDT)
Subject: [Swift-commit] r4553 - trunk/tests/functions
Message-ID: <20110603164901.8516E9CC9C@svn.ci.uchicago.edu>
Author: wozniak
Date: 2011-06-03 11:49:01 -0500 (Fri, 03 Jun 2011)
New Revision: 4553
Added:
trunk/tests/functions/450-tracef.swift
Removed:
trunk/tests/functions/tracef.swift
Log:
Add sort key to tracef test
Copied: trunk/tests/functions/450-tracef.swift (from rev 4552, trunk/tests/functions/tracef.swift)
===================================================================
--- trunk/tests/functions/450-tracef.swift (rev 0)
+++ trunk/tests/functions/450-tracef.swift 2011-06-03 16:49:01 UTC (rev 4553)
@@ -0,0 +1,4 @@
+
+tracef("3:%i\n", 3);
+tracef("4:%s\n", "4");
+tracef("fraction:%f\n", 22.0/7.0);
Deleted: trunk/tests/functions/tracef.swift
===================================================================
--- trunk/tests/functions/tracef.swift 2011-06-03 16:46:02 UTC (rev 4552)
+++ trunk/tests/functions/tracef.swift 2011-06-03 16:49:01 UTC (rev 4553)
@@ -1,4 +0,0 @@
-
-tracef("3:%i\n", 3);
-tracef("4:%s\n", "4");
-tracef("fraction:%f\n", 22.0/7.0);
From wozniak at ci.uchicago.edu Fri Jun 3 11:58:36 2011
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 11:58:36 -0500 (CDT)
Subject: [Swift-commit] r4554 - trunk/tests
Message-ID: <20110603165836.CE26D9CC9C@svn.ci.uchicago.edu>
Author: wozniak
Date: 2011-06-03 11:58:36 -0500 (Fri, 03 Jun 2011)
New Revision: 4554
Modified:
trunk/tests/suite.sh
Log:
Fix error in usage notes
Modified: trunk/tests/suite.sh
===================================================================
--- trunk/tests/suite.sh 2011-06-03 16:49:01 UTC (rev 4553)
+++ trunk/tests/suite.sh 2011-06-03 16:58:36 UTC (rev 4554)
@@ -119,7 +119,7 @@
# to files in RUNDIR and environment variables from suite.sh
# such as $GROUP. Thus, you can:
# bring in input files: cp $GROUP/input-file.txt .
-# check output: grep TEXT1 exec.out
+# check output: grep TEXT1 stdout.txt
# grep TEXT2 output-file.txt
# clean up (optional): rm output-file.txt
# The results are added to the HTML output, etc., automatically.
From davidk at ci.uchicago.edu Fri Jun 3 12:32:56 2011
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 12:32:56 -0500 (CDT)
Subject: [Swift-commit] r4555 - trunk/docs
Message-ID: <20110603173256.603ED9CCC8@svn.ci.uchicago.edu>
Author: davidk
Date: 2011-06-03 12:32:56 -0500 (Fri, 03 Jun 2011)
New Revision: 4555
Modified:
trunk/docs/build_docs.sh
Log:
Fix for build_docs to copy all files in docs directory rather than just html and pdf output (may include charts, graphics, etc)
Modified: trunk/docs/build_docs.sh
===================================================================
--- trunk/docs/build_docs.sh 2011-06-03 16:58:36 UTC (rev 4554)
+++ trunk/docs/build_docs.sh 2011-06-03 17:32:56 UTC (rev 4555)
@@ -49,18 +49,12 @@
mkdir $INSTALLATION_DIRECTORY/$VERSION/$directory || crash "Unable to create directory $INSTALLATION_DIRECTORY/$VERSION/$directory"
fi
- # Copy HTML output
- for copyfile in `ls *.html 2>/dev/null`
+ # Copy all files to destination (may include graphics, etc)
+ for copyfile in `ls * 2>/dev/null`
do
cp $copyfile $INSTALLATION_DIRECTORY/$VERSION/$directory || crash "Unable to copy $copyfile to $INSTALLATION_DIRECTORY/$VERSION/$directory"
done
- # Copy PDF output
- for copyfile in `ls *.pdf 2>/dev/null`
- do
- cp $copyfile $INSTALLATION_DIRECTORY/$VERSION/$directory || crash "Unable to copy $copyfile to $INSTALLATION_DIRECTORY/$VERSION/$directory"
- done
-
popd > /dev/null 2>&1
done
popd > /dev/null 2>&1
From tga at ci.uchicago.edu Fri Jun 3 13:40:35 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 13:40:35 -0500 (CDT)
Subject: [Swift-commit] r4556 - in SwiftApps/SwiftR/Swift: R exec
Message-ID: <20110603184035.8B3ED9CC9C@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-03 13:40:35 -0500 (Fri, 03 Jun 2011)
New Revision: 4556
Modified:
SwiftApps/SwiftR/Swift/R/Apply.R
SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
SwiftApps/SwiftR/Swift/exec/start-swift
SwiftApps/SwiftR/Swift/exec/start-swift-daemon
Log:
No longer assumes that /tmp/$USER is writable - falls back to a random temporary directory and gives meaningful error messages when misconfigured.
Modified: SwiftApps/SwiftR/Swift/R/Apply.R
===================================================================
--- SwiftApps/SwiftR/Swift/R/Apply.R 2011-06-03 17:32:56 UTC (rev 4555)
+++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-06-03 18:40:35 UTC (rev 4556)
@@ -199,7 +199,16 @@
requestid = getOption(".swift.requestid") + 1;
}
else {
- requestdirbase = file.path(tmpdir, Sys.info()[["user"]],"SwiftR",
+ topdir <- file.path(tmpdir, Sys.info()[["user"]],"SwiftR")
+ if (!dir.create(topdir,recursive=TRUE,showWarnings=FALSE,
+ mode=kDIR_MODE)) {
+ oldtopdir <- topdir
+ topdir <- tempdir()
+ warning(paste("Could not create working directory", oldtopdir,
+ "instead using", topdir))
+ }
+
+ requestdirbase = file.path(topdir,
sprintf("requests.P%.5d",Sys.getpid()))
dir.create(requestdirbase,recursive=TRUE,showWarnings=FALSE,
mode=kDIR_MODE)
Modified: SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh
===================================================================
--- SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-06-03 17:32:56 UTC (rev 4555)
+++ SwiftApps/SwiftR/Swift/exec/EvalRBatchPersistent.sh 2011-06-03 18:40:35 UTC (rev 4556)
@@ -88,6 +88,19 @@
# Ensure that the dir for this slot exists.
BASEDIR=$tmp/$(id -nu)/SwiftR/Rworkers
+if mkdir -p $BASEDIR; then
+ :
+else
+ OLD_BASEDIR=$BASEDIR
+ BASEDIR=$tmp/SwiftR.${SWIFT_WORKER_PID}.Rworkers
+ mkdir -p $BASEDIR
+ basedir_ok=$?
+ if [ $basedir_ok != 0 ]; then
+ echo "Could not create worker directory in either $OLD_BASEDIR or $BASEDIR"
+ exit 1
+ fi
+fi
+
WORKERDIR=$BASEDIR/worker.$SWIFT_WORKER_PID
SLOTDIR=$WORKERDIR/${SWIFT_JOB_SLOT}
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-03 17:32:56 UTC (rev 4555)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-03 18:40:35 UTC (rev 4556)
@@ -42,7 +42,7 @@
{
get-contact
LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env?
- LOGDIR=$tmp/$USER/SwiftR/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way?
+ LOGDIR=$topdir/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way?
# mkdir -p $LOGDIR # is done with the ssh command, below
@@ -319,7 +319,7 @@
{
get-contact
LOGDIR=$(pwd)/swiftworkerlogs # full path. FIXME: Generate this with remote-side paths if not shared dir env?
- LOGDIR=/tmp/$USER/SwiftR/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way?
+ LOGDIR=$topdir/swiftworkerlogs # FIXME: left this in /tmp so it works on any host. Better way?
mkdir -p $LOGDIR
@@ -434,7 +434,7 @@
argname=$1; shift
#check if positive integer
if [ "$1" -gt 0 ] 2> /dev/null; then
- time_secs=$time
+ time_secs=$(( $time * 60 ))
# echo time_secs: $time_secs
return 0
fi
@@ -580,26 +580,43 @@
source $SWIFTRBIN/compat-setup
SWIFTBIN=$SWIFTRBIN/../swift/bin # This depends on ~/SwiftR/Swift/swift being a symlink to swift in RLibrary/Swift
-rundir=$tmp/$USER/SwiftR/swift.$server # rundir prefix # FIXME: handle multiple concurent independent swift servers per user
-mkdir -p $(dirname $rundir)
+
# Setup a working directory
if [ "$workdir" = NONE ]
then
+ topdir=$tmp/$USER/SwiftR
+
+ if mkdir -p $topdir 2> /dev/null ; then
+ :
+ else
+ topdir=$(mktemp -d $tmp/SwiftR.XXXXXX)
+ created_ok=$?
+ if [ "$created_ok" -ne 0 ]; then
+ echo "Could not create temporary directory under $tmp"
+ exit 1
+ fi
+ fi
+
+ rundir=$topdir/swift.$server # rundir prefix
doack=FALSE
- trundir=$(mktemp -d $rundir.XXXX) # FIXME: check success
- if [ "$?" != "0" ]
+ trundir=$(mktemp -d $rundir.XXXX)
+ created_ok=$?
+ if [ "$created_ok" -ne 0 ]
then
- echo "Could not create temporary directory under $tmp/$USER/SwiftR"
+ echo "Could not create temporary directory under $topdir/SwiftR"
exit 1
fi
else
+ topdir=`dirname "$workdir"`
+ rundir=$topdir/swift.$server # rundir prefix
doack=TRUE # let -daemon script know when we are done
echo Working in $workdir
trundir=$workdir
mkdir -p $workdir
fi
+# link temporary dir with different server name
rm -f $rundir
ln -s $trundir $rundir
Modified: SwiftApps/SwiftR/Swift/exec/start-swift-daemon
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift-daemon 2011-06-03 17:32:56 UTC (rev 4555)
+++ SwiftApps/SwiftR/Swift/exec/start-swift-daemon 2011-06-03 18:40:35 UTC (rev 4556)
@@ -6,20 +6,33 @@
# This script forks off a child process, detaches it
# and then, as the only thing written to stdout, echoes
# the pid of start-swift
+set -x
ssscript=`dirname $0`/start-swift
# Choose a working directory
tmp=${SWIFTR_TMP:-/tmp}
-mkdir -p $tmp/$USER/SwiftR
-workdir=$(mktemp -d $tmp/$USER/SwiftR/swift.XXXX)
+topdir=$tmp/$USER/SwiftR
+if mkdir -p $topdir 2> /dev/null ; then
+ :
+else
+ topdir=$(mktemp -d $tmp/SwiftR.XXXXXX)
+ created_ok=$?
+ if [ $created_ok -ne 0 ]; then
+ echo "Could not create temporary directory under $tmp"
+ exit 1
+ fi
+fi
+
+workdir=$(mktemp -d $topdir/swift.XXXX)
+
ackfifo=$workdir/ackfifo
mkfifo $ackfifo
if [ "$?" != "0" ]
then
- echo "Could not create temporary directory under $tmp/$USER/SwiftR"
+ echo "Could not create temporary directory under $topdir"
exit 1
fi
From tga at ci.uchicago.edu Fri Jun 3 13:46:22 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 13:46:22 -0500 (CDT)
Subject: [Swift-commit] r4557 - SwiftApps/SwiftR/Swift/exec
Message-ID: <20110603184622.896049CC9C@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-03 13:46:22 -0500 (Fri, 03 Jun 2011)
New Revision: 4557
Modified:
SwiftApps/SwiftR/Swift/exec/start-swift-daemon
Log:
didn't disable tracing.
Modified: SwiftApps/SwiftR/Swift/exec/start-swift-daemon
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift-daemon 2011-06-03 18:40:35 UTC (rev 4556)
+++ SwiftApps/SwiftR/Swift/exec/start-swift-daemon 2011-06-03 18:46:22 UTC (rev 4557)
@@ -6,7 +6,6 @@
# This script forks off a child process, detaches it
# and then, as the only thing written to stdout, echoes
# the pid of start-swift
-set -x
ssscript=`dirname $0`/start-swift
# Choose a working directory
From tga at ci.uchicago.edu Fri Jun 3 13:54:49 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 13:54:49 -0500 (CDT)
Subject: [Swift-commit] r4558 - SwiftApps/SwiftR/Swift/R
Message-ID: <20110603185449.D95E59CC9C@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-03 13:54:44 -0500 (Fri, 03 Jun 2011)
New Revision: 4558
Modified:
SwiftApps/SwiftR/Swift/R/Apply.R
Log:
Did not correctly detect when request directory already existed.
Modified: SwiftApps/SwiftR/Swift/R/Apply.R
===================================================================
--- SwiftApps/SwiftR/Swift/R/Apply.R 2011-06-03 18:46:22 UTC (rev 4557)
+++ SwiftApps/SwiftR/Swift/R/Apply.R 2011-06-03 18:54:44 UTC (rev 4558)
@@ -202,10 +202,13 @@
topdir <- file.path(tmpdir, Sys.info()[["user"]],"SwiftR")
if (!dir.create(topdir,recursive=TRUE,showWarnings=FALSE,
mode=kDIR_MODE)) {
- oldtopdir <- topdir
- topdir <- tempdir()
- warning(paste("Could not create working directory", oldtopdir,
- "instead using", topdir))
+ # Might already have existed
+ if (!file.exists(topdir) && file.info(topdir)$isdir) {
+ oldtopdir <- topdir
+ topdir <- tempdir()
+ warning(paste("Could not create working directory", oldtopdir,
+ "instead using", topdir))
+ }
}
requestdirbase = file.path(topdir,
From wozniak at ci.uchicago.edu Fri Jun 3 14:10:23 2011
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 14:10:23 -0500 (CDT)
Subject: [Swift-commit] r4559 -
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript
Message-ID: <20110603191023.79FAC9CC9C@svn.ci.uchicago.edu>
Author: wozniak
Date: 2011-06-03 14:10:23 -0500 (Fri, 03 Jun 2011)
New Revision: 4559
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java
Log:
Update tracef(), sprintf() functionality
Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java 2011-06-03 18:54:44 UTC (rev 4558)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Sprintf.java 2011-06-03 19:10:23 UTC (rev 4559)
@@ -55,13 +55,14 @@
return result;
}
- private String format(DSHandle[] args)
+ public static String format(DSHandle[] args)
throws ExecutionException {
if (! (args[0].getType() == Types.STRING))
throw new ExecutionException
("First argument to sprintf() must be a string!");
- String spec = args[0].toString();
+ String spec = (String) args[0].getValue();
+ logger.debug("spec: " + spec);
DSHandle[] vars = copyArray(args, 1, args.length-1);
StringBuilder output = new StringBuilder();
@@ -71,7 +72,7 @@
}
public static DSHandle[] copyArray(DSHandle[] src,
- int offset, int length)
+ int offset, int length)
{
DSHandle[] result = new DSHandle[length];
@@ -99,7 +100,7 @@
}
else if (c == '\\') {
char d = spec.charAt(++i);
- escape(d, output);
+ escape(i, spec, d, output);
}
else {
output.append(c);
@@ -107,7 +108,7 @@
i++;
}
}
-
+
private static int append(char c, int arg, DSHandle[] vars,
StringBuilder output)
throws ExecutionException {
@@ -147,7 +148,7 @@
return arg+1;
}
- private static void append_M(DSHandle arg, StringBuilder output)
+ private static void append_M(DSHandle arg, StringBuilder output)
throws ExecutionException {
try {
synchronized (arg.getRoot()) {
@@ -167,7 +168,7 @@
private static void append_f(DSHandle arg, StringBuilder output)
throws ExecutionException {
if (arg.getType() == Types.FLOAT) {
- output.append(arg).toString();
+ output.append(arg.getValue());
}
else {
throw new ExecutionException
@@ -178,7 +179,8 @@
private static void append_i(DSHandle arg, StringBuilder output)
throws ExecutionException {
if (arg.getType() == Types.INT) {
- output.append(arg).toString();
+ Double d = (Double) arg.getValue();
+ output.append(new Integer(d.intValue()));
}
else {
throw new ExecutionException
@@ -197,7 +199,7 @@
String entry = ""+i;
DSHandle handle =
node.getField(Path.parse(entry));
- output.append(handle);
+ output.append(handle.getValue());
if (i < size-1)
output.append(",");
}
@@ -218,7 +220,7 @@
private static void append_s(DSHandle arg, StringBuilder output)
throws ExecutionException {
if (arg.getType() == Types.STRING) {
- output.append(arg).toString();
+ output.append(arg.getValue());
}
else {
throw new ExecutionException
@@ -226,7 +228,12 @@
}
}
- private static void escape(char c, StringBuilder output)
+ /**
+ * @param i Only used for error messages
+ * @param spec Only used for error messages
+ */
+ private static void escape(int i, String spec,
+ char c, StringBuilder output)
throws ExecutionException {
if (c == '\\') {
output.append('\\');
@@ -239,7 +246,9 @@
}
else {
throw new ExecutionException
- ("tracef(): unknown backslash escape sequence!");
+ ("tracef(): unknown backslash escape sequence! " +
+ "(\\" + c + ")\n" +
+ "\t in " + spec + " character: " + i);
}
}
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2011-06-03 18:54:44 UTC (rev 4558)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Tracef.java 2011-06-03 19:10:23 UTC (rev 4559)
@@ -37,12 +37,7 @@
VDLFunction.waitFor(stack, handle);
}
- String spec = args[0].toString();
- DSHandle[] vars = Sprintf.copyArray(args, 1, args.length-1);
-
- StringBuilder output = new StringBuilder();
- Sprintf.format(spec, vars, output);
- String msg = output.toString();
+ String msg = Sprintf.format(args);
logger.info(msg);
System.out.print(msg);
return null;
From wozniak at ci.uchicago.edu Fri Jun 3 14:13:53 2011
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 14:13:53 -0500 (CDT)
Subject: [Swift-commit] r4560 - trunk/tests/functions
Message-ID: <20110603191353.4EFCF9CC9C@svn.ci.uchicago.edu>
Author: wozniak
Date: 2011-06-03 14:13:53 -0500 (Fri, 03 Jun 2011)
New Revision: 4560
Added:
trunk/tests/functions/450-tracef.check.sh
Modified:
trunk/tests/functions/450-tracef.swift
Log:
Update tracef() tests
Added: trunk/tests/functions/450-tracef.check.sh
===================================================================
--- trunk/tests/functions/450-tracef.check.sh (rev 0)
+++ trunk/tests/functions/450-tracef.check.sh 2011-06-03 19:13:53 UTC (rev 4560)
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -x
+
+grep "int:3:3" stdout.txt || exit 1
+grep "string:4:4" stdout.txt || exit 1
+grep "fraction:3.14" stdout.txt || exit 1
+grep "file:file:.*/test.txt" stdout.txt || exit 1
+grep "array:\[9.0,91.0,19.0\]" stdout.txt || exit 1
+grep "pointer:.*Closed" stdout.txt || exit 1
+
+[[ $( grep -c "WORD" stdout.txt ) == 2 ]] || exit 1
+[[ $( grep "WORD" stdout.txt | wc -w ) == 5 ]] || exit 1
+
+exit 0
Property changes on: trunk/tests/functions/450-tracef.check.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: trunk/tests/functions/450-tracef.swift
===================================================================
--- trunk/tests/functions/450-tracef.swift 2011-06-03 19:10:23 UTC (rev 4559)
+++ trunk/tests/functions/450-tracef.swift 2011-06-03 19:13:53 UTC (rev 4560)
@@ -1,4 +1,20 @@
-tracef("3:%i\n", 3);
-tracef("4:%s\n", "4");
+tracef("int:3:%i\n", 3);
+tracef("string:4:%s\n", "4");
tracef("fraction:%f\n", 22.0/7.0);
+
+type file;
+file f<"test.txt">;
+tracef("file:%M\n", f);
+
+int i[];
+i[0] = 9;
+i[1] = 91;
+i[2] = 19;
+tracef("array:%q\n", i);
+
+tracef("pointer:%p\n", 3);
+tracef("spacing: WORD\tWORD\nWORD\tWORD\n");
+
+// TODO: Debug the Swift parser- it handles backslash strangely
+// tracef("backslash2: \\ \ qq \\\\ \\n");
From wozniak at ci.uchicago.edu Fri Jun 3 15:07:13 2011
From: wozniak at ci.uchicago.edu (wozniak at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 15:07:13 -0500 (CDT)
Subject: [Swift-commit] r4561 - trunk/docs/userguide
Message-ID: <20110603200713.8DF5D9CCC8@svn.ci.uchicago.edu>
Author: wozniak
Date: 2011-06-03 15:07:13 -0500 (Fri, 03 Jun 2011)
New Revision: 4561
Modified:
trunk/docs/userguide/userguide.txt
Log:
Document tracef()
Modified: trunk/docs/userguide/userguide.txt
===================================================================
--- trunk/docs/userguide/userguide.txt 2011-06-03 19:13:53 UTC (rev 4560)
+++ trunk/docs/userguide/userguide.txt 2011-06-03 20:07:13 UTC (rev 4561)
@@ -1661,7 +1661,6 @@
used with @arg to pass input parameters to a SwiftScript program as
integers.
-
Built-in procedure reference
----------------------------
This section details built-in procedures that are available for use in
@@ -1733,6 +1732,38 @@
log message. The particular output format should not be relied upon.
(since Swift 0.4)
+tracef
+~~~~~~
+
++tracef(_spec_, _variable list_)+ will log its parameters as formatted
+by the formatter _spec_. _spec_ must be a string. Checks the type of
+the specifiers arguments against the variable list and allows for
+certain escape characters.
+
+Example:
+----
+int i = 3;
+tracef("%s: %i\n", "the value is", i);
+----
+
+Specifiers:
+
++%s+:: Format a string.
++%i+:: Format a number as an integer.
++%f+:: Format a number as a floating point number.
++%q+:: Format an array.
++%M+:: Format a mapped variable's filename.
++%k+:: Wait for the given variable but do not format it.
++%p+:: Format variable according to an internal format.
+
+Escape sequences:
+
++\n+:: Produce a newline.
++\t+:: Produce a tab.
+
+Known issues: :: Swift does not correctly scan certain backslash
+sequences such as +\\+.
+
writeData
~~~~~~~~~
writeData will write out data structures in the format described for
From tga at ci.uchicago.edu Fri Jun 3 15:27:02 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 15:27:02 -0500 (CDT)
Subject: [Swift-commit] r4562 - in SwiftApps/SwiftR/Swift: R exec
Message-ID: <20110603202702.14EDD9CC9C@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-03 15:27:01 -0500 (Fri, 03 Jun 2011)
New Revision: 4562
Modified:
SwiftApps/SwiftR/Swift/R/Workers.R
SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
SwiftApps/SwiftR/Swift/exec/configure-server-crayxt
SwiftApps/SwiftR/Swift/exec/configure-server-local
SwiftApps/SwiftR/Swift/exec/configure-server-pbs
SwiftApps/SwiftR/Swift/exec/configure-server-pbsauto
SwiftApps/SwiftR/Swift/exec/configure-server-sge
SwiftApps/SwiftR/Swift/exec/configure-server-ssh
SwiftApps/SwiftR/Swift/exec/rserver.swift
SwiftApps/SwiftR/Swift/exec/start-swift
Log:
Added configurable number of task retries.
found out that qsub sometimes returns code of 0 even on failure, added extra code to catch this case.
Modified: SwiftApps/SwiftR/Swift/R/Workers.R
===================================================================
--- SwiftApps/SwiftR/Swift/R/Workers.R 2011-06-03 20:07:13 UTC (rev 4561)
+++ SwiftApps/SwiftR/Swift/R/Workers.R 2011-06-03 20:27:01 UTC (rev 4562)
@@ -52,6 +52,7 @@
kernel=getOption("swift.kernel"),
workmode=getOption("swift.workmode"),
throttle=getOption("swift.throttle"),
+ retries=getOption("swift.retries"),
queue=getOption("swift.queue"),
rcmd=getOption("swift.rcmd"), time=getOption("swift.time"),
workerLogging=getOption("swift.workerLogging"),
@@ -138,6 +139,9 @@
if(! is.null(workmode) ) {
cmdString <- paste(cmdString, "-m", shQuote(workmode))
}
+ if (! is.null(retries) ) {
+ cmdString <- paste(cmdString, "-retries", shQuote(retries))
+ }
if (server == "local")
nodes <- 1
Modified: SwiftApps/SwiftR/Swift/exec/configure-server-cobalt
===================================================================
--- SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-06-03 20:07:13 UTC (rev 4561)
+++ SwiftApps/SwiftR/Swift/exec/configure-server-cobalt 2011-06-03 20:27:01 UTC (rev 4562)
@@ -27,7 +27,7 @@
cat >cf <cf <cf <cf <cf <cf <cf < $jobidfile
succ=$?
+
fi
if [ $succ -eq 0 ]
then
- echo Started workers from batch job $(cat $jobidfile)
+
+ # torque's error code is 0 sometimes when failed: make sure there
+ # is something in job id file
+ jobid=$(cat $jobidfile)
+ jobid=`echo $jobid`
+ if [ -z "$jobid" ]; then
+ echo Batch queue submission failed, exiting.
+ if [ "$doack" = TRUE ]; then
+ echo "Error: batch submission failed: qsub returned $succ" > ackfifo
+ fi
+ stdcleanup_start
+ stdcleanup_end
+ exit 1
+ fi
+ echo Started workers from batch job $jobid
if [ "$doack" = TRUE ]; then
echo ok > ackfifo
fi
@@ -531,6 +546,7 @@
defaultSshCores=4
defaultClusterCores=8
throttle=10
+num_retries=0
hosts=no-hosts-specified
queue=NONE
project=NONE
@@ -571,6 +587,7 @@
--tc.file) tc_file=$2; shift ;;
--sites.file) sites_file=$2; shift ;;
--cf.file) cf_file=$2; shift ;;
+ -retries) num_retries=$2; verify-is-numeric num_retries $num_retries; shift ;;
*) usage; exit 1 ;;
esac
shift
From tga at ci.uchicago.edu Fri Jun 3 15:38:29 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Fri, 3 Jun 2011 15:38:29 -0500 (CDT)
Subject: [Swift-commit] r4563 - SwiftApps/SwiftR/Swift/exec
Message-ID: <20110603203829.68D609CC9C@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-03 15:38:29 -0500 (Fri, 03 Jun 2011)
New Revision: 4563
Modified:
SwiftApps/SwiftR/Swift/exec/start-swift
Log:
eliminated extraneous error message
Modified: SwiftApps/SwiftR/Swift/exec/start-swift
===================================================================
--- SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-03 20:27:01 UTC (rev 4562)
+++ SwiftApps/SwiftR/Swift/exec/start-swift 2011-06-03 20:38:29 UTC (rev 4563)
@@ -387,7 +387,7 @@
# torque's error code is 0 sometimes when failed: make sure there
# is something in job id file
- jobid=$(cat $jobidfile)
+ jobid=$(cat $jobidfile 2> /dev/null)
jobid=`echo $jobid`
if [ -z "$jobid" ]; then
echo Batch queue submission failed, exiting.
@@ -834,7 +834,7 @@
coasterservicepid="" # null: saved in case we go back to using coaster servers
trap - $TRAPEVENTS
- jobid=$(cat $jobidfile)
+ jobid=$(cat $jobidfile 2> /dev/null)
echo Terminating worker processes starter $starterpid and batch job $jobid
if [ "_$starterpid" != _ ]; then
kill $starterpid &> /dev/null
From jonmon at ci.uchicago.edu Sat Jun 4 16:37:18 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Sat, 4 Jun 2011 16:37:18 -0500 (CDT)
Subject: [Swift-commit] r4564 - trunk/bin
Message-ID: <20110604213718.9BE929CC9C@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-04 16:37:18 -0500 (Sat, 04 Jun 2011)
New Revision: 4564
Modified:
trunk/bin/gensites
Log:
Added internalhostname variable for gensites
Modified: trunk/bin/gensites
===================================================================
--- trunk/bin/gensites 2011-06-03 20:38:29 UTC (rev 4563)
+++ trunk/bin/gensites 2011-06-04 21:37:18 UTC (rev 4564)
@@ -175,6 +175,9 @@
"#site $TEMPLATE execution_url="*|'#site slots='*)
EXECUTION_URL=`get_value $line`
;;
+ "#site $TEMPLATE internalhostname="*|'#site internalhostname='*)
+ INTERNALHOSTNAME=`get_value $line`
+ ;;
esac
done < $PROPERTIES_FILE
@@ -182,7 +185,7 @@
[[ ${WORK} == "" ]] && crash "Not specified: WORK"
# Verify that the variables by the template are defined
-for TOKEN in PROJECT QUEUE N_GRAN N_MAX SLOTS
+for TOKEN in PROJECT QUEUE N_GRAN N_MAX SLOTS INTERNALHOSTNAME
do
# Need but can't find, due
if grep _${TOKEN}_ $TEMPLATE_PATH > /dev/null; then
@@ -203,6 +206,7 @@
echo "s/_N_GRAN_/${N_GRAN}/"
echo "s/_N_MAX_/${N_MAX}/"
echo "s/_SLOTS_/${SLOTS}/"
+ echo "s/_INTERNALHOSTNAME_/${INTERNALHOSTNAME}/"
echo "s/_MAXTIME_/${MAXTIME}/"
echo "s at _EXECUTION_URL_@${EXECUTION_URL}@"
echo "s at _SERVICE_COASTERS_@${SERVICE_COASTERS:-NO_URL_GIVEN}@"
From davidk at ci.uchicago.edu Sun Jun 5 00:05:50 2011
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Sun, 5 Jun 2011 00:05:50 -0500 (CDT)
Subject: [Swift-commit] r4565 - trunk/bin
Message-ID: <20110605050550.3C0AD9CC9C@svn.ci.uchicago.edu>
Author: davidk
Date: 2011-06-05 00:05:50 -0500 (Sun, 05 Jun 2011)
New Revision: 4565
Modified:
trunk/bin/gensites
Log:
Fixed a few issues with gensites working with start-coaster-service
Modified: trunk/bin/gensites
===================================================================
--- trunk/bin/gensites 2011-06-04 21:37:18 UTC (rev 4564)
+++ trunk/bin/gensites 2011-06-05 05:05:50 UTC (rev 4565)
@@ -172,7 +172,7 @@
"#site $TEMPLATE slots="*|'#site slots='*)
SLOTS=`get_value $line`
;;
- "#site $TEMPLATE execution_url="*|'#site slots='*)
+ "#site $TEMPLATE execution_url="*|'#site execution_url='*)
EXECUTION_URL=`get_value $line`
;;
"#site $TEMPLATE internalhostname="*|'#site internalhostname='*)
@@ -185,7 +185,7 @@
[[ ${WORK} == "" ]] && crash "Not specified: WORK"
# Verify that the variables by the template are defined
-for TOKEN in PROJECT QUEUE N_GRAN N_MAX SLOTS INTERNALHOSTNAME
+for TOKEN in NODES HOST WORK PROJECT QUEUE N_GRAN N_MAX SLOTS INTERNALHOSTNAME MAXTIME EXECUTION_URL
do
# Need but can't find, due
if grep _${TOKEN}_ $TEMPLATE_PATH > /dev/null; then
From jonmon at ci.uchicago.edu Mon Jun 6 10:54:26 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Mon, 6 Jun 2011 10:54:26 -0500 (CDT)
Subject: [Swift-commit] r4567 - SwiftApps/Montage/scripts
Message-ID: <20110606155426.775469CCA5@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-06 10:54:26 -0500 (Mon, 06 Jun 2011)
New Revision: 4567
Modified:
SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
Log:
Modifications to the Montage wrappers
o Fixed typo in mShrink app procedure
o Switched to not explicitly mapping a file in some cases of the batch functions
Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-06 15:19:20 UTC (rev 4566)
+++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-06 15:54:26 UTC (rev 4567)
@@ -73,7 +73,7 @@
app ( Image shrunk ) mShrink( Image original, int factor )
{
- mShrink @orginial @shrunk factor;
+ mShrink @original @shrunk factor;
}
/*----------------------------------------------------------------------------------*/
Modified: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-06 15:19:20 UTC (rev 4566)
+++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-06 15:54:26 UTC (rev 4567)
@@ -1,7 +1,6 @@
( Image corr_imgs[] ) mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl )
{
- Table tmp <"back_tmp.tbl">;
- tmp = Background_list( img_tbl, corr_tbl );
+ Table back_list = Background_list( img_tbl, corr_tbl );
BackgroundStruct back_struct[] = readData2( tmp );
@@ -41,8 +40,7 @@
{
Status stats[] ;
- Table status_tbl <"stats.tbl">;
- status_tbl = create_status_table( diff_tbl );
+ Table status_tbl = create_status_table( diff_tbl );
foreach img, i in stats
{
From jonmon at ci.uchicago.edu Mon Jun 6 10:57:09 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Mon, 6 Jun 2011 10:57:09 -0500 (CDT)
Subject: [Swift-commit] r4568 - SwiftApps/Montage
Message-ID: <20110606155709.D74429CCA5@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-06 10:57:09 -0500 (Mon, 06 Jun 2011)
New Revision: 4568
Modified:
SwiftApps/Montage/TODO
Log:
Deleted some items from the Montage TODO list
Modified: SwiftApps/Montage/TODO
===================================================================
--- SwiftApps/Montage/TODO 2011-06-06 15:54:26 UTC (rev 4567)
+++ SwiftApps/Montage/TODO 2011-06-06 15:57:09 UTC (rev 4568)
@@ -3,12 +3,10 @@
-- Need to fix mProjectBatch. Currently calls mProject blindly. Need to figure out how to choose between mProject and mProjectPP
o exec/montage.sh
- -- Fix how to calculate the run directory
-- Figure out how to specify what site location that are going to be run on and do gensites on the templates
- -- Make sure all environment variables are set that are needed to run the execution script
o optimizations
- -- Create different pool entries. One for projectiog, calculating difference images, and background rectification. Each one will require different number of workers to execute with so different pool entries will be required. This will also change the tc.data files.
+ -- Try different sites.xml settings. Try different throttleing parameters as well as different queue sizes.
o templates
- -- Add more templates for other machines. For sure needs localhost
+ -- Add more templates for other machines. Add one for Beagle
From jonmon at ci.uchicago.edu Mon Jun 6 12:24:42 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Mon, 6 Jun 2011 12:24:42 -0500 (CDT)
Subject: [Swift-commit] r4569 - SwiftApps/Montage/scripts
Message-ID: <20110606172442.AE45D9CCA5@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-06 12:24:42 -0500 (Mon, 06 Jun 2011)
New Revision: 4569
Modified:
SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
Log:
Forgot to update a variable name in Montage wrappers
Modified: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-06 15:57:09 UTC (rev 4568)
+++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-06 17:24:42 UTC (rev 4569)
@@ -2,7 +2,7 @@
{
Table back_list = Background_list( img_tbl, corr_tbl );
- BackgroundStruct back_struct[] = readData2( tmp );
+ BackgroundStruct back_struct[] = readData2( back_list );
foreach background_entry, i in back_struct
{
From davidk at ci.uchicago.edu Mon Jun 6 13:26:29 2011
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Mon, 6 Jun 2011 13:26:29 -0500 (CDT)
Subject: [Swift-commit] r4570 - in trunk: bin etc
Message-ID: <20110606182629.995239CCA5@svn.ci.uchicago.edu>
Author: davidk
Date: 2011-06-06 13:26:29 -0500 (Mon, 06 Jun 2011)
New Revision: 4570
Modified:
trunk/bin/start-coaster-service
trunk/bin/stop-coaster-service
trunk/etc/coaster-service.conf
Log:
Modifications to coaster scripts to test with futuregrid
Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service 2011-06-06 17:24:42 UTC (rev 4569)
+++ trunk/bin/start-coaster-service 2011-06-06 18:26:29 UTC (rev 4570)
@@ -8,6 +8,46 @@
exit 1
}
+# Start futuregrid workers
+start-workers-futuregrid()
+{
+ PORT=$1
+ EXECUTION_URL=http://localhost:$PORT
+
+ if [ ! -d "$SWIFTVMBOOT_DIR" ] || [ ! -x "$SWIFTVMBOOT_DIR/bin/bootit.sh" ]; then
+ crash "SWIFTVMBOOT_DIR incorrectly defined in coaster-service.conf"
+ fi
+
+ echo Starting virtual machines.. please wait
+ #$SWIFTVMBOOT_DIR/bin/bootit.sh
+ SWIFTVMBOOT_OUTPUT=$SWIFTVMBOOT_DIR/output.json
+
+ if [ ! -f "$SWIFTVMBOOT_OUTPUT" ]; then
+ crash "Error: Swift VM output file $SWIFTVMBOOT_OUTPUT does not exist!"
+ fi
+
+ SWIFTVM_INSTANCES=`grep instance_id $SWIFTVMBOOT_OUTPUT |awk '{print $2}'|sed 's/\"//g;s/,//g;s/null//g'`
+ echo $SWIFTVM_INSTANCES > .swiftvm_instances
+
+ WORKER_HOSTS=`grep hostname $SWIFTVMBOOT_OUTPUT |awk '{print $2}'|sed 's/\"//g;s/,//g;s/null//g'`
+
+ # Start worker script
+ for MACHINE in $WORKER_HOSTS
+ do
+ # Enable ssh tunneling if needed
+ if [ $SSH_TUNNELING == "yes" ]; then
+ ssh -R *:$PORT:localhost:$PORT $WORKER_USERNAME@$MACHINE sleep 999 &
+ echo $! >> $PID_FILE
+ fi
+
+ # Copy and start worker script
+ scp $SWIFT_BIN/$WORKER $WORKER_USERNAME@$MACHINE:$WORKER_WORK > /dev/null 2>&1
+ echo "Starting worker on $MACHINE"
+ ssh $WORKER_USERNAME@$MACHINE $WORKER_WORK/$WORKER $EXECUTION_URL $MACHINE $LOG_DIR &
+ echo $! >> $PID_FILE
+ done
+}
+
# Start SSH workers
start-workers-ssh()
{
@@ -44,6 +84,9 @@
start-workers-cobalt()
{
PORT=$1
+ if [ -z "$PORT" ]; then
+ crash "start-workers-cobalt: Port number not specified, giving up"
+ fi
EXECUTION_URL=http://$IPADDR:$PORT
local TIMESTAMP=$(date "+%Y.%m%d.%H%M%S")
local -Z 5 R=${RANDOM}
@@ -77,7 +120,7 @@
# Determine information needed about this machine
if [ -z "$IPADDR" ]; then
if [ -x "/sbin/ifconfig" ]; then
- IPADDR=$( /sbin/ifconfig | grep inet | head -1 | cut -d ':' -f 2 | awk '{print $1}' )
+ IPADDR=$( /sbin/ifconfig | grep 'inet addr' | grep -v 127.0.0.1 | cut -d ':' -f 2 | awk '{print $1}' |head -1)
else
crash "Unable to determine IP address of system. Please add to coaster-service.conf"
fi
@@ -174,9 +217,27 @@
cobalt)
start-workers-cobalt $LOCAL_PORT
;;
+ futuregrid)
+ start-workers-futuregrid $LOCAL_PORT
+ ;;
+ *)
+ crash "Unknown WORKER_MODE. Please modify coaster-service.conf"
+ ;;
esac
# Generate sites.xml
export EXECUTION_URL="http://$IPADDR:$SERVICE_PORT"
echo Generating sites.xml..
gensites persistent-coasters -p $CONFIG_FILE > $RUN_DIR/sites.xml
+
+# Generate config file
+if [ $SHARED_FILESYSTEM == "no" ]; then
+cat > $RUN_DIR/cf << EOF
+use.provider.staging=true
+wrapperlog.always.transfer=false
+execution.retries=0
+provider.staging.pin.swiftfiles=false
+EOF
+else
+ echo > $RUN_DIR/cf
+fi
Modified: trunk/bin/stop-coaster-service
===================================================================
--- trunk/bin/stop-coaster-service 2011-06-06 17:24:42 UTC (rev 4569)
+++ trunk/bin/stop-coaster-service 2011-06-06 18:26:29 UTC (rev 4570)
@@ -1,8 +1,26 @@
#!/bin/bash
+# Files relative to location of this script
+PID_FILE=".coaster-service-pids"
+SWIFTVM_INSTANCES=".swiftvm_instances"
+CONFIG_FILE="../etc/coaster-service.conf"
+
+# crash: Report a problem and exit
+crash()
+{
+ MSG=$1
+ echo ${MSG} >&2
+ exit 1
+}
+
pushd $(dirname $(readlink -f $0)) > /dev/null 2>&1
-PID_FILE=".coaster-service-pids"
+# Read config file to determine worker mode
+if [ -f "$CONFIG_FILE" ]; then
+ source "$CONFIG_FILE"
+else
+ crash "Cannot find coaster-service.conf!"
+fi
echo Ending coaster processes..
if [ -f "$PID_FILE" ]; then
@@ -20,4 +38,13 @@
fi
echo Done
+if [ $WORKER_MODE == "futuregrid" ] && [ -f "$SWIFTVM_INSTANCES" ]; then
+ for INSTANCE in `cat $SWIFTVM_INSTANCES`
+ do
+ $SWIFTVMBOOT_DIR/ve/bin/cloudinitd terminate $INSTANCE
+ done
+ rm $SWIFTVM_INSTANCES > /dev/null 2>&1
+fi
+
+
popd > /dev/null 2>&1
Modified: trunk/etc/coaster-service.conf
===================================================================
--- trunk/etc/coaster-service.conf 2011-06-06 17:24:42 UTC (rev 4569)
+++ trunk/etc/coaster-service.conf 2011-06-06 18:26:29 UTC (rev 4570)
@@ -1,33 +1,45 @@
# Keep all interesting settings in one place
# User should modify this to fit environment
-# Location of SWIFT. If empty, PATH is referenced
+# Location of SWIFT. If empty, PATH is searched
export SWIFT=
-# Where to place/launch worker.pl on the remote machine for sites.xml
-export WORKER_WORK=/home/${USER}/work
+# Where to copy worker.pl on the remote machine for sites.xml
+export WORKER_WORK=$HOME/swiftwork
-# How to launch workers: local, ssh, or cobalt
+# How to launch workers: local, ssh, cobalt, or futuregrid
export WORKER_MODE=ssh
# Worker logging setting passed to worker.pl for sites.xml
export WORKER_LOGGING=INFO
-# Worker host names for ssh
+# SSH hosts to start workers on (ssh mode only)
export WORKER_HOSTS="host1 host2 host3"
+# Do all the worker nodes you're using have a shared filesystem? (yes/no)
+export SHARED_FILESYSTEM=no
+
+# Username to use on worker nodes
+export WORKER_USERNAME=$USER
+
+# Enable SSH tunneling? (yes/no)
+export SSH_TUNNELING=no
+
# Directory to keep log files, relative to working directory when launching start-coaster-service
export LOG_DIR=logs
-# Manually define ports. If not specified, ports will be automatically generated
+# Manually define ports. If not specified, an available port will be used
export LOCAL_PORT=
export SERVICE_PORT=
-# start-coaster-service tries to automatically detect IP address. Specify here if auto detection is not working correctly
+# start-coaster-service tries to automatically detect the IP address of this system. Specify here if you have multiple network interfaces
export IPADDR=
-# Below are various settings to give information about how to create sites.xml
-export WORK=$HOME/work
+# Location of the swift-vm-boot scripts
+export SWIFTVMBOOT_DIR=$HOME/swift-vm-boot
+
+# Swift information for creating sites.xml
+export WORK=$HOME/swiftwork
export QUEUE=prod-devel
export MAXTIME=20
export NODE=64
From davidk at ci.uchicago.edu Mon Jun 6 14:15:11 2011
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Mon, 6 Jun 2011 14:15:11 -0500 (CDT)
Subject: [Swift-commit] r4571 - in trunk: bin etc
Message-ID: <20110606191511.B1DE59CCA5@svn.ci.uchicago.edu>
Author: davidk
Date: 2011-06-06 14:15:11 -0500 (Mon, 06 Jun 2011)
New Revision: 4571
Modified:
trunk/bin/start-coaster-service
trunk/etc/coaster-service.conf
Log:
Modified start-coaster-service to only create config file when needed
Better defaults on coaster-service.conf
Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service 2011-06-06 18:26:29 UTC (rev 4570)
+++ trunk/bin/start-coaster-service 2011-06-06 19:15:11 UTC (rev 4571)
@@ -227,17 +227,16 @@
# Generate sites.xml
export EXECUTION_URL="http://$IPADDR:$SERVICE_PORT"
-echo Generating sites.xml..
+echo Generating sites.xml
gensites persistent-coasters -p $CONFIG_FILE > $RUN_DIR/sites.xml
# Generate config file
if [ $SHARED_FILESYSTEM == "no" ]; then
+echo Generating config file
cat > $RUN_DIR/cf << EOF
use.provider.staging=true
wrapperlog.always.transfer=false
execution.retries=0
provider.staging.pin.swiftfiles=false
EOF
-else
- echo > $RUN_DIR/cf
fi
Modified: trunk/etc/coaster-service.conf
===================================================================
--- trunk/etc/coaster-service.conf 2011-06-06 18:26:29 UTC (rev 4570)
+++ trunk/etc/coaster-service.conf 2011-06-06 19:15:11 UTC (rev 4571)
@@ -17,7 +17,7 @@
export WORKER_HOSTS="host1 host2 host3"
# Do all the worker nodes you're using have a shared filesystem? (yes/no)
-export SHARED_FILESYSTEM=no
+export SHARED_FILESYSTEM=yes
# Username to use on worker nodes
export WORKER_USERNAME=$USER
From jonmon at ci.uchicago.edu Mon Jun 6 15:30:40 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Mon, 6 Jun 2011 15:30:40 -0500 (CDT)
Subject: [Swift-commit] r4572 - SwiftApps/Montage/scripts
Message-ID: <20110606203040.5B7DD9CCAC@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-06 15:30:40 -0500 (Mon, 06 Jun 2011)
New Revision: 4572
Modified:
SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
Log:
Added more functionality
o Now can use the mDiffFit fucntion from Montage
Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-06 19:15:11 UTC (rev 4571)
+++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-06 20:30:40 UTC (rev 4572)
@@ -29,6 +29,12 @@
}
+app ( Image diff_img, Status stat ) mDiffFit( Image img_1, Image img_2, MosaicData hdr )
+{
+ mDiffFit "-n" "-s" @stat @img_1 @img_2 @diff_img @hdr;
+}
+
+
app ( Status stat ) mFitplane( Image diff_img )
{
// mFitplane "-s" @stat @diff_img;
Modified: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-06 19:15:11 UTC (rev 4571)
+++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-06 20:30:40 UTC (rev 4572)
@@ -36,6 +36,30 @@
}
+( Image diff_imgs[], Table fits_tbl ) mDiffFitBatch( string src_dir, string dest_dir, Table diff_tbl, MosaicData hdr )
+{
+ DiffStruct diffs[] ;
+ Status stats[];
+
+ Table status_tbl = create_status_table( diff_tbl );
+
+ foreach d_entry, i in diffs
+ {
+ Image img_1 ;
+ Image img_2 ;
+
+ Image diff_img ;
+ Status stat ;
+
+ ( diff_img, stat ) = mDiffFit( img_1, img_2, hdr );
+
+ diff_imgs[ i ] = diff_img;
+ stats[ i ] = stat;
+ }
+ fits_tbl = mConcatFit( status_tbl, stats );
+}
+
+
( Table fits_tbl ) mFitBatch( Image diff_imgs[], Table diff_tbl )
{
Status stats[] ;
@@ -56,8 +80,8 @@
foreach img, i in raw_imgs
{
Image proj_img ;
+ proj_img = mProject( img, hdr );
// proj_img = mProjectPP( img, hdr );
- proj_img = mProject( img, hdr );
proj_imgs[ i ] = proj_img;
}
From davidk at ci.uchicago.edu Mon Jun 6 15:58:31 2011
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Mon, 6 Jun 2011 15:58:31 -0500 (CDT)
Subject: [Swift-commit] r4573 - trunk/bin
Message-ID: <20110606205832.0188B9CCAC@svn.ci.uchicago.edu>
Author: davidk
Date: 2011-06-06 15:58:31 -0500 (Mon, 06 Jun 2011)
New Revision: 4573
Modified:
trunk/bin/start-coaster-service
Log:
Have start-coaster-service create a simple example tc file
Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service 2011-06-06 20:30:40 UTC (rev 4572)
+++ trunk/bin/start-coaster-service 2011-06-06 20:58:31 UTC (rev 4573)
@@ -19,7 +19,7 @@
fi
echo Starting virtual machines.. please wait
- #$SWIFTVMBOOT_DIR/bin/bootit.sh
+ $SWIFTVMBOOT_DIR/bin/bootit.sh
SWIFTVMBOOT_OUTPUT=$SWIFTVMBOOT_DIR/output.json
if [ ! -f "$SWIFTVMBOOT_OUTPUT" ]; then
@@ -240,3 +240,17 @@
provider.staging.pin.swiftfiles=false
EOF
fi
+
+# Generate TC file
+echo Generating tc.data
+cat > $RUN_DIR/tc.data << EOF
+persistent-coasters echo /bin/echo null null null
+persistent-coasters cat /bin/cat null null null
+persistent-coasters ls /bin/ls null null null
+persistent-coasters grep /bin/grep null null null
+persistent-coasters sort /bin/sort null null null
+persistent-coasters paste /bin/paste null null null
+persistent-coasters cp /bin/cp null null null
+persistent-coasters wc /usr/bin/wc null null null
+EOF
+
From jonmon at ci.uchicago.edu Tue Jun 7 13:33:55 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Tue, 7 Jun 2011 13:33:55 -0500 (CDT)
Subject: [Swift-commit] r4574 - SwiftApps/Montage/scripts
Message-ID: <20110607183355.60B5C9CCA5@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-07 13:33:55 -0500 (Tue, 07 Jun 2011)
New Revision: 4574
Modified:
SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
Log:
Added fix so that the area file generated by mProject are also mapped for stageout
Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-06 20:58:31 UTC (rev 4573)
+++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-07 18:33:55 UTC (rev 4574)
@@ -72,7 +72,8 @@
}
-app ( Image proj_img ) mProject( Image raw_img, MosaicData hdr )
+//app ( Image proj_img ) mProject( Image raw_img, MosaicData hdr )
+app ( Image proj_img, Image area_img ) mProject( Image raw_img, MosaicData hdr )
{
mProject "-X" @raw_img @proj_img @hdr;
}
Modified: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-06 20:58:31 UTC (rev 4573)
+++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-07 18:33:55 UTC (rev 4574)
@@ -1,4 +1,4 @@
-( Image corr_imgs[] ) mBgBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl )
+( Image corr_imgs[] ) mBackgroundBatch( Image bg_imgs[], Table img_tbl, Table corr_tbl )
{
Table back_list = Background_list( img_tbl, corr_tbl );
@@ -42,18 +42,18 @@
Status stats[];
Table status_tbl = create_status_table( diff_tbl );
-
+
foreach d_entry, i in diffs
{
Image img_1 ;
Image img_2 ;
Image diff_img ;
- Status stat ;
-
+ Status stat ;
+
( diff_img, stat ) = mDiffFit( img_1, img_2, hdr );
- diff_imgs[ i ] = diff_img;
+ diff_imgs[ i ] = diff_img;
stats[ i ] = stat;
}
fits_tbl = mConcatFit( status_tbl, stats );
@@ -75,14 +75,17 @@
}
-( Image proj_imgs[] ) mProjectBatch( Image raw_imgs[], MosaicData hdr )
+( Image proj_imgs[], Image area_imgs[] ) mProjectBatch( Image raw_imgs[], MosaicData hdr )
{
foreach img, i in raw_imgs
{
Image proj_img ;
- proj_img = mProject( img, hdr );
+ Image area_img ;
+// proj_img = mProject( img, hdr );
// proj_img = mProjectPP( img, hdr );
+ ( proj_img, area_img ) = mProject( img, hdr );
proj_imgs[ i ] = proj_img;
+ area_imgs[ i ] = area_img;
}
}
From davidk at ci.uchicago.edu Tue Jun 7 14:02:17 2011
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 7 Jun 2011 14:02:17 -0500 (CDT)
Subject: [Swift-commit] r4575 - in trunk: bin docs/newuser-guide examples
Message-ID: <20110607190217.7CA249CCA5@svn.ci.uchicago.edu>
Author: davidk
Date: 2011-06-07 14:02:17 -0500 (Tue, 07 Jun 2011)
New Revision: 4575
Added:
trunk/docs/newuser-guide/futuregrid-quickstart.txt
trunk/examples/hostsn.swift
Modified:
trunk/bin/start-coaster-service
trunk/docs/newuser-guide/pads-quickstart.txt
Log:
More content for the new user guide
More apps for start-coaster-service tc generation
Include hostsn script in examples
Modified: trunk/bin/start-coaster-service
===================================================================
--- trunk/bin/start-coaster-service 2011-06-07 18:33:55 UTC (rev 4574)
+++ trunk/bin/start-coaster-service 2011-06-07 19:02:17 UTC (rev 4575)
@@ -244,13 +244,14 @@
# Generate TC file
echo Generating tc.data
cat > $RUN_DIR/tc.data << EOF
-persistent-coasters echo /bin/echo null null null
-persistent-coasters cat /bin/cat null null null
-persistent-coasters ls /bin/ls null null null
-persistent-coasters grep /bin/grep null null null
-persistent-coasters sort /bin/sort null null null
-persistent-coasters paste /bin/paste null null null
-persistent-coasters cp /bin/cp null null null
-persistent-coasters wc /usr/bin/wc null null null
+persistent-coasters echo /bin/echo null null null
+persistent-coasters cat /bin/cat null null null
+persistent-coasters ls /bin/ls null null null
+persistent-coasters grep /bin/grep null null null
+persistent-coasters sort /bin/sort null null null
+persistent-coasters paste /bin/paste null null null
+persistent-coasters cp /bin/cp null null null
+persistent-coasters wc /usr/bin/wc null null null
+persistent-coasters hostname /bin/hostname null null null
EOF
Added: trunk/docs/newuser-guide/futuregrid-quickstart.txt
===================================================================
--- trunk/docs/newuser-guide/futuregrid-quickstart.txt (rev 0)
+++ trunk/docs/newuser-guide/futuregrid-quickstart.txt 2011-06-07 19:02:17 UTC (rev 4575)
@@ -0,0 +1,134 @@
+Futuregrid Quickstart Guide
+===========================
+
+Downloading and Building Swift
+------------------------------
+The most recent versions of Swift can be found at
+http://www.ci.uchicago.edu/swift/downloads/index.php. To run on futuregrid,
+you will need to download the development/trunk version from SVN.
+
+Adding Swift to your PATH
+-------------------------
+Once you have installed Swift, add the Swift binary to your PATH so you can
+easily run it from any directory.
+
+In your home directory, edit the file ".bashrc".
+
+If you have installed Swift via a source repository, add the following line
+at the bottom of .bashrc.
+
+-----
+export PATH=$PATH:$HOME/cog/modules/swift/dist/swift-svn/bin
+-----
+
+Requesting Futuregrid Access
+----------------------------
+If you do not already have a futuregrid account, you can follow the
+instructions at https://portal.futuregrid.org/gettingstarted to get started.
+This page provides information on how to create an account, how to join
+a project, how to set up your SSH keys, and how to create a new project.
+
+Downloading Swift VM Tools
+--------------------------
+A set of scripts based around cloudinitd are used to easily start virtual
+machines. To download, change to your home directory and run the
+following command:
+
+-----
+$ svn co https://svn.ci.uchicago.edu/svn/vdl2/usertools/swift-vm-boot
+-----
+
+Download your Credentials
+-------------------------
+Run the following commands to retrieve your credentials:
+-----
+$ scp yourusername at hotel.futuregrid.org:nimbus_creds.tar.gz .
+$ tar xvfz nimbus_creds.tar.gz
+-----
+When you extract your credential file, look at the file called
+hotel.conf. Near the bottom of this file will be two settings
+called vws.repository.s3id and vws.repository.s3key. Copy these
+values for the next step.
+
+Configuring Swift VM Tools
+--------------------------
+Change directories into the swift-vm-boot directory. Edit the file
+called env.sh. There are two settings here that you will need to
+modify: FUTUREGRID_IAAS_ACCESS_KEY and FUTUREGRID_IAAS_SECRET_KEY.
+Paste your access key (s3id) and secret key (s3key) here and save
+the file. By default, env.sh requests 2 nodes on hotel and 2 nodes
+on sierra. Change these values as needed. Run install.sh from the
+swift-vm-boot directory to complete the installation of Swift VM
+Tools. The Swift VM Tools may require a fairly recent version of
+Python. If you run into problems while running the install.sh
+script, please try a more recent version of Python and associated
+libraries.
+
+Configuring coaster-service.conf
+--------------------------------
+Within your swift distribution's etc directory is a file called
+coaster-service.conf. There are many options here you can modify
+as needed, but these are the settings that will be required
+to run on futuregrid:
+
+-----
+export WORKER_WORK=/tmp
+export WORKER_MODE=futuregrid
+export SHARED_FILESYSTEM=no
+export WORKER_USERNAME=root
+export SSH_TUNNELING=yes
+export SWIFTVMBOOT_DIR=$HOME/swift-vm-boot
+export WORK=/tmp
+-----
+
+These is a brief description of these settings in the coaster-service.conf file.
+Tunneling should be used when you are restricted by a firewall. If your local
+machine has multiple network interfaces, you should also manually set the IP
+address of your machine with export IPADDR=your.ip.address.here.
+
+Starting the Coaster Service Script
+-----------------------------------
+Now that everything is configured, change to a temporary directory
+and run this command to start the coaster service:
+
+-----
+$ start-coaster-service
+-----
+
+This command will start the VMs, start the required processes on the worker nodes,
+and generate Swift configuration files for you to use. The configuration files
+will be generated in your current directory. These files are sites.xml, tc.data,
+and cf.
+
+Running Swift
+-------------
+Now that you have all of your configuration files generated, run the following command:
+
+-----
+$ swift -sites.file sites.xml -tc.file tc.data -config cf
+-----
+
+If you would like to create a custom tc file for repeated use, rename it to something other
+than tc.data to prevent it from being overwritten. The sites.xml however will need to be
+regenerated every time you start the coaster service. If you need to repeatedly modify some
+sites.xml options, you may edit the template in Swift's etc/sites/persistent-coasters. You
+may also create your own custom tc files with the hostname of persistent-coasters. More
+information about this can be found in the Swift userguide at
+http://www.ci.uchicago.edu/swift/guides/trunk/userguide/userguide.html.
+
+Stopping the Coaster Service Script
+-----------------------------------
+To stop the coaster service, run the following command:
+-----
+$ stop-coaster-service
+-----
+
+This will kill the coaster service, kill the worker scripts on remote systems and terminate
+the virtual machines that were created during start-coaster-service.
+
+More Help
+---------
+The best place for additional help is the Swift user mailing list. You can subscribe to this list at
+http://mail.ci.uchicago.edu/mailman/listinfo/swift-user. When submitting information, please send
+your sites.xml file, your tc.data, and any error messages you run into.
+
Modified: trunk/docs/newuser-guide/pads-quickstart.txt
===================================================================
--- trunk/docs/newuser-guide/pads-quickstart.txt 2011-06-07 18:33:55 UTC (rev 4574)
+++ trunk/docs/newuser-guide/pads-quickstart.txt 2011-06-07 19:02:17 UTC (rev 4575)
@@ -39,7 +39,7 @@
@default
-----
-Downloading and building swift
+Downloading and building Swift
------------------------------
The most recent versions of Swift can be found at
http://www.ci.uchicago.edu/swift/downloads/index.php. Follow the instructions
Added: trunk/examples/hostsn.swift
===================================================================
--- trunk/examples/hostsn.swift (rev 0)
+++ trunk/examples/hostsn.swift 2011-06-07 19:02:17 UTC (rev 4575)
@@ -0,0 +1,11 @@
+type file;
+
+app (file o) hostname ()
+{
+ hostname stdout=@o;
+}
+
+file out[];
+foreach j in [1:@toint(@arg("n","1"))] {
+ out[j] = hostname();
+}
From ketan at ci.uchicago.edu Tue Jun 7 14:02:51 2011
From: ketan at ci.uchicago.edu (ketan at ci.uchicago.edu)
Date: Tue, 7 Jun 2011 14:02:51 -0500 (CDT)
Subject: [Swift-commit] r4576 - trunk/docs/newuser-guide
Message-ID: <20110607190251.8C1A09CCA5@svn.ci.uchicago.edu>
Author: ketan
Date: 2011-06-07 14:02:50 -0500 (Tue, 07 Jun 2011)
New Revision: 4576
Added:
trunk/docs/newuser-guide/beagle-quickstart.txt
Log:
Added beagle quickstart guide to swift newuser guide
Added: trunk/docs/newuser-guide/beagle-quickstart.txt
===================================================================
--- trunk/docs/newuser-guide/beagle-quickstart.txt (rev 0)
+++ trunk/docs/newuser-guide/beagle-quickstart.txt 2011-06-07 19:02:50 UTC (rev 4576)
@@ -0,0 +1,49 @@
+PADS Quickstart Guide
+=====================
+
+Requesting Access
+-----------------
+If you do not already have a Computation Institute account, you can request
+one at https://www.ci.uchicago.edu/accounts/. This page will give you a list
+of resources you can request access to.
+You already have an existing CI account, but do not have access to Beagle,
+send an email to support at ci.uchicago.edu to request access.
+
+Connecting to a login node
+--------------------------
+Once you have account, you should be able to access a Beagle login
+node with the following command:
+
+-----
+ssh yourusername at login.beagle.ci.uchicago.edu
+-----
+
+Follow the steps outlined below to get started with Swift on Beagle:
+
+*step 1.* Load the Swift module on Beagle as follows: +module load swift+
+
+*step 2.* Create and change to a directory where your Swift related work will
+stay. (say, +mkdir swift-lab+, followed by, +cd swift-lab+)
+
+*step 3.* To get started with a simple example running +/bin/cat+ to read an
+input file +data.txt+ and write to an output file +f.nnn.out+, copy the folder
+at +/home/ketan/catsn+ to the above directory. (+cp -r /home/ketan/catsn
+.+ followed by +cd catsn+).
+
+*step 4.* In the sites file: +beagle-coaster.xml+, make the following two
+changes: *1)* change the path of +workdirectory+ to your preferred location
+(say to +/lustre/beagle/$USER/swift-lab/swift.workdir+) and *2)* Change the
+project name to your project (+CI-CCR000013+) . The workdirectory will contain
+execution data related to each run, e.g. wrapper scripts, system information,
+inputs and outputs.
+
+*step 5.* Run the example using following commandline (also found in run.sh):
++swift -config cf -tc.file tc -sites.file beagle-coaster.xml catsn.swift -n=1+
+. You can further change the value of +-n+ to any arbitrary number to run that
+many number of concurrent +cat+
+
+*step 6.* Check the output in the generated +outdir+ directory (+ls outdir+)
+
+Note: Running from sandbox node or requesting 1 hour walltime for upto 3 nodes
+will get fast prioritized execution. Good for small tests.
+
From jonmon at ci.uchicago.edu Tue Jun 7 14:47:37 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Tue, 7 Jun 2011 14:47:37 -0500 (CDT)
Subject: [Swift-commit] r4577 - SwiftApps/Montage/scripts
Message-ID: <20110607194737.8BC4E9CFD6@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-07 14:47:37 -0500 (Tue, 07 Jun 2011)
New Revision: 4577
Modified:
SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
SwiftApps/Montage/scripts/mDiff_wrap.sh
Log:
Changed several interfaces to allow the mapping of the area files in the Montage scripts
Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-07 19:02:50 UTC (rev 4576)
+++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-07 19:47:37 UTC (rev 4577)
@@ -4,9 +4,10 @@
}
-app ( Image bg_img ) mBackground( Image img, float a, float b, float c )
+app ( Image bg_img ) mBackground( Image img, Image area_img, float a, float b, float c )
{
- mBackground "-n" @img @bg_img a b c;
+// mBackground "-n" @img @bg_img a b c;
+ mBackground @img @bg_img a b c;
}
@@ -22,16 +23,18 @@
}
-app ( Image diff_img ) mDiff( Image proj_img_1, Image proj_img_2, MosaicData hdr )
+app ( Image diff_img ) mDiff( Image proj_img_1, Image area_img_1, Image proj_img_2, Image area_img_2, MosaicData hdr )
{
// mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr;
- mDiff_wrap "-n" @proj_img_1 @proj_img_2 @diff_img @hdr;
+// mDiff_wrap "-n" @proj_img_1 @proj_img_2 @diff_img @hdr;
+ mDiff_wrap @proj_img_1 @proj_img_2 @diff_img @hdr;
}
-app ( Image diff_img, Status stat ) mDiffFit( Image img_1, Image img_2, MosaicData hdr )
+app ( Image diff_img, Status stat ) mDiffFit( Image img_1, Image area_img_1, Image img_2, Image area_img_2, MosaicData hdr )
{
- mDiffFit "-n" "-s" @stat @img_1 @img_2 @diff_img @hdr;
+// mDiffFit "-n" "-s" @stat @img_1 @img_2 @diff_img @hdr;
+ mDiffFit "-s" @stat @img_1 @img_2 @diff_img @hdr;
}
@@ -56,7 +59,7 @@
app ( JPEG mos_img_jpg ) mJPEG( Image mos_img )
{
- mJPEG "-gray" @mos_img "20%" "99.98%" "loglog" "-out" @mos_img_jpg;
+ mJPEG "-gray" @mos_img "0s" "max" "gaussian-log" "-out" @mos_img_jpg;
}
Modified: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-07 19:02:50 UTC (rev 4576)
+++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-07 19:47:37 UTC (rev 4577)
@@ -7,12 +7,13 @@
foreach background_entry, i in back_struct
{
Image proj_img ;
+ Image area_img ;
Image corr_img ;
float a = background_entry.a;
float b = background_entry.b;
float c = background_entry.c;
- corr_img = mBackground( proj_img, a, b, c );
+ corr_img = mBackground( proj_img, area_img, a, b, c );
corr_imgs[ i ] = corr_img;
}
@@ -27,9 +28,11 @@
{
Image img_1 ;
Image img_2 ;
+ Image area_img_1 ;
+ Image area_img_2 ;
Image diff_img ;
- diff_img = mDiff( img_1, img_2, hdr );
+ diff_img = mDiff( img_1, area_img_1, img_2, area_img_2, hdr );
diff_imgs[ i ] = diff_img;
}
@@ -47,11 +50,13 @@
{
Image img_1 ;
Image img_2 ;
+ Image area_img_1 ;
+ Image area_img_2 ;
Image diff_img ;
Status stat ;
- ( diff_img, stat ) = mDiffFit( img_1, img_2, hdr );
+ ( diff_img, stat ) = mDiffFit( img_1, area_img_1, img_2, area_img_2, hdr );
diff_imgs[ i ] = diff_img;
stats[ i ] = stat;
@@ -80,7 +85,7 @@
foreach img, i in raw_imgs
{
Image proj_img ;
- Image area_img ;
+ Image area_img ;
// proj_img = mProject( img, hdr );
// proj_img = mProjectPP( img, hdr );
( proj_img, area_img ) = mProject( img, hdr );
Modified: SwiftApps/Montage/scripts/mDiff_wrap.sh
===================================================================
--- SwiftApps/Montage/scripts/mDiff_wrap.sh 2011-06-07 19:02:50 UTC (rev 4576)
+++ SwiftApps/Montage/scripts/mDiff_wrap.sh 2011-06-07 19:47:37 UTC (rev 4577)
@@ -1,8 +1,8 @@
#!/bin/bash
-mDiff $1 $2 $3 $4 $5
+mDiff $1 $2 $3 $4
mkdir -p `dirname $4`
-touch $4
+touch $3
exit 0
From jonmon at ci.uchicago.edu Tue Jun 7 14:50:56 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Tue, 7 Jun 2011 14:50:56 -0500 (CDT)
Subject: [Swift-commit] r4578 - in SwiftApps/Montage: . scripts
Message-ID: <20110607195056.32F639CFD6@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-07 14:50:56 -0500 (Tue, 07 Jun 2011)
New Revision: 4578
Modified:
SwiftApps/Montage/TODO
SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
Log:
removed commented out lines during the testing of mapping the area files out. Updated TODO file
Modified: SwiftApps/Montage/TODO
===================================================================
--- SwiftApps/Montage/TODO 2011-06-07 19:47:37 UTC (rev 4577)
+++ SwiftApps/Montage/TODO 2011-06-07 19:50:56 UTC (rev 4578)
@@ -1,5 +1,4 @@
o scripts/Swift_Montage_Batch.swift
- -- Must map out the area files as well. Then change all the app procedures accordingly to remove the -n option.(-n stands for ignore the area files)
-- Need to fix mProjectBatch. Currently calls mProject blindly. Need to figure out how to choose between mProject and mProjectPP
o exec/montage.sh
Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-07 19:47:37 UTC (rev 4577)
+++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-07 19:50:56 UTC (rev 4578)
@@ -6,7 +6,6 @@
app ( Image bg_img ) mBackground( Image img, Image area_img, float a, float b, float c )
{
-// mBackground "-n" @img @bg_img a b c;
mBackground @img @bg_img a b c;
}
@@ -25,22 +24,18 @@
app ( Image diff_img ) mDiff( Image proj_img_1, Image area_img_1, Image proj_img_2, Image area_img_2, MosaicData hdr )
{
-// mDiff "-n" @proj_img_1 @proj_img_2 @diff_img @hdr;
-// mDiff_wrap "-n" @proj_img_1 @proj_img_2 @diff_img @hdr;
mDiff_wrap @proj_img_1 @proj_img_2 @diff_img @hdr;
}
app ( Image diff_img, Status stat ) mDiffFit( Image img_1, Image area_img_1, Image img_2, Image area_img_2, MosaicData hdr )
{
-// mDiffFit "-n" "-s" @stat @img_1 @img_2 @diff_img @hdr;
mDiffFit "-s" @stat @img_1 @img_2 @diff_img @hdr;
}
app ( Status stat ) mFitplane( Image diff_img )
{
-// mFitplane "-s" @stat @diff_img;
mFitplane_wrap "-s" @stat @diff_img;
}
@@ -75,7 +70,6 @@
}
-//app ( Image proj_img ) mProject( Image raw_img, MosaicData hdr )
app ( Image proj_img, Image area_img ) mProject( Image raw_img, MosaicData hdr )
{
mProject "-X" @raw_img @proj_img @hdr;
From davidk at ci.uchicago.edu Tue Jun 7 17:07:37 2011
From: davidk at ci.uchicago.edu (davidk at ci.uchicago.edu)
Date: Tue, 7 Jun 2011 17:07:37 -0500 (CDT)
Subject: [Swift-commit] r4580 - in trunk/docs: . userguide
Message-ID: <20110607220737.E902F9CCA5@svn.ci.uchicago.edu>
Author: davidk
Date: 2011-06-07 17:07:37 -0500 (Tue, 07 Jun 2011)
New Revision: 4580
Removed:
trunk/docs/userguide/overview.txt
Modified:
trunk/docs/build_docs.sh
Log:
Fix for group permissions in build_docs.sh
Removed duplicate chapter from userguide
Modified: trunk/docs/build_docs.sh
===================================================================
--- trunk/docs/build_docs.sh 2011-06-07 20:47:31 UTC (rev 4579)
+++ trunk/docs/build_docs.sh 2011-06-07 22:07:37 UTC (rev 4580)
@@ -8,6 +8,9 @@
exit 1
}
+# Change file permissions to values set below
+CHMOD_VALUE="664"
+
# Verify correct arguments
if [ -n "$1" ]; then
INSTALLATION_DIRECTORY=$1
@@ -53,6 +56,7 @@
for copyfile in `ls * 2>/dev/null`
do
cp $copyfile $INSTALLATION_DIRECTORY/$VERSION/$directory || crash "Unable to copy $copyfile to $INSTALLATION_DIRECTORY/$VERSION/$directory"
+ chmod $CHMOD_VALUE $INSTALLATION_DIRECTORY/$VERSION/$directory/$copyfile > /dev/null 2>&1
done
popd > /dev/null 2>&1
Deleted: trunk/docs/userguide/overview.txt
===================================================================
--- trunk/docs/userguide/overview.txt 2011-06-07 20:47:31 UTC (rev 4579)
+++ trunk/docs/userguide/overview.txt 2011-06-07 22:07:37 UTC (rev 4580)
@@ -1,20 +0,0 @@
-Overview
---------
-This manual provides reference material for Swift: the SwiftScript
-language and the Swift runtime system. For introductory material,
-consult the Swift tutorial
-.
-
-Swift is a data-oriented coarse grained scripting language that supports
-dataset typing and mapping, dataset iteration, conditional branching,
-and procedural composition.
-
-Swift programs (or workflows) are written in a language called
-SwiftScript.
-
-SwiftScript programs are dataflow oriented - they are primarily
-concerned with processing (possibly large) collections of data files, by
-invoking programs to do that processing. Swift handles execution of such
-programs on remote sites by choosing sites, handling the staging of
-input and output files to and from the chosen sites and remote execution
-of program code.
From tga at ci.uchicago.edu Tue Jun 7 23:41:46 2011
From: tga at ci.uchicago.edu (tga at ci.uchicago.edu)
Date: Tue, 7 Jun 2011 23:41:46 -0500 (CDT)
Subject: [Swift-commit] r4581 - SwiftApps/SwiftR/Swift
Message-ID: <20110608044146.AE3899CCA5@svn.ci.uchicago.edu>
Author: tga
Date: 2011-06-07 23:41:46 -0500 (Tue, 07 Jun 2011)
New Revision: 4581
Modified:
SwiftApps/SwiftR/Swift/DESCRIPTION
Log:
Bumping version number for next release.
Modified: SwiftApps/SwiftR/Swift/DESCRIPTION
===================================================================
--- SwiftApps/SwiftR/Swift/DESCRIPTION 2011-06-07 22:07:37 UTC (rev 4580)
+++ SwiftApps/SwiftR/Swift/DESCRIPTION 2011-06-08 04:41:46 UTC (rev 4581)
@@ -1,12 +1,12 @@
Package: Swift
Type: Package
Title: R interface to Swift parallel scripting languaage
-Version: 0.2.4
-Date: 2011-05-06
+Version: 0.2.5
+Date: 2011-06-07
Author: Michael Wilde
Maintainer: Michael Wilde
Description: Routines to invoke R functions on remote resources through Swift.
License: Apache License
LazyLoad: yes
-Packaged: 2011-05-06; Tim Armstrong
+Packaged: 2011-06-07; Tim Armstrong
From jonmon at ci.uchicago.edu Wed Jun 8 12:47:06 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 8 Jun 2011 12:47:06 -0500 (CDT)
Subject: [Swift-commit] r4582 - SwiftApps/Montage/scripts
Message-ID: <20110608174706.10B7C9CFD6@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-08 12:47:05 -0500 (Wed, 08 Jun 2011)
New Revision: 4582
Modified:
SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
Log:
the different table is now called on the raw image data. Calling the difference table on the projected images was causing some problems in which the raw images didn't overlap but the projected images did. this caused problems for mDiff and mDiffFit as files weren't being created. wrote wrapper scripts to solve this issue but this solution is much nicer
Modified: SwiftApps/Montage/scripts/Swift_Montage_Batch.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-08 04:41:46 UTC (rev 4581)
+++ SwiftApps/Montage/scripts/Swift_Montage_Batch.swift 2011-06-08 17:47:05 UTC (rev 4582)
@@ -39,7 +39,8 @@
}
-( Image diff_imgs[], Table fits_tbl ) mDiffFitBatch( string src_dir, string dest_dir, Table diff_tbl, MosaicData hdr )
+( Image diff_imgs[], Table fits_tbl ) mDiffFitBatch( string src_dir, string dest_dir, string files[], Table diff_tbl, MosaicData hdr )
+//( Image diff_imgs[], Table fits_tbl ) mDiffFitBatch( string src_dir, string dest_dir, Table diff_tbl, MosaicData hdr )
{
DiffStruct diffs[] ;
Status stats[];
@@ -48,8 +49,8 @@
foreach d_entry, i in diffs
{
- Image img_1 ;
- Image img_2 ;
+ Image img_1 ;
+ Image img_2 ;
Image area_img_1 ;
Image area_img_2 ;
From jonmon at ci.uchicago.edu Wed Jun 8 12:49:33 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 8 Jun 2011 12:49:33 -0500 (CDT)
Subject: [Swift-commit] r4583 - SwiftApps/Montage/scripts
Message-ID: <20110608174933.8E57C9CFD6@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-08 12:49:33 -0500 (Wed, 08 Jun 2011)
New Revision: 4583
Modified:
SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
Log:
Should not need to use the the wrapper scripts anymore.
Modified: SwiftApps/Montage/scripts/Swift_Montage_Apps.swift
===================================================================
--- SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-08 17:47:05 UTC (rev 4582)
+++ SwiftApps/Montage/scripts/Swift_Montage_Apps.swift 2011-06-08 17:49:33 UTC (rev 4583)
@@ -24,7 +24,7 @@
app ( Image diff_img ) mDiff( Image proj_img_1, Image area_img_1, Image proj_img_2, Image area_img_2, MosaicData hdr )
{
- mDiff_wrap @proj_img_1 @proj_img_2 @diff_img @hdr;
+ mDiff at proj_img_1 @proj_img_2 @diff_img @hdr;
}
@@ -36,7 +36,7 @@
app ( Status stat ) mFitplane( Image diff_img )
{
- mFitplane_wrap "-s" @stat @diff_img;
+ mFitplane"-s" @stat @diff_img;
}
From jonmon at ci.uchicago.edu Wed Jun 8 16:45:21 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 8 Jun 2011 16:45:21 -0500 (CDT)
Subject: [Swift-commit] r4584 - SwiftApps/Montage/exec
Message-ID: <20110608214521.B642E9CFD6@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-08 16:45:21 -0500 (Wed, 08 Jun 2011)
New Revision: 4584
Modified:
SwiftApps/Montage/exec/montage.sh
Log:
Minor edits to the execution script
Modified: SwiftApps/Montage/exec/montage.sh
===================================================================
--- SwiftApps/Montage/exec/montage.sh 2011-06-08 17:49:33 UTC (rev 4583)
+++ SwiftApps/Montage/exec/montage.sh 2011-06-08 21:45:21 UTC (rev 4584)
@@ -9,26 +9,31 @@
fi
# Fix this. Not sure how to set this variable if it not set or even what the error message should be
-if [ "X$SWIFT_MONTAGE" == "X" ]; then
+if [ "X$SWIFT_MONTAGE" = "X" ] ; then
echo "Please set the SWIFT_MONTAGE environment variable to where the Swift_Montage wrappers home directory is."
exit 1
fi
+if [ "X$SWIFT_MONTAGE_WORK" = "X" ] ; then
+ echo "Please set the SWIFT_MONTAGE_WORK environment varible to where the montage work should be done."
+ exit 1
+fi
+
RUN_DIRECTORY=(`ls | grep run | tail -n 1`)
NUM=`echo $RUN_DIRECTORY | awk -F . '{print $NF}'`
WORK_DIRECTORY=(`printf "run.%04d" $((NUM+1))`)
mkdir ${WORK_DIRECTORY}
-if [ $HOSTNAME == "login2.pads.ci.uchicago.edu" ]; then
- echo "#site pads-proj-coasters.xml internalhostname=192.5.86.6" >> sites.properties
- echo "#site pads-diff-coasters.xml internalhostname=192.5.86.6" >> sites.properties
- echo "#site pads-fit-coasters.xml internalhostname=192.5.86.6" >> sites.properties
-elif [ $HOSTNAME == "login1.pads.ci.uchicago.edu" ]; then
- echo "#site pads-proj-coasters.xml internalhostname=192.5.86.5" >> sites.properties
- echo "#site pads-diff-coasters.xml internalhostname=192.5.86.5" >> sites.properties
- echo "#site pads-fit-coasters.xml internalhostname=192.5.86.5" >> sites.properties
-fi
+# if [ $HOSTNAME == "login2.pads.ci.uchicago.edu" ]; then
+# echo "#site pads-proj-coasters.xml internalhostname=192.5.86.6" >> sites.properties
+# echo "#site pads-diff-coasters.xml internalhostname=192.5.86.6" >> sites.properties
+# echo "#site pads-fit-coasters.xml internalhostname=192.5.86.6" >> sites.properties
+# elif [ $HOSTNAME == "login1.pads.ci.uchicago.edu" ]; then
+# echo "#site pads-proj-coasters.xml internalhostname=192.5.86.5" >> sites.properties
+# echo "#site pads-diff-coasters.xml internalhostname=192.5.86.5" >> sites.properties
+# echo "#site pads-fit-coasters.xml internalhostname=192.5.86.5" >> sites.properties
+# fi
echo "#site pads-proj-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
echo "#site pads-proj-coasters.xml slots=$1" >> sites.properties
@@ -36,11 +41,12 @@
echo "#site pads-diff-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
echo "#site pads-diff-coasters.xml slots=$2" >> sites.properties
-echo "#site pads-fit-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
-echo "#site pads-fit-coasters.xml slots=$3" >> sites.properties
+echo "#site pads-fit-coasters.xml work=/gpfs/pads/jonmon/Swift/work/pads" >> sites.properties
+echo "#site pads-fit-coasters.xml slots=$3" >> sites.properties
-echo "#site local.xml work=/gpfs/pads/jonmon/Swift/work/local" >> sites.properties
+echo "#site local.xml work=/gpfs/pads/jonmon/Swift/work/local" >> sites.properties
+
echo "" > sites.xml
echo "" >> sites.xml
@@ -62,8 +68,8 @@
mv sites.xml ${WORK_DIRECTORY}
cp tc.data ${WORK_DIRECTORY}
-cp $4 ${WORK_DIRECTORY}
+cp -r $4 ${WORK_DIRECTORY}
cd ${WORK_DIRECTORY}
-#swift -tc.file tc.data -sites.file sites.xml -config ${SWIFT_MONTAGE}/etc swift_montage.properties
+echo "swift -tc.file tc.data -sites.file sites.xml -config ${SWIFT_MONTAGE}/etc swift_montage.properties ${SWIFT_MONTAGE}/scripts/montage.swift"
#clear ; swiftrun -sites localhost pads -input raw_dir header.hdr -script m101_montage.swift
From jonmon at ci.uchicago.edu Wed Jun 8 16:48:30 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 8 Jun 2011 16:48:30 -0500 (CDT)
Subject: [Swift-commit] r4585 - SwiftApps/Montage/exec
Message-ID: <20110608214830.8D00E9CFD6@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-08 16:48:30 -0500 (Wed, 08 Jun 2011)
New Revision: 4585
Modified:
SwiftApps/Montage/exec/montage.sh
Log:
Added change so now the tc file must be added at the command line. Might change this later
Modified: SwiftApps/Montage/exec/montage.sh
===================================================================
--- SwiftApps/Montage/exec/montage.sh 2011-06-08 21:45:21 UTC (rev 4584)
+++ SwiftApps/Montage/exec/montage.sh 2011-06-08 21:48:30 UTC (rev 4585)
@@ -1,6 +1,6 @@
#!/bin/bash
-EXPECTED_ARGS=4
+EXPECTED_ARGS=5
if [ $# -ne $EXPECTED_ARGS ]
then
@@ -67,8 +67,8 @@
rm -f sites.properties
mv sites.xml ${WORK_DIRECTORY}
-cp tc.data ${WORK_DIRECTORY}
-cp -r $4 ${WORK_DIRECTORY}
+cp $4 ${WORK_DIRECTORY}
+cp -r $5 ${WORK_DIRECTORY}
cd ${WORK_DIRECTORY}
echo "swift -tc.file tc.data -sites.file sites.xml -config ${SWIFT_MONTAGE}/etc swift_montage.properties ${SWIFT_MONTAGE}/scripts/montage.swift"
From jonmon at ci.uchicago.edu Wed Jun 8 16:56:42 2011
From: jonmon at ci.uchicago.edu (jonmon at ci.uchicago.edu)
Date: Wed, 8 Jun 2011 16:56:42 -0500 (CDT)
Subject: [Swift-commit] r4586 - in SwiftApps/Montage: exec scripts
Message-ID: <20110608215642.95E6A9CFD6@svn.ci.uchicago.edu>
Author: jonmon
Date: 2011-06-08 16:56:42 -0500 (Wed, 08 Jun 2011)
New Revision: 4586
Added:
SwiftApps/Montage/scripts/montage.swift
Modified:
SwiftApps/Montage/exec/montage.sh
Log:
Fixed error in the execution scirpt. erroneus space in the config file path. added the general montage workflow to the scripts repository
Modified: SwiftApps/Montage/exec/montage.sh
===================================================================
--- SwiftApps/Montage/exec/montage.sh 2011-06-08 21:48:30 UTC (rev 4585)
+++ SwiftApps/Montage/exec/montage.sh 2011-06-08 21:56:42 UTC (rev 4586)
@@ -67,9 +67,10 @@
rm -f sites.properties
mv sites.xml ${WORK_DIRECTORY}
+cp ${SWIFT_MONTAGE}/scripts/montage.swift ${WORK_DIRECTORY}
cp $4 ${WORK_DIRECTORY}
cp -r $5 ${WORK_DIRECTORY}
cd ${WORK_DIRECTORY}
-echo "swift -tc.file tc.data -sites.file sites.xml -config ${SWIFT_MONTAGE}/etc swift_montage.properties ${SWIFT_MONTAGE}/scripts/montage.swift"
+echo "swift -tc.file tc.data -sites.file sites.xml -config ${SWIFT_MONTAGE}/etc/swift_montage.properties ${SWIFT_MONTAGE}/scripts/montage.swift | tee -a swift.out"
#clear ; swiftrun -sites localhost pads -input raw_dir header.hdr -script m101_montage.swift
Added: SwiftApps/Montage/scripts/montage.swift
===================================================================
--- SwiftApps/Montage/scripts/montage.swift (rev 0)
+++ SwiftApps/Montage/scripts/montage.swift 2011-06-08 21:56:42 UTC (rev 4586)
@@ -0,0 +1,35 @@
+import "Swift_Montage_Types";
+import "Swift_Montage_Apps";
+import "Swift_Montage_Batch";
+
+MosaicData header <"header.hdr">;
+Table images_tbl <"images.tbl">;
+Table raw_images_tbl <"raw-images.tbl">;
+Table difference_tbl <"diffs.tbl">;
+Table fits_images_tbl <"fits.tbl">;
+Table corrections_tbl <"corrections.tbl">;
+Table stat_tbl <"stats.tbl">;
+Image corrected_mos <"final/corrected.fits">;
+JPEG corrected_jpg_img <"final/corrected.jpg">;
+Image projected_images[];
+Image projected_areas[];
+Image difference_images[];
+Image corrected_images[];
+
+Image raw_image_files[] ;
+
+difference_tbl = mOverlaps( raw_images_tbl );
+
+( projected_images, projected_areas ) = mProjectBatch( raw_image_files, header );
+
+images_tbl = mImgtbl( projected_images );
+
+( difference_images, fits_images_tbl ) = mDiffFitBatch( "proj_dir", "diff_dir", @filenames(projected_images), difference_tbl, header );
+
+corrections_tbl = mBgModel( images_tbl, fits_images_tbl );
+
+corrected_images = mBackgroundBatch( projected_images, images_tbl, corrections_tbl );
+
+corrected_mos = mAdd( corrected_images, images_tbl, header );
+
+corrected_jpg_img = mJPEG( corrected_mos );
From hategan at ci.uchicago.edu Wed Jun 8 21:13:07 2011
From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu)
Date: Wed, 8 Jun 2011 21:13:07 -0500 (CDT)
Subject: [Swift-commit] r4587 - trunk/src/org/griphyn/vdl/karajan/lib
Message-ID: <20110609021307.4B8949CFD6@svn.ci.uchicago.edu>
Author: hategan
Date: 2011-06-08 21:13:07 -0500 (Wed, 08 Jun 2011)
New Revision: 4587
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/Stageout.java
Log:
1. wait for static return arrays to be initialized before starting the app (i.e. make sure the size is known); 2. the list of stageouts for complex types (and arrays) was broken (e.g. it returned the leaf var instead of the root var; fixed both issues
Modified: trunk/src/org/griphyn/vdl/karajan/lib/Stageout.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Stageout.java 2011-06-08 21:56:42 UTC (rev 4586)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Stageout.java 2011-06-09 02:13:07 UTC (rev 4587)
@@ -9,17 +9,15 @@
import org.apache.log4j.Logger;
import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.arguments.Arg.Channel;
import org.globus.cog.karajan.arguments.ArgUtil;
import org.globus.cog.karajan.arguments.NamedArguments;
-import org.globus.cog.karajan.arguments.Arg.Channel;
-import org.globus.cog.karajan.stack.VariableNotFoundException;
import org.globus.cog.karajan.stack.VariableStack;
import org.globus.cog.karajan.workflow.ExecutionException;
import org.globus.cog.karajan.workflow.futures.FutureNotYetAvailable;
import org.griphyn.vdl.mapping.AbstractDataNode;
import org.griphyn.vdl.mapping.DSHandle;
import org.griphyn.vdl.mapping.HandleOpenException;
-import org.griphyn.vdl.mapping.InvalidPathException;
import org.griphyn.vdl.mapping.MappingDependentException;
import org.griphyn.vdl.mapping.Path;
@@ -40,8 +38,8 @@
.isPrimitive());
}
- private List list(Path p, DSHandle var) {
- ArrayList l = new ArrayList(2);
+ private List> list(Path p, DSHandle var) {
+ ArrayList
+
+
+
+
+
+
Both the Montage binaries as well as the Swift binaries must be included
+in your PATH environment variable for the wrappers to function properly.
+
+
Swift Montage API
Types
-
These are all the Types that are declared for Swift
+
These are all the Types that are declared for Swift. They are located in the
+file Swift_Montage_types.swift.
Image
@@ -804,9 +814,17 @@
App Procedures
-
These are all the app procedures for Swift Montage. Currently the options to
-the Montage functions are hard coded and the user cannot choose which options
-to run.
+
These are all the app procedures for Swift Montage. They are located in the
+file Swift_Montage_Apps.swift
+
+
+
+
+
+
Currently the options to the Montage functions are hard coded and the
+user cannot choose which options to run.
+
+
mAdd
@@ -980,7 +998,8 @@
Batch Wrappers
These are the Batch scripts. These scripts do most of the file mapping
-and is where the parallelization is taking place.
+and is where the parallelization is taking place. They are located in the file
+Swift_Montage_Batch.swift