[Swift-commit] r6276 - in SwiftApps/modis: . conf tutorial
davidk at ci.uchicago.edu
davidk at ci.uchicago.edu
Fri Feb 15 04:36:37 CST 2013
Author: davidk
Date: 2013-02-15 04:36:37 -0600 (Fri, 15 Feb 2013)
New Revision: 6276
Added:
SwiftApps/modis/conf/beagle-ssh.cf
SwiftApps/modis/tutorial/hello.swift
Modified:
SwiftApps/modis/README
SwiftApps/modis/conf/beagle.cf
SwiftApps/modis/conf/local.cf
SwiftApps/modis/conf/midway.cf
SwiftApps/modis/demo
SwiftApps/modis/tutorial/modis01.swift
SwiftApps/modis/tutorial/modis02.swift
SwiftApps/modis/tutorial/modis03.swift
SwiftApps/modis/tutorial/modis04.swift
Log:
Some simplifications of the app calls
hello world script
beagle-ssh config
Some more info in README
Modified: SwiftApps/modis/README
===================================================================
--- SwiftApps/modis/README 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/README 2013-02-15 10:36:37 UTC (rev 6276)
@@ -1,15 +1,63 @@
-tutorial/modis01.swift
+Setup
-----
-Runs a single landuse on h00v09.pgm.gz. Creates landuse/h00v08.landuse.byfreq as output
+To install modis, use SVN to check out the required scripts:
-tutorial/modis02.swift
------
-Runs landuse on all data files. Creates multiple landuse/*.byfreq files.
+$ svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/modis
-tutorial/modis03.swift
------
-Runs analyzelanduse on output data. Generates topselected.txt and selectedtiles.txt
+Once you do this, you all also need the associated modis data.
+To download the images, run:
+$ cd modis/data/modis/2002
+$ wget http://www.ci.uchicago.edu/swift/modis/modis-2002.tar
+$ tar xf modis-2002.tar
+
+Hello world
+-----------
+The script tutorial/hello.swift demonstrates a simple "hello world"
+script. It demonstrates how to define apps, passing arguments to
+an app, and how to work with files. In this example, a file called
+hello.txt gets created with with the contents of "Hello, world!".
+
+To run the application, the command is:
+
+$ swiftrun -site local tutorial/hello.swift
+
+modis01.swift
+-------------
+The first modis example defines one app function called getLandUse.
+This app takes a sallite image (data/modis/2002/h00v09.pgm.gz) as
+input. getLandUse creates a text file called landuse/h00v08.landuse.byfreq
+that counts the frequency of each land type defined in the input image.
+
+To run modis01.swift:
+
+$ swiftrun -site local tutorial/modis01.swift
+
+modis02.swift
+-------------
+The second modis example expands upon the first example by running getLandUse
+with multiple (317) input files. Ouptut files are stored in the landuse directory.
+
+To run modis02.swift:
+
+$ swiftrun -site local tutorial/modis02.swift
+
+modis03.swift
+-------------
+The third modis example builds on the previous example. It defines a new app function
+called analyzeLandUse. The analyzeLandUse app examines the data generated by getLandUse
+and creates two summary files called topselected.txt and selectedtiles.txt. These
+files contain information about the top 10 urban areas.
+
+In the previous example, you have have noticed that running all 317 input files on
+your laptop, even with 4 tasks a time, is not very efficient. In the next example,
+instead of running locally, we will use a cluster called midway at the University of
+Chicago to improve performance.
+
+To run modis03.swift:
+
+$ swiftrun -site midway tutorial/modis03.swift
+
tutorial/modis04.swift
-----
Also creates a map with top section selected
Added: SwiftApps/modis/conf/beagle-ssh.cf
===================================================================
--- SwiftApps/modis/conf/beagle-ssh.cf (rev 0)
+++ SwiftApps/modis/conf/beagle-ssh.cf 2013-02-15 10:36:37 UTC (rev 6276)
@@ -0,0 +1,15 @@
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+status.mode=provider
+use.provider.staging=true
+provider.staging.pin.swiftfiles=false
+
+#site beagle-ssh WALLTIME=00:05:00
+
+#app getlanduse=/lustre/beagle/davidk/modis/bin/getlanduse.sh
+#app analyzelanduse=/lustre/beagle/davidk/modis/bin/analyzelanduse.sh
+#app colormodis=/lustre/beagle/davidk/modis/bin/colormodis.sh
+#app assemble=/lustre/beagle/davidk/modis/bin/assemble2.sh
+#app markmap=/lustre/beagle/davidk/modis/bin/markmap.sh
+#app echo=/bin/echo
Modified: SwiftApps/modis/conf/beagle.cf
===================================================================
--- SwiftApps/modis/conf/beagle.cf 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/conf/beagle.cf 2013-02-15 10:36:37 UTC (rev 6276)
@@ -12,3 +12,4 @@
#app colormodis=$PWD/../bin/colormodis.sh
#app assemble=$PWD/../bin/assemble2.sh
#app markmap=$PWD/../bin/markmap.sh
+#app echo=/bin/echo
Modified: SwiftApps/modis/conf/local.cf
===================================================================
--- SwiftApps/modis/conf/local.cf 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/conf/local.cf 2013-02-15 10:36:37 UTC (rev 6276)
@@ -12,4 +12,4 @@
#app colormodis=$PWD/../bin/colormodis.sh
#app assemble=$PWD/../bin/assemble2.sh
#app markmap=$PWD/../bin/markmap.sh
-
+#app echo=/bin/echo
Modified: SwiftApps/modis/conf/midway.cf
===================================================================
--- SwiftApps/modis/conf/midway.cf 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/conf/midway.cf 2013-02-15 10:36:37 UTC (rev 6276)
@@ -13,3 +13,4 @@
#app colormodis=$PWD/../bin/colormodis.sh
#app assemble=$PWD/../bin/assemble2.sh
#app markmap=$PWD/../bin/markmap.sh
+#app echo=/bin/echo
Modified: SwiftApps/modis/demo
===================================================================
--- SwiftApps/modis/demo 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/demo 2013-02-15 10:36:37 UTC (rev 6276)
@@ -49,13 +49,14 @@
echo
echo "Available sites"
prompt="Pick an option: "
-options=("local" "midway" "beagle")
+options=("local" "midway" "beagle" "beagle-ssh")
PS3=$prompt
select opt in "${options[@]}" "Quit"; do
case "$REPLY" in
1) SITE="local"; break;;
2) SITE="midway"; break;;
3) SITE="beagle"; break;;
+ 4) SITE="beagle-ssh"; break;;
$(( ${#options[@]}+1 )) ) echo "Goodbye!"; break;;
*) echo "Invalid option. Try another one.";continue;;
esac
@@ -69,16 +70,13 @@
echo "Select script"
PS3="$prompt "
select opt in "${options[@]}" "Quit"; do
-
case "$REPLY" in
-
- 1 ) run tutorial/modis01.swift; break;;
- 2 ) run tutorial/modis02.swift; break;;
- 3 ) run tutorial/modis03.swift; break;;
- 4 ) run tutorial/modis04.swift; break;;
- $(( ${#options[@]}+1 )) ) echo "Goodbye!"; break;;
- *) echo "Invalid option. Try another one.";continue;;
-
+ 1 ) run tutorial/modis01.swift; break;;
+ 2 ) run tutorial/modis02.swift; break;;
+ 3 ) run tutorial/modis03.swift; break;;
+ 4 ) run tutorial/modis04.swift; break;;
+ $(( ${#options[@]}+1 )) ) echo "Goodbye!"; break;;
+ *) echo "Invalid option. Try another one.";continue;;
esac
done
Added: SwiftApps/modis/tutorial/hello.swift
===================================================================
--- SwiftApps/modis/tutorial/hello.swift (rev 0)
+++ SwiftApps/modis/tutorial/hello.swift 2013-02-15 10:36:37 UTC (rev 6276)
@@ -0,0 +1,8 @@
+type file;
+
+app (file f) greeting() {
+ echo "Hello, world!" stdout=@filename(f);
+}
+
+file outfile <"hello.txt">;
+outfile = greeting();
Modified: SwiftApps/modis/tutorial/modis01.swift
===================================================================
--- SwiftApps/modis/tutorial/modis01.swift 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/tutorial/modis01.swift 2013-02-15 10:36:37 UTC (rev 6276)
@@ -1,14 +1,11 @@
type imagefile;
type landuse;
-app (landuse output) getLandUse (imagefile input, int sortfield, string mpath)
+app (landuse output) getLandUse (imagefile input)
{
- getlanduse @input sortfield mpath stdout=@output;
+ getlanduse @filename(input) stdout=@filename(output);
}
-# Command line arguments
-string MODISdir = @arg("modisdir");
-
imagefile modisImage <"../data/modis/2002/h00v09.pgm.gz">;
landuse result <"landuse/h00v08.landuse.byfreq">;
-result = getLandUse(modisImage, 1, MODISdir);
+result = getLandUse(modisImage);
Modified: SwiftApps/modis/tutorial/modis02.swift
===================================================================
--- SwiftApps/modis/tutorial/modis02.swift 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/tutorial/modis02.swift 2013-02-15 10:36:37 UTC (rev 6276)
@@ -1,14 +1,14 @@
type imagefile;
type landuse;
-app (landuse output) getLandUse (imagefile input, int sortfield, string mpath)
+app (landuse output) getLandUse (imagefile input)
{
- getlanduse @input sortfield mpath stdout=@output;
+ getlanduse @filenames(input) stdout=@filename(output);
}
# Constants and command line arguments
-int nFiles = @toInt(@arg("nfiles"));
-string MODISdir = @arg("modisdir");
+int nFiles = @toInt(@arg("nfiles", "1000"));
+string MODISdir = @arg("modisdir", "../data/modis/2002");
# Input Dataset
imagefile geos[] <ext; exec="../bin/modis.mapper", location=MODISdir, suffix=".pgm.gz", n=nFiles>;
@@ -17,6 +17,6 @@
landuse land[] <structured_regexp_mapper; source=geos, match="(h..v..)", transform=@strcat("landuse/\\1.landuse.byfreq")>;
foreach g,i in geos {
- land[i] = getLandUse(g,1,MODISdir);
+ land[i] = getLandUse(g);
}
Modified: SwiftApps/modis/tutorial/modis03.swift
===================================================================
--- SwiftApps/modis/tutorial/modis03.swift 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/tutorial/modis03.swift 2013-02-15 10:36:37 UTC (rev 6276)
@@ -2,9 +2,9 @@
type imagefile;
type landuse;
-app (landuse output) getLandUse (imagefile input, int sortfield, string mpath)
+app (landuse output) getLandUse (imagefile input)
{
- getlanduse @input sortfield mpath stdout=@output;
+ getlanduse @filename(input) stdout=@filename(output);
}
app (file output, file tilelist) analyzeLandUse (landuse input[], string usetype, int maxnum)
@@ -13,11 +13,10 @@
}
# Constants and command line arguments
-int nFiles = @toInt(@arg("nfiles"));
-int nSelect = @toInt(@arg("nselect"));
-string landType = @arg("landtype");
-string MODISdir = @arg("modisdir");
-string webDir = @arg("webdir");
+int nFiles = @toInt(@arg("nfiles", "1000"));
+int nSelect = @toInt(@arg("nselect", "10"));
+string landType = @arg("landtype", "urban");
+string MODISdir = @arg("modisdir", "../data/modis/2002");
# Input Dataset
imagefile geos[] <ext; exec="../bin/modis.mapper", location=MODISdir, suffix=".pgm.gz", n=nFiles>;
@@ -26,7 +25,7 @@
landuse land[] <structured_regexp_mapper; source=geos, match="(h..v..)", transform=@strcat("landuse/\\1.landuse.byfreq")>;
foreach g,i in geos {
- land[i] = getLandUse(g,1,MODISdir);
+ land[i] = getLandUse(g);
}
# Find the top N tiles (by total area of selected landuse types)
Modified: SwiftApps/modis/tutorial/modis04.swift
===================================================================
--- SwiftApps/modis/tutorial/modis04.swift 2013-02-15 10:34:41 UTC (rev 6275)
+++ SwiftApps/modis/tutorial/modis04.swift 2013-02-15 10:36:37 UTC (rev 6276)
@@ -2,9 +2,9 @@
type imagefile;
type landuse;
-app (landuse output) getLandUse (imagefile input, int sortfield, string mpath)
+app (landuse output) getLandUse (imagefile input)
{
- getlanduse @input sortfield mpath stdout=@output;
+ getlanduse @filename(input) stdout=@filename(output);
}
app (file output, file tilelist) analyzeLandUse (landuse input[], string usetype, int maxnum)
@@ -22,7 +22,6 @@
int nSelect = @toInt(@arg("nselect"));
string landType = @arg("landtype");
string MODISdir = @arg("modisdir");
-string webDir = @arg("webdir");
# Input Dataset
imagefile geos[] <ext; exec="../bin/modis.mapper", location=MODISdir, suffix=".pgm.gz", n=nFiles>;
@@ -31,7 +30,7 @@
landuse land[] <structured_regexp_mapper; source=geos, match="(h..v..)", transform=@strcat("landuse/\\1.landuse.byfreq")>;
foreach g,i in geos {
- land[i] = getLandUse(g,1,MODISdir);
+ land[i] = getLandUse(g);
}
# Find the top N tiles (by total area of selected landuse types)
More information about the Swift-commit
mailing list