[Swift-commit] r5489 - SwiftApps/modis
davidk at ci.uchicago.edu
davidk at ci.uchicago.edu
Sat Jan 7 01:48:37 CST 2012
Author: davidk
Date: 2012-01-07 01:48:36 -0600 (Sat, 07 Jan 2012)
New Revision: 5489
Added:
SwiftApps/modis/cf.local
SwiftApps/modis/cf.pads
SwiftApps/modis/demo.local
SwiftApps/modis/demo.pads
SwiftApps/modis/modis.swift
SwiftApps/modis/sites.pads.xml
SwiftApps/modis/tc.pads
Removed:
SwiftApps/modis/cf
SwiftApps/modis/demo5
SwiftApps/modis/demo5.local
SwiftApps/modis/modis5.swift
SwiftApps/modis/selectedtiles.txt
SwiftApps/modis/tc
Log:
Some updates to run modis on pads
Deleted: SwiftApps/modis/cf
===================================================================
--- SwiftApps/modis/cf 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/cf 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,9 +0,0 @@
-wrapperlog.always.transfer=true
-sitedir.keep=true
-execution.retries=0
-lazy.errors=false
-status.mode=provider
-use.provider.staging=false
-provider.staging.pin.swiftfiles=false
-#throttle.transfers=16
-#throttle.file.operations=16
Added: SwiftApps/modis/cf.local
===================================================================
--- SwiftApps/modis/cf.local (rev 0)
+++ SwiftApps/modis/cf.local 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,9 @@
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+lazy.errors=false
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
+#throttle.transfers=16
+#throttle.file.operations=16
Added: SwiftApps/modis/cf.pads
===================================================================
--- SwiftApps/modis/cf.pads (rev 0)
+++ SwiftApps/modis/cf.pads 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,7 @@
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+lazy.errors=false
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
Added: SwiftApps/modis/demo.local
===================================================================
--- SwiftApps/modis/demo.local (rev 0)
+++ SwiftApps/modis/demo.local 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,15 @@
+# demo4 forest 50 5 pbs
+
+landtype=${1:-urban}
+nfiles=${2:-999}
+nselect=${3:-10}
+site=${4:-local}
+
+runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
+
+echo runid=$runid
+
+swift -tc.file tc.local \
+ -sites.file sites.local.xml \
+ modis.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
+ -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo
Property changes on: SwiftApps/modis/demo.local
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/modis/demo.pads
===================================================================
--- SwiftApps/modis/demo.pads (rev 0)
+++ SwiftApps/modis/demo.pads 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,15 @@
+# demo4 forest 50 5 pbs
+
+landtype=${1:-urban}
+nfiles=${2:-999}
+nselect=${3:-10}
+site=${4:-local}
+
+runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
+
+echo runid=$runid
+
+swift -tc.file tc.pads \
+ -sites.file sites.pads.xml \
+ modis.swift -config cf.pads -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
+ -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo
Property changes on: SwiftApps/modis/demo.pads
___________________________________________________________________
Added: svn:executable
+ *
Deleted: SwiftApps/modis/demo5
===================================================================
--- SwiftApps/modis/demo5 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/demo5 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,18 +0,0 @@
-# demo4 forest 50 5 pbs
-
-landtype=${1:-urban}
-nfiles=${2:-50}
-nselect=${3:-10}
-site=${4:-local}
-
-runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
-
-echo runid=$runid
-
-swift -tc.file tc.$site \
- -sites.file sites.xml \
- modis5.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
- -MODISdir=/home/wilde/bigdata/data/modis/2002 -webdir=/home/wilde/public_html/geo
-exit
-
-swift -tc.file tc.local -sites.file sites.local.xml modis2.swift -landtype=$1 -nfiles=$2 -nselect=$3
Deleted: SwiftApps/modis/demo5.local
===================================================================
--- SwiftApps/modis/demo5.local 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/demo5.local 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,15 +0,0 @@
-# demo4 forest 50 5 pbs
-
-landtype=${1:-urban}
-nfiles=${2:-999}
-nselect=${3:-10}
-site=${4:-local}
-
-runid=modis-$(date +%Y.%m%d.%H%M)-$landtype-$nfiles-$nselect
-
-echo runid=$runid
-
-swift -tc.file tc.local \
- -sites.file sites.local.xml \
- modis5.swift -landtype=$landtype -nfiles=$nfiles -nselect=$nselect -runid=$runid \
- -modisdir=$PWD/data/modis/2002 -webdir=$HOME/public_html/geo
Copied: SwiftApps/modis/modis.swift (from rev 5488, SwiftApps/modis/modis5.swift)
===================================================================
--- SwiftApps/modis/modis.swift (rev 0)
+++ SwiftApps/modis/modis.swift 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,93 @@
+type file;
+type imagefile;
+type landuse;
+
+# Define application program interfaces
+
+app (landuse output) getLandUse (imagefile input, int sortfield)
+{
+ getlanduse @input sortfield stdout=@output ;
+}
+
+app (file output, file tilelist) analyzeLandUse
+ (landuse input[], string usetype, int maxnum)
+{
+ analyzelanduse @output @tilelist usetype maxnum @filenames(input);
+}
+
+app (imagefile output) colorMODIS (imagefile input)
+{
+ colormodis @input @output;
+}
+
+app (imagefile output) assemble
+ (file selected, imagefile image[], string webdir)
+{
+ assemble @output @selected @filename(image[0]) webdir;
+}
+
+app (imagefile grid) markMap (file tilelist)
+{
+ markmap @tilelist @grid;
+}
+
+# Constants and command line arguments
+
+int nFiles = @toint(@arg("nfiles","1000"));
+int nSelect = @toint(@arg("nselect","12"));
+string landType = @arg("landtype","urban");
+string runID = @arg("runid","modis-run");
+string MODISdir= @arg("modisdir","/home/wilde/bigdata/data/modis/2002");
+string webDir = @arg("webdir","/home/wilde/public_html/geo/");
+
+string suffix=".tif";
+
+# Input Dataset
+
+imagefile geos[] <ext; exec="modis.mapper",
+ location=MODISdir, suffix=".tif", n=nFiles >; # site=site
+
+# Compute the land use summary of each MODIS tile
+
+landuse land[] <structured_regexp_mapper; source=geos, match="(h..v..)",
+ transform=@strcat(runID,"/\\1.landuse.byfreq")>;
+
+foreach g,i in geos {
+ land[i] = getLandUse(g,1);
+}
+
+# Find the top N tiles (by total area of selected landuse types)
+
+file topSelected<"topselected.txt">;
+file selectedTiles<"selectedtiles.txt">;
+(topSelected, selectedTiles) = analyzeLandUse(land, landType, nSelect);
+
+# Mark the top N tiles on a sinusoidal gridded map
+
+imagefile gridMap<"markedGrid.gif">;
+gridMap = markMap(topSelected);
+
+# Create multi-color images for all tiles
+
+imagefile colorImage[] <structured_regexp_mapper;
+ source=geos, match="(h..v..)",
+ transform="landuse/\\1.color.png">;
+
+foreach g, i in geos {
+ colorImage[i] = colorMODIS(g);
+}
+
+# Assemble a montage of the top selected areas
+
+imagefile montage <single_file_mapper; file=@strcat(runID,"/","map.png") >; # @arg
+montage = assemble(selectedTiles,colorImage,webDir);
+
+# future args:
+
+int selectHiThreshold;
+int selectLowThreshold;
+string upperLeftTile;
+string lowerRightTile;
+float imageSizes[];
+string displayOptions;
+
Deleted: SwiftApps/modis/modis5.swift
===================================================================
--- SwiftApps/modis/modis5.swift 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/modis5.swift 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,93 +0,0 @@
-type file;
-type imagefile;
-type landuse;
-
-# Define application program interfaces
-
-app (landuse output) getLandUse (imagefile input, int sortfield)
-{
- getlanduse @input sortfield stdout=@output ;
-}
-
-app (file output, file tilelist) analyzeLandUse
- (landuse input[], string usetype, int maxnum)
-{
- analyzelanduse @output @tilelist usetype maxnum @filenames(input);
-}
-
-app (imagefile output) colorMODIS (imagefile input)
-{
- colormodis @input @output;
-}
-
-app (imagefile output) assemble
- (file selected, imagefile image[], string webdir)
-{
- assemble @output @selected @filename(image[0]) webdir;
-}
-
-app (imagefile grid) markMap (file tilelist)
-{
- markmap @tilelist @grid;
-}
-
-# Constants and command line arguments
-
-int nFiles = @toint(@arg("nfiles","1000"));
-int nSelect = @toint(@arg("nselect","12"));
-string landType = @arg("landtype","urban");
-string runID = @arg("runid","modis-run");
-string MODISdir= @arg("modisdir","/home/wilde/bigdata/data/modis/2002");
-string webDir = @arg("webdir","/home/wilde/public_html/geo/");
-
-string suffix=".tif";
-
-# Input Dataset
-
-imagefile geos[] <ext; exec="modis.mapper",
- location=MODISdir, suffix=".tif", n=nFiles >; # site=site
-
-# Compute the land use summary of each MODIS tile
-
-landuse land[] <structured_regexp_mapper; source=geos, match="(h..v..)",
- transform=@strcat(runID,"/\\1.landuse.byfreq")>;
-
-foreach g,i in geos {
- land[i] = getLandUse(g,1);
-}
-
-# Find the top N tiles (by total area of selected landuse types)
-
-file topSelected<"topselected.txt">;
-file selectedTiles<"selectedtiles.txt">;
-(topSelected, selectedTiles) = analyzeLandUse(land, landType, nSelect);
-
-# Mark the top N tiles on a sinusoidal gridded map
-
-imagefile gridMap<"markedGrid.gif">;
-gridMap = markMap(topSelected);
-
-# Create multi-color images for all tiles
-
-imagefile colorImage[] <structured_regexp_mapper;
- source=geos, match="(h..v..)",
- transform="landuse/\\1.color.png">;
-
-foreach g, i in geos {
- colorImage[i] = colorMODIS(g);
-}
-
-# Assemble a montage of the top selected areas
-
-imagefile montage <single_file_mapper; file=@strcat(runID,"/","map.png") >; # @arg
-montage = assemble(selectedTiles,colorImage,webDir);
-
-# future args:
-
-int selectHiThreshold;
-int selectLowThreshold;
-string upperLeftTile;
-string lowerRightTile;
-float imageSizes[];
-string displayOptions;
-
Deleted: SwiftApps/modis/selectedtiles.txt
===================================================================
--- SwiftApps/modis/selectedtiles.txt 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/selectedtiles.txt 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,10 +0,0 @@
-/home/wilde/bigdata/data/modis/2002//h30v07.tif
-/home/wilde/bigdata/data/modis/2002//h08v04.tif
-/home/wilde/bigdata/data/modis/2002//h20v12.tif
-/home/wilde/bigdata/data/modis/2002//h19v12.tif
-/home/wilde/bigdata/data/modis/2002//h13v13.tif
-/home/wilde/bigdata/data/modis/2002//h21v10.tif
-/home/wilde/bigdata/data/modis/2002//h10v10.tif
-/home/wilde/bigdata/data/modis/2002//h08v06.tif
-/home/wilde/bigdata/data/modis/2002//h10v08.tif
-/home/wilde/bigdata/data/modis/2002//h24v05.tif
Added: SwiftApps/modis/sites.pads.xml
===================================================================
--- SwiftApps/modis/sites.pads.xml (rev 0)
+++ SwiftApps/modis/sites.pads.xml 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,16 @@
+<config>
+ <pool handle="pads">
+ <execution provider="coaster" jobmanager="local:pbs" url="login.pads.ci.uchicago.edu" />
+ <filesystem provider="local" />
+ <workdirectory>/autonfs/gpfs-pads/projects/CI-CCR000013/davidk/swiftwork</workdirectory>
+ <profile namespace="globus" key="maxtime">3600</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="slots">64</profile> <!-- Max number of jobs for the fast queue on PADS => 192 -->
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="queue">fast</profile>
+ <profile namespace="karajan" key="jobThrottle">.6399</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="env" key="SWIFT_GEN_SCRIPTS">KEEP</profile>
+ </pool>
+</config>
Deleted: SwiftApps/modis/tc
===================================================================
--- SwiftApps/modis/tc 2012-01-06 21:39:41 UTC (rev 5488)
+++ SwiftApps/modis/tc 2012-01-07 07:48:36 UTC (rev 5489)
@@ -1,34 +0,0 @@
-#This is the transformation catalog.
-#
-#It comes pre-configured with a number of simple transformations with
-#paths that are likely to work on a linux box. However, on some systems,
-#the paths to these executables will be different (for example, sometimes
-#some of these programs are found in /usr/bin rather than in /bin)
-#
-#NOTE WELL: fields in this file must be separated by tabs, not spaces; and
-#there must be no trailing whitespace at the end of each line.
-#
-# site transformation path obsolete fields for compatibility
-
-localhost echo /bin/echo null null null
-localhost cat /bin/cat null null null
-localhost ls /bin/ls null null null
-localhost grep /bin/grep null null null
-localhost sort /bin/sort null null null
-localhost paste /bin/paste null null null
-localhost pwd /bin/pwd null null null
-
-# For cluster usage
-
-pbs convert /usr/bin/convert null null null
-pbs getlanduse /home/wilde/swift/demo/modis/bin/getlanduse.sh null null null
-pbs analyzelanduse /home/wilde/swift/demo/modis/bin/analyzelandusep1.sh null null null
-pbs colormodis /home/wilde/swift/demo/modis/bin/colormodis.sh null null null
-pbs assemble /home/wilde/swift/demo/modis/bin/assemble.sh null null null
-
-# For localhost testing
-
-#localhost convert /usr/bin/convert null null null
-#localhost getlanduse /home/wilde/bigdata/bin/getlanduse.sh null null null
-#localhost analyzelanduse /home/wilde/bigdata/bin/analyzelanduse.sh null null null
-#localhost colormodis /home/wilde/bigdata/bin/colormodis.sh null null null
Added: SwiftApps/modis/tc.pads
===================================================================
--- SwiftApps/modis/tc.pads (rev 0)
+++ SwiftApps/modis/tc.pads 2012-01-07 07:48:36 UTC (rev 5489)
@@ -0,0 +1,26 @@
+#This is the transformation catalog.
+#
+#It comes pre-configured with a number of simple transformations with
+#paths that are likely to work on a linux box. However, on some systems,
+#the paths to these executables will be different (for example, sometimes
+#some of these programs are found in /usr/bin rather than in /bin)
+#
+#NOTE WELL: fields in this file must be separated by tabs, not spaces; and
+#there must be no trailing whitespace at the end of each line.
+#
+# site transformation path obsolete fields for compatibility
+
+localhost echo /bin/echo null null null
+localhost cat /bin/cat null null null
+localhost ls /bin/ls null null null
+localhost grep /bin/grep null null null
+localhost sort /bin/sort null null null
+localhost paste /bin/paste null null null
+localhost pwd /bin/pwd null null null
+
+# For cluster usage
+pads getlanduse /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/getlanduse.sh null null null
+pads analyzelanduse /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/analyzelanduse2.sh null null null
+pads colormodis /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/colormodis.sh null null null
+pads assemble /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/assemble2.sh null null null
+pads markmap /autonfs/gpfs-pads/projects/CI-CCR000013/davidk/modis/bin/markmap.sh null null null
More information about the Swift-commit
mailing list