[Swift-commit] r7693 - in branches/release-0.94/tests: groups sites stress/apps stress/apps/modis_osgc
davidk at ci.uchicago.edu
davidk at ci.uchicago.edu
Tue Mar 11 14:22:25 CDT 2014
Author: davidk
Date: 2014-03-11 14:22:25 -0500 (Tue, 11 Mar 2014)
New Revision: 7693
Added:
branches/release-0.94/tests/stress/apps/modis_osgc/
branches/release-0.94/tests/stress/apps/modis_osgc/clean
branches/release-0.94/tests/stress/apps/modis_osgc/getlanduse.pl
branches/release-0.94/tests/stress/apps/modis_osgc/input/
branches/release-0.94/tests/stress/apps/modis_osgc/makeinput
branches/release-0.94/tests/stress/apps/modis_osgc/modis.args
branches/release-0.94/tests/stress/apps/modis_osgc/modis.kml
branches/release-0.94/tests/stress/apps/modis_osgc/modis.setup.sh
branches/release-0.94/tests/stress/apps/modis_osgc/modis.source.sh
branches/release-0.94/tests/stress/apps/modis_osgc/modis.swift
branches/release-0.94/tests/stress/apps/modis_osgc/modis.timeout
branches/release-0.94/tests/stress/apps/modis_osgc/run
branches/release-0.94/tests/stress/apps/modis_osgc/sites.template.xml
branches/release-0.94/tests/stress/apps/modis_osgc/swift.properties
branches/release-0.94/tests/stress/apps/modis_osgc/tc.template.data
branches/release-0.94/tests/stress/apps/modis_osgc/title.txt
Removed:
branches/release-0.94/tests/sites/uc3/
branches/release-0.94/tests/stress/apps/modis_uc3/
Modified:
branches/release-0.94/tests/groups/group-apps.sh
Log:
uc3-sub is being taken down and no longer functioning, use osgconnect instead
Modified: branches/release-0.94/tests/groups/group-apps.sh
===================================================================
--- branches/release-0.94/tests/groups/group-apps.sh 2014-03-11 18:44:06 UTC (rev 7692)
+++ branches/release-0.94/tests/groups/group-apps.sh 2014-03-11 19:22:25 UTC (rev 7693)
@@ -5,8 +5,8 @@
$TESTDIR/stress/apps/modis_beagle \
$TESTDIR/stress/apps/modis_local \
$TESTDIR/stress/apps/modis_midway \
- $TESTDIR/stress/apps/modis_uc3 \
- $TESTDIR/stress/apps/modis_multiple\
+ $TESTDIR/stress/apps/modis_osgc \
+ $TESTDIR/stress/apps/modis_multiple \
)
checkvars WORK
Added: branches/release-0.94/tests/stress/apps/modis_osgc/clean
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/clean (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/clean 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,3 @@
+#! /bin/sh
+
+rm -rf *~ *log *.d ../*.{kml,swiftx} *.out landuse
Property changes on: branches/release-0.94/tests/stress/apps/modis_osgc/clean
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.94/tests/stress/apps/modis_osgc/getlanduse.pl
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/getlanduse.pl (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/getlanduse.pl 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,29 @@
+#!/usr/bin/perl -w
+
+# Input to this program should be a raw, greyscale RGB file
+# Usage: rgb_histogram.pl myfile.rgb
+
+my $image_filename = shift;
+open(IMAGEFILE, "$image_filename") || die "Unable to open $image_filename!\n";
+binmode IMAGEFILE;
+
+my @pixelcount;
+foreach my $count (0..255) { $pixelcount[$count] = 0; }
+
+# Read values, three bytes at a time
+$/ = \3;
+foreach(<IMAGEFILE>) {
+ $pixelcount[unpack('C', $_)]++;
+}
+close(IMAGEFILE);
+
+ at sortedcount = sort {$b <=> $a} @pixelcount;
+
+foreach my $count (0..255) {
+ if($sortedcount[$count] == 0) { next; }
+ printf("%d %d %02x\n", $sortedcount[$count], $count, $count);
+}
+
+system("hostname -f 2>&1");
+
+sleep(1);
Property changes on: branches/release-0.94/tests/stress/apps/modis_osgc/getlanduse.pl
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.94/tests/stress/apps/modis_osgc/makeinput
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/makeinput (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/makeinput 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,22 @@
+#! /bin/sh
+
+nfiles=${1:-10}
+
+rm -rf input
+mkdir input
+cp ../data/t? input/
+( cd input
+ n=0
+ for h in $(seq -w 00 99); do
+ for v in $(seq -w 00 99); do
+ n=$((n+1))
+ if [ $n -gt $nfiles ]; then
+ break;
+ else
+ f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/')
+ ln $f h${h}v${v}.rgb
+ fi
+ done
+ done
+ rm t?
+)
Property changes on: branches/release-0.94/tests/stress/apps/modis_osgc/makeinput
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.94/tests/stress/apps/modis_osgc/modis.args
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/modis.args (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/modis.args 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1 @@
+-modisdir=input/
\ No newline at end of file
Added: branches/release-0.94/tests/stress/apps/modis_osgc/modis.kml
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/modis.kml (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/modis.kml 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,200 @@
+<project><!-- CACHE ID 5384750e-183f-421c-bdc7-502c4e357001-no-provenance -->
+ <import file="sys.xml"/>
+ <import file="scheduler.xml"/>
+ <import file="rlog.xml"/>
+ <import file="vdl.k"/>
+ <types>
+ <xs:schema targetNamespace="http://ci.uchicago.edu/swift/2009/02/swiftscript" xmlns="http://ci.uchicago.edu/swift/2009/02/swiftscript" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xs="http://www.w3.org/2001/XMLSchema">
+ <xs:simpleType name="imagefile">
+ <xs:restriction base="string"/>
+ </xs:simpleType>
+ <xs:simpleType name="landuse">
+ <xs:restriction base="string"/>
+ </xs:simpleType>
+ <xs:simpleType name="perlscript">
+ <xs:restriction base="string"/>
+ </xs:simpleType>
+ </xs:schema>
+ </types>
+ <global name="swift#string#17009">
+ <vdl:new type="string" value="landuse/\1.landuse.byfreq" />
+ </global>
+ <global name="swift#string#17002">
+ <vdl:new type="string" value="modisdir" />
+ </global>
+ <global name="swift#string#17007">
+ <vdl:new type="string" value="(h..v..)" />
+ </global>
+ <global name="swift#string#17000">
+ <vdl:new type="string" value="nfiles" />
+ </global>
+ <global name="swift#string#17003">
+ <vdl:new type="string" value="../data/modis/2002" />
+ </global>
+ <global name="swift#string#17005">
+ <vdl:new type="string" value=".rgb" />
+ </global>
+ <global name="swift#string#17001">
+ <vdl:new type="string" value="1000" />
+ </global>
+ <element name="get_land_use" arguments="output,input,ps" _defline="7">
+
+
+
+ <unitStart name="get_land_use" line="7" type="PROCEDURE" outputs="output"/>
+ <vdl:execute>
+ <vdl:tr>perl</vdl:tr>
+ <vdl:stagein var="{input}"/>
+ <vdl:stagein var="{ps}"/>
+ <vdl:stageout var="{output}"/>
+
+ <vdl:arguments>
+ <swiftscript:filename _traceline="9">
+ <variable>ps</variable>
+ </swiftscript:filename>
+ <swiftscript:filename _traceline="9">
+ <variable>input</variable>
+ </swiftscript:filename>
+ </vdl:arguments>
+ <vdl:stdout>
+ <swiftscript:filename _traceline="9">
+ <variable>output</variable>
+ </swiftscript:filename>
+ </vdl:stdout>
+ </vdl:execute>
+ <vdl:closedataset var="{output}"/>
+ <unitEnd name="get_land_use" line="7" type="PROCEDURE"/></element>
+
+ <set name="getlanduse_pl">
+ <vdl:new type="perlscript" dbgname="getlanduse_pl" _defline="5" input="true">
+ <vdl:mapping descriptor="single_file_mapper">
+ <vdl:parameter name="file" value="getlanduse.pl"/>
+ </vdl:mapping>
+ </vdl:new>
+ </set>
+
+ <set name="nFiles">
+ <vdl:new type="int" dbgname="nFiles" waitCount="1" _defline="13"/>
+ </set>
+
+ <set name="MODISdir">
+ <vdl:new type="string" dbgname="MODISdir" waitCount="1" _defline="14"/>
+ </set>
+
+ <set name="swift#mapper#17004">
+ <vdl:new type="string" dbgname="swift#mapper#17004" _defline=""/>
+ </set>
+
+ <set name="geos">
+ <vdl:new type="imagefile[]" dbgname="geos" _defline="17" input="true">
+ <vdl:mapping descriptor="filesys_mapper">
+ <vdl:parameter name="location"><variable>MODISdir</variable></vdl:parameter>
+ <vdl:parameter name="suffix"><vdl:new type="string" value=".rgb" /></vdl:parameter>
+ </vdl:mapping>
+ </vdl:new>
+ </set>
+
+ <set name="swift#mapper#17006">
+ <vdl:new type="string" dbgname="swift#mapper#17006" _defline=""/>
+ </set>
+
+ <set name="swift#mapper#17008">
+ <vdl:new type="string" dbgname="swift#mapper#17008" _defline=""/>
+ </set>
+
+ <set name="land">
+ <vdl:new type="landuse[]" dbgname="land" waitCount="1" _defline="20">
+ <vdl:mapping descriptor="structured_regexp_mapper">
+ <vdl:parameter name="source"><variable>geos</variable></vdl:parameter>
+ <vdl:parameter name="match"><vdl:new type="string" value="(h..v..)" /></vdl:parameter>
+ <vdl:parameter name="transform"><variable>swift#mapper#17008</variable></vdl:parameter>
+ </vdl:mapping>
+ </vdl:new>
+ </set>
+
+ <restartLog>
+ <vdl:mains>
+ <vdl:startprogressticker />
+ <vdl:mainp>
+ <uparallel>
+ <sequential>
+ <vdl:setfieldvalue _traceline="14">
+ <variable>nFiles</variable>
+ <swiftscript:to_int _traceline="13">
+ <swiftscript:arg _traceline="13">
+ <variable>swift#string#17000</variable><variable>swift#string#17001</variable>
+ </swiftscript:arg>
+ </swiftscript:to_int>
+ </vdl:setfieldvalue>
+ <partialCloseDataset var="{nFiles}"/>
+ </sequential>
+ <sequential>
+ <vdl:setfieldvalue _traceline="17">
+ <variable>MODISdir</variable>
+ <swiftscript:arg _traceline="14">
+ <variable>swift#string#17002</variable><variable>swift#string#17003</variable>
+ </swiftscript:arg>
+ </vdl:setfieldvalue>
+ <partialCloseDataset var="{MODISdir}"/>
+ </sequential>
+ <sequential>
+ <vdl:setfieldvalue _traceline="-1">
+ <variable>swift#mapper#17004</variable>
+ <variable>swift#string#17005</variable>
+ </vdl:setfieldvalue>
+
+ </sequential>
+ <sequential>
+ <vdl:setfieldvalue _traceline="-1">
+ <variable>swift#mapper#17006</variable>
+ <variable>swift#string#17007</variable>
+ </vdl:setfieldvalue>
+
+ </sequential>
+ <sequential>
+ <vdl:setfieldvalue _traceline="-1">
+ <variable>swift#mapper#17008</variable>
+ <swiftscript:strcat _traceline="21">
+ <variable>swift#string#17009</variable>
+ </swiftscript:strcat>
+ </vdl:setfieldvalue>
+
+ </sequential>
+ <sequential>
+ <vdl:tparallelFor name="$"_kvar="i" _vvar="g" refs="land 1">
+ <getarrayiterator>
+ <variable>geos</variable>
+ </getarrayiterator>
+ <set names="$$, g">
+ <each items="{$}"/>
+ </set>
+ <set name="i">
+ <vdl:new type="int" value="{$$}"/>
+ </set>
+ <unitStart line="23" type="FOREACH_IT"/>
+ <sequentialWithID>
+ <sequential>
+ <get_land_use _traceline="24">
+ <vdl:getfieldsubscript>
+ <argument name="var"><variable>land</variable></argument>
+ <argument name="subscript"><variable>i</variable></argument>
+ </vdl:getfieldsubscript>
+ <variable>g</variable>
+ <variable>getlanduse_pl</variable>
+ </get_land_use>
+ <partialCloseDataset var="{land}"/>
+ </sequential>
+ </sequentialWithID>
+ <unitEnd line="23" type="FOREACH_IT"/>
+ </vdl:tparallelFor>
+ </sequential>
+ </uparallel>
+ </vdl:mainp>
+ <vdl:stopprogressticker />
+ </vdl:mains>
+ </restartLog>
+ <vdl:cleandataset var="{geos}"/>
+ <vdl:cleandataset var="{getlanduse_pl}"/>
+ <vdl:cleandataset var="{land}"/>
+ <vdl:cleandataset shutdown="true"/>
+</project>
Added: branches/release-0.94/tests/stress/apps/modis_osgc/modis.setup.sh
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/modis.setup.sh (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/modis.setup.sh 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+USERNAME=$UC3_USERNAME
+
+if [[ -z $USERNAME ]]
+then
+ echo "Remote username not provided. Skipping sites configs"
+else
+ ls *xml
+ cat sites.xml | sed "s/{env.USER}/$USERNAME/" > tmp && mv tmp sites.xml
+fi
+
+case $STRESS in
+ "S1")
+ FILES=10
+ ;;
+ "S2")
+ FILES=1000
+ ;;
+ "S3")
+ FILES=10000
+ ;;
+ "S4")
+ FILES=10000
+ ;;
+ *)
+ FILES=1000
+ ;;
+esac
+
+nfiles=${FILES:-10}
+#OVERRIDE_GLOBUS_HOSTNAME "swift.rcc.uchicago.edu"
+rm -rf input
+mkdir input
+cp $(dirname $GROUP)/data/t? input/
+cp $GROUP/getlanduse.pl ./
+( cd input
+ n=0
+ for h in $(seq -w 00 99); do
+ for v in $(seq -w 00 99); do
+ n=$((n+1))
+ if [ $n -gt $nfiles ]; then
+ break;
+ else
+ f=t$(echo $RANDOM | sed -e 's/.*\(.\)/\1/')
+ ln $f h${h}v${v}.rgb
+ fi
+ done
+ done
+ rm t?
+)
+
+#makeinput
\ No newline at end of file
Property changes on: branches/release-0.94/tests/stress/apps/modis_osgc/modis.setup.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.94/tests/stress/apps/modis_osgc/modis.source.sh
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/modis.source.sh (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/modis.source.sh 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,6 @@
+#!/bin/bash
+if [ "midway001" == "midway001" ]
+then
+ export GLOBUS_HOSTNAME=swift.rcc.uchicago.edu
+ export GLOBUS_TCP_PORT_RANGE=50000,51000
+fi;
Property changes on: branches/release-0.94/tests/stress/apps/modis_osgc/modis.source.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.94/tests/stress/apps/modis_osgc/modis.swift
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/modis.swift (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/modis.swift 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,25 @@
+type imagefile;
+type landuse;
+type perlscript;
+
+perlscript getlanduse_pl <"getlanduse.pl">;
+
+app (landuse output) getLandUse (imagefile input, perlscript ps)
+{
+ perl @ps @filename(input) stdout=@filename(output);
+}
+
+# Constants and command line arguments
+int nFiles = @toInt(@arg("nfiles", "1000"));
+string MODISdir = @arg("modisdir", "../data/modis/2002");
+
+# Input Dataset
+imagefile geos[] <filesys_mapper; location=MODISdir, suffix=".rgb">;
+
+# Compute the land use summary of each MODIS tile
+landuse land[] <structured_regexp_mapper; source=geos, match="(h..v..)",
+ transform=@strcat("landuse/\\1.landuse.byfreq")>;
+
+foreach g,i in geos {
+ land[i] = getLandUse(g, getlanduse_pl);
+}
Added: branches/release-0.94/tests/stress/apps/modis_osgc/modis.timeout
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/modis.timeout (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/modis.timeout 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1 @@
+1800
Added: branches/release-0.94/tests/stress/apps/modis_osgc/run
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/run (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/run 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+set -x
+
+swift -config swift.properties -sites.file sites.template.xml -tc.file tc.template.data modis.swift -modisdir=input/
Property changes on: branches/release-0.94/tests/stress/apps/modis_osgc/run
___________________________________________________________________
Added: svn:executable
+ *
Added: branches/release-0.94/tests/stress/apps/modis_osgc/sites.template.xml
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/sites.template.xml (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/sites.template.xml 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config xmlns="http://www.ci.uchicago.edu/swift/SwiftSites">
+
+ <pool handle="osgc">
+ <execution provider="coaster" url="login01.osgconnect.net" jobmanager="ssh-cl:condor"/>
+ <profile namespace="karajan" key="jobThrottle">10.00</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobsPerNode">1</profile>
+ <profile namespace="globus" key="maxtime">3600</profile>
+ <profile namespace="globus" key="maxWalltime">00:30:00</profile>
+ <profile namespace="globus" key="highOverAllocation">100</profile>
+ <profile namespace="globus" key="lowOverAllocation">100</profile>
+ <profile namespace="globus" key="slots">5</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="jobType">nonshared</profile>
+ <profile namespace="globus" key="condor.+ProjectName">"Swift"</profile>
+ <workdirectory>.</workdirectory>
+ </pool>
+</config>
Added: branches/release-0.94/tests/stress/apps/modis_osgc/swift.properties
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/swift.properties (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/swift.properties 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1,8 @@
+
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=0
+lazy.errors=false
+status.mode=provider
+use.provider.staging=true
+provider.staging.pin.swiftfiles=false
Added: branches/release-0.94/tests/stress/apps/modis_osgc/tc.template.data
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/tc.template.data (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/tc.template.data 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1 @@
+osgc perl /usr/bin/perl null null null
Added: branches/release-0.94/tests/stress/apps/modis_osgc/title.txt
===================================================================
--- branches/release-0.94/tests/stress/apps/modis_osgc/title.txt (rev 0)
+++ branches/release-0.94/tests/stress/apps/modis_osgc/title.txt 2014-03-11 19:22:25 UTC (rev 7693)
@@ -0,0 +1 @@
+Modis UC3
More information about the Swift-commit
mailing list