[Swift-commit] r5732 - in SwiftApps/ParVis: . HiRAMTools
wilde at ci.uchicago.edu
wilde at ci.uchicago.edu
Fri Mar 30 11:26:08 CDT 2012
Author: wilde
Date: 2012-03-30 11:26:08 -0500 (Fri, 30 Mar 2012)
New Revision: 5732
Added:
SwiftApps/ParVis/HiRAMTools/
SwiftApps/ParVis/HiRAMTools/checkgroups.sh
SwiftApps/ParVis/HiRAMTools/combine.chk.sh
SwiftApps/ParVis/HiRAMTools/combine.orig.sh
SwiftApps/ParVis/HiRAMTools/combine.sh
SwiftApps/ParVis/HiRAMTools/combine.swift
SwiftApps/ParVis/HiRAMTools/combine2.swift
SwiftApps/ParVis/HiRAMTools/combine_realization.sh
SwiftApps/ParVis/HiRAMTools/find_anomalies.sh
SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh
SwiftApps/ParVis/HiRAMTools/get_real_copy.sh
SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh
SwiftApps/ParVis/HiRAMTools/makeyearly.sh
SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh
SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh
SwiftApps/ParVis/HiRAMTools/runall.sh
SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh
SwiftApps/ParVis/HiRAMTools/runcombine.sh
SwiftApps/ParVis/HiRAMTools/y1.sh
Log:
Initial version.
Added: SwiftApps/ParVis/HiRAMTools/checkgroups.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/checkgroups.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/checkgroups.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,21 @@
+#! /bin/bash
+
+# /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history/19500101/atmos_4xdaily.tile1.nc
+
+sed -e 's/^.*history.//' groups | sort >todo
+find /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016 -type f -print | fgrep .nc | sed -e 's/^.*run016.//' | sort >done
+
+echo
+echo diff: "<todo >done"
+echo
+diff todo done
+
+echo
+echo last 20 files produced were:
+echo
+ls -ltd /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016/*/* | head -20
+
+echo
+echo File lengths:
+echo
+ls -ltd /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016/*/* | awk '{print $5}' | sort | uniq -c
Property changes on: SwiftApps/ParVis/HiRAMTools/checkgroups.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/combine.chk.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine.chk.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/combine.chk.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,33 @@
+#! /bin/bash
+
+set -o errexit
+
+mkdir -p /scratch/wilde
+
+tmp=$(mktemp -d /scratch/wilde/combine_XXXXXX)
+
+echo mktemp error code: $?
+
+cd $tmp
+
+outdir=$1
+inpattern=$2
+date=$(basename $(dirname $inpattern))
+basepattern=$(basename $inpattern)
+
+infiles="$2*"
+for f in $infiles; do
+ dd if=$f of=$(basename $f) bs=4M
+done
+
+/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
+
+mkdir -p $outdir/$date
+dd if=out.nc of=$outdir/$date/$basepattern.trans bs=4M
+mv $outdir/$date/$basepattern.trans $outdir/$date/$basepattern
+
+cd ..
+rm -rf $tmp
+
+
+
Property changes on: SwiftApps/ParVis/HiRAMTools/combine.chk.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/combine.orig.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine.orig.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/combine.orig.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,28 @@
+#! /bin/bash
+
+set -x
+
+mkdir -p /scratch/wilde
+tmp=$(mktemp -d /scratch/wilde/combine_XXXX)
+cd $tmp
+
+outdir=$1
+inpattern=$2
+date=$(basename $(dirname $inpattern))
+basepattern=$(basename $inpattern)
+
+infiles="$2*"
+for f in $infiles; do
+ dd if=$f of=$(basename $f) bs=4M
+done
+
+/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
+
+mkdir -p $outdir/$date
+dd if=out.nc of=$outdir/$date/$basepattern bs=4M
+
+cd ..
+rm -rf $tmp
+
+
+
Property changes on: SwiftApps/ParVis/HiRAMTools/combine.orig.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/combine.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/combine.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,29 @@
+#! /bin/bash
+
+set -x
+
+mkdir -p /scratch/wilde
+tmp=$(mktemp -d /scratch/wilde/combine_XXXX)
+cd $tmp
+
+outdir=$1
+inpattern=$2
+date=$(basename $(dirname $inpattern))
+basepattern=$(basename $inpattern)
+
+infiles="$2*"
+for f in $infiles; do
+ dd if=$f of=$(basename $f) bs=4M
+done
+
+/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
+
+mkdir -p $outdir/$date
+dd if=out.nc of=$outdir/$date/$basepattern.trans bs=4M
+mv $outdir/$date/$basepattern.trans $outdir/$date/$basepattern
+
+cd ..
+rm -rf $tmp
+
+
+
Property changes on: SwiftApps/ParVis/HiRAMTools/combine.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/combine.swift
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine.swift (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/combine.swift 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,16 @@
+
+type file;
+
+string rundir=@arg("rundir","NO-OUTDIR-SPECIFIED");
+string groupfile=@arg("groupfile","groups");
+
+app combine (string dir, string pattern)
+{
+ combine dir pattern;
+}
+
+string group[] = readData(groupfile);
+
+foreach g in group {
+ combine(rundir,g);
+}
Added: SwiftApps/ParVis/HiRAMTools/combine2.swift
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine2.swift (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/combine2.swift 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,18 @@
+
+type file;
+
+string rundir=@arg("rundir","NO-OUTDIR-SPECIFIED");
+string groupfile=@arg("groupfile","groups");
+string realizatoinfile=@arg("ralizationsfile","realizations");
+
+app combine (string dir, string pattern)
+{
+ combine dir pattern;
+}
+
+string group[] = readData(groupfile);
+string reals[] = readData(realizationFile);
+
+foreach g in group {
+ combine(rundir,g);
+}
Added: SwiftApps/ParVis/HiRAMTools/combine_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine_realization.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,82 @@
+#! /bin/bash
+
+hdir=$1
+outdir=$2
+
+/bin/ls -1 -d ${hdir}* >dirs
+
+echo '=== ' $(basename $0) doing dirs:
+cat dirs
+
+for d in $(cat dirs); do
+ find $d | grep .nc.[0-9][0-9][0-9][0-9] | sed -e 's/.....$//' | sort | uniq
+done >groups
+
+# Create new runNNN directory
+
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+mkdir $rundir
+
+cat >$rundir/cf <<END
+
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=10
+lazy.errors=true
+
+END
+
+cat >$rundir/sites.xml <<END
+
+<config>
+
+ <pool handle="local">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">.01</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>$PWD/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="cobalt">
+ <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
+ <filesystem provider="local" />
+ <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
+ <profile namespace="globus" key="project">MTCScienceApps</profile>
+ <profile namespace="globus" key="queue">default</profile>
+ <profile namespace="karajan" key="jobthrottle">2.56</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobsPerNode">8</profile>
+ <!-- <profile namespace="globus" key="workerLoggingLevel">DEBUG</profile> -->
+ <!-- <profile namespace="globus" key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
+ <profile namespace="globus" key="slots">32</profile>
+ <profile namespace="globus" key="maxTime">28000</profile> <!-- ~ 8hrs -->
+ <profile namespace="globus" key="maxWallTime">00:20:00</profile>
+ <profile namespace="globus" key="lowoverallocation">100</profile>
+ <profile namespace="globus" key="highoverallocation">100</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
+ </pool>
+
+</config>
+
+END
+
+cat >$rundir/tc <<END
+
+#local combine $PWD/combine.sh
+cobalt combine $PWD/combine.sh
+
+END
+
+cp groups combine.swift $rundir
+
+cd $rundir
+echo Running in directory $rundir
+runid=$(basename $rundir)
+mkdir -p $outdir/$runid
+
+swift -config cf -tc.file tc -sites.file sites.xml combine.swift -rundir=$outdir/$runid -groupfile=groups >& swift.out
+
+
Property changes on: SwiftApps/ParVis/HiRAMTools/combine_realization.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/find_anomalies.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/find_anomalies.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/find_anomalies.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,18 @@
+#! /bin/sh
+
+reals=/intrepid-fs0/users/wilde/persistent/LZ/combined
+cd $reals
+
+for real in *; do
+ echo Realization: $real
+ ( cd $real
+ for y in *; do
+ echo " Year: " $y
+ ( cd $y
+ for f in *.nc; do
+ echo -n $f; ncdump -h $f | grep 'time =' || echo
+ done | awk '{print " ",$1,$7}' | grep 'atmos.*\.tile[0-9].* (1' | grep -v scalar
+ )
+ done
+ )
+done
Property changes on: SwiftApps/ParVis/HiRAMTools/find_anomalies.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,17 @@
+#! /bin/bash
+
+src=/intrepid-fs0/users/lzamboni/persistent
+dest=/intrepid-fs0/users/wilde/persistent/LZ/combined
+
+f=atmos_scalar.nc
+
+reals=$(cat /home/wilde/LZ/realizations)
+
+for r in $reals; do
+ cd $src/$r/history
+ years=$( /bin/ls -1d [0-9]??????? )
+ for y in $years; do
+ cp $y/$f $dest/$r/$y
+ done
+done
+
Property changes on: SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/get_real_copy.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/get_real_copy.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/get_real_copy.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,26 @@
+#! /bin/bash
+
+# src=/intrepid-fs0/users/lzamboni/persistent
+
+src=/intrepid-fs0/users/wilde/persistent/LZ/combined
+dest=$PWD
+
+real=$1
+
+if [ _$1 = _ ]; then
+ echo realization to copy was not specified - exiting; exit 1
+fi
+
+if [ -e _$1 ]; then
+ echo $1 already exits - exiting; exit 1
+fi
+
+cd $src/$real
+dirs=$(find * -type d -print)
+
+for d in $dirs; do
+ mkdir -p $dest/$real/$d
+ for f in $d/*.nc; do
+ ln -s $PWD/$f $dest/$real/$f
+ done
+done
Property changes on: SwiftApps/ParVis/HiRAMTools/get_real_copy.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,26 @@
+#! /bin/bash
+
+# src=/intrepid-fs0/users/lzamboni/persistent
+
+src=/intrepid-fs0/users/wilde/persistent/LZ/combined
+dest=$PWD
+
+real=$1
+
+if [ _$1 = _ ]; then
+ echo realization to copy was not specified - exiting; exit 1
+fi
+
+if [ -e _$1 ]; then
+ echo $1 already exits - exiting; exit 1
+fi
+
+cd $src/$real
+dirs=$(find * -type d -print)
+
+for d in $dirs; do
+ mkdir -p $dest/$real/history/$d
+ for f in $d/*.nc; do
+ ln -s $PWD/$f $dest/$real/history/$f
+ done
+done
Property changes on: SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/makeyearly.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,231 @@
+#! /bin/bash
+
+set -x
+
+infile=$1 # must be full path
+outdir=$2 # must be full path
+
+realdir=$(dirname $(dirname $1)) # realization dir, e.g., en*
+realname=$(basename $realdir) # realization name
+f=$(basename $1)
+d1=$(dirname $1)
+yfirst=$(echo $(basename $d1) | sed -e 's/....$//')
+ymid=$((yfirst+2))
+#ydir1=${yfirst}0101
+d2=$realdir/${ymid}0701
+
+period=$(basename $f .nc | sed -e 's/^.*_//' -e 's/\..*$//' -e 's/_.*//')
+
+leap()
+{
+ case $1 in
+ 1952|1956|1960|1964|1968|1972|1976|1980|1984|1988|1992|1996|2000) echo 1 ;;
+ *) echo 0 ;;
+ esac
+}
+
+units()
+{
+ p=$1 # period
+ y=$2 # year
+ case $period in
+ 4xdaily) echo $((4*(365+$(leap $y)))) ;;
+ 8xdaily) echo $((8*(365+$(leap $y)))) ;;
+ daily) echo $((1*(365+$(leap $y)))) ;;
+ month) echo 12 ;;
+ * )
+ esac
+}
+
+mkdir -p /scratch/wilde
+tmp=$(mktemp -d /scratch/wilde/makeyearly_XXXXXX)
+startdir=$PWD
+cd $tmp
+
+export PATH=$PATH:/soft/apps/nco-3.9.9/bin
+# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/apps/nco-3.9.9/lib
+
+o=$outdir/$realname
+
+yd1=$o/$((yfirst+0))0101
+yd2=$o/$((yfirst+1))0101
+yd3=$o/$((yfirst+2))0101
+yd4=$o/$((yfirst+3))0101
+yd5=$o/$((yfirst+4))0101
+
+y1=$((yfirst+0))
+y2=$((yfirst+1))
+y3=$((yfirst+2))
+y4=$((yfirst+3))
+y5=$((yfirst+4))
+
+set +x
+e=echo
+
+$e mkdir -p $yd1 $yd2 $yd3 $yd4 $yd5
+
+$e dd if=$d1/$f of=$f bs=4M # copy in first 2.5 year fle
+
+su=1 # start unit
+u1=$(units $period $y1)
+eu=$u1 # end unit
+
+$e ncks -F -d time,$su,$eu $f out.nc # extract year 1
+$e dd if=out.nc of=$yd1/$f bs=4M
+$e rm out.nc
+
+su=$((eu+1))
+u2=$(units $period $y2)
+eu=$((eu+$u2))
+
+$e ncks -F -d time,$su,$eu $f out.nc # extract year 2
+$e dd if=out.nc of=$yd2/$f bs=4M
+$e rm out.nc
+
+su=$((eu+1))
+u3=$(units $period $y3)
+heu=$((u3/2))
+eu=$((eu+$heu))
+
+$e ncks -F -d time,$su,$eu $f tmp.a.nc # extract year 3 part a
+$e dd if=$d2/$f of=$f bs=4M # copy in second 2.y year file
+
+su=1
+eu=$heu
+
+$e ncks -F -d time,$su,$eu $f tmp.b.nc # extract year 3 part b
+
+$e ncrcat tmp.a.nc tmp.b.nc out.nc # merge year3 parts a and b
+$e dd if=out.nc of=$yd3/$f bs=4M
+$e rm out.nc tmp.a.nc tmp.b.nc
+
+su=$((eu+1))
+u4=$(units $period $y4)
+eu=$((eu+$u4))
+
+$e ncks -F -d time,$su,$eu $f out.nc # extract year 4
+$e dd if=out.nc of=$yd4/$f bs=4M
+$e rm out.nc
+
+su=$((eu+1))
+u5=$(units $period $y5)
+eu=$((eu+$u4))
+
+$e ncks -F -d time,$su,$eu $f out.nc # extract year 5
+$e dd if=out.nc of=$yd5/$f bs=4M
+$e rm out.nc $f
+
+$e cd $startdir
+rm -rf $tmp
+
+exit
+
+######################
+#
+
+the origin of the problem is that in the first 6 month of the year (Jan-Jun)
+there are less days than in the second half of the year (Jul-Dec), and that
+2.5 years refers to the # of months.
+
+The difference is 3 days in a normal year, 2 in a leap year.
+
+So the variations are not simply due by the presence of a leap year. These are
+the categories. Let me define these before:
+
+# normal= no leap in the 2.5 directory.
+# 1st half=directoryName is yyyy0101
+# 2nd half=directoryName is yyyy0701
+#
+# N1 normal, 1st half -->3644 steps (911 days*4)
+# N2 normal, 2nd half -->3656 steps (914 days*4)
+# L1 leap, 1st half -->3648 steps (912 days*4)
+# L2 leap, 2nd half -->3660 steps. (915 days*4)
+
+# leap: 1952 1956 1960 1964 1968 1972 1976 1980 1984 1988 1992 1996 2000
+
+# Jan, Feb, Mar, Apr, May, June = (31+28+31+30+31+30) = 181
+
+# N-year: 181+184 x4= 724 + 736 = 1460
+# L-year: 182+184 x4= 728 + 736 = 1474
+
+
+19500101 1 L1: 3648
+1951 1
+1952 / L
+
+19520701 / N2: 3656
+1953 1
+1954 1
+----
+19550101 1 L1: 3648
+1956 1 L
+1957 /
+
+19570701 + N2: 3656
+1958 1
+1959 1
+----
+19600101 1 L L1: 3648
+1961 1
+1962 /
+
+19620701 + L2: 3660
+1963 1
+1964 1 L
+----
+19650101 1 N1: 3648
+1966 1
+1967 /
+
+19670701 + L2: 3660
+1968 1 L
+1969 1
+----
+19700101 1 L1: 3648
+1971 1
+1972 / L
+
+19720701 + N2: 3656
+1973 1
+1974 1
+----
+19750101 1 L1: 3648
+1976 1 L
+1977 /
+
+19770701 + N2: 3656
+1978 1
+1979 1
+----
+19800101 1 L L1: 3648
+1981 1
+1982 /
+
+19820701 + L2: 3660
+1983 1
+1984 1 L
+----
+19850101 1 N1: 3648
+1986 1
+1987 /
+
+19870701 + L2: 3660
+1988 1 L
+1989 1
+----
+19900101 1 L1: 3648
+1991 1
+1992 / L
+
+19920701 + N2: 3656
+1993 1
+1994 1
+----
+19950101 1 L1: 3648
+1995 1
+1996 / L
+
+19970701 + N2: 3656
+1998 1
+1999 1
+----
Property changes on: SwiftApps/ParVis/HiRAMTools/makeyearly.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,83 @@
+#! /bin/bash
+
+cdir=$1 # combined dir
+outdir=$2 # outdir for equivalent yearly files
+
+
+for y in $(seq 1950 5 1997); do
+ startyear=${y}0101;
+ nextyear=$((y+2))0701
+
+ find $cdir/$startyear -type f | fgrep .nc
+done >ncfiles
+
+exit
+
+# Create new runNNN directory
+
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+mkdir $rundir
+
+cat >$rundir/cf <<END
+
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=10
+lazy.errors=true
+
+END
+
+cat >$rundir/sites.xml <<END
+
+<config>
+
+ <pool handle="local">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">.01</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>$PWD/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="cobalt">
+ <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
+ <filesystem provider="local" />
+ <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
+ <profile namespace="globus" key="project">MTCScienceApps</profile>
+ <profile namespace="globus" key="queue">default</profile>
+ <profile namespace="karajan" key="jobthrottle">2.56</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobsPerNode">8</profile>
+ <!-- <profile namespace="globus" key="workerLoggingLevel">DEBUG</profile> -->
+ <!-- <profile namespace="globus" key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
+ <profile namespace="globus" key="slots">32</profile>
+ <profile namespace="globus" key="maxTime">28000</profile> <!-- ~ 8hrs -->
+ <profile namespace="globus" key="maxWallTime">00:20:00</profile>
+ <profile namespace="globus" key="lowoverallocation">100</profile>
+ <profile namespace="globus" key="highoverallocation">100</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
+ </pool>
+
+</config>
+
+END
+
+cat >$rundir/tc <<END
+
+#local combine $PWD/combine.sh
+cobalt combine $PWD/combine.sh
+
+END
+
+cp ncfiles makeyearly.swift $rundir
+
+cd $rundir
+echo Running in directory $rundir
+runid=$(basename $rundir)
+mkdir -p $outdir/$runid
+
+swift -config cf -tc.file tc -sites.file sites.xml makeyearly.swift -rundir=$outdir/$runid -ncfiles=ncfiles >& swift.out
+
+
Property changes on: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+./combine_realization.sh \
+ /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history/19 \
+ /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/
Property changes on: SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,48 @@
+#! /bin/sh
+
+# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]????
+
+# The following can be made into parameters to this script. Set here for now.
+
+realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations
+realtodo=real.todo # File of names of realization subdirs to process
+realdone=real.done # File of names of realization subdirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined # Base dir for combined output
+
+# Helpful patterns for filename matching
+
+yyyy="[0-9][0-9][0-9][0-9]"
+yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
+
+mkdir -p $outdir
+
+while true; do
+
+ real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
+ if [ _$real = _ ]; then
+ break
+ fi
+
+ echo Combining $real
+
+ find $realdir/$real/history/$yyyymmdd |
+ grep .nc.$yyyy |
+ sed -e 's/.....$//' | sort | uniq > $real.groups
+
+ echo Created $real.groups
+
+ echo ./combine_realization.sh \
+ "$realdir/$real/history/$yyyymmdd" \
+ $outdir/$real
+
+ ./combine_realization.sh \
+ "$realdir/$real/history/$yyyymmdd" \
+ $outdir/$real
+
+ echo Completed $real
+ echo $real >>$realdone
+
+done
+
+echo All combines are done.
+
Property changes on: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/runall.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/runall.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,48 @@
+#! /bin/sh
+
+# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]????
+
+# The following can be made into parameters to this script. Set here for now.
+
+realdir=/intrepid-fs0/users/wilde/persistent/LZ/combined # Base dir of realizations
+realtodo=yearly.todo # File of names of realization subdirs to process
+realdone=yearly.done # File of names of realization subdirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for combined output
+
+# Helpful patterns for filename matching
+
+yyyy="[0-9][0-9][0-9][0-9]"
+yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
+
+mkdir -p $outdir
+
+while true; do
+
+ real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
+ if [ _$real = _ ]; then
+ break
+ fi
+
+ echo RunAll: Processing $real
+
+ find $realdir/$real/history/$yyyymmdd |
+ grep .nc.$yyyy |
+ sed -e 's/.....$//' | sort | uniq > $real.groups
+
+ echo Created $real.groups
+
+ echo ./makeyearly_realization.sh \
+ "$realdir/$real/$yyyymmdd" \
+ $outdir/$real
+
+ ./makeyearly_realization.sh \
+ "$realdir/$real/$yyyymmdd" \
+ $outdir/$real
+
+ echo RunAll: Completed $real
+ echo $real >>$realdone
+
+done
+
+echo RunAll: all realizations are done.
+
Property changes on: SwiftApps/ParVis/HiRAMTools/runall.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,48 @@
+#! /bin/sh
+
+# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]????
+
+# The following can be made into parameters to this script. Set here for now.
+
+realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations
+realtodo=real4.todo # File of names of realization subdirs to process
+realdone=real4.done # File of names of realization subdirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined4 # Base dir for combined output
+
+# Helpful patterns for filename matching
+
+yyyy="[0-9][0-9][0-9][0-9]"
+yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
+
+mkdir -p $outdir
+
+while true; do
+
+ real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
+ if [ _$real = _ ]; then
+ break
+ fi
+
+ echo Combining $real
+
+ find $realdir/$real/history/$yyyymmdd |
+ grep .nc.$yyyy |
+ sed -e 's/.....$//' | sort | uniq > $real.groups
+
+ echo Created $real.groups
+
+ echo ./combine_realization.sh \
+ "$realdir/$real/history/$yyyymmdd" \
+ $outdir/$real
+
+ ./combine_realization.sh \
+ "$realdir/$real/history/$yyyymmdd" \
+ $outdir/$real
+
+ echo Completed $real
+ echo $real >>$realdone
+
+done
+
+echo All combines are done.
+
Property changes on: SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/runcombine.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runcombine.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/runcombine.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+$HOME/LZ/combine.csh \
+ -a /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history \
+ -s 19500101 -n 0 -m 30 # -q
Property changes on: SwiftApps/ParVis/HiRAMTools/runcombine.sh
___________________________________________________________________
Added: svn:executable
+
Added: SwiftApps/ParVis/HiRAMTools/y1.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/y1.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/y1.sh 2012-03-30 16:26:08 UTC (rev 5732)
@@ -0,0 +1,29 @@
+#! /bin/sh
+
+export PATH=$PATH:/soft/apps/nco-3.9.9/bin
+# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/apps/nco-3.9.9/lib
+
+d1=19500101
+d2=19520701
+f=ice_month.nc
+
+o=testyears
+y1=$o/19500101
+y2=$o/19510101
+y3=$o/19520101
+y4=$o/19530101
+y5=$o/19540101
+
+mkdir -p $y1 $y2 $y3 $y4 $y5
+
+ncks -F -d time,1,12 $d1/$f $y1/$f
+
+ncks -F -d time,13,24 $d1/$f $y2/$f
+
+ncks -F -d time,25,30 $d1/$f $y3/$f.a
+ncks -F -d time,1,6 $d2/$f $y3/$f.b
+ncrcat $y3/$f.a $y3/$f.b $y3/$f
+
+ncks -F -d time,7,18 $d2/$f $y4/$f
+
+ncks -F -d time,19,30 $d2/$f $y5/$f
Property changes on: SwiftApps/ParVis/HiRAMTools/y1.sh
___________________________________________________________________
Added: svn:executable
+
More information about the Swift-commit
mailing list