[Swift-commit] r5804 - in SwiftApps/ParVis/HiRAMTools: . JUNK
wilde at ci.uchicago.edu
wilde at ci.uchicago.edu
Fri Jun 8 13:10:40 CDT 2012
Author: wilde
Date: 2012-06-08 13:10:40 -0500 (Fri, 08 Jun 2012)
New Revision: 5804
Added:
SwiftApps/ParVis/HiRAMTools/JUNK/
SwiftApps/ParVis/HiRAMTools/JUNK/checkgroups.sh
SwiftApps/ParVis/HiRAMTools/JUNK/combine.chk.sh
SwiftApps/ParVis/HiRAMTools/JUNK/combine.orig.sh
SwiftApps/ParVis/HiRAMTools/JUNK/find_anomalies.sh
SwiftApps/ParVis/HiRAMTools/JUNK/get_atmos_scalar.sh
SwiftApps/ParVis/HiRAMTools/JUNK/get_real_copy.sh
SwiftApps/ParVis/HiRAMTools/JUNK/get_real_copyL.sh
SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly-cdo-fail.sh
SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly.sh
SwiftApps/ParVis/HiRAMTools/JUNK/run.combine_realization.sh
SwiftApps/ParVis/HiRAMTools/JUNK/runall.combine_realizations.config
SwiftApps/ParVis/HiRAMTools/JUNK/runall.combine_realizations.sh
SwiftApps/ParVis/HiRAMTools/JUNK/runall4.combine_realizations.sh
SwiftApps/ParVis/HiRAMTools/JUNK/runcombine.sh
SwiftApps/ParVis/HiRAMTools/JUNK/test-makeyearly-cdo.sh
SwiftApps/ParVis/HiRAMTools/JUNK/testfailures.sh
SwiftApps/ParVis/HiRAMTools/JUNK/y1.sh
SwiftApps/ParVis/HiRAMTools/combine.sh
SwiftApps/ParVis/HiRAMTools/combine_realization.sh
SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh
SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
SwiftApps/ParVis/HiRAMTools/runall.sh
Removed:
SwiftApps/ParVis/HiRAMTools/checkgroups.sh
SwiftApps/ParVis/HiRAMTools/combine.chk.sh
SwiftApps/ParVis/HiRAMTools/combine.orig.sh
SwiftApps/ParVis/HiRAMTools/combine.sh
SwiftApps/ParVis/HiRAMTools/combine_realization.sh
SwiftApps/ParVis/HiRAMTools/find_anomalies.sh
SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh
SwiftApps/ParVis/HiRAMTools/get_real_copy.sh
SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh
SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh
SwiftApps/ParVis/HiRAMTools/makeyearly.sh
SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh
SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config
SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh
SwiftApps/ParVis/HiRAMTools/runall.sh
SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh
SwiftApps/ParVis/HiRAMTools/runcombine.sh
SwiftApps/ParVis/HiRAMTools/y1.sh
Modified:
SwiftApps/ParVis/HiRAMTools/README
SwiftApps/ParVis/HiRAMTools/runall.config
Log:
Cleanup and documentation of all scripts.
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/checkgroups.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/checkgroups.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/checkgroups.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/checkgroups.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,21 @@
+#! /bin/bash
+
+# /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history/19500101/atmos_4xdaily.tile1.nc
+
+sed -e 's/^.*history.//' groups | sort >todo
+find /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016 -type f -print | fgrep .nc | sed -e 's/^.*run016.//' | sort >done
+
+echo
+echo diff: "<todo >done"
+echo
+diff todo done
+
+echo
+echo last 20 files produced were:
+echo
+ls -ltd /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016/*/* | head -20
+
+echo
+echo File lengths:
+echo
+ls -ltd /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016/*/* | awk '{print $5}' | sort | uniq -c
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/combine.chk.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/combine.chk.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/combine.chk.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/combine.chk.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,33 @@
+#! /bin/bash
+
+set -o errexit
+
+mkdir -p /scratch/$USER
+
+tmp=$(mktemp -d /scratch/$USER/combine_XXXXXX)
+
+echo mktemp error code: $?
+
+cd $tmp
+
+outdir=$1
+inpattern=$2
+date=$(basename $(dirname $inpattern))
+basepattern=$(basename $inpattern)
+
+infiles="$2*"
+for f in $infiles; do
+ dd if=$f of=$(basename $f) bs=4M
+done
+
+/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
+
+mkdir -p $outdir/$date
+dd if=out.nc of=$outdir/$date/$basepattern.trans bs=4M
+mv $outdir/$date/$basepattern.trans $outdir/$date/$basepattern
+
+cd ..
+rm -rf $tmp
+
+
+
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/combine.orig.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/combine.orig.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/combine.orig.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/combine.orig.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,28 @@
+#! /bin/bash
+
+set -x
+
+mkdir -p /scratch/wilde
+tmp=$(mktemp -d /scratch/wilde/combine_XXXX)
+cd $tmp
+
+outdir=$1
+inpattern=$2
+date=$(basename $(dirname $inpattern))
+basepattern=$(basename $inpattern)
+
+infiles="$2*"
+for f in $infiles; do
+ dd if=$f of=$(basename $f) bs=4M
+done
+
+/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
+
+mkdir -p $outdir/$date
+dd if=out.nc of=$outdir/$date/$basepattern bs=4M
+
+cd ..
+rm -rf $tmp
+
+
+
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/find_anomalies.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/find_anomalies.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/find_anomalies.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/find_anomalies.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,18 @@
+#! /bin/sh
+
+reals=/intrepid-fs0/users/wilde/persistent/LZ/combined
+cd $reals
+
+for real in *; do
+ echo Realization: $real
+ ( cd $real
+ for y in *; do
+ echo " Year: " $y
+ ( cd $y
+ for f in *.nc; do
+ echo -n $f; ncdump -h $f | grep 'time =' || echo
+ done | awk '{print " ",$1,$7}' | grep 'atmos.*\.tile[0-9].* (1' | grep -v scalar
+ )
+ done
+ )
+done
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/get_atmos_scalar.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/get_atmos_scalar.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/get_atmos_scalar.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,17 @@
+#! /bin/bash
+
+src=/intrepid-fs0/users/lzamboni/persistent
+dest=/intrepid-fs0/users/wilde/persistent/LZ/combined
+
+f=atmos_scalar.nc
+
+reals=$(cat /home/wilde/LZ/realizations)
+
+for r in $reals; do
+ cd $src/$r/history
+ years=$( /bin/ls -1d [0-9]??????? )
+ for y in $years; do
+ cp $y/$f $dest/$r/$y
+ done
+done
+
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/get_real_copy.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/get_real_copy.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/get_real_copy.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/get_real_copy.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,26 @@
+#! /bin/bash
+
+# src=/intrepid-fs0/users/lzamboni/persistent
+
+src=/intrepid-fs0/users/wilde/persistent/LZ/combined
+dest=$PWD
+
+real=$1
+
+if [ _$1 = _ ]; then
+ echo realization to copy was not specified - exiting; exit 1
+fi
+
+if [ -e _$1 ]; then
+ echo $1 already exits - exiting; exit 1
+fi
+
+cd $src/$real
+dirs=$(find * -type d -print)
+
+for d in $dirs; do
+ mkdir -p $dest/$real/$d
+ for f in $d/*.nc; do
+ ln -s $PWD/$f $dest/$real/$f
+ done
+done
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/get_real_copyL.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/get_real_copyL.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/get_real_copyL.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,26 @@
+#! /bin/bash
+
+# src=/intrepid-fs0/users/lzamboni/persistent
+
+src=/intrepid-fs0/users/wilde/persistent/LZ/combined
+dest=$PWD
+
+real=$1
+
+if [ _$1 = _ ]; then
+ echo realization to copy was not specified - exiting; exit 1
+fi
+
+if [ -e _$1 ]; then
+ echo $1 already exits - exiting; exit 1
+fi
+
+cd $src/$real
+dirs=$(find * -type d -print)
+
+for d in $dirs; do
+ mkdir -p $dest/$real/history/$d
+ for f in $d/*.nc; do
+ ln -s $PWD/$f $dest/$real/history/$f
+ done
+done
Added: SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly-cdo-fail.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly-cdo-fail.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly-cdo-fail.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,13 @@
+#! /bin/bash
+
+set -o errexit
+
+# Create a fast (local-filesystem) working dir
+
+echo About to run failing mkdir
+
+mkdir -p /NOscratch/wilde
+
+echo Should not get here!
+exit 42
+
Property changes on: SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly-cdo-fail.sh
___________________________________________________________________
Added: svn:executable
+ *
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/makeyearly.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/makeyearly.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,231 @@
+#! /bin/bash
+
+set -x
+
+infile=$1 # must be full path
+outdir=$2 # must be full path
+
+realdir=$(dirname $(dirname $1)) # realization dir, e.g., en*
+realname=$(basename $realdir) # realization name
+f=$(basename $1)
+d1=$(dirname $1)
+yfirst=$(echo $(basename $d1) | sed -e 's/....$//')
+ymid=$((yfirst+2))
+#ydir1=${yfirst}0101
+d2=$realdir/${ymid}0701
+
+period=$(basename $f .nc | sed -e 's/^.*_//' -e 's/\..*$//' -e 's/_.*//')
+
+leap()
+{
+ case $1 in
+ 1952|1956|1960|1964|1968|1972|1976|1980|1984|1988|1992|1996|2000) echo 1 ;;
+ *) echo 0 ;;
+ esac
+}
+
+units()
+{
+ p=$1 # period
+ y=$2 # year
+ case $period in
+ 4xdaily) echo $((4*(365+$(leap $y)))) ;;
+ 8xdaily) echo $((8*(365+$(leap $y)))) ;;
+ daily) echo $((1*(365+$(leap $y)))) ;;
+ month) echo 12 ;;
+ * )
+ esac
+}
+
+mkdir -p /scratch/wilde
+tmp=$(mktemp -d /scratch/wilde/makeyearly_XXXXXX)
+startdir=$PWD
+cd $tmp
+
+export PATH=$PATH:/soft/apps/nco-3.9.9/bin
+# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/apps/nco-3.9.9/lib
+
+o=$outdir/$realname
+
+yd1=$o/$((yfirst+0))0101
+yd2=$o/$((yfirst+1))0101
+yd3=$o/$((yfirst+2))0101
+yd4=$o/$((yfirst+3))0101
+yd5=$o/$((yfirst+4))0101
+
+y1=$((yfirst+0))
+y2=$((yfirst+1))
+y3=$((yfirst+2))
+y4=$((yfirst+3))
+y5=$((yfirst+4))
+
+set +x
+e=echo
+
+$e mkdir -p $yd1 $yd2 $yd3 $yd4 $yd5
+
+$e dd if=$d1/$f of=$f bs=4M # copy in first 2.5 year fle
+
+su=1 # start unit
+u1=$(units $period $y1)
+eu=$u1 # end unit
+
+$e ncks -F -d time,$su,$eu $f out.nc # extract year 1
+$e dd if=out.nc of=$yd1/$f bs=4M
+$e rm out.nc
+
+su=$((eu+1))
+u2=$(units $period $y2)
+eu=$((eu+$u2))
+
+$e ncks -F -d time,$su,$eu $f out.nc # extract year 2
+$e dd if=out.nc of=$yd2/$f bs=4M
+$e rm out.nc
+
+su=$((eu+1))
+u3=$(units $period $y3)
+heu=$((u3/2))
+eu=$((eu+$heu))
+
+$e ncks -F -d time,$su,$eu $f tmp.a.nc # extract year 3 part a
+$e dd if=$d2/$f of=$f bs=4M # copy in second 2.y year file
+
+su=1
+eu=$heu
+
+$e ncks -F -d time,$su,$eu $f tmp.b.nc # extract year 3 part b
+
+$e ncrcat tmp.a.nc tmp.b.nc out.nc # merge year3 parts a and b
+$e dd if=out.nc of=$yd3/$f bs=4M
+$e rm out.nc tmp.a.nc tmp.b.nc
+
+su=$((eu+1))
+u4=$(units $period $y4)
+eu=$((eu+$u4))
+
+$e ncks -F -d time,$su,$eu $f out.nc # extract year 4
+$e dd if=out.nc of=$yd4/$f bs=4M
+$e rm out.nc
+
+su=$((eu+1))
+u5=$(units $period $y5)
+eu=$((eu+$u4))
+
+$e ncks -F -d time,$su,$eu $f out.nc # extract year 5
+$e dd if=out.nc of=$yd5/$f bs=4M
+$e rm out.nc $f
+
+$e cd $startdir
+rm -rf $tmp
+
+exit
+
+######################
+#
+
+the origin of the problem is that in the first 6 month of the year (Jan-Jun)
+there are less days than in the second half of the year (Jul-Dec), and that
+2.5 years refers to the # of months.
+
+The difference is 3 days in a normal year, 2 in a leap year.
+
+So the variations are not simply due by the presence of a leap year. These are
+the categories. Let me define these before:
+
+# normal= no leap in the 2.5 directory.
+# 1st half=directoryName is yyyy0101
+# 2nd half=directoryName is yyyy0701
+#
+# N1 normal, 1st half -->3644 steps (911 days*4)
+# N2 normal, 2nd half -->3656 steps (914 days*4)
+# L1 leap, 1st half -->3648 steps (912 days*4)
+# L2 leap, 2nd half -->3660 steps. (915 days*4)
+
+# leap: 1952 1956 1960 1964 1968 1972 1976 1980 1984 1988 1992 1996 2000
+
+# Jan, Feb, Mar, Apr, May, June = (31+28+31+30+31+30) = 181
+
+# N-year: 181+184 x4= 724 + 736 = 1460
+# L-year: 182+184 x4= 728 + 736 = 1474
+
+
+19500101 1 L1: 3648
+1951 1
+1952 / L
+
+19520701 / N2: 3656
+1953 1
+1954 1
+----
+19550101 1 L1: 3648
+1956 1 L
+1957 /
+
+19570701 + N2: 3656
+1958 1
+1959 1
+----
+19600101 1 L L1: 3648
+1961 1
+1962 /
+
+19620701 + L2: 3660
+1963 1
+1964 1 L
+----
+19650101 1 N1: 3648
+1966 1
+1967 /
+
+19670701 + L2: 3660
+1968 1 L
+1969 1
+----
+19700101 1 L1: 3648
+1971 1
+1972 / L
+
+19720701 + N2: 3656
+1973 1
+1974 1
+----
+19750101 1 L1: 3648
+1976 1 L
+1977 /
+
+19770701 + N2: 3656
+1978 1
+1979 1
+----
+19800101 1 L L1: 3648
+1981 1
+1982 /
+
+19820701 + L2: 3660
+1983 1
+1984 1 L
+----
+19850101 1 N1: 3648
+1986 1
+1987 /
+
+19870701 + L2: 3660
+1988 1 L
+1989 1
+----
+19900101 1 L1: 3648
+1991 1
+1992 / L
+
+19920701 + N2: 3656
+1993 1
+1994 1
+----
+19950101 1 L1: 3648
+1995 1
+1996 / L
+
+19970701 + N2: 3656
+1998 1
+1999 1
+----
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/run.combine_realization.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/run.combine_realization.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/run.combine_realization.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+./combine_realization.sh \
+ /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history/19 \
+ /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/runall.combine_realizations.config (from rev 5750, SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/runall.combine_realizations.config (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/runall.combine_realizations.config 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,4 @@
+realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations
+realtodo=real.todo # File of names of realization subdirs to process
+realdone=real.done # File of names of realization subdirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined4 # Base dir for combined output
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/runall.combine_realizations.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/runall.combine_realizations.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/runall.combine_realizations.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,50 @@
+#! /bin/sh
+
+# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]????
+
+# The following can be made into parameters to this script. Set here for now.
+
+realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations
+realtodo=real.todo # File of names of realization subdirs to process
+realdone=real.done # File of names of realization subdirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined # Base dir for combined output
+
+source ./runall.combine_realizations.config # can overwrite above config params
+
+# Helpful patterns for filename matching
+
+yyyy="[0-9][0-9][0-9][0-9]"
+yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
+
+mkdir -p $outdir
+
+while true; do
+
+ real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
+ if [ _$real = _ ]; then
+ break
+ fi
+
+ echo Combining $real
+
+ find $realdir/$real/history/$yyyymmdd |
+ grep .nc.$yyyy |
+ sed -e 's/.....$//' | sort | uniq > $real.groups
+
+ echo Created $real.groups
+
+ echo ./combine_realization.sh \
+ "$realdir/$real/history/$yyyymmdd" \
+ $outdir/$real
+
+ ./combine_realization.sh \
+ "$realdir/$real/history/$yyyymmdd" \
+ $outdir/$real
+
+ echo Completed $real
+ echo $real >>$realdone
+
+done
+
+echo All combines are done.
+
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/runall4.combine_realizations.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/runall4.combine_realizations.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/runall4.combine_realizations.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,48 @@
+#! /bin/sh
+
+# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]????
+
+# The following can be made into parameters to this script. Set here for now.
+
+realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations
+realtodo=real4.todo # File of names of realization subdirs to process
+realdone=real4.done # File of names of realization subdirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined4 # Base dir for combined output
+
+# Helpful patterns for filename matching
+
+yyyy="[0-9][0-9][0-9][0-9]"
+yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
+
+mkdir -p $outdir
+
+while true; do
+
+ real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
+ if [ _$real = _ ]; then
+ break
+ fi
+
+ echo Combining $real
+
+ find $realdir/$real/history/$yyyymmdd |
+ grep .nc.$yyyy |
+ sed -e 's/.....$//' | sort | uniq > $real.groups
+
+ echo Created $real.groups
+
+ echo ./combine_realization.sh \
+ "$realdir/$real/history/$yyyymmdd" \
+ $outdir/$real
+
+ ./combine_realization.sh \
+ "$realdir/$real/history/$yyyymmdd" \
+ $outdir/$real
+
+ echo Completed $real
+ echo $real >>$realdone
+
+done
+
+echo All combines are done.
+
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/runcombine.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/runcombine.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/runcombine.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/runcombine.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,5 @@
+#! /bin/sh
+
+$HOME/LZ/combine.csh \
+ -a /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history \
+ -s 19500101 -n 0 -m 30 # -q
Added: SwiftApps/ParVis/HiRAMTools/JUNK/test-makeyearly-cdo.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/test-makeyearly-cdo.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/test-makeyearly-cdo.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,32 @@
+# ./makeyearly-cdo.sh /intrepid-fs0/users/wilde/persistent/LZ/combined/en1eo12Ic1/19500101/grid_spec.tile4.nc \
+# /intrepid-fs0/users/wilde/persistent/LZ/combined/en1eo12Ic1/19520701/grid_spec.tile4.nc
+
+#./makeyearly-cdo.sh /intrepid-fs0/users/wilde/persistent/LZ/combined/en1eo12Ic1/19500101/atmos_8xdaily_instant.tile3.nc \
+# /intrepid-fs0/users/wilde/persistent/LZ/combined/en1eo12Ic1/19520701/atmos_8xdaily_instant.tile3.nc
+
+#./makeyearly-cdo.sh /intrepid-fs0/users/lzamboni/persistent/combined/Climo/run002/19500101/atmos_month.tile1.nc \
+# /intrepid-fs0/users/lzamboni/persistent/combined/Climo/run002/19520701/atmos_month.tile1.nc
+
+#./makeyearly-cdo.sh /intrepid-fs0/users/lzamboni/persistent/combined/Climo2/run004/19500101/atmos_daily.tile1.nc \
+# /intrepid-fs0/users/lzamboni/persistent/combined/Climo2/run004/19520701/atmos_daily.tile1.nc \
+# /home/wilde/LZ/tests/y2
+
+#./makeyearly-cdo.sh /intrepid-fs0/users/lzamboni/persistent/combined/Climo2/run004/19500101/atmos_scalar.nc \
+# /intrepid-fs0/users/lzamboni/persistent/combined/Climo2/run004/19520701/atmos_scalar.nc \
+# /home/wilde/LZ/tests/y2
+
+./makeyearly-cdo.sh /intrepid-fs0/users/lzamboni/persistent/combined/Climo2/run004/19500101/grid_spec.tile1.nc \
+ /intrepid-fs0/users/lzamboni/persistent/combined/Climo2/run004/19520701/grid_spec.tile1.nc \
+ /home/wilde/LZ/tests/y2
+
+
+
+
+
+
+
+
+
+
+
+
Added: SwiftApps/ParVis/HiRAMTools/JUNK/testfailures.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/testfailures.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/testfailures.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,98 @@
+#! /bin/bash
+
+cdir=$1 # combined dir
+outbase=$2 # base of outdir for equivalent yearly files
+
+bindir=$(dirname $0)
+bindir=$(cd $bindir; pwd)
+
+echo bindir=$bindir
+
+# Enable one of the following patterns to name the output dir:
+
+outsuffix=$(basename $cdir) # rname
+outsuffix=$(basename $(dirname $cdir)) # rname/rundir, use rname
+outsuffix=$(basename $(dirname $cdir))/$(basename $cdir) # rname/rundir, use both
+
+# generate the list of files to process: do every half-decade
+
+echo f1 f2 >ncfiles
+echo f1 f2 >>ncfiles
+
+# for y in $(seq 1950 5 1997); do
+# startyear=${y}0101;
+# nextyear=$((y+2))0701
+# find $cdir/$startyear -type f | fgrep .nc | sed -e "s,^\\(.*\\)/$startyear/\\(.*\\),\\1/$startyear/\\2 \\1/$nextyear/\\2 $outdir,"
+# done >>ncfiles
+
+# Create new runNNN directory
+
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+mkdir $rundir
+
+# Generate Swift config files (cf, tc, sites.xml):
+
+cat >$rundir/cf <<END
+
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=10
+lazy.errors=true
+status.mode=provider
+
+END
+
+cat >$rundir/sites.xml <<END
+
+<config>
+
+ <pool handle="local">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">.01</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>$PWD/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="cobalt">
+ <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
+ <filesystem provider="local" />
+ <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
+ <profile namespace="globus" key="project">MTCScienceApps</profile>
+ <profile namespace="globus" key="queue">default</profile>
+ <profile namespace="karajan" key="jobthrottle">2.56</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobsPerNode">8</profile>
+ <!-- <profile namespace="globus" key="workerLoggingLevel">DEBUG</profile> -->
+ <!-- <profile namespace="globus" key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
+ <profile namespace="globus" key="slots">32</profile>
+ <profile namespace="globus" key="maxTime">28000</profile> <!-- ~ 8hrs -->
+ <profile namespace="globus" key="maxWallTime">00:20:00</profile>
+ <profile namespace="globus" key="lowoverallocation">100</profile>
+ <profile namespace="globus" key="highoverallocation">100</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
+ </pool>
+
+</config>
+
+END
+
+cat >$rundir/tc <<END
+
+cobalt makeyearly $bindir/makeyearly-cdo-fail.sh
+
+END
+
+# Place file list and swift script in run dir, set params, make output dir, and run script
+
+cp ncfiles $bindir/makeyearly.swift $rundir
+
+cd $rundir
+echo Running in directory $rundir
+runid=$(basename $rundir) # un-needed?
+outdir=$outbase/$outsuffix/$runid
+mkdir -p $outdir
+
+swift -config cf -tc.file tc -sites.file sites.xml makeyearly.swift -rundir=$outdir -ncfiles=ncfiles >& swift.out
Property changes on: SwiftApps/ParVis/HiRAMTools/JUNK/testfailures.sh
___________________________________________________________________
Added: svn:executable
+ *
Copied: SwiftApps/ParVis/HiRAMTools/JUNK/y1.sh (from rev 5750, SwiftApps/ParVis/HiRAMTools/y1.sh)
===================================================================
--- SwiftApps/ParVis/HiRAMTools/JUNK/y1.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/JUNK/y1.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,29 @@
+#! /bin/sh
+
+export PATH=$PATH:/soft/apps/nco-3.9.9/bin
+# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/apps/nco-3.9.9/lib
+
+d1=19500101
+d2=19520701
+f=ice_month.nc
+
+o=testyears
+y1=$o/19500101
+y2=$o/19510101
+y3=$o/19520101
+y4=$o/19530101
+y5=$o/19540101
+
+mkdir -p $y1 $y2 $y3 $y4 $y5
+
+ncks -F -d time,1,12 $d1/$f $y1/$f
+
+ncks -F -d time,13,24 $d1/$f $y2/$f
+
+ncks -F -d time,25,30 $d1/$f $y3/$f.a
+ncks -F -d time,1,6 $d2/$f $y3/$f.b
+ncrcat $y3/$f.a $y3/$f.b $y3/$f
+
+ncks -F -d time,7,18 $d2/$f $y4/$f
+
+ncks -F -d time,19,30 $d2/$f $y5/$f
Modified: SwiftApps/ParVis/HiRAMTools/README
===================================================================
--- SwiftApps/ParVis/HiRAMTools/README 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/README 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,24 +1,84 @@
-1. Add Swift to your path. For now we will use a Swift version maintained by
- Swift team member Jon Monette:
+The top-level script in this toolkit is "runall.sh". This is the only script
+that you should need to run directly.
-PATH=/home/jonmon/Library/Swift/swift-0.93/bin:/home/jonmon/Library/jdk1.7.0_03/bin:$PATH
+runall.sh takes a config file which specifies what realization processing
+script to run, where to find the input realizations, and where to place the
+processed output realizations.
-2. Edit runall.combine_realizations.config
+ makeyearly_realization.sh => annualizes a realization
+ combine_realization.sh => combines a realization
-3. Place the list of realizations to process in a file named "real.todo" and
- create and empty file "real.done", like so (for example, to process 2
- realizations, Climo_001 and Climo_023):
+The script calling structure is:
-cat >real.todo
-Climo_001
-Climo_023
-^D
+ runall.sh # called by the user, runs the scripts below
-cat >real.done
-^D
+ makeyearly_realization.sh
+ makeyearly.swift
+ makeyearly-cdo.sh (app, specified in tc)
+
+ combine_realization.sh
+ combine.swift
+ combine.chk.sh (app, specified in tc)
-3. run:
- ./runall.combine_realizations.sh >& runall.out
+To process a set of realizations (for example, to combine them), perform these
+steps:
+1. Add Swift and a recent Sun Java to your path. For now we will use a Swift
+ and Java version maintained by Swift team member Jon Monette:
+
+ PATH=/home/jonmon/Library/Swift/swift-0.93/bin:/home/jonmon/Library/jdk1.7.0_03/bin:$PATH
+
+2. Create and cd to a working directory below which runall.sh can make per-run
+ directories:
+
+ mkdir $HOME/swiftruns
+ cd $HOME/swiftruns
+
+3. Check out these HiRAM tools from svn:
+
+ mkdir $HOME/HiRAMTools
+ cd $HOME/HiRAMTools
+
+ svn co https://svn.ci.uchicago.edu/svn/vdl2/SwiftApps/ParVis/HiRAMTools
+
+4. Copy runall.config to your swiftruns dir and edit it to specify your input
+ and output directories. It contains for example:
+
+
+ realtodo=real.todo # File of names of realization subdirs to process
+ realdone=real.done # File of names of realization subdirs completed
+ # (this should be an empty file at start)
+
+ outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for yearly output
+
+ # script=combine_realization.sh # Script to run for each realization
+ script=makeyearly_realization.sh # Script to run for each realization
+
+
+5. Place the list of realizations to process in a file named "real.todo" and
+ create and empty file "real.done". For example, to process 2 realizations,
+ Climo_001 and Climo_023):
+
+ cat >real.todo
+ Climo_001
+ Climo_023
+ ^D
+
+ >real.done # create an empty real.done file
+
+6. run:
+
+ ./runall.combine_realizations.sh >& runall.out
+
+Then look for the most recently created run directory, and do:
+
+ tail -f swift.out
+
+
+OPEN ISSUES:
+
+- We see straggler scripts on Eureka nodes. Seems to be doing very slow IO on
+shared disk for unexplained reasons. Ticket is open with ALCF support, being
+investigated by Andrew Cherry.
Deleted: SwiftApps/ParVis/HiRAMTools/checkgroups.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/checkgroups.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/checkgroups.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,21 +0,0 @@
-#! /bin/bash
-
-# /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history/19500101/atmos_4xdaily.tile1.nc
-
-sed -e 's/^.*history.//' groups | sort >todo
-find /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016 -type f -print | fgrep .nc | sed -e 's/^.*run016.//' | sort >done
-
-echo
-echo diff: "<todo >done"
-echo
-diff todo done
-
-echo
-echo last 20 files produced were:
-echo
-ls -ltd /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016/*/* | head -20
-
-echo
-echo File lengths:
-echo
-ls -ltd /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/run016/*/* | awk '{print $5}' | sort | uniq -c
Deleted: SwiftApps/ParVis/HiRAMTools/combine.chk.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine.chk.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/combine.chk.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,33 +0,0 @@
-#! /bin/bash
-
-set -o errexit
-
-mkdir -p /scratch/$USER
-
-tmp=$(mktemp -d /scratch/$USER/combine_XXXXXX)
-
-echo mktemp error code: $?
-
-cd $tmp
-
-outdir=$1
-inpattern=$2
-date=$(basename $(dirname $inpattern))
-basepattern=$(basename $inpattern)
-
-infiles="$2*"
-for f in $infiles; do
- dd if=$f of=$(basename $f) bs=4M
-done
-
-/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
-
-mkdir -p $outdir/$date
-dd if=out.nc of=$outdir/$date/$basepattern.trans bs=4M
-mv $outdir/$date/$basepattern.trans $outdir/$date/$basepattern
-
-cd ..
-rm -rf $tmp
-
-
-
Deleted: SwiftApps/ParVis/HiRAMTools/combine.orig.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine.orig.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/combine.orig.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,28 +0,0 @@
-#! /bin/bash
-
-set -x
-
-mkdir -p /scratch/wilde
-tmp=$(mktemp -d /scratch/wilde/combine_XXXX)
-cd $tmp
-
-outdir=$1
-inpattern=$2
-date=$(basename $(dirname $inpattern))
-basepattern=$(basename $inpattern)
-
-infiles="$2*"
-for f in $infiles; do
- dd if=$f of=$(basename $f) bs=4M
-done
-
-/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
-
-mkdir -p $outdir/$date
-dd if=out.nc of=$outdir/$date/$basepattern bs=4M
-
-cd ..
-rm -rf $tmp
-
-
-
Deleted: SwiftApps/ParVis/HiRAMTools/combine.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/combine.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,29 +0,0 @@
-#! /bin/bash
-
-set -x
-
-mkdir -p /scratch/wilde
-tmp=$(mktemp -d /scratch/wilde/combine_XXXX)
-cd $tmp
-
-outdir=$1
-inpattern=$2
-date=$(basename $(dirname $inpattern))
-basepattern=$(basename $inpattern)
-
-infiles="$2*"
-for f in $infiles; do
- dd if=$f of=$(basename $f) bs=4M
-done
-
-/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
-
-mkdir -p $outdir/$date
-dd if=out.nc of=$outdir/$date/$basepattern.trans bs=4M
-mv $outdir/$date/$basepattern.trans $outdir/$date/$basepattern
-
-cd ..
-rm -rf $tmp
-
-
-
Added: SwiftApps/ParVis/HiRAMTools/combine.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/combine.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,29 @@
+#! /bin/bash
+
+set -x
+
+mkdir -p /scratch/wilde
+tmp=$(mktemp -d /scratch/wilde/combine_XXXX)
+cd $tmp
+
+outdir=$1
+inpattern=$2
+date=$(basename $(dirname $inpattern))
+basepattern=$(basename $inpattern)
+
+infiles="$2*"
+for f in $infiles; do
+ dd if=$f of=$(basename $f) bs=4M
+done
+
+/home/ckerr/bin/fms/tools/x86_64/bin/mppnccombine.exe out.nc ${basepattern}*
+
+mkdir -p $outdir/$date
+dd if=out.nc of=$outdir/$date/$basepattern.trans bs=4M
+mv $outdir/$date/$basepattern.trans $outdir/$date/$basepattern
+
+cd ..
+rm -rf $tmp
+
+
+
Deleted: SwiftApps/ParVis/HiRAMTools/combine_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,82 +0,0 @@
-#! /bin/bash
-
-hdir=$1
-outdir=$2
-
-/bin/ls -1 -d ${hdir}* >dirs
-
-echo '=== ' $(basename $0) doing dirs:
-cat dirs
-
-for d in $(cat dirs); do
- find $d | grep .nc.[0-9][0-9][0-9][0-9] | sed -e 's/.....$//' | sort | uniq
-done >groups
-
-# Create new runNNN directory
-
-rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
-mkdir $rundir
-
-cat >$rundir/cf <<END
-
-wrapperlog.always.transfer=true
-sitedir.keep=true
-execution.retries=10
-lazy.errors=true
-
-END
-
-cat >$rundir/sites.xml <<END
-
-<config>
-
- <pool handle="local">
- <execution provider="local" />
- <profile namespace="karajan" key="jobThrottle">.01</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local"/>
- <workdirectory>$PWD/swiftwork</workdirectory>
- </pool>
-
- <pool handle="cobalt">
- <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
- <filesystem provider="local" />
- <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
- <!-- <profile namespace="globus" key="project">MTCScienceApps</profile> -->
- <profile namespace="globus" key="queue">default</profile>
- <profile namespace="karajan" key="jobthrottle">2.56</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <profile namespace="globus" key="jobsPerNode">8</profile>
- <!-- <profile namespace="globus" key="workerLoggingLevel">DEBUG</profile> -->
- <!-- <profile namespace="globus" key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
- <profile namespace="globus" key="slots">32</profile>
- <profile namespace="globus" key="maxTime">28000</profile> <!-- ~ 8hrs -->
- <profile namespace="globus" key="maxWallTime">00:20:00</profile>
- <profile namespace="globus" key="lowoverallocation">100</profile>
- <profile namespace="globus" key="highoverallocation">100</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
- </pool>
-
-</config>
-
-END
-
-cat >$rundir/tc <<END
-
-#local combine $PWD/combine.sh
-cobalt combine $PWD/combine.chk.sh
-
-END
-
-cp groups combine.swift $rundir
-
-cd $rundir
-echo Running in directory $rundir
-runid=$(basename $rundir)
-mkdir -p $outdir/$runid
-
-swift -config cf -tc.file tc -sites.file sites.xml combine.swift -rundir=$outdir/$runid -groupfile=groups >& swift.out
-
-
Added: SwiftApps/ParVis/HiRAMTools/combine_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/combine_realization.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/combine_realization.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,82 @@
+#! /bin/bash
+
+hdir=$1
+outdir=$2
+
+/bin/ls -1 -d ${hdir}* >dirs
+
+echo '=== ' $(basename $0) doing dirs:
+cat dirs
+
+for d in $(cat dirs); do
+ find $d | grep .nc.[0-9][0-9][0-9][0-9] | sed -e 's/.....$//' | sort | uniq
+done >groups
+
+# Create new runNNN directory
+
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+mkdir $rundir
+
+cat >$rundir/cf <<END
+
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=10
+lazy.errors=true
+
+END
+
+cat >$rundir/sites.xml <<END
+
+<config>
+
+ <pool handle="local">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">.01</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>$PWD/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="cobalt">
+ <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
+ <filesystem provider="local" />
+ <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
+ <!-- <profile namespace="globus" key="project">MTCScienceApps</profile> -->
+ <profile namespace="globus" key="queue">default</profile>
+ <profile namespace="karajan" key="jobthrottle">2.56</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobsPerNode">8</profile>
+ <!-- <profile namespace="globus" key="workerLoggingLevel">DEBUG</profile> -->
+ <!-- <profile namespace="globus" key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
+ <profile namespace="globus" key="slots">32</profile>
+ <profile namespace="globus" key="maxTime">28000</profile> <!-- ~ 8hrs -->
+ <profile namespace="globus" key="maxWallTime">00:20:00</profile>
+ <profile namespace="globus" key="lowoverallocation">100</profile>
+ <profile namespace="globus" key="highoverallocation">100</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
+ </pool>
+
+</config>
+
+END
+
+cat >$rundir/tc <<END
+
+#local combine $PWD/combine.sh
+cobalt combine $PWD/combine.chk.sh
+
+END
+
+cp groups combine.swift $rundir
+
+cd $rundir
+echo Running in directory $rundir
+runid=$(basename $rundir)
+mkdir -p $outdir/$runid
+
+swift -config cf -tc.file tc -sites.file sites.xml combine.swift -rundir=$outdir/$runid -groupfile=groups >& swift.out
+
+
Deleted: SwiftApps/ParVis/HiRAMTools/find_anomalies.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/find_anomalies.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/find_anomalies.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,18 +0,0 @@
-#! /bin/sh
-
-reals=/intrepid-fs0/users/wilde/persistent/LZ/combined
-cd $reals
-
-for real in *; do
- echo Realization: $real
- ( cd $real
- for y in *; do
- echo " Year: " $y
- ( cd $y
- for f in *.nc; do
- echo -n $f; ncdump -h $f | grep 'time =' || echo
- done | awk '{print " ",$1,$7}' | grep 'atmos.*\.tile[0-9].* (1' | grep -v scalar
- )
- done
- )
-done
Deleted: SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/get_atmos_scalar.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,17 +0,0 @@
-#! /bin/bash
-
-src=/intrepid-fs0/users/lzamboni/persistent
-dest=/intrepid-fs0/users/wilde/persistent/LZ/combined
-
-f=atmos_scalar.nc
-
-reals=$(cat /home/wilde/LZ/realizations)
-
-for r in $reals; do
- cd $src/$r/history
- years=$( /bin/ls -1d [0-9]??????? )
- for y in $years; do
- cp $y/$f $dest/$r/$y
- done
-done
-
Deleted: SwiftApps/ParVis/HiRAMTools/get_real_copy.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/get_real_copy.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/get_real_copy.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,26 +0,0 @@
-#! /bin/bash
-
-# src=/intrepid-fs0/users/lzamboni/persistent
-
-src=/intrepid-fs0/users/wilde/persistent/LZ/combined
-dest=$PWD
-
-real=$1
-
-if [ _$1 = _ ]; then
- echo realization to copy was not specified - exiting; exit 1
-fi
-
-if [ -e _$1 ]; then
- echo $1 already exits - exiting; exit 1
-fi
-
-cd $src/$real
-dirs=$(find * -type d -print)
-
-for d in $dirs; do
- mkdir -p $dest/$real/$d
- for f in $d/*.nc; do
- ln -s $PWD/$f $dest/$real/$f
- done
-done
Deleted: SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/get_real_copyL.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,26 +0,0 @@
-#! /bin/bash
-
-# src=/intrepid-fs0/users/lzamboni/persistent
-
-src=/intrepid-fs0/users/wilde/persistent/LZ/combined
-dest=$PWD
-
-real=$1
-
-if [ _$1 = _ ]; then
- echo realization to copy was not specified - exiting; exit 1
-fi
-
-if [ -e _$1 ]; then
- echo $1 already exits - exiting; exit 1
-fi
-
-cd $src/$real
-dirs=$(find * -type d -print)
-
-for d in $dirs; do
- mkdir -p $dest/$real/history/$d
- for f in $d/*.nc; do
- ln -s $PWD/$f $dest/$real/history/$f
- done
-done
Deleted: SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,56 +0,0 @@
-#! /bin/bash
-
-# Create a fast (local-filesystem) working dir
-
-if mkdir -p /scratch/wilde; then
- scratch=/scratch/wilde
-elif mkdir -p $HOME/scratch; then
- scratch=$HOME/scratch
-else
- echo $0: cant create scratch dir
- exit 11
-fi
-
-tmp=$(mktemp -d $scratch/makeyearly_XXXXXX)
-startdir=$PWD
-cd $tmp
-
-echo running in $PWD
-
-qtrdec1=$1 # 1st quarter-decade dir
-qtrdec2=$2 # 2nd quarter-decade dir
-destbase=$3 # base dir for output years
-
-echo destbase=$destbase
-
-export PATH=/soft/apps/cdo-1.4.7/bin:$PATH
-export LD_LIBRARY_PATH=/soft/apps/intel-fc-11.0.074/lib/intel64:/soft/apps/intel-cc-11.0.074/lib/intel64:/soft/apps/netcdf-3.6.3/lib:/usr/lib:/usr/X11R6/lib:/usr/local/lib
-
-fname=$(basename $qtrdec1 .nc)
-firstyear=$(basename $(dirname $qtrdec1) | sed -e 's/....$//')
-
-if echo $fname | egrep -v 'grid_spec|scalar'; then
- cdo mergetime $qtrdec1 $qtrdec2 ${fname}.nc
- cdo splityear ${fname}.nc ${fname}.
- # Yields, eg, from atmos_month.tile1:
- # atmos_month.tile1.1950.nc
- # atmos_month.tile1.1951.nc
- # atmos_month.tile1.1952.nc
- # atmos_month.tile1.1953.nc
- # atmos_month.tile1.1954.nc
-fi
-
-for (( y=0; y<5; y++ ))
-do
- year=$((firstyear+y))
- yeardir=${year}0101
- destdir=$destbase/$yeardir
- mkdir -p $destdir
- if echo $fname | egrep -v 'grid_spec|scalar'; then
- dd if=$fname.$year.nc of=$destdir/$fname.nc bs=8M
- else
- dd if=$qtrdec1 of=$destdir/$fname.nc bs=8M
- fi
-done
-
-rm -rf $tmp
Added: SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,82 @@
+#! /bin/bash
+
+# Create a fast (local-filesystem) working dir
+
+if mkdir -p /scratch/wilde; then
+ scratch=/scratch/wilde
+elif mkdir -p $HOME/scratch; then
+ scratch=$HOME/scratch
+else
+ echo $0: cant create scratch dir
+ exit 11
+fi
+
+tmp=$(mktemp -d $scratch/makeyearly_XXXXXX)
+startdir=$PWD
+cd $tmp
+
+echo running in $PWD
+
+qtrdec1=$1 # 1st quarter-decade dir
+qtrdec2=$2 # 2nd quarter-decade dir
+destbase=$3 # base dir for output years
+
+echo destbase=$destbase
+
+# Add cdo and nco tools to PATH and Intel compiler v11 libs to LD_LIBRARY_PATH
+
+export PATH=/soft/apps/cdo-1.4.7/bin:/soft/apps/nco-4.0.9/bin:$PATH
+export LD_LIBRARY_PATH=/soft/apps/intel-fc-11.0.074/lib/intel64:/soft/apps/intel-cc-11.0.074/lib/intel64:/soft/apps/netcdf-3.6.3/lib:/usr/lib:/usr/X11R6/lib:/usr/local/lib
+
+fname=$(basename $qtrdec1 .nc)
+firstyear=$(basename $(dirname $qtrdec1) | sed -e 's/....$//')
+
+# process only the time-based files
+
+if echo $fname | egrep -v 'grid_spec|scalar'; then
+
+ # Merge 2 quarter-decade files into one half-decade file
+
+ ncrcat $qtrdec1 $qtrdec2 5years.nc
+
+ # Extract each year from the half-decade file
+
+ for (( y=$firstyear; y<$firstyear+5; y++ )); do
+ ncks -d time,"${y}-01-01 00:00:00","${y}-12-31 23:59:59" 5years.nc $y.nc
+ done
+
+fi
+
+# Copy the years from temporary to permanent filesystem
+
+for (( y=$firstyear; y<$firstyear+5; y++ ))
+do
+ destdir=$destbase/${y}0101
+ mkdir -p $destdir
+ if echo $fname | egrep -v 'grid_spec|scalar'; then
+ dd if=$y.nc of=$destdir/$fname.nc bs=8M
+ else
+ dd if=$qtrdec1 of=$destdir/$fname.nc bs=8M
+ fi
+done
+
+rm -rf $tmp
+
+# Delete when this works:
+
+ # Old method:
+ #
+ # cdo mergetime $qtrdec1 $qtrdec2 ${fname}.nc
+ # cdo splityear ${fname}.nc ${fname}.
+ # Yields, eg, from atmos_month.tile1:
+ # atmos_month.tile1.1950.nc
+ # atmos_month.tile1.1951.nc
+ # atmos_month.tile1.1952.nc
+ # atmos_month.tile1.1953.nc
+ # atmos_month.tile1.1954.nc
+
+ # New method:
+ #
+ # ncrcat input1.nc input2.nc 5years.nc
+ # ncks -d time,"$yyyy-01-01 00:00:00","$yyyy-12-31 23:59:59" 5years.nc year1.nc
+
Deleted: SwiftApps/ParVis/HiRAMTools/makeyearly.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,231 +0,0 @@
-#! /bin/bash
-
-set -x
-
-infile=$1 # must be full path
-outdir=$2 # must be full path
-
-realdir=$(dirname $(dirname $1)) # realization dir, e.g., en*
-realname=$(basename $realdir) # realization name
-f=$(basename $1)
-d1=$(dirname $1)
-yfirst=$(echo $(basename $d1) | sed -e 's/....$//')
-ymid=$((yfirst+2))
-#ydir1=${yfirst}0101
-d2=$realdir/${ymid}0701
-
-period=$(basename $f .nc | sed -e 's/^.*_//' -e 's/\..*$//' -e 's/_.*//')
-
-leap()
-{
- case $1 in
- 1952|1956|1960|1964|1968|1972|1976|1980|1984|1988|1992|1996|2000) echo 1 ;;
- *) echo 0 ;;
- esac
-}
-
-units()
-{
- p=$1 # period
- y=$2 # year
- case $period in
- 4xdaily) echo $((4*(365+$(leap $y)))) ;;
- 8xdaily) echo $((8*(365+$(leap $y)))) ;;
- daily) echo $((1*(365+$(leap $y)))) ;;
- month) echo 12 ;;
- * )
- esac
-}
-
-mkdir -p /scratch/wilde
-tmp=$(mktemp -d /scratch/wilde/makeyearly_XXXXXX)
-startdir=$PWD
-cd $tmp
-
-export PATH=$PATH:/soft/apps/nco-3.9.9/bin
-# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/apps/nco-3.9.9/lib
-
-o=$outdir/$realname
-
-yd1=$o/$((yfirst+0))0101
-yd2=$o/$((yfirst+1))0101
-yd3=$o/$((yfirst+2))0101
-yd4=$o/$((yfirst+3))0101
-yd5=$o/$((yfirst+4))0101
-
-y1=$((yfirst+0))
-y2=$((yfirst+1))
-y3=$((yfirst+2))
-y4=$((yfirst+3))
-y5=$((yfirst+4))
-
-set +x
-e=echo
-
-$e mkdir -p $yd1 $yd2 $yd3 $yd4 $yd5
-
-$e dd if=$d1/$f of=$f bs=4M # copy in first 2.5 year fle
-
-su=1 # start unit
-u1=$(units $period $y1)
-eu=$u1 # end unit
-
-$e ncks -F -d time,$su,$eu $f out.nc # extract year 1
-$e dd if=out.nc of=$yd1/$f bs=4M
-$e rm out.nc
-
-su=$((eu+1))
-u2=$(units $period $y2)
-eu=$((eu+$u2))
-
-$e ncks -F -d time,$su,$eu $f out.nc # extract year 2
-$e dd if=out.nc of=$yd2/$f bs=4M
-$e rm out.nc
-
-su=$((eu+1))
-u3=$(units $period $y3)
-heu=$((u3/2))
-eu=$((eu+$heu))
-
-$e ncks -F -d time,$su,$eu $f tmp.a.nc # extract year 3 part a
-$e dd if=$d2/$f of=$f bs=4M # copy in second 2.y year file
-
-su=1
-eu=$heu
-
-$e ncks -F -d time,$su,$eu $f tmp.b.nc # extract year 3 part b
-
-$e ncrcat tmp.a.nc tmp.b.nc out.nc # merge year3 parts a and b
-$e dd if=out.nc of=$yd3/$f bs=4M
-$e rm out.nc tmp.a.nc tmp.b.nc
-
-su=$((eu+1))
-u4=$(units $period $y4)
-eu=$((eu+$u4))
-
-$e ncks -F -d time,$su,$eu $f out.nc # extract year 4
-$e dd if=out.nc of=$yd4/$f bs=4M
-$e rm out.nc
-
-su=$((eu+1))
-u5=$(units $period $y5)
-eu=$((eu+$u4))
-
-$e ncks -F -d time,$su,$eu $f out.nc # extract year 5
-$e dd if=out.nc of=$yd5/$f bs=4M
-$e rm out.nc $f
-
-$e cd $startdir
-rm -rf $tmp
-
-exit
-
-######################
-#
-
-the origin of the problem is that in the first 6 month of the year (Jan-Jun)
-there are less days than in the second half of the year (Jul-Dec), and that
-2.5 years refers to the # of months.
-
-The difference is 3 days in a normal year, 2 in a leap year.
-
-So the variations are not simply due by the presence of a leap year. These are
-the categories. Let me define these before:
-
-# normal= no leap in the 2.5 directory.
-# 1st half=directoryName is yyyy0101
-# 2nd half=directoryName is yyyy0701
-#
-# N1 normal, 1st half -->3644 steps (911 days*4)
-# N2 normal, 2nd half -->3656 steps (914 days*4)
-# L1 leap, 1st half -->3648 steps (912 days*4)
-# L2 leap, 2nd half -->3660 steps. (915 days*4)
-
-# leap: 1952 1956 1960 1964 1968 1972 1976 1980 1984 1988 1992 1996 2000
-
-# Jan, Feb, Mar, Apr, May, June = (31+28+31+30+31+30) = 181
-
-# N-year: 181+184 x4= 724 + 736 = 1460
-# L-year: 182+184 x4= 728 + 736 = 1474
-
-
-19500101 1 L1: 3648
-1951 1
-1952 / L
-
-19520701 / N2: 3656
-1953 1
-1954 1
-----
-19550101 1 L1: 3648
-1956 1 L
-1957 /
-
-19570701 + N2: 3656
-1958 1
-1959 1
-----
-19600101 1 L L1: 3648
-1961 1
-1962 /
-
-19620701 + L2: 3660
-1963 1
-1964 1 L
-----
-19650101 1 N1: 3648
-1966 1
-1967 /
-
-19670701 + L2: 3660
-1968 1 L
-1969 1
-----
-19700101 1 L1: 3648
-1971 1
-1972 / L
-
-19720701 + N2: 3656
-1973 1
-1974 1
-----
-19750101 1 L1: 3648
-1976 1 L
-1977 /
-
-19770701 + N2: 3656
-1978 1
-1979 1
-----
-19800101 1 L L1: 3648
-1981 1
-1982 /
-
-19820701 + L2: 3660
-1983 1
-1984 1 L
-----
-19850101 1 N1: 3648
-1986 1
-1987 /
-
-19870701 + L2: 3660
-1988 1 L
-1989 1
-----
-19900101 1 L1: 3648
-1991 1
-1992 / L
-
-19920701 + N2: 3656
-1993 1
-1994 1
-----
-19950101 1 L1: 3648
-1995 1
-1996 / L
-
-19970701 + N2: 3656
-1998 1
-1999 1
-----
Deleted: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,96 +0,0 @@
-#! /bin/bash
-
-cdir=$1 # combined dir
-outbase=$2 # base of outdir for equivalent yearly files
-
-bindir=$(dirname $0)
-bindir=$(cd $bindir; pwd)
-
-echo bindir=$bindir
-
-# Enable one of the following patterns to name the output dir:
-
-outsuffix=$(basename $cdir) # rname
-outsuffix=$(basename $(dirname $cdir)) # rname/rundir, use rname
-outsuffix=$(basename $(dirname $cdir))/$(basename $cdir) # rname/rundir, use both
-
-# generate the list of files to process: do every half-decade
-
-echo f1 f2 >ncfiles
-for y in $(seq 1950 5 1997); do
- startyear=${y}0101;
- nextyear=$((y+2))0701
- find $cdir/$startyear -type f | fgrep .nc | sed -e "s,^\\(.*\\)/$startyear/\\(.*\\),\\1/$startyear/\\2 \\1/$nextyear/\\2 $outdir,"
-done >>ncfiles
-
-# Create new runNNN directory
-
-rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
-mkdir $rundir
-
-# Generate Swift config files (cf, tc, sites.xml):
-
-cat >$rundir/cf <<END
-
-wrapperlog.always.transfer=true
-sitedir.keep=true
-execution.retries=10
-lazy.errors=true
-status.mode=provider
-
-END
-
-cat >$rundir/sites.xml <<END
-
-<config>
-
- <pool handle="local">
- <execution provider="local" />
- <profile namespace="karajan" key="jobThrottle">.01</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <filesystem provider="local"/>
- <workdirectory>$PWD/swiftwork</workdirectory>
- </pool>
-
- <pool handle="cobalt">
- <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
- <filesystem provider="local" />
- <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
- <profile namespace="globus" key="project">MTCScienceApps</profile>
- <profile namespace="globus" key="queue">default</profile>
- <profile namespace="karajan" key="jobthrottle">2.56</profile>
- <profile namespace="karajan" key="initialScore">10000</profile>
- <profile namespace="globus" key="jobsPerNode">8</profile>
- <!-- <profile namespace="globus" key="workerLoggingLevel">DEBUG</profile> -->
- <!-- <profile namespace="globus" key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
- <profile namespace="globus" key="slots">32</profile>
- <profile namespace="globus" key="maxTime">28000</profile> <!-- ~ 8hrs -->
- <profile namespace="globus" key="maxWallTime">00:20:00</profile>
- <profile namespace="globus" key="lowoverallocation">100</profile>
- <profile namespace="globus" key="highoverallocation">100</profile>
- <profile namespace="globus" key="nodeGranularity">1</profile>
- <profile namespace="globus" key="maxNodes">1</profile>
- <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
- </pool>
-
-</config>
-
-END
-
-cat >$rundir/tc <<END
-
-cobalt makeyearly $bindir/makeyearly-cdo.sh
-
-END
-
-# Place file list and swift script in run dir, set params, make output dir, and run script
-
-cp ncfiles $bindir/makeyearly.swift $rundir
-
-cd $rundir
-echo Running in directory $rundir
-runid=$(basename $rundir) # un-needed?
-outdir=$outbase/$outsuffix/$runid
-mkdir -p $outdir
-
-swift -config cf -tc.file tc -sites.file sites.xml makeyearly.swift -rundir=$outdir -ncfiles=ncfiles >& swift.out
Added: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,97 @@
+#! /bin/bash
+
+cdir=$1 # combined dir
+outbase=$2 # base of outdir for equivalent yearly files
+
+bindir=$(dirname $0)
+bindir=$(cd $bindir; pwd)
+
+echo bindir=$bindir
+
+# Enable one of the following patterns to name the output dir:
+
+outsuffix=$(basename $cdir) # rname
+outsuffix=$(basename $(dirname $cdir)) # rname/rundir, use rname
+outsuffix=$(basename $(dirname $cdir))/$(basename $cdir) # rname/rundir, use both
+
+# generate the list of files to process: do every half-decade
+
+echo f1 f2 >ncfiles
+for y in $(seq 1950 5 1997); do
+ startyear=${y}0101;
+ nextyear=$((y+2))0701
+ find $cdir/$startyear -type f | fgrep .nc | sed -e "s,^\\(.*\\)/$startyear/\\(.*\\),\\1/$startyear/\\2 \\1/$nextyear/\\2 $outdir,"
+done >>ncfiles
+
+# Create new runNNN directory
+
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+mkdir $rundir
+
+# Generate Swift config files (cf, tc, sites.xml):
+
+cat >$rundir/cf <<END
+
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=10
+lazy.errors=true
+status.mode=provider
+
+END
+
+cat >$rundir/sites.xml <<END
+
+<config>
+
+ <pool handle="local">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">.01</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>$PWD/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="cobalt">
+ <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
+ <filesystem provider="local" />
+ <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
+ <!-- <profile namespace="globus" key="project">MTCScienceApps</profile> -->
+ <profile namespace="globus" key="project">SU_Climate</profile>
+ <profile namespace="globus" key="queue">default</profile>
+ <profile namespace="karajan" key="jobthrottle">2.56</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobsPerNode">8</profile>
+ <!-- <profile namespace="globus" key="workerLoggingLevel">DEBUG</profile> -->
+ <!-- <profile namespace="globus" key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
+ <profile namespace="globus" key="slots">32</profile>
+ <profile namespace="globus" key="maxTime">28000</profile> <!-- ~ 8hrs -->
+ <profile namespace="globus" key="maxWallTime">00:20:00</profile>
+ <profile namespace="globus" key="lowoverallocation">100</profile>
+ <profile namespace="globus" key="highoverallocation">100</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
+ </pool>
+
+</config>
+
+END
+
+cat >$rundir/tc <<END
+
+cobalt makeyearly $bindir/makeyearly-cdo.sh
+
+END
+
+# Place file list and swift script in run dir, set params, make output dir, and run script
+
+cp ncfiles $bindir/makeyearly.swift $rundir
+
+cd $rundir
+echo Running in directory $rundir
+runid=$(basename $rundir) # un-needed?
+outdir=$outbase/$outsuffix/$runid
+mkdir -p $outdir
+
+swift -config cf -tc.file tc -sites.file sites.xml makeyearly.swift -rundir=$outdir -ncfiles=ncfiles >& swift.out
Deleted: SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/run.combine_realization.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,5 +0,0 @@
-#! /bin/sh
-
-./combine_realization.sh \
- /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history/19 \
- /intrepid-fs0/users/wilde/persistent/en1eo8Ic1/combined/
Deleted: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.config 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,4 +0,0 @@
-realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations
-realtodo=real.todo # File of names of realization subdirs to process
-realdone=real.done # File of names of realization subdirs completed
-outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined4 # Base dir for combined output
Deleted: SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/runall.combine_realizations.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,50 +0,0 @@
-#! /bin/sh
-
-# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]????
-
-# The following can be made into parameters to this script. Set here for now.
-
-realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations
-realtodo=real.todo # File of names of realization subdirs to process
-realdone=real.done # File of names of realization subdirs completed
-outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined # Base dir for combined output
-
-source ./runall.combine_realizations.config # can overwrite above config params
-
-# Helpful patterns for filename matching
-
-yyyy="[0-9][0-9][0-9][0-9]"
-yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
-
-mkdir -p $outdir
-
-while true; do
-
- real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
- if [ _$real = _ ]; then
- break
- fi
-
- echo Combining $real
-
- find $realdir/$real/history/$yyyymmdd |
- grep .nc.$yyyy |
- sed -e 's/.....$//' | sort | uniq > $real.groups
-
- echo Created $real.groups
-
- echo ./combine_realization.sh \
- "$realdir/$real/history/$yyyymmdd" \
- $outdir/$real
-
- ./combine_realization.sh \
- "$realdir/$real/history/$yyyymmdd" \
- $outdir/$real
-
- echo Completed $real
- echo $real >>$realdone
-
-done
-
-echo All combines are done.
-
Modified: SwiftApps/ParVis/HiRAMTools/runall.config
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.config 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/runall.config 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,6 +1,9 @@
+
realtodo=real.todo # File of names of realization subdirs to process
realdone=real.done # File of names of realization subdirs completed
+ # (this should be an empty file at start)
+
outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for yearly output
-# script=combine_realizations.sh
-script=makeyearly_realization.sh
+# script=combine_realization.sh # Script to run for each realization
+script=makeyearly_realization.sh # Script to run for each realization
Deleted: SwiftApps/ParVis/HiRAMTools/runall.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/runall.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,54 +0,0 @@
-#! /bin/sh
-
-bindir=$(dirname $0)
-bindir=$(cd $bindir; pwd)
-
-echo bindir=$bindir
-
-realtodo=real.todo # File of full path names of realization dirs to process
-realdone=real.done # File of full path names of realization dirs completed
-outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for processed output
-
-source $bindir/runall.config # set default params
-if [ -f runall.config ]; then
- source runall.config # set local params
-fi
-
-echo bindir=$bindir script=$script
-
-# Helpful patterns for filename matching
-
-yyyy="[0-9][0-9][0-9][0-9]"
-yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
-
-mkdir -p $outdir
-
-while true; do
-
- real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
- if [ _$real = _ ]; then
- break
- fi
-
- echo RunAll: Processing $real
-
-# find $realdir/$real/history/$yyyymmdd |
-# grep .nc.$yyyy |
-# sed -e 's/.....$//' | sort | uniq > $real.groups
-#
-# echo Created $real.groups
-
- echo Running: $bindir/$script \
- $real \
- $outdir
-
- $bindir/$script \
- $real \
- $outdir
-
- echo RunAll: Completed $real
- echo $real >>$realdone
-
-done
-
-echo RunAll: all realizations are done.
Added: SwiftApps/ParVis/HiRAMTools/runall.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/runall.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -0,0 +1,54 @@
+#! /bin/sh
+
+bindir=$(dirname $0)
+bindir=$(cd $bindir; pwd)
+
+echo bindir=$bindir
+
+realtodo=real.todo # File of full path names of realization dirs to process
+realdone=real.done # File of full path names of realization dirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for processed output
+
+source $bindir/runall.config # set default params
+if [ -f ./runall.config ]; then
+ source ./runall.config # set local params
+fi
+
+echo bindir=$bindir script=$script
+
+# Helpful patterns for filename matching
+
+yyyy="[0-9][0-9][0-9][0-9]"
+yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
+
+mkdir -p $outdir
+
+while true; do
+
+ real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
+ if [ _$real = _ ]; then
+ break
+ fi
+
+ echo RunAll: Processing $real
+
+# find $realdir/$real/history/$yyyymmdd |
+# grep .nc.$yyyy |
+# sed -e 's/.....$//' | sort | uniq > $real.groups
+#
+# echo Created $real.groups
+
+ echo Running: $bindir/$script \
+ $real \
+ $outdir
+
+ $bindir/$script \
+ $real \
+ $outdir
+
+ echo RunAll: Completed $real
+ echo $real >>$realdone
+
+done
+
+echo RunAll: all realizations are done.
Deleted: SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/runall4.combine_realizations.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,48 +0,0 @@
-#! /bin/sh
-
-# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]????
-
-# The following can be made into parameters to this script. Set here for now.
-
-realdir=/intrepid-fs0/users/lzamboni/persistent # Base dir of realizations
-realtodo=real4.todo # File of names of realization subdirs to process
-realdone=real4.done # File of names of realization subdirs completed
-outdir=/intrepid-fs0/users/wilde/persistent/LZ/combined4 # Base dir for combined output
-
-# Helpful patterns for filename matching
-
-yyyy="[0-9][0-9][0-9][0-9]"
-yyyymmdd="${yyyy}[0-9][0-9][0-9][0-9]"
-
-mkdir -p $outdir
-
-while true; do
-
- real=$(diff $realtodo $realdone | grep '^<' | sed -e 's/^..//' | head -1) # next realization to do
- if [ _$real = _ ]; then
- break
- fi
-
- echo Combining $real
-
- find $realdir/$real/history/$yyyymmdd |
- grep .nc.$yyyy |
- sed -e 's/.....$//' | sort | uniq > $real.groups
-
- echo Created $real.groups
-
- echo ./combine_realization.sh \
- "$realdir/$real/history/$yyyymmdd" \
- $outdir/$real
-
- ./combine_realization.sh \
- "$realdir/$real/history/$yyyymmdd" \
- $outdir/$real
-
- echo Completed $real
- echo $real >>$realdone
-
-done
-
-echo All combines are done.
-
Deleted: SwiftApps/ParVis/HiRAMTools/runcombine.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runcombine.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/runcombine.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,5 +0,0 @@
-#! /bin/sh
-
-$HOME/LZ/combine.csh \
- -a /intrepid-fs0/users/lzamboni/persistent/en1eo8Ic1/history \
- -s 19500101 -n 0 -m 30 # -q
Deleted: SwiftApps/ParVis/HiRAMTools/y1.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/y1.sh 2012-06-05 16:45:58 UTC (rev 5803)
+++ SwiftApps/ParVis/HiRAMTools/y1.sh 2012-06-08 18:10:40 UTC (rev 5804)
@@ -1,29 +0,0 @@
-#! /bin/sh
-
-export PATH=$PATH:/soft/apps/nco-3.9.9/bin
-# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/soft/apps/nco-3.9.9/lib
-
-d1=19500101
-d2=19520701
-f=ice_month.nc
-
-o=testyears
-y1=$o/19500101
-y2=$o/19510101
-y3=$o/19520101
-y4=$o/19530101
-y5=$o/19540101
-
-mkdir -p $y1 $y2 $y3 $y4 $y5
-
-ncks -F -d time,1,12 $d1/$f $y1/$f
-
-ncks -F -d time,13,24 $d1/$f $y2/$f
-
-ncks -F -d time,25,30 $d1/$f $y3/$f.a
-ncks -F -d time,1,6 $d2/$f $y3/$f.b
-ncrcat $y3/$f.a $y3/$f.b $y3/$f
-
-ncks -F -d time,7,18 $d2/$f $y4/$f
-
-ncks -F -d time,19,30 $d2/$f $y5/$f
More information about the Swift-commit
mailing list