[Swift-commit] r5751 - SwiftApps/ParVis/HiRAMTools
wilde at ci.uchicago.edu
wilde at ci.uchicago.edu
Sat Apr 21 09:53:11 CDT 2012
Author: wilde
Date: 2012-04-21 09:53:10 -0500 (Sat, 21 Apr 2012)
New Revision: 5751
Added:
SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh
SwiftApps/ParVis/HiRAMTools/makeyearly.swift
SwiftApps/ParVis/HiRAMTools/runall.config
Modified:
SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
SwiftApps/ParVis/HiRAMTools/runall.sh
Log:
Add scripts to reorganize quarter0decade directory structures into yearly structures.
Added: SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh 2012-04-21 14:53:10 UTC (rev 5751)
@@ -0,0 +1,48 @@
+#! /bin/bash
+
+set -o errexit
+
+# Create a fast (local-filesystem) working dir
+
+mkdir -p /scratch/wilde
+tmp=$(mktemp -d /scratch/wilde/makeyearly_XXXXXX)
+startdir=$PWD
+cd $tmp
+
+echo running in $PWD
+
+qtrdec1=$1 # 1st quarter-decade dir
+qtrdec2=$2 # 2nd quarter-decade dir
+destbase=$3 # base dir for output years
+
+echo destbase=$destbase
+
+export PATH=/soft/apps/cdo-1.4.7/bin:$PATH
+export LD_LIBRARY_PATH=/soft/apps/intel-fc-11.0.074/lib/intel64:/soft/apps/intel-cc-11.0.074/lib/intel64:/soft/apps/netcdf-3.6.3/lib:/usr/lib:/usr/X11R6/lib:/usr/local/lib
+
+fname=$(basename $qtrdec1 .nc)
+firstyear=$(basename $(dirname $qtrdec1) | sed -e 's/....$//')
+
+if echo $fname | egrep -v 'grid_spec|scalar'; then
+ cdo mergetime $qtrdec1 $qtrdec2 ${fname}.nc
+ cdo splityear ${fname}.nc ${fname}.
+ # Yields, eg, from atmos_month.tile1:
+ # atmos_month.tile1.1950.nc
+ # atmos_month.tile1.1951.nc
+ # atmos_month.tile1.1952.nc
+ # atmos_month.tile1.1953.nc
+ # atmos_month.tile1.1954.nc
+fi
+
+for (( y=0; y<5; y++ ))
+do
+ year=$((firstyear+y))
+ yeardir=${year}0101
+ destdir=$destbase/$yeardir
+ mkdir -p $destdir
+ if echo $fname | egrep -v 'grid_spec|scalar'; then
+ dd if=$fname.$year.nc of=$destdir/$fname.nc bs=8M
+ else
+ dd if=$qtrdec1 of=$destdir/$fname.nc bs=8M
+ fi
+done
Property changes on: SwiftApps/ParVis/HiRAMTools/makeyearly-cdo.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/ParVis/HiRAMTools/makeyearly.swift
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly.swift (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly.swift 2012-04-21 14:53:10 UTC (rev 5751)
@@ -0,0 +1,21 @@
+
+type file;
+
+type ncrec {
+ string f1;
+ string f2;
+}
+
+string rundir=@arg("rundir","NO-OUTDIR-SPECIFIED");
+string ncfile=@arg("ncfile","ncfiles");
+
+app makeyearly (string dir, string f1, string f2)
+{
+ makeyearly f1 f2 dir;
+}
+
+ncrec nc[] = readData(ncfile);
+
+foreach n in nc {
+ makeyearly(rundir,n.f1,n.f2);
+}
Modified: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2012-04-17 15:11:35 UTC (rev 5750)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh 2012-04-21 14:53:10 UTC (rev 5751)
@@ -1,23 +1,35 @@
#! /bin/bash
-cdir=$1 # combined dir
-outdir=$2 # outdir for equivalent yearly files
+cdir=$1 # combined dir
+outbase=$2 # base of outdir for equivalent yearly files
+bindir=$(dirname $0)
+bindir=$(cd $bindir; pwd)
+echo bindir=$bindir
+
+# Enable one of the following patterns to name the output dir:
+
+outsuffix=$(basename $cdir) # rname
+outsuffix=$(basename $(dirname $cdir)) # rname/rundir, use rname
+outsuffix=$(basename $(dirname $cdir))/$(basename $cdir) # rname/rundir, use both
+
+# generate the list of files to process: do every half-decade
+
+echo f1 f2 >ncfiles
for y in $(seq 1950 5 1997); do
startyear=${y}0101;
nextyear=$((y+2))0701
+ find $cdir/$startyear -type f | fgrep .nc | sed -e "s,^\\(.*\\)/$startyear/\\(.*\\),\\1/$startyear/\\2 \\1/$nextyear/\\2 $outdir,"
+done >>ncfiles
- find $cdir/$startyear -type f | fgrep .nc
-done >ncfiles
-
-exit
-
# Create new runNNN directory
rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
mkdir $rundir
+# Generate Swift config files (cf, tc, sites.xml):
+
cat >$rundir/cf <<END
wrapperlog.always.transfer=true
@@ -66,18 +78,18 @@
cat >$rundir/tc <<END
-#local combine $PWD/combine.sh
-cobalt combine $PWD/combine.sh
+cobalt makeyearly $bindir/makeyearly-cdo.sh
END
-cp ncfiles makeyearly.swift $rundir
+# Place file list and swift script in run dir, set params, make output dir, and run script
+cp ncfiles $bindir/makeyearly.swift $rundir
+
cd $rundir
echo Running in directory $rundir
-runid=$(basename $rundir)
-mkdir -p $outdir/$runid
+runid=$(basename $rundir) # un-needed?
+outdir=$outbase/$outsuffix/$runid
+mkdir -p $outdir
-swift -config cf -tc.file tc -sites.file sites.xml makeyearly.swift -rundir=$outdir/$runid -ncfiles=ncfiles >& swift.out
-
-
+swift -config cf -tc.file tc -sites.file sites.xml makeyearly.swift -rundir=$outdir -ncfiles=ncfiles >& swift.out
Added: SwiftApps/ParVis/HiRAMTools/runall.config
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.config (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/runall.config 2012-04-21 14:53:10 UTC (rev 5751)
@@ -0,0 +1,6 @@
+realtodo=real.todo # File of names of realization subdirs to process
+realdone=real.done # File of names of realization subdirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for yearly output
+
+# script=combine_realizations.sh
+script=makeyearly_realization.sh
Modified: SwiftApps/ParVis/HiRAMTools/runall.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/runall.sh 2012-04-17 15:11:35 UTC (rev 5750)
+++ SwiftApps/ParVis/HiRAMTools/runall.sh 2012-04-21 14:53:10 UTC (rev 5751)
@@ -1,14 +1,21 @@
#! /bin/sh
-# ls -1d en*/history/[0-9][0-9][0-9][0-9]???? eur$ ls -1d en*/history/[0-9][0-9][0-9][0-9]????
+bindir=$(dirname $0)
+bindir=$(cd $bindir; pwd)
-# The following can be made into parameters to this script. Set here for now.
+echo bindir=$bindir
-realdir=/intrepid-fs0/users/wilde/persistent/LZ/combined # Base dir of realizations
-realtodo=yearly.todo # File of names of realization subdirs to process
-realdone=yearly.done # File of names of realization subdirs completed
-outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for combined output
+realtodo=real.todo # File of full path names of realization dirs to process
+realdone=real.done # File of full path names of realization dirs completed
+outdir=/intrepid-fs0/users/wilde/persistent/LZ/yearly # Base dir for processed output
+source $bindir/runall.config # set default params
+if [ -f runall.config ]; then
+ source runall.config # set local params
+fi
+
+echo bindir=$bindir script=$script
+
# Helpful patterns for filename matching
yyyy="[0-9][0-9][0-9][0-9]"
@@ -25,24 +32,23 @@
echo RunAll: Processing $real
- find $realdir/$real/history/$yyyymmdd |
- grep .nc.$yyyy |
- sed -e 's/.....$//' | sort | uniq > $real.groups
+# find $realdir/$real/history/$yyyymmdd |
+# grep .nc.$yyyy |
+# sed -e 's/.....$//' | sort | uniq > $real.groups
+#
+# echo Created $real.groups
- echo Created $real.groups
+ echo Running: $bindir/$script \
+ $real \
+ $outdir
- echo ./makeyearly_realization.sh \
- "$realdir/$real/$yyyymmdd" \
- $outdir/$real
+ $bindir/$script \
+ $real \
+ $outdir
- ./makeyearly_realization.sh \
- "$realdir/$real/$yyyymmdd" \
- $outdir/$real
-
echo RunAll: Completed $real
echo $real >>$realdone
done
echo RunAll: all realizations are done.
-
More information about the Swift-commit
mailing list