[Swift-commit] r7117 - SwiftApps/ParVis/HiRAMTools
tmoore at ci.uchicago.edu
tmoore at ci.uchicago.edu
Mon Sep 30 22:42:35 CDT 2013
Author: tmoore
Date: 2013-09-30 22:42:34 -0500 (Mon, 30 Sep 2013)
New Revision: 7117
Added:
SwiftApps/ParVis/HiRAMTools/averager.sh
SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.bak
SwiftApps/ParVis/HiRAMTools/genpfrepps1deg6month.sh
SwiftApps/ParVis/HiRAMTools/genpfreppsMain.sh
SwiftApps/ParVis/HiRAMTools/genpfreppsSelected.sh
SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh.bak
SwiftApps/ParVis/HiRAMTools/pfrepps.swift.orig
Modified:
SwiftApps/ParVis/HiRAMTools/genpfrepps.sh
Log:
updated hiram tools with backup copies of modified scripts
Added: SwiftApps/ParVis/HiRAMTools/averager.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/averager.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/averager.sh 2013-10-01 03:42:34 UTC (rev 7117)
@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# $1 should be name of the realization
+
+PATHTODIR="/intrepid-fs0/projects/SU_Climate/post/data/pfrepp/"
+PATHTODIR+=$1
+PATHTODIR+=/atmos/av/annual_1year
+
+PATHTOTARGETDIR="/intrepid-fs0/projects/SU_Climate/post/data/averages/"
+PATHTOTARGETDIR+=$1
+PATHTOTARGETFILE="$PATHTOTARGETDIR/atmos.1979-1999.ann.nc"
+
+echo $PATHTODIR
+echo $PATHTOTARGETDIR
+echo $PATHTOTARGETFILE
+#build the target directory if needed.
+if [ ! -d "$PATHTOTARGETDIR" ]; then
+ mkdir $PATHTOTARGETDIR
+fi
+
+if [ -f "$PATHTOTARGETFILE" ]; then
+ rm $PATHTOTARGETFILE
+fi
+
+cd $PATHTODIR
+
+ncra -v precip atmos.1979.ann.nc atmos.1980.ann.nc atmos.1981.ann.nc atmos.1982.ann.nc atmos.1983.ann.nc atmos.1984.ann.nc atmos.1985.ann.nc atmos.1986.ann.nc atmos.1987.ann.nc atmos.1988.ann.nc atmos.1989.ann.nc atmos.1990.ann.nc atmos.1991.ann.nc atmos.1992.ann.nc atmos.1993.ann.nc atmos.1994.ann.nc atmos.1995.ann.nc atmos.1996.ann.nc atmos.1997.ann.nc atmos.1998.ann.nc atmos.1999.ann.nc "$PATHTOTARGETFILE"
Property changes on: SwiftApps/ParVis/HiRAMTools/averager.sh
___________________________________________________________________
Added: svn:executable
+ *
Modified: SwiftApps/ParVis/HiRAMTools/genpfrepps.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/genpfrepps.sh 2013-09-30 22:23:53 UTC (rev 7116)
+++ SwiftApps/ParVis/HiRAMTools/genpfrepps.sh 2013-10-01 03:42:34 UTC (rev 7117)
@@ -16,7 +16,7 @@
-i /intrepid-fs0/users/lzamboni/persistent/bin/fms/pfrepp/remap_file/version1_C90_to_180x90/C90_mosaic.nc \
-x 360 -y 180 \
-r /intrepid-fs0/projects/SU_Climate/post/HiRAMTools/C90_to_360x180/C90_to_360x180_remap_file.nc \
- -m 12 -n 50 -q \
+ -m 12 -n 4 -q \
-u
echo $0: scriptfilter=$scriptfilter
Added: SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.bak
===================================================================
--- SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.bak (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.bak 2013-10-01 03:42:34 UTC (rev 7117)
@@ -0,0 +1,59 @@
+#! /bin/sh
+
+realid=$1
+realdir=$2
+outdir=$3
+scriptlist=$PWD/$4
+scriptfilter=$5
+
+mkdir -p $outdir/$realid/scripts
+cd $outdir/$realid/scripts
+
+tcsh /home/wilde/LZ/HiRAMTools/pfrepp.csh \
+ -a $realdir \
+ -b $outdir/$realid \
+ -s 1950 \
+ -i /intrepid-fs0/users/lzamboni/persistent/bin/fms/pfrepp/remap_file/C90_to_180x90/C90_mosaic.nc \
+ -x 180 -y 90 \
+ -r /intrepid-fs0/users/lzamboni/persistent/bin/fms/pfrepp/remap_file/C90_to_180x90/C90_to_180x90.nc \
+ -m 12 -n 50 -q \
+ -u
+
+echo $0: scriptfilter=$scriptfilter
+
+if [ "_$scriptfilter" != _ ]; then
+ echo $0: using scriptfilter
+ scripts=$(/bin/ls -1 pfrepp_script* | sh -c "$scriptfilter" )
+else
+ scripts=$(/bin/ls -1 pfrepp_script* )
+fi
+
+echo id path >$scriptlist
+for f in $scripts; do
+ echo $realid $f >>$scriptlist
+done
+
+exit
+
+# -a /intrepid-fs0/users/lzamboni/persistent/yearly-nco/en1eo12Ic3/run001/run001/ \
+
+# datasets needed by LZ:
+
+Pfrepp-month.time_average
+Pfrepp-month.time_series
+Pfrepp-daily.time_series (4X???)
+Pfrepp-8xdaily.time_series
+Pfrepp-8xdaily_instant.time_series
+Pfrepp-4xdaily.time_series
+
+which means:
+
+pfrepp_script.atmos_month.time_average.1950*
+pfrepp_script.atmos_month.time_series.1950*
+pfrepp_script.atmos_4xdaily.time_series.1950*
+pfrepp_script.atmos_8xdaily.time_series.1950*
+pfrepp_script.atmos_8xdaily_instant.time_series.1950*
+pfrepp_script.atmos_daily.time_series.1950*
+
+pfrepp_script.ice_month.time_average.1950*
+pfrepp_script.ice_month.time_series.1950*
Property changes on: SwiftApps/ParVis/HiRAMTools/genpfrepps.sh.bak
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/ParVis/HiRAMTools/genpfrepps1deg6month.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/genpfrepps1deg6month.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/genpfrepps1deg6month.sh 2013-10-01 03:42:34 UTC (rev 7117)
@@ -0,0 +1,59 @@
+#! /bin/sh
+
+realid=$1
+realdir=$2
+outdir=$3
+scriptlist=$PWD/$4
+scriptfilter=$5
+
+mkdir -p $outdir/$realid/scripts
+cd $outdir/$realid/scripts
+
+tcsh /home/wilde/LZ/HiRAMTools/pfrepp.csh \
+ -a $realdir \
+ -b $outdir/$realid \
+ -s 1950 \
+ -i /intrepid-fs0/users/lzamboni/persistent/bin/fms/pfrepp/remap_file/version1_C90_to_180x90/C90_mosaic.nc \
+ -x 360 -y 180 \
+ -r /intrepid-fs0/projects/SU_Climate/post/HiRAMTools/C90_to_360x180/C90_to_360x180_remap_file.nc \
+ -m 6 -n 50 -q \
+ -u
+
+echo $0: scriptfilter=$scriptfilter
+
+if [ "_$scriptfilter" != _ ]; then
+ echo $0: using scriptfilter
+ scripts=$(/bin/ls -1 pfrepp_script* | sh -c "$scriptfilter" )
+else
+ scripts=$(/bin/ls -1 pfrepp_script* )
+fi
+
+echo id path >$scriptlist
+for f in $scripts; do
+ echo $realid $f >>$scriptlist
+done
+
+exit
+
+# -a /intrepid-fs0/users/lzamboni/persistent/yearly-nco/en1eo12Ic3/run001/run001/ \
+
+# datasets needed by LZ:
+
+Pfrepp-month.time_average
+Pfrepp-month.time_series
+Pfrepp-daily.time_series (4X???)
+Pfrepp-8xdaily.time_series
+Pfrepp-8xdaily_instant.time_series
+Pfrepp-4xdaily.time_series
+
+which means:
+
+pfrepp_script.atmos_month.time_average.1950*
+pfrepp_script.atmos_month.time_series.1950*
+pfrepp_script.atmos_4xdaily.time_series.1950*
+pfrepp_script.atmos_8xdaily.time_series.1950*
+pfrepp_script.atmos_8xdaily_instant.time_series.1950*
+pfrepp_script.atmos_daily.time_series.1950*
+
+pfrepp_script.ice_month.time_average.1950*
+pfrepp_script.ice_month.time_series.1950*
Property changes on: SwiftApps/ParVis/HiRAMTools/genpfrepps1deg6month.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/ParVis/HiRAMTools/genpfreppsMain.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/genpfreppsMain.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/genpfreppsMain.sh 2013-10-01 03:42:34 UTC (rev 7117)
@@ -0,0 +1,59 @@
+#! /bin/sh
+
+realid=$1
+realdir=$2
+outdir=$3
+scriptlist=$PWD/$4
+scriptfilter=$5
+
+mkdir -p $outdir/$realid/scripts
+cd $outdir/$realid/scripts
+
+tcsh /home/wilde/LZ/HiRAMTools/pfrepp.csh \
+ -a $realdir \
+ -b $outdir/$realid \
+ -s 1950 \
+ -i /intrepid-fs0/users/lzamboni/persistent/bin/fms/pfrepp/remap_file/version1_C90_to_180x90/C90_mosaic.nc \
+ -x 360 -y 180 \
+ -r /intrepid-fs0/projects/SU_Climate/post/HiRAMTools/C90_to_360x180/C90_to_360x180_remap_file.nc \
+ -m 12 -n 50 -q \
+ -u
+
+echo $0: scriptfilter=$scriptfilter
+
+if [ "_$scriptfilter" != _ ]; then
+ echo $0: using scriptfilter
+ scripts=$(/bin/ls -1 pfrepp_script* | sh -c "$scriptfilter" )
+else
+ scripts=$(/bin/ls -1 pfrepp_script* )
+fi
+
+echo id path >$scriptlist
+for f in $scripts; do
+ echo $realid $f >>$scriptlist
+done
+
+exit
+
+# -a /intrepid-fs0/users/lzamboni/persistent/yearly-nco/en1eo12Ic3/run001/run001/ \
+
+# datasets needed by LZ:
+
+Pfrepp-month.time_average
+Pfrepp-month.time_series
+Pfrepp-daily.time_series (4X???)
+Pfrepp-8xdaily.time_series
+Pfrepp-8xdaily_instant.time_series
+Pfrepp-4xdaily.time_series
+
+which means:
+
+pfrepp_script.atmos_month.time_average.1950*
+pfrepp_script.atmos_month.time_series.1950*
+pfrepp_script.atmos_4xdaily.time_series.1950*
+pfrepp_script.atmos_8xdaily.time_series.1950*
+pfrepp_script.atmos_8xdaily_instant.time_series.1950*
+pfrepp_script.atmos_daily.time_series.1950*
+
+pfrepp_script.ice_month.time_average.1950*
+pfrepp_script.ice_month.time_series.1950*
Property changes on: SwiftApps/ParVis/HiRAMTools/genpfreppsMain.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/ParVis/HiRAMTools/genpfreppsSelected.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/genpfreppsSelected.sh (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/genpfreppsSelected.sh 2013-10-01 03:42:34 UTC (rev 7117)
@@ -0,0 +1,59 @@
+#! /bin/sh
+
+realid=$1
+realdir=$2
+outdir=$3
+scriptlist=$PWD/$4
+scriptfilter=$5
+
+mkdir -p $outdir/$realid/scripts
+cd $outdir/$realid/scripts
+
+tcsh /home/wilde/LZ/HiRAMTools/pfrepp.csh \
+ -a $realdir \
+ -b $outdir/$realid \
+ -s 1950 \
+ -i /intrepid-fs0/users/lzamboni/persistent/bin/fms/pfrepp/remap_file/version1_C90_to_180x90/C90_mosaic.nc \
+ -x 360 -y 180 \
+ -r /intrepid-fs0/projects/SU_Climate/post/HiRAMTools/C90_to_360x180/C90_to_360x180_remap_file.nc \
+ -m 12 -n 5 -q \
+ -u
+
+echo $0: scriptfilter=$scriptfilter
+
+if [ "_$scriptfilter" != _ ]; then
+ echo $0: using scriptfilter
+ scripts=$(/bin/ls -1 pfrepp_script* | sh -c "$scriptfilter" )
+else
+ scripts=$(/bin/ls -1 pfrepp_script* )
+fi
+
+echo id path >$scriptlist
+for f in $scripts; do
+ echo $realid $f >>$scriptlist
+done
+
+exit
+
+# -a /intrepid-fs0/users/lzamboni/persistent/yearly-nco/en1eo12Ic3/run001/run001/ \
+
+# datasets needed by LZ:
+
+Pfrepp-month.time_average
+Pfrepp-month.time_series
+Pfrepp-daily.time_series (4X???)
+Pfrepp-8xdaily.time_series
+Pfrepp-8xdaily_instant.time_series
+Pfrepp-4xdaily.time_series
+
+which means:
+
+pfrepp_script.atmos_month.time_average.1950*
+pfrepp_script.atmos_month.time_series.1950*
+pfrepp_script.atmos_4xdaily.time_series.1950*
+pfrepp_script.atmos_8xdaily.time_series.1950*
+pfrepp_script.atmos_8xdaily_instant.time_series.1950*
+pfrepp_script.atmos_daily.time_series.1950*
+
+pfrepp_script.ice_month.time_average.1950*
+pfrepp_script.ice_month.time_series.1950*
Property changes on: SwiftApps/ParVis/HiRAMTools/genpfreppsSelected.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh.bak
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh.bak (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh.bak 2013-10-01 03:42:34 UTC (rev 7117)
@@ -0,0 +1,106 @@
+#! /bin/bash
+
+cdir=$1 # combined dir
+outbase=$2 # base of outdir for equivalent yearly files
+
+bindir=$(dirname $0)
+bindir=$(cd $bindir; pwd)
+
+echo bindir=$bindir
+
+# Enable one of the following patterns to name the output dir:
+
+outsuffix=$(basename $cdir) # cdir ends in rname, use rname
+outsuffix=$(basename $(dirname $cdir)) # cdir ends in rname/rundir, use rname
+outsuffix=$(basename $(dirname $cdir))/$(basename $cdir) # cdir ends in rname/rundir, use rname/rundir
+ # Default assumes cdir is a rundir from combine script
+ # New rundir will be appended to end of outsuffix
+
+# Strip out the "done" directory in the outsuffix if present
+
+STRIPTEXT="\/done"
+
+outsuffix=${outsuffix//$STRIPTEXT/}
+
+# generate the list of files to process: do every half-decade
+
+echo f1 f2 >ncfiles
+for y in $(seq 1950 5 1997); do
+ startyear=${y}0101;
+ nextyear=$((y+2))0701
+ find $cdir/$startyear -type f | fgrep .nc | sed -e "s,^\\(.*\\)/$startyear/\\(.*\\),\\1/$startyear/\\2 \\1/$nextyear/\\2 $outdir,"
+done >>ncfiles
+
+# Create new runNNN directory
+
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+mkdir $rundir
+
+# Generate Swift config files (cf, tc, sites.xml):
+
+cat >$rundir/cf <<END
+
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=10
+lazy.errors=true
+status.mode=provider
+use.wrapper.staging=false
+
+END
+
+cat >$rundir/sites.xml <<END
+
+<config>
+
+ <pool handle="local">
+ <execution provider="local" />
+ <profile namespace="karajan" key="jobThrottle">.01</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>$PWD/swiftwork</workdirectory>
+ </pool>
+
+ <pool handle="cobalt">
+ <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
+ <filesystem provider="local" />
+ <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
+ <!-- <profile namespace="globus" key="project">MTCScienceApps</profile> -->
+ <profile namespace="globus" key="project">prec_sense</profile>
+ <profile namespace="globus" key="queue">default</profile>
+ <profile namespace="karajan" key="jobthrottle">2.56</profile>
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <profile namespace="globus" key="jobsPerNode">8</profile>
+ <!-- <profile namespace="globus" key="workerLoggingLevel">DEBUG</profile> -->
+ <!-- <profile namespace="globus" key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
+ <profile namespace="globus" key="slots">32</profile>
+ <profile namespace="globus" key="maxTime">28000</profile> <!-- ~ 8hrs -->
+ <profile namespace="globus" key="maxWallTime">00:20:00</profile>
+ <profile namespace="globus" key="lowoverallocation">100</profile>
+ <profile namespace="globus" key="highoverallocation">100</profile>
+ <profile namespace="globus" key="nodeGranularity">1</profile>
+ <profile namespace="globus" key="maxNodes">1</profile>
+ <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
+ </pool>
+
+</config>
+
+END
+
+cat >$rundir/tc <<END
+
+cobalt makeyearly $bindir/makeyearly.sh
+
+END
+
+# Place file list and swift script in run dir, set params, make output dir, and run script
+
+cp ncfiles $bindir/makeyearly.swift $rundir
+
+cd $rundir
+echo Running in directory $rundir
+runid=$(basename $rundir) # un-needed?
+outdir=$outbase/$outsuffix/$runid
+mkdir -p $outdir
+
+swift -config cf -tc.file tc -sites.file sites.xml makeyearly.swift -rundir=$outdir -ncfiles=ncfiles >& swift.out
Property changes on: SwiftApps/ParVis/HiRAMTools/makeyearly_realization.sh.bak
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/ParVis/HiRAMTools/pfrepps.swift.orig
===================================================================
--- SwiftApps/ParVis/HiRAMTools/pfrepps.swift.orig (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/pfrepps.swift.orig 2013-10-01 03:42:34 UTC (rev 7117)
@@ -0,0 +1,60 @@
+type file;
+
+type realization {
+ string path;
+ string id;
+}
+
+type script {
+ string path;
+ string id;
+}
+
+global string outdir = @arg("outdir", "");
+ string reallist = @arg("reallist", "");
+ string scriptfilter = @arg("scriptfilter","grep pfrepp_script.atmos_|grep -v atmos_daily.SKIPME");
+ string dryrun = @arg("dryrun","true"); # default to doing a "dry run"
+
+app (file scripts, file std_out, file std_err) genpfrepps (string realid, string realpath, string filter)
+{
+ genpfrepps realid realpath outdir @scripts filter stdout=@std_out stderr=@std_err;
+}
+
+app (file std_out, file std_err) sh (string script)
+{
+ sh "-c" script stdout=@std_out stderr=@std_err;
+}
+
+if(outdir == "") {
+ tracef("pfrepps.swift: ERROR: output directory (swift argument outdir) not specified.");
+}
+
+if(reallist == "") {
+ tracef("pfrepps.swift: ERROR: realization list (swift argument reallist) not specified");
+}
+
+if(reallist != "" && outdir != "") {
+ realization reali[] = readData(reallist);
+ foreach r,i in reali {
+ tracef("%s %s\n", r.id, r.path);
+ file sofile <single_file_mapper; file=@strcat(outdir,"/",r.id,".gen.out")>;
+ file sefile <single_file_mapper; file=@strcat(outdir,"/",r.id,".gen.err")>;
+ file scriptlist <single_file_mapper; file=@strcat(outdir,"/",r.id,"/scripts/scriptlist")>;
+ (scriptlist,sofile,sefile) = genpfrepps(r.id, r.path, scriptfilter);
+ script scripts[] = readData(scriptlist);
+ foreach s in scripts {
+ string cmd = @strcat(outdir,"/",r.id,"/scripts/",s.path);
+ file ssofile <single_file_mapper; file=@strcat(s.path,".out")>;
+ file ssefile <single_file_mapper; file=@strcat(s.path,".err")>;
+ if( dryrun != "true" ) {
+ (ssofile, ssefile) = sh(cmd);
+ }
+ else {
+ tracef("Running script: %s: %s\n", s.id, s.path);
+ }
+ }
+ }
+}
+else {
+ tracef("pfrepps.swift: EXIT: exiting due to argument errors.");
+}
\ No newline at end of file
More information about the Swift-commit
mailing list