[Swift-commit] r6009 - SwiftApps/ParVis/HiRAMTools

[tmoore at ci.uchicago.edu]

Author: tmoore
Date: 2012-11-08 08:41:23 -0600 (Thu, 08 Nov 2012)
New Revision: 6009

Added:
   SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh
Log:
added file not included in a previous commit - a version of makeyearly_realization script designed to work with the 2-year realization files

Added: SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh
===================================================================
--- SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh	                        (rev 0)
+++ SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh	2012-11-08 14:41:23 UTC (rev 6009)
@@ -0,0 +1,113 @@
+#! /bin/bash
+
+cdir=$1     # combined dir
+outbase=$2  # base of outdir for equivalent yearly files
+
+bindir=$(dirname $0)
+bindir=$(cd $bindir; pwd)
+
+echo bindir=$bindir
+
+# Enable one of the following patterns to name the output dir:
+
+outsuffix=$(basename $cdir)                              # cdir ends in rname, use rname
+outsuffix=$(basename $(dirname $cdir))                   # cdir ends in rname/rundir, use rname
+outsuffix=$(basename $(dirname $cdir))/$(basename $cdir) # cdir ends in rname/rundir, use rname/rundir
+                                                         # Default assumes cdir is a rundir from combine script
+                                                         # New rundir will be appended to end of outsuffix
+
+STRIPPATH="\/done"
+outsuffix=${outsuffix//$STRIPPATH/}
+# generate the list of files to process: do every half-decade
+
+echo f1  >ncfiles
+for y in $(seq 1950 2 1998); do
+  startyear=${y}0101; 
+#  nextyear=$((y+1))0101
+
+#this line does the following at sed
+#    ^\\(.*\\)/$startyear/\\(.*\\), searches for everything before startyear and saves it to 1
+#    \\(.*\\) searches for everything after startyear and saves as 2
+# builds a new line with full path for first file, full path for second and targets the output directory
+#  find $cdir/$startyear -type f | fgrep .nc | sed -e "s,^\\(.*\\)/$startyear/\\(.*\\),\\1/$startyear/\\2 \\1/$nextyear/\\2 $outdir,"
+
+#in the 2 year version, we only want the first file
+	find $cdir/$startyear -type f | fgrep .nc | sed -e "s,^\\(.*\\)/$startyear/\\(.*\\),\\1/$startyear/\\2 $outdir,"
+
+done >>ncfiles
+#now we have a list of the 2 year files and the outdir
+
+# Create new runNNN directory
+
+rundir=$( echo run??? | sed -e 's/^.*run//' | awk '{ printf("run%03d\n", $1+1)}' )
+mkdir $rundir
+
+# Generate Swift config files (cf, tc, sites.xml):
+
+cat >$rundir/cf <<END
+
+wrapperlog.always.transfer=true
+sitedir.keep=true
+execution.retries=10
+lazy.errors=true
+#status.mode=provider
+use.wrapper.staging=false
+
+END
+
+cat >$rundir/sites.xml <<END
+
+<config>
+
+   <pool handle="local">
+     <execution provider="local" />
+     <profile namespace="karajan" key="jobThrottle">.01</profile>
+     <profile namespace="karajan" key="initialScore">10000</profile>
+     <filesystem provider="local"/>
+     <workdirectory>$PWD/swiftwork</workdirectory>
+   </pool>
+
+  <pool handle="cobalt">
+    <execution provider="coaster" jobmanager="local:cobalt" url="eureka.alcf.anl.gov" />
+    <filesystem provider="local" />
+    <!-- <profile namespace="globus"key="internalHostname">172.17.3.11</profile> -->
+    <!-- <profile namespace="globus"  key="project">MTCScienceApps</profile> -->
+    <profile namespace="globus"  key="project">SU_Climate</profile>
+    <profile namespace="globus"  key="queue">default</profile>
+    <profile namespace="karajan" key="jobthrottle">2.56</profile>
+    <profile namespace="karajan" key="initialScore">10000</profile>
+    <profile namespace="globus"  key="jobsPerNode">8</profile>
+    <!-- <profile namespace="globus"  key="workerLoggingLevel">DEBUG</profile> -->
+    <!-- <profile namespace="globus"  key="workerLoggingDirectory">/home/jonmon/Workspace/Swift/tests/catsnsleep</profile> -->
+    <profile namespace="globus"  key="slots">32</profile>
+    <profile namespace="globus"  key="maxTime">28000</profile> <!-- ~ 8hrs -->
+    <profile namespace="globus"  key="maxWallTime">00:20:00</profile>
+    <profile namespace="globus"  key="lowoverallocation">100</profile>
+    <profile namespace="globus"  key="highoverallocation">100</profile>
+    <profile namespace="globus"  key="nodeGranularity">1</profile>
+    <profile namespace="globus"  key="maxNodes">1</profile>
+    <workdirectory>$PWD/$rundir/swiftwork</workdirectory>
+  </pool>
+
+</config>
+
+END
+
+cat >$rundir/tc <<END
+
+cobalt makeyearly_2yr $bindir/makeyearly_2yr.sh
+
+END
+
+# Place file list and swift script in run dir, set params, make output dir, and run script
+
+cp ncfiles $bindir/makeyearly2yr.swift $rundir
+
+cd $rundir
+echo Running in directory $rundir
+runid=$(basename $rundir) # un-needed?
+outdir=$outbase/$outsuffix/$runid
+mkdir -p $outdir
+
+swift -config cf -tc.file tc -sites.file sites.xml makeyearly2yr.swift -rundir=$outdir -ncfiles=ncfiles >& swift.out
+


Property changes on: SwiftApps/ParVis/HiRAMTools/makeyearly_realization_2yr.sh
___________________________________________________________________
Added: svn:executable
   + *