[Swift-commit] r7101 - SwiftApps/Swift-MapRed/psims
ketan at ci.uchicago.edu
ketan at ci.uchicago.edu
Sun Sep 22 12:08:38 CDT 2013
Author: ketan
Date: 2013-09-22 12:08:38 -0500 (Sun, 22 Sep 2013)
New Revision: 7101
Added:
SwiftApps/Swift-MapRed/psims/RunpSIMS.sh
SwiftApps/Swift-MapRed/psims/RunpSIMS.swift
SwiftApps/Swift-MapRed/psims/cf
SwiftApps/Swift-MapRed/psims/midway.xml
SwiftApps/Swift-MapRed/psims/tc.data
Log:
Added: SwiftApps/Swift-MapRed/psims/RunpSIMS.sh
===================================================================
--- SwiftApps/Swift-MapRed/psims/RunpSIMS.sh (rev 0)
+++ SwiftApps/Swift-MapRed/psims/RunpSIMS.sh 2013-09-22 17:08:38 UTC (rev 7101)
@@ -0,0 +1,156 @@
+#!/bin/bash -x
+
+out=$1
+executable=$2
+outtypes=$3
+postprocess=$4
+tappwth=$5
+tappinp=$6
+tappcamp=$7
+grid1=$8
+grid2=$9
+model=${10}
+shift 10
+
+# All other arguments beyond this are lists of input files
+#mkdir -p data
+for var in "$@" ; do
+ for file in $var ; do
+# ln -s $PWD/$file data/$( basename $file )
+ ln -s $PWD/$file $( basename $file )
+done ; done
+
+# Make all python, perl, and bash scripts executable. If tapps or other stuff in any
+# other language, add suffix here.
+chmod +x *.py *.pl *.sh *.EXE *.exe
+
+##################################################################################
+# Run tappcamp application to generate the experiment file from the campaign file.
+# Input campaign file called something like campaign.nc; defines the spatial experiment.
+# This file is a single netcdf file that contains arbitary number of variables
+# that must be changed spatially in the experiment files. The experiment
+# file is model agnostic and always called experiment.json.
+tappcamp="$( echo $tappcamp | sed s/:/' '/g ) --grid1 $grid1 --grid2 $grid2 "
+echo Campaign translator command: $tappcamp
+ls -l experiment.json 1>&2
+#strace -o strace.log
+./$tappcamp
+ls -l experiment.json 1>&2
+#ln -s $PWD/experiment.json data/experiment.json
+
+###############################################################################
+# Run tappinp application to generate the input files (.XXX and .SOL for DSSAT
+# and .apsim for APSIM) from the experiment.json file
+if [ "$model" == "dssat45" ] ; then suff=".SOL .MZX .RIX .WHX .SBX" ; fi
+if [ "$model" == "apsim75" ] ; then suff=".apsim" ; fi
+if [ "$model" == "cenw" ] ; then suff=".PJ!" ; fi
+for sf in $suff ; do ls -l *$sf 1>&2 ; done
+
+tappinp="$( echo $tappinp | sed s/:/' '/g ) "
+echo Input translator command: $tappinp
+./$tappinp # run the input translator app
+
+#for sf in $suff ; do # since tappinp can create multipl files, loop over suffixes
+# for file in $( ls *$sf ) ; do # link the input files to the data directory
+# ln -s $PWD/$file data/$( basename $file )
+# done ; done
+for sf in $suff ; do ls -l *$sf 1>&2 ; done
+
+###############################################################################
+# Run tappwth application to generate the weather file from the .psims file
+# input files usually of form grid1_grid2.psims.nc and outputs files with
+# generic names like GENERIC.WTH (for DSSAT) and Generic.met (for APSIM).
+if [ "$model" == "dssat45" ] ; then suff=".WTH" ; fi
+if [ "$model" == "apsim75" ] ; then suff=".met" ; fi
+if [ "$model" == "cenw" ] ; then suff=".CL!" ; fi
+ls -l *$suff 1>&2
+
+for file in $( ls *.psims.nc ) ; do # build the command line for the weather translator
+ tappwth="$( echo $tappwth | sed s/:/' '/g ) -i $file"
+ echo Weather translator command: $tappwth
+ ./$tappwth # run weather translator app
+done
+#for file in $( ls *$suff ) ; do # link the .wth type files to the data directory
+# ln -s $PWD/$file data/$( basename $file )
+#done
+ls -l *$suff 1>&2
+
+#########################################################
+################# Run the impact model #################
+#cd data
+executable=$( echo $executable | sed s/:/' '/g )
+
+ ###########
+ # DSSAT45 #
+if [ "$model" == "dssat45" ] ; then
+ commandToRun="$executable"
+ echo Executable: $commandToRun
+ ./$commandToRun > RESULT.OUT 2>&1
+fi
+
+ ###########
+ # APSIM75 #
+if [ "$model" == "apsim75" ] ; then
+ tar -xzvf Apsim75.exe.tar.gz # expand exe & files needed to run (have correct permissions)
+ mv *.xml Model/ # If user adds custom [crop].xml file, overwrites the default
+ mv ./Model/Apsim.xml ./
+ ./paths.sh # Sets boost and mono and ld_lib paths for the worker node
+ mono ./Model/ApsimToSim.exe Generic.apsim # converts .apsim file into N .sim files
+ for file in $( ls *.sim ) ; do
+ commandToRun="$executable $file"
+ echo Executable: $commandToRun # run the model on each of the .sim files
+ $commandToRun >> RESULT.out 2>&1
+ done
+fi
+
+ ##########
+ # CENW40 #
+if [ "$model" == "cenw" ]; then
+ mv CenW.CL! CenW-T.CL!
+ for tt in {0..95..5} ; do
+ let "tlin = ( 129 - $tt )*1461/4"
+ lin=$( printf "%0.f\n" $tlin )
+ tailcom="tail -$lin CenW-T.CL! "
+ $tailcom > CenW.CL!
+ for cap in {1..7} ; do
+ tail -3652 CenW-T.CL! >> CenW.CL!
+ done
+ for hh in {1..9} ; do
+ cp PJHead.TMP CenW.PJ!
+ cat Scene-$hh.PJ! >> CenW.PJ!
+ ./$commandToRun > RESULT-T.OUT 2>&1
+ echo "Scenario $hh planting year $tt" >> RESULT.OUT
+ cat RESULT-T.OUT >> RESULT.OUT
+ rm RESULT-T.OUT
+ if [ $tt -eq 0 -a $hh -eq 1 ] ; then
+ head -105 CenW.DT! > CenW-101.DT!
+ else
+ tail -102 CenW.DT! | head -101 >> CenW-101.DT!
+ fi
+ rm CenW.DT!
+ done
+ done
+fi
+exit_status=$?
+#cd ..
+
+# Tar and compress output
+mkdir -p output
+for oo in $( echo $outtypes | sed s/,/' '/g ) ; do
+#cp data/
+cp *$oo output
+done
+tar czf $out output
+
+#################################################################################
+# Extract data from output files into a single 'psims.nc' file with all variables
+if [ _$postprocess != __ ]; then
+ postprocess=$( echo $postprocess | sed s/:/' '/g )
+ mkdir -p parts
+ mkdir -p parts/$grid1
+ postprocessToRun="$postprocess --grid1 $grid1 --grid2 $grid2 --output parts/$grid1/$grid2.psims.nc"
+ ./$postprocessToRun
+ exit_status=$?
+fi
+
+exit 0
Property changes on: SwiftApps/Swift-MapRed/psims/RunpSIMS.sh
___________________________________________________________________
Added: svn:executable
+ *
Added: SwiftApps/Swift-MapRed/psims/RunpSIMS.swift
===================================================================
--- SwiftApps/Swift-MapRed/psims/RunpSIMS.swift (rev 0)
+++ SwiftApps/Swift-MapRed/psims/RunpSIMS.swift 2013-09-22 17:08:38 UTC (rev 7101)
@@ -0,0 +1,34 @@
+type file;
+
+app (file tar_out, file part_out, file outlog, file errlog) RunpSIMS (file scenario_in[], file weather_in[], file soils_in[], file common_in[], file binary_in[], file wrapper_file, string grid1, string grid2)
+{
+ bash "-c" @strcat("chmod +x ./RunpSIMS.sh ; ./RunpSIMS.sh ", @tar_out, " ",
+ @arg("executable"), " ", @arg("outtypes"), " ", @arg("postprocess"), " ",
+ @arg("tappwth"), " ", @arg("tappinp"), " ", @arg("tappcamp"), " ",
+ grid1, " ", grid2, " ", @arg("model"), " ",
+ @scenario_in, " ", @weather_in, " ", @soils_in, " ", @common_in, " ", @binary_in)
+ stdout=@outlog stderr=@errlog;
+}
+
+string gridLists[] = readData("gridList.txt");
+
+foreach g,i in gridLists {
+
+string gridNames[] = @strsplit(g, "/");
+
+ // Input files
+ file scenario_input[] <filesys_mapper; location=@arg("campaign"), pattern="*">;
+ file weather_input[] <filesys_mapper; location=@strcat(@arg("weather"), "/", gridLists[i]), pattern="*">;
+ file soils_input[] <filesys_mapper; location=@strcat(@arg("soils"), "/", gridLists[i]), pattern="*">;
+ file common_input[] <filesys_mapper; location=@arg("refdata"), pattern="*">;
+ file binary_input[] <fixed_array_mapper; files=@arg("bintransfer")>;
+ file wrapper_input <single_file_mapper; file="RunpSIMS.sh">;
+
+ // Output files
+ file stdout_output <single_file_mapper; file=@strcat("logs/", gridLists[i], "o.log")>;
+ file stderr_output <single_file_mapper; file=@strcat("logs/", gridLists[i], "e.log")>;
+ file tar_output <single_file_mapper; file=@strcat("output/", gridLists[i], "output.tar.gz")>;
+ file part_output <single_file_mapper; file=@strcat("parts/", gridLists[i], ".psims.nc")>;
+
+ (tar_output, part_output, stdout_output, stderr_output) = RunpSIMS(scenario_input, weather_input, soils_input, common_input, binary_input, wrapper_input, gridNames[0], gridNames[1]);
+}
Added: SwiftApps/Swift-MapRed/psims/cf
===================================================================
--- SwiftApps/Swift-MapRed/psims/cf (rev 0)
+++ SwiftApps/Swift-MapRed/psims/cf 2013-09-22 17:08:38 UTC (rev 7101)
@@ -0,0 +1,11 @@
+use.wrapper.staging=false
+wrapperlog.always.transfer=true
+sitedir.keep=false
+execution.retries=0
+lazy.errors=true
+status.mode=provider
+use.provider.staging=false
+provider.staging.pin.swiftfiles=false
+foreach.max.threads=2000
+
+#app bash=/bin/bash
Added: SwiftApps/Swift-MapRed/psims/midway.xml
===================================================================
--- SwiftApps/Swift-MapRed/psims/midway.xml (rev 0)
+++ SwiftApps/Swift-MapRed/psims/midway.xml 2013-09-22 17:08:38 UTC (rev 7101)
@@ -0,0 +1,19 @@
+<config>
+ <pool handle="midway">
+ <execution provider="coaster" jobmanager="local:slurm"/>
+ <profile namespace="globus" key="jobsPerNode">16</profile> <!-- Maximum number of RunpSIMS.sh tasks per node -->
+ <profile namespace="globus" key="maxTime">43200</profile> <!-- Maximum time, in seconds of a coaster block -->
+ <profile namespace="globus" key="maxwalltime">00:30:00</profile> <!-- Maximum time, in hh:mm:ss of any one RunpSIMS.sh app call -->
+ <profile namespace="globus" key="lowOverallocation">100</profile> <!-- Don't worry too much about these, they should both always be set to 100 -->
+ <profile namespace="globus" key="highOverallocation">100</profile>
+ <profile namespace="globus" key="queue">sandyb</profile> <!-- Slurm partition to use -->
+ <profile namespace="globus" key="slots">20</profile> <!-- Maximum number of Slurm jobs -->
+ <!-- <profile namespace="globus" key="nodeGranularity">1</profile> --> <!-- Request nodes in blocks of nodeGranularity -->
+ <!-- <profile namespace="globus" key="maxNodes">1</profile> --> <!-- Maximum number of nodes per Slurm job -->
+ <profile namespace="karajan" key="jobThrottle">10</profile> <!-- Multiply this number by 100 to get the total number of active jobs before Swift starts throttling -->
+ <profile namespace="globus" key="slurm.exclusive">false</profile> <!-- When set to false, it's easier to get nodes, but things may run slower -->
+ <profile namespace="karajan" key="initialScore">10000</profile>
+ <filesystem provider="local"/>
+ <workdirectory>/scratch/midway/maheshwari/psims/run003/swiftwork</workdirectory>
+ </pool>
+</config>
Added: SwiftApps/Swift-MapRed/psims/tc.data
===================================================================
--- SwiftApps/Swift-MapRed/psims/tc.data (rev 0)
+++ SwiftApps/Swift-MapRed/psims/tc.data 2013-09-22 17:08:38 UTC (rev 7101)
@@ -0,0 +1 @@
+midway bash /bin/bash null null null
More information about the Swift-commit
mailing list