#!/bin/tcsh
### Change to the current working directory:
#$ -S /bin/tcsh
#$ -cwd
#$ -j y

### SET YOUR JOB NAME HERE
#$ -N WRF-co4-job

### SET YOUR NUMBER OF CPUS HERE
#$ -pe hydra 4

### LOAD ANY NECESSARY MODULES HERE
#source /etc/csh/login.d/modules-tcsh.csh 
module purge
module load wrf-mpich2-new

cat $PE_HOSTFILE

### PUT YOUR MPICH JOB HERE
##  mpiexec.hydra ./wrf.exe

## Do our best to get sufficient stack memory

echo "LIMIT-----"
limit 
echo "ENDLIMIT-----"
limit stacksize unlimited

echo "setting variables 1"

set startdate=200101
##set enddate=200102
set case=test1
set metdir = /cluster/home/lxu/WRF/MET/$startdate
##set startmetdir=/home/lxu/WRF/MET/$startdate
set rundir = /cluster/home/lxu/WRF/WRF_co4/WRF_acasa_ready/run 

echo "cd to rundir, linking stuff"
cd $rundir
##ln -s $startmetdir/met* .
ln -s $metdir/met* .

## Create the namelist

echo "writing namelist"
cat >! namelist.input << EOF
 &time_control
 run_days                            = 31,
 run_hours                           = 0,
 run_minutes                         = 0,
 run_seconds                         = 0,
 start_year                          = 2001, 2001, 2000,
 start_month                         = 01,   01,   01,
 start_day                           = 01,   01,   24,
 start_hour                          = 00,   00,   12,
 start_minute                        = 00,   00,   00,
 start_second                        = 00,   00,   00,
 end_year                            = 2001, 2001, 2000, end_month                           = 01,   01,   01, end_day                             = 31,   31,   25,
 end_hour                            = 00,   00,   12,
 end_minute                          = 00,   00,   00,
 end_second                          = 00,   00,   00,
 interval_seconds                    = 10800,
 input_from_file                     = .true.,.true.,.true.,
 history_interval                    = 180,  180,   60,
 frames_per_outfile                  = 56, 56, 1000,
 restart                             = .false.,
 restart_interval                    = 10080,
 io_form_history                     = 2,
 io_form_restart                     = 2,
 io_form_input                       = 2,
 io_form_boundary                    = 2,
 debug_level                         = 0,
 /

 &domains
 time_step                           = 80,
 time_step_fract_num                 = 0,
 time_step_fract_den                 = 1,
 max_dom                             = 2,
 e_we                                = 100,   149,   94,
 e_sn                                = 100,   149,    91,
 e_vert                              = 31,    31,    28,
 num_metgrid_levels                  = 30,
 num_metgrid_soil_levels             = 4,
 dx                                  = 16000, 4000,  3333.33,
 dy                                  = 16000, 4000,  3333.33,
 grid_id                             = 1,     2,     3,
 parent_id                           = 0,     1,     2,
 i_parent_start                      = 1,     28,    30,
 j_parent_start                      = 1,     32,    30,
 parent_grid_ratio                   = 1,     4,     3,
 parent_time_step_ratio              = 1,     4,     3,
 feedback                            = 1,
 smooth_option                       = 0,
 p_top_requested                     = 10000,
 /

 &physics
 mp_physics                          = 3,     3,     3,
 ra_lw_physics                       = 1,     1,     1,
 ra_sw_physics                       = 1,     1,     1,
 radt                                = 30,    30,    30,
 sf_sfclay_physics                   = 2,     2,     1,
 sf_surface_physics                  = 4,     4,     2,
 bl_pbl_physics                      = 2,     2,     1,
 bldt                                = 30,     30,     0,
 cu_physics                          = 1,     1,     0,
 cudt                                = 5,     5,     5,
 isfflx                              = 1,
 ifsnow                              = 0,
 icloud                              = 1,
 surface_input_source                = 1,
 num_soil_layers                     = 4,
 sf_urban_physics                    = 0,
 maxiens                             = 1,
 maxens                              = 3,
 maxens2                             = 3,
 maxens3                             = 16,
 ensdim                              = 144,
 num_land_cat			     = 20,
 num_soil_cat                        = 16,
 /

 &fdda
grid_fdda = 1,
gfdda_inname = "wrffdda_d<domain>",
gfdda_interval_m = 360,
gfdda_end_h = 24,
io_form_gfdda = 2,
 /

 &dynamics
 w_damping                           = 0,
 diff_opt                            = 1,
 km_opt                              = 4,
 diff_6th_opt                        = 0,      0,      0,
 diff_6th_factor                     = 0.12,   0.12,   0.12,
 base_temp                           = 290.
 damp_opt                            = 0,
 zdamp                               = 5000.,  5000.,  5000.,
 dampcoef                            = 0.2,    0.2,    0.2
 khdif                               = 0,      0,      0,
 kvdif                               = 0,      0,      0,
 non_hydrostatic                     = .true., .true., .true.,
 moist_adv_opt                       = 1,      1,      1,     
 scalar_adv_opt                      = 1,      1,      1,     
 /

 &bdy_control
 spec_bdy_width                      = 5,
 spec_zone                           = 1,
 relax_zone                          = 4,
 specified                           = .true., .false.,.false.,
 nested                              = .false., .true., .true.,
 /

 &grib2
 /

 &namelist_quilt
 nio_tasks_per_group = 0,
 nio_groups = 1,
 /
EOF

echo "Before running WRF"

## Run real.exe
### Machine file:

### Put your MPICH code here
##mpiexec -machinefile $TMPDIR/machines -n $NSLOTS ./real.exe

which mpiexec
mpiexec -verbose ./real.exe

### Cleanup if needed


rm rsl*

##mpiexec -machinefile $TMPDIR/machines -n $NSLOTS ./wrf.exe
mpiexec -verbose ./wrf.exe

exit 0

##mpirun -np 48 $rundir/real.exe >&! $rundir/$case.real.out || echo "WRF real failed" && exit 1
##sudo /cluster/opt/mvapich2-1.5rc1-ofed-1.5.1/bin/mpirun -np 28 $rundir/real.exe >&! $rundir/$case.real.out || echo "WRF real failed" && exit 1

##echo "real.exe successful"

##exit 1
##sudo rm rsl.*


## Run wrf.exe
##sudo /cluster/opt/mvapich2-1.5rc1-ofed-1.5.1/bin/mpirun -np 28 $rundir/wrf.exe >&! $rundir/$case.wrf.out || echo "WRF wrf failed" && exit 1

##sudo /cluster/opt/mvapich2-1.5rc1-ofed-1.5.1/bin/mpirun -np 32 ./wrf.exe < /dev/null >& wrf.log &

echo "wrf.exe successful"

exit 0