module purge

#LOAD OPENFOAM
export OPENFOAM_root=/ccs/home/arpans/OpenFOAM
#source $OPENFOAM_root/bash_OFv2312
module load PrgEnv-gnu
module load gcc
#module load valgrind4hpc
module load xpmem
module load craype-x86-trento
module load amd-mixed/5.7.1 cpe/23.09
module load cray-mpich/8.1.26 #/8.1.28
source ~/OpenFOAM/OpenFOAM-v2312/etc/bashrc

#LOAD GPU ENABLING TOOLS
#module load PrgEnv-gnu
module load craype-accel-amd-gfx90a
module unload darshan-runtime
#module load amd-mixed/5.7.1
#module load amd-mixed/5.6.0
#module load amd-mixed/5.7.1 cpe/23.12 #From Steve Abbott
export MPICH_GPU_SUPPORT_ENABLED=0
export HIPFLAGS=--offload-arch=gfx90a

#INCLUDE FRONTIER AMD LIBS
#export LIBRARY_PATH=/opt/rocm-5.7.1/llvm/lib
export PATH=$PATH:-I${MPICH_DIR}/include
export LIBRARY_PATH=$LIBRARY_PATH:${PE_MPICH_GTL_DIR_amd_gfx90a}
export LIBRARY_PATH=$LIBRARY_PATH:-L${MPICH_DIR}/lib 
export LIBRARY_PATH=$LIBRARY_PATH:${PE_MPICH_GTL_LIBS_amd_gfx90a}
export LIBRARY_PATH=$LIBRARY_PATH:-lmpi
export LD_LIBRARY_PATH=${CRAY_LD_LIBRARY_PATH}:${LD_LIBRARY_PATH} #From Steve Abbott

export PE_MPICH_GTL_DIR_amd_gfx90a="-L${CRAY_MPICH_ROOTDIR}/gtl/lib"
export PE_MPICH_GTL_LIBS_amd_gfx90a="-lmpi_gtl_hsa"

#export GTLDIR=PE_MPICH_GTL_DIR_amd_gfx90a
#export GTLLIBS=PE_MPICH_GTL_LIBS_amd_gfx90a

#SET PATHS FOR PETSC
export PETSC_DIR=$OPENFOAM_root/petsc
export PETSC_ARCH_PATH=$PETSC_DIR/$WM_OPTIONS
export PETSC_LIB_DIR=$PETSC_DIR/linux64GccDPInt32Opt/lib/
export PETSC_INC_DIR=$PETSC_DIR/linux64GccDPInt32Opt/include/
export LD_LIBRARY_PATH=$PETSC_DIR/linux64GccDPInt32Opt/lib/:$LD_LIBRARY_PATH

#SET PATHS FOR PETSC4FOAM
#export PETSC_ARCH_PATH=/ccs/home/arpans/OpenFOAM/ThirdParty-v2312/platforms/linux64GccDPInt32/petsc-3.16.2