[petsc-users] Problem with NVIDIA compiler and OpenACC

[Satish Balay]

Or you can skip fortran - if you are not using PETSc from it [or any external package that requires it], but you would need cxx for cuda

--with-fc=0 --download-f2cblaslapack --with-cxx=0 --with-cudac=0

or

--with-fc=0 --download-f2cblaslapack --with-cudac=nvcc LIBS=-lstdc++

Satish

On Fri, 5 Apr 2024, Satish Balay wrote:

> >>>
> Executing: mpifort  -o /tmp/petsc-nopi85m9/config.compilers/conftest  -v   -KPIC -O2 -g /tmp/petsc-nopi85m9/config.compilers/conftest.o
> stdout:
> Export NVCOMPILER=/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7
> Export PGI=/software/sse2/tetralith_el9/manual/nvhpc/23.7
> /software/sse2/generic/manual/ssetools/v1.9.5/wrappers/ld /usr/lib64/crt1.o /usr/lib64/crti.o /software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib/trace_init.o /usr/lib/gcc/x86_64-redhat-linux/11//crtbegin.o /software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib/f90main.o --eh-frame-hdr -m elf_x86_64 -dynamic-linker /lib64/ld-linux-x86-64.so.2 -T /software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib/nvhpc.ld -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/mpi/lib -L/software/sse2/tetralith_el9/manual/FFTW/3.3.10/nv23.7/hpc1/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nvshmem/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nccl/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/mpi/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/math_libs/lib64 -L/soft
 ware/sse
 2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/extras/qd/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/extras/CUPTI/lib64 -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/lib64 -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib -L/usr/lib64 -L/usr/lib/gcc/x86_64-redhat-linux/11/ /tmp/petsc-nopi85m9/config.compilers/conftest.o -rpath /software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/mpi/lib -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -lmpi -rpath /software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib -o /tmp/petsc-nopi85m9/config.compilers/conftest -L/usr/lib/gcc/x86_64-redhat-linux/11//../../../../lib64 -lnvf -lnvomp -ldl --as-needed -lnvhpcatm -latomic --no-as-needed -lpthread -lnvcpumath -lnsnvc -lnvc -lrt -lpthread -lgcc -lc -lgcc_s -lm /us
 r/lib/gc
 c/x86_64-redhat-linux/11//crtend.o /usr/lib64/crtn.o
> 
>           compilers: Libraries needed to link Fortran code with the C linker: ['-Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/mpi/lib', '-L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/mpi/lib', '-Wl,-rpath,/software/sse2/tetralith_el9/manual/FFTW/3.3.10/nv23.7/hpc1/lib', '-L/software/sse2/tetralith_el9/manual/FFTW/3.3.10/nv23.7/hpc1/lib', '-Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nvshmem/lib', '-L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nvshmem/lib', '-Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nccl/lib', '-L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nccl/lib', '-Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/math_libs/lib64', '-L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/math_libs/lib64', '-Wl,-rpath,/sof
 tware/ss
 e2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib', '-L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib', '-Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/extras/qd/lib', '-L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/extras/qd/lib', '-Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/extras/CUPTI/lib64', '-L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/extras/CUPTI/lib64', '-Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/lib64', '-L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/lib64', '-Wl,-rpath,/usr/lib/gcc/x86_64-redhat-linux/11', '-L/usr/lib/gcc/x86_64-redhat-linux/11', '-Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/mpi/lib', '-lmpi_usempif08', '-lmpi_usempi_ignore_tkr', '-lmpi_mpifh', '-lmpi', '-Wl,-rpath,/
 software
 /sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23..7/compilers/lib', '-lnvf', '-lnvomp', '-ldl', '-lnvhpcatm', '-latomic', '-lpthread', '-lnvcpumath', '-lnsnvc', '-lnvc', '-lrt', '-lgcc_s', '-lm']
> 
> 
> PETSC_WITH_EXTERNAL_LIB = -Wl,-rpath,/proj/nsc/users/bramkamp/petsc_install/petsc_barry_fix_nvclib_no_cuda/lib -L/proj/nsc/users/bramkamp/petsc_install/petsc_barry_fix_nvclib_no_cuda/lib -Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/mpi/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/mpi/lib -Wl,-rpath,/software/sse2/tetralith_el9/manual/FFTW/3.3.10/nv23.7/hpc1/lib -L/software/sse2/tetralith_el9/manual/FFTW/3.3.10/nv23.7/hpc1/lib -Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nvshmem/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nvshmem/lib -Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nccl/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/comm_libs/nccl/lib -Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/math_libs/lib64 -L/software/sse2/tetr
 alith_el
 9/manual/nvhpc/23.7/Linux_x86_64/23.7/math_libs/lib64 -Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/lib -Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/extras/qd/lib -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/compilers/extras/qd/lib -Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/extras/CUPTI/lib64 -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/extras/CUPTI/lib64 -Wl,-rpath,/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/lib64 -L/software/sse2/tetralith_el9/manual/nvhpc/23.7/Linux_x86_64/23.7/cuda/lib64 -Wl,-rpath,/usr/lib/gcc/x86_64-redhat-linux/11 -L/usr/lib/gcc/x86_64-redhat-linux/11 -lpetsc -lflapack -lfblas -lX11 -lmpi_usempif08 -lmpi_usempi_ignore_tkr -lmpi_mpifh -lmpi -lnvf -lnvomp -ldl -lnvhpcatm -latomic -lp
 thread -
 lnvcpumath -lnsnvc -lnvc -lrt -lgcc_s -lm -lstdc++ -lquadmath
> <<<
> 
> You'll probably want to skip lot more than just -lnvc
> 
> Try the following and see if it works
> 
> --with-cudac=0 LIBS="-lmpi_mpifh -lnvf -lstdc++"
> 
> [or specify the correct list of libraries for the fortran MPI/Compiler libraries - with dependencies - as needed]
> 
> Satish
> 
> On Fri, 5 Apr 2024, Frank Bramkamp wrote:
> 
> > Dear Barry, I tried your fix for -lnvc. Unfortunately it did not work so far. Here I send you the configure. log file again. One can see that you try to skip something, but later it still always includes -lnvc for the linker. In the
> > file petscvariables
> > ZjQcmQRYFpfptBannerStart
> > This Message Is From an External Sender
> > This message came from outside your organization.
> >  
> > ZjQcmQRYFpfptBannerEnd
> > 
> > Dear Barry,
> > 
> > I tried your fix for -lnvc.  Unfortunately it did not work so far.
> > Here I send you the configure.log file again.
> > 
> > One can see that you try to skip something, but later it still always includes -lnvc for the linker.
> > In the file petscvariables it also appears as before.
> > 
> > As I see it, it lists the linker options including -lnvc also before you try to skip it.
> > Maybe it is already in the linker options before the skipping.
> > 
> > 
> > Greetings, Frank 
> > 
> > 
> > 
> > 
>