Script started on 2021-07-16 15:47:36-04:00 [TERM="xterm-256color" TTY="/dev/pts/34" COLUMNS="283" LINES="92"] 15:47 main *= /gpfs/alpine/csc314/scratch/adams/petsc$ exit~/arch-spock-opt-cray-kokkos.pyhostname~/arch-spock-opt-cray-kokkos.pyexitmake PETSC_DIR=/gpfs/alpine/csc314/scratch/adams/petsc PETSC_ARCH=arch-spock-opt-cray-kokkos check Running check examples to verify correct installation Using PETSC_DIR=/gpfs/alpine/csc314/scratch/adams/petsc and PETSC_ARCH=arch-spock-opt-cray-kokkos gmake[3]: [/gpfs/alpine/csc314/scratch/adams/petsc/lib/petsc/conf/rules:301: ex19.PETSc] Error 2 (ignored) *******************Error detected during compile or link!******************* See http://www.mcs.anl.gov/petsc/documentation/faq.html /gpfs/alpine/csc314/scratch/adams/petsc/src/snes/tutorials ex19 ********************************************************************************* cc -L/sw/spock/spack-envs/views/rocm-4.1.0/lib -lhsa-runtime64 -L/sw/spock/spack-envs/views/rocm-4.1.0/lib -lamdhip64 -lhsa-runtime64 -fPIC -Wall -Wwrite-strings -Wno-strict-aliasing -Wno-unknown-pragmas -fstack-protector -Qunused-arguments -fvisibility=hidden -g -O2 -fPIC -Wall -Wwrite-strings -Wno-strict-aliasing -Wno-unknown-pragmas -fstack-protector -Qunused-arguments -fvisibility=hidden -g -O2 -I/sw/spock/spack-envs/views/rocm-4.1.0/include -I/gpfs/alpine/csc314/scratch/adams/petsc/include -I/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include -I/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/include -I/sw/spock/spack-envs/views/rocm-4.1.0/include -I/sw/spock/spack-envs/views/rocm-4.1.0/include ex19.c -Wl,-rpath,/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -L/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -Wl,-rpath,/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -L/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -Wl,-rpath,/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/lib -L/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/lib -Wl,-rpath,/sw/spock/spack-envs/views/rocm-4.1.0/lib -L/sw/spock/spack-envs/views/rocm-4.1.0/lib -Wl,-rpath,/opt/gcc/8.1.0/snos/lib64 -L/opt/gcc/8.1.0/snos/lib64 -Wl,-rpath,/opt/cray/pe/libsci/21.04.1.1/CRAY/9.0/x86_64/lib -L/opt/cray/pe/libsci/21.04.1.1/CRAY/9.0/x86_64/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.4/ofi/cray/9.1/lib -L/opt/cray/pe/mpich/8.1.4/ofi/cray/9.1/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.4/gtl/lib -L/opt/cray/pe/mpich/8.1.4/gtl/lib -Wl,-rpath,/opt/cray/pe/pmi/6.0.10/lib -L/opt/cray/pe/pmi/6.0.10/lib -Wl,-rpath,/opt/cray/pe/dsmml/0.1.4/dsmml/lib -L/opt/cray/pe/dsmml/0.1.4/dsmml/lib -Wl,-rpath,/opt/cray/pe/cce/11.0.4/cce/x86_64/lib -L/opt/cray/pe/cce/11.0.4/cce/x86_64/lib -Wl,-rpath,/opt/cray/xpmem/2.2.40-2.1_2.7__g3cf3325.shasta/lib64 -L/opt/cray/xpmem/2.2.40-2.1_2.7__g3cf3325.shasta/lib64 -Wl,-rpath,/opt/cray/pe/cce/11.0.4/cce-clang/x86_64/lib/clang/11.0.0/lib/linux -L/opt/cray/pe/cce/11.0.4/cce-clang/x86_64/lib/clang/11.0.0/lib/linux -Wl,-rpath,/opt/gcc/8.1.0/snos/lib/gcc/x86_64-suse-linux/8.1.0 -L/opt/gcc/8.1.0/snos/lib/gcc/x86_64-suse-linux/8.1.0 -Wl,-rpath,/opt/cray/pe/cce/11.0.4/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -L/opt/cray/pe/cce/11.0.4/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -lpetsc -lkokkoskernels -lkokkoscontainers -lkokkoscore -lp4est -lsc -lz -lhipsparse -lhipblas -lrocsparse -lrocsolver -lrocblas -lamdhip64 -lhsa-runtime64 -lstdc++ -ldl -lpmi -lsci_cray_mpi -lsci_cray -lmpifort_cray -lmpi_cray -lmpi_gtl_hsa -lxpmem -ldsmml -lpgas-shmem -lquadmath -lcrayacc_amdgpu -lopenacc -lmodules -lfi -lcraymath -lf -lu -lcsup -lgfortran -lpthread -lgcc_eh -lm -lclang_rt.craypgo-x86_64 -lclang_rt.builtins-x86_64 -lquadmath -lstdc++ -ldl -o ex19 ld.lld: error: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib/libpetsc.so: undefined reference to .omp_offloading.img_start.cray_amdgcn-amd-amdhsa [--no-allow-shlib-undefined] ld.lld: error: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib/libpetsc.so: undefined reference to .omp_offloading.img_size.cray_amdgcn-amd-amdhsa [--no-allow-shlib-undefined] ld.lld: error: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib/libpetsc.so: undefined reference to .omp_offloading.img_cache.cray_amdgcn-amd-amdhsa [--no-allow-shlib-undefined] clang-11: error: linker command failed with exit code 1 (use -v to see invocation) gmake[4]: *** [: ex19] Error 1 gmake[3]: [/gpfs/alpine/csc314/scratch/adams/petsc/lib/petsc/conf/rules:301: ex3k.PETSc] Error 2 (ignored) *******************Error detected during compile or link!******************* See http://www.mcs.anl.gov/petsc/documentation/faq.html /gpfs/alpine/csc314/scratch/adams/petsc/src/snes/tutorials ex3k ********************************************************************************* MPICH_CXX="hipcc" OMPI_CXX="hipcc" CC -L/sw/spock/spack-envs/views/rocm-4.1.0/lib -lhsa-runtime64 -L/sw/spock/spack-envs/views/rocm-4.1.0/lib -lamdhip64 -lhsa-runtime64 -Wall -Wwrite-strings -Wno-strict-aliasing -Wno-unknown-pragmas -fstack-protector -fvisibility=hidden -g -O2 -fPIC -Wall -Wwrite-strings -Wno-strict-aliasing -Wno-unknown-pragmas -fvisibility=hidden -g -O3 -std=c++14 -I/gpfs/alpine/csc314/scratch/adams/petsc/include -I/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include -I/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/include -I/sw/spock/spack-envs/views/rocm-4.1.0/include -I/sw/spock/spack-envs/views/rocm-4.1.0/include ex3k.kokkos.cxx -Wl,-rpath,/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -L/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -Wl,-rpath,/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -L/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -Wl,-rpath,/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/lib -L/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/lib -Wl,-rpath,/sw/spock/spack-envs/views/rocm-4.1.0/lib -L/sw/spock/spack-envs/views/rocm-4.1.0/lib -Wl,-rpath,/opt/gcc/8.1.0/snos/lib64 -L/opt/gcc/8.1.0/snos/lib64 -Wl,-rpath,/opt/cray/pe/libsci/21.04.1.1/CRAY/9.0/x86_64/lib -L/opt/cray/pe/libsci/21.04.1.1/CRAY/9.0/x86_64/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.4/ofi/cray/9.1/lib -L/opt/cray/pe/mpich/8.1.4/ofi/cray/9.1/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.4/gtl/lib -L/opt/cray/pe/mpich/8.1.4/gtl/lib -Wl,-rpath,/opt/cray/pe/pmi/6.0.10/lib -L/opt/cray/pe/pmi/6.0.10/lib -Wl,-rpath,/opt/cray/pe/dsmml/0.1.4/dsmml/lib -L/opt/cray/pe/dsmml/0.1.4/dsmml/lib -Wl,-rpath,/opt/cray/pe/cce/11.0.4/cce/x86_64/lib -L/opt/cray/pe/cce/11.0.4/cce/x86_64/lib -Wl,-rpath,/opt/cray/xpmem/2.2.40-2.1_2.7__g3cf3325.shasta/lib64 -L/opt/cray/xpmem/2.2.40-2.1_2.7__g3cf3325.shasta/lib64 -Wl,-rpath,/opt/cray/pe/cce/11.0.4/cce-clang/x86_64/lib/clang/11.0.0/lib/linux -L/opt/cray/pe/cce/11.0.4/cce-clang/x86_64/lib/clang/11.0.0/lib/linux -Wl,-rpath,/opt/gcc/8.1.0/snos/lib/gcc/x86_64-suse-linux/8.1.0 -L/opt/gcc/8.1.0/snos/lib/gcc/x86_64-suse-linux/8.1.0 -Wl,-rpath,/opt/cray/pe/cce/11.0.4/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -L/opt/cray/pe/cce/11.0.4/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -lpetsc -lkokkoskernels -lkokkoscontainers -lkokkoscore -lp4est -lsc -lz -lhipsparse -lhipblas -lrocsparse -lrocsolver -lrocblas -lamdhip64 -lhsa-runtime64 -lstdc++ -ldl -lpmi -lsci_cray_mpi -lsci_cray -lmpifort_cray -lmpi_cray -lmpi_gtl_hsa -lxpmem -ldsmml -lpgas-shmem -lquadmath -lcrayacc_amdgpu -lopenacc -lmodules -lfi -lcraymath -lf -lu -lcsup -lgfortran -lpthread -lgcc_eh -lm -lclang_rt.craypgo-x86_64 -lclang_rt.builtins-x86_64 -lquadmath -lstdc++ -ldl -o ex3k In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:212: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Exchange.hpp:259:11: error: no member named 'lock_address_host_space' in namespace 'Kokkos::Impl::Kokkos::Impl'; did you mean simply 'lock_address_host_space'? while (!Impl::lock_address_host_space((void*)dest)) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ lock_address_host_space /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_HostSpace.hpp:83:6: note: 'lock_address_host_space' declared here bool lock_address_host_space(void* ptr); ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:212: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Exchange.hpp:261:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:212: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Exchange.hpp:277:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:212: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Exchange.hpp:278:3: error: no member named 'unlock_address_host_space' in namespace 'Kokkos::Impl::Kokkos::Impl'; did you mean simply 'unlock_address_host_space'? Impl::unlock_address_host_space((void*)dest); ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ unlock_address_host_space /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_HostSpace.hpp:91:6: note: 'unlock_address_host_space' declared here void unlock_address_host_space(void* ptr); ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:212: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Exchange.hpp:346:11: error: no member named 'lock_address_host_space' in namespace 'Kokkos::Impl::Kokkos::Impl'; did you mean simply 'lock_address_host_space'? while (!Impl::lock_address_host_space((void*)dest)) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ lock_address_host_space /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_HostSpace.hpp:83:6: note: 'lock_address_host_space' declared here bool lock_address_host_space(void* ptr); ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:212: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Exchange.hpp:348:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:212: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Exchange.hpp:356:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:212: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Exchange.hpp:357:3: error: no member named 'unlock_address_host_space' in namespace 'Kokkos::Impl::Kokkos::Impl'; did you mean simply 'unlock_address_host_space'? Impl::unlock_address_host_space((void*)dest); ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ unlock_address_host_space /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_HostSpace.hpp:91:6: note: 'unlock_address_host_space' declared here void unlock_address_host_space(void* ptr); ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:222: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp:273:11: error: no member named 'lock_address_host_space' in namespace 'Kokkos::Impl::Kokkos::Impl'; did you mean simply 'lock_address_host_space'? while (!Impl::lock_address_host_space((void*)dest)) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ lock_address_host_space /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_HostSpace.hpp:83:6: note: 'lock_address_host_space' declared here bool lock_address_host_space(void* ptr); ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:222: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp:275:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:222: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp:292:5: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:222: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp:294:3: error: no member named 'unlock_address_host_space' in namespace 'Kokkos::Impl::Kokkos::Impl'; did you mean simply 'unlock_address_host_space'? Impl::unlock_address_host_space((void*)dest); ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ unlock_address_host_space /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_HostSpace.hpp:91:6: note: 'unlock_address_host_space' declared here void unlock_address_host_space(void* ptr); ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:222: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp:350:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:222: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp:352:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:222: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp:360:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:222: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Compare_Exchange_Strong.hpp:367:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:224: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Generic.hpp:292:11: error: no member named 'lock_address_host_space' in namespace 'Kokkos::Impl::Kokkos::Impl'; did you mean simply 'lock_address_host_space'? while (!Impl::lock_address_host_space((void*)dest)) ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ lock_address_host_space /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_HostSpace.hpp:83:6: note: 'lock_address_host_space' declared here bool lock_address_host_space(void* ptr); ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:224: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Generic.hpp:294:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ In file included from ex3k.kokkos.cxx:3: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscdmda_kokkos.hpp:4: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec_kokkos.hpp:14: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscvec.h:9: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsys.h:42: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/include/petscsystypes.h:255: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Complex.hpp:47: In file included from /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/Kokkos_Atomic.hpp:224: /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Atomic_Generic.hpp:297:3: error: no member named 'memory_fence' in namespace 'Kokkos::Impl::Kokkos'; did you mean simply 'memory_fence'? Kokkos::memory_fence(); ^~~~~~~~~~~~~~~~~~~~ memory_fence /gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include/impl/Kokkos_Memory_Fence.hpp:53:6: note: 'memory_fence' declared here void memory_fence() { ^ fatal error: too many errors emitted, stopping now [-ferror-limit=] 20 errors generated. gmake[4]: *** [/gpfs/alpine/csc314/scratch/adams/petsc/lib/petsc/conf/test:26: ex3k] Error 1 gmake[3]: [makefile:102: runex3k_kokkos] Error 2 (ignored) 1,25c1,6 < atol=1e-50, rtol=1e-08, stol=1e-08, maxit=50, maxf=10000 < Vec Object: Exact Solution 2 MPI processes < type: mpikokkos < Process [0] < 0. < 0.015625 < 0.125 < Process [1] < 0.421875 < 1. < Vec Object: Forcing function 2 MPI processes < type: mpikokkos < Process [0] < 1e-72 < 1.50024 < 3.01563 < Process [1] < 4.67798 < 7. < 0 SNES Function norm 5.414682427127e+00 < 1 SNES Function norm 2.952582418265e-01 < 2 SNES Function norm 4.502293658739e-04 < 3 SNES Function norm 1.389665806646e-09 < Number of SNES iterations = 3 < Norm of error 1.49752e-10 Iterations 3 --- > srun: job 272850 queued and waiting for resources > srun: job 272850 has been allocated resources > slurmstepd: error: execve(): /gpfs/alpine/csc314/scratch/adams/petsc/src/snes/tutorials/./ex3k: No such file or directory > slurmstepd: error: execve(): /gpfs/alpine/csc314/scratch/adams/petsc/src/snes/tutorials/./ex3k: No such file or directory > srun: error: spock12: tasks 0-1: Exited with exit code 2 > srun: launch/slurm: _step_signal: Terminating StepId=272850.0 /gpfs/alpine/csc314/scratch/adams/petsc/src/snes/tutorials Possible problem with ex3k running with kokkos-kernels, diffs above ========================================= *******************Error detected during compile or link!******************* See http://www.mcs.anl.gov/petsc/documentation/faq.html /gpfs/alpine/csc314/scratch/adams/petsc/src/snes/tutorials ex5f ********************************************************* ftn -L/sw/spock/spack-envs/views/rocm-4.1.0/lib -lhsa-runtime64 -fPIC -g -O2 -fPIC -g -O2 -I/gpfs/alpine/csc314/scratch/adams/petsc/include -I/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/include -I/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/include -I/sw/spock/spack-envs/views/rocm-4.1.0/include ex5f.F90 -Wl,-rpath,/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -L/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -Wl,-rpath,/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -L/gpfs/alpine/csc314/scratch/adams/petsc/arch-spock-opt-cray-kokkos/lib -Wl,-rpath,/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/lib -L/sw/spock/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/zlib-1.2.11-xujqx3ri73tbwmcwoolo3jnn6ti6vwsh/lib -Wl,-rpath,/sw/spock/spack-envs/views/rocm-4.1.0/lib -L/sw/spock/spack-envs/views/rocm-4.1.0/lib -Wl,-rpath,/opt/gcc/8.1.0/snos/lib64 -L/opt/gcc/8.1.0/snos/lib64 -Wl,-rpath,/opt/cray/pe/libsci/21.04.1.1/CRAY/9.0/x86_64/lib -L/opt/cray/pe/libsci/21.04.1.1/CRAY/9.0/x86_64/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.4/ofi/cray/9.1/lib -L/opt/cray/pe/mpich/8.1.4/ofi/cray/9.1/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.4/gtl/lib -L/opt/cray/pe/mpich/8.1.4/gtl/lib -Wl,-rpath,/opt/cray/pe/pmi/6.0.10/lib -L/opt/cray/pe/pmi/6.0.10/lib -Wl,-rpath,/opt/cray/pe/dsmml/0.1.4/dsmml/lib -L/opt/cray/pe/dsmml/0.1.4/dsmml/lib -Wl,-rpath,/opt/cray/pe/cce/11.0.4/cce/x86_64/lib -L/opt/cray/pe/cce/11.0.4/cce/x86_64/lib -Wl,-rpath,/opt/cray/xpmem/2.2.40-2.1_2.7__g3cf3325.shasta/lib64 -L/opt/cray/xpmem/2.2.40-2.1_2.7__g3cf3325.shasta/lib64 -Wl,-rpath,/opt/cray/pe/cce/11.0.4/cce-clang/x86_64/lib/clang/11.0.0/lib/linux -L/opt/cray/pe/cce/11.0.4/cce-clang/x86_64/lib/clang/11.0.0/lib/linux -Wl,-rpath,/opt/gcc/8.1.0/snos/lib/gcc/x86_64-suse-linux/8.1.0 -L/opt/gcc/8.1.0/snos/lib/gcc/x86_64-suse-linux/8.1.0 -Wl,-rpath,/opt/cray/pe/cce/11.0.4/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -L/opt/cray/pe/cce/11.0.4/binutils/x86_64/x86_64-pc-linux-gnu/..//x86_64-unknown-linux-gnu/lib -lpetsc -lkokkoskernels -lkokkoscontainers -lkokkoscore -lp4est -lsc -lz -lhipsparse -lhipblas -lrocsparse -lrocsolver -lrocblas -lamdhip64 -lhsa-runtime64 -lstdc++ -ldl -lpmi -lsci_cray_mpi -lsci_cray -lmpifort_cray -lmpi_cray -lmpi_gtl_hsa -lxpmem -ldsmml -lpgas-shmem -lquadmath -lcrayacc_amdgpu -lopenacc -lmodules -lfi -lcraymath -lf -lu -lcsup -lgfortran -lpthread -lgcc_eh -lm -lclang_rt.craypgo-x86_64 -lclang_rt.builtins-x86_64 -lquadmath -lstdc++ -ldl -o ex5f /opt/cray/pe/cce/11.0.4/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: warning: alignment 128 of symbol `$host_init$$runtime_init_for_iso_c_binding$iso_c_binding_' in /opt/cray/pe/cce/11.0.4/cce/x86_64/lib/libmodules.so is smaller than 256 in /tmp/pe_136574/ex5f_1.o /opt/cray/pe/cce/11.0.4/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: warning: alignment 64 of symbol `$data_init$iso_c_binding_' in /opt/cray/pe/cce/11.0.4/cce/x86_64/lib/libmodules.so is smaller than 256 in /tmp/pe_136574/ex5f_1.o Possible error running Fortran example src/snes/tutorials/ex5f with 1 MPI process See http://www.mcs.anl.gov/petsc/documentation/faq.html srun: job 272851 queued and waiting for resources srun: job 272851 has been allocated resources Number of SNES iterations = 4 Completed test examples 15:49 main *= /gpfs/alpine/csc314/scratch/adams/petsc$ moexit exit Script done on 2021-07-16 15:50:09-04:00 [COMMAND_EXIT_CODE="0"]