[Darshan-users] darshan3.1.5 issue on Cray XC40 cle6.up05
Bilel Hadri
bilel.hadri at kaust.edu.sa
Mon Feb 26 23:34:53 CST 2018
Hi Phil,
Sure, test_scalapack.f90 call ScaLAPACK routines and uses inside MPI library. The output of the failing ftn command with the craype-verbose flag
hadrib at cdl1:~> ftn -o exe test_scalapack.f90 -craype-verbose
ftn_driver.exe -hcpu=haswell -hstatic -D__CRAYXC -D__CRAY_HASWELL -D__CRAYXT_COMPUTE_LINUX_TARGET -hnetwork=aries -o exe test_scalapack.f90 -Wl,-rpath=/opt/cray/pe/cce/8.6.5/cce/x86_64/lib -ulast_user_arg -I /opt/cray/pe/cce/8.6.5/cce/x86_64/include/craylibs -I/usr/include -L/opt/gcc/6.1.0/snos/lib64 -Wl,-rpath=/opt/cray/pe/gcc-libs -L/usr/lib64 -L/lib64 -ugcc_base=/opt/gcc/6.1.0/snos -uno_driver_libs -Wl,/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/libmodules.a,/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/libomp.a,/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/libopenacc.a, -I /usr/include -I/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/include -I/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/include -I/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/include -I/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/include -I/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/include -I/opt/cray/gni-headers/5.0.12-6.0.5.0_2.15__g2ef1ebc.ari/include -I/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/include -I/opt/cray/pe/pmi/5.0.13/include -I/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/include -I/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/include -I/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/include -I/opt/cray/krca/2.2.2-6.0.5.0_5.46__g4614cf3.ari/include -I/opt/cray-hss-devel/8.0.0/include -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib -L/sw/xc40cle6/darshan/3.1.4/sles12.3_gcc7.2.0/lib -L/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64 -L/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64 -L/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64 -L/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64 -L/opt/cray/pe/pmi/5.0.13/lib64 -L/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64 -L/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64 -L/opt/cray/pe/atp/2.1.1/libApp -L/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/pkgconfig/../ -L/lib64 -L/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64 -Wl,@/sw/xc40cle6/darshan/3.1.4/sles12.3_gcc7.2.0/share/ld-opts/darshan-base-ld-opts -lfmpich -lmpichcxx -Wl,--start-group -ldarshan -ldarshan-stubs -Wl,--end-group -lz -Wl,--no-as-needed,-lAtpSigHandler,-lAtpSigHCommData -Wl,--undefined=_ATP_Data_Globals -Wl,--undefined=__atpHandlerInstall -lpthread -lmpichf90_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mpi_mp -lm -lf -lsci_cray_mp -lmpich_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mp -lcraymp -lm -lpthread -lf -lhugetlbfs -lpgas-dmapp -lfi -lu -lrt -Wl,--undefined=dmapp_get_flag_nbi -ldmapp -lugni -ludreg -lpthread -lm -lcray-c++-rts -lstdc++ -lxpmem -ldmapp -lpthread -lpmi -lpthread -lalpslli -lpthread -lwlm_detect -lugni -lpthread -lalpsutil -lpthread -lrca -ludreg -lquadmath -lm -lomp -lcraymp -lpthread -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lmodules -lm -lfi -lm -lquadmath -lcraymath -lm -lgfortran -lquadmath -lf -lm -lpthread -lu -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lcsup -Wl,--as-needed,-latomic,--no-as-needed -Wl,--as-needed,-lgfortran,--no-as-needed -lstdc++ -lpthread -Wl,--start-group,-lc,-lcsup,-lgcc_eh,-lm,-lgcc,--end-group -Wl,-T/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/2.23.1.cce.ld
hadrib at cdl1:~>
hadrib at cdl1:~>
hadrib at cdl1:~> module swap darshan/3.1.4 darshan/3.1.5
hadrib at cdl1:~>
hadrib at cdl1:~>
hadrib at cdl1:~> ftn -o exe test_scalapack.f90 -craype-verbose
ftn_driver.exe -hcpu=haswell -hstatic -D__CRAYXC -D__CRAY_HASWELL -D__CRAYXT_COMPUTE_LINUX_TARGET -hnetwork=aries -o exe test_scalapack.f90 -Wl,-rpath=/opt/cray/pe/cce/8.6.5/cce/x86_64/lib -ulast_user_arg -I /opt/cray/pe/cce/8.6.5/cce/x86_64/include/craylibs -I/usr/include -L/opt/gcc/6.1.0/snos/lib64 -Wl,-rpath=/opt/cray/pe/gcc-libs -L/usr/lib64 -L/lib64 -ugcc_base=/opt/gcc/6.1.0/snos -uno_driver_libs -Wl,/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/libmodules.a,/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/libomp.a,/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/libopenacc.a, -I /usr/include -I/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/include -I/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/include -I/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/include -I/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/include -I/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/include -I/opt/cray/gni-headers/5.0.12-6.0.5.0_2.15__g2ef1ebc.ari/include -I/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/include -I/opt/cray/pe/pmi/5.0.13/include -I/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/include -I/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/include -I/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/include -I/opt/cray/krca/2.2.2-6.0.5.0_5.46__g4614cf3.ari/include -I/opt/cray-hss-devel/8.0.0/include -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib -L/sw/xc40cle6/darshan/3.1.5/sles12.3_gcc7.2.0/lib -L/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64 -L/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64 -L/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64 -L/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64 -L/opt/cray/pe/pmi/5.0.13/lib64 -L/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64 -L/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64 -L/opt/cray/pe/atp/2.1.1/libApp -L/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/pkgconfig/../ -L/lib64 -L/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64 -Wl,@/sw/xc40cle6/darshan/3.1.5/sles12.3_gcc7.2.0/share/ld-opts/darshan-base-ld-opts -lfmpich -lmpichcxx -Wl,--start-group -ldarshan -ldarshan-stubs -Wl,--end-group -lz -Wl,--no-as-needed,-lAtpSigHandler,-lAtpSigHCommData -Wl,--undefined=_ATP_Data_Globals -Wl,--undefined=__atpHandlerInstall -lpthread -lmpichf90_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mpi_mp -lm -lf -lsci_cray_mp -lmpich_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mp -lcraymp -lm -lpthread -lf -lhugetlbfs -lpgas-dmapp -lfi -lu -lrt -Wl,--undefined=dmapp_get_flag_nbi -ldmapp -lugni -ludreg -lpthread -lm -lcray-c++-rts -lstdc++ -lxpmem -ldmapp -lpthread -lpmi -lpthread -lalpslli -lpthread -lwlm_detect -lugni -lpthread -lalpsutil -lpthread -lrca -ludreg -lquadmath -lm -lomp -lcraymp -lpthread -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lmodules -lm -lfi -lm -lquadmath -lcraymath -lm -lgfortran -lquadmath -lf -lm -lpthread -lu -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lcsup -Wl,--as-needed,-latomic,--no-as-needed -Wl,--as-needed,-lgfortran,--no-as-needed -lstdc++ -lpthread -Wl,--start-group,-lc,-lcsup,-lgcc_eh,-lm,-lgcc,--end-group -Wl,-T/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/2.23.1.cce.ld
/opt/cray/dmapp/default/lib64/libdmapp.a(dmapp_internal.o): In function `_dmappi_is_pure_dmapp_job':
/home/abuild/rpmbuild/BUILD/cray-dmapp-7.1.1/src/dmapp_internal.c:1401: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib/libsci_cray_mpi_mp.a(blacs_exit_.o): In function `blacs_exit_':
/b/worker/csml-libsci-sles/build/mp/scalapack/BLACS/SRC/blacs_exit_.c:42: undefined reference to `__wrap_MPI_Finalize'
/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib/libsci_cray_mpi_mp.a(blacs_pinfo_.o): In function `blacs_pinfo_':
/b/worker/csml-libsci-sles/build/mp/scalapack/BLACS/SRC/blacs_pinfo_.c:18: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib/libsci_cray_mpi_mp.a(blacs_pinfo_.oo): In function `Cblacs_pinfo':
/b/worker/csml-libsci-sles/build/mp/scalapack/BLACS/SRC/blacs_pinfo_.c:18: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: link errors found, deleting executable `exe'
/usr/bin/sha1sum: exe: No such file or directory
Darshan 3.1.5 darshan-runtime.pc
# This pkgconfig file was built for use with the Cray 2.x Programming
# Environment. It has not been tested for general purpose use.
Name: darshan
Description: Darshan runtime library
Version: 3.1.5
Requires: zlib mpich
Requires.private:
darshan_prefix=/sw/xc40cle6/darshan/3.1.5/sles12.3_gcc7.2.0
darshan_share=/sw/xc40cle6/darshan/3.1.5/sles12.3_gcc7.2.0/share
darshan_includedir=
darshan_libdir= -L${darshan_prefix}/lib
darshan_linkopts="-Wl,@${darshan_share}/ld-opts/darshan-base-ld-opts"
Cflags:
Libs:
Libs.private: ${darshan_linkopts} ${darshan_libdir} -lfmpich -lmpichcxx -Wl,--start-group -ldarshan -ldarshan-stubs -Wl,--end-group
Darshan 3.1.4 darshan-runtime.pc
hadrib at cdl1:~> more /sw/xc40cle6/darshan/3.1.4/sles12.3_gcc7.2.0/darshan-3.1.4/darshan-runtime/lib/pkgconfig/darshan-runtime.pc
# This pkgconfig file was built for use with the Cray 2.x Programming
# Environment. It has not been tested for general purpose use.
Name: darshan
Description: Darshan runtime library
Version: 3.1.4
Requires: zlib mpich
Requires.private:
darshan_prefix=/sw/xc40cle6/darshan/3.1.4/sles12.3_gcc7.2.0
darshan_share=/sw/xc40cle6/darshan/3.1.4/sles12.3_gcc7.2.0/share
darshan_includedir=
darshan_libdir= -L${darshan_prefix}/lib
darshan_linkopts="-Wl,@${darshan_share}/ld-opts/darshan-base-ld-opts"
Cflags:
Libs:
Libs.private: ${darshan_linkopts} ${darshan_libdir} -lfmpich -lmpichcxx -Wl,--start-group -ldarshan -ldarshan-stubs -Wl,--end-group
Darshan modules.
#%Module
#
# Module darshan
#
#module load texlive/2017
if { [ info exists env(PE_ENV) ] } {
set PRGENV $env(PE_ENV)
} else {
set PRGENV "GNU"
}
set PREFIX /sw/xc40cle6/darshan/3.1.5/sles12.3_gcc7.2.0
#PE_PKGCONFIG_LIBS contain a list of all pc files that apply to all supported languages
prepend-path PE_PKGCONFIG_LIBS darshan-runtime
prepend-path PKG_CONFIG_PATH $PREFIX/lib/pkgconfig
prepend-path LD_LIBRARY_PATH $PREFIX/lib
prepend-path PATH $PREFIX/bin
By the way, only 2 tests fails at compilations so far with darshan3.1.5, ( the above scalapack test) and a simple petsc code:
cat petsc.c
static char help[] = "Solves 2D Poisson equation using multigrid.\n\n";
#include <petscdm.h>
#include <petscdmda.h>
#include <petscksp.h>
#include <petscsys.h>
#include <petscvec.h>
extern PetscErrorCode ComputeJacobian(KSP,Mat,Mat,void*);
extern PetscErrorCode ComputeRHS(KSP,Vec,void*);
extern PetscErrorCode ComputeTrueSolution(DM, Vec);
extern PetscErrorCode VecView_VTK(Vec, const char [], const char []);
typedef enum {DIRICHLET, NEUMANN} BCType;
typedef struct {
PetscScalar uu, tt;
BCType bcType;
} UserContext;
#undef __FUNCT__
#define __FUNCT__ "main"
int main(int argc,char **argv)
{
KSP ksp;
DM da;
UserContext user;
PetscInt bc;
PetscErrorCode ierr;
PetscInitialize(&argc,&argv,(char*)0,help);
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp);CHKERRQ(ierr);
ierr = DMDACreate2d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-11,-11,PETSC_DECIDE,PETSC_DECIDE,1,1,NULL,NULL,&da);CHKERRQ(ierr);
ierr = KSPSetDM(ksp,(DM)da);
ierr = DMSetApplicationContext(da,&user);CHKERRQ(ierr);
user.uu = 1.0;
user.tt = 1.0;
bc = (PetscInt)NEUMANN;
user.bcType = (BCType)bc;
ierr = KSPSetComputeRHS(ksp,ComputeRHS,&user);CHKERRQ(ierr);
ierr = KSPSetComputeOperators(ksp,ComputeJacobian,&user);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp);CHKERRQ(ierr);
ierr = KSPSolve(ksp,NULL,NULL);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = PetscFinalize();CHKERRQ(ierr);
return 0;
}
#undef __FUNCT__
#define __FUNCT__ "ComputeRHS"
PetscErrorCode ComputeRHS(KSP ksp,Vec b,void *ctx)
{
UserContext *user = (UserContext*)ctx;
PetscErrorCode ierr;
PetscInt i,j,M,N,xm,ym,xs,ys;
PetscScalar Hx,Hy,pi,uu,tt;
PetscScalar **array;
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da, 0, &M, &N, 0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
uu = user->uu; tt = user->tt;
pi = 4*atan(1.0);
Hx = 1.0/(PetscReal)(M);
Hy = 1.0/(PetscReal)(N);
ierr = DMDAGetCorners(da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
ierr = DMDAVecGetArray(da, b, &array);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
array[j][i] = -PetscCosScalar(uu*pi*((PetscReal)i+0.5)*Hx)*PetscCosScalar(tt*pi*((PetscReal)j+0.5)*Hy)*Hx*Hy;
}
}
ierr = DMDAVecRestoreArray(da, b, &array);CHKERRQ(ierr);
ierr = VecAssemblyBegin(b);CHKERRQ(ierr);
ierr = VecAssemblyEnd(b);CHKERRQ(ierr);
if (user->bcType == NEUMANN) {
MatNullSpace nullspace;
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);CHKERRQ(ierr);
ierr = MatNullSpaceRemove(nullspace,b);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
#undef __FUNCT__
#define __FUNCT__ "ComputeJacobian"
PetscErrorCode ComputeJacobian(KSP ksp,Mat J, Mat jac,void *ctx)
{
UserContext *user = (UserContext*)ctx;
PetscErrorCode ierr;
PetscInt i, j, M, N, xm, ym, xs, ys, num, numi, numj;
PetscScalar v[5], Hx, Hy, HydHx, HxdHy;
MatStencil row, col[5];
DM da;
PetscFunctionBeginUser;
ierr = KSPGetDM(ksp,&da);CHKERRQ(ierr);
ierr = DMDAGetInfo(da,0,&M,&N,0,0,0,0,0,0,0,0,0,0);CHKERRQ(ierr);
Hx = 1.0 / (PetscReal)(M);
Hy = 1.0 / (PetscReal)(N);
HxdHy = Hx/Hy;
HydHx = Hy/Hx;
ierr = DMDAGetCorners(da,&xs,&ys,0,&xm,&ym,0);CHKERRQ(ierr);
for (j=ys; j<ys+ym; j++) {
for (i=xs; i<xs+xm; i++) {
row.i = i; row.j = j;
if (i==0 || j==0 || i==M-1 || j==N-1) {
if (user->bcType == DIRICHLET) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Dirichlet boundary conditions not supported !\n");
else if (user->bcType == NEUMANN) {
num=0; numi=0; numj=0;
if (j!=0) {
v[num] = -HxdHy; col[num].i = i; col[num].j = j-1;
num++; numj++;
}
if (i!=0) {
v[num] = -HydHx; col[num].i = i-1; col[num].j = j;
num++; numi++;
}
if (i!=M-1) {
v[num] = -HydHx; col[num].i = i+1; col[num].j = j;
num++; numi++;
}
if (j!=N-1) {
v[num] = -HxdHy; col[num].i = i; col[num].j = j+1;
num++; numj++;
}
v[num] = ((PetscReal)(numj)*HxdHy + (PetscReal)(numi)*HydHx); col[num].i = i; col[num].j = j;
num++;
ierr = MatSetValuesStencil(jac,1,&row,num,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
} else {
v[0] = -HxdHy; col[0].i = i; col[0].j = j-1;
v[1] = -HydHx; col[1].i = i-1; col[1].j = j;
v[2] = 2.0*(HxdHy + HydHx); col[2].i = i; col[2].j = j;
v[3] = -HydHx; col[3].i = i+1; col[3].j = j;
v[4] = -HxdHy; col[4].i = i; col[4].j = j+1;
ierr = MatSetValuesStencil(jac,1,&row,5,col,v,INSERT_VALUES);CHKERRQ(ierr);
}
}
}
ierr = MatAssemblyBegin(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(jac,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
if (user->bcType == NEUMANN) {
MatNullSpace nullspace;
ierr = MatNullSpaceCreate(PETSC_COMM_WORLD,PETSC_TRUE,0,0,&nullspace);CHKERRQ(ierr);
ierr = MatSetNullSpace(jac,nullspace);CHKERRQ(ierr);
ierr = MatNullSpaceDestroy(&nullspace);CHKERRQ(ierr);
}
PetscFunctionReturn(0);
}
hadrib at cdl1:~> cc -o exe_petsc petsc.c -craype-verbose
driver.cc -hcpu=haswell -hstatic -D__CRAYXC -D__CRAY_HASWELL -D__CRAYXT_COMPUTE_LINUX_TARGET -hnetwork=aries -o exe_petsc petsc.c -Wl,-rpath=/opt/cray/pe/cce/8.6.5/cce/x86_64/lib -hlast_user_arg -nostdinc -uno_driver_includes -isystem /opt/cray/pe/cce/8.6.5/cce/x86_64/include/craylibs -isystem /opt/cray/pe/cce/8.6.5/cce/x86_64/include/basic -isystem /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0/include -isystem /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0/include-fixed -isystem /opt/gcc/6.1.0/snos/include -L/opt/gcc/6.1.0/snos/lib64 -Wl,-rpath=/opt/cray/pe/gcc-libs -L/usr/lib64 -L/lib64 -ugcc_base=/opt/gcc/6.1.0/snos -uno_driver_libs -isystem /usr/include -I/opt/cray/pe/petsc/3.7.6.2/real/CRAY/8.6/haswell/include -I/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/include -I/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/include -I/opt/cray/pe/tpsl/17.11.1/CRAY/8.6/haswell/include -I/opt/cray/pe/hdf5-parallel/1.10.1.1/CRAY/8.6/include -I/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/include -I/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/include -I/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/include -I/opt/cray/gni-headers/5.0.12-6.0.5.0_2.15__g2ef1ebc.ari/include -I/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/include -I/opt/cray/pe/pmi/5.0.13/include -I/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/include -I/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/include -I/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/include -I/opt/cray/krca/2.2.2-6.0.5.0_5.46__g4614cf3.ari/include -I/opt/cray-hss-devel/8.0.0/include -L/opt/cray/pe/petsc/3.7.6.2/real/CRAY/8.6/haswell/lib -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib -L/opt/cray/pe/tpsl/17.11.1/CRAY/8.6/haswell/lib -L/opt/cray/pe/hdf5-parallel/1.10.1.1/CRAY/8.6/lib -L/sw/xc40cle6/darshan/3.1.5/sles12.3_gcc7.2.0/lib -L/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64 -L/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64 -L/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64 -L/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64 -L/opt/cray/pe/pmi/5.0.13/lib64 -L/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64 -L/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64 -L/opt/cray/pe/atp/2.1.1/libApp -L/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/pkgconfig/../ -L/lib64 -L/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64 -Wl,@/sw/xc40cle6/darshan/3.1.5/sles12.3_gcc7.2.0/share/ld-opts/darshan-base-ld-opts -lfmpich -lmpichcxx -Wl,--start-group -ldarshan -ldarshan-stubs -Wl,--end-group -lz -Wl,--no-as-needed,-lAtpSigHandler,-lAtpSigHCommData -Wl,--undefined=_ATP_Data_Globals -Wl,--undefined=__atpHandlerInstall -lpthread -lcraypetsc_cray_real -homp -lm -lmodules -lf -lHYPRE_cray -lcmumps_cray -ldmumps_cray -lesmumps_cray -lmatio_cray -lptscotcherrexit_cray -lptscotchparmetis_cray -lscotcherrexit_cray -lscotchmetis_cray -lsmumps_cray -lsundials_arkode_cray -lsundials_cvode_cray -lsundials_cvodes_cray -lsundials_fnvecopenmp_cray -lsundials_fnvecparallel_cray -lsundials_fnvecserial_cray -lsundials_ida_cray -lsundials_idas_cray -lsundials_kinsol_cray -lsuperlu_cray -lsuperlu_dist_cray -lzmumps_cray -lsundials_nvecopenmp_cray -lsundials_nvecparallel_cray -lsundials_nvecserial_cray -lmumps_common_cray -lptscotch_cray -lparmetis_cray -lptesmumps_cray -lpord_cray -lmetis_cray -lscotch_cray -lptscotcherr_cray -lscotcherr_cray -lcraymp -lpthread -lm -lomp64 -lf -lz -lrt -lhdf5_parallel -lrt -lz -ldl -lm -lpthread -lmpichf90_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mpi_mp -lm -lf -lsci_cray_mp -lmpich_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mp -lcraymp -lm -lpthread -lf -lhugetlbfs -lpgas-dmapp -lfi -lu -lrt -Wl,--undefined=dmapp_get_flag_nbi -ldmapp -lugni -ludreg -lpthread -lm -lcray-c++-rts -lstdc++ -lxpmem -ldmapp -lpthread -lpmi -lpthread -lalpslli -lpthread -lwlm_detect -lugni -lpthread -lalpsutil -lpthread -lrca -ludreg -lquadmath -lm -lomp -lcraymp -lpthread -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lmodules -lm -lfi -lm -lquadmath -lcraymath -lm -lgfortran -lquadmath -lf -lm -lpthread -lu -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lcsup -Wl,--as-needed,-latomic,--no-as-needed -lstdc++ -lpthread -Wl,--start-group,-lc,-lcsup,-lgcc_eh,-lm,-lgcc,--end-group -Wl,-T/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/2.23.1.cce.ld
/opt/cray/pe/hdf5-parallel/1.10.1.1/CRAY/8.6/lib/libhdf5_parallel.a(H5PL.o): In function `H5PL__open$$CFE_id_56395c9c_a2f1556b':
/b/hdf5-support/rpm/BUILD/cray-hdf5-1.10.1.1-201709131735.a6e5c5fe22823-cce1-parallel/hdf5-1.10.1/src/H5PL.c:878: warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
/opt/cray/dmapp/default/lib64/libdmapp.a(dmapp_internal.o): In function `_dmappi_is_pure_dmapp_job':
/home/abuild/rpmbuild/BUILD/cray-dmapp-7.1.1/src/dmapp_internal.c:1401: undefined reference to `__wrap_MPI_Init'
/home/abuild/rpmbuild/BUILD/cray-dmapp-7.1.1/src/dmapp_internal.c:1401: undefined reference to `__wrap_MPI_Init_thread'
/opt/cray/pe/petsc/3.7.6.2/real/CRAY/8.6/haswell/lib/libcraypetsc_cray_real.a(pinit.o): In function `PetscInitialize':
/b/cray-petsc/.cray-build/CRAY/86/haswell/cray-petsc/petsc-3.7.6/src/sys/objects/pinit.c:779: undefined reference to `__wrap_MPI_Init_thread'
/opt/cray/pe/petsc/3.7.6.2/real/CRAY/8.6/haswell/lib/libcraypetsc_cray_real.a(pinit.o): In function `PetscFinalize':
/b/cray-petsc/.cray-build/CRAY/86/haswell/cray-petsc/petsc-3.7.6/src/sys/objects/pinit.c:1428: undefined reference to `__wrap_MPI_Finalize'
/opt/cray/pe/tpsl/17.11.1/CRAY/8.6/haswell/lib/libHYPRE_cray.a(mpistubs.o): In function `hypre_MPI_Init':
/b/ulib/tpsl/.cray-build/CRAY/86/haswell/hypre/hypre-2.11.2/src/utilities/mpistubs.c:663: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/tpsl/17.11.1/CRAY/8.6/haswell/lib/libHYPRE_cray.a(mpistubs.o): In function `hypre_MPI_Finalize':
/b/ulib/tpsl/.cray-build/CRAY/86/haswell/hypre/hypre-2.11.2/src/utilities/mpistubs.c:669: undefined reference to `__wrap_MPI_Finalize'
/opt/cray/pe/tpsl/17.11.1/CRAY/8.6/haswell/lib/libparmetis_cray.a(gkmpi.c.o): In function `libparmetis__gkMPI_Finalize':
/b/ulib/tpsl/.cray-build/CRAY/86/haswell/parmetis/parmetis-4.0.2/libparmetis/gkmpi.c:288: undefined reference to `__wrap_MPI_Finalize'
/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib/libsci_cray_mpi_mp.a(blacs_pinfo_.o): In function `blacs_pinfo_':
/b/worker/csml-libsci-sles/build/mp/scalapack/BLACS/SRC/blacs_pinfo_.c:18: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib/libsci_cray_mpi_mp.a(blacs_pinfo_.oo): In function `Cblacs_pinfo':
/b/worker/csml-libsci-sles/build/mp/scalapack/BLACS/SRC/blacs_pinfo_.c:18: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: link errors found, deleting executable `exe_petsc'
/usr/bin/sha1sum: exe_petsc: No such file or directory
hadrib at cdl1:~>
hadrib at cdl1:~>
hadrib at cdl1:~>
hadrib at cdl1:~> module swap darshan/3.1.5 darshan/3.1.4
hadrib at cdl1:~>
hadrib at cdl1:~>
hadrib at cdl1:~> cc -o exe_petsc petsc.c -craype-verbose
driver.cc -hcpu=haswell -hstatic -D__CRAYXC -D__CRAY_HASWELL -D__CRAYXT_COMPUTE_LINUX_TARGET -hnetwork=aries -o exe_petsc petsc.c -Wl,-rpath=/opt/cray/pe/cce/8.6.5/cce/x86_64/lib -hlast_user_arg -nostdinc -uno_driver_includes -isystem /opt/cray/pe/cce/8.6.5/cce/x86_64/include/craylibs -isystem /opt/cray/pe/cce/8.6.5/cce/x86_64/include/basic -isystem /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0/include -isystem /opt/gcc/6.1.0/snos/lib/gcc/x86_64-suse-linux/6.1.0/include-fixed -isystem /opt/gcc/6.1.0/snos/include -L/opt/gcc/6.1.0/snos/lib64 -Wl,-rpath=/opt/cray/pe/gcc-libs -L/usr/lib64 -L/lib64 -ugcc_base=/opt/gcc/6.1.0/snos -uno_driver_libs -isystem /usr/include -I/opt/cray/pe/petsc/3.7.6.2/real/CRAY/8.6/haswell/include -I/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/include -I/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/include -I/opt/cray/pe/tpsl/17.11.1/CRAY/8.6/haswell/include -I/opt/cray/pe/hdf5-parallel/1.10.1.1/CRAY/8.6/include -I/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/include -I/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/include -I/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/include -I/opt/cray/gni-headers/5.0.12-6.0.5.0_2.15__g2ef1ebc.ari/include -I/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/include -I/opt/cray/pe/pmi/5.0.13/include -I/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/include -I/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/include -I/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/include -I/opt/cray/krca/2.2.2-6.0.5.0_5.46__g4614cf3.ari/include -I/opt/cray-hss-devel/8.0.0/include -L/opt/cray/pe/petsc/3.7.6.2/real/CRAY/8.6/haswell/lib -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L/opt/cray/dmapp/default/lib64 -L/opt/cray/pe/mpt/7.7.0/gni/mpich-cray/8.6/lib -L/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib -L/opt/cray/pe/tpsl/17.11.1/CRAY/8.6/haswell/lib -L/opt/cray/pe/hdf5-parallel/1.10.1.1/CRAY/8.6/lib -L/sw/xc40cle6/darshan/3.1.4/sles12.3_gcc7.2.0/lib -L/opt/cray/rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari/lib64 -L/opt/cray/alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari/lib64 -L/opt/cray/xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari/lib64 -L/opt/cray/dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari/lib64 -L/opt/cray/pe/pmi/5.0.13/lib64 -L/opt/cray/ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari/lib64 -L/opt/cray/udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari/lib64 -L/opt/cray/pe/atp/2.1.1/libApp -L/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/pkgconfig/../ -L/lib64 -L/opt/cray/wlm_detect/1.3.2-6.0.5.0_3.1__g388ccd5.ari/lib64 -Wl,@/sw/xc40cle6/darshan/3.1.4/sles12.3_gcc7.2.0/share/ld-opts/darshan-base-ld-opts -lfmpich -lmpichcxx -Wl,--start-group -ldarshan -ldarshan-stubs -Wl,--end-group -lz -Wl,--no-as-needed,-lAtpSigHandler,-lAtpSigHCommData -Wl,--undefined=_ATP_Data_Globals -Wl,--undefined=__atpHandlerInstall -lpthread -lcraypetsc_cray_real -homp -lm -lmodules -lf -lHYPRE_cray -lcmumps_cray -ldmumps_cray -lesmumps_cray -lmatio_cray -lptscotcherrexit_cray -lptscotchparmetis_cray -lscotcherrexit_cray -lscotchmetis_cray -lsmumps_cray -lsundials_arkode_cray -lsundials_cvode_cray -lsundials_cvodes_cray -lsundials_fnvecopenmp_cray -lsundials_fnvecparallel_cray -lsundials_fnvecserial_cray -lsundials_ida_cray -lsundials_idas_cray -lsundials_kinsol_cray -lsuperlu_cray -lsuperlu_dist_cray -lzmumps_cray -lsundials_nvecopenmp_cray -lsundials_nvecparallel_cray -lsundials_nvecserial_cray -lmumps_common_cray -lptscotch_cray -lparmetis_cray -lptesmumps_cray -lpord_cray -lmetis_cray -lscotch_cray -lptscotcherr_cray -lscotcherr_cray -lcraymp -lpthread -lm -lomp64 -lf -lz -lrt -lhdf5_parallel -lrt -lz -ldl -lm -lpthread -lmpichf90_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mpi_mp -lm -lf -lsci_cray_mp -lmpich_cray -lrt -lpthread -lugni -lpmi -lsci_cray_mp -lcraymp -lm -lpthread -lf -lhugetlbfs -lpgas-dmapp -lfi -lu -lrt -Wl,--undefined=dmapp_get_flag_nbi -ldmapp -lugni -ludreg -lpthread -lm -lcray-c++-rts -lstdc++ -lxpmem -ldmapp -lpthread -lpmi -lpthread -lalpslli -lpthread -lwlm_detect -lugni -lpthread -lalpsutil -lpthread -lrca -ludreg -lquadmath -lm -lomp -lcraymp -lpthread -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lmodules -lm -lfi -lm -lquadmath -lcraymath -lm -lgfortran -lquadmath -lf -lm -lpthread -lu -lrt -ldl -lcray-c++-rts -lstdc++ -lm -lcsup -Wl,--as-needed,-latomic,--no-as-needed -lstdc++ -lpthread -Wl,--start-group,-lc,-lcsup,-lgcc_eh,-lm,-lgcc,--end-group -Wl,-T/opt/cray/pe/cce/8.6.5/cce/x86_64/lib/2.23.1.cce.ld
/opt/cray/pe/hdf5-parallel/1.10.1.1/CRAY/8.6/lib/libhdf5_parallel.a(H5PL.o): In function `H5PL__open$$CFE_id_56395c9c_a2f1556b':
/b/hdf5-support/rpm/BUILD/cray-hdf5-1.10.1.1-201709131735.a6e5c5fe22823-cce1-parallel/hdf5-1.10.1/src/H5PL.c:878: warning: Using 'dlopen' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking
hadrib at cdl1:~>
Sorry for the lengthy email :D
Regards,
Bilel
-------
Bilel Hadri, PhD
Computational Scientist
KAUST Supercomputing Lab
Al Khawarizmi Bldg. (1) Office 126
4700 King Abdullah University of Science and Technology
Thuwal 23955-6900
Kingdom of Saudi Arabia
Office Phone: +966 12 808 0654
Cell Phone: + 966 544 700 893
From: Phil Carns <carns at mcs.anl.gov>
Date: Monday, February 26, 2018 at 8:18 PM
To: "darshan-users at lists.mcs.anl.gov" <darshan-users at lists.mcs.anl.gov>, "bilel.hadri at kaust.edu.sa" <bilel.hadri at kaust.edu.sa>
Subject: Re: [Darshan-users] darshan3.1.5 issue on Cray XC40 cle6.up05
Actually, it turns out that we don't have a up05 system to test on :)
Can you show us the output of the failing ftn command if you add the -craype-verbose command line option? It might also be helpful to see the contents of the darshan-runtime.pc and module file.
We'll try building scalapack on some up04 systems to see if we either see the same problem or can tell what the environment difference is if not.
thanks,
-Phil
On 02/26/2018 10:18 AM, Phil Carns wrote:
Hi Bilel,
Thanks for the bug report. We can try to reproduce here and confirm.
In the mean time, can you tell me if the test_scalapack.f90 code is using MPI or not?
thanks,
-Phil
On 02/24/2018 02:25 PM, Bilel Hadri wrote:
Dear Darshan colleagues,
I recently installed darshan 3.1.5 on Shaheen, Cray XC40, we recently upgraded the OS to CLE6.up05 and using the 17.12 PrgEnv.
Compiling Scalapack with Cray Libsci fails with the error shown below with all programming environment. Similar issue was observed for other codes, like a simple petsc code.
After digging, it seems that it is related to darshan3.1.5 version recently installed on Shaheen. When unloading darshan, the compilation works fine with no issue.
The error is not appearing when using darshan 3.1.4.
ftn -o exe test_scalapack.f90
/opt/cray/dmapp/default/lib64/libdmapp.a(dmapp_internal.o): In function `_dmappi_is_pure_dmapp_job':
/home/abuild/rpmbuild/BUILD/cray-dmapp-7.1.1/src/dmapp_internal.c:1401: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib/libsci_cray_mpi_mp.a(blacs_exit_.o): In function `blacs_exit_':
/b/worker/csml-libsci-sles/build/mp/scalapack/BLACS/SRC/blacs_exit_.c:42: undefined reference to `__wrap_MPI_Finalize'
/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib/libsci_cray_mpi_mp.a(blacs_pinfo_.o): In function `blacs_pinfo_':
/b/worker/csml-libsci-sles/build/mp/scalapack/BLACS/SRC/blacs_pinfo_.c:18: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/libsci/17.12.1/CRAY/8.6/x86_64/lib/libsci_cray_mpi_mp.a(blacs_pinfo_.oo): In function `Cblacs_pinfo':
/b/worker/csml-libsci-sles/build/mp/scalapack/BLACS/SRC/blacs_pinfo_.c:18: undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/cce/8.6.5/binutils/x86_64/x86_64-pc-linux-gnu/bin/ld: link errors found, deleting executable `exe'
/usr/bin/sha1sum: exe: No such file or directory
hadrib at cdl1:~> ll /usr/bin/sha1
sha1pass sha1sum
hadrib at cdl1:~> ll /usr/bin/sha1sum
-rwxr-xr-x 1 root root 43912 Aug 6 2016 /usr/bin/sha1sum
hadrib at cdl1:~> which sha1sum
/usr/bin/sha1sum
hadrib at cdl1:~> module list
Currently Loaded Modulefiles:
1) modules/3.2.10.6 9) pmi/5.0.13 17) atp/2.1.1
2) eproxy/2.0.22-6.0.5.0_2.1__g1ebe45c.ari 10) dmapp/7.1.1-6.0.5.0_49.8__g1125556.ari 18) perftools-base/7.0.0
3) cce/8.6.5 11) gni-headers/5.0.12-6.0.5.0_2.15__g2ef1ebc.ari 19) PrgEnv-cray/6.0.4
4) craype-network-aries 12) xpmem/2.2.4-6.0.5.0_4.8__g35d5e73.ari 20) cray-mpich/7.7.0
5) craype/2.5.13 13) job/2.2.2-6.0.5.0_8.47__g3c644b5.ari 21) slurm/slurm
6) cray-libsci/17.12.1 14) dvs/2.7_2.2.52-6.0.5.2_17.6__g5170dea 22) craype-haswell
7) udreg/2.3.2-6.0.5.0_13.12__ga14955a.ari 15) alps/6.5.28-6.0.5.0_18.6__g13a91b6.ari 23) texlive/2017
8) ugni/6.0.14-6.0.5.0_16.9__g19583bb.ari 16) rca/2.2.16-6.0.5.0_15.34__g5e09e6d.ari 24) darshan/3.1.5
hadrib at cdl1:~> module swap PrgEnv-cray/6.0.4 PrgEnv-intel
hadrib at cdl1:~> ftn -o exe_i test_scalapack.f90
/opt/cray/pe/libsci/17.12.1/INTEL/16.0/x86_64/lib/libsci_intel_mpi.a(blacs_exit_.o): In function `blacs_exit_':
blacs_exit_.c:(.text+0xe9): undefined reference to `__wrap_MPI_Finalize'
/opt/cray/pe/libsci/17.12.1/INTEL/16.0/x86_64/lib/libsci_intel_mpi.a(blacs_pinfo_.o): In function `blacs_pinfo_':
blacs_pinfo_.c:(.text+0x9b): undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/libsci/17.12.1/INTEL/16.0/x86_64/lib/libsci_intel_mpi.a(blacs_pinfo_.oo): In function `Cblacs_pinfo':
blacs_pinfo_.c:(.text+0x9b): undefined reference to `__wrap_MPI_Init'
hadrib at cdl1:~>
hadrib at cdl1:~>
hadrib at cdl1:~> module swap PrgEnv-intel/6.0.4 PrgEnv-gnu
PrgEnv-gnu PrgEnv-gnu/6.0.4
hadrib at cdl1:~> module swap PrgEnv-intel/6.0.4 PrgEnv-gnu
hadrib at cdl1:~> ftn -o exe_i test_scalapack.f90
/opt/cray/pe/libsci/17.12.1/GNU/6.1/x86_64/lib/libsci_gnu_61_mpi.a(blacs_exit_.o): In function `blacs_exit_':
blacs_exit_.c:(.text+0xdb): undefined reference to `__wrap_MPI_Finalize'
/opt/cray/pe/libsci/17.12.1/GNU/6.1/x86_64/lib/libsci_gnu_61_mpi.a(blacs_pinfo_.o): In function `blacs_pinfo_':
blacs_pinfo_.c:(.text+0xb3): undefined reference to `__wrap_MPI_Init'
/opt/cray/pe/libsci/17.12.1/GNU/6.1/x86_64/lib/libsci_gnu_61_mpi.a(blacs_pinfo_.oo): In function `Cblacs_pinfo':
blacs_pinfo_.c:(.text+0xb3): undefined reference to `__wrap_MPI_Init'
=====
implicit none
integer :: n, nb ! problem size and block size
integer :: myunit ! local output unit number
integer :: myArows, myAcols ! size of local subset of global array
integer :: i,j, igrid,jgrid, iproc,jproc, myi,myj, p
real*8, dimension(:,:), allocatable :: myA,myB,myC
integer :: numroc ! blacs routine
integer :: me, procs, icontxt, prow, pcol, myrow, mycol ! blacs data
integer :: info ! scalapack return value
integer, dimension(9) :: ides_a, ides_b, ides_c ! scalapack array desc
open(unit=1,file="ABCp.dat",status="old",form="formatted")
read(1,*)prow
read(1,*)pcol
read(1,*)n
read(1,*)nb
close(1)
if (((n/nb) < prow) .or. ((n/nb) < pcol)) then
print *,"Problem size too small for processor set!"
stop 100
endif
call blacs_pinfo (me,procs)
call blacs_get (0, 0, icontxt)
call blacs_gridinit(icontxt, 'R', prow, pcol)
call blacs_gridinfo(icontxt, prow, pcol, myrow, mycol)
myunit = 10+me
write(myunit,*)"--------"
write(myunit,*)"Output for processor ",me," to unit ",myunit
write(myunit,*)"Proc ",me,": myrow, mycol in p-array is ", &
myrow, mycol
myArows = numroc(n, nb, myrow, 0, prow)
myAcols = numroc(n, nb, mycol, 0, pcol)
write(myunit,*)"Size of global array is ",n," x ",n
write(myunit,*)"Size of block is ",nb," x ",nb
write(myunit,*)"Size of local array is ",myArows," x ",myAcols
allocate(myA(myArows,myAcols))
allocate(myB(myArows,myAcols))
allocate(myC(myArows,myAcols))
do i=1,n
call g2l(i,n,prow,nb,iproc,myi)
if (myrow==iproc) then
do j=1,n
call g2l(j,n,pcol,nb,jproc,myj)
if (mycol==jproc) then
myA(myi,myj) = real(i+j)
myB(myi,myj) = real(i-j)
myC(myi,myj) = 0.d0
endif
enddo
endif
enddo
! Prepare array descriptors for ScaLAPACK
ides_a(1) = 1 ! descriptor type
ides_a(2) = icontxt ! blacs context
ides_a(3) = n ! global number of rows
ides_a(4) = n ! global number of columns
ides_a(5) = nb ! row block size
ides_a(6) = nb ! column block size
ides_a(7) = 0 ! initial process row
ides_a(8) = 0 ! initial process column
ides_a(9) = myArows ! leading dimension of local array
do i=1,9
ides_b(i) = ides_a(i)
ides_c(i) = ides_a(i)
enddo
! Call ScaLAPACK library routine
call pdgemm('T','T',n,n,n,1.0d0, myA,1,1,ides_a, &
myB,1,1,ides_b,0.d0, &
myC,1,1,ides_c )
! Print results
call g2l(n,n,prow,nb,iproc,myi)
call g2l(n,n,pcol,nb,jproc,myj)
if ((myrow==iproc) .and. (mycol==jproc)) &
write(*,*) 'c(',n,n,')=',myC(myi,myj)
! Deallocate the local arrays
deallocate(myA, myB, myC)
! End blacs for processors that are used
call blacs_gridexit(icontxt)
call blacs_exit(0)
end
! convert global index to local index in block-cyclic distribution
subroutine g2l(i,n,np,nb,p,il)
implicit none
integer :: i ! global array index, input
integer :: n ! global array dimension, input
integer :: np ! processor array dimension, input
integer :: nb ! block size, input
integer :: p ! processor array index, output
integer :: il ! local array index, output
integer :: im1
im1 = i-1
p = mod((im1/nb),np)
il = (im1/(np*nb))*nb + mod(im1,nb) + 1
return
end
! convert local index to global index in block-cyclic distribution
subroutine l2g(il,p,n,np,nb,i)
implicit none
integer :: il ! local array index, input
integer :: p ! processor array index, input
integer :: n ! global array dimension, input
integer :: np ! processor array dimension, input
integer :: nb ! block size, input
integer :: i ! global array index, output
integer :: ilm1
ilm1 = il-1
i = (((ilm1/nb) * np) + p)*nb + mod(ilm1,nb) + 1
return
end
-------
Bilel Hadri, PhD
Computational Scientist
KAUST Supercomputing Lab
Al Khawarizmi Bldg. (1) Office 126
4700 King Abdullah University of Science and Technology
Thuwal 23955-6900
Kingdom of Saudi Arabia
Office Phone: +966 12 808 0654
Cell Phone: + 966 544 700 893
________________________________
This message and its contents including attachments are intended solely for the original recipient. If you are not the intended recipient or have received this message in error, please notify me immediately and delete this message from your computer system. Any unauthorized use or distribution is prohibited. Please consider the environment before printing this email.
_______________________________________________
Darshan-users mailing list
Darshan-users at lists.mcs.anl.gov<mailto:Darshan-users at lists.mcs.anl.gov>
https://lists.mcs.anl.gov/mailman/listinfo/darshan-users
_______________________________________________
Darshan-users mailing list
Darshan-users at lists.mcs.anl.gov<mailto:Darshan-users at lists.mcs.anl.gov>
https://lists.mcs.anl.gov/mailman/listinfo/darshan-users
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-users/attachments/20180227/551b4192/attachment-0001.html>
More information about the Darshan-users
mailing list