<div dir="ltr"><div><div><div><div><div>Hi All,<br><br></div>I am having a problem running Petsc3.6 and Slepc3.6 on Stampede.  My code should be a simple eigenvalue solver, but when I attempt to solve large problems (8488x8488 matrices) I get errors:<br><br>--------------------- Error Message --------------------------------------------------------------<br>[1]Total space allocated 1736835920 bytes<br>[1]PETSC ERROR: Out of memory. This could be due to allocating<br>[1]PETSC ERROR: too large an object or bleeding by not properly<br>[1]PETSC ERROR: destroying unneeded objects.<br>[1]PETSC ERROR: Memory allocated 1736835920 Memory used by process 1769742336<br>[1]PETSC ERROR: [0]PETSC ERROR: Memory requested 864587796<br>[1]PETSC ERROR: [0]PETSC ERROR: See <a href="http://www.mcs.anl.gov/petsc/documentation/faq.html">http://www.mcs.anl.gov/petsc/documentation/faq.html</a> for trouble shooting.<br>[1]PETSC ERROR: #8 STSetUp() line 305 in /work/03324/hlammiv/slepc-3.6.0/src/sys/classes/st/interface/stsolve.c<br>[1]PETSC ERROR: [0]PETSC ERROR: #1 MatDuplicateNoCreate_SeqAIJ() line 4030 in /home1/apps/intel15/mvapich2_2_1/petsc/3.6/src/mat/impls/aij/seq/aij.c<br>[1]PETSC ERROR: #2 PetscTrMallocDefault() line 188 in /home1/apps/intel15/mvapich2_2_1/petsc/3.6/src/sys/memory/mtr.c<br>[1]PETSC ERROR: #4 MatDuplicate_SeqAIJ() line 4103 in /home1/apps/intel15/mvapich2_2_1/petsc/3.6/src/mat/impls/aij/seq/aij.c<br>[1]PETSC ERROR: #5 MatDuplicate() line 4252 in /home1/apps/intel15/mvapich2_2_1/petsc/3.6/src/mat/interface/matrix.c<br>[1]PETSC ERROR: #6 STMatMAXPY_Private() line 379 in /work/03324/hlammiv/slepc-3.6.0/src/sys/classes/st/interface/stsolve.c<br>[1]PETSC ERROR: #7 STSetUp_Sinvert() line 131 in /work/03324/hlammiv/slepc-3.6.0/src/sys/classes/st/impls/sinvert/sinvert.c<br>[1]PETSC ERROR: #8 STSetUp() line 305 in /work/03324/hlammiv/slepc-3.6.0/src/sys/classes/st/interface/stsolve.c<br>[1]PETSC ERROR: #9 EPSSliceGetInertia() line 295 in /work/03324/hlammiv/slepc-3.6.0/src/eps/impls/krylov/krylovschur/ks-slice.c<br>[1]PETSC ERROR: #10 EPSSetUp_KrylovSchur_Slice() line 425 in /work/03324/hlammiv/slepc-3.6.0/src/eps/impls/krylov/krylovschur/ks-slice.c<br>[1]PETSC ERROR: #11 EPSSetUp_KrylovSchur() line 89 in /work/03324/hlammiv/slepc-3.6.0/src/eps/impls/krylov/krylovschur/krylovschur.c<br>[1]PETSC ERROR: #12 EPSSetUp() line 121 in /work/03324/hlammiv/slepc-3.6.0/src/eps/interface/epssetup.c<br>[1]PETSC ERROR: #13 EPSSliceGetEPS() line 267 in /work/03324/hlammiv/slepc-3.6.0/src/eps/impls/krylov/krylovschur/ks-slice.c<br>[1]PETSC ERROR: #14 EPSSetUp_KrylovSchur_Slice() line 368 in /work/03324/hlammiv/slepc-3.6.0/src/eps/impls/krylov/krylovschur/ks-slice.c<br>[1]PETSC ERROR: #15 EPSSetUp_KrylovSchur() line 89 in /work/03324/hlammiv/slepc-3.6.0/src/eps/impls/krylov/krylovschur/krylovschur.c<br>[1]PETSC ERROR: #16 EPSSetUp() line 121 in /work/03324/hlammiv/slepc-3.6.0/src/eps/interface/epssetup.c<br>[1]PETSC ERROR: #17 EPSSolve() line 88 in /work/03324/hlammiv/slepc-3.6.0/src/eps/interface/epssolve.c<br>[1]PETSC ERROR: #18 eigensolver() line 64 in /work/03324/hlammiv/TMSWIFT/src/solver.cpp<br>[1]Current space PetscMalloc()ed 1.73683e+09, max space PetscMalloced() 1.73684e+09<br>[1]Current process memory 1.76979e+09 max process memory 1.76979e+09<br><br><br></div>The curious thing about this error, is that it seems that if I increase the number of nodes, from 32 to 64 to 128, the amount of memory per node doesn't decrease.  I have used valgrind and it doesn't seem to a memory leak.<br><br></div>The relevant code piece is:<br><br>void eigensolver(PetscErrorCode ierr, params *params, Mat &H, int argc, char **argv)<br>{<br>    <br><br>    EPS        eps;             /* eigenproblem solver context */<br>      EPSType        type;<br>      ST             st;<br>    KSP            ksp;<br>      PC             pc; <br>    PetscReal    tol,error;<br>    PetscReal    lower,upper;<br>      //PetscInt       nev=dim,maxit,its;<br>      PetscInt           nev,maxit,its,nconv;<br>      Vec                xr,xi;<br>      PetscScalar       kr,ki;<br>    PetscReal     re,im;<br>      PetscViewer        viewer;<br>    PetscInt rank;<br>    PetscInt size;<br>    std::string eig_file_n;<br>    std::ofstream eig_file;    <br>    char ofile[100];<br><br>        MPI_Comm_rank(PETSC_COMM_WORLD,&rank);<br>        MPI_Comm_size(PETSC_COMM_WORLD,&size);<br><br>      ierr = PetscPrintf(PETSC_COMM_WORLD,"---Beginning Eigenvalue Solver---\n");CHKERRV(ierr);<br>     ierr = EPSCreate(PETSC_COMM_WORLD,&eps);CHKERRV(ierr);<br><br>        eig_file_n.append(params->ofile_n);<br>        eig_file_n.append("_eval");<br>        eig_file.open(eig_file_n.c_str(),std::ofstream::trunc);<br><br>    //Set operators. In this case, it is a standard eigenvalue problem<br>    ierr = EPSSetOperators(eps,H,NULL);CHKERRV(ierr);<br>    ierr = EPSSetProblemType(eps,EPS_HEP);CHKERRV(ierr);  <br><br>    ierr = EPSSetType(eps,EPSKRYLOVSCHUR);CHKERRV(ierr);<br><br>      ierr = EPSGetST(eps,&st);CHKERRV(ierr);<br>      ierr = STSetType(st,STSINVERT);CHKERRV(ierr);<br>  <br>      ierr = STGetKSP(st,&ksp);CHKERRV(ierr);<br>      ierr = KSPSetType(ksp,KSPPREONLY);CHKERRV(ierr);<br>      ierr = KSPGetPC(ksp,&pc);CHKERRV(ierr);<br>      ierr = PCSetType(pc,PCCHOLESKY);CHKERRV(ierr);<br>    ierr = EPSKrylovSchurSetPartitions(eps,size);CHKERRV(ierr);<br><br>    for(PetscInt i=0;i<params->nf;i++){<br>    lower=std::pow(2.0*params->m[i]-params->m[i]*params->alpha*params->alpha,2.0);<br>    upper=4.0*params->m[i]*params->m[i];<br>    ierr = EPSSetInterval(eps,lower,upper);<br>    ierr = EPSSetWhichEigenpairs(eps,EPS_ALL);<br>        //Set solver parameters at runtime<br>      ierr = EPSSetFromOptions(eps);CHKERRV(ierr);<br>//     ierr = EPSSetWhichEigenpairs(eps,EPS_SMALLEST_REAL);<br><br>    ierr = MatCreateVecs(H,NULL,&xr);CHKERRV(ierr);<br>    ierr = MatCreateVecs(H,NULL,&xi);CHKERRV(ierr);<br><br><br>       ierr = EPSSolve(eps);CHKERRV(ierr);<br><br>       ierr = EPSGetIterationNumber(eps,&its);CHKERRV(ierr);<br>      ierr = PetscPrintf(PETSC_COMM_WORLD," Number of iterations of the method: %D\n",its);CHKERRV(ierr);<br><br> <br>       //Optional: Get some information from the solver and display it<br>      ierr = EPSGetType(eps,&type);CHKERRV(ierr);<br>      ierr = PetscPrintf(PETSC_COMM_WORLD," Solution method: %s\n\n",type);CHKERRV(ierr);<br>      ierr = EPSGetDimensions(eps,&nev,NULL,NULL);CHKERRV(ierr);<br>      ierr = PetscPrintf(PETSC_COMM_WORLD," Number of requested eigenvalues: %D\n",nev);CHKERRV(ierr);<br>      ierr = EPSGetTolerances(eps,&tol,&maxit);CHKERRV(ierr);<br>      ierr = PetscPrintf(PETSC_COMM_WORLD," Stopping condition: tol=%.4g, maxit=%D\n",tol,maxit);CHKERRV(ierr);<br><br>    ierr = EPSGetConverged(eps,&nconv);CHKERRV(ierr);<br>      ierr = PetscPrintf(PETSC_COMM_WORLD," Number of converged eigenpairs: %D\n\n",nconv);CHKERRV(ierr);<br>        <br>    strcpy(ofile,params->ofile_n);<br>          strcat(ofile,"_evecr");<br><br>        ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD,ofile,&viewer);CHKERRV(ierr);<br><br>    if (nconv>0) <br>    {<br>            ierr = PetscPrintf(PETSC_COMM_WORLD,<br>                 "           k          ||Ax-kx||/||kx||\n"<br>                 "   ----------------- ------------------\n");CHKERRV(ierr);<br><br>        for (PetscInt i=0;i<nconv;i++)<br>        {<br>            //Get converged eigenpairs: i-th eigenvalue is stored in kr (real part) and ki (imaginary part)<br>                  ierr = EPSGetEigenpair(eps,i,&kr,&ki,xr,xi);CHKERRV(ierr);<br>                 //Compute the relative error associated to each eigenpair<br>                 ierr = EPSComputeError(eps,i,EPS_ERROR_RELATIVE,&error);CHKERRV(ierr);<br>                <br>            #if defined(PETSC_USE_COMPLEX)<br>                      re = PetscRealPart(kr);<br>                      im = PetscImaginaryPart(kr);<br>            #else<br>                      re = kr;<br>                      im = ki;<br>            #endif<br><br>                  if (im!=0.0)<br>            {<br>                <br>                ierr = PetscPrintf(PETSC_COMM_WORLD," %9f%+9f j %12g\n",re,im,error);CHKERRV(ierr);<br>                if(rank==0) eig_file << re << " " << im << " " << error << std::endl;<br>            } else <br>            {<br>                    ierr = PetscPrintf(PETSC_COMM_WORLD,"   %12f       %12g\n",re,error);CHKERRV(ierr);<br>                if(rank==0) eig_file << re << " " << 0 << " " << error << std::endl;<br>                 }<br><br>                        ierr = VecView(xr,viewer);CHKERRV(ierr);<br><br>        }<br>            ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRV(ierr);<br>      }<br>    }<br>    eig_file.close();<br>    ierr = EPSDestroy(&eps);CHKERRV(ierr);<br>    ierr = PetscViewerDestroy(&viewer);CHKERRV(ierr);<br>    ierr = VecDestroy(&xr);CHKERRV(ierr);<br>    ierr = VecDestroy(&xi);CHKERRV(ierr);<br>    <br>      ierr = PetscPrintf(PETSC_COMM_WORLD,"---Finishing Eigenvalue Solver---\n");CHKERRV(ierr);<br>}<br><br></div><div><br></div><div><br></div>Thanks,<br></div>Hank<br></div>