/*
 * commproblem.cpp
 *
 *  Created on: Mar 20, 2014
 *      Author: fdkong
 */

#include <petscsnes.h>
#include <petscdm.h>
#include <petsc-private/snesimpl.h>

static char help[] = "Need helps.\n\n";

#undef __FUNCT__
#define __FUNCT__ "main"
int main(int argc,char **argv)
{
   Vec               vec;
   MPI_Request       request;
   MPI_Status        status;
   PetscMPIInt       tag =123;
   MPI_Comm          comm;
   PetscMPIInt       rank, size;
   PetscInt           recv =10;
   PetscInt           send = 0;
   DM                 dm;
   PetscErrorCode    ierr;

   PetscInitialize(&argc,&argv,(char *)0,help);
   comm =  PETSC_COMM_WORLD;
   ierr =  MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
   ierr =  MPI_Comm_size(comm, &size);CHKERRQ(ierr);
   // create an object
   ierr =  VecCreate(comm, &vec);CHKERRQ(ierr);
   // take a comm from that object
   comm =  ((PetscObject) vec)->comm;
   // if we set comm back to PETSC_COMM_WORLD, the code should work fine
   //comm =  PETSC_COMM_WORLD;
   // receive messages from rank 0
   if( rank!=0)
   {
	  ierr = MPI_Irecv(&recv, 1, MPIU_INT, 0,tag, comm, &request);CHKERRQ(ierr);
   }
   if(!rank)
   {
	   //send messages to all others
	   for(PetscMPIInt i =1; i<size;i++)
	   {
		  ierr = MPI_Isend(&send, 1, MPIU_INT, i, tag,  comm, &request);CHKERRQ(ierr);
	   }
   }
   // rank 0 doest not need to wait, it could continue to do other things.
   if(rank !=0)
   {
     ierr =  MPI_Waitall(1,&request, &status);CHKERRQ(ierr);
   }
   ierr = VecDestroy(&vec);CHKERRQ(ierr);
   ierr = PetscFinalize();CHKERRQ(ierr);
}