[MOAB-dev] r2762 - in MOAB/branches/parallel_ghosting: . parallel tools/mbcoupler
tautges at mcs.anl.gov
tautges at mcs.anl.gov
Thu Mar 26 16:54:59 CDT 2009
Author: tautges
Date: 2009-03-26 16:54:59 -0500 (Thu, 26 Mar 2009)
New Revision: 2762
Modified:
MOAB/branches/parallel_ghosting/MBEntityType.h
MOAB/branches/parallel_ghosting/MBTypes.h
MOAB/branches/parallel_ghosting/parallel/MBParallelComm.cpp
MOAB/branches/parallel_ghosting/parallel/MBParallelComm.hpp
MOAB/branches/parallel_ghosting/parallel/parallel_unit_tests.cpp
MOAB/branches/parallel_ghosting/tools/mbcoupler/mbcoupler_test.cpp
Log:
- Added checks so that parallel_unit_test fails if ghosting tests fail
- added MBParallelComm::check_all_shared_handles(), which calls
exchange_all_shared_handles and checks results
- Fixed a couple of bugs in MBParallelComm::exchange_ghost_cells,
mostly with populating the sharedEnts structure.
Modified: MOAB/branches/parallel_ghosting/MBEntityType.h
===================================================================
--- MOAB/branches/parallel_ghosting/MBEntityType.h 2009-03-26 21:42:37 UTC (rev 2761)
+++ MOAB/branches/parallel_ghosting/MBEntityType.h 2009-03-26 21:54:59 UTC (rev 2762)
@@ -44,16 +44,16 @@
};
#ifdef __cplusplus
-} // extern "C"
+} /* extern "C" */
-//! prefix increment operator for MBEntityType
+/* prefix increment operator for MBEntityType */
inline MBEntityType & operator++(MBEntityType &type)
{
return type = static_cast<MBEntityType>(type+1);
}
-//! postfix increment operator for MBEntityType
+/* postfix increment operator for MBEntityType */
inline MBEntityType operator++(MBEntityType &type, int)
{
MBEntityType oldval = type;
@@ -61,13 +61,13 @@
return oldval;
}
-//! prefix increment operator for MBEntityType
+/* prefix increment operator for MBEntityType */
inline MBEntityType & operator--(MBEntityType &type)
{
return type = static_cast<MBEntityType>(type-1);
}
-//! postfix increment operator for MBEntityType
+/* postfix increment operator for MBEntityType */
inline MBEntityType operator--(MBEntityType &type, int)
{
MBEntityType oldval = type;
Modified: MOAB/branches/parallel_ghosting/MBTypes.h
===================================================================
--- MOAB/branches/parallel_ghosting/MBTypes.h 2009-03-26 21:42:37 UTC (rev 2761)
+++ MOAB/branches/parallel_ghosting/MBTypes.h 2009-03-26 21:54:59 UTC (rev 2762)
@@ -93,7 +93,7 @@
};
#ifdef __cplusplus
-} // extern "C"
+} /* extern "C" */
#endif
/*@}*/
Modified: MOAB/branches/parallel_ghosting/parallel/MBParallelComm.cpp
===================================================================
--- MOAB/branches/parallel_ghosting/parallel/MBParallelComm.cpp 2009-03-26 21:42:37 UTC (rev 2761)
+++ MOAB/branches/parallel_ghosting/parallel/MBParallelComm.cpp 2009-03-26 21:54:59 UTC (rev 2762)
@@ -921,6 +921,12 @@
owned_shared.merge(owned_ents);
}
+ // else if we're sending interface ents, add all the entities
+ // to the ownedShared list
+ else if (!is_ghost)
+ owned_shared.merge(entities);
+
+
if (store_remote_handles) {
// pack the local handles
int tmp_space = RANGE_SIZE(entities);
@@ -3132,7 +3138,7 @@
if (sharing_procs[0] < (int)procConfig.proc_rank()) {
for (unsigned int j = 0; j < sharing_procs.size() && sharing_procs[j] != -1; j++) {
if (sharing_procs[j] == (int)procConfig.proc_rank()) continue;
- int ind = get_buffers(sharing_procs[j]);
+ get_buffers(sharing_procs[j]);
sharedEnts2.insert(this_ent);
}
}
@@ -3142,7 +3148,7 @@
for (unsigned int j = 0; j < sharing_procs.size() && sharing_procs[j] != -1; j++) {
if (sharing_procs[j] == (int)procConfig.proc_rank()) continue;
assert(sharing_procs[j] > (int)procConfig.proc_rank());
- int ind = get_buffers(sharing_procs[j]);
+ get_buffers(sharing_procs[j]);
sharedEnts2.insert(this_ent);
}
}
@@ -4999,6 +5005,144 @@
return 0;
}
+MBErrorCode MBParallelComm::exchange_all_shared_handles( shared_entity_map& result )
+{
+ MBErrorCode rval;
+ int ierr;
+ const int tag = 0x4A41534E;
+ const MPI_Comm comm = procConfig.proc_comm();
+ std::set<unsigned int> exch_procs;
+ rval = get_comm_procs(exch_procs);
+ if (MB_SUCCESS != rval)
+ return rval;
+ const int num_proc = exch_procs.size();
+ std::vector<MPI_Request> send_req(num_proc), recv_req(num_proc);
+ const std::vector<int> procs( exch_procs.begin(), exch_procs.end() );
+
+ // get all shared entities
+ MBRange all_shared;
+ MBTag pstatus = pstatus_tag();
+ for (MBEntityType type = MBVERTEX; type < MBENTITYSET; ++type) {
+ rval = get_moab()->get_entities_by_type_and_tag( 0, type, &pstatus, 0, 1, all_shared,
+ MBInterface::UNION);
+ if (MB_SUCCESS != rval)
+ return rval;
+ }
+
+ // build up send buffers
+ shared_entity_map send_data;
+ int ent_procs[MAX_SHARING_PROCS];
+ MBEntityHandle handles[MAX_SHARING_PROCS];
+ int num_sharing;
+ SharedEntityData tmp;
+ for (MBRange::iterator i = all_shared.begin(); i != all_shared.end(); ++i) {
+ tmp.remote = *i; // swap local/remote so they're correct on the remote proc.
+ rval = get_owner( *i, tmp.owner );
+ if (MB_SUCCESS != rval)
+ return rval;
+
+ rval = get_sharing_parts( *i, ent_procs, num_sharing, handles );
+ for (int j = 0; j < num_sharing; ++j) {
+ if (ent_procs[j] == (int)proc_config().proc_rank())
+ continue;
+ tmp.local = handles[j];
+ send_data[ent_procs[j]].push_back( tmp );
+ }
+ }
+
+ // set up to receive sizes
+ std::vector<int> sizes_send(num_proc), sizes_recv(num_proc);
+ for (int i = 0; i < num_proc; ++i) {
+ ierr = MPI_Irecv( &sizes_recv[i], 1, MPI_INT, procs[i], tag, comm, &recv_req[i] );
+ if (ierr)
+ return MB_FILE_WRITE_ERROR;
+ }
+
+ // send sizes
+ for (int i = 0; i < num_proc; ++i) {
+ sizes_send[i] = send_data[procs[i]].size();
+ ierr = MPI_Isend( &sizes_send[i], 1, MPI_INT, procs[i], tag, comm, &send_req[i] );
+ if (ierr)
+ return MB_FILE_WRITE_ERROR;
+ }
+
+ // receive sizes
+ std::vector<MPI_Status> stat(num_proc);
+ ierr = MPI_Waitall( num_proc, &recv_req[0], &stat[0] );
+ if (ierr)
+ return MB_FILE_WRITE_ERROR;
+
+ // wait until all sizes are sent (clean up pending req's)
+ ierr = MPI_Waitall( num_proc, &send_req[0], &stat[0] );
+ if (ierr)
+ return MB_FILE_WRITE_ERROR;
+
+ // set up to receive data
+ for (int i = 0; i < num_proc; ++i) {
+ result[procs[i]].resize( sizes_recv[i] );
+ ierr = MPI_Irecv( &result[procs[i]][0],
+ sizeof(SharedEntityData)*sizes_recv[i],
+ MPI_UNSIGNED_CHAR,
+ procs[i], tag, comm, &recv_req[i] );
+ if (ierr)
+ return MB_FILE_WRITE_ERROR;
+ }
+
+ // send data
+ for (int i = 0; i < num_proc; ++i) {
+ ierr = MPI_Isend( &send_data[procs[i]][0],
+ sizeof(SharedEntityData)*sizes_send[i],
+ MPI_UNSIGNED_CHAR,
+ procs[i], tag, comm, &send_req[i] );
+ if (ierr)
+ return MB_FILE_WRITE_ERROR;
+ }
+
+ // receive data
+ ierr = MPI_Waitall( num_proc, &recv_req[0], &stat[0] );
+ if (ierr)
+ return MB_FILE_WRITE_ERROR;
+
+ // wait until everything is sent to release send buffers
+ ierr = MPI_Waitall( num_proc, &send_req[0], &stat[0] );
+ if (ierr)
+ return MB_FILE_WRITE_ERROR;
+
+ return MB_SUCCESS;
+}
+
+MBErrorCode MBParallelComm::check_all_shared_handles()
+{
+ // get all shared ent data from other procs
+ shared_entity_map shents;
+ MBErrorCode result;
+ result = exchange_all_shared_handles(shents);
+ if (MB_SUCCESS != result)
+ return result;
+ else if (shents.empty())
+ return MB_SUCCESS;
+
+ // now check against what I think data should be
+ MBRange dum_range, bad_ents, local_shared;
+ for (shared_entity_map::iterator mit = shents.begin(); mit != shents.end(); mit++) {
+ int other_proc = (*mit).first;
+ int ind = get_buffers(other_proc);
+ if (-1 == ind) return MB_FAILURE;
+ local_shared = shared_ents()[ind].ownedShared;
+ shared_entity_vec &shvec = (*mit).second;
+ for (shared_entity_vec::iterator vit = shvec.begin(); vit != shvec.end(); vit++) {
+ MBEntityHandle localh = vit->local, remoteh = vit->remote, dumh;
+ local_shared.erase(localh);
+ result = get_remote_handles(true, &localh, &dumh, 1, other_proc, dum_range);
+ if (MB_SUCCESS != result || dumh != remoteh)
+ bad_ents.insert(localh);
+ }
+ }
+
+ if (!bad_ents.empty() || !local_shared.empty()) return MB_FAILURE;
+ else return MB_SUCCESS;
+}
+
#ifdef TEST_PARALLELCOMM
#include <iostream>
Modified: MOAB/branches/parallel_ghosting/parallel/MBParallelComm.hpp
===================================================================
--- MOAB/branches/parallel_ghosting/parallel/MBParallelComm.hpp 2009-03-26 21:42:37 UTC (rev 2761)
+++ MOAB/branches/parallel_ghosting/parallel/MBParallelComm.hpp 2009-03-26 21:54:59 UTC (rev 2762)
@@ -778,6 +778,30 @@
//! add vertices adjacent to entities in this list
MBErrorCode add_verts(MBRange &sent_ents);
+ struct SharedEntityData {
+ MBEntityHandle local;
+ MBEntityHandle remote;
+ int owner;
+ };
+
+ typedef std::vector< SharedEntityData > shared_entity_vec;
+
+ //! Map indexed by processor ID and containing, for each processor ID,
+ //! a list of <local,remote> handle pairs, where the local handle is
+ //! the handle on this processor and the remove handle is the handle on
+ //! the processor ID indicated by the map index.
+ typedef std::map< int, shared_entity_vec > shared_entity_map;
+
+ //! Every processor sends shared entity handle data to every other processor
+ //! that it shares entities with. Passed back map is all received data,
+ //! indexed by processor ID. This function is intended to be used for
+ //! debugging.
+ MBErrorCode exchange_all_shared_handles( shared_entity_map& result );
+
+ //! Call exchange_all_shared_handles, then compare the results with tag data
+ //! on local shared entities.
+ MBErrorCode check_all_shared_handles();
+
//! replace handles in from_vec with corresponding handles on
//! to_proc (by checking shared[p/h]_tag and shared[p/h]s_tag;
//! if no remote handle and new_ents is non-null, substitute
Modified: MOAB/branches/parallel_ghosting/parallel/parallel_unit_tests.cpp
===================================================================
--- MOAB/branches/parallel_ghosting/parallel/parallel_unit_tests.cpp 2009-03-26 21:42:37 UTC (rev 2761)
+++ MOAB/branches/parallel_ghosting/parallel/parallel_unit_tests.cpp 2009-03-26 21:54:59 UTC (rev 2762)
@@ -593,22 +593,14 @@
shared_ents.merge(pcomm->shared_ents()[i].localHandles);
}
- // over all interfaces, the owned and shared lists should be mutually exclusive
- // and the interface ents should be contained in one of those
- if (!(owned_ents.intersect(shared_ents)).empty()) {
- std::cerr << "Contents of localHandles and ownedShared not consistent on proc "
- << pcomm->proc_config().proc_rank() << std::endl;
+ rval = pcomm->check_all_shared_handles();
+ if (MB_SUCCESS != rval) {
my_error = 1;
- }
- MBRange rem_ents = iface_ents.subtract(shared_ents);
- rem_ents = rem_ents.subtract(owned_ents);
- if (!rem_ents.empty()) {
- std::cerr << "Interface entities inconsistent with sharedEnts on proc "
+ std::cerr << "check_all_shared_handles test failed on proc "
<< pcomm->proc_config().proc_rank() << std::endl;
- my_error = 1;
}
PCHECK(!my_error);
-
+
// finally, check adjacencies just to make sure they're consistent
rval = mb_instance.check_adjacencies();
if (MB_SUCCESS != rval) my_error = 1;
@@ -939,6 +931,10 @@
MPI_INT, pcomm->proc_config().proc_comm() );
PCHECK(!error);
+ rval = pcomm->check_all_shared_handles();
+ if (MB_SUCCESS != rval) error = 1;
+ PCHECK(!error);
+
// for each ghost entity, check owning processor and list of
// sharing processors.
int k = 0;
@@ -972,7 +968,8 @@
break;
}
}
-
+ PCHECK(!error);
+
// done
return MB_SUCCESS;
}
Modified: MOAB/branches/parallel_ghosting/tools/mbcoupler/mbcoupler_test.cpp
===================================================================
--- MOAB/branches/parallel_ghosting/tools/mbcoupler/mbcoupler_test.cpp 2009-03-26 21:42:37 UTC (rev 2761)
+++ MOAB/branches/parallel_ghosting/tools/mbcoupler/mbcoupler_test.cpp 2009-03-26 21:54:59 UTC (rev 2762)
@@ -151,7 +151,7 @@
// now figure out which vertices are shared
for (unsigned int p = 0; p < pcs.size(); p++) {
for (int i = 0; i < 4; i++) {
- tmp_result = pcs[p]->get_iface_entities(-1, i, iface_ents[i]);
+ tmp_result = pcs[p]->get_iface_entities(-1, iface_ents[i], i);
if (MB_SUCCESS != tmp_result) {
std::cerr << "get_iface_entities returned error on proc "
More information about the moab-dev
mailing list