[MOAB-dev] r1922 - in MOAB/trunk: . parallel tools/iMesh
tautges at mcs.anl.gov
tautges at mcs.anl.gov
Wed Jun 25 00:20:16 CDT 2008
Author: tautges
Date: 2008-06-25 00:20:15 -0500 (Wed, 25 Jun 2008)
New Revision: 1922
Modified:
MOAB/trunk/MBParallelConventions.h
MOAB/trunk/Tqdcfr.cpp
MOAB/trunk/Tqdcfr.hpp
MOAB/trunk/mbparallelcomm_test.cpp
MOAB/trunk/parallel/MBParallelComm.cpp
MOAB/trunk/parallel/MBParallelComm.hpp
MOAB/trunk/parallel/ReadParallel.cpp
MOAB/trunk/parallel/ReadParallel.hpp
MOAB/trunk/parallel/pcomm_unit.cpp
MOAB/trunk/tools/iMesh/iMeshP_MOAB.cpp
Log:
Major change to how partition and interface sets are cached. Now, any
MBParallelComm object which gets created gets cached in a tag on the
interface instance. By default, a parallel reader (ReadParallel) will
take the first MBParallelComm it gets, or create one if it doesn't
find one. The interface and partition sets for a given pcomm get
stored on that pcomm object in ranges.
This change was motivated by the need to read in multiple files and
establish partitions/interfaces for those files in parallel. The
method I currently use to resolve shared vertices between parts in a
partition relies on global ids of the vertices. When reading in more
than one file, the global id space will typically collide. That means
that reading in both files then computing the shared interfaces will
get the wrong answer, because some vertices will have identical global
ids when they're not really the same vertex.
Now, each file can get read, then resolve the shared interfaces for
the contents of that file, without regard to mesh which may already
exist in a given instance. This commit also enables reading mesh into
a communicator which is not COMM_WORLD.
At least, those things should work.
This commit also fixes an error which prevented reading more than one
.cub file into a given moab instance.
For now, I've only tested reading one file, and it works in parallel
as before. Will test on multiple files next.
make check also works.
Modified: MOAB/trunk/MBParallelConventions.h
===================================================================
--- MOAB/trunk/MBParallelConventions.h 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/MBParallelConventions.h 2008-06-25 05:20:15 UTC (rev 1922)
@@ -69,14 +69,19 @@
#define PSTATUS_INTERFACE 0x4
#define PSTATUS_GHOST 0x8
-/** \brief Tag storing interface sets
+/** \brief Tag storing parallel communication objects
*
- * This tag stores interface sets allocated for a particular
- * interface instance. This is tag of length MAX_SHARING_PROCS
- * (defined in parallel/MBParallelComm.hpp) which stores interface
- * set handles (entries set to 0 by default).
+ * This tag stores pointers to MBParallelComm communication
+ * objects; one of these is allocated for each different
+ * communicator used to read mesh. MBParallelComm stores
+ * partition and interface sets corresponding to its parallel mesh.
+ * By default, a parallel read uses the first MBParallelComm object
+ * on the interface instance; if instantiated with one, ReadParallel
+ * adds this object to the interface instance too.
*
+ * Tag type: opaque
+ * Tag size: MAX_SHARING_PROCS*sizeof(MBParallelComm*)
*/
-#define PARALLEL_IFACE_SETS_TAG_NAME "PARALLEL_IFACE_SETS"
+#define PARALLEL_COMM_TAG_NAME "PARALLEL_COMM"
#endif
Modified: MOAB/trunk/Tqdcfr.cpp
===================================================================
--- MOAB/trunk/Tqdcfr.cpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/Tqdcfr.cpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -160,6 +160,7 @@
readUtilIface = reinterpret_cast<MBReadUtilIface*>(ptr);
currVHandleOffset = -1;
+ firstVHandle = 0;
for (MBEntityType this_type = MBVERTEX; this_type < MBMAXTYPE; this_type++)
currElementIdOffset[this_type] = -1;
@@ -981,6 +982,9 @@
uint_buf[0],
readUtilIface->parallel_rank(),
vhandle, arrays);
+
+ if (!firstVHandle) firstVHandle = vhandle;
+
// get node x's in arrays[0]
FREADDA(entity->nodeCt, arrays[0]);
// get node y's in arrays[1]
@@ -1010,9 +1014,10 @@
if (!cubMOABVertexMap &&
(currVHandleOffset != vhandle_offset || !contig)) {
// get all vertices, removing ones in this batch
- MBRange vrange;
+ MBRange vrange, tmp_range(dum_range);
result = mdbImpl->get_entities_by_type(0, MBVERTEX, vrange); RR;
- vrange = vrange.subtract(dum_range);
+ if (1 != firstVHandle) tmp_range.insert(1, firstVHandle-1);
+ vrange = vrange.subtract(tmp_range);
// compute the max cid; map is indexed by cid, so size is max_cid+1
#define MAX(a,b) (a > b ? a : b)
#define MIN(a,b) (a < b ? a : b)
Modified: MOAB/trunk/Tqdcfr.hpp
===================================================================
--- MOAB/trunk/Tqdcfr.hpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/Tqdcfr.hpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -274,6 +274,7 @@
std::vector<ModelEntry> modelEntries;
MetaDataContainer modelMetaData;
long currVHandleOffset;
+ MBEntityHandle firstVHandle;
long currElementIdOffset[MBMAXTYPE];
MBTag globalIdTag, cubIdTag, geomTag, uniqueIdTag, blockTag, nsTag, ssTag,
attribVectorTag, entityNameTag, categoryTag, hasMidNodesTag;
Modified: MOAB/trunk/mbparallelcomm_test.cpp
===================================================================
--- MOAB/trunk/mbparallelcomm_test.cpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/mbparallelcomm_test.cpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -173,7 +173,8 @@
if (0 == rank) rtime = MPI_Wtime();
if (MB_SUCCESS == tmp_result && 4 != this_opt) {
// now figure out which vertices are shared
- MBParallelComm *pcomm = new MBParallelComm(mbImpl);
+ MBParallelComm *pcomm = MBParallelComm::get_pcomm(mbImpl, 0);
+ assert(pcomm);
MBRange iface_ents[6];
for (int i = 0; i < 4; i++) {
Modified: MOAB/trunk/parallel/MBParallelComm.cpp
===================================================================
--- MOAB/trunk/parallel/MBParallelComm.cpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/parallel/MBParallelComm.cpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -135,6 +135,8 @@
// mpi not initialized yet - initialize here
retval = MPI_Init(&argc, &argv);
}
+
+ add_pcomm(this);
}
MBParallelComm::MBParallelComm(MBInterface *impl,
@@ -154,8 +156,49 @@
// mpi not initialized yet - initialize here
retval = MPI_Init(&argc, &argv);
}
+
+ add_pcomm(this);
}
+MBParallelComm::~MBParallelComm()
+{
+ remove_pcomm(this);
+}
+
+void MBParallelComm::add_pcomm(MBParallelComm *pc)
+{
+ // add this pcomm to instance tag
+ std::vector<MBParallelComm *> pc_array(MAX_SHARING_PROCS,
+ (MBParallelComm*)NULL);
+ MBTag pc_tag = pcomm_tag(mbImpl, true);
+
+ MBErrorCode result = mbImpl->tag_get_data(pc_tag, 0, 0, (void*)&pc_array[0]);
+ if (MB_SUCCESS != result && MB_TAG_NOT_FOUND != result) return;
+ int index = 0;
+ while (index < MAX_SHARING_PROCS && pc_array[index]) index++;
+ if (index == MAX_SHARING_PROCS)
+ assert(false);
+ else {
+ pc_array[index] = pc;
+ mbImpl->tag_set_data(pc_tag, 0, 0, (void*)&pc_array[0]);
+ }
+}
+
+void MBParallelComm::remove_pcomm(MBParallelComm *pc)
+{
+ // remove this pcomm from instance tag
+ std::vector<MBParallelComm *> pc_array(MAX_SHARING_PROCS);
+ MBTag pc_tag = pcomm_tag(mbImpl, true);
+
+ MBErrorCode result = mbImpl->tag_get_data(pc_tag, 0, 0, (void*)&pc_array[0]);
+ std::vector<MBParallelComm*>::iterator pc_it =
+ std::find(pc_array.begin(), pc_array.end(), pc);
+ assert(MB_SUCCESS == result &&
+ pc_it != pc_array.end());
+ *pc_it = NULL;
+ mbImpl->tag_set_data(pc_tag, 0, 0, (void*)&pc_array[0]);
+}
+
//! assign a global id space, for largest-dimension or all entities (and
//! in either case for vertices too)
MBErrorCode MBParallelComm::assign_global_ids(MBEntityHandle this_set,
@@ -2212,17 +2255,14 @@
return MB_SUCCESS;
}
-MBErrorCode MBParallelComm::resolve_shared_ents(int resolve_dim,
+MBErrorCode MBParallelComm::resolve_shared_ents(MBEntityHandle this_set,
+ int resolve_dim,
int shared_dim)
{
MBErrorCode result;
MBRange proc_ents;
// get the entities in the partition sets
- MBRange part_sets;
- result = get_partition_sets(part_sets);
- RRA(" ");
-
- for (MBRange::iterator rit = part_sets.begin(); rit != part_sets.end(); rit++) {
+ for (MBRange::iterator rit = partitionSets.begin(); rit != partitionSets.end(); rit++) {
MBRange tmp_ents;
result = mbImpl->get_entities_by_handle(*rit, tmp_ents, true);
if (MB_SUCCESS != result) return result;
@@ -2247,10 +2287,11 @@
// must call even if we don't have any entities, to make sure
// collective comm'n works
- return resolve_shared_ents(proc_ents, resolve_dim, shared_dim);
+ return resolve_shared_ents(this_set, proc_ents, resolve_dim, shared_dim);
}
-MBErrorCode MBParallelComm::resolve_shared_ents(MBRange &proc_ents,
+MBErrorCode MBParallelComm::resolve_shared_ents(MBEntityHandle this_set,
+ MBRange &proc_ents,
int resolve_dim,
int shared_dim)
{
@@ -2432,8 +2473,7 @@
// create the sets for each interface; store them as tags on
// the interface instance
MBRange iface_sets;
- result = create_interface_sets(proc_nranges, resolve_dim, shared_dim,
- &iface_sets);
+ result = create_interface_sets(proc_nranges, this_set, resolve_dim, shared_dim);
RRA("Trouble creating iface sets.");
// resolve shared entity remote handles; implemented in ghost cell exchange
@@ -2442,22 +2482,22 @@
RRA("Trouble resolving shared entity remote handles.");
// now set the shared/interface tag on non-vertex entities on interface
- result = tag_iface_entities(iface_sets);
+ result = tag_iface_entities();
RRA("Failed to tag iface entities.");
// now build parent/child links for interface sets
- result = create_iface_pc_links(iface_sets);
+ result = create_iface_pc_links();
RRA("Trouble creating interface parent/child links.");
// done
return result;
}
-MBErrorCode MBParallelComm::tag_iface_entities(MBRange &iface_ents)
+MBErrorCode MBParallelComm::tag_iface_entities()
{
MBRange all_ents, if_ents;
MBErrorCode result;
- for (MBRange::iterator if_it = iface_ents.begin(); if_it != iface_ents.end(); if_it++) {
+ for (MBRange::iterator if_it = interfaceSets.begin(); if_it != interfaceSets.end(); if_it++) {
// get all ents
result = mbImpl->get_entities_by_handle(*if_it, if_ents);
RRA("Trouble getting iface entities.");
@@ -2511,8 +2551,8 @@
}
MBErrorCode MBParallelComm::create_interface_sets(std::map<std::vector<int>, MBRange> &proc_nranges,
- int resolve_dim, int shared_dim,
- MBRange *iface_sets_ptr)
+ MBEntityHandle this_set,
+ int resolve_dim, int shared_dim)
{
if (proc_nranges.empty()) return MB_SUCCESS;
@@ -2523,18 +2563,6 @@
RRA("Trouble getting shared proc tags in create_interface_sets.");
MBRange::iterator rit;
- MBRange psets;
- if (!iface_sets_ptr) iface_sets_ptr = &psets;
-
- // get all partition sets and mark contents with iface set tag;
- // pre-use iface_sets
- MBTag pset_tag = partition_tag();
- MBRange tmp_ents, tmp_ents2;
-
- result = mbImpl->get_entities_by_type_and_tag(0, MBENTITYSET, &pset_tag, NULL, 1,
- psets);
- RRA("Couldn't get PARALLEL_PARTITION sets.");
-
// create interface sets, tag them, and tag their contents with iface set tag
std::vector<MBEntityHandle> tag_vals;
std::vector<unsigned char> pstatus;
@@ -2544,7 +2572,7 @@
MBEntityHandle new_set;
result = mbImpl->create_meshset(MESHSET_SET, new_set);
RRA("Failed to create interface set.");
- iface_sets_ptr->insert(new_set);
+ interfaceSets.insert(new_set);
int nump = 0;
while (((*mit).first)[nump] != -1 && nump < MAX_SHARING_PROCS) nump++;
@@ -2575,21 +2603,10 @@
RRA("Failed to tag interface set entities with pstatus.");
}
- // set tag on interface instance holding all interface sets for this instance
- MBEntityHandle tmp_iface_sets[MAX_SHARING_PROCS];
- std::fill(tmp_iface_sets, tmp_iface_sets+MAX_SHARING_PROCS, 0);
- unsigned int i;
- for (rit = iface_sets_ptr->begin(), i = 0; rit != iface_sets_ptr->end();
- rit++, i++)
- tmp_iface_sets[i] = *rit;
- result = mbImpl->tag_set_data(iface_sets_tag(), NULL, 0,
- tmp_iface_sets);
- RRA("Couldn't set iface sets tag on MOAB interface.");
-
return MB_SUCCESS;
}
-MBErrorCode MBParallelComm::create_iface_pc_links(MBRange &iface_sets)
+MBErrorCode MBParallelComm::create_iface_pc_links()
{
// now that we've resolved the entities in the iface sets,
// set parent/child links between the iface sets
@@ -2606,7 +2623,7 @@
std::vector<MBEntityHandle> tag_vals;
MBRange::iterator rit;
- for (rit = iface_sets.begin(); rit != iface_sets.end(); rit++) {
+ for (rit = interfaceSets.begin(); rit != interfaceSets.end(); rit++) {
// tag entities with interface set
iface_ents.clear();
result = mbImpl->get_entities_by_handle(*rit, iface_ents);
@@ -2623,7 +2640,7 @@
// now go back through interface sets and add parent/child links
MBRange tmp_ents2;
for (int d = 2; d >= 0; d--) {
- for (rit = iface_sets.begin(); rit != iface_sets.end(); rit++) {
+ for (rit = interfaceSets.begin(); rit != interfaceSets.end(); rit++) {
// get entities on this interface
iface_ents.clear();
result = mbImpl->get_entities_by_handle(*rit, iface_ents, true);
@@ -2842,10 +2859,10 @@
{
MBRange iface_sets;
std::vector<int> iface_procs;
- MBErrorCode result = get_interface_sets_procs(iface_sets, iface_procs);
+ MBErrorCode result = get_interface_procs(iface_procs);
RRA("Failed to get iface sets/procs.");
- for (MBRange::iterator rit = iface_sets.begin(); rit != iface_sets.end(); rit++) {
+ for (MBRange::iterator rit = interfaceSets.begin(); rit != interfaceSets.end(); rit++) {
if (-1 != other_proc && !is_iface_proc(*rit, other_proc)) continue;
if (-1 == dim) result = mbImpl->get_entities_by_handle(*rit, iface_ents);
@@ -2856,51 +2873,16 @@
return MB_SUCCESS;
}
- //! return partition sets; if tag_name is input, gets sets with
- //! that tag name, otherwise uses PARALLEL_PARTITION tag
-MBErrorCode MBParallelComm::get_partition_sets(MBRange &part_sets,
- const char *tag_name)
+ //! get processors with which this processor communicates; sets are sorted by processor
+MBErrorCode MBParallelComm::get_interface_procs(std::vector<int> &iface_procs)
{
- MBErrorCode result;
-
- if (NULL != tag_name) {
- result = MB_NOT_IMPLEMENTED;
- RRA("Specified tag name not yet implemented.");
- }
-
- MBTag part_tag = partition_tag();
- result = mbImpl->get_entities_by_type_and_tag(0, MBENTITYSET, &part_tag,
- NULL, 1, part_sets);
- return result;
-}
-
- //! get communication interface sets and the processors with which
- //! this processor communicates; sets are sorted by processor
-MBErrorCode MBParallelComm::get_interface_sets_procs(MBRange &iface_sets,
- std::vector<int> &iface_procs)
-{
- MBTag iface_tag = iface_sets_tag();
- if (0 == iface_tag) return MB_FAILURE;
-
// make sure the sharing procs vector is empty
iface_procs.clear();
- // get the iface sets, which are stored on the instance
- int tag_sz;
- MBErrorCode result = mbImpl->tag_get_size(iface_tag, tag_sz);
- RRA("Failed to get iface tag size.");
- tag_sz /= sizeof(MBEntityHandle);
- std::vector<MBEntityHandle> iface_sets_vec(tag_sz);
- result = mbImpl->tag_get_data(iface_tag, NULL, 0, &iface_sets_vec[0]);
- RRA("Failed to find iface tag.");
- // can do a straight copy because ranges don't keep 0-values
- std::copy(iface_sets_vec.begin(), iface_sets_vec.end(),
- mb_range_inserter(iface_sets));
-
// pre-load vector of single-proc tag values
unsigned int i, j;
- std::vector<int> iface_proc(iface_sets.size());
- result = mbImpl->tag_get_data(sharedp_tag(), iface_sets, &iface_proc[0]);
+ std::vector<int> iface_proc(interfaceSets.size());
+ MBErrorCode result = mbImpl->tag_get_data(sharedp_tag(), interfaceSets, &iface_proc[0]);
RRA("Failed to get iface_proc for iface sets.");
// get sharing procs either from single-proc vector or by getting
@@ -2909,7 +2891,7 @@
std::set<int> procs_set;
std::fill(tmp_iface_procs, tmp_iface_procs+MAX_SHARING_PROCS, -1);
MBRange::iterator rit;
- for (rit = iface_sets.begin(), i = 0; rit != iface_sets.end(); rit++, i++) {
+ for (rit = interfaceSets.begin(), i = 0; rit != interfaceSets.end(); rit++, i++) {
if (-1 != iface_proc[i]) procs_set.insert(iface_proc[i]);
else {
// get the sharing_procs tag
@@ -2980,7 +2962,19 @@
RRA("Failed to create/get gid tag handle.");
}
- else if (MB_ALREADY_ALLOCATED != result) {
+ MBRange dum_range;
+ if (MB_ALREADY_ALLOCATED == result) {
+ void *tag_ptr = &def_val;
+ MBErrorCode tmp_result = mbImpl->get_entities_by_type_and_tag(this_set, MBVERTEX,
+ &gid_tag, &tag_ptr, 1,
+ dum_range);
+ if (MB_SUCCESS != tmp_result) {
+ result = tmp_result;
+ RRA("Failed to get gid tag.");
+ }
+ }
+
+ if (MB_ALREADY_ALLOCATED != result || !dum_range.empty()) {
// just created it, so we need global ids
result = assign_global_ids(this_set, dimension, start_id, largest_dim_only,
parallel);
@@ -3109,9 +3103,8 @@
int success;
// get all procs interfacing to this proc
- MBRange iface_sets;
std::vector<int> iface_procs;
- result = get_interface_sets_procs(iface_sets, iface_procs);
+ result = get_interface_procs(iface_procs);
RRA("Failed to get iface sets, procs");
// post ghost irecv's for all interface procs
@@ -3145,7 +3138,7 @@
MBRange bridge_ents;
// get bridge ents on interface(s)
- for (MBRange::iterator rit = iface_sets.begin(); rit != iface_sets.end();
+ for (MBRange::iterator rit = interfaceSets.begin(); rit != interfaceSets.end();
rit++) {
if (!is_iface_proc(*rit, *vit)) continue;
@@ -3175,7 +3168,7 @@
// receive/unpack entities
// number of incoming messages depends on whether we're getting back
// remote handles
- int num_incoming = iface_sets.size() * (store_remote_handles ? 2 : 1);
+ int num_incoming = interfaceSets.size() * (store_remote_handles ? 2 : 1);
while (num_incoming) {
int ind;
@@ -3231,7 +3224,7 @@
result = set_remote_data(&sent_ents_tmp[0], &remote_handles_v[0], sent_ents_tmp.size(),
buffProcs[ind-MAX_SHARING_PROCS]);
RRA("Trouble setting remote data range on sent entities in ghost exchange.");
- result = update_iface_sets(iface_sets, sent_ents[ind-MAX_SHARING_PROCS],
+ result = update_iface_sets(sent_ents[ind-MAX_SHARING_PROCS],
remote_handles_v, buffProcs[ind-MAX_SHARING_PROCS]);
RRA("Trouble updating iface sets.");
break;
@@ -3276,8 +3269,7 @@
return MB_SUCCESS;
}
-MBErrorCode MBParallelComm::update_iface_sets(MBRange &iface_sets,
- MBRange &sent_ents,
+MBErrorCode MBParallelComm::update_iface_sets(MBRange &sent_ents,
std::vector<MBEntityHandle> &remote_handles,
int from_proc)
{
@@ -3288,7 +3280,7 @@
if (!*remote_it) ents_to_remove.insert(*sent_it);
}
- for (MBRange::iterator set_it = iface_sets.begin(); set_it != iface_sets.end(); set_it++) {
+ for (MBRange::iterator set_it = interfaceSets.begin(); set_it != interfaceSets.end(); set_it++) {
if (!is_iface_proc(*set_it, from_proc)) continue;
MBErrorCode result = mbImpl->remove_entities(*set_it, ents_to_remove);
RRA("Couldn't remove entities from iface set in update_iface_sets.");
@@ -3378,22 +3370,6 @@
return pstatusTag;
}
- //! return iface set tag
-MBTag MBParallelComm::iface_sets_tag()
-{
- if (!ifaceSetsTag) {
- MBErrorCode result = mbImpl->tag_create(PARALLEL_IFACE_SETS_TAG_NAME,
- MAX_SHARING_PROCS*sizeof(MBEntityHandle),
- MB_TAG_SPARSE,
- MB_TYPE_HANDLE, ifaceSetsTag,
- NULL, true);
- if (MB_SUCCESS != result && MB_ALREADY_ALLOCATED != result)
- return 0;
- }
-
- return ifaceSetsTag;
-}
-
//! return partition set tag
MBTag MBParallelComm::partition_tag()
{
@@ -3410,6 +3386,35 @@
return partitionTag;
}
+ //! return pcomm tag; passes in impl 'cuz this is a static function
+MBTag MBParallelComm::pcomm_tag(MBInterface *impl,
+ bool create_if_missing)
+{
+ MBTag this_tag = 0;
+ MBErrorCode result = impl->tag_create(PARALLEL_COMM_TAG_NAME,
+ MAX_SHARING_PROCS*sizeof(MBParallelComm*),
+ MB_TAG_SPARSE,
+ MB_TYPE_OPAQUE, this_tag,
+ NULL, create_if_missing);
+ if (MB_SUCCESS != result && MB_ALREADY_ALLOCATED != result)
+ return 0;
+
+ return this_tag;
+}
+
+ //! get the indexed pcomm object from the interface
+MBParallelComm *MBParallelComm::get_pcomm(MBInterface *impl, const int index)
+{
+ MBParallelComm *pc_array[MAX_SHARING_PROCS];
+ MBTag pc_tag = pcomm_tag(impl, false);
+ if (0 == pc_tag) return NULL;
+
+ MBErrorCode result = impl->tag_get_data(pc_tag, 0, 0, (void*)pc_array);
+ if (MB_SUCCESS != result) return NULL;
+
+ return pc_array[index];
+}
+
#ifdef TEST_PARALLELCOMM
#include <iostream>
Modified: MOAB/trunk/parallel/MBParallelComm.hpp
===================================================================
--- MOAB/trunk/parallel/MBParallelComm.hpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/parallel/MBParallelComm.hpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -57,6 +57,9 @@
std::vector<unsigned char> &tmp_buff,
MPI_Comm comm = MPI_COMM_WORLD);
+ //! destructor
+ ~MBParallelComm();
+
static unsigned char PROC_SHARED, PROC_OWNER;
//! assign a global id space, for largest-dimension or all entities (and
@@ -164,7 +167,8 @@
* \param proc_ents Entities for which to resolve shared entities
* \param shared_dim Maximum dimension of shared entities to look for
*/
- MBErrorCode resolve_shared_ents(MBRange &proc_ents,
+ MBErrorCode resolve_shared_ents(MBEntityHandle this_set,
+ MBRange &proc_ents,
int resolve_dim = -1,
int shared_dim = -1);
@@ -179,7 +183,8 @@
* \param dim Dimension of entities in the partition
* \param shared_dim Maximum dimension of shared entities to look for
*/
- MBErrorCode resolve_shared_ents(int resolve_dim = 3,
+ MBErrorCode resolve_shared_ents(MBEntityHandle this_set,
+ int resolve_dim = 3,
int shared_dim = -1);
/** \brief Get entities with the given pstatus bit(s) set
@@ -222,13 +227,12 @@
//! return partition sets; if tag_name is input, gets sets with
//! that tag name, otherwise uses PARALLEL_PARTITION tag
- MBErrorCode get_partition_sets(MBRange &part_sets,
+ MBErrorCode get_partition_sets(MBEntityHandle this_set,
+ MBRange &part_sets,
const char *tag_name = NULL);
- //! get communication interface sets and the processors with which
- //! this processor communicates; sets are sorted by processor
- MBErrorCode get_interface_sets_procs(MBRange &iface_sets,
- std::vector<int> &iface_procs);
+ //! get processors with which this processor communicates; sets are sorted by processor
+ MBErrorCode get_interface_procs(std::vector<int> &iface_procs);
//! pack the buffer with ALL data for orig_ents; return entities actually
//! packed (including reference sub-entities) in final_ents
@@ -260,7 +264,11 @@
MBTag &sharedh_tag,
MBTag &sharedhs_tag,
MBTag &pstatus_tag);
-
+
+ //! return partition, interface set ranges
+ MBRange &partition_sets() {return partitionSets;}
+ MBRange &interface_sets() {return interfaceSets;}
+
//! return sharedp tag
MBTag sharedp_tag();
@@ -276,8 +284,13 @@
//! return pstatus tag
MBTag pstatus_tag();
- //! return iface_set tag
- MBTag iface_sets_tag();
+ //! return pcomm tag; static because might not have a pcomm before going
+ //! to look for one on the interface
+ static MBTag pcomm_tag(MBInterface *impl,
+ bool create_if_missing = true);
+
+ //! get the indexed pcomm object from the interface
+ static MBParallelComm *get_pcomm(MBInterface *impl, const int index);
//! return partitions set tag
MBTag partition_tag();
@@ -416,15 +429,15 @@
// returned; NOTE: a subsequent step is used to verify entities on the interface
// and remove them if they're not shared
MBErrorCode create_interface_sets(std::map<std::vector<int>, MBRange> &proc_nranges,
- int resolve_dim, int shared_dim,
- MBRange *iface_sets_ptr = NULL);
+ MBEntityHandle this_set,
+ int resolve_dim, int shared_dim);
// after verifying shared entities, now parent/child links between sets can be established
- MBErrorCode create_iface_pc_links(MBRange &iface_sets);
+ MBErrorCode create_iface_pc_links();
//! resolve remote handles for shared non-vertex ents, assuming
//! this has already been done for vertices
- MBErrorCode resolve_ent_remote_handles(MBRange &iface_sets);
+ MBErrorCode resolve_ent_remote_handles();
//! pack a range map with keys in this_range and values a contiguous series
//! of handles starting at actual_start
@@ -449,8 +462,7 @@
//! for any remote_handles set to zero, remove corresponding sent_ents from
//! iface_sets corresponding to from_proc
- MBErrorCode update_iface_sets(MBRange &iface_sets,
- MBRange &sent_ents,
+ MBErrorCode update_iface_sets(MBRange &sent_ents,
std::vector<MBEntityHandle> &remote_handles,
int from_proc);
@@ -517,8 +529,14 @@
/** \brief Set pstatus tag interface bit on entities in sets passed in
*/
- MBErrorCode tag_iface_entities(MBRange &iface_ents);
+ MBErrorCode tag_iface_entities();
+
+ //! add a pc to the iface instance tag PARALLEL_COMM
+ void add_pcomm(MBParallelComm *pc);
+ //! remove a pc from the iface instance tag PARALLEL_COMM
+ void remove_pcomm(MBParallelComm *pc);
+
//! MB interface associated with this writer
MBInterface *mbImpl;
@@ -543,6 +561,9 @@
MPI_Request sendReqs[2*MAX_SHARING_PROCS];
std::vector<int> buffProcs;
+
+ //! the partition, interface sets for this comm'n instance
+ MBRange partitionSets, interfaceSets;
//! tags used to save sharing procs and handles
MBTag sharedpTag, sharedpsTag, sharedhTag, sharedhsTag, pstatusTag,
Modified: MOAB/trunk/parallel/ReadParallel.cpp
===================================================================
--- MOAB/trunk/parallel/ReadParallel.cpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/parallel/ReadParallel.cpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -37,7 +37,10 @@
MBParallelComm *pc)
: mbImpl(impl), myPcomm(pc)
{
- if (!myPcomm) myPcomm = new MBParallelComm(mbImpl);
+ if (!myPcomm) {
+ myPcomm = MBParallelComm::get_pcomm(mbImpl, 0);
+ if (NULL == myPcomm) myPcomm = new MBParallelComm(mbImpl);
+ }
}
MBErrorCode ReadParallel::load_file(const char **file_names,
@@ -364,6 +367,10 @@
partition_tag_vals,
distrib,
file_set);
+ if (debug) {
+ std::cerr << "Delete nonlocal done; entities:" << std::endl;
+ mbImpl->list_entities(0, 0);
+ }
break;
//==================
@@ -379,7 +386,7 @@
if (debug)
std::cout << "Resolving shared entities." << std::endl;
- tmp_result = myPcomm->resolve_shared_ents(resolve_dim, shared_dim);
+ tmp_result = myPcomm->resolve_shared_ents(file_set, resolve_dim, shared_dim);
break;
//==================
@@ -438,7 +445,7 @@
result = mbImpl->get_entities_by_type_and_tag(file_set, MBENTITYSET,
&ptag, NULL, 1,
- partition_sets);
+ myPcomm->partition_sets());
RR("Failed to get sets with partition-type tag.");
int proc_sz = myPcomm->proc_config().proc_size();
@@ -447,45 +454,45 @@
if (!ptag_vals.empty()) {
// values input, get sets with those values
MBRange tmp_sets;
- std::vector<int> tag_vals(partition_sets.size());
- result = mbImpl->tag_get_data(ptag, partition_sets, &tag_vals[0]);
+ std::vector<int> tag_vals(myPcomm->partition_sets().size());
+ result = mbImpl->tag_get_data(ptag, myPcomm->partition_sets(), &tag_vals[0]);
RR("Failed to get tag data for partition vals tag.");
for (std::vector<int>::iterator pit = tag_vals.begin();
pit != tag_vals.end(); pit++) {
std::vector<int>::iterator pit2 = std::find(ptag_vals.begin(),
ptag_vals.end(), *pit);
if (pit2 != ptag_vals.end())
- tmp_sets.insert(partition_sets[pit - tag_vals.begin()]);
+ tmp_sets.insert(myPcomm->partition_sets()[pit - tag_vals.begin()]);
}
- partition_sets.swap(tmp_sets);
+ myPcomm->partition_sets().swap(tmp_sets);
}
if (distribute) {
// for now, require that number of partition sets be greater
// than number of procs
- if (partition_sets.size() < (unsigned int) proc_sz) {
+ if (myPcomm->partition_sets().size() < (unsigned int) proc_sz) {
result = MB_FAILURE;
RR("Number of procs greater than number of partitions.");
}
MBRange tmp_sets;
// distribute the partition sets
- unsigned int num_sets = partition_sets.size() / proc_sz;
- if (proc_rk < (int) (partition_sets.size() % proc_sz)) num_sets++;
+ unsigned int num_sets = myPcomm->partition_sets().size() / proc_sz;
+ if (proc_rk < (int) (myPcomm->partition_sets().size() % proc_sz)) num_sets++;
for (unsigned int i = 0; i < num_sets; i++)
- tmp_sets.insert(partition_sets[i*proc_sz + proc_rk]);
+ tmp_sets.insert(myPcomm->partition_sets()[i*proc_sz + proc_rk]);
- partition_sets.swap(tmp_sets);
+ myPcomm->partition_sets().swap(tmp_sets);
}
if (debug) {
std::cerr << "My partition sets: ";
- partition_sets.print();
+ myPcomm->partition_sets().print();
}
- result = delete_nonlocal_entities(partition_sets, file_set); RR(" ");
+ result = delete_nonlocal_entities(file_set); RR(" ");
if (ptag_name != PARALLEL_PARTITION_TAG_NAME) {
// tag the partition sets with a standard tag name
@@ -501,24 +508,23 @@
result = mbImpl->get_entities_by_type_and_tag(file_set, MBENTITYSET, &ptag,
(const void* const*)&proc_rk_ptr, 1,
tagged_sets); RR(" ");
- if (!tagged_sets.empty() && tagged_sets != partition_sets) {
+ if (!tagged_sets.empty() && tagged_sets != myPcomm->partition_sets()) {
result = mbImpl->tag_delete_data(ptag, tagged_sets); RR(" ");
}
- else if (tagged_sets == partition_sets) return MB_SUCCESS;
+ else if (tagged_sets == myPcomm->partition_sets()) return MB_SUCCESS;
}
// if we get here, we need to assign the tag
- std::vector<int> values(partition_sets.size());
- for (unsigned int i = 0; i < partition_sets.size(); i++)
+ std::vector<int> values(myPcomm->partition_sets().size());
+ for (unsigned int i = 0; i < myPcomm->partition_sets().size(); i++)
values[i] = proc_rk;
- result = mbImpl->tag_set_data(ptag, partition_sets, &values[0]); RR(" ");
+ result = mbImpl->tag_set_data(ptag, myPcomm->partition_sets(), &values[0]); RR(" ");
}
return result;
}
-MBErrorCode ReadParallel::delete_nonlocal_entities(MBRange &partition_sets,
- MBEntityHandle file_set)
+MBErrorCode ReadParallel::delete_nonlocal_entities(MBEntityHandle file_set)
{
MBErrorCode result;
@@ -532,7 +538,7 @@
if (debug) std::cout << "Gathering related entities." << std::endl;
- result = read_iface->gather_related_ents(partition_sets, partition_ents,
+ result = read_iface->gather_related_ents(myPcomm->partition_sets(), partition_ents,
&all_sets);
RR("Failure gathering related entities.");
@@ -569,30 +575,5 @@
result = mbImpl->delete_entities(deletable_ents);
RR("Failure deleting entities in delete_nonlocal_entities.");
- // mark partition sets with partition tag, needed later for
- // establishing interface sets
- MBTag partition_set_tag;
- result = mbImpl->tag_create(PARALLEL_PARTITION_TAG_NAME, sizeof(int),
- MB_TAG_SPARSE, MB_TYPE_INTEGER,
- partition_set_tag, NULL, true);
- if (MB_SUCCESS != result && MB_ALREADY_ALLOCATED != result) {
- RR("Couldn't create/get partition set tag.");
- }
-
- std::vector<int> pset_vals(partition_sets.size(), -1);
- if (MB_ALREADY_ALLOCATED == result) {
- // if all partition sets already marked, don't need to mark again
- result = mbImpl->tag_get_data(partition_set_tag, partition_sets,
- &pset_vals[0]);
- if (MB_SUCCESS == result &&
- std::find(pset_vals.begin(), pset_vals.end(), -1) ==
- pset_vals.end()) return MB_SUCCESS;
- }
-
- std::fill(pset_vals.begin(), pset_vals.end(), myPcomm->proc_config().proc_rank());
- result = mbImpl->tag_set_data(partition_set_tag, partition_sets,
- &pset_vals[0]);
- RR("Couldn't set partition set tag value.");
-
return result;
}
Modified: MOAB/trunk/parallel/ReadParallel.hpp
===================================================================
--- MOAB/trunk/parallel/ReadParallel.hpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/parallel/ReadParallel.hpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -64,8 +64,7 @@
bool distribute,
MBEntityHandle file_set);
- MBErrorCode delete_nonlocal_entities(MBRange &partition_sets,
- MBEntityHandle file_set);
+ MBErrorCode delete_nonlocal_entities(MBEntityHandle file_set);
MBInterface *mbImpl;
Modified: MOAB/trunk/parallel/pcomm_unit.cpp
===================================================================
--- MOAB/trunk/parallel/pcomm_unit.cpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/parallel/pcomm_unit.cpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -68,22 +68,24 @@
}
MBRange tmp_range;
- MBParallelComm pcomm( &moab );
+ MBParallelComm *pcomm = new MBParallelComm( &moab );
int size = 0;
std::vector<unsigned char> buff;
- rval = pcomm.pack_buffer( entities, false, true, false, false,
- -1, tmp_range, buff, size );
+ rval = pcomm->pack_buffer( entities, false, true, false, false,
+ -1, tmp_range, buff, size );
CHECK_ERR(rval);
+ delete pcomm;
moab.~MBCore();
+
new (&moab) MBCore();
+ pcomm = new MBParallelComm( &moab);
- pcomm.~MBParallelComm();
- new (&pcomm) MBParallelComm( &moab);
-
entities.clear();
- rval = pcomm.unpack_buffer( &buff[0], false, -1, entities );
+ rval = pcomm->unpack_buffer( &buff[0], false, -1, entities );
CHECK_ERR(rval);
+
+ delete pcomm;
}
void pack_unpack_mesh( MBCore& moab )
{
Modified: MOAB/trunk/tools/iMesh/iMeshP_MOAB.cpp
===================================================================
--- MOAB/trunk/tools/iMesh/iMeshP_MOAB.cpp 2008-06-24 18:48:52 UTC (rev 1921)
+++ MOAB/trunk/tools/iMesh/iMeshP_MOAB.cpp 2008-06-25 05:20:15 UTC (rev 1922)
@@ -139,16 +139,17 @@
/*out*/ int *part_handles_size,
int *err)
{
- MBParallelComm pc(MBI);
+ MBParallelComm *pc = MBParallelComm::get_pcomm(MBI, 0);
+ if (!pc) RETURN(iBase_ERROR_MAP[MB_FAILURE]);
+
MBRange part_sets;
- MBErrorCode result = pc.get_partition_sets(part_sets);
- if (MB_SUCCESS != result) RETURN(iBase_ERROR_MAP[result]);
- *part_handles_size = part_sets.size();
+ *part_handles_size = pc->partition_sets().size();
CHECK_SIZE(*part_handles, *part_handles_allocated, *part_handles_size, iMeshP_PartHandle,);
MBRange::iterator rit;
int i;
- for (i = 0, rit = part_sets.begin(); rit != part_sets.end(); rit++, i++)
+ for (i = 0, rit = pc->partition_sets().begin();
+ rit != pc->partition_sets().end(); rit++, i++)
(*part_handles)[i] = CAST_TO_VOID(*rit);
RETURN(iBase_SUCCESS);
More information about the moab-dev
mailing list