[MOAB-dev] r1907 - in MOAB/trunk: . parallel tools/iMesh
tautges at mcs.anl.gov
tautges at mcs.anl.gov
Tue Jun 17 16:31:59 CDT 2008
Author: tautges
Date: 2008-06-17 16:31:59 -0500 (Tue, 17 Jun 2008)
New Revision: 1907
Modified:
MOAB/trunk/MBParallelConventions.h
MOAB/trunk/mbparallelcomm_test.cpp
MOAB/trunk/parallel/MBParallelComm.cpp
MOAB/trunk/parallel/MBParallelComm.hpp
MOAB/trunk/parallel/MBProcConfig.cpp
MOAB/trunk/tools/iMesh/iMeshP_MOAB.cpp
MOAB/trunk/tools/iMesh/iMesh_MOAB.cpp
Log:
A start at fixing ghost exchange, and a few other parallel-related things. mbparallelcomm_test works for 2 procs, but not yet for 4.
iMesh_MOAB: in newMesh, if PARALLEL option is passed, test for MPI
initialization, and init if necessary.
iMeshP_MOAB: flip interpretation of PSTATUS_NOT_OWNED
MBParallelConventions.h: add shared tag, for convenience
mbparallelcomm_test.cpp: correct parsing of distribute option; add
PARALLEL_RESOLVE_SHARED_ENTS to options list, rather than assuming
that's done by default
MBProcConfig: make crystal router non-const
parallel/MBParallelComm.cpp:
- init MPI in constructor if it isn't yet done
- add iface_layer argument to several functions associated with
packing/unpacking; affects whether already-shared entities are sent or
not (when resolving iface entities, you want to send them even though
they may be shared)
- more precise (hopefully) setting of pstatus tags for various
entities
- adding proc_config() function so other code can use this
- adding tag_iface_entities convenience function
Modified: MOAB/trunk/MBParallelConventions.h
===================================================================
--- MOAB/trunk/MBParallelConventions.h 2008-06-17 20:27:58 UTC (rev 1906)
+++ MOAB/trunk/MBParallelConventions.h 2008-06-17 21:31:59 UTC (rev 1907)
@@ -58,8 +58,9 @@
* not shared with any other processors, the pstatus is 0, otherwise it's > 0
*
* bit 0: !owned (0=owned, 1=not owned)
- * bit 1: interface (0=not interface, 1=interface)
- * bit 2: ghost (0=not ghost, 1=ghost)
+ * bit 1: shared (0=not shared, 1=shared)
+ * bit 2: interface (0=not interface, 1=interface)
+ * bit 3: ghost (0=not ghost, 1=ghost)
*/
#define PARALLEL_STATUS_TAG_NAME "PARALLEL_STATUS"
Modified: MOAB/trunk/mbparallelcomm_test.cpp
===================================================================
--- MOAB/trunk/mbparallelcomm_test.cpp 2008-06-17 20:27:58 UTC (rev 1906)
+++ MOAB/trunk/mbparallelcomm_test.cpp 2008-06-17 21:31:59 UTC (rev 1907)
@@ -141,8 +141,8 @@
if (npos < argc) tag_name = argv[npos++];
if (npos < argc) tag_val = strtol(argv[npos++], NULL, 0);
if (npos < argc) distrib = strtol(argv[npos++], NULL, 0);
+ else distrib = 1;
if (npos < argc) with_ghosts = strtol(argv[npos++], NULL, 0);
- else distrib = 1;
tmp_result = read_file(mbImpl, filename, tag_name, tag_val,
distrib, parallel_option, with_ghosts);
if (MB_SUCCESS != tmp_result) {
@@ -358,7 +358,7 @@
options << ";PARTITION_DISTRIBUTE";
if (1 == with_ghosts)
- options << ";PARALLEL_GHOSTS=3.0.1";
+ options << ";PARALLEL_RESOLVE_SHARED_ENTS;PARALLEL_GHOSTS=3.0.1";
options << ";CPUTIME";
@@ -731,7 +731,7 @@
MBParallelComm *pcomm = new MBParallelComm(mbImpl);
std::vector<unsigned char> buff(1024);
int buff_size;
- result = pcomm->pack_buffer(ents, false, true, false, -1,
+ result = pcomm->pack_buffer(ents, false, true, false, false, -1,
whole_range, buff, buff_size);
RRA("Packing buffer count (non-stored handles) failed.");
Modified: MOAB/trunk/parallel/MBParallelComm.cpp
===================================================================
--- MOAB/trunk/parallel/MBParallelComm.cpp 2008-06-17 20:27:58 UTC (rev 1906)
+++ MOAB/trunk/parallel/MBParallelComm.cpp 2008-06-17 21:31:59 UTC (rev 1907)
@@ -19,6 +19,7 @@
#include <numeric>
#define MIN(a,b) (a < b ? a : b)
+#define MAX(a,b) (a > b ? a : b)
const bool debug = false;
const bool debug_packing = false;
@@ -124,6 +125,16 @@
tagServer = dynamic_cast<MBCore*>(mbImpl)->tag_server();
sequenceManager = dynamic_cast<MBCore*>(mbImpl)->sequence_manager();
+
+ int flag = 1;
+ int retval = MPI_Initialized(&flag);
+ if (MPI_SUCCESS != retval || !flag) {
+ int argc = 0;
+ char **argv = NULL;
+
+ // mpi not initialized yet - initialize here
+ retval = MPI_Init(&argc, &argv);
+ }
}
MBParallelComm::MBParallelComm(MBInterface *impl,
@@ -134,6 +145,15 @@
partitionTag(0)
{
myBuffer.swap(tmp_buff);
+ int flag = 1;
+ int retval = MPI_Initialized(&flag);
+ if (MPI_SUCCESS != retval || !flag) {
+ int argc = 0;
+ char **argv = NULL;
+
+ // mpi not initialized yet - initialize here
+ retval = MPI_Init(&argc, &argv);
+ }
}
//! assign a global id space, for largest-dimension or all entities (and
@@ -342,7 +362,7 @@
int buff_size;
result = pack_buffer(orig_ents, adjacencies, tags,
- store_remote_handles, to_proc,
+ store_remote_handles, iface_layer, to_proc,
final_ents, send_buff, buff_size);
RRA("Failed to pack buffer in pack_send.");
@@ -473,7 +493,7 @@
std::vector<unsigned char> buff;
if ((int)procConfig.proc_rank() == from_proc) {
result = pack_buffer( entities, adjacencies, tags,
- false, -1,
+ false, false, -1,
whole_range, buff, buff_size);
RRA("Failed to compute buffer size in broadcast_entities.");
}
@@ -505,6 +525,7 @@
const bool adjacencies,
const bool tags,
const bool store_remote_handles,
+ const bool iface_layer,
const int to_proc,
MBRange &final_ents,
std::vector<unsigned char> &buff,
@@ -579,8 +600,9 @@
// entities
result = pack_entities(orig_ents, rit, final_ents, buff_ptr,
- buff_size, true, store_remote_handles, to_proc,
- ent_types, all_ranges, verts_per_entity);
+ buff_size, true, store_remote_handles,
+ iface_layer, to_proc, ent_types, all_ranges,
+ verts_per_entity);
RRA("Packing entities (count) failed.");
// sets
@@ -617,8 +639,9 @@
// entities
result = pack_entities(orig_ents, rit, final_ents, buff_ptr,
- buff_size, false, store_remote_handles, to_proc,
- ent_types, all_ranges, verts_per_entity);
+ buff_size, false, store_remote_handles,
+ iface_layer, to_proc, ent_types,
+ all_ranges, verts_per_entity);
RRA("Packing entities (real) failed.");
#ifdef DEBUG_PACKING
std::cerr << "pack_entities buffer space: " << buff_ptr - &buff[0] << " bytes." << std::endl;
@@ -722,6 +745,7 @@
int &count,
const bool just_count,
const bool store_remote_handles,
+ const bool iface_layer,
const int to_proc,
std::vector<MBEntityType> &ent_types,
std::vector<MBRange> &all_ranges,
@@ -792,10 +816,6 @@
}
MBRange::const_iterator end_rit = start_rit;
- // If we return now because there are no elements, then MBMAXTYPE
- // doesn't get appended and the unpack code fails. -- j.kraftcheck
- //if (allRanges[0].size() == entities.size()) return MB_SUCCESS;
-
std::vector<MBRange>::iterator allr_it = all_ranges.begin();
// pack entities
@@ -829,7 +849,7 @@
std::copy(start_rit, end_rit, mb_range_inserter(*all_ranges.rbegin()));
// remove any which are already shared
- if (store_remote_handles) {
+ if (store_remote_handles && !iface_layer) {
result = remove_nonowned_shared(*all_ranges.rbegin(), to_proc, false);
RRA("Failed nonowned_shared test.");
}
@@ -1462,21 +1482,14 @@
result = rmv_remote_proc(local_ents[i], remote_procs, remote_handles,
other_proc);
RRA(" ");
- if (-1 == remote_procs[0])
- pstatus_vals[i] = (pstatus_vals[i] & (!PSTATUS_SHARED));
}
else {
result = add_remote_proc(local_ents[i], remote_procs, remote_handles,
other_proc, remote_ents[i]);
RRA(" ");
- pstatus_vals[i] |= PSTATUS_SHARED;
}
}
- result = mbImpl->tag_set_data(pstatus_tag, local_ents,
- num_ents, &pstatus_vals[0]);
- RRA("Couldn't set shared tag (vector)");
-
return MB_SUCCESS;
}
@@ -1502,6 +1515,9 @@
// since they're sparse tags
result = mbImpl->tag_set_data(sharedp_tag(), &ent, 1, remote_procs);
result = mbImpl->tag_set_data(sharedh_tag(), &ent, 1, remote_hs);
+ // if it's not shared, it's also not a ghost, interface, or !owned entity
+ unsigned char not_shared = 0;
+ result = mbImpl->tag_set_data(pstatus_tag(), &ent, 1, ¬_shared);
}
else if (i == 2) {
// went from 2 to 1, need to unset 1 and set the other
@@ -2419,6 +2435,10 @@
result = exchange_ghost_cells(-1, -1, 0, true, true);
RRA("Trouble resolving shared entity remote handles.");
+ // now set the shared/interface tag on non-vertex entities on interface
+ result = tag_iface_entities(iface_sets);
+ RRA("Failed to tag iface entities.");
+
// now build parent/child links for interface sets
result = create_iface_pc_links(iface_sets);
RRA("Trouble creating interface parent/child links.");
@@ -2427,6 +2447,28 @@
return result;
}
+MBErrorCode MBParallelComm::tag_iface_entities(MBRange &iface_ents)
+{
+ MBRange all_ents, if_ents;
+ MBErrorCode result;
+ for (MBRange::iterator if_it = iface_ents.begin(); if_it != iface_ents.end(); if_it++) {
+ // get all ents
+ result = mbImpl->get_entities_by_handle(*if_it, if_ents);
+ RRA("Trouble getting iface entities.");
+ }
+ std::vector<unsigned char> tvals(if_ents.size());
+ result = mbImpl->tag_get_data(pstatus_tag(), if_ents, &tvals[0]);
+ RRA("Failed to get pstatus tag values.");
+
+ for (unsigned int i = 0; i < tvals.size(); i++)
+ tvals[i] |= PSTATUS_INTERFACE;
+
+ result = mbImpl->tag_set_data(pstatus_tag(), if_ents, &tvals[0]);
+ RRA("Failed to set pstatus tag values.");
+
+ return MB_SUCCESS;
+}
+
MBErrorCode MBParallelComm::set_pstatus_entities(MBRange &pstatus_ents,
unsigned char pstatus_val,
bool lower_dim_ents,
@@ -2489,6 +2531,7 @@
// create interface sets, tag them, and tag their contents with iface set tag
std::vector<MBEntityHandle> tag_vals;
+ std::vector<unsigned char> pstatus;
for (std::map<std::vector<int>,MBRange>::iterator mit = proc_nranges.begin();
mit != proc_nranges.end(); mit++) {
// create the set
@@ -2514,10 +2557,16 @@
// get the owning proc, then set the pstatus tag on iface set
int min_proc = ((*mit).first)[0];
- unsigned char pstatus = (PSTATUS_SHARED | PSTATUS_INTERFACE);
- if (min_proc < (int) procConfig.proc_rank()) pstatus |= PSTATUS_NOT_OWNED;
- result = mbImpl->tag_set_data(pstatus_tag, &new_set, 1, &pstatus);
+ unsigned char pval = (PSTATUS_SHARED | PSTATUS_INTERFACE);
+ if (min_proc < (int) procConfig.proc_rank()) pval |= PSTATUS_NOT_OWNED;
+ result = mbImpl->tag_set_data(pstatus_tag, &new_set, 1, &pval);
RRA("Failed to tag interface set with pstatus.");
+
+ // tag the entities with the same thing
+ pstatus.resize((*mit).second.size());
+ std::fill(pstatus.begin(), pstatus.end(), pval);
+ result = mbImpl->tag_set_data(pstatus_tag, (*mit).second, &pstatus[0]);
+ RRA("Failed to tag interface set entities with pstatus.");
}
// set tag on interface instance holding all interface sets for this instance
@@ -2706,7 +2755,7 @@
&sharing_hs[0]);
RRA("Failed to set sharedhs_tag on non-vertex skin entity.");
}
-
+
// reset sharing proc(s) tags
std::fill(sharing_procs.begin(), sharing_procs.end(), -1);
}
@@ -2767,13 +2816,9 @@
}
RRA("Failed setting shared_procs tag on skin vertices.");
+ // tag the entity as shared & interface too (this function only
+ // ever called for interface vertices)
unsigned char share_flag = PSTATUS_SHARED;
- if (!proc_verts.empty() && proc_verts.find(this_ent) == proc_verts.end())
- share_flag |= (PSTATUS_NOT_OWNED | PSTATUS_GHOST);
-
- else if (proc_verts.empty() && (int) procConfig.proc_rank() > sharing_procs[0])
- share_flag |= PSTATUS_NOT_OWNED;
-
result = mbImpl->tag_set_data(pstatus_tag, &this_ent, 1, &share_flag);
RRA("Couldn't set shared tag on shared vertex.");
@@ -3084,6 +3129,10 @@
// make sendReqs vector to simplify initialization
std::fill(sendReqs, sendReqs+2*MAX_SHARING_PROCS, MPI_REQUEST_NULL);
MBRange recd_ents[MAX_SHARING_PROCS], sent_ents[MAX_SHARING_PROCS];
+
+ // keep track of new ghosted and ghost ents so we can tag them later
+ MBRange new_ghosted, new_ghosts;
+
for (vit = iface_procs.begin(); vit != iface_procs.end(); vit++) {
int ind = get_buffers(*vit);
@@ -3108,11 +3157,13 @@
// pack-send; this also posts receives if store_remote_handles is true
result = pack_send_entities(*vit, bridge_ents, false, true,
- store_remote_handles, (0 == num_layers),
+ store_remote_handles, (0 == num_layers),
ownerSBuffs[ind], ownerRBuffs[MAX_SHARING_PROCS+ind],
sendReqs[ind], recv_reqs[MAX_SHARING_PROCS+ind],
sent_ents[ind]);
RRA("Failed to pack-send in ghost exchange.");
+
+ if (0 != num_layers) new_ghosted.merge(sent_ents[ind]);
}
// receive/unpack entities
@@ -3156,6 +3207,7 @@
ghostRBuffs[ind], ghostSBuffs[ind],
sendReqs[ind], recd_ents[ind]);
RRA("Failed to recv-unpack message.");
+ if (0 != num_layers) new_ghosts.merge(recd_ents[ind]);
break;
case MB_MESG_REMOTE_HANDLES_VECTOR:
// incoming remote handles; use to set remote handles
@@ -3192,6 +3244,19 @@
}
}
+ if (0 != num_layers) {
+ unsigned char pval = (PSTATUS_SHARED | PSTATUS_GHOST | PSTATUS_NOT_OWNED);
+ std::vector<unsigned char> pvals(MAX(new_ghosts.size(), new_ghosted.size()), pval);
+ // new ghost entities are always new
+ result = mbImpl->tag_set_data(pstatus_tag(), new_ghosts, &pvals[0]);
+ RRA("Trouble setting tags for new ghost entities");
+
+ pval = PSTATUS_SHARED;
+ std::fill(pvals.begin(), pvals.end(), pval);
+ result = mbImpl->tag_set_data(pstatus_tag(), new_ghosted, &pvals[0]);
+ RRA("Trouble setting tags for new ghosted entities");
+ }
+
// ok, now wait if requested
if (wait_all) {
MPI_Status status[2*MAX_SHARING_PROCS];
@@ -3381,7 +3446,7 @@
if (MB_SUCCESS != result) return result;
int buff_size;
- result = pcomm.pack_buffer(all_mesh, false, true, true, whole_range, buff_size);
+ result = pcomm.pack_buffer(all_mesh, false, true, true, false, whole_range, buff_size);
PM;
@@ -3390,7 +3455,7 @@
// pack the actual buffer
int actual_buff_size;
- result = pcomm.pack_buffer(whole_range, false, true, false, all_mesh,
+ result = pcomm.pack_buffer(whole_range, false, true, false, false, all_mesh,
actual_buff_size);
PM;
Modified: MOAB/trunk/parallel/MBParallelComm.hpp
===================================================================
--- MOAB/trunk/parallel/MBParallelComm.hpp 2008-06-17 20:27:58 UTC (rev 1906)
+++ MOAB/trunk/parallel/MBParallelComm.hpp 2008-06-17 21:31:59 UTC (rev 1907)
@@ -236,6 +236,7 @@
const bool adjacencies,
const bool tags,
const bool store_remote_handles,
+ const bool iface_layer,
const int to_proc,
MBRange &final_ents,
std::vector<unsigned char> &buff,
@@ -250,6 +251,9 @@
//! Get proc config for this communication object
const MBProcConfig &proc_config() const {return procConfig;}
+ //! Get proc config for this communication object
+ MBProcConfig &proc_config() {return procConfig;}
+
//! return the tags used to indicate shared procs and handles
MBErrorCode get_shared_proc_tags(MBTag &sharedp_tag,
MBTag &sharedps_tag,
@@ -336,6 +340,7 @@
int &count,
const bool just_count,
const bool store_remote_handles,
+ const bool iface_layer,
const int to_proc,
std::vector<MBEntityType> &ent_types,
std::vector<MBRange> &all_ranges,
@@ -510,6 +515,10 @@
int remote_proc,
MBEntityHandle remote_handle);
+ /** \brief Set pstatus tag interface bit on entities in sets passed in
+ */
+ MBErrorCode tag_iface_entities(MBRange &iface_ents);
+
//! MB interface associated with this writer
MBInterface *mbImpl;
Modified: MOAB/trunk/parallel/MBProcConfig.cpp
===================================================================
--- MOAB/trunk/parallel/MBProcConfig.cpp 2008-06-17 20:27:58 UTC (rev 1906)
+++ MOAB/trunk/parallel/MBProcConfig.cpp 2008-06-17 21:31:59 UTC (rev 1907)
@@ -32,7 +32,7 @@
#endif
}
-crystal_data *MBProcConfig::crystal_router(bool construct_if_missing)
+crystal_data *MBProcConfig::crystal_router(bool construct_if_missing)
{
if (!crystalInit && construct_if_missing)
#ifdef USE_MPI
Modified: MOAB/trunk/tools/iMesh/iMeshP_MOAB.cpp
===================================================================
--- MOAB/trunk/tools/iMesh/iMeshP_MOAB.cpp 2008-06-17 20:27:58 UTC (rev 1906)
+++ MOAB/trunk/tools/iMesh/iMeshP_MOAB.cpp 2008-06-17 21:31:59 UTC (rev 1907)
@@ -190,8 +190,8 @@
&pstatus);
if (MB_SUCCESS != result) RETURN(result);
- if (pstatus & PSTATUS_NOT_OWNED) *is_owner = 1;
- else *is_owner = 0;
+ if (pstatus & PSTATUS_NOT_OWNED) *is_owner = 0;
+ else *is_owner = 1;
RETURN(iBase_SUCCESS);
}
@@ -217,8 +217,8 @@
CHECK_SIZE(*is_owner, *is_owner_allocated, *is_owner_size, int, );
for (int i = 0; i < entity_handles_size; i++) {
- if (pstatus[i] & PSTATUS_NOT_OWNED) (*is_owner)[i] = 1;
- else (*is_owner)[i] = 0;
+ if (pstatus[i] & PSTATUS_NOT_OWNED) (*is_owner)[i] = 0;
+ else (*is_owner)[i] = 1;
}
RETURN(iBase_SUCCESS);
Modified: MOAB/trunk/tools/iMesh/iMesh_MOAB.cpp
===================================================================
--- MOAB/trunk/tools/iMesh/iMesh_MOAB.cpp 2008-06-17 20:27:58 UTC (rev 1906)
+++ MOAB/trunk/tools/iMesh/iMesh_MOAB.cpp 2008-06-17 21:31:59 UTC (rev 1907)
@@ -256,6 +256,7 @@
iMesh_Instance *instance, int *err, int options_len)
{
std::string tmp_options(options, options_len);
+ eatwhitespace(tmp_options);
FileOptions opts(tmp_options.c_str());
MBInterface* core;
@@ -263,6 +264,15 @@
MBErrorCode result = opts.get_null_option("PARALLEL");
if (MB_SUCCESS == result) {
#ifdef USE_MPI
+ int flag = 1;
+ int retval = MPI_Initialized(&flag);
+ if (MPI_SUCCESS != retval || !flag) {
+ int argc = 0;
+ char **argv = NULL;
+
+ // mpi not initialized yet - initialize here
+ retval = MPI_Init(&argc, &argv);
+ }
int rank, size;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
@@ -282,11 +292,6 @@
RETURN(iBase_FAILURE);
}
- if (0 != options_len) {
- iMesh_processError(iBase_NOT_SUPPORTED, "No options for iMesh factory have been implemented.");
- RETURN(iBase_NOT_SUPPORTED);
- }
-
RETURN(iBase_SUCCESS);
}
More information about the moab-dev
mailing list