[MOAB-dev] commit/MOAB: danwu: Added a new read option TRIVIAL_PARTITION_SHIFT to NC reader (for HOMME, MPAS and GCRM), which can be used to rotate trivial partition across processors. In parallel unit test of NC writer, localGidVertsOwned.psize() is always 1 for HOMME after trivial partition, so we can apply this option to test cases where localGidVertsOwned.psize() > 1 (and get better code coverage).
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Wed Jun 11 15:18:10 CDT 2014
1 new commit in MOAB:
https://bitbucket.org/fathomteam/moab/commits/21b3e843c6b5/
Changeset: 21b3e843c6b5
Branch: master
User: danwu
Date: 2014-06-11 22:17:53
Summary: Added a new read option TRIVIAL_PARTITION_SHIFT to NC reader (for HOMME, MPAS and GCRM), which can be used to rotate trivial partition across processors. In parallel unit test of NC writer, localGidVertsOwned.psize() is always 1 for HOMME after trivial partition, so we can apply this option to test cases where localGidVertsOwned.psize() > 1 (and get better code coverage).
Affected #: 6 files
diff --git a/src/io/NCHelperGCRM.cpp b/src/io/NCHelperGCRM.cpp
index 14c4b5f..13103a8 100644
--- a/src/io/NCHelperGCRM.cpp
+++ b/src/io/NCHelperGCRM.cpp
@@ -248,6 +248,7 @@ ErrorCode NCHelperGCRM::check_existing_mesh()
ErrorCode NCHelperGCRM::create_mesh(Range& faces)
{
int& gatherSetRank = _readNC->gatherSetRank;
+ int& trivialPartitionShift = _readNC->trivialPartitionShift;
bool& noEdges = _readNC->noEdges;
DebugOutput& dbgOut = _readNC->dbgOut;
@@ -267,19 +268,24 @@ ErrorCode NCHelperGCRM::create_mesh(Range& faces)
createGatherSet = true;
if (procs >= 2) {
+ // Shift rank to obtain a rotated trivial partition
+ int shifted_rank = rank;
+ if (trivialPartitionShift > 0)
+ shifted_rank = (rank + trivialPartitionShift) % procs;
+
// Compute the number of local cells on this proc
nLocalCells = int(std::floor(1.0 * nCells / procs));
// The starting global cell index in the GCRM file for this proc
- int start_cell_idx = rank * nLocalCells;
+ int start_cell_idx = shifted_rank * nLocalCells;
// Number of extra cells after equal split over procs
int iextra = nCells % procs;
// Allocate extra cells over procs
- if (rank < iextra)
+ if (shifted_rank < iextra)
nLocalCells++;
- start_cell_idx += std::min(rank, iextra);
+ start_cell_idx += std::min(shifted_rank, iextra);
start_cell_idx++; // 0 based -> 1 based
diff --git a/src/io/NCHelperHOMME.cpp b/src/io/NCHelperHOMME.cpp
index 85f3a69..a296314 100644
--- a/src/io/NCHelperHOMME.cpp
+++ b/src/io/NCHelperHOMME.cpp
@@ -258,6 +258,7 @@ ErrorCode NCHelperHOMME::create_mesh(Range& faces)
DebugOutput& dbgOut = _readNC->dbgOut;
bool& spectralMesh = _readNC->spectralMesh;
int& gatherSetRank = _readNC->gatherSetRank;
+ int& trivialPartitionShift = _readNC->trivialPartitionShift;
int rank = 0;
int procs = 1;
@@ -355,24 +356,29 @@ ErrorCode NCHelperHOMME::create_mesh(Range& faces)
tmp_conn[4 * i + 3] = tmp_conn2[i + 3 * num_quads];
}
- // Need to know whether we'll be creating gather mesh later, to make sure we allocate enough space
- // in one shot
+ // Need to know whether we'll be creating gather mesh later, to make sure
+ // we allocate enough space in one shot
bool create_gathers = false;
if (rank == gatherSetRank)
create_gathers = true;
+ // Shift rank to obtain a rotated trivial partition
+ int shifted_rank = rank;
+ if (procs >= 2 && trivialPartitionShift > 0)
+ shifted_rank = (rank + trivialPartitionShift) % procs;
+
// Compute the number of local quads, accounting for coarse or fine representation
// spectral_unit is the # fine quads per coarse quad, or spectralOrder^2
int spectral_unit = (spectralMesh ? _spectralOrder * _spectralOrder : 1);
// num_coarse_quads is the number of quads instantiated in MOAB; if !spectralMesh, num_coarse_quads = num_fine_quads
num_coarse_quads = int(std::floor(1.0 * num_quads / (spectral_unit * procs)));
// start_idx is the starting index in the HommeMapping connectivity list for this proc, before converting to coarse quad representation
- start_idx = 4 * rank * num_coarse_quads * spectral_unit;
+ start_idx = 4 * shifted_rank * num_coarse_quads * spectral_unit;
// iextra = # coarse quads extra after equal split over procs
int iextra = num_quads % (procs * spectral_unit);
- if (rank < iextra)
+ if (shifted_rank < iextra)
num_coarse_quads++;
- start_idx += 4 * spectral_unit * std::min(rank, iextra);
+ start_idx += 4 * spectral_unit * std::min(shifted_rank, iextra);
// num_fine_quads is the number of quads in the connectivity list in HommeMapping file assigned to this proc
num_fine_quads = spectral_unit * num_coarse_quads;
diff --git a/src/io/NCHelperMPAS.cpp b/src/io/NCHelperMPAS.cpp
index a02e4ce..b453b82 100644
--- a/src/io/NCHelperMPAS.cpp
+++ b/src/io/NCHelperMPAS.cpp
@@ -299,6 +299,7 @@ ErrorCode NCHelperMPAS::create_mesh(Range& faces)
{
Interface*& mbImpl = _readNC->mbImpl;
int& gatherSetRank = _readNC->gatherSetRank;
+ int& trivialPartitionShift = _readNC->trivialPartitionShift;
bool& noMixedElements = _readNC->noMixedElements;
bool& noEdges = _readNC->noEdges;
@@ -318,19 +319,24 @@ ErrorCode NCHelperMPAS::create_mesh(Range& faces)
createGatherSet = true;
if (procs >= 2) {
+ // Shift rank to obtain a rotated trivial partition
+ int shifted_rank = rank;
+ if (trivialPartitionShift > 0)
+ shifted_rank = (rank + trivialPartitionShift) % procs;
+
// Compute the number of local cells on this proc
nLocalCells = int(std::floor(1.0 * nCells / procs));
// The starting global cell index in the MPAS file for this proc
- int start_cell_idx = rank * nLocalCells;
+ int start_cell_idx = shifted_rank * nLocalCells;
// Number of extra cells after equal split over procs
int iextra = nCells % procs;
// Allocate extra cells over procs
- if (rank < iextra)
+ if (shifted_rank < iextra)
nLocalCells++;
- start_cell_idx += std::min(rank, iextra);
+ start_cell_idx += std::min(shifted_rank, iextra);
start_cell_idx++; // 0 based -> 1 based
diff --git a/src/io/ReadNC.cpp b/src/io/ReadNC.cpp
index cd2c004..4543b64 100644
--- a/src/io/ReadNC.cpp
+++ b/src/io/ReadNC.cpp
@@ -25,7 +25,7 @@ ReadNC::ReadNC(Interface* impl) :
myPcomm(NULL),
#endif
noMesh(false), noVars(false), spectralMesh(false), noMixedElements(false), noEdges(false),
- gatherSetRank(-1), tStepBase(-1), myHelper(NULL)
+ gatherSetRank(-1), tStepBase(-1), trivialPartitionShift(0), myHelper(NULL)
{
assert(impl != NULL);
impl->query_interface(readMeshIface);
@@ -289,6 +289,12 @@ ErrorCode ReadNC::parse_options(const FileOptions& opts, std::vector<std::string
return rval;
}
+ rval = opts.get_int_option("TRIVIAL_PARTITION_SHIFT", 1, trivialPartitionShift);
+ if (MB_TYPE_OUT_OF_RANGE == rval) {
+ readMeshIface->report_error("Invalid value for TRIVIAL_PARTITION_SHIFT option");
+ return rval;
+ }
+
#ifdef USE_MPI
isParallel = (opts.match_option("PARALLEL","READ_PART") != MB_ENTITY_NOT_FOUND);
diff --git a/src/io/ReadNC.hpp b/src/io/ReadNC.hpp
index 1b81d15..b4ec60b 100644
--- a/src/io/ReadNC.hpp
+++ b/src/io/ReadNC.hpp
@@ -209,6 +209,7 @@ private:
bool noEdges;
int gatherSetRank;
int tStepBase;
+ int trivialPartitionShift;
//! Helper class instance
NCHelper* myHelper;
diff --git a/test/io/write_nc.cpp b/test/io/write_nc.cpp
index 7dd8d4f..d0ee024 100644
--- a/test/io/write_nc.cpp
+++ b/test/io/write_nc.cpp
@@ -465,8 +465,10 @@ void test_homme_read_write_T()
// Read non-set variable T and set variable lat
read_opts += ";VARIABLE=T,lat;DEBUG_IO=0";
- if (procs > 1)
- read_opts += ";PARALLEL_RESOLVE_SHARED_ENTS";
+ if (procs > 1) {
+ // Rotate trivial partition, otherwise localGidVertsOwned.psize() is always 1
+ read_opts += ";PARALLEL_RESOLVE_SHARED_ENTS;TRIVIAL_PARTITION_SHIFT=1";
+ }
rval = mb.load_file(example_homme, &set, read_opts.c_str());
CHECK_ERR(rval);
Repository URL: https://bitbucket.org/fathomteam/moab/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the moab-dev
mailing list