[MOAB-dev] commit/MOAB: danwu: Updated NC writer to make it work for HOMME in serial. Parallel support on Euler and HOMME will be added later.
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Tue Apr 15 16:32:51 CDT 2014
1 new commit in MOAB:
https://bitbucket.org/fathomteam/moab/commits/48d8c8710b5b/
Changeset: 48d8c8710b5b
Branch: ncwriter
User: danwu
Date: 2014-04-15 23:32:36
Summary: Updated NC writer to make it work for HOMME in serial. Parallel support on Euler and HOMME will be added later.
Affected #: 12 files
diff --git a/src/io/NCWriteEuler.cpp b/src/io/NCWriteEuler.cpp
index 4671ed2..41661a7 100644
--- a/src/io/NCWriteEuler.cpp
+++ b/src/io/NCWriteEuler.cpp
@@ -20,7 +20,7 @@ NCWriteEuler::~NCWriteEuler()
// TODO Auto-generated destructor stub
}
-ErrorCode NCWriteEuler::write_values(std::vector<std::string>& var_names, EntityHandle fileSet)
+ErrorCode NCWriteEuler::write_values(std::vector<std::string>& var_names)
{
Interface*& mbImpl = _writeNC->mbImpl;
std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates;
@@ -79,7 +79,7 @@ ErrorCode NCWriteEuler::write_values(std::vector<std::string>& var_names, Entity
switch (variableData.entLoc) {
case WriteNC::ENTLOCFACE:
// Faces
- rval = mbImpl->get_entities_by_dimension(fileSet, 2, ents);
+ rval = mbImpl->get_entities_by_dimension(_fileSet, 2, ents);
ERRORR(rval, "Can't get entities for faces.");
break;
default:
@@ -117,19 +117,8 @@ ErrorCode NCWriteEuler::write_values(std::vector<std::string>& var_names, Entity
ERRORS(success, "Failed to write double data.");
break;
}
- case NC_INT: {
- std::vector<int> tmpintdata(ni*nj*nk);
- // Transpose (lat, lon, lev) back to (lev, lat, lon)
- jik_to_kji(ni, nj, nk, &tmpintdata[0], (int*)(dataptr));
- success = NCFUNCAP(_vara_int)(_fileId, variableData.varId,
- &variableData.writeStarts[0], &variableData.writeCounts[0],
- &tmpintdata[0]);
- ERRORS(success, "Failed to write int data.");
- break;
- }
default:
- success = 1;
- break;
+ ERRORR(MB_FAILURE, "Not implemented yet.");
}
}
}
@@ -141,14 +130,8 @@ ErrorCode NCWriteEuler::write_values(std::vector<std::string>& var_names, Entity
&variableData.writeCounts[0], (double*)(variableData.memoryHogs[0]));
ERRORS(success, "Failed to write double data.");
break;
- case NC_INT:
- success = NCFUNCAP(_vara_int)(_fileId, variableData.varId, &variableData.writeStarts[0],
- &variableData.writeCounts[0], (int*)(variableData.memoryHogs[0]));
- ERRORS(success, "Failed to write int data.");
- break;
default:
- success = 1;
- break;
+ ERRORR(MB_FAILURE, "Not implemented yet.");
}
}
}
diff --git a/src/io/NCWriteEuler.hpp b/src/io/NCWriteEuler.hpp
index b1c5c78..e7009e7 100644
--- a/src/io/NCWriteEuler.hpp
+++ b/src/io/NCWriteEuler.hpp
@@ -22,7 +22,7 @@ public:
virtual ~NCWriteEuler();
private:
- ErrorCode write_values(std::vector<std::string>& var_names, EntityHandle fileSet);
+ ErrorCode write_values(std::vector<std::string>& var_names);
};
} // namespace moab
diff --git a/src/io/NCWriteFV.cpp b/src/io/NCWriteFV.cpp
index 1eca105..39e325e 100644
--- a/src/io/NCWriteFV.cpp
+++ b/src/io/NCWriteFV.cpp
@@ -20,7 +20,7 @@ NCWriteFV::~NCWriteFV()
// TODO Auto-generated destructor stub
}
-ErrorCode NCWriteFV::write_values(std::vector<std::string>& var_names, EntityHandle fileSet)
+ErrorCode NCWriteFV::write_values(std::vector<std::string>& var_names)
{
return MB_NOT_IMPLEMENTED;
}
diff --git a/src/io/NCWriteFV.hpp b/src/io/NCWriteFV.hpp
index 6d7f022..c9c6a7d 100644
--- a/src/io/NCWriteFV.hpp
+++ b/src/io/NCWriteFV.hpp
@@ -22,7 +22,7 @@ public:
virtual ~NCWriteFV();
private:
- ErrorCode write_values(std::vector<std::string>& var_names, EntityHandle fileSet);
+ ErrorCode write_values(std::vector<std::string>& var_names);
};
} // namespace moab
diff --git a/src/io/NCWriteHOMME.cpp b/src/io/NCWriteHOMME.cpp
index 8625f14..bc92cc4 100644
--- a/src/io/NCWriteHOMME.cpp
+++ b/src/io/NCWriteHOMME.cpp
@@ -20,9 +20,10 @@ NCWriteHOMME::~NCWriteHOMME()
// TODO Auto-generated destructor stub
}
-ErrorCode NCWriteHOMME::write_values(std::vector<std::string>& var_names, EntityHandle fileSet)
+ErrorCode NCWriteHOMME::write_values(std::vector<std::string>& var_names)
{
Interface*& mbImpl = _writeNC->mbImpl;
+ Tag& mGlobalIdTag = _writeNC->mGlobalIdTag;
std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates;
std::set<std::string>& dummyVarNames = _writeNC->dummyVarNames;
std::map<std::string, WriteNC::VarData>& varInfo = _writeNC->varInfo;
@@ -62,8 +63,124 @@ ErrorCode NCWriteHOMME::write_values(std::vector<std::string>& var_names, Entity
// Now look at requested var_names; if they have time, we will have a list, and write one at a time
// Need to transpose from lev dimension
+ Range localGidVerts;
+ Range local_verts;
+ ErrorCode rval = mbImpl->get_entities_by_dimension(_fileSet, 0, local_verts);
+ ERRORR(rval, "Trouble getting local vertices in current file set.");
- // Write data, to be implemented ...
+ if (!local_verts.empty()) {
+ std::vector<int> gids(local_verts.size());
+
+ rval = mbImpl->tag_get_data(mGlobalIdTag, local_verts, &gids[0]);
+ ERRORR(rval, "Trouble getting global IDs on local vertices.");
+
+ // Restore localGidVerts
+ std::copy(gids.rbegin(), gids.rend(), range_inserter(localGidVerts));
+ }
+
+ // For each variable tag in the indexed lists, write a time step data
+ // Assume the first dimension is time (need to check); if not, just write regularly
+ for (size_t i = 0; i < var_names.size(); i++) {
+ std::map<std::string, WriteNC::VarData>::iterator vit = varInfo.find(var_names[i]);
+ if (vit == varInfo.end())
+ ERRORR(MB_FAILURE, "Can't find variable requested.");
+
+ WriteNC::VarData& variableData = vit->second;
+ int numTimeSteps = (int)variableData.varTags.size();
+ if (variableData.has_tsteps) {
+ // Get entities of this variable
+ Range ents;
+ switch (variableData.entLoc) {
+ case WriteNC::ENTLOCVERT:
+ // Vertices
+ rval = mbImpl->get_entities_by_dimension(_fileSet, 0, ents);
+ ERRORR(rval, "Can't get entities for vertices.");
+ break;
+ default:
+ ERRORR(MB_FAILURE, "Unexpected entity location type for HOMME non-set variable.");
+ }
+
+ // A typical variable has 3 dimensions as (time, lev, ncol)
+ // At each timestep, we need to transpose tag format (ncol, lev) back
+ // to NC format (lev, ncol) for writing
+ int nLocalVertices = localGidVerts.size(); // ncol
+ size_t numLev = variableData.writeCounts[1]; // lev
+
+ variableData.writeCounts[0] = 1; // We will write one time step
+ for (int j = 0; j < numTimeSteps; j++) {
+ // We will write one time step, and count will be one; start will be different
+ // We will write values directly from tag_iterate, but we should also transpose for level
+ // so that means deep copy for transpose
+ variableData.writeStarts[0] = j; // This is time, again
+ int count;
+ void* dataptr;
+ rval = mbImpl->tag_iterate(variableData.varTags[j], ents.begin(), ents.end(), count, dataptr);
+ assert(count == (int)ents.size());
+
+#ifdef PNETCDF_FILE
+ size_t nb_writes = localGidVerts.psize();
+ std::vector<int> requests(nb_writes), statuss(nb_writes);
+ size_t idxReq = 0;
+#endif
+
+ // Now write from memory directly
+ int success = 0;
+ switch (variableData.varDataType) {
+ case NC_DOUBLE: {
+ std::vector<double> tmpdoubledata(nLocalVertices * numLev);
+ // Transpose (ncol, lev) back to (lev, ncol)
+ jik_to_kji(nLocalVertices, 1, numLev, &tmpdoubledata[0], (double*)(dataptr));
+
+ size_t indexInDoubleArray = 0;
+ size_t ic = 0;
+ for (Range::pair_iterator pair_iter = localGidVerts.pair_begin();
+ pair_iter != localGidVerts.pair_end(); ++pair_iter, ic++) {
+ EntityHandle starth = pair_iter->first;
+ EntityHandle endh = pair_iter->second;
+ variableData.writeStarts[2] = (NCDF_SIZE)(starth - 1);
+ variableData.writeCounts[2] = (NCDF_SIZE)(endh - starth + 1);
+
+ // Do a partial write, in each subrange
+#ifdef PNETCDF_FILE
+ // Wait outside this loop
+ success = NCFUNCREQP(_vara_double)(_fileId, variableData.varId,
+ &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
+ &(tmpdoubledata[indexInDoubleArray]), &requests[idxReq++]);
+#else
+ success = NCFUNCAP(_vara_double)(_fileId, variableData.varId,
+ &(variableData.writeStarts[0]), &(variableData.writeCounts[0]),
+ &(tmpdoubledata[indexInDoubleArray]));
+#endif
+ ERRORS(success, "Failed to read double data in loop");
+ // We need to increment the index in double array for the
+ // next subrange
+ indexInDoubleArray += (endh - starth + 1) * numLev;
+ }
+ assert(ic == localGidVerts.psize());
+#ifdef PNETCDF_FILE
+ success = ncmpi_wait_all(_fileId, requests.size(), &requests[0], &statuss[0]);
+ ERRORS(success, "Failed on wait_all.");
+#endif
+ break;
+ }
+ default:
+ ERRORR(MB_FAILURE, "Not implemented yet.");
+ }
+ }
+ } // if (variableData.has_tsteps)
+ else {
+ int success = 0;
+ switch (variableData.varDataType) {
+ case NC_DOUBLE:
+ success = NCFUNCAP(_vara_double)(_fileId, variableData.varId, &variableData.writeStarts[0],
+ &variableData.writeCounts[0], (double*)(variableData.memoryHogs[0]));
+ ERRORS(success, "Failed to write double data.");
+ break;
+ default:
+ ERRORR(MB_FAILURE, "Not implemented yet.");
+ }
+ }
+ }
return MB_SUCCESS;
}
diff --git a/src/io/NCWriteHOMME.hpp b/src/io/NCWriteHOMME.hpp
index c0db566..4cdf299 100644
--- a/src/io/NCWriteHOMME.hpp
+++ b/src/io/NCWriteHOMME.hpp
@@ -22,7 +22,7 @@ public:
virtual ~NCWriteHOMME();
private:
- ErrorCode write_values(std::vector<std::string>& var_names, EntityHandle fileSet);
+ ErrorCode write_values(std::vector<std::string>& var_names);
};
} // namespace moab
diff --git a/src/io/NCWriteHelper.hpp b/src/io/NCWriteHelper.hpp
index 157c2dc..d190922 100644
--- a/src/io/NCWriteHelper.hpp
+++ b/src/io/NCWriteHelper.hpp
@@ -24,7 +24,7 @@ public:
static NCWriteHelper* get_nc_helper(WriteNC* writeNC, int fileId, const FileOptions& opts, EntityHandle fileSet);
//! Take the info from VarData and write first the coordinates, then the actual variables
- virtual ErrorCode write_values(std::vector<std::string>& var_names, EntityHandle fileSet) = 0;
+ virtual ErrorCode write_values(std::vector<std::string>& var_names) = 0;
protected:
template <typename T> void jik_to_kji(size_t ni, size_t nj, size_t nk, T* dest, T* source)
diff --git a/src/io/NCWriteMPAS.cpp b/src/io/NCWriteMPAS.cpp
index a385968..f544e96 100644
--- a/src/io/NCWriteMPAS.cpp
+++ b/src/io/NCWriteMPAS.cpp
@@ -20,7 +20,7 @@ NCWriteMPAS::~NCWriteMPAS()
// TODO Auto-generated destructor stub
}
-ErrorCode NCWriteMPAS::write_values(std::vector<std::string>& var_names, EntityHandle fileSet)
+ErrorCode NCWriteMPAS::write_values(std::vector<std::string>& var_names)
{
return MB_NOT_IMPLEMENTED;
}
diff --git a/src/io/NCWriteMPAS.hpp b/src/io/NCWriteMPAS.hpp
index 815292a..3d9e829 100644
--- a/src/io/NCWriteMPAS.hpp
+++ b/src/io/NCWriteMPAS.hpp
@@ -22,7 +22,7 @@ public:
virtual ~NCWriteMPAS();
private:
- ErrorCode write_values(std::vector<std::string>& var_names, EntityHandle fileSet);
+ ErrorCode write_values(std::vector<std::string>& var_names);
};
} // namespace moab
diff --git a/src/io/WriteNC.cpp b/src/io/WriteNC.cpp
index e05dadf..af20151 100644
--- a/src/io/WriteNC.cpp
+++ b/src/io/WriteNC.cpp
@@ -116,7 +116,7 @@ ErrorCode WriteNC::write_file(const char* file_name,
rval = initialize_file(var_names);
ERRORR(rval, "Failed to initialize file.");
- rval = myHelper->write_values(var_names, *file_set);
+ rval = myHelper->write_values(var_names);
ERRORR(rval, "Failed to write values.");
success = NCFUNC(close)(fileId);
diff --git a/src/io/WriteNC.hpp b/src/io/WriteNC.hpp
index 855bc6e..47debd5 100644
--- a/src/io/WriteNC.hpp
+++ b/src/io/WriteNC.hpp
@@ -41,6 +41,9 @@
//! Collective I/O mode put
#define NCFUNCAP(func) ncmpi_put ## func ## _all
+//! Nonblocking put (request aggregation), used so far only for ucd mesh
+#define NCFUNCREQP(func) ncmpi_iput ## func
+
#define NCDF_SIZE MPI_Offset
#define NCDF_DIFF MPI_Offset
#else
diff --git a/test/io/write_nc.cpp b/test/io/write_nc.cpp
index 5261532..a9b9e0c 100644
--- a/test/io/write_nc.cpp
+++ b/test/io/write_nc.cpp
@@ -45,7 +45,7 @@ int main(int argc, char* argv[])
result += RUN_TEST(test_eul_read_write_T);
result += RUN_TEST(test_eul_check_T);
result += RUN_TEST(test_homme_read_write_T);
- //result += RUN_TEST(test_homme_check_T);
+ result += RUN_TEST(test_homme_check_T);
#ifdef USE_MPI
fail = MPI_Finalize();
@@ -57,7 +57,7 @@ int main(int argc, char* argv[])
}
// We also read and write set variable gw, which is required to create the mesh
-// In test_eul_check_T_values(), we need to load the output file with mesh
+// In test_eul_check_T(), we need to load the output file with mesh
void test_eul_read_write_T()
{
Core moab;
@@ -150,7 +150,7 @@ void test_eul_check_T()
}
// We also read and write set variables lat and lon, which are are required to create the mesh
-// In test_homme_check_T_values(), we need to load the output file with mesh
+// In test_homme_check_T(), we need to load the output file with mesh
void test_homme_read_write_T()
{
Core moab;
Repository URL: https://bitbucket.org/fathomteam/moab/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the moab-dev
mailing list