[MOAB-dev] commit/MOAB: iulian07: first pass at append option
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Tue Apr 29 16:35:45 CDT 2014
1 new commit in MOAB:
https://bitbucket.org/fathomteam/moab/commits/f48ec38dbf3e/
Changeset: f48ec38dbf3e
Branch: master
User: iulian07
Date: 2014-04-29 23:32:58
Summary: first pass at append option
small test now for euler only
maybe it should be tested for more formats
assumes that if a dimension is already defined, there should be
a variable with the same name (coordinate)
Affected #: 5 files
diff --git a/src/io/NCWriteHelper.cpp b/src/io/NCWriteHelper.cpp
index 4f95ef8..923fe02 100644
--- a/src/io/NCWriteHelper.cpp
+++ b/src/io/NCWriteHelper.cpp
@@ -209,7 +209,7 @@ ErrorCode NCWriteHelper::collect_variable_data(std::vector<std::string>& var_nam
return MB_SUCCESS;
}
-ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names)
+ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names, bool append)
{
std::vector<std::string>& dimNames = _writeNC->dimNames;
std::set<std::string>& usedCoordinates = _writeNC->usedCoordinates;
@@ -221,6 +221,9 @@ ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names)
int tDim_in_dimNames = tDim;
int levDim_in_dimNames = levDim;
+ // if append mode, make sure we are in define mode; a simple open will not allow creation of new variables
+ if (append)
+ NCFUNC(redef)(_fileId);
// First initialize all coordinates, then fill VarData for actual variables (and dimensions)
// Check that for used coordinates we have found the tags
for (std::set<std::string>::iterator setIt = usedCoordinates.begin();
@@ -234,6 +237,40 @@ ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names)
WriteNC::VarData& varCoordData = vit->second;
varCoordData.varDims.resize(1);
+ // if not append, create it for sure
+ // if append, we might already have it, including the tag / variable with the same name
+ /*
+ * int ncmpi_inq_dimid(int ncid, const char *name, int *idp);
+ */
+ if (append)
+ {
+ int dimId;
+ if (NCFUNC(inq_dimid)(_fileId, coordName.c_str(), &dimId) == NC_NOERR)
+ // if not found, create it later
+ {
+ varCoordData.varDims[0] = dimId;
+ // check that the coordinate is a variable too
+ // Skip dummy coordinate variables (e.g. ncol)
+ dbgOut.tprintf(2, " file already has coordName %s dim id is %d \n", coordName.c_str(), (int)varCoordData.varDims[0]);
+ if (dummyVarNames.find(coordName) != dummyVarNames.end())
+ continue;
+ // inquire for a variable with the same name
+
+ int varId;
+ if (NCFUNC(inq_varid)(_fileId, coordName.c_str(), &varId) != NC_NOERR)
+ ERRORR(MB_FAILURE, "we do not have a variable with the same name.");
+ // we should also check that this variable has one dimension, and it is dimId
+ varCoordData.varId = varId;
+ dbgOut.tprintf(2, " file already has coordinate %s and varId is %d \n", coordName.c_str(), varId);
+ // Update tDim and levDim to actual dimension id
+ if (coordName == dimNames[tDim_in_dimNames])
+ tDim = varCoordData.varDims[0];
+ else if (coordName == dimNames[levDim_in_dimNames])
+ levDim = varCoordData.varDims[0];
+ continue; // maybe more checks are needed here
+ }
+
+ }
/* int nc_def_dim (int ncid, const char *name, size_t len, int *dimidp);
* example: status = nc_def_dim(fileId, "lat", 18L, &latid);
*/
@@ -297,9 +334,10 @@ ErrorCode NCWriteHelper::init_file(std::vector<std::string>& var_names)
}
// Define the variable now:
- if (NCFUNC(def_var)(_fileId, var_names[i].c_str(), variableData.varDataType,
+ int errCode = NCFUNC(def_var)(_fileId, var_names[i].c_str(), variableData.varDataType,
(int)variableData.varDims.size(), &(variableData.varDims[0]),
- &variableData.varId) != NC_NOERR)
+ &variableData.varId);
+ if ( errCode != NC_NOERR)
ERRORR(MB_FAILURE, "Failed to create coordinate variable.");
dbgOut.tprintf(2, " for variable %s variable id is %d \n", var_names[i].c_str(), variableData.varId);
diff --git a/src/io/NCWriteHelper.hpp b/src/io/NCWriteHelper.hpp
index b05fe12..ed2443a 100644
--- a/src/io/NCWriteHelper.hpp
+++ b/src/io/NCWriteHelper.hpp
@@ -34,7 +34,7 @@ public:
//! Initialize file: this is where all defines are done
//! The VarData dimension ids are filled up after define
- ErrorCode init_file(std::vector<std::string>& var_names);
+ ErrorCode init_file(std::vector<std::string>& var_names, bool _append);
protected:
template <typename T> void jik_to_kji(size_t ni, size_t nj, size_t nk, T* dest, T* source)
@@ -50,7 +50,7 @@ protected:
//! Allow NCWriteHelper to directly access members of WriteNC
WriteNC* _writeNC;
- //! Cache some information from ReadNC
+ //! Cache some information from WriteNC
int _fileId;
const FileOptions& _opts;
EntityHandle _fileSet;
diff --git a/src/io/WriteNC.cpp b/src/io/WriteNC.cpp
index e665e26..98f4e3b 100644
--- a/src/io/WriteNC.cpp
+++ b/src/io/WriteNC.cpp
@@ -33,7 +33,7 @@ WriteNC::WriteNC(Interface* impl) :
#ifdef USE_MPI
myPcomm(NULL),
#endif
- noMesh(false), noVars(false),
+ noMesh(false), noVars(false), append(false),
mGlobalIdTag(0), isParallel(false),
myHelper(NULL)
{
@@ -84,23 +84,42 @@ ErrorCode WriteNC::write_file(const char* file_name,
rval = process_conventional_tags(*file_set);
ERRORR(rval, "Trouble processing conventional tags.");
- // Create the file ; assume we will overwrite always, for the time being
- dbgOut.tprintf(1, "creating file %s\n", file_name);
+ // Create or append the file
+ if (append)
+ dbgOut.tprintf(1, "opening file %s for appending \n", file_name);
+ else
+ dbgOut.tprintf(1, "creating file %s\n", file_name);
fileName = file_name;
int success;
+ if (append)
+ {
#ifdef PNETCDF_FILE
- int cmode = overwrite ? NC_CLOBBER : NC_NOCLOBBER;
- if (isParallel)
- success = NCFUNC(create)(myPcomm->proc_config().proc_comm(), file_name, cmode, MPI_INFO_NULL, &fileId);
- else
- success = NCFUNC(create)(MPI_COMM_SELF, file_name, cmode, MPI_INFO_NULL, &fileId);
+ int omode = NC_WRITE;
+ if (isParallel)
+ success = NCFUNC(open)(myPcomm->proc_config().proc_comm(), file_name, omode, MPI_INFO_NULL, &fileId);
+ else
+ success = NCFUNC(open)(MPI_COMM_SELF, file_name, omode, MPI_INFO_NULL, &fileId);
#else
- // This is a regular netcdf file
- success = NCFUNC(create)(file_name, overwrite ? NC_CLOBBER : NC_NOCLOBBER, &fileId);
+ // This is a regular netcdf file
+ success = NCFUNC(open)(file_name, overwrite ? NC_CLOBBER : NC_NOCLOBBER, &fileId);
#endif
- ERRORS(success, "Failed to create file.");
-
+ ERRORS(success, "Failed to open file for appending.");
+ }
+ else // case when the file is new, will be overwritten, most likely
+ {
+#ifdef PNETCDF_FILE
+ int cmode = overwrite ? NC_CLOBBER : NC_NOCLOBBER;
+ if (isParallel)
+ success = NCFUNC(create)(myPcomm->proc_config().proc_comm(), file_name, cmode, MPI_INFO_NULL, &fileId);
+ else
+ success = NCFUNC(create)(MPI_COMM_SELF, file_name, cmode, MPI_INFO_NULL, &fileId);
+#else
+ // This is a regular netcdf file
+ success = NCFUNC(create)(file_name, overwrite ? NC_CLOBBER : NC_NOCLOBBER, &fileId);
+#endif
+ ERRORS(success, "Failed to create file.");
+ }
if (NULL != myHelper)
delete myHelper;
@@ -116,7 +135,7 @@ ErrorCode WriteNC::write_file(const char* file_name,
rval = myHelper->collect_variable_data(var_names);
ERRORR(rval, "Trouble collecting variable data.");
- rval = myHelper->init_file(var_names);
+ rval = myHelper->init_file(var_names, append);
ERRORR(rval, "Failed to initialize file.");
rval = myHelper->write_values(var_names);
@@ -148,6 +167,10 @@ ErrorCode WriteNC::parse_options(const FileOptions& opts, std::vector<std::strin
if (MB_SUCCESS == rval)
noMesh = true;
+ rval = opts.get_null_option("APPEND");
+ if (MB_SUCCESS == rval)
+ append = true;
+
if (2 <= dbgOut.get_verbosity()) {
if (!var_names.empty()) {
std::cerr << "Variables requested: ";
diff --git a/src/io/WriteNC.hpp b/src/io/WriteNC.hpp
index 1c5d78e..5827680 100644
--- a/src/io/WriteNC.hpp
+++ b/src/io/WriteNC.hpp
@@ -185,6 +185,7 @@ private:
//! Write options
bool noMesh;
bool noVars;
+ bool append;
//! Cached tags for writing. This will be important for ordering the data, in parallel
Tag mGlobalIdTag;
diff --git a/test/io/write_nc.cpp b/test/io/write_nc.cpp
index 2d9bdae..f48e4c6 100644
--- a/test/io/write_nc.cpp
+++ b/test/io/write_nc.cpp
@@ -36,6 +36,7 @@ static const char example_mpas[] = "/io/mpasx1.642.t.2.nc";
// CAM-EUL
void test_eul_read_write_T();
void test_eul_check_T();
+void test_eul_read_write_append();
// CAM-FV
void test_fv_read_write_T();
@@ -68,12 +69,12 @@ int main(int argc, char* argv[])
result += RUN_TEST(test_eul_read_write_T);
result += RUN_TEST(test_eul_check_T);
+ result += RUN_TEST(test_eul_read_write_append);
result += RUN_TEST(test_fv_read_write_T);
- result += RUN_TEST(test_fv_check_T);
result += RUN_TEST(test_homme_read_write_T);
result += RUN_TEST(test_homme_check_T);
- //result += RUN_TEST(test_mpas_read_write_vars);
- //result += RUN_TEST(test_mpas_check_vars);
+ // result += RUN_TEST(test_mpas_read_write_vars);
+ // result += RUN_TEST(test_mpas_check_vars);
#ifdef USE_MPI
fail = MPI_Finalize();
@@ -234,6 +235,61 @@ void test_eul_check_T()
}
}
}
+// We read and write variables T and U; U after we write T, so we append
+// we will also write gw, just to test the file after writing
+void test_eul_read_write_append()
+{
+ int procs = 1;
+#ifdef USE_MPI
+ MPI_Comm_size(MPI_COMM_WORLD, &procs);
+#endif
+
+// We will not test NC writer in parallel without pnetcdf support
+#ifndef PNETCDF_FILE
+ if (procs > 1)
+ return;
+#endif
+
+ Core moab;
+ Interface& mb = moab;
+
+ std::string read_opts;
+ get_eul_read_options(read_opts);
+
+ EntityHandle set;
+ ErrorCode rval = mb.create_meshset(MESHSET_SET, set);
+ CHECK_ERR(rval);
+
+ // Load non-set variable T, set variable gw, and the mesh
+ read_opts += ";DEBUG_IO=0;VARIABLE=T,U,gw";
+ rval = mb.load_file(example_eul, &set, read_opts.c_str());
+ CHECK_ERR(rval);
+
+ // Write variables T and gw
+ std::string write_opts;
+ write_opts = std::string(";;VARIABLE=T,gw;DEBUG_IO=0");
+#ifdef USE_MPI
+ // Use parallel options
+ write_opts += std::string(";PARALLEL=WRITE_PART");
+#endif
+ if (procs > 1)
+ rval = mb.write_file("test_par_eul_TU.nc", 0, write_opts.c_str(), &set, 1);
+ else
+ rval = mb.write_file("test_eul_TU.nc", 0, write_opts.c_str(), &set, 1);
+ CHECK_ERR(rval);
+ // append to the file variable U
+ std::string write_opts2;
+ write_opts2 = std::string(";;VARIABLE=U;DEBUG_IO=0;APPEND");
+#ifdef USE_MPI
+ // Use parallel options
+ write_opts2 += std::string(";PARALLEL=WRITE_PART");
+#endif
+ if (procs > 1)
+ rval = mb.write_file("test_par_eul_TU.nc", 0, write_opts2.c_str(), &set, 1);
+ else
+ rval = mb.write_file("test_eul_TU.nc", 0, write_opts2.c_str(), &set, 1);
+ CHECK_ERR(rval);
+}
// We also write coordinate variables slat and slon to the output file, so that
// it can be recognized by NC reader later in test_fv_check_T()
Repository URL: https://bitbucket.org/fathomteam/moab/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the moab-dev
mailing list