[MOAB-dev] commit/MOAB: danwu: Updated NC reader and writer to support reading variables whose timesteps spread across files and writing them to a file.
Grindeanu, Iulian R.
iulian at mcs.anl.gov
Thu May 15 13:40:31 CDT 2014
that is the plan, Danqing is working at a test case; the code is in place, mostly, to read from multiple files (time series)
and write to one file.
Iulian
________________________________________
From: moab-dev-bounces at mcs.anl.gov [moab-dev-bounces at mcs.anl.gov] on behalf of Robert Jacob [jacob at mcs.anl.gov]
Sent: Thursday, May 15, 2014 1:37 PM
To: commits-noreply at bitbucket.org; moab-dev at mcs.anl.gov
Subject: Re: [MOAB-dev] commit/MOAB: danwu: Updated NC reader and writer to support reading variables whose timesteps spread across files and writing them to a file.
Does this mean we can now read a multi-file timeseries entirely in to
memory?
Rob
On 5/15/14 12:57 PM, commits-noreply at bitbucket.org wrote:
> 1 new commit in MOAB:
>
> https://bitbucket.org/fathomteam/moab/commits/a5907807ffc2/
> Changeset: a5907807ffc2
> Branch: master
> User: danwu
> Date: 2014-05-15 19:57:18
> Summary: Updated NC reader and writer to support reading variables whose timesteps spread across files and writing them to a file.
>
> Affected #: 5 files
>
> diff --git a/src/io/NCHelper.cpp b/src/io/NCHelper.cpp
> index 59d6df4..ccca80c 100644
> --- a/src/io/NCHelper.cpp
> +++ b/src/io/NCHelper.cpp
> @@ -61,7 +61,8 @@ NCHelper* NCHelper::get_nc_helper(ReadNC* readNC, int fileId, const FileOptions&
> return NULL;
> }
>
> -ErrorCode NCHelper::create_conventional_tags(const std::vector<int>& tstep_nums) {
> +ErrorCode NCHelper::create_conventional_tags(const std::vector<int>& tstep_nums)
> +{
> Interface*& mbImpl = _readNC->mbImpl;
> std::vector<std::string>& dimNames = _readNC->dimNames;
> std::vector<int>& dimLens = _readNC->dimLens;
> @@ -327,6 +328,58 @@ ErrorCode NCHelper::create_conventional_tags(const std::vector<int>& tstep_nums)
> return MB_SUCCESS;
> }
>
> +ErrorCode NCHelper::update_time_tag_vals()
> +{
> + Interface*& mbImpl = _readNC->mbImpl;
> + std::vector<std::string>& dimNames = _readNC->dimNames;
> +
> + ErrorCode rval;
> +
> + // The time tag might be a dummy one (e.g. 'Time' for MPAS)
> + std::string time_tag_name = dimNames[tDim];
> + if (dummyVarNames.find(time_tag_name) != dummyVarNames.end())
> + return MB_SUCCESS;
> +
> + Tag time_tag = 0;
> + const void* data = NULL;
> + int time_tag_size = 0;
> + rval = mbImpl->tag_get_handle(time_tag_name.c_str(), 0, MB_TYPE_DOUBLE, time_tag, MB_TAG_VARLEN);
> + ERRORR(rval, "Trouble getting time tag.");
> + rval = mbImpl->tag_get_by_ptr(time_tag, &_fileSet, 1, &data, &time_tag_size);
> + ERRORR(rval, "Trouble getting values for time tag.");
> + const double* time_tag_vals = static_cast<const double*>(data);
> +
> + // Merge tVals (read from current file) to existing time tag
> + // Assume that time_tag_vals and tVals are both sorted
> + std::vector<double> merged_time_vals;
> + merged_time_vals.reserve(time_tag_size + nTimeSteps);
> + int i = 0;
> + int j = 0;
> +
> + // Merge time values from time_tag_vals and tVals
> + while (i < time_tag_size && j < nTimeSteps) {
> + if (time_tag_vals[i] < tVals[j])
> + merged_time_vals.push_back(time_tag_vals[i++]);
> + else
> + merged_time_vals.push_back(tVals[j++]);
> + }
> +
> + // Append remaining time values of time_tag_vals (if any)
> + while (i < time_tag_size)
> + merged_time_vals.push_back(time_tag_vals[i++]);
> +
> + // Append remaining time values of tVals (if any)
> + while (j < nTimeSteps)
> + merged_time_vals.push_back(tVals[j++]);
> +
> + data = &merged_time_vals[0];
> + time_tag_size = merged_time_vals.size();
> + rval = mbImpl->tag_set_by_ptr(time_tag, &_fileSet, 1, &data, &time_tag_size);
> + ERRORR(rval, "Failed to set data for time tag.");
> +
> + return MB_SUCCESS;
> +}
> +
> ErrorCode NCHelper::read_variable_setup(std::vector<std::string>& var_names, std::vector<int>& tstep_nums,
> std::vector<ReadNC::VarData>& vdatas, std::vector<ReadNC::VarData>& vsetdatas)
> {
> @@ -636,6 +689,10 @@ ErrorCode NCHelper::get_tag_to_set(ReadNC::VarData& var_data, int tstep_num, Tag
> {
> Interface*& mbImpl = _readNC->mbImpl;
> DebugOutput& dbgOut = _readNC->dbgOut;
> + int& tStepBase = _readNC->tStepBase;
> +
> + if (tStepBase > 0)
> + tstep_num += tStepBase;
>
> std::ostringstream tag_name;
> if (var_data.has_tsteps)
> @@ -673,6 +730,10 @@ ErrorCode NCHelper::get_tag_to_nonset(ReadNC::VarData& var_data, int tstep_num,
> {
> Interface*& mbImpl = _readNC->mbImpl;
> DebugOutput& dbgOut = _readNC->dbgOut;
> + int& tStepBase = _readNC->tStepBase;
> +
> + if (tStepBase > 0)
> + tstep_num += tStepBase;
>
> std::ostringstream tag_name;
> tag_name << var_data.varName << tstep_num;
>
> diff --git a/src/io/NCHelper.hpp b/src/io/NCHelper.hpp
> index 817257c..d134249 100644
> --- a/src/io/NCHelper.hpp
> +++ b/src/io/NCHelper.hpp
> @@ -35,6 +35,9 @@ public:
> //! Create NC conventional tags
> ErrorCode create_conventional_tags(const std::vector<int>& tstep_nums);
>
> + //! Update time tag values if timesteps spread across files
> + ErrorCode update_time_tag_vals();
> +
> protected:
> //! Separate set and non-set variables (common to scd mesh and ucd mesh)
> ErrorCode read_variable_setup(std::vector<std::string>& var_names,
>
> diff --git a/src/io/NCWriteHelper.cpp b/src/io/NCWriteHelper.cpp
> index 635b595..028bce4 100644
> --- a/src/io/NCWriteHelper.cpp
> +++ b/src/io/NCWriteHelper.cpp
> @@ -224,12 +224,19 @@ ErrorCode NCWriteHelper::collect_variable_data(std::vector<std::string>& var_nam
> }
> else {
> // The number of coordinates should be exactly the same as dimension length
> - assert(sizeCoordinate == coordDimLen);
> + // However, if timesteps spread across files and time tag has been updated,
> + // sizeCoordinate will be larger
> + if (varCoordData.varDims[0] != tDim)
> + assert(sizeCoordinate == coordDimLen);
> }
>
> // For time, the actual output size and values are determined by tstep_nums
> if (varCoordData.varDims[0] == tDim) {
> - assert(tstep_nums.size() > 0 && tstep_nums.size() <= (size_t)sizeCoordinate);
> + // Does not apply to dummy time tag (e.g. 'Time' tag of MPAS), when timesteps
> + // spread across files
> + if (NULL != data)
> + assert(tstep_nums.size() > 0 && tstep_nums.size() <= (size_t)sizeCoordinate);
> +
> sizeCoordinate = tstep_nums.size();
>
> if (NULL != data) {
>
> diff --git a/src/io/ReadNC.cpp b/src/io/ReadNC.cpp
> index 0a41cfa..98bf6b3 100644
> --- a/src/io/ReadNC.cpp
> +++ b/src/io/ReadNC.cpp
> @@ -25,7 +25,7 @@ ReadNC::ReadNC(Interface* impl) :
> myPcomm(NULL),
> #endif
> noMesh(false), noVars(false), spectralMesh(false), noMixedElements(false), noEdges(false),
> - gatherSetRank(-1), myHelper(NULL)
> + gatherSetRank(-1), tStepBase(-1), myHelper(NULL)
> {
> assert(impl != NULL);
> impl->query_interface(readMeshIface);
> @@ -119,7 +119,9 @@ ErrorCode ReadNC::load_file(const char* file_name, const EntityHandle* file_set,
> }
>
> // Create some conventional tags, e.g. __NUM_DIMS
> - // Keep a flag on the file set to prevent conventional tags from being created again on a second read
> + // For multiple reads to a specified file set, we assume a single file, or a series of
> + // files with separated timesteps. Keep a flag on the file set to prevent conventional
> + // tags from being created again on a second read
> Tag convTagsCreated = 0;
> int def_val = 0;
> rval = mbImpl->tag_get_handle("__CONV_TAGS_CREATED", 1, MB_TYPE_INTEGER, convTagsCreated,
> @@ -127,6 +129,7 @@ ErrorCode ReadNC::load_file(const char* file_name, const EntityHandle* file_set,
> ERRORR(rval, "Trouble getting _CONV_TAGS_CREATED tag.");
> int create_conv_tags_flag = 0;
> rval = mbImpl->tag_get_data(convTagsCreated, &tmp_set, 1, &create_conv_tags_flag);
> + // The first read to the file set
> if (0 == create_conv_tags_flag) {
> // Read dimension variables to create tags like __<var_name>_DIMS
> rval = myHelper->read_variables(dimNames, tstep_nums);
> @@ -139,6 +142,15 @@ ErrorCode ReadNC::load_file(const char* file_name, const EntityHandle* file_set,
> rval = mbImpl->tag_set_data(convTagsCreated, &tmp_set, 1, &create_conv_tags_flag);
> ERRORR(rval, "Trouble setting data for _CONV_TAGS_CREATED tag.");
> }
> + // Another read to the file set
> + else {
> + if (tStepBase > -1) {
> + // If timesteps spread across files, merge time values read
> + // from current file to existing time tag
> + rval = myHelper->update_time_tag_vals();
> + ERRORR(rval, "Trouble updating time tag values.");
> + }
> + }
>
> // Create mesh vertex/edge/face sequences
> Range faces;
> @@ -205,8 +217,10 @@ ErrorCode ReadNC::parse_options(const FileOptions& opts, std::vector<std::string
> noVars = true;
> else
> noVars = false;
> +
> opts.get_ints_option("TIMESTEP", tstep_nums);
> opts.get_reals_option("TIMEVAL", tstep_vals);
> +
> rval = opts.get_null_option("NOMESH");
> if (MB_SUCCESS == rval)
> noMesh = true;
> @@ -230,12 +244,14 @@ ErrorCode ReadNC::parse_options(const FileOptions& opts, std::vector<std::string
> std::cerr << var_names[i];
> std::cerr << std::endl;
> }
> +
> if (!tstep_nums.empty()) {
> std::cerr << "Timesteps requested: ";
> for (unsigned int i = 0; i < tstep_nums.size(); i++)
> std::cerr << tstep_nums[i];
> std::cerr << std::endl;
> }
> +
> if (!tstep_vals.empty()) {
> std::cerr << "Time vals requested: ";
> for (unsigned int i = 0; i < tstep_vals.size(); i++)
> @@ -250,6 +266,12 @@ ErrorCode ReadNC::parse_options(const FileOptions& opts, std::vector<std::string
> return rval;
> }
>
> + rval = opts.get_int_option("TIMESTEPBASE", 0, tStepBase);
> + if (MB_TYPE_OUT_OF_RANGE == rval) {
> + readMeshIface->report_error("Invalid value for TIMESTEPBASE option");
> + return rval;
> + }
> +
> #ifdef USE_MPI
> isParallel = (opts.match_option("PARALLEL","READ_PART") != MB_ENTITY_NOT_FOUND);
>
>
> diff --git a/src/io/ReadNC.hpp b/src/io/ReadNC.hpp
> index 1a0ae91..1b81d15 100644
> --- a/src/io/ReadNC.hpp
> +++ b/src/io/ReadNC.hpp
> @@ -208,6 +208,7 @@ private:
> bool noMixedElements;
> bool noEdges;
> int gatherSetRank;
> + int tStepBase;
>
> //! Helper class instance
> NCHelper* myHelper;
>
> Repository URL: https://bitbucket.org/fathomteam/moab/
>
> --
>
> This is a commit notification from bitbucket.org. You are receiving
> this because you have the service enabled, addressing the recipient of
> this email.
>
More information about the moab-dev
mailing list