[MOAB-dev] commit/MOAB: danwu: Updated NC reader and writer to support reading variables whose timesteps spread across files and writing them to a file.

Robert Jacob jacob at mcs.anl.gov
Thu May 15 13:37:19 CDT 2014


Does this mean we can now read a multi-file timeseries entirely in to 
memory?

Rob


On 5/15/14 12:57 PM, commits-noreply at bitbucket.org wrote:
> 1 new commit in MOAB:
>
> https://bitbucket.org/fathomteam/moab/commits/a5907807ffc2/
> Changeset:   a5907807ffc2
> Branch:      master
> User:        danwu
> Date:        2014-05-15 19:57:18
> Summary:     Updated NC reader and writer to support reading variables whose timesteps spread across files and writing them to a file.
>
> Affected #:  5 files
>
> diff --git a/src/io/NCHelper.cpp b/src/io/NCHelper.cpp
> index 59d6df4..ccca80c 100644
> --- a/src/io/NCHelper.cpp
> +++ b/src/io/NCHelper.cpp
> @@ -61,7 +61,8 @@ NCHelper* NCHelper::get_nc_helper(ReadNC* readNC, int fileId, const FileOptions&
>     return NULL;
>   }
>
> -ErrorCode NCHelper::create_conventional_tags(const std::vector<int>& tstep_nums) {
> +ErrorCode NCHelper::create_conventional_tags(const std::vector<int>& tstep_nums)
> +{
>     Interface*& mbImpl = _readNC->mbImpl;
>     std::vector<std::string>& dimNames = _readNC->dimNames;
>     std::vector<int>& dimLens = _readNC->dimLens;
> @@ -327,6 +328,58 @@ ErrorCode NCHelper::create_conventional_tags(const std::vector<int>& tstep_nums)
>     return MB_SUCCESS;
>   }
>
> +ErrorCode NCHelper::update_time_tag_vals()
> +{
> +  Interface*& mbImpl = _readNC->mbImpl;
> +  std::vector<std::string>& dimNames = _readNC->dimNames;
> +
> +  ErrorCode rval;
> +
> +  // The time tag might be a dummy one (e.g. 'Time' for MPAS)
> +  std::string time_tag_name = dimNames[tDim];
> +  if (dummyVarNames.find(time_tag_name) != dummyVarNames.end())
> +    return MB_SUCCESS;
> +
> +  Tag time_tag = 0;
> +  const void* data = NULL;
> +  int time_tag_size = 0;
> +  rval = mbImpl->tag_get_handle(time_tag_name.c_str(), 0, MB_TYPE_DOUBLE, time_tag, MB_TAG_VARLEN);
> +  ERRORR(rval, "Trouble getting time tag.");
> +  rval = mbImpl->tag_get_by_ptr(time_tag, &_fileSet, 1, &data, &time_tag_size);
> +  ERRORR(rval, "Trouble getting values for time tag.");
> +  const double* time_tag_vals = static_cast<const double*>(data);
> +
> +  // Merge tVals (read from current file) to existing time tag
> +  // Assume that time_tag_vals and tVals are both sorted
> +  std::vector<double> merged_time_vals;
> +  merged_time_vals.reserve(time_tag_size + nTimeSteps);
> +  int i = 0;
> +  int j = 0;
> +
> +  // Merge time values from time_tag_vals and tVals
> +  while (i < time_tag_size && j < nTimeSteps) {
> +    if (time_tag_vals[i] < tVals[j])
> +      merged_time_vals.push_back(time_tag_vals[i++]);
> +    else
> +      merged_time_vals.push_back(tVals[j++]);
> +  }
> +
> +  // Append remaining time values of time_tag_vals (if any)
> +  while (i < time_tag_size)
> +    merged_time_vals.push_back(time_tag_vals[i++]);
> +
> +  // Append remaining time values of tVals (if any)
> +  while (j < nTimeSteps)
> +    merged_time_vals.push_back(tVals[j++]);
> +
> +  data = &merged_time_vals[0];
> +  time_tag_size = merged_time_vals.size();
> +  rval = mbImpl->tag_set_by_ptr(time_tag, &_fileSet, 1, &data, &time_tag_size);
> +  ERRORR(rval, "Failed to set data for time tag.");
> +
> +  return MB_SUCCESS;
> +}
> +
>   ErrorCode NCHelper::read_variable_setup(std::vector<std::string>& var_names, std::vector<int>& tstep_nums,
>                                           std::vector<ReadNC::VarData>& vdatas, std::vector<ReadNC::VarData>& vsetdatas)
>   {
> @@ -636,6 +689,10 @@ ErrorCode NCHelper::get_tag_to_set(ReadNC::VarData& var_data, int tstep_num, Tag
>   {
>     Interface*& mbImpl = _readNC->mbImpl;
>     DebugOutput& dbgOut = _readNC->dbgOut;
> +  int& tStepBase = _readNC->tStepBase;
> +
> +  if (tStepBase > 0)
> +    tstep_num += tStepBase;
>
>     std::ostringstream tag_name;
>     if (var_data.has_tsteps)
> @@ -673,6 +730,10 @@ ErrorCode NCHelper::get_tag_to_nonset(ReadNC::VarData& var_data, int tstep_num,
>   {
>     Interface*& mbImpl = _readNC->mbImpl;
>     DebugOutput& dbgOut = _readNC->dbgOut;
> +  int& tStepBase = _readNC->tStepBase;
> +
> +  if (tStepBase > 0)
> +    tstep_num += tStepBase;
>
>     std::ostringstream tag_name;
>     tag_name << var_data.varName << tstep_num;
>
> diff --git a/src/io/NCHelper.hpp b/src/io/NCHelper.hpp
> index 817257c..d134249 100644
> --- a/src/io/NCHelper.hpp
> +++ b/src/io/NCHelper.hpp
> @@ -35,6 +35,9 @@ public:
>     //! Create NC conventional tags
>     ErrorCode create_conventional_tags(const std::vector<int>& tstep_nums);
>
> +  //! Update time tag values if timesteps spread across files
> +  ErrorCode update_time_tag_vals();
> +
>   protected:
>     //! Separate set and non-set variables (common to scd mesh and ucd mesh)
>     ErrorCode read_variable_setup(std::vector<std::string>& var_names,
>
> diff --git a/src/io/NCWriteHelper.cpp b/src/io/NCWriteHelper.cpp
> index 635b595..028bce4 100644
> --- a/src/io/NCWriteHelper.cpp
> +++ b/src/io/NCWriteHelper.cpp
> @@ -224,12 +224,19 @@ ErrorCode NCWriteHelper::collect_variable_data(std::vector<std::string>& var_nam
>       }
>       else {
>         // The number of coordinates should be exactly the same as dimension length
> -      assert(sizeCoordinate == coordDimLen);
> +      // However, if timesteps spread across files and time tag has been updated,
> +      // sizeCoordinate will be larger
> +      if (varCoordData.varDims[0] != tDim)
> +        assert(sizeCoordinate == coordDimLen);
>       }
>
>       // For time, the actual output size and values are determined by tstep_nums
>       if (varCoordData.varDims[0] == tDim) {
> -      assert(tstep_nums.size() > 0 && tstep_nums.size() <= (size_t)sizeCoordinate);
> +      // Does not apply to dummy time tag (e.g. 'Time' tag of MPAS), when timesteps
> +      // spread across files
> +      if (NULL != data)
> +        assert(tstep_nums.size() > 0 && tstep_nums.size() <= (size_t)sizeCoordinate);
> +
>         sizeCoordinate = tstep_nums.size();
>
>         if (NULL != data) {
>
> diff --git a/src/io/ReadNC.cpp b/src/io/ReadNC.cpp
> index 0a41cfa..98bf6b3 100644
> --- a/src/io/ReadNC.cpp
> +++ b/src/io/ReadNC.cpp
> @@ -25,7 +25,7 @@ ReadNC::ReadNC(Interface* impl) :
>     myPcomm(NULL),
>   #endif
>     noMesh(false), noVars(false), spectralMesh(false), noMixedElements(false), noEdges(false),
> -  gatherSetRank(-1), myHelper(NULL)
> +  gatherSetRank(-1), tStepBase(-1), myHelper(NULL)
>   {
>     assert(impl != NULL);
>     impl->query_interface(readMeshIface);
> @@ -119,7 +119,9 @@ ErrorCode ReadNC::load_file(const char* file_name, const EntityHandle* file_set,
>     }
>
>     // Create some conventional tags, e.g. __NUM_DIMS
> -  // Keep a flag on the file set to prevent conventional tags from being created again on a second read
> +  // For multiple reads to a specified file set, we assume a single file, or a series of
> +  // files with separated timesteps. Keep a flag on the file set to prevent conventional
> +  // tags from being created again on a second read
>     Tag convTagsCreated = 0;
>     int def_val = 0;
>     rval = mbImpl->tag_get_handle("__CONV_TAGS_CREATED", 1, MB_TYPE_INTEGER, convTagsCreated,
> @@ -127,6 +129,7 @@ ErrorCode ReadNC::load_file(const char* file_name, const EntityHandle* file_set,
>     ERRORR(rval, "Trouble getting _CONV_TAGS_CREATED tag.");
>     int create_conv_tags_flag = 0;
>     rval = mbImpl->tag_get_data(convTagsCreated, &tmp_set, 1, &create_conv_tags_flag);
> +  // The first read to the file set
>     if (0 == create_conv_tags_flag) {
>       // Read dimension variables to create tags like __<var_name>_DIMS
>       rval = myHelper->read_variables(dimNames, tstep_nums);
> @@ -139,6 +142,15 @@ ErrorCode ReadNC::load_file(const char* file_name, const EntityHandle* file_set,
>       rval = mbImpl->tag_set_data(convTagsCreated, &tmp_set, 1, &create_conv_tags_flag);
>       ERRORR(rval, "Trouble setting data for _CONV_TAGS_CREATED tag.");
>     }
> +  // Another read to the file set
> +  else {
> +    if (tStepBase > -1) {
> +      // If timesteps spread across files, merge time values read
> +      // from current file to existing time tag
> +      rval = myHelper->update_time_tag_vals();
> +      ERRORR(rval, "Trouble updating time tag values.");
> +    }
> +  }
>
>     // Create mesh vertex/edge/face sequences
>     Range faces;
> @@ -205,8 +217,10 @@ ErrorCode ReadNC::parse_options(const FileOptions& opts, std::vector<std::string
>       noVars = true;
>     else
>       noVars = false;
> +
>     opts.get_ints_option("TIMESTEP", tstep_nums);
>     opts.get_reals_option("TIMEVAL", tstep_vals);
> +
>     rval = opts.get_null_option("NOMESH");
>     if (MB_SUCCESS == rval)
>       noMesh = true;
> @@ -230,12 +244,14 @@ ErrorCode ReadNC::parse_options(const FileOptions& opts, std::vector<std::string
>           std::cerr << var_names[i];
>         std::cerr << std::endl;
>       }
> +
>       if (!tstep_nums.empty()) {
>         std::cerr << "Timesteps requested: ";
>         for (unsigned int i = 0; i < tstep_nums.size(); i++)
>           std::cerr << tstep_nums[i];
>         std::cerr << std::endl;
>       }
> +
>       if (!tstep_vals.empty()) {
>         std::cerr << "Time vals requested: ";
>         for (unsigned int i = 0; i < tstep_vals.size(); i++)
> @@ -250,6 +266,12 @@ ErrorCode ReadNC::parse_options(const FileOptions& opts, std::vector<std::string
>       return rval;
>     }
>
> +  rval = opts.get_int_option("TIMESTEPBASE", 0, tStepBase);
> +  if (MB_TYPE_OUT_OF_RANGE == rval) {
> +    readMeshIface->report_error("Invalid value for TIMESTEPBASE option");
> +    return rval;
> +  }
> +
>   #ifdef USE_MPI
>     isParallel = (opts.match_option("PARALLEL","READ_PART") != MB_ENTITY_NOT_FOUND);
>
>
> diff --git a/src/io/ReadNC.hpp b/src/io/ReadNC.hpp
> index 1a0ae91..1b81d15 100644
> --- a/src/io/ReadNC.hpp
> +++ b/src/io/ReadNC.hpp
> @@ -208,6 +208,7 @@ private:
>     bool noMixedElements;
>     bool noEdges;
>     int gatherSetRank;
> +  int tStepBase;
>
>     //! Helper class instance
>     NCHelper* myHelper;
>
> Repository URL: https://bitbucket.org/fathomteam/moab/
>
> --
>
> This is a commit notification from bitbucket.org. You are receiving
> this because you have the service enabled, addressing the recipient of
> this email.
>


More information about the moab-dev mailing list