[MOAB-dev] commit/MOAB: 2 new changesets
commits-noreply at bitbucket.org
commits-noreply at bitbucket.org
Sun Mar 30 10:36:58 CDT 2014
2 new commits in MOAB:
https://bitbucket.org/fathomteam/moab/commits/9bfb3fe9c697/
Changeset: 9bfb3fe9c697
Branch: None
User: iulian07
Date: 2014-03-30 03:24:41
Summary: process attributes for each variable and global attributes
these are saved when the original file is read with nomesh, novars option
the attributes are concatenated, I saved them all as char
do we really need them
AttData is different compared to ReadNC
why save the name again?
well, not sure anybody needs them yet
Affected #: 2 files
diff --git a/src/io/WriteNC.cpp b/src/io/WriteNC.cpp
index a65548a..5af6fdf 100644
--- a/src/io/WriteNC.cpp
+++ b/src/io/WriteNC.cpp
@@ -13,6 +13,7 @@
#include <set>
#include <iostream>
+#include <sstream>
#define ERRORR(rval, str) \
if (MB_SUCCESS != rval) { mWriteIface->report_error("%s", str); return rval; }
@@ -365,6 +366,7 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet)
ERRORR(rval, " size of dimensions for variable");
dbgOut.tprintf(2, "var name: %s has %d dimensions \n", var_name.c_str(), sz);
+ variableDataStruct.varDims.resize(sz);
//std::vector<const pcdim*> dims(sz, NULL);
const void* ptr = NULL;
rval = mbImpl->tag_get_by_ptr(dims_tag, &fileSet, 1, &ptr);
@@ -376,123 +378,124 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet)
rval = mbImpl->tag_get_name(ptags[j], dim_name);
ERRORR(rval, "name of tag for dimension");
dbgOut.tprintf(2, "var name: %s has %s as dimension \n", var_name.c_str(), dim_name.c_str() );
+ std::vector<std::string>::iterator vit=std::find(dimNames.begin(), dimNames.end(), dim_name);
+ if (vit==dimNames.end())
+ ERRORR(MB_FAILURE, "dimension not found\n");
+ variableDataStruct.varDims[j]=(int)(vit-dimNames.begin()); // will be used for writing
+ // do we have a variable for each dimension? I mean, a tag?
//dims[j] = &(get_dim(dim_name));
}
- /*insert(var_name,
- *(new fvar(var_name, moab::MB_TYPE_DOUBLE, dims,
- locmap[varLoc[nthVar++]])));*/
+
+ // attributes for this variable
+ std::stringstream ssTagName;
+ ssTagName << "__" << var_name << "_ATTRIBS";
+ tag_name = ssTagName.str();
+ Tag varAttTag = 0;
+ rval = mbImpl->tag_get_handle(tag_name.c_str(), 0, MB_TYPE_OPAQUE, varAttTag, MB_TAG_SPARSE | MB_TAG_VARLEN);
+ ERRORR(rval, "Trouble getting __<var_name>_ATTRIBS tag.");
+ std::string varAttVal;
+ std::vector<int> varAttLen;
+ const void* varAttPtr = 0;
+ int varAttSz = 0;
+ rval = mbImpl->tag_get_by_ptr(varAttTag, &fileSet, 1, &varAttPtr, &varAttSz);
+ ERRORR(rval, "Trouble setting data for __<var_name>_ATTRIBS tag.");
+ if (MB_SUCCESS == rval)
+ dbgOut.tprintf(2, "Tag retrieved for variable %s\n", tag_name.c_str());
+
+ ssTagName << "_LEN";
+ tag_name = ssTagName.str();
+ Tag varAttLenTag = 0;
+ rval = mbImpl->tag_get_handle(tag_name.c_str(), 0, MB_TYPE_INTEGER, varAttLenTag, MB_TAG_ANY);
+ ERRORR(rval, "Trouble getting __<var_name>_ATTRIBS_LEN tag.");
+ int varAttLenSz=0;
+ rval = mbImpl->tag_get_length(varAttLenTag, varAttLenSz);
+ ERRORR(rval, "Trouble getting __<var_name>_ATTRIBS_LEN length.");
+ varAttLen.resize(varAttLenSz);
+
+ rval = mbImpl->tag_get_data(varAttLenTag, &fileSet, 1, &varAttLen[0]);
+ ERRORR(rval, "Trouble getting data for __<var_name>_ATTRIBS_LEN tag.");
+
+ rval = process_concatenated_attribute(varAttPtr, varAttSz, varAttLen, variableDataStruct.varAtts);
+ ERRORR(rval, " trouble processing global attributes ");
+
+ if (MB_SUCCESS == rval)
+ dbgOut.tprintf(2, "Tag metadata for variable %s\n", tag_name.c_str());
+ // end attribute
start = i + 1;
idxVar++;
}
}
// attributes
-#if 0
- std::vector<std::string> nameVec(num_vars+1);
- nameVec[0] = "GLOBAL";
- std::map<std::string, const fvar*>::iterator it = m_vars.begin();
- for (std::size_t i=1; it != m_vars.end(); ++it, ++i)
- nameVec[i] = it->first;
-
- for (std::size_t vec_counter = 0; vec_counter != nameVec.size(); ++vec_counter)
- {
- // read __<var_name>_ATTRIBS tag
- moab::Tag tag = 0;
- std::string tag_name = "__"+nameVec[vec_counter]+"_ATTRIBS";
- const void * data = NULL;
- int sz = 0;
- moab::ErrorCode rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_OPAQUE, tag, moab::MB_TAG_ANY);
- if (rval != moab::MB_SUCCESS)
- throw pargal_except("Error: " + m_mb.get_error_string(rval),
- __FILE__, __LINE__, __PRETTY_FUNCTION__);
-
- rval = m_mb.tag_get_by_ptr(tag, &m_file_set, 1, &data, &sz);
- const char * p = static_cast<const char *>(data);
- std::string att_val(&p[0], sz);
- if (vec_counter == 0) nameVec[0]="MOAB_GLOBAL";
- const std::string& var_name = nameVec[vec_counter];
-
- // read __<var_name>_ATTRIBS_LEN tag
- moab::Tag attLenTag = 0;
- tag_name = tag_name + "_LEN";
- const void * len_data = NULL;
- int len_sz = 0;
- rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_INTEGER, attLenTag, moab::MB_TAG_ANY);
- rval = m_mb.tag_get_by_ptr(attLenTag, &m_file_set, 1, &len_data, &len_sz);
- const int * len_p = static_cast<const int *>(len_data);
- std::vector<int> attLen(len_sz);
- std::copy(len_p, len_p+len_sz, attLen.begin());
-
- // create attribute
- insert(var_name, *(new pcatt(var_name, att_val, attLen)));
-#endif
+ tag_name = "__GLOBAL_ATTRIBS";
+ Tag globalAttTag = 0;
+ rval = mbImpl->tag_get_handle(tag_name.c_str(), 0, MB_TYPE_OPAQUE, globalAttTag, MB_TAG_SPARSE | MB_TAG_VARLEN);
+ ERRORR(rval, "Trouble getting __GLOBAL_ATTRIBS tag.");
+ std::string gattVal;
+ std::vector<int> gattLen;
+
+ const void* gattptr;
+ int globalAttSz=0;
+ rval = mbImpl->tag_get_by_ptr(globalAttTag, &fileSet, 1, &gattptr, &globalAttSz);
+ ERRORR(rval, "Trouble getting data for __GLOBAL_ATTRIBS tag.");
+
+ if (MB_SUCCESS == rval)
+ dbgOut.tprintf(2, "Tag value retrieved for %s size %d\n", tag_name.c_str(), globalAttSz);
+
+ // <__GLOBAL_ATTRIBS_LEN>
+ tag_name = "__GLOBAL_ATTRIBS_LEN";
+ Tag globalAttLenTag = 0;
+
+ rval = mbImpl->tag_get_handle(tag_name.c_str(), 0, MB_TYPE_INTEGER, globalAttLenTag, MB_TAG_ANY);
+ ERRORR(rval, "Trouble getting __GLOBAL_ATTRIBS_LEN tag.");
+ int sizeGAtt =0;
+ rval = mbImpl->tag_get_length(globalAttLenTag, sizeGAtt);
+ ERRORR(rval, "Trouble getting length of __GLOBAL_ATTRIBS_LEN tag ");
+ gattLen.resize(sizeGAtt);
+ rval = mbImpl->tag_get_data(globalAttLenTag, &fileSet, 1, &gattLen[0]);
+ ERRORR(rval, "Trouble setting data for __GLOBAL_ATTRIBS_LEN tag.");
+ if (MB_SUCCESS == rval)
+ dbgOut.tprintf(2, "Tag retrieved for variable %s\n", tag_name.c_str());
+
+ rval = process_concatenated_attribute(gattptr, globalAttSz, gattLen, globalAtts);
+ ERRORR(rval, " trouble processing global attributes ");
+
return MB_SUCCESS;
}
-#if 0
+ErrorCode WriteNC::process_concatenated_attribute(const void * gattptr, int globalAttSz, std::vector<int> & gattLen,
+ std::map<std::string, AttData> & attributes)
+{
- void
- fileinfo::init_atts()
- {
- std::vector<std::string> nameVec(m_vars.size()+1);
- nameVec[0] = "GLOBAL";
- std::map<std::string, const fvar*>::iterator it = m_vars.begin();
- for (std::size_t i=1; it != m_vars.end(); ++it, ++i)
- nameVec[i] = it->first;
-
- for (std::size_t vec_counter = 0; vec_counter != nameVec.size(); ++vec_counter)
- {
- // read __<var_name>_ATTRIBS tag
- moab::Tag tag = 0;
- std::string tag_name = "__"+nameVec[vec_counter]+"_ATTRIBS";
- const void * data = NULL;
- int sz = 0;
- moab::ErrorCode rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_OPAQUE, tag, moab::MB_TAG_ANY);
- if (rval != moab::MB_SUCCESS)
- throw pargal_except("Error: " + m_mb.get_error_string(rval),
- __FILE__, __LINE__, __PRETTY_FUNCTION__);
-
- rval = m_mb.tag_get_by_ptr(tag, &m_file_set, 1, &data, &sz);
- const char * p = static_cast<const char *>(data);
- std::string att_val(&p[0], sz);
- if (vec_counter == 0) nameVec[0]="MOAB_GLOBAL";
- const std::string& var_name = nameVec[vec_counter];
-
- // read __<var_name>_ATTRIBS_LEN tag
- moab::Tag attLenTag = 0;
- tag_name = tag_name + "_LEN";
- const void * len_data = NULL;
- int len_sz = 0;
- rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_INTEGER, attLenTag, moab::MB_TAG_ANY);
- rval = m_mb.tag_get_by_ptr(attLenTag, &m_file_set, 1, &len_data, &len_sz);
- const int * len_p = static_cast<const int *>(len_data);
- std::vector<int> attLen(len_sz);
- std::copy(len_p, len_p+len_sz, attLen.begin());
-
- // create attribute
- insert(var_name, *(new pcatt(var_name, att_val, attLen)));
- }
- }
+ std::size_t start = 0;
+ std::size_t att_counter = 0;
+ std::string concatString( (char*)gattptr, (char*)gattptr+globalAttSz);
- void
- fileinfo::init_grid_type()
+ for (std::size_t i = 0; i != (size_t)globalAttSz; ++i)
+ {
+ if (concatString[i] == '\0')
{
- moab::Tag tag = 0;
- std::string tag_name = "__MESH_TYPE";
- const void * data = NULL;
- int sz = 0;
- moab::ErrorCode rval;
- rval = m_mb.tag_get_handle(tag_name.c_str(), 0, moab::MB_TYPE_OPAQUE, tag, moab::MB_TAG_ANY);
- rval = m_mb.tag_get_by_ptr(tag, &m_file_set, 1, &data, &sz);
- if (rval != moab::MB_SUCCESS)
- throw pargal_except("Error: " + m_mb.get_error_string(rval),
- __FILE__, __LINE__, __PRETTY_FUNCTION__);
- const char * p = static_cast<const char *>(data);
- m_grid_type = std::string(&p[0], sz);
+ std::string att_name(&concatString[start], i - start);
+ start = i + 1;
+ while (concatString[i] != ';')
+ ++i;
+ std::string data_type(&concatString[start], i - start);
+ ++i;
+ start = i;
+ i = gattLen[att_counter];
+ if (concatString[i] != ';')
+ ERRORR(MB_FAILURE, "Error parsing attributes ");
+
+ std::string data_val(&concatString[start], i - start);
+ start = i + 1;
+ AttData attrib = attributes[att_name];
+ attrib.attValue = data_val;
+ ++att_counter;
+ dbgOut.tprintf(2, " Process attribute %s with value %s \n",att_name.c_str(), data_val.c_str() );
}
+ }
return MB_SUCCESS;
}
-#endif
} // end moab namespace
diff --git a/src/io/WriteNC.hpp b/src/io/WriteNC.hpp
index c3b893a..6b75d59 100644
--- a/src/io/WriteNC.hpp
+++ b/src/io/WriteNC.hpp
@@ -110,7 +110,7 @@ private:
NCDF_SIZE attLen;
int attVarId;
nc_type attDataType;
- std::string attName;
+ std::string attValue;
};
class VarData
@@ -152,6 +152,9 @@ private:
* ErrorCode NCHelper::create_conventional_tags
*/
ErrorCode process_conventional_tags(EntityHandle fileSet);
+
+ ErrorCode process_concatenated_attribute(const void * gattptr, int globalAttSz, std::vector<int> & gattLen,
+ std::map<std::string, AttData> & globalAtts);
// interface instance
Interface *mbImpl;
WriteUtilIface* mWriteIface;
https://bitbucket.org/fathomteam/moab/commits/e64001b99718/
Changeset: e64001b99718
Branch: ncwriter
User: iulian07
Date: 2014-03-30 17:23:36
Summary: start collecting data
will have to populate VarData with appropriate info;
start gathering tag info for nonset variables
"set" variables are coordinate variables (dimensions), usually
time is integer (time steps, or time values?)
Affected #: 2 files
diff --git a/src/io/WriteNC.cpp b/src/io/WriteNC.cpp
index 5af6fdf..05fcc3f 100644
--- a/src/io/WriteNC.cpp
+++ b/src/io/WriteNC.cpp
@@ -122,6 +122,9 @@ ErrorCode WriteNC::write_file(const char *file_name,
const size_t count[], const double *dp);
*/
+ //
+ rval = collect_variable_data(var_names, tstep_nums, tstep_vals, *file_set);
+ ERRORR(rval, "Trouble collecting data.");
return MB_SUCCESS;
}
@@ -348,7 +351,11 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet)
dbgOut.tprintf(2, "var name: %s index %d \n", var_name.c_str(), idxVar);
// process var name:
// this will create/initiate map; we will populate variableDataStruct wit info about dims, tags, etc
- VarData variableDataStruct = varInfo[var_name];
+ // reference & is important; otherwise variableDataStruct will go out of scope, and deleted :(
+ VarData & variableDataStruct = varInfo[var_name];
+
+ dbgOut.tprintf(2, "at var name %s varInfo size %d \n", var_name.c_str(), varInfo.size() );
+
sz = 0;
Tag dims_tag = 0;
std::string dim_names = "__" + var_name + "_DIMS";
@@ -371,6 +378,7 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet)
const void* ptr = NULL;
rval = mbImpl->tag_get_by_ptr(dims_tag, &fileSet, 1, &ptr);
//
+
const Tag * ptags = static_cast<const moab::Tag*>(ptr);
for (std::size_t j = 0; j != static_cast<std::size_t>(sz); ++j)
{
@@ -381,7 +389,8 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet)
std::vector<std::string>::iterator vit=std::find(dimNames.begin(), dimNames.end(), dim_name);
if (vit==dimNames.end())
ERRORR(MB_FAILURE, "dimension not found\n");
- variableDataStruct.varDims[j]=(int)(vit-dimNames.begin()); // will be used for writing
+ variableDataStruct.varDims[j]= (int)(vit-dimNames.begin()) ; // will be used for writing
+
// do we have a variable for each dimension? I mean, a tag?
//dims[j] = &(get_dim(dim_name));
}
@@ -462,7 +471,7 @@ ErrorCode WriteNC::process_conventional_tags(EntityHandle fileSet)
return MB_SUCCESS;
}
-
+// reverse process from create_attrib_string
ErrorCode WriteNC::process_concatenated_attribute(const void * gattptr, int globalAttSz, std::vector<int> & gattLen,
std::map<std::string, AttData> & attributes)
{
@@ -498,4 +507,106 @@ ErrorCode WriteNC::process_concatenated_attribute(const void * gattptr, int glob
return MB_SUCCESS;
}
+ErrorCode WriteNC::collect_variable_data( std::vector<std::string>& var_names, std::vector<int>& tstep_nums,
+ std::vector<double>& tstep_vals, EntityHandle fileSet)
+{
+ // in general, in netcdf, variables that have the same name as their only dimension are called
+ // coordinate variables
+ // for the time being, check if all dimensions for variables are coordinate variables
+ ErrorCode rval;
+
+ usedCoordinates.clear();
+
+ for (size_t i=0; i<var_names.size(); i++)
+ {
+ std::string varname=var_names[i];
+ std::map<std::string, VarData>::iterator vit = varInfo.find(varname);
+ if (vit==varInfo.end())
+ ERRORR(MB_FAILURE, "can't find one variable");
+
+ size_t sizeVar = 1;// get multiplied by dim lengths
+ VarData & currentVarData = vit->second;
+ dbgOut.tprintf(2, " for variable %s varDims.size %d \n", varname.c_str(), (int)currentVarData.varDims.size() );
+ for (size_t j =0; j<currentVarData.varDims.size(); j++)
+ {
+ std::string dimName= dimNames[ currentVarData.varDims[j] ];
+ vit = varInfo.find(dimName);
+ if (vit==varInfo.end())
+ ERRORR(MB_FAILURE, "can't find one coordinate variable");
+
+ if( (dimName == "time" || dimName == "Time" || dimName == "t") &&
+ currentVarData.varDims.size()>1 ) // so it is not time itself
+ currentVarData.has_tsteps=true;
+
+ // probably will have to look at tstep_vals to match them
+ sizeVar *= dimLens[j];
+ usedCoordinates.insert(dimName); // collect those used, we will need to write them to the file
+ dbgOut.tprintf(2, " for variable %s need dimension %s with length %d\n", varname.c_str(), dimName.c_str(), dimLens[j] );
+ }
+
+ currentVarData.sz=sizeVar;
+
+ if (currentVarData.has_tsteps)
+ {
+
+ int index=0;
+ while(1)
+ {
+ Tag indexedTag;
+ std::stringstream ssTagNameWithIndex;
+ ssTagNameWithIndex << varname << index;
+ rval = mbImpl->tag_get_handle(ssTagNameWithIndex.str().c_str(), indexedTag);
+ if (rval!=MB_SUCCESS)
+ break;
+ dbgOut.tprintf(2, " found indexed tag %d with name %s\n", index, ssTagNameWithIndex.str().c_str());
+ currentVarData.varTags.push_back(indexedTag);
+ index++; // we should get out of the loop at some point
+ // we will have to collect data for these tags; maybe even allocate memory again
+ }
+ }
+ else
+ {
+ // get the tag with varname
+ Tag coordtag=0;
+ rval = mbImpl->tag_get_handle(varname.c_str(), coordtag);
+ ERRORR(rval, "can't find one tag");
+ currentVarData.varTags.push_back(coordtag); // really, only one for these
+ const void * data;
+ int sizeCoordinate;
+ rval = mbImpl->tag_get_by_ptr(coordtag, &fileSet, 1, &data, &sizeCoordinate);
+ ERRORR(rval, "can't get coordinate values");
+ assert(currentVarData.memoryHogs.size()==0);// nothing so far
+ currentVarData.memoryHogs.push_back((void*)data);
+ }
+
+ }
+
+ // check that for used coordinates we have found the tags
+ for (std::set<std::string>::iterator setIt = usedCoordinates.begin(); setIt!=usedCoordinates.end(); setIt++)
+ {
+ std::string coordName=*setIt; // a deep copy ; is it needed?
+
+ std::map<std::string, VarData>::iterator vit = varInfo.find(coordName);
+ if (vit==varInfo.end())
+ ERRORR(MB_FAILURE, "can't find one coordinate variable");
+
+ VarData & varCoordData = vit->second;
+ Tag coordtag=0;
+ rval = mbImpl->tag_get_handle(coordName.c_str(), coordtag);
+ ERRORR(rval, "can't find one tag");
+ varCoordData.varTags.push_back(coordtag); // really, only one for these
+
+ const void * data;
+ int sizeCoordinate;
+ rval = mbImpl->tag_get_by_ptr(coordtag, &fileSet, 1, &data, &sizeCoordinate);
+ ERRORR(rval, "can't get coordinate values");
+ dbgOut.tprintf(2, " found coordinate tag with name %s and length %d\n", coordName.c_str(),
+ sizeCoordinate);
+
+ assert(varCoordData.memoryHogs.size()==0);// nothing so far
+ varCoordData.memoryHogs.push_back((void*)data);
+ }
+ return MB_SUCCESS;
+}
+
} // end moab namespace
diff --git a/src/io/WriteNC.hpp b/src/io/WriteNC.hpp
index 6b75d59..df89af1 100644
--- a/src/io/WriteNC.hpp
+++ b/src/io/WriteNC.hpp
@@ -139,6 +139,9 @@ private:
//! Dimension lengths
std::vector<int> dimLens;
+ // will collect used dimensions (ccordinate variables)
+ std::set<std::string> usedCoordinates;
+
//! Global attribs
std::map<std::string, AttData> globalAtts;
@@ -155,6 +158,11 @@ private:
ErrorCode process_concatenated_attribute(const void * gattptr, int globalAttSz, std::vector<int> & gattLen,
std::map<std::string, AttData> & globalAtts);
+
+ // will collect data; it should be only on gather processor, but for the time being, collect
+ // for everybody
+ ErrorCode collect_variable_data( std::vector<std::string>& var_names, std::vector<int>& tstep_nums,
+ std::vector<double>& tstep_vals, EntityHandle fileSet);
// interface instance
Interface *mbImpl;
WriteUtilIface* mWriteIface;
Repository URL: https://bitbucket.org/fathomteam/moab/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the moab-dev
mailing list