[MOAB-dev] r1651 - in MOAB/trunk: . parallel test/h5file
kraftche at mcs.anl.gov
kraftche at mcs.anl.gov
Thu Mar 13 15:34:46 CDT 2008
Author: kraftche
Date: 2008-03-13 15:34:46 -0500 (Thu, 13 Mar 2008)
New Revision: 1651
Added:
MOAB/trunk/test/h5file/varlen_ll.cpp
Modified:
MOAB/trunk/WriteHDF5.cpp
MOAB/trunk/WriteHDF5.hpp
MOAB/trunk/parallel/WriteHDF5Parallel.cpp
MOAB/trunk/test/h5file/Makefile.am
Log:
o Fix bug writing tags parallel
o Add support for writing variable-length tags in parallel
Modified: MOAB/trunk/WriteHDF5.cpp
===================================================================
--- MOAB/trunk/WriteHDF5.cpp 2008-03-13 19:52:36 UTC (rev 1650)
+++ MOAB/trunk/WriteHDF5.cpp 2008-03-13 20:34:46 UTC (rev 1651)
@@ -1949,7 +1949,14 @@
const std::list<SparseTag>::iterator tag_end = tagList.end();
for ( ; tag_iter != tag_end; ++tag_iter)
{
- rval = create_tag( *tag_iter );
+ int s;
+ unsigned long var_len_total;
+ if (MB_VARIABLE_DATA_LENGTH == iFace->tag_get_size( tag_iter->tag_id, s )) {
+ rval = get_tag_data_length( *tag_iter, var_len_total );
+ CHK_MB_ERR_0(rval);
+ }
+
+ rval = create_tag( tag_iter->tag_id, tag_iter->range.size(), var_len_total );
CHK_MB_ERR_0(rval);
} // for(tags)
@@ -2215,15 +2222,30 @@
return rval;
for (size_t i= 0; i < remaining; ++i)
result += size_buffer[i];
+
+ MBDataType type;
+ rval = iFace->tag_get_data_type( tag_info.tag_id, type );
+ if (MB_SUCCESS != rval)
+ return rval;
+ switch (type) {
+ case MB_TYPE_INTEGER: result /= sizeof(int); break;
+ case MB_TYPE_DOUBLE: result /= sizeof(double); break;
+ case MB_TYPE_HANDLE: result /= sizeof(MBEntityHandle); break;
+ case MB_TYPE_OPAQUE: break;
+ // We fail for MB_TYPE_BIT because MOAB currently does
+ // not support variable-length bit tags.
+ default: return MB_FAILURE;
+ }
+
return MB_SUCCESS;
}
-MBErrorCode WriteHDF5::create_tag( const SparseTag& tag_info )
+MBErrorCode WriteHDF5::create_tag( MBTag tag_id,
+ unsigned long num_sparse_entities,
+ unsigned long data_table_size )
{
- MBTag tag_id = tag_info.tag_id;
- unsigned long num_sparse_entities = tag_info.range.size();
MBTagType storage;
MBDataType mb_type;
mhdf_TagDataType mhdf_type;
@@ -2260,13 +2282,6 @@
}
else if (MB_SUCCESS != rval)
return rval;
-
- // for variable-length tags, need to calculate total data table size
- unsigned long data_table_size = 0;
- if (tag_size == MB_VARIABLE_LENGTH) {
- rval = get_tag_data_length( tag_info, data_table_size );
- CHK_MB_ERR_0(rval);
- }
// for handle-type tags, need to convert from handles to file ids
if (MB_TYPE_HANDLE == mb_type) {
@@ -2337,7 +2352,7 @@
mhdf_createVarLenTagData( filePtr,
tag_name.c_str(),
num_sparse_entities,
- data_table_size / elem_size,
+ data_table_size,
handles,
&status );
CHK_MHDF_ERR_0(status);
Modified: MOAB/trunk/WriteHDF5.hpp
===================================================================
--- MOAB/trunk/WriteHDF5.hpp 2008-03-13 19:52:36 UTC (rev 1650)
+++ MOAB/trunk/WriteHDF5.hpp 2008-03-13 20:34:46 UTC (rev 1651)
@@ -254,8 +254,13 @@
*
* Write tag meta-info and create zero-ed table where
* tag values will be written.
+ *\param num_entities Number of entities for which to write tag data.
+ *\param var_len_total For variable-length tags, the total number of values
+ * in the data table.
*/
- MBErrorCode create_tag( const SparseTag& tag_info );
+ MBErrorCode create_tag( MBTag tag_id,
+ unsigned long num_entities,
+ unsigned long var_len_total );
/** Get possibly compacted list of IDs for passed entities
*
@@ -285,6 +290,11 @@
* the ID to the passed list.
*/
MBErrorCode get_adjacencies( MBEntityHandle entity, std::vector<id_t>& adj );
+
+ //! get sum of lengths of tag values (as number of type) for
+ //! variable length tag data.
+ MBErrorCode get_tag_data_length( const SparseTag& tag_info,
+ unsigned long& result );
private:
@@ -359,11 +369,6 @@
MBRange::const_iterator end,
int nodes_per_element,
id_t* id_data_out );
-
- //! get sum of lengths of tag values (in bytes) for
- //! variable length tag data.
- MBErrorCode get_tag_data_length( const SparseTag& tag_info,
- unsigned long& result );
//! Get size data for tag
//!\param tag MOAB tag ID
Modified: MOAB/trunk/parallel/WriteHDF5Parallel.cpp
===================================================================
--- MOAB/trunk/parallel/WriteHDF5Parallel.cpp 2008-03-13 19:52:36 UTC (rev 1650)
+++ MOAB/trunk/parallel/WriteHDF5Parallel.cpp 2008-03-13 20:34:46 UTC (rev 1651)
@@ -1,5 +1,5 @@
-#undef DEBUG
+#define DEBUG
#ifdef DEBUG
# include <stdio.h>
@@ -93,7 +93,7 @@
int rank;
MPI_Comm_rank( MPI_COMM_WORLD, &rank );
MBEntityType type = MBMAXTYPE;
- for (MBRange::const_pair_iterator i = r.pair_begin(); i != r.pair_end(); ++i)
+ for (MBRange::const_pair_iterator i = r.const_pair_begin(); i != r.const_pair_end(); ++i)
{
MBEntityHandle a, b;
a = (*i).first;
@@ -128,9 +128,9 @@
iFace->tag_get_handle( MATERIAL_SET_TAG_NAME, bid );
iFace->tag_get_handle( DIRICHLET_SET_TAG_NAME, nid );
iFace->tag_get_handle( NEUMANN_SET_TAG_NAME, sid );
- iFace->tag_get_handle( PARALLEL_INTERFACE_TAG_NAME, iid );
+ iFace->tag_get_handle( PARALLEL_PARTITION_TAG_NAME, iid );
MBRange typesets[10];
- const char* typenames[] = {"Block", "Sideset", "NodeSet", "Vertex", "Curve", "Surface", "Volume", "Body", "Interfaces", "Other"};
+ const char* typenames[] = {"Block", "Sideset", "NodeSet", "Vertex", "Curve", "Surface", "Volume", "Body", "Partition", "Other"};
for (MBRange::iterator riter = sets.begin(); riter != sets.end(); ++riter)
{
unsigned dim, id, proc[2], oldsize;
@@ -165,22 +165,22 @@
std::string line(typenames[ii]);
if (typesets[ii].empty())
continue;
- sprintf(num, "(%u):", typesets[ii].size());
+ sprintf(num, "(%lu):",(unsigned long)typesets[ii].size());
line += num;
- for (MBRange::const_pair_iterator piter = typesets[ii].pair_begin();
- piter != typesets[ii].pair_end(); ++piter)
+ for (MBRange::const_pair_iterator piter = typesets[ii].const_pair_begin();
+ piter != typesets[ii].const_pair_end(); ++piter)
{
- sprintf(num," %d", (*piter).first);
+ sprintf(num," %lx", (unsigned long)(*piter).first);
line += num;
if ((*piter).first != (*piter).second) {
- sprintf(num,"-%d", (*piter).second);
+ sprintf(num,"-%lx", (unsigned long)(*piter).second);
line += num;
}
}
printdebug ("%s\n", line.c_str());
}
- printdebug("Total: %u\n", sets.size());
+ printdebug("Total: %lu\n", (unsigned long)sets.size());
}
#endif
@@ -262,7 +262,7 @@
MBTag iface_tag, geom_tag;
int i, proc_pair[2];
- START_SERIAL;
+ //START_SERIAL;
printdebug( "Pre-interface mesh:\n");
printrange(nodeSet.range);
for (std::list<ExportSet>::iterator eiter = exportList.begin();
@@ -275,7 +275,10 @@
// Get tag handles
result = iFace->tag_get_handle( PARALLEL_SHARED_PROC_TAG_NAME, iface_tag );
- if (MB_SUCCESS != result) return result;
+ if (MB_SUCCESS != result) {
+ iface_tag = 0;
+ return MB_SUCCESS;
+ }
result = iFace->tag_get_handle( GEOM_DIMENSION_TAG_NAME, geom_tag );
if (MB_SUCCESS != result) return result;
@@ -413,7 +416,7 @@
printrange(eiter->range);
printrange(setSet.range);
- END_SERIAL;
+ //END_SERIAL;
return MB_SUCCESS;
}
@@ -515,44 +518,85 @@
std::list<SparseTag>::iterator tag_iter;
sort_tags_by_name();
const int num_tags = tagList.size();
- std::vector<int> tag_offsets(num_tags), tag_counts(num_tags);
- std::vector<int>::iterator tag_off_iter = tag_counts.begin();
- for (tag_iter = tagList.begin(); tag_iter != tagList.end(); ++tag_iter, ++tag_off_iter)
+
+ // Construct vector (tag_counts) containing a pair of values for each
+ // tag, where the first value in the pair is the number of entities on
+ // this processor for which the tag has been set. The second value is
+ // zero for normal tags. For variable-length tags it is the total number
+ // of tag values set for all entities on this processor.
+ std::vector<unsigned long> tag_offsets(2*num_tags), tag_counts(2*num_tags);
+ std::vector<unsigned long>::iterator tag_off_iter = tag_counts.begin();
+ for (tag_iter = tagList.begin(); tag_iter != tagList.end(); ++tag_iter) {
+ int s;
*tag_off_iter = tag_iter->range.size();
+ ++tag_off_iter;
+ if (MB_VARIABLE_DATA_LENGTH == iFace->tag_get_size( tag_iter->tag_id, s )) {
+ unsigned long total_len;
+ rval = get_tag_data_length( *tag_iter, total_len );
+ if (MB_SUCCESS != rval)
+ return rval;
+
+ *tag_off_iter = total_len;
+ assert(total_len == *tag_off_iter);
+ }
+ else {
+ *tag_off_iter = 0;
+ }
+ ++tag_off_iter;
+ }
+ // Populate proc_tag_offsets on root processor with the values from
+ // tag_counts on each processor.
printdebug("Exchanging tag data for %d tags.\n", num_tags);
- std::vector<int> proc_tag_offsets(num_tags*numProc);
- result = MPI_Gather( &tag_counts[0], num_tags, MPI_INT,
- &proc_tag_offsets[0], num_tags, MPI_INT,
+ std::vector<unsigned long> proc_tag_offsets(2*num_tags*numProc);
+ result = MPI_Gather( &tag_counts[0], 2*num_tags, MPI_UNSIGNED_LONG,
+ &proc_tag_offsets[0], 2*num_tags, MPI_UNSIGNED_LONG,
0, MPI_COMM_WORLD );
assert(MPI_SUCCESS == result);
+ // Calculate the total counts over all processors (tag_counts)
+ // and the offset at which each processor should begin writing
+ // its data (proc_tag_offsets). Both lists contain a pair of
+ // values, where the first is the number of entities and the second
+ // is either unused for fixed-length tags or the total data table
+ // size for variable-length tags.
tag_iter = tagList.begin();
for (int i = 0; i < num_tags; ++i, ++tag_iter)
{
- tag_counts[i] = 0;
- int next_offset = 0;
+ tag_counts[2*i] = tag_counts[2*i+1] = 0;
+ unsigned long next_offset = 0;
+ unsigned long next_var_len_offset = 0;
for (int j = 0; j < numProc; j++)
{
- int count = proc_tag_offsets[i + j*num_tags];
- proc_tag_offsets[i + j*num_tags] = next_offset;
+ unsigned long count = proc_tag_offsets[2*i + j*2*num_tags];
+ proc_tag_offsets[2*i + j*2*num_tags] = next_offset;
next_offset += count;
- tag_counts[i] += count;
+ tag_counts[2*i] += count;
+
+ count = proc_tag_offsets[2*i + 1 + j*2*num_tags];
+ proc_tag_offsets[2*i + 1 + j*2*num_tags] = next_var_len_offset;
+ next_var_len_offset += count;
+ tag_counts[2*i + 1] += count;
}
if (0 == myRank)
{
- rval = create_tag(*tag_iter);
+ rval = create_tag(tag_iter->tag_id, next_offset, next_var_len_offset);
assert(MB_SUCCESS == rval);
- printdebug( "Creating table of size %d for tag 0x%lx\n", (int)next_offset, (unsigned long)tag_iter->tag_id);
+ printdebug( "Creating table of size %lu for tag 0x%lx\n",
+ next_var_len_offset ? next_var_len_offset : next_offset,
+ (unsigned long)tag_iter->tag_id );
}
}
- result = MPI_Bcast( &tag_counts[0], num_tags, MPI_INT, 0, MPI_COMM_WORLD );
+ // Send total counts to all processors. This is necessary because all
+ // processors need to know if we are not writing anything for the tag (count == 0).
+ result = MPI_Bcast( &tag_counts[0], 2*num_tags, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD );
assert(MPI_SUCCESS == result);
- result = MPI_Scatter( &proc_tag_offsets[0], num_tags, MPI_INT,
- &tag_offsets[0], num_tags, MPI_INT,
+ // Send to each processor its per-tag offset values.
+ result = MPI_Scatter( &proc_tag_offsets[0], 2*num_tags, MPI_UNSIGNED_LONG,
+ &tag_offsets[0], 2*num_tags, MPI_UNSIGNED_LONG,
0, MPI_COMM_WORLD );
assert(MPI_SUCCESS == result);
@@ -560,20 +604,21 @@
tag_iter = tagList.begin();
for (int i = 0; i < num_tags; ++i, ++tag_iter)
{
- tag_iter->offset = tag_offsets[i];
- tag_iter->write = tag_counts[i] > 0;
+ tag_iter->offset = tag_offsets[2*i];
+ tag_iter->write = tag_counts[2*i] > 0;
+ tag_iter->varDataOffset = tag_offsets[2*i + 1];
}
#ifdef DEBUG
START_SERIAL;
- printdebug("Tags: %16s %8s %8s %8s\n", "Name", "Count", "Offset", "Handle");
+ printdebug("Tags: %12s %8s %8s %8s %8s %8s\n", "Name", "Count", "Offset", "Var Off", "Var Len", "Handle");
tag_iter = tagList.begin();
for (int i = 0; i < num_tags; ++i, ++tag_iter)
{
std::string name;
iFace->tag_get_name( tag_iter->tag_id, name );
- printdebug(" %16s %8d %8d %8lx\n", name.c_str(), tag_counts[i], tag_offsets[i], (unsigned long)tag_iter->tag_id );
+ printdebug("%18s %8lu %8lu %8lu %8lu 0x%7lx\n", name.c_str(), tag_counts[2*i], tag_offsets[2*i], tag_offsets[2*i+1], tag_counts[2*i+1], (unsigned long)tag_iter->tag_id );
}
END_SERIAL;
#endif
@@ -1060,7 +1105,7 @@
MBErrorCode rval;
int i, result;
MBRange::iterator riter;
-
+
rval = iFace->tag_get_handle( tags.filterTag.c_str(), data.filter_tag );
if (rval != MB_SUCCESS) return rval;
if (tags.useFilterValue)
Modified: MOAB/trunk/test/h5file/Makefile.am
===================================================================
--- MOAB/trunk/test/h5file/Makefile.am 2008-03-13 19:52:36 UTC (rev 1650)
+++ MOAB/trunk/test/h5file/Makefile.am 2008-03-13 20:34:46 UTC (rev 1651)
@@ -1,14 +1,18 @@
DEFS = $(DEFINES)
INCLUDES += -I$(top_srcdir) -I$(top_srcdir)/mhdf/include -I$(top_builddir)
check_PROGRAMS = h5test h5legacy h5varlen
+parallel_programs = parallel varlen_ll
+
if PARALLEL_HDF5
- check_PROGRAMS += parallel
+ check_PROGRAMS += $(parallel_programs)
INCLUDES += -I$(top_srcdir)/parallel
endif
+
TESTS = $(check_PROGRAMS)
+LDADD = $(top_builddir)/libMOAB.la
-LDADD = $(top_builddir)/libMOAB.la
h5test_SOURCES = h5file_test.cpp
h5legacy_SOURCES = h5legacy.cpp
h5varlen_SOURCES = h5varlen.cpp
parallel_SOURCES = parallel.cpp
+varlen_ll_SOURCES = varlen_ll.cpp
Added: MOAB/trunk/test/h5file/varlen_ll.cpp
===================================================================
--- MOAB/trunk/test/h5file/varlen_ll.cpp (rev 0)
+++ MOAB/trunk/test/h5file/varlen_ll.cpp 2008-03-13 20:34:46 UTC (rev 1651)
@@ -0,0 +1,170 @@
+/**\file varlen_ll.cpp
+ * \brief Test HDF5 parallel I/O of variable-length tag data.
+ * \author Jason Kraftcheck <kraftche at cae.wisc.edu>
+ */
+
+#include "MBCore.hpp"
+#define TEST_WITH_MPI
+#include "TestUtil.hpp"
+#include "WriteHDF5Parallel.hpp"
+#include "FileOptions.hpp"
+#include "MBParallelConventions.h"
+
+bool keep_files = false; // controllable with -k flag
+bool wait_on_start = false; // start all procs and wait for input on root node
+
+void test_var_length_parallel();
+
+int main(int argc, char* argv[])
+{
+ MPI_Init( &argc, &argv );
+
+ for (int i = 1; i < argc; ++i) {
+ if (!strcmp(argv[i], "-k"))
+ keep_files = true;
+ else if (!strcmp(argv[i], "-w"))
+ wait_on_start = true;
+ else {
+ fprintf( stderr, "Usage: %s [-k] [-w]\n", argv[0] );
+ abort();
+ }
+ }
+
+ if (wait_on_start) {
+ int rank;
+ MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+ if (0 == rank) {
+ puts( "Press <enter> to begin: ");
+ fflush( stdout );
+ getchar();
+ }
+ MPI_Barrier(MPI_COMM_WORLD);
+ }
+
+ int err_count = 0;
+ err_count += RUN_TEST( test_var_length_parallel );
+ return err_count;
+}
+
+
+void test_var_length_parallel()
+{
+ MBRange::const_iterator i;
+ MBErrorCode rval;
+ MBCore moab;
+ MBInterface &mb = moab;
+ MBRange verts;
+ MBTag vartag;
+ const char* filename = "var-len-para.h5m";
+ const char* tagname = "ParVar";
+
+ // If this tag doesn't exist, writer will fail
+ MBTag junk_tag;
+ mb.tag_create( PARALLEL_GID_TAG_NAME, sizeof(int), MB_TAG_DENSE, MB_TYPE_INTEGER, junk_tag, 0 );
+
+ int numproc, rank;
+ MPI_Comm_size( MPI_COMM_WORLD, &numproc );
+ MPI_Comm_rank( MPI_COMM_WORLD, &rank );
+
+ // Create N+1 vertices on each processor, where N is the rank
+ std::vector<double> coords( 3*rank+3, (double)rank );
+ rval = mb.create_vertices( &coords[0], rank+1, verts );
+ CHECK_ERR(rval);
+
+ // Create a var-len tag
+ rval = mb.tag_create_variable_length( tagname, MB_TAG_DENSE, MB_TYPE_INTEGER, vartag );
+ CHECK_ERR(rval);
+
+ // Write data on each vertex:
+ // { n, rank, rank+1, ..., rank+n-1 } where n >= 1
+ std::vector<int> data;
+ rval = MB_SUCCESS;
+ for (i = verts.begin(); i != verts.end(); ++i) {
+ MBEntityHandle h = *i;
+ const int n = h % 7 + 1;
+ data.resize( n+1 );
+ data[0] = n;
+ for (int j = 0; j < n; ++j)
+ data[j+1] = rank + j;
+ const int s = (n + 1) * sizeof(int);
+ const void* ptrarr[] = { &data[0] };
+ MBErrorCode tmperr = mb.tag_set_data( vartag, &h, 1, ptrarr, &s );
+ if (MB_SUCCESS != tmperr)
+ rval = tmperr;
+ }
+ CHECK_ERR(rval);
+
+ // Write file
+ WriteHDF5Parallel writer( &mb );
+ FileOptions opts("");
+ std::vector<std::string> qa_records;
+ rval = writer.write_file( filename,
+ true,
+ opts,
+ 0, 0,
+ qa_records );
+ CHECK_ERR(rval);
+
+ // Read file. We only reset and re-read the file on the
+ // root processsor. All other processors keep the pre-write
+ // mesh. This allows many of the tests to be run on all
+ // processors. Running the tests on the pre-write mesh on
+ // non-root processors allows us to verify that any problems
+ // are due to the file API rather than some other bug.
+ MBErrorCode rval2 = rval = MB_SUCCESS;
+ if (!rank) {
+ moab.~MBCore();
+ new (&moab) MBCore;
+ rval = mb.load_mesh( filename );
+ if (!keep_files)
+ remove( filename );
+ rval2 = mb.tag_get_handle( tagname, vartag );
+ }
+ CHECK_ERR(rval);
+ CHECK_ERR(rval2);
+
+ // Check that tag is correct
+ int tag_size;
+ rval = mb.tag_get_size( vartag, tag_size );
+ CHECK_EQUAL( MB_VARIABLE_DATA_LENGTH, rval );
+ MBTagType storage;
+ rval = mb.tag_get_type( vartag, storage );
+ CHECK_EQUAL( MB_TAG_DENSE, storage );
+ MBDataType type;
+ rval = mb.tag_get_data_type( vartag, type);
+ CHECK_EQUAL( MB_TYPE_INTEGER, type );
+
+ // get vertices
+ verts.clear();
+ rval = mb.get_entities_by_type( 0, MBVERTEX, verts );
+ CHECK_ERR(rval);
+
+ // Check consistency of tag data on each vertex
+ // and count the number of vertices for each rank.
+ std::vector<int> vtx_counts( numproc, 0 );
+ for (i = verts.begin(); i != verts.end(); ++i) {
+ MBEntityHandle h = *i;
+ int size = -1;
+ const void* ptrarr[1] = { 0 };
+ rval = mb.tag_get_data( vartag, &h, 1, ptrarr, &size );
+ size /= sizeof(int);
+ CHECK_ERR( rval );
+ const int* data = reinterpret_cast<const int*>(ptrarr[0]);
+ CHECK( size >= 2 );
+ CHECK( NULL != data );
+ CHECK_EQUAL( size-1, data[0] );
+ CHECK( data[1] >= 0 && data[1] < numproc );
+ ++vtx_counts[data[1]];
+ for (int j = 1; j < size-1; ++j)
+ CHECK_EQUAL( data[1]+j, data[1+j] );
+ }
+
+ // Check number of vertices for each rank
+ for (int j = 0; j < numproc; ++j) {
+ // Only root should have data for other processors.
+ if (rank == 0 || rank == j)
+ CHECK_EQUAL( j+1, vtx_counts[j] );
+ else
+ CHECK_EQUAL( 0, vtx_counts[j] );
+ }
+}
More information about the moab-dev
mailing list