[MOAB-dev] r1985 - in MOAB/trunk: . mhdf/src parallel tools/mbcoupler
tautges at mcs.anl.gov
tautges at mcs.anl.gov
Mon Jul 7 14:04:46 CDT 2008
Author: tautges
Date: 2008-07-07 14:04:46 -0500 (Mon, 07 Jul 2008)
New Revision: 1985
Modified:
MOAB/trunk/ReadHDF5.cpp
MOAB/trunk/mhdf/src/adjacency.c
MOAB/trunk/mhdf/src/connectivity.c
MOAB/trunk/mhdf/src/file-handle.h
MOAB/trunk/mhdf/src/file.c
MOAB/trunk/mhdf/src/nodes.c
MOAB/trunk/mhdf/src/sets.c
MOAB/trunk/mhdf/src/tags.c
MOAB/trunk/mhdf/src/util.c
MOAB/trunk/mhdf/src/util.h
MOAB/trunk/parallel/ReadParallel.cpp
MOAB/trunk/parallel/ReadParallel.hpp
MOAB/trunk/parallel/mbparallelcomm_test.cpp
MOAB/trunk/tools/mbcoupler/mbcoupler_test.cpp
Log:
Adding MPI collective IO option as a driver for ReadHDF5. If the
PARALLEL=READ_DELETE option is used and USE_MPIO is also in the option
list, MPI collective IO will be used to read the HDF5 data. This
could potentially save lots of time for large reads on lots of
processors. Will test this out and see if this really works. In
workstation-parallel environments, this degrades performance by a
coupla percent max.
tools/mbcoupler/mbcoupler_test.cpp: moved call to MPI_Finalize, in an
attempt to get rid of warning about calling MPI functions after
MPI_Finalize; didn't work.
ReadHDF5: implemented parallel MPI-based read.
parallel/mbparallelcomm_test.cpp: changed syntax for calling the test,
to allow specification of multiple files for '3' option and to turn
on/off parallel MPI-based read
parallel/ReadParallel.cpp: parsing of MPI-based parallel read
mhdf files: adding ability to specify MPI collective IO as IO driver
Passed make check both in serial and in parallel.
Modified: MOAB/trunk/ReadHDF5.cpp
===================================================================
--- MOAB/trunk/ReadHDF5.cpp 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/ReadHDF5.cpp 2008-07-07 19:04:46 UTC (rev 1985)
@@ -27,11 +27,18 @@
#include <assert.h>
#include <H5Tpublic.h>
+#include <H5Ppublic.h>
#include "MBInterface.hpp"
#include "MBInternals.hpp"
#include "MBTagConventions.hpp"
#include "ReadHDF5.hpp"
#include "MBCN.hpp"
+#include "FileOptions.hpp"
+#ifdef USE_MPI
+#include "ReadParallel.hpp"
+#include <H5FDmpi.h>
+#include <H5FDmpio.h>
+#endif
//#include "WriteHDF5.hpp"
#include <stdlib.h>
@@ -112,7 +119,7 @@
MBErrorCode ReadHDF5::load_file( const char* filename,
MBEntityHandle& file_set,
- const FileOptions&,
+ const FileOptions&opts,
const int*,
const int num_blocks )
{
@@ -136,8 +143,33 @@
DEBUGOUT( "Opening File\n" );
+#ifdef USE_MPI
+ int parallel_mode;
+ MBErrorCode result = opts.match_option( "PARALLEL", ReadParallel::parallelOptsNames,
+ parallel_mode );
+ if (MB_FAILURE == result) {
+ readUtil->report_error("Unexpected value for 'PARALLEL' option\n");
+ return MB_FAILURE;
+ }
+ else if (MB_ENTITY_NOT_FOUND == result) {
+ parallel_mode = 0;
+ }
+
+ bool use_mpio = false;
+ result = opts.get_null_option("USE_MPIO");
+ if (MB_SUCCESS == result) use_mpio = true;
+
+ if (parallel_mode == ReadParallel::POPT_READ_DELETE && use_mpio) {
+ hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS);
+ H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL);
+ filePtr = mhdf_openFileWithOpt( filename, 0, NULL, plist_id, &status );
+ }
+ else filePtr = mhdf_openFile( filename, 0, NULL, &status );
+
+#else
// Open the file
filePtr = mhdf_openFile( filename, 0, NULL, &status );
+#endif
if (!filePtr)
{
readUtil->report_error( mhdf_message( &status ));
@@ -1987,7 +2019,6 @@
MBErrorCode ReadHDF5::read_qa( MBEntityHandle import_set )
{
- MBErrorCode rval;
mhdf_Status status;
std::vector<std::string> qa_list;
Modified: MOAB/trunk/mhdf/src/adjacency.c
===================================================================
--- MOAB/trunk/mhdf/src/adjacency.c 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/adjacency.c 2008-07-07 19:04:46 UTC (rev 1985)
@@ -137,7 +137,7 @@
{
table_id = mhdf_open_table( file_ptr->hdf_handle,
NODE_ADJCY_PATH,
- 1, &dim,
+ 1, file_ptr->parallel, &dim,
status );
}
else
@@ -146,7 +146,7 @@
if (elem_id < 0) return -1;
table_id = mhdf_open_table( elem_id,
ADJACENCY_NAME,
- 1, &dim,
+ 1, file_ptr->parallel, &dim,
status );
H5Gclose( elem_id );
}
Modified: MOAB/trunk/mhdf/src/connectivity.c
===================================================================
--- MOAB/trunk/mhdf/src/connectivity.c 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/connectivity.c 2008-07-07 19:04:46 UTC (rev 1985)
@@ -113,7 +113,7 @@
if (elem_id < 0) return -1;
table_id = mhdf_open_table2( elem_id, CONNECTIVITY_NAME,
- 2, dims,
+ 2, file_ptr->parallel, dims,
first_elem_id_out, status );
H5Gclose( elem_id );
@@ -276,13 +276,13 @@
if (elem_id < 0) return ;
index_id = mhdf_open_table( elem_id, POLY_INDEX_NAME,
- 1, &row_count, status );
+ 1, file_ptr->parallel, &row_count, status );
if (index_id < 0)
{ H5Gclose(elem_id); return ; }
*num_poly_out = (int)row_count;
table_id = mhdf_open_table( elem_id, CONNECTIVITY_NAME,
- 1, &row_count, status );
+ 1, file_ptr->parallel, &row_count, status );
H5Gclose( elem_id );
if (table_id < 0)
Modified: MOAB/trunk/mhdf/src/file-handle.h
===================================================================
--- MOAB/trunk/mhdf/src/file-handle.h 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/file-handle.h 2008-07-07 19:04:46 UTC (rev 1985)
@@ -32,6 +32,7 @@
int open_handle_count;
long max_id;
+ int parallel;
} FileHandle;
FileHandle* mhdf_alloc_FileHandle( hid_t hdf_handle, mhdf_Status* status );
Modified: MOAB/trunk/mhdf/src/file.c
===================================================================
--- MOAB/trunk/mhdf/src/file.c 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/file.c 2008-07-07 19:04:46 UTC (rev 1985)
@@ -22,6 +22,10 @@
#include <H5Spublic.h>
#include <H5Tpublic.h>
#include <H5Apublic.h>
+#ifdef USE_MPI
+#include <H5FDmpi.h>
+#include <H5FDmpio.h>
+#endif
#include "mhdf.h"
#include "status.h"
#include "names-and-paths.h"
@@ -331,7 +335,17 @@
if (max_id_out)
*max_id_out = file_ptr->max_id;
-
+
+ file_ptr->parallel = 0;
+
+#ifdef USE_MPI
+ MPI_Comm comm;
+ MPI_Info info;
+ if (0 <= H5Pget_fapl_mpio(access_prop, &comm, &info) &&
+ MPI_COMM_NULL != comm && MPI_INFO_NULL != info)
+ file_ptr->parallel = 1;
+#endif
+
mhdf_setOkay( status );
API_END_H(1);
return file_ptr;
Modified: MOAB/trunk/mhdf/src/nodes.c
===================================================================
--- MOAB/trunk/mhdf/src/nodes.c 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/nodes.c 2008-07-07 19:04:46 UTC (rev 1985)
@@ -95,7 +95,7 @@
return -1;
table_id = mhdf_open_table2( file_ptr->hdf_handle,
- NODE_COORD_PATH, 2,
+ NODE_COORD_PATH, 2, file_ptr->parallel,
dims, first_id_out, status );
if (table_id < 0)
return -1;
Modified: MOAB/trunk/mhdf/src/sets.c
===================================================================
--- MOAB/trunk/mhdf/src/sets.c 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/sets.c 2008-07-07 19:04:46 UTC (rev 1985)
@@ -179,7 +179,7 @@
return -1;
table_id = mhdf_open_table2( file_ptr->hdf_handle,
- SET_META_PATH, 2,
+ SET_META_PATH, 2, file_ptr->parallel,
dims, first_id_out, status );
if (table_id < 0)
return -1;
@@ -618,7 +618,7 @@
table_id = mhdf_open_table( file_ptr->hdf_handle,
SET_DATA_PATH,
- 1, &dim,
+ 1, file_ptr->parallel, &dim,
status );
*data_list_size_out = (long)dim;
@@ -706,7 +706,7 @@
table_id = mhdf_open_table( file_ptr->hdf_handle,
SET_CHILD_PATH,
- 1, &dim,
+ 1, file_ptr->parallel, &dim,
status );
*child_list_size = (long)dim;
@@ -766,7 +766,7 @@
table_id = mhdf_open_table( file_ptr->hdf_handle,
SET_PARENT_PATH,
- 1, &dim,
+ 1, file_ptr->parallel, &dim,
status );
*parent_list_size = (long)dim;
Modified: MOAB/trunk/mhdf/src/tags.c
===================================================================
--- MOAB/trunk/mhdf/src/tags.c 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/tags.c 2008-07-07 19:04:46 UTC (rev 1985)
@@ -1400,7 +1400,7 @@
strcpy( path, DENSE_TAG_SUBGROUP );
mhdf_name_to_path( tag_name, path + dir_len, name_len + 1 );
- data_id = mhdf_open_table( elem_id, path, 1, &size, status );
+ data_id = mhdf_open_table( elem_id, path, 1, file_ptr->parallel, &size, status );
free( path );
H5Gclose( elem_id );
*num_values_out = (long)size;
@@ -1615,14 +1615,15 @@
tag_id = get_tag( file_handle, tag_name, status );
if (tag_id < 0) return ;
- index_id = mhdf_open_table( tag_id, SPARSE_ENTITY_NAME, 1, &num_ent, status );
+ FileHandle *file_ptr = (FileHandle*)file_handle;
+ index_id = mhdf_open_table( tag_id, SPARSE_ENTITY_NAME, 1, file_ptr->parallel, &num_ent, status );
if (index_id < 0)
{
H5Gclose( tag_id );
return ;
}
- data_id = mhdf_open_table( tag_id, SPARSE_VALUES_NAME, 1, &data_size, status );
+ data_id = mhdf_open_table( tag_id, SPARSE_VALUES_NAME, 1, file_ptr->parallel, &data_size, status );
if (data_id < 0)
{
H5Gclose( tag_id );
@@ -1641,7 +1642,7 @@
/* If variable length... */
if (rval) {
- offset_id = mhdf_open_table( tag_id, TAG_VAR_INDICES, 1, &num_data, status );
+ offset_id = mhdf_open_table( tag_id, TAG_VAR_INDICES, 1, file_ptr->parallel, &num_data, status );
if (offset_id < 0) {
H5Gclose( tag_id );
H5Dclose( index_id );
Modified: MOAB/trunk/mhdf/src/util.c
===================================================================
--- MOAB/trunk/mhdf/src/util.c 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/util.c 2008-07-07 19:04:46 UTC (rev 1985)
@@ -21,6 +21,10 @@
#include "util.h"
#include "status.h"
#include "names-and-paths.h"
+#ifdef USE_MPI
+#include <H5FDmpi.h>
+#include <H5FDmpio.h>
+#endif
void* mhdf_malloc( size_t size, mhdf_Status* status )
{
@@ -549,6 +553,7 @@
mhdf_open_table( hid_t group_id,
const char* path,
int columns,
+ int parallel,
hsize_t* rows_out,
mhdf_Status* status )
{
@@ -561,6 +566,12 @@
#else
table_id = H5Dopen( group_id, path );
#endif
+
+#ifdef USE_MPI
+ if (parallel)
+ H5Pset_dxpl_mpio(group_id, H5FD_MPIO_COLLECTIVE);
+#endif
+
if (table_id < 0)
{
mhdf_setFail( status, "HDF5 DataSet creation failed.");
@@ -602,6 +613,7 @@
mhdf_open_table2( hid_t group_id,
const char* path,
int rank,
+ int parallel,
hsize_t* dims_out,
long* start_id_out,
mhdf_Status* status )
@@ -613,6 +625,12 @@
#else
table_id = H5Dopen( group_id, path );
#endif
+
+#ifdef USE_MPI
+ if (parallel)
+ H5Pset_dxpl_mpio(group_id, H5FD_MPIO_COLLECTIVE);
+#endif
+
if (table_id < 0)
{
mhdf_setFail( status, "HDF5 DataSet creation failed.");
Modified: MOAB/trunk/mhdf/src/util.h
===================================================================
--- MOAB/trunk/mhdf/src/util.h 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/mhdf/src/util.h 2008-07-07 19:04:46 UTC (rev 1985)
@@ -115,6 +115,7 @@
mhdf_open_table( hid_t group,
const char* path,
int columns,
+ int parallel,
hsize_t* rows_out,
mhdf_Status* status );
@@ -122,6 +123,7 @@
mhdf_open_table2( hid_t group,
const char* path,
int rank,
+ int parallel,
hsize_t* dims_out,
long* start_id_out,
mhdf_Status* status );
Modified: MOAB/trunk/parallel/ReadParallel.cpp
===================================================================
--- MOAB/trunk/parallel/ReadParallel.cpp 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/parallel/ReadParallel.cpp 2008-07-07 19:04:46 UTC (rev 1985)
@@ -34,6 +34,10 @@
"PARALLEL EXCHANGE_GHOSTS"
};
+const char* ReadParallel::parallelOptsNames[] = { "NONE", "BCAST", "BCAST_DELETE",
+ "READ_DELETE", "READ_PARALLEL",
+ "FORMAT", 0 };
+
ReadParallel::ReadParallel(MBInterface* impl,
MBParallelComm *pc)
: mbImpl(impl), myPcomm(pc)
@@ -55,14 +59,7 @@
// Get parallel settings
int parallel_mode;
- const char* parallel_opts[] = { "NONE", "BCAST", "BCAST_DELETE",
- "READ_DELETE", "READ_PARALLEL",
- "FORMAT", 0 };
- enum ParallelOpts {POPT_NONE=0, POPT_BCAST, POPT_BCAST_DELETE,
- POPT_READ_DELETE, POPT_READ_PARALLEL,
- POPT_FORMAT, POPT_LAST};
-
- MBErrorCode result = opts.match_option( "PARALLEL", parallel_opts,
+ MBErrorCode result = opts.match_option( "PARALLEL", parallelOptsNames,
parallel_mode );
if (MB_FAILURE == result) {
merror->set_last_error( "Unexpected value for 'PARALLEL' option\n" );
Modified: MOAB/trunk/parallel/ReadParallel.hpp
===================================================================
--- MOAB/trunk/parallel/ReadParallel.hpp 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/parallel/ReadParallel.hpp 2008-07-07 19:04:46 UTC (rev 1985)
@@ -37,6 +37,12 @@
//! Destructor
virtual ~ReadParallel() {}
+ static const char *parallelOptsNames[];
+
+ enum ParallelOpts {POPT_NONE=0, POPT_BCAST, POPT_BCAST_DELETE,
+ POPT_READ_DELETE, POPT_READ_PARALLEL,
+ POPT_FORMAT, POPT_LAST};
+
protected:
private:
Modified: MOAB/trunk/parallel/mbparallelcomm_test.cpp
===================================================================
--- MOAB/trunk/parallel/mbparallelcomm_test.cpp 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/parallel/mbparallelcomm_test.cpp 2008-07-07 19:04:46 UTC (rev 1985)
@@ -47,7 +47,7 @@
MBErrorCode read_file(MBInterface *mbImpl, std::vector<std::string> &filenames,
const char *tag_name, int tag_val, int distrib,
- int parallel_option, int resolve_shared, int with_ghosts);
+ int parallel_option, int resolve_shared, int with_ghosts, int use_mpio);
MBErrorCode test_packing(MBInterface *mbImpl, const char *filename);
@@ -91,7 +91,7 @@
<< "=== =====" << std::endl
<< " 1 <linear_ints> <shared_verts> " << std::endl
<< " 2 <n_ints> " << std::endl
- << " 3* <file_name> [<tag_name>=\"MATERIAL_SET\" [tag_val] [distribute=1] [resolve_shared=1] [with_ghosts=1]" << std::endl
+ << " 3* <# files> <file_names...> [<tag_name>=\"MATERIAL_SET\" [tag_val] [distribute=1] [resolve_shared=1] [with_ghosts=1] [use_mpio=0]" << std::endl
<< " 4 <file_name> " << std::endl
<< "*Note: if opt 3 is used, it must be the last one." << std::endl;
@@ -99,10 +99,11 @@
return 1;
}
- int npos = 1, tag_val, distrib, with_ghosts = 1, resolve_shared = 1;
+ int npos = 1, tag_val, distrib, with_ghosts = 1, resolve_shared = 1, use_mpio = 0;
const char *tag_name;
std::vector<std::string> filenames;
int parallel_option = 0;
+ int num_files;
while (npos != argc) {
MBErrorCode tmp_result;
@@ -120,17 +121,20 @@
// read a file in parallel from the filename on the command line
tag_name = "MATERIAL_SET";
tag_val = -1;
- filenames.push_back(std::string(argv[npos++]));
+ num_files = strtol(argv[npos++], NULL, 0);
+ while (num_files-- && npos < argc)
+ filenames.push_back(std::string(argv[npos++]));
if (npos < argc) tag_name = argv[npos++];
if (npos < argc) tag_val = strtol(argv[npos++], NULL, 0);
if (npos < argc) distrib = strtol(argv[npos++], NULL, 0);
else distrib = 1;
if (npos < argc) resolve_shared = strtol(argv[npos++], NULL, 0);
if (npos < argc) with_ghosts = strtol(argv[npos++], NULL, 0);
+ if (npos < argc) use_mpio = strtol(argv[npos++], NULL, 0);
tmp_result = read_file(mbImpl, filenames, tag_name, tag_val,
distrib, parallel_option,
- resolve_shared, with_ghosts);
+ resolve_shared, with_ghosts, use_mpio);
if (MB_SUCCESS != tmp_result) {
result = tmp_result;
std::cerr << "Couldn't read mesh; error message:" << std::endl;
@@ -161,7 +165,7 @@
filenames.push_back(std::string(argv[npos++]));
tmp_result = read_file(mbImpl, filenames, tag_name, tag_val,
distrib, parallel_option, resolve_shared,
- with_ghosts);
+ with_ghosts, use_mpio);
if (MB_SUCCESS != tmp_result) {
result = tmp_result;
std::cerr << "Couldn't read mesh; error message:" << std::endl;
@@ -276,7 +280,7 @@
std::vector<std::string> &filenames,
const char *tag_name, int tag_val,
int distrib, int parallel_option, int resolve_shared,
- int with_ghosts)
+ int with_ghosts, int use_mpio)
{
std::ostringstream options;
switch (parallel_option) {
@@ -308,6 +312,9 @@
if (1 == with_ghosts)
options << ";PARALLEL_GHOSTS=3.0.1";
+ if (1 == use_mpio)
+ options << ";USE_MPIO";
+
options << ";CPUTIME";
std::vector<MBEntityHandle> filesets(filenames.size());
@@ -321,7 +328,8 @@
result = rps[i]->load_file(filenames[i].c_str(), filesets[i],
FileOptions(options.str().c_str()), NULL, 0);
- PRINT_LAST_ERROR;
+ if (MB_SUCCESS != result)
+ PRINT_LAST_ERROR;
if (MB_SUCCESS != result) {
MPI_Abort(MPI_COMM_WORLD, result);
Modified: MOAB/trunk/tools/mbcoupler/mbcoupler_test.cpp
===================================================================
--- MOAB/trunk/tools/mbcoupler/mbcoupler_test.cpp 2008-07-07 18:08:47 UTC (rev 1984)
+++ MOAB/trunk/tools/mbcoupler/mbcoupler_test.cpp 2008-07-07 19:04:46 UTC (rev 1985)
@@ -128,7 +128,6 @@
}
std::cout << "Success." << std::endl;
- err = MPI_Finalize();
for (unsigned int i = 0; i < filenames.size(); i++) {
delete rps[i];
@@ -137,6 +136,8 @@
delete mbImpl;
+ err = MPI_Finalize();
+
return 0;
}
More information about the moab-dev
mailing list