diff -ruN flash-argonne-20060913/hdf5/flash_benchmark_io.F90 flash-argonne-ornl-updated/hdf5/flash_benchmark_io.F90 --- flash-argonne-20060913/hdf5/flash_benchmark_io.F90 2006-03-15 17:24:18.000000000 -0500 +++ flash-argonne-ornl-updated/hdf5/flash_benchmark_io.F90 2007-07-09 15:14:47.000000000 -0400 @@ -63,6 +63,10 @@ ! setup the file properties basenm = "flash_io_test_" +#ifdef JAGUAR +#error "change 'basenm' to the correct basename for your file system" + basenm = "/tmp/work/wyu/hdf5/flash_io_test_" +#endif #ifdef CHIBA #error "change 'basenm' to the correct basename for your file system" diff -ruN flash-argonne-20060913/hdf5/h5_file_interface.c flash-argonne-ornl-updated/hdf5/h5_file_interface.c --- flash-argonne-20060913/hdf5/h5_file_interface.c 2006-03-15 17:24:18.000000000 -0500 +++ flash-argonne-ornl-updated/hdf5/h5_file_interface.c 2007-07-09 15:18:19.000000000 -0400 @@ -85,6 +85,16 @@ ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "cb_buffer_size", "4194304"); #endif +#ifdef JAGUAR + ierr = H5Pset_sieve_buf_size(acc_template, 524288); + ierr = H5Pset_alignment(acc_template, 524288, 262144); + + /*ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "cb_nodes", "64");*/ + ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "romio_cb_write", "ENABLE"); + ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "romio_cb_read", "ENABLE"); + ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "cb_buffer_size", "33554432"); +#endif + #ifdef BGL /* TODO: what are the good hints for BGL? */ #endif diff -ruN flash-argonne-20060913/hdf5/h5_parallel_write.c flash-argonne-ornl-updated/hdf5/h5_parallel_write.c --- flash-argonne-20060913/hdf5/h5_parallel_write.c 2006-09-07 17:14:39.000000000 -0400 +++ flash-argonne-ornl-updated/hdf5/h5_parallel_write.c 2007-07-09 15:18:41.000000000 -0400 @@ -918,6 +918,11 @@ ierr = H5Pset_preserve(dxfer_template, 0u); #endif +#ifdef JAGUAR + ierr = H5Pset_dxpl_mpio(dxfer_template, H5FD_MPIO_COLLECTIVE); + ierr = H5Pset_preserve(dxfer_template, 0u); +#endif + #ifdef TFLOPS ierr = H5Pset_dxpl_mpio(dxfer_template, H5FD_MPIO_COLLECTIVE); ierr = H5Pset_preserve(dxfer_template, 0u); diff -ruN flash-argonne-20060913/hdf5/h5_parallel_write_single.c flash-argonne-ornl-updated/hdf5/h5_parallel_write_single.c --- flash-argonne-20060913/hdf5/h5_parallel_write_single.c 2006-09-07 17:14:39.000000000 -0400 +++ flash-argonne-ornl-updated/hdf5/h5_parallel_write_single.c 2007-07-09 15:19:24.000000000 -0400 @@ -770,6 +770,11 @@ ierr = H5Pset_preserve(dxfer_template, 0u); #endif +#ifdef JAGUAR + ierr = H5Pset_dxpl_mpio(dxfer_template, H5FD_MPIO_COLLECTIVE); + ierr = H5Pset_preserve(dxfer_template, 0u); +#endif + #ifdef CHIBA ierr = H5Pset_dxpl_mpio(dxfer_template, H5FD_MPIO_INDEPENDENT); ierr = H5Pset_preserve(dxfer_template, 0u); diff -ruN flash-argonne-20060913/hdf5/Makefile.jaguar flash-argonne-ornl-updated/hdf5/Makefile.jaguar --- flash-argonne-20060913/hdf5/Makefile.jaguar 1969-12-31 19:00:00.000000000 -0500 +++ flash-argonne-ornl-updated/hdf5/Makefile.jaguar 2007-07-09 15:08:08.000000000 -0400 @@ -0,0 +1,60 @@ +# Makefile for the I/O benchmark -- generic linux + +# objects +OBJS = get_mfluid_property.o \ + flash_release.o \ + flash_benchmark_io.o + +OBJS_HDF5_PARALLEL_IO = checkpoint_hdf5_parallel.o \ + plotfile_hdf5_parallel.o \ + h5_file_interface.o \ + h5_parallel_write.o \ + h5_parallel_write_single.o + +# compiler and linker commands +FCOMP = mpif90 +CCOMP = mpicc +CPPCOMP = mpicxx +LINK = mpif90 + + +# library locations +HDF5path = /apps/HDF5/1.6.5/xt3_pgi605_par +ZLIBpath = /apps/zlib/1.2.1/xt3_pgi60 + +#Usage: ftn test.f90 ${HDF5_FLIB} +# or cc test.c ${HDF5_CLIB} + +# compiler flags +FFLAGS = -c -fast -r8 -DN_DIM=3 -DMAXBLOCKS=100 -DIONMAX=13 -DJAGUAR + +F90FLAGS = + +CFLAGS = -c -I $(HDF5path)/include/ -DN_DIM=3 -DMAXBLOCKS=100 -DIONMAX=13 -DJAGUAR + + +# linker flags +LFLAGS = -o + + + +# libraries to include +LIB = -L $(HDF5path)/lib -lhdf5 -L$(ZLIBpath)/lib -lz + +.SUFFIXES: .f .F .f90 .F90 .c .C .o + +.F90.o : + $(FCOMP) $(FFLAGS) $< + +.f.o : + $(FCOMP) $(FFLAGS) $< + +.c.o : + $(CCOMP) $(CFLAGS) $< + +flash_benchmark_io: $(OBJS) \ + $(OBJS_HDF5_PARALLEL_IO) + $(LINK) $(LFLAGS) $@ $(OBJS) $(OBJS_HDF5_PARALLEL_IO) $(LIB) + +clean:: + rm -f *.o flash_benchmark_io diff -ruN flash-argonne-20060913/README.jaguar flash-argonne-ornl-updated/README.jaguar --- flash-argonne-20060913/README.jaguar 1969-12-31 19:00:00.000000000 -0500 +++ flash-argonne-ornl-updated/README.jaguar 2007-07-09 15:11:09.000000000 -0400 @@ -0,0 +1,35 @@ +0. Start with flash-argonne-20060913 + cd flash-argonne-20060913 + +1. file path editing + search for this line in the file: hdf5/flash_benchmark_io.F90 + Change this path name: /tmp/work/wyu/hdf5, for this line + basenm = "/tmp/work/wyu/hdf5/flash_io_test_" +to + basenm = "/tmp/work/oral/hdf5/flash_io_test_" + +2. To run flash IO + cd hdf + make -f Makefile.jaguar + yod -sz 128 ./flash_benchmark_io + +3. Efficiency tuning on Lustre striping width and size. + Looks the block protected with #ifdef JAGUAR #endif + in hdf5/h5_file_interface.c + +#ifdef JAGUAR + ierr = H5Pset_sieve_buf_size(acc_template, 524288); + ierr = H5Pset_alignment(acc_template, 524288, 262144); + + /*ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "cb_nodes", "64");*/ + ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "romio_cb_write", "ENABLE"); + ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "romio_cb_read", "ENABLE"); + ierr = MPI_Info_set(FILE_INFO_TEMPLATE, "cb_buffer_size", "33554432"); +#endif + + 2.a) To disable collective buffering, delete the line for romio_cb_write + 2.b) when collective buffering is on, set the number of IO processes with + the line for "cb_nodes" + 2.c) when collective buffering is on, adjust the buffer size with + the line for "cb_buffer_size" + 2.d) You may also explore for H5Pset_alignment and H5Pset_sieve_buf_size + +4. For questions, send your inquiries to Weikuan Yu (wyu@ornl.gov)