/*=========================================================================

  Program:   Visualization Toolkit
  Module:    POSIXSharedFile.cxx

  Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
  All rights reserved.
  See Copyright.txt or http://www.kitware.com/Copyright.htm for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notice for more information.

=========================================================================*/
// .NAME POSIXSharedFile.cxx -- Unbuffered POSIX I/O on a single file.
//
// .SECTION Description
//  A simple benchmark using unbuffered POSIX I/O on a single file.

#include <iostream>
#include <sstream>
#include <fstream>
#include <cassert>
#include <mpi.h>
#include <cmath>
#include <limits>

// System includes
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <fcntl.h>

#define POSIX_OPEN_WRITE  0
#define POSIX_WRITE       1
#define POSIX_CLOSE_WRITE 2
#define POSIX_OPEN_READ   3
#define POSIX_READ        4
#define POSIX_CLOSE_READ  5

// Number of doubles each process will write which is equal to 1048576
// that is 1MB and corresponds to the stripe size set in the directory
// where the data will be stored.
#define NUMDOUBLES 131072

// Stripe size: 33554432
//#define NUMDOUBLES 4194304

// Description:
// Logs a message -- only process 0 writes the message
void Log( const char *msg );

// Description:
// Initialize Data
void InitializeData( double *data, const int N, const int rank );

// Description:
// Checks the data
void CheckData( double *a, double *b, const int N );

// Description:
// Writes data
void WriteData( double *data, const int N, const int rank );

// Description:
// Reads data
void ReadData( double *data, const int N, const int rank );

// Description:
// Gather statistics
void GatherStatistics();

// Description:
// Returns a string represenation of the given timer index corresponding
// to Program.statistics.
std::string GetStringFromIdx( const int idx );

// Description:
// Program data-structure
struct {
    int Rank;
    int NumProcessors;
    int N;
    double *dataToWrite;
    double *dataRead;
    double *statistics;
} Program;


// Description:
// Program main
int main( int argc, char **argv )
{
  MPI_Init( &argc, &argv );
  MPI_Comm_rank( MPI_COMM_WORLD, &Program.Rank );
  MPI_Comm_size( MPI_COMM_WORLD, &Program.NumProcessors );

  // STEP 0: Initialize data
  Program.N           = NUMDOUBLES;
  Program.dataRead    = new double[ Program.N ];
  Program.dataToWrite = new double[ Program.N ];
  Program.statistics  = new double[ 6 ];
  for( int i=0; i < 6; Program.statistics[i++]=0.0 );
  InitializeData(Program.dataToWrite, Program.N, Program.Rank );
  MPI_Barrier( MPI_COMM_WORLD );

  // STEP 1: Write Data with POSIX
  Log( "Write data with POSIX" );
  WriteData( Program.dataToWrite, Program.N, Program.Rank );

  // STEP 2: Read Data with POSIX
  Log( "Read data with POSIX" );
  ReadData( Program.dataRead, Program.N, Program.Rank );

  // STEP 3: Check data
  Log( "Checking data from POSIX I/O" );
  CheckData( Program.dataToWrite, Program.dataRead, Program.N );

  // STEP 5: Gathering statistics
  Log( "Gathering statistics" );
  GatherStatistics();

  // STEP 4: Clean Up
  delete [] Program.dataRead;
  delete [] Program.dataToWrite;
  delete [] Program.statistics;

  MPI_Finalize();
  return 0;
}

//==============================================================================
//      F U N C T I O N   P R O T O T Y P E   I M P L E M E N T A T I O N
//==============================================================================

std::string GetStringFromIdx( const int idx )
{
/*
#define POSIX_OPEN_WRITE  0
#define POSIX_WRITE       1
#define POSIX_CLOSE_WRITE 2
#define POSIX_OPEN_READ   3
#define POSIX_READ        4
#define POSIX_CLOSE_READ  5
*/
  switch( idx )
    {
      case 0:
        return( std::string( "POSIX_OPEN_WRITE" ) );
        break;
      case 1:
        return( std::string( "POSIX_WRITE" ) );
        break;
      case 2:
        return( std::string( "POSIX_CLOSE_WRITE" ) );
        break;
      case 3:
        return( std::string( "POSIX_OPEN_READ" ) );
        break;
      case 4:
        return( std::string( "POSIX_READ" ) );
        break;
      case 5:
        return( std::string( "POSIX_CLOSE_READ" ) );
        break;
      default:
        assert( "Code should not reach here!" && false );
        return( std::string( "" ) );
    }
}

//------------------------------------------------------------------------------
void GatherStatistics()
{
  double *StatisticsPerRank = NULL;
  if( Program.Rank == 0 )
    StatisticsPerRank = new double[ 6*Program.NumProcessors ];

  MPI_Gather(
      Program.statistics, 6, MPI_DOUBLE,
      StatisticsPerRank, 6, MPI_DOUBLE,
      0, MPI_COMM_WORLD );

  if( Program.Rank == 0 )
    {
      // Find max
      double maxStats[ 6 ];
      for( int i=0; i < 6; maxStats[ i++ ]=std::numeric_limits<double>::min() );

      for( int i=0; i < Program.NumProcessors; ++i )
        {
          for( int j=0; j < 6; ++j )
            {
              if( maxStats[ j ] < StatisticsPerRank[i*6+j] )
                maxStats[ j ] = StatisticsPerRank[ i*6+j ];
            }
        } // END for all processors

      // Write the max out
      std::ofstream ofs;
      ofs.open( "MaxStatistics.txt" );
      assert( ofs.is_open() );

      for( int i=0; i < 6; ++i )
        ofs << GetStringFromIdx( i ) << ": " << maxStats[ i ] << std::endl;

      ofs.close();
      delete [] StatisticsPerRank;
    }

  MPI_Barrier( MPI_COMM_WORLD );

}

//------------------------------------------------------------------------------
// Description:
// Writes data
void WriteData( double *data, const int N, const int rank )
{
  assert( "pre: data array is NULL" && (data != NULL) );

  double start, end;
  int fd;
  std::ostringstream oss;
  oss << "posixdata_shared.dat";

  // STEP 0: Open File
  start  = MPI_Wtime();
  // Let process 0 create the file if it does not exist
  if( rank == 0 )
    {
      fd = open(oss.str().c_str(),O_WRONLY|O_CREAT|O_TRUNC,S_IRUSR|S_IWUSR);
      assert( "Cannot opent file to write data!" && (fd >= 0) );
    }
  MPI_Barrier( MPI_COMM_WORLD );

  // Everyone else opens the file with write-only
  if( rank != 0 )
    {
      fd = open( oss.str().c_str(),O_WRONLY);
      assert( "Cannot opent file to write data!" && (fd >= 0) );
    }
  end    = MPI_Wtime();
  Program.statistics[ POSIX_OPEN_WRITE ] = end-start;

  // STEP 1: Write Data
  size_t bytesize    = sizeof(double)*N;
  start              = MPI_Wtime();
  int rc = lseek(fd, rank*N*sizeof(double), SEEK_SET);
  assert( "Error seeking to file" && (rc >= 0) );
  size_t bytesWriten = write(fd,data,bytesize);
  end                = MPI_Wtime();
  assert( "Failed writing to file!" && (bytesWriten==bytesize) );
  Program.statistics[ POSIX_WRITE ]= end-start;

  // STEP 2: Close file
  start = MPI_Wtime();
  close( fd );
  end   = MPI_Wtime();
  Program.statistics[ POSIX_CLOSE_WRITE ] = end-start;
}

//------------------------------------------------------------------------------
// Description:
// Reads data
void ReadData( double *data, const int N, const int rank )
{
  assert( "pre: data array is NULL" && (data != NULL) );

  double start, end;
  std::ostringstream oss;
  oss << "posixdata_shared.dat";

  // STEP 0: Open File
  start  = MPI_Wtime();
  int fd = open(oss.str().c_str(), O_RDONLY );
  end    = MPI_Wtime();
  assert( "Cannot open file to read data!" && (fd >= 0) );
  Program.statistics[ POSIX_OPEN_READ ] = end-start;

  // STEP 1: Read data
  size_t bytesize    = sizeof(double)*N;
  start              = MPI_Wtime();
  int rc = lseek(fd, rank*N*sizeof(double), SEEK_SET);
  assert( "Error seeking to file" && (rc >= 0) );
  size_t bytesRead   = read(fd,data,bytesize);
  end                = MPI_Wtime();
  assert( "Failed writing to file!" && (bytesRead==bytesize) );
  Program.statistics[ POSIX_READ ]= end-start;

  // STEP 2: Close file
  start = MPI_Wtime();
  close( fd );
  end   = MPI_Wtime();
  Program.statistics[ POSIX_CLOSE_READ ] = end-start;
}

//------------------------------------------------------------------------------
// Description:
// Initializes the data to be written
void CheckData( double *a, double *b, const int N )
{
  assert( "pre: array is NULL" && (a != NULL) );
  assert( "pre: array is NULL" && (b != NULL) );

  for( int i=0; i < N; ++i )
    {
      assert( a[ i ] == b[ i ] );
    } // END for all data

}

//------------------------------------------------------------------------------
// Description:
// Initializes the data to be written
void InitializeData( double *data, const int N, const int rank )
{
  assert( "pre: data != NULL" && (data != NULL) );

  for( int i=0; i < N; ++i )
   data[ i ] = rank*10+i;
}

//------------------------------------------------------------------------------
// Description:
// Logs message to STDOUT
void Log( const char *msg )
{
  if( Program.Rank == 0 )
    {
      std::cout << msg << std::endl;
      std::cout.flush();
    }
  MPI_Barrier( MPI_COMM_WORLD );
}