#include "mpi.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <errno.h>

#define ROWS                (32768)
#define COLS                (32768)
#define MPI_DATATYPE        (MPI_DOUBLE)
#define C_DATATYPE          double
#define DIMS                (2)
#define COLL_BUFSIZE        (8388608)

#define MPI_ERR_CHECK(error_code) if((error_code) != MPI_SUCCESS) { \
                                    MPI_Error_string(error_code, string, &len); \
                                    fprintf(stderr, "file: %s, func: %s, line:%d error_code: %s\n", __FILE__, __func__, __LINE__, string); \
                                    return error_code; \
                                  }


int main(int argc, char **argv) {
  char          string[MPI_MAX_ERROR_STRING] = {'\0'}, fname[] = "pvfs2:/home/mdl/patrick/pvfs2/testfile",
	        stripe_unit_str[MPI_MAX_INFO_VAL] = {'\0'}, stripe_factor_str[MPI_MAX_INFO_VAL] = {'\0'};
  int           i = 0, j = 0, k = 0, l = 0, nprocs = 0, mynod = 0, error_code = 0, provided = 0, c_size = 0, mpi_size = 0,
		ret = 0, len = 0, incr_cnt = 0, iterations = 0,
	        array_size[] = {0, 0}, array_subsize[] = {0, 0}, array_start[] = {0, 0},
	        stripe_unit = 0, stripe_factor = 0;
  long          rows = 0l, cols = 0l, coll_bufsize = 0l, rows_collbuf = 0l, cols_collbuf = 0l, elts_collbuf = 0l;
  unsigned long filesize = 0l;
  double        *buffer = NULL, count = 0.0;
  MPI_File      fhandle;
  MPI_Status    status;
  MPI_Datatype  subarray;
  MPI_Info      file_info;

  MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
  MPI_Comm_rank(MPI_COMM_WORLD, &mynod);
  MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
  MPI_Type_size(MPI_DATATYPE, &mpi_size);
  c_size     = sizeof(C_DATATYPE);
  if(c_size != mpi_size) {
    fprintf(stderr, "Datatypes in MPI and C do not match\n");
    MPI_Abort(MPI_COMM_WORLD, EIO);
  }

  rows             = ROWS;
  cols             = COLS;
  coll_bufsize     = COLL_BUFSIZE;
  cols_collbuf     = (cols / nprocs);
  rows_collbuf     = (coll_bufsize / (cols_collbuf * mpi_size));
  elts_collbuf     = (coll_bufsize / mpi_size);
  filesize         = (rows * cols * mpi_size);
  array_size[0]    = rows;
  array_size[1]    = cols;
  array_subsize[0] = rows;
  array_subsize[1] = cols_collbuf;
  array_start[0]   = 0;
  array_start[1]   = cols_collbuf * mynod;

  if(0 == mynod)
    fprintf(stdout, "Filesize         = %lu\n", filesize);

  fprintf(stdout, "rows             = %ld\n"
                  "cols             = %ld\n"
		  "Element size     = %d\n"
		  "Filesize         = %lu\n"
		  "Coll buf size    = %ld\n"
		  "Rows in coll buf = %ld\n"
		  "Cols in coll buf = %ld\n"
                  "array_size[]     = {%d, %d}\n"
                  "array_subsize[]  = {%d, %d}\n"
		  "array_start[]    = {%d, %d}\n",
		  rows, cols, mpi_size, filesize, coll_bufsize, rows_collbuf, cols_collbuf,
		  array_size[0], array_size[1], array_subsize[0], array_subsize[1], array_start[0], array_start[1]);

  buffer = (C_DATATYPE *)calloc(elts_collbuf, sizeof(C_DATATYPE));
  if(!buffer) {
    fprintf(stderr, "calloc error\n");
    MPI_Abort(MPI_COMM_WORLD, ENOMEM);
  }

  MPI_Info_create(&file_info);

  if(argc > 1) {
    stripe_unit = atoi(argv[1]);
    sprintf(stripe_unit_str, "%d", stripe_unit);
    MPI_Info_set(file_info, "striping_unit", stripe_unit_str);
  }

  if(argc > 2) {
    stripe_factor = atoi(argv[2]);
    sprintf(stripe_factor_str, "%d", stripe_factor);
    MPI_Info_set(file_info, "striping_factor", stripe_factor_str);
  }

  MPI_File_open(MPI_COMM_WORLD, fname, MPI_MODE_CREATE | MPI_MODE_WRONLY, file_info, &fhandle);

  MPI_Type_create_subarray(DIMS, array_size, array_subsize, array_start, MPI_ORDER_C, MPI_DATATYPE, &subarray);
  MPI_Type_commit(&subarray);
  MPI_File_set_view(fhandle, 0, MPI_DATATYPE, subarray, "native", MPI_INFO_NULL);

  incr_cnt = (nprocs - 1) * cols_collbuf;
  count = mynod * cols_collbuf;
  iterations = rows / rows_collbuf;

  for(i = 0; i < iterations; i++) {
    for (j = 0, l = 0; j < rows_collbuf; j++) {
      for(k = 0; k < cols_collbuf; k++)
	buffer[l++] = (double)count++;
      count += incr_cnt;
    }
    error_code = MPI_File_write_all(fhandle, buffer, elts_collbuf, MPI_DATATYPE, &status); MPI_ERR_CHECK(error_code);
  }

  free(buffer);
  MPI_File_close(&fhandle);
  MPI_Finalize();

  return 0;
}