<div dir="ltr"><div dir="ltr"><div><font size="1">Hello, </font></div><div><font size="1"><br></font></div><div><font size="1">i'm using mpich2 in a C program that reads several buffers from a data text files. <br>I noticed that even when i free the buffer at the end, the global memory usage still increase but the memory usage per process still the same. <br>
I changed the MPI_File_read_at() with fseek/fread and the problem has dissapeared. </font></div><div><font size="1">I'm wondering if that i should do some extra instriuctions to avoid that because it seems that mpi_read allocate an intermediate buffer for reading (sieving optimization maybe) from the user space memory and the OS ( windosws 7) don't get it back. <br>
i'm using MPICH2 1.4 on w windows 7 with NTFS file system.</font></div><div><font size="1"><br></font></div><div><font size="1">Thank you for your help. </font></div><div><font size="1"><br></font></div><div><font size="1"><br>
</font></div><div><font size="1">here is my code : </font></div><div><font size="1"><div>#include <stdio.h></div><div>#include <stdlib.h></div><div>#include <mpi.h></div><div>#include <omp.h></div>
<div>#include <string.h></div><div>#include <time.h></div><div>#include "queue.h"</div><div>#include "KhiopsNativeInterface.h"</div><div>#include <windows.h></div><div><br></div><div>
#define maxLength 100000</div><div><br></div><div>double CPUtime(){ return ((double) clock())/CLOCKS_PER_SEC;}</div><div><br></div><div><br></div><div>int main(int argc, char* argv []){</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if(argc != 5) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("\t[Dictionary file] [Dictionary] [Input file] [Buffer size]\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>exit(0);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><br></div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>char* sDictionaryFileName =<span class="Apple-tab-span" style="white-space:pre"> </span>argv[1];</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>char* sDictionaryName<span class="Apple-tab-span" style="white-space:pre"> </span>= argv[2];</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>char* filename<span class="Apple-tab-span" style="white-space:pre"> </span>= <span class="Apple-tab-span" style="white-space:pre"> </span>argv[3];</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int Mbuffer<span class="Apple-tab-span" style="white-space:pre"> </span>= atoi(argv[4]);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int maxBuffer = Mbuffer*1024*1024;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int over <span class="Apple-tab-span" style="white-space:pre"> </span> = 10000;</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int rank,numprocess;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>long int offset;</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>char* buffer;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>char* opbuffer;</div>
<div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>double tstart=CPUtime();</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Init( &argc, &argv );</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Comm_rank( MPI_COMM_WORLD, &rank );</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* mpi version */</div><div>
<span class="Apple-tab-span" style="white-space:pre"> </span>/* open the file*/</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_File fh;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int err;</div>
<div> err = MPI_File_open(MPI_COMM_WORLD, filename, MPI_MODE_RDONLY, MPI_INFO_NULL, &fh);</div><div> if (err != MPI_SUCCESS) {</div><div> char errstr[MPI_MAX_ERROR_STRING];</div><div> int errlen;</div>
<div> MPI_Error_string(err, errstr, &errlen);</div><div> printf("Error at opening file %s (%s)\n",filename,errstr);</div><div> MPI_Finalize();</div><div> exit(1);</div><div> }</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>// get offsets and buffer size</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Offset sfile;</div><div> MPI_File_get_size(fh,&sfile);</div>
<div> MPI_Status status;</div><div><br></div><div><br></div><div> /* C version */</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/*FILE* fh;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>long int sfile;</div>
<div> fh =fopen( filename,"rb");</div><div> if (fh==NULL) {</div><div> printf("Error at opening file %s\n",filename);</div><div> exit(1);</div><div> }</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>// get offsets and buffer size</div>
<div> fseek(fh, 0L, SEEK_END);</div><div> sfile = ftell(fh);</div><div> fseek(fh, 0L, SEEK_SET);*/</div><div><br></div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Comm_size( MPI_COMM_WORLD, &numprocess );</div>
<div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* number of iterations */</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>long int data_size = (long int)(sfile/(numprocess));</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int nbIter = data_size/maxBuffer;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if(nbIter<=1){</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>nbIter = 1;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>maxBuffer = data_size;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* offsets */</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>offset = data_size*(rank);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>long int cursor = offset;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>char* header;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if(rank==0){</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>FILE* fh;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span> fh =fopen( filename,"rb");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span> if (fh==NULL) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span> printf("Error at opening file %s\n",filename);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span> exit(1);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span> }</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* read the header and broadcast it */</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>header = malloc(sizeof(char)*1000);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fgets(header,1000,fh);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fclose(fh);</div>
<div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>//broadcast header</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int sndHeader = strlen(header);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>//cursor+=sndHeader;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int process_counter;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>for(process_counter=1;process_counter<numprocess;process_counter++){</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int ierr = MPI_Send(&sndHeader,sizeof(int), MPI_INT, process_counter, 42,MPI_COMM_WORLD);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if (ierr != MPI_SUCCESS) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int errclass,resultlen;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>char err_buffer[MPI_MAX_ERROR_STRING];</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Error_class(ierr,&errclass);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if (errclass== MPI_ERR_RANK) {</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr,"Invalid rank used in MPI send call\n");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Error_string(ierr,err_buffer,&resultlen);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fprintf(stderr,err_buffer);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Finalize();</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Send(header, sndHeader, MPI_CHAR, process_counter, 43, MPI_COMM_WORLD);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>else{</div><div> <span class="Apple-tab-span" style="white-space:pre"> </span>/* receive the header */</div>
<div> <span class="Apple-tab-span" style="white-space:pre"> </span>int sizeofHeader;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Status s ;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Recv(&sizeofHeader,sizeof(int),MPI_INT,0,42,MPI_COMM_WORLD,&s);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>header = malloc (sizeof(char)*sizeofHeader+1);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Recv(header,sizeofHeader,MPI_CHAR,0,43,MPI_COMM_WORLD,&s);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><br></div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* Synchronization barrier */</div><div>
<span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Barrier(MPI_COMM_WORLD);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int count;</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>opbuffer = malloc(sizeof(char)*maxBuffer);</div>
<div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* C version */</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>//fseek(fh,cursor,SEEK_SET);</div><div><br></div><div>
<span class="Apple-tab-span" style="white-space:pre"> </span>for(count=0;count<nbIter;count++){</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if(count==0 && rank==numprocess-1){ //init ring</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>//send the token to p0</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int token=1;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Send(&token,sizeof(int),MPI_INT,0,55,MPI_COMM_WORLD);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>//recv</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int token;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int sender;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if(rank==0)</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>sender = numprocess-1;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>else</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>sender=rank-1;</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Status s;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Recv(&token,sizeof(int),MPI_INT,sender,55,MPI_COMM_WORLD,&s);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>fflush(stdout);printf("P%d got the token at %G\n",rank,CPUtime());</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>//read</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>double start=CPUtime();</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/*double readtime;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>double sread=CPUtime();//read time*/</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>//read</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if(token==1){</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>/* MPI version */</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int err=MPI_File_read_at(fh, cursor,opbuffer, sizeof(char)*maxBuffer, MPI_CHAR, &status);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if(err!=MPI_SUCCESS){</div><div><span class="Apple-tab-span" style="white-space:pre"> </span> char errstr[MPI_MAX_ERROR_STRING];</div><div><span class="Apple-tab-span" style="white-space:pre"> </span> int errlen;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span> MPI_Error_string(err, errstr, &errlen);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span> printf("Error reading file %s (%s)\n",filename,errstr);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Finalize();</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>exit(0);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* C version of read */</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/*int k=fread(opbuffer,sizeof(char),maxBuffer,fh);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if(k==0)</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>perror("fread");*/</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>cursor+=maxBuffer;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>buffer=opbuffer;</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>else{</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Error token!\n");</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>token=1;</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>//printf("P%d readtime=%G\n",rank,CPUtime()-sread);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>//Isend</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>int next = (rank+1)%numprocess;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Send(&token,sizeof(int),MPI_INT,next,55,MPI_COMM_WORLD);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>//i perform processing buffer here! </div>
<div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>free(opbuffer);</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* mpi version */</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int er=MPI_File_close(&fh);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>if(er!=MPI_SUCCESS){</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Error closing file\n");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Finalize();</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>exit(1);</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>/* c version */</div><div><span class="Apple-tab-span" style="white-space:pre"> </span>//fclose(fh);</div><div><br></div><div><br></div><div>
<span class="Apple-tab-span" style="white-space:pre"> </span>MPI_Finalize();</div><div><br></div><div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Global time : %G\n",CPUtime()-tstart);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>return 0;</div><div>}</div><div><br></div></font></div><div><font size="1"><br></font></div><div><font size="1">Best Regards</font></div></div></div><br>