[Darshan-commits] [Git][darshan/darshan][lustre-mod] 5 commits: add stubbed out lustre source and header files
Shane Snyder
xgitlab at cels.anl.gov
Thu Feb 25 10:09:04 CST 2016
Shane Snyder pushed to branch lustre-mod at darshan / darshan
Commits:
62ed332a by Shane Snyder at 2016-02-23T16:55:27-06:00
add stubbed out lustre source and header files
- - - - -
1eecb3b1 by Shane Snyder at 2016-02-23T16:55:27-06:00
clean up headers for some modules
- - - - -
ffeeed89 by Shane Snyder at 2016-02-23T16:56:24-06:00
some initial luste mod logic and makefile changes
- - - - -
21c1b5d0 by Shane Snyder at 2016-02-24T22:27:10-06:00
cleanup bgq module to use fs_info struct
- - - - -
657c5b56 by Shane Snyder at 2016-02-24T22:28:58-06:00
initial runtime lustre module implementation
the POSIX module will call into the Lustre module for each file
record created that corresponds to a file from a Lustre FS. No
counters are currently stored but most of the module strucutre
has been stubbed.
- - - - -
13 changed files:
- darshan-bgq-log-format.h
- darshan-log-format.h
- + darshan-lustre-log-format.h
- darshan-runtime/Makefile.in
- darshan-runtime/darshan.h
- darshan-runtime/lib/darshan-bgq.c
- darshan-runtime/lib/darshan-core-init-finalize.c
- darshan-runtime/lib/darshan-core.c
- darshan-runtime/lib/darshan-hdf5.c
- + darshan-runtime/lib/darshan-lustre.c
- darshan-runtime/lib/darshan-mpiio.c
- darshan-runtime/lib/darshan-pnetcdf.c
- darshan-runtime/lib/darshan-posix.c
Changes:
=====================================
darshan-bgq-log-format.h
=====================================
--- a/darshan-bgq-log-format.h
+++ b/darshan-bgq-log-format.h
@@ -69,7 +69,6 @@ struct darshan_bgq_record
{
darshan_record_id f_id;
int64_t rank;
- int alignment;
int64_t counters[BGQ_NUM_INDICES];
double fcounters[BGQ_F_NUM_INDICES];
};
=====================================
darshan-log-format.h
=====================================
--- a/darshan-log-format.h
+++ b/darshan-log-format.h
@@ -103,6 +103,7 @@ struct darshan_record
#include "darshan-hdf5-log-format.h"
#include "darshan-pnetcdf-log-format.h"
#include "darshan-bgq-log-format.h"
+#include "darshan-lustre-log-format.h"
/* X-macro for keeping module ordering consistent */
/* NOTE: first val used to define module enum values,
@@ -120,7 +121,8 @@ struct darshan_record
X(DARSHAN_MPIIO_MOD, "MPI-IO", DARSHAN_MPIIO_VER, &mpiio_logutils) \
X(DARSHAN_HDF5_MOD, "HDF5", DARSHAN_HDF5_VER, &hdf5_logutils) \
X(DARSHAN_PNETCDF_MOD, "PNETCDF", DARSHAN_PNETCDF_VER, &pnetcdf_logutils) \
- X(DARSHAN_BGQ_MOD, "BG/Q", DARSHAN_BGQ_VER, &bgq_logutils)
+ X(DARSHAN_BGQ_MOD, "BG/Q", DARSHAN_BGQ_VER, &bgq_logutils) \
+ X(DARSHAN_LUSTRE_MOD, "LUSTRE", DARSHAN_LUSTRE_VER, NULL)
/* unique identifiers to distinguish between available darshan modules */
/* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]
=====================================
darshan-lustre-log-format.h
=====================================
--- /dev/null
+++ b/darshan-lustre-log-format.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#ifndef __DARSHAN_LUSTRE_LOG_FORMAT_H
+#define __DARSHAN_LUSTRE_LOG_FORMAT_H
+
+/* current Lustre log format version */
+#define DARSHAN_LUSTRE_VER 1
+
+/* TODO: add integer counters here (e.g., counter for stripe width, stripe size, etc etc) */
+#define LUSTRE_COUNTERS \
+ /* end of counters */\
+ X(LUSTRE_NUM_INDICES)
+
+#define X(a) a,
+/* integer statistics for Lustre file records */
+enum darshan_lustre_indices
+{
+ LUSTRE_COUNTERS
+};
+#undef X
+
+/* record structure for the Lustre module. a record is created and stored for
+ * every file opened that belongs to a Lustre file system. This record includes:
+ * - a corresponding record identifier (created by hashing the file path)
+ * - the rank of the process which opened the file (-1 for shared files)
+ * - integer file I/O statistics (stripe size, width, # of OSTs, etc.)
+ */
+struct darshan_lustre_record
+{
+ darshan_record_id rec_id;
+ int64_t rank;
+ int64_t counters[LUSTRE_NUM_INDICES];
+};
+
+#endif /* __DARSHAN_LUSTRE_LOG_FORMAT_H */
+
=====================================
darshan-runtime/Makefile.in
=====================================
--- a/darshan-runtime/Makefile.in
+++ b/darshan-runtime/Makefile.in
@@ -36,6 +36,10 @@ LIBS = -lz @LIBBZ2@
DARSHAN_STATIC_MOD_OBJS = lib/darshan-posix.o lib/darshan-mpiio.o lib/darshan-hdf5.o lib/darshan-pnetcdf.o
DARSHAN_DYNAMIC_MOD_OBJS = lib/darshan-posix.po lib/darshan-mpiio.po lib/darshan-hdf5.po lib/darshan-pnetcdf.po
+# TODO: make the lustre module enabled using config options
+DARSHAN_STATIC_MOD_OBJS += lib/darshan-lustre.o
+DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-lustre.po
+
ifdef DARSHAN_USE_BGQ
DARSHAN_STATIC_MOD_OBJS += lib/darshan-bgq.o
DARSHAN_DYNAMIC_MOD_OBJS += lib/darshan-bgq.po
@@ -49,63 +53,69 @@ lib::
lib/darshan-core-init-finalize.o: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-core-init-finalize.po: lib/darshan-core-init-finalize.c darshan.h darshan-dynamic.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-core.o: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-core.po: lib/darshan-core.c darshan.h darshan-dynamic.h darshan-core.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
-lib/darshan-common.o: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-common.o: lib/darshan-common.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-common.po: lib/darshan-common.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
+lib/darshan-common.po: lib/darshan-common.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
-lib/darshan-null.o: lib/darshan-null.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
+lib/darshan-null.o: lib/darshan-null.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-null.po: lib/darshan-null.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
+lib/darshan-null.po: lib/darshan-null.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-null-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
-lib/darshan-posix.o: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
+lib/darshan-posix.o: lib/darshan-posix.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-posix.po: lib/darshan-posix.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
+lib/darshan-posix.po: lib/darshan-posix.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-posix-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
-lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
+lib/darshan-mpiio.o: lib/darshan-mpiio.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
+lib/darshan-mpiio.po: lib/darshan-mpiio.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
-lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
+lib/darshan-bgq.o: lib/darshan-bgq.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
+lib/darshan-bgq.po: lib/darshan-bgq.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-mpiio-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
-lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
+lib/darshan-hdf5.o: lib/darshan-hdf5.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
+lib/darshan-hdf5.po: lib/darshan-hdf5.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-hdf5-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-hdf5-stubs.o: lib/darshan-hdf5-stubs.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
+lib/darshan-pnetcdf.o: lib/darshan-pnetcdf.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
$(CC) $(CFLAGS) -c $< -o $@
-lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
+lib/darshan-pnetcdf.po: lib/darshan-pnetcdf.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-pnetcdf-log-format.h | lib
$(CC) $(CFLAGS_SHARED) -c $< -o $@
lib/darshan-pnetcdf-stubs.o: lib/darshan-pnetcdf-stubs.c darshan.h $(DARSHAN_LOG_FORMAT) | lib
$(CC) $(CFLAGS) -c $< -o $@
+lib/darshan-lustre.o: lib/darshan-lustre.c darshan.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-lustre-log-format.h | lib
+ $(CC) $(CFLAGS) -c $< -o $@
+
+lib/darshan-lustre.po: lib/darshan-hdf5.c darshan.h darshan-dynamic.h darshan-common.h $(DARSHAN_LOG_FORMAT) $(srcdir)/../darshan-lustre-log-format.h | lib
+ $(CC) $(CFLAGS_SHARED) -c $< -o $@
+
lib/lookup3.o: lib/lookup3.c
$(CC) $(CFLAGS) -c $< -o $@
=====================================
darshan-runtime/darshan.h
=====================================
--- a/darshan-runtime/darshan.h
+++ b/darshan-runtime/darshan.h
@@ -80,6 +80,13 @@ struct darshan_module_funcs
void (*shutdown)(void);
};
+/* stores FS info from statfs calls for a given mount point */
+struct darshan_fs_info
+{
+ int fs_type;
+ int block_size;
+};
+
/* paths that darshan will not trace */
extern char* darshan_path_exclusions[]; /* defined in lib/darshan-core.c */
@@ -134,7 +141,7 @@ void darshan_core_register_record(
int printable_flag,
int mod_limit_flag,
darshan_record_id *rec_id,
- int *file_alignment);
+ struct darshan_fs_info *fs_info);
/* darshan_core_unregister_record()
*
=====================================
darshan-runtime/lib/darshan-bgq.c
=====================================
--- a/darshan-runtime/lib/darshan-bgq.c
+++ b/darshan-runtime/lib/darshan-bgq.c
@@ -48,7 +48,6 @@ static int instrumentation_disabled = 0;
/* my_rank indicates the MPI rank of this process */
static int my_rank = -1;
-static int darshan_mem_alignment = 1;
/* internal helper functions for the BGQ module */
void bgq_runtime_initialize(void);
@@ -127,7 +126,7 @@ void bgq_runtime_initialize()
&bgq_mod_fns,
&my_rank,
&mem_limit,
- &darshan_mem_alignment);
+ NULL);
/* return if no memory assigned by darshan-core */
if(mem_limit == 0)
@@ -162,7 +161,7 @@ void bgq_runtime_initialize()
1,
0,
&bgq_runtime->record.f_id,
- &bgq_runtime->record.alignment);
+ NULL);
/* if record is set to 0, darshan-core is out of space and will not
* track this record, so we should avoid tracking it, too
=====================================
darshan-runtime/lib/darshan-core-init-finalize.c
=====================================
--- a/darshan-runtime/lib/darshan-core-init-finalize.c
+++ b/darshan-runtime/lib/darshan-core-init-finalize.c
@@ -15,6 +15,7 @@
#include "darshan.h"
#include "darshan-core.h"
+#include "darshan-dynamic.h"
#ifdef DARSHAN_PRELOAD
=====================================
darshan-runtime/lib/darshan-core.c
=====================================
--- a/darshan-runtime/lib/darshan-core.c
+++ b/darshan-runtime/lib/darshan-core.c
@@ -79,9 +79,9 @@ void (*mod_static_init_fns[])(void) =
#define DARSHAN_MAX_MNT_TYPE 32
struct mnt_data
{
- int block_size;
char path[DARSHAN_MAX_MNT_PATH];
char type[DARSHAN_MAX_MNT_TYPE];
+ struct darshan_fs_info fs_info;
};
static struct mnt_data mnt_data_array[DARSHAN_MAX_MNTS];
static int mnt_data_count = 0;
@@ -96,8 +96,8 @@ static void darshan_get_exe_and_mounts_root(
int space_left);
static char* darshan_get_exe_and_mounts(
struct darshan_core_runtime *core);
-static void darshan_block_size_from_path(
- const char *path, int *block_size);
+static void darshan_fs_info_from_path(
+ const char *path, struct darshan_fs_info *fs_info);
static void darshan_get_shared_records(
struct darshan_core_runtime *core, darshan_record_id **shared_recs,
int *shared_rec_cnt);
@@ -952,12 +952,13 @@ static void add_entry(char* trailing_data, int* space_left, struct mntent *entry
#define LL_SUPER_MAGIC 0x0BD00BD0
#endif
ret = statfs(entry->mnt_dir, &statfsbuf);
+ mnt_data_array[mnt_data_count].fs_info.fs_type = statfsbuf.f_type;
if(ret == 0 && statfsbuf.f_type != LL_SUPER_MAGIC)
- mnt_data_array[mnt_data_count].block_size = statfsbuf.f_bsize;
+ mnt_data_array[mnt_data_count].fs_info.block_size = statfsbuf.f_bsize;
else if(ret == 0 && statfsbuf.f_type == LL_SUPER_MAGIC)
- mnt_data_array[mnt_data_count].block_size = 1024*1024;
+ mnt_data_array[mnt_data_count].fs_info.block_size = 1024*1024;
else
- mnt_data_array[mnt_data_count].block_size = 4096;
+ mnt_data_array[mnt_data_count].fs_info.block_size = 4096;
/* store mount information for use in header of darshan log */
ret = snprintf(tmp_mnt, 256, "\n%s\t%s",
@@ -1097,16 +1098,17 @@ static char* darshan_get_exe_and_mounts(struct darshan_core_runtime *core)
return(trailing_data);
}
-static void darshan_block_size_from_path(const char *path, int *block_size)
+static void darshan_fs_info_from_path(const char *path, struct darshan_fs_info *fs_info)
{
int i;
- *block_size = -1;
+ fs_info->fs_type = -1;
+ fs_info->block_size = -1;
for(i=0; i<mnt_data_count; i++)
{
if(!(strncmp(mnt_data_array[i].path, path, strlen(mnt_data_array[i].path))))
{
- *block_size = mnt_data_array[i].block_size;
+ *fs_info = mnt_data_array[i].fs_info;
return;
}
}
@@ -1695,7 +1697,7 @@ void darshan_core_register_record(
int printable_flag,
int mod_limit_flag,
darshan_record_id *rec_id,
- int *file_alignment)
+ struct darshan_fs_info *fs_info)
{
darshan_record_id tmp_rec_id;
struct darshan_core_record_ref *ref;
@@ -1743,10 +1745,10 @@ void darshan_core_register_record(
DARSHAN_MOD_FLAG_SET(ref->mod_flags, mod_id);
DARSHAN_CORE_UNLOCK();
- if(file_alignment)
- darshan_block_size_from_path(name, file_alignment);
-
*rec_id = tmp_rec_id;
+ if(fs_info)
+ darshan_fs_info_from_path(name, fs_info);
+
return;
}
=====================================
darshan-runtime/lib/darshan-hdf5.c
=====================================
--- a/darshan-runtime/lib/darshan-hdf5.c
+++ b/darshan-runtime/lib/darshan-hdf5.c
@@ -4,6 +4,9 @@
*
*/
+#define _XOPEN_SOURCE 500
+#define _GNU_SOURCE
+
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
@@ -16,7 +19,6 @@
#include <errno.h>
#include <search.h>
#include <assert.h>
-#define __USE_GNU
#include <pthread.h>
#include "uthash.h"
=====================================
darshan-runtime/lib/darshan-lustre.c
=====================================
--- /dev/null
+++ b/darshan-runtime/lib/darshan-lustre.c
@@ -0,0 +1,223 @@
+/*
+ * Copyright (C) 2015 University of Chicago.
+ * See COPYRIGHT notice in top-level directory.
+ *
+ */
+
+#define _XOPEN_SOURCE 500
+#define _GNU_SOURCE
+
+#include "darshan-runtime-config.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <pthread.h>
+
+#include "uthash.h"
+
+#include "darshan.h"
+#include "darshan-dynamic.h"
+
+/* TODO: once mmap merge is complete, we can just use an array
+ * to store record data rather than a hash table -- in that
+ * branch, register_record() returns whether the record
+ * already exists, at which point we won't need to instrument
+ * more data, since the Lustre record data is immutable. records
+ * could just be appended to the array if there is no need to
+ * search for a specific record.
+ */
+struct lustre_record_ref
+{
+ struct darshan_lustre_record *record;
+ UT_hash_handle hlink;
+};
+
+struct lustre_runtime
+{
+ struct lustre_record_ref *ref_array;
+ struct darshan_lustre_record *record_array;
+ int record_array_size;
+ int record_array_ndx;
+ struct lustre_record_ref *record_hash;
+};
+
+static struct lustre_runtime *lustre_runtime = NULL;
+static pthread_mutex_t lustre_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
+static int instrumentation_disabled = 0;
+static int my_rank = -1;
+
+static void lustre_runtime_initialize(void);
+
+static void lustre_begin_shutdown(void);
+static void lustre_get_output_data(MPI_Comm mod_comm, darshan_record_id *shared_recs,
+ int shared_rec_count, void **lustre_buf, int *lustre_buf_sz);
+static void lustre_shutdown(void);
+
+#define LUSTRE_LOCK() pthread_mutex_lock(&lustre_runtime_mutex)
+#define LUSTRE_UNLOCK() pthread_mutex_unlock(&lustre_runtime_mutex)
+
+/* TODO: is there any way we can further compact Lustre data to save space?
+ * e.g., are all files in the same directory guaranteed same striping parameters?
+ * if so, can we store stripe parameters on per-directory basis and the OST
+ * list on a per-file basis? maybe the storage savings are small enough this isn't
+ * worth it, but nice to keep in mind
+ */
+
+void darshan_instrument_lustre_file(char *filepath)
+{
+ struct lustre_record_ref *lustre_ref;
+ darshan_record_id rec_id;
+ int limit_flag;
+
+ LUSTRE_LOCK();
+ /* make sure the lustre module is already initialized */
+ lustre_runtime_initialize();
+
+ limit_flag = (lustre_runtime->record_array_ndx >= lustre_runtime->record_array_size);
+
+ /* register a Lustre file record with Darshan */
+ darshan_core_register_record(
+ (void *)filepath,
+ strlen(filepath),
+ DARSHAN_LUSTRE_MOD,
+ 1,
+ limit_flag,
+ &rec_id,
+ NULL);
+
+ /* if record id is 0, darshan has no more memory for instrumenting */
+ if(rec_id == 0)
+ return;
+
+ HASH_FIND(hlink, lustre_runtime->record_hash, &rec_id,
+ sizeof(darshan_record_id), lustre_ref);
+ if(!lustre_ref)
+ {
+ /* no existing record, allocate a new one and add it to the hash */
+ lustre_ref = &(lustre_runtime->ref_array[lustre_runtime->record_array_ndx]);
+ lustre_ref->record = &(lustre_runtime->record_array[lustre_runtime->record_array_ndx]);
+ lustre_ref->record->rec_id = rec_id;
+ lustre_ref->record->rank = my_rank;
+
+ /* TODO: gather lustre data, store in record hash */
+ /* counters in lustre_ref->record->counters */
+
+ HASH_ADD(hlink, lustre_runtime->record_hash, record->rec_id,
+ sizeof(darshan_record_id), lustre_ref);
+ lustre_runtime->record_array_ndx++;
+ }
+
+ LUSTRE_UNLOCK();
+ return;
+}
+
+static void lustre_runtime_initialize()
+{
+ int mem_limit;
+ struct darshan_module_funcs lustre_mod_fns =
+ {
+ .begin_shutdown = &lustre_begin_shutdown,
+ .get_output_data = &lustre_get_output_data,
+ .shutdown = &lustre_shutdown
+ };
+
+ /* don't do anything if already initialized or instrumenation is disabled */
+ if(lustre_runtime || instrumentation_disabled)
+ return;
+
+ /* register the lustre module with darshan-core */
+ darshan_core_register_module(
+ DARSHAN_LUSTRE_MOD,
+ &lustre_mod_fns,
+ &my_rank,
+ &mem_limit,
+ NULL);
+
+ /* return if no memory assigned by darshan core */
+ if(mem_limit == 0)
+ return;
+
+ lustre_runtime = malloc(sizeof(*lustre_runtime));
+ if(!lustre_runtime)
+ return;
+ memset(lustre_runtime, 0, sizeof(*lustre_runtime));
+
+ /* allocate array of Lustre records according to the amount of memory
+ * assigned by Darshan
+ */
+ lustre_runtime->record_array_size = mem_limit / sizeof(struct darshan_lustre_record);
+
+ lustre_runtime->ref_array = malloc(lustre_runtime->record_array_size *
+ sizeof(struct lustre_record_ref));
+ lustre_runtime->record_array = malloc(lustre_runtime->record_array_size *
+ sizeof(struct darshan_lustre_record));
+ if(!lustre_runtime->ref_array || !lustre_runtime->record_array)
+ {
+ lustre_runtime->record_array_size = 0;
+ return;
+ }
+ memset(lustre_runtime->ref_array, 0, lustre_runtime->record_array_size *
+ sizeof(struct lustre_record_ref));
+ memset(lustre_runtime->record_array, 0, lustre_runtime->record_array_size *
+ sizeof(struct darshan_lustre_record));
+
+ return;
+}
+
+/**************************************************************************
+ * Functions exported by Lustre module for coordinating with darshan-core *
+ **************************************************************************/
+
+static void lustre_begin_shutdown(void)
+{
+ assert(lustre_runtime);
+
+ LUSTRE_LOCK();
+ /* disable further instrumentation while Darshan shuts down */
+ instrumentation_disabled = 1;
+ LUSTRE_UNLOCK();
+
+ return;
+}
+
+static void lustre_get_output_data(
+ MPI_Comm mod_comm,
+ darshan_record_id *shared_recs,
+ int shared_rec_count,
+ void **lustre_buf,
+ int *lustre_buf_sz)
+{
+ assert(lustre_runtime);
+
+ /* TODO: determine lustre record shared across all processes,
+ * and have only rank 0 write these records out. No shared
+ * reductions should be necessary as the Lustre data for a
+ * given file should be the same on each process
+ */
+
+ return;
+}
+
+static void lustre_shutdown(void)
+{
+ assert(lustre_runtime);
+
+ /* TODO: free data structures */
+
+ lustre_runtime = NULL;
+
+ return;
+}
+
+/*
+ * Local variables:
+ * c-indent-level: 4
+ * c-basic-offset: 4
+ * End:
+ *
+ * vim: ts=8 sts=4 sw=4 expandtab
+ */
=====================================
darshan-runtime/lib/darshan-mpiio.c
=====================================
--- a/darshan-runtime/lib/darshan-mpiio.c
+++ b/darshan-runtime/lib/darshan-mpiio.c
@@ -4,6 +4,9 @@
*
*/
+#define _XOPEN_SOURCE 500
+#define _GNU_SOURCE
+
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
@@ -17,7 +20,6 @@
#include <errno.h>
#include <search.h>
#include <assert.h>
-#define __USE_GNU
#include <pthread.h>
#include "uthash.h"
=====================================
darshan-runtime/lib/darshan-pnetcdf.c
=====================================
--- a/darshan-runtime/lib/darshan-pnetcdf.c
+++ b/darshan-runtime/lib/darshan-pnetcdf.c
@@ -4,6 +4,9 @@
*
*/
+#define _XOPEN_SOURCE 500
+#define _GNU_SOURCE
+
#include "darshan-runtime-config.h"
#include <stdio.h>
#include <unistd.h>
@@ -16,7 +19,6 @@
#include <errno.h>
#include <search.h>
#include <assert.h>
-#define __USE_GNU
#include <pthread.h>
#include "uthash.h"
=====================================
darshan-runtime/lib/darshan-posix.c
=====================================
--- a/darshan-runtime/lib/darshan-posix.c
+++ b/darshan-runtime/lib/darshan-posix.c
@@ -176,6 +176,11 @@ struct posix_runtime
struct posix_file_runtime_ref* fd_hash;
};
+/* XXX modules don't expose an API for other modules, so use extern to get
+ * Lustre instrumentation function
+ */
+extern void darshan_instrument_lustre_file(char *filepath);
+
static struct posix_runtime *posix_runtime = NULL;
static pthread_mutex_t posix_runtime_mutex = PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP;
static int instrumentation_disabled = 0;
@@ -1505,7 +1510,7 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
struct posix_file_runtime *file = NULL;
char *newname = NULL;
darshan_record_id file_id;
- int file_alignment;
+ struct darshan_fs_info fs_info;
int limit_flag;
if(!posix_runtime || instrumentation_disabled)
@@ -1525,7 +1530,7 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
1,
limit_flag,
&file_id,
- &file_alignment);
+ &fs_info);
/* the file record id is set to 0 if no memory is available for tracking
* new records -- just fall through and ignore this record
@@ -1552,12 +1557,19 @@ static struct posix_file_runtime* posix_file_by_name(const char *name)
file->file_record->f_id = file_id;
file->file_record->rank = my_rank;
file->file_record->counters[POSIX_MEM_ALIGNMENT] = darshan_mem_alignment;
- file->file_record->counters[POSIX_FILE_ALIGNMENT] = file_alignment;
+ file->file_record->counters[POSIX_FILE_ALIGNMENT] = fs_info.block_size;
/* add new record to file hash table */
HASH_ADD(hlink, posix_runtime->file_hash, file_record->f_id, sizeof(darshan_record_id), file);
posix_runtime->file_array_ndx++;
+ /* XXX: check for lustre and call in */
+#ifndef LL_SUPER_MAGIC
+#define LL_SUPER_MAGIC 0x0BD00BD0
+#endif
+ if(fs_info.fs_type == LL_SUPER_MAGIC)
+ darshan_instrument_lustre_file(newname);
+
if(newname != name)
free(newname);
return(file);
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/compare/d14cd7589d20f229ff43daebd79c980677592e54...657c5b565dd8c7b847c956a39c61f8711ce21833
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20160225/33cd9ebb/attachment-0001.html>
More information about the Darshan-commits
mailing list