[Darshan-commits] [Git][darshan/darshan][master] 2 commits: PyDarshan: Record Collections
Shane Snyder
xgitlab at cels.anl.gov
Thu Mar 25 11:13:43 CDT 2021
Shane Snyder pushed to branch master at darshan / darshan
Commits:
92501e87 by Jakob Luettgau at 2021-03-25T11:13:34-05:00
PyDarshan: Record Collections
- - - - -
054de670 by Shane Snyder at 2021-03-25T11:13:34-05:00
Merge branch 'pydarshan-RecordCollection' into 'master'
PyDarshan: Record Collections
See merge request darshan/darshan!86
- - - - -
18 changed files:
- darshan-util/pydarshan/darshan/__init__.py
- + darshan-util/pydarshan/darshan/experimental/aggregators/create_dxttimeline.py
- darshan-util/pydarshan/darshan/experimental/aggregators/name_records_summary.py
- darshan-util/pydarshan/darshan/experimental/aggregators/filter.py → darshan-util/pydarshan/darshan/experimental/operations/filter.py
- darshan-util/pydarshan/darshan/experimental/aggregators/merge.py → darshan-util/pydarshan/darshan/experimental/operations/merge.py
- + darshan-util/pydarshan/darshan/experimental/operations/potential_functions.py.txt
- darshan-util/pydarshan/darshan/experimental/aggregators/reduce.py → darshan-util/pydarshan/darshan/experimental/operations/reduce.py
- darshan-util/pydarshan/darshan/experimental/plots/matplotlib.py
- darshan-util/pydarshan/darshan/report.py
- darshan-util/pydarshan/devel/build-all.sh
- darshan-util/pydarshan/examples/01_darshan-introduction.ipynb
- darshan-util/pydarshan/examples/02_darshan-plotting-matplotlib.ipynb
- darshan-util/pydarshan/examples/04_darshan-low-level-cffi-interface.ipynb
- darshan-util/pydarshan/examples/99-Debugging-PyDarshan-via-Logging-Interface.ipynb
- darshan-util/pydarshan/examples/99_darshan-experimental.ipynb
- darshan-util/pydarshan/examples/99_darshan-experimental_report-algebra.ipynb
- darshan-util/pydarshan/examples/99_darshan-low-level-direct-libdarshanutils-interaction.ipynb
- darshan-util/pydarshan/setup.py
Changes:
=====================================
darshan-util/pydarshan/darshan/__init__.py
=====================================
@@ -33,16 +33,17 @@ def enable_experimental(verbose=False):
import importlib
import darshan
- paths = glob.glob(darshan.__path__[0] + "/experimental/aggregators/*.py")
- for path in paths:
- base = os.path.basename(path)
- name = os.path.splitext(base)[0]
+ for subdir in ['aggregators', 'operations']:
+ paths = glob.glob(darshan.__path__[0] + f"/experimental/{subdir}/*.py")
+ for path in paths:
+ base = os.path.basename(path)
+ name = os.path.splitext(base)[0]
+
+ if name == "__init__":
+ continue
+
+ mod = importlib.import_module(f"darshan.experimental.{subdir}.{name}")
+ setattr(DarshanReport, name, getattr(mod, name))
- if name == "__init__":
- continue
-
- mod = importlib.import_module('darshan.experimental.aggregators.{0}'.format(name))
- setattr(DarshanReport, name, getattr(mod, name))
-
- if verbose:
- print("Added method {} to DarshanReport.".format(name))
+ if verbose:
+ print(f"Added method {mod.__name__} to DarshanReport.")
=====================================
darshan-util/pydarshan/darshan/experimental/aggregators/create_dxttimeline.py
=====================================
@@ -0,0 +1,116 @@
+from darshan.report import *
+
+def create_dxttimeline(self, group_by='rank', mode="append"):
+ """
+ Generate/update a timeline from dxt tracing records of current report.
+
+ Args:
+ group_by (str): By which factor to group entries (default: rank)
+ Allowed Parameters: rank, filename
+ """
+
+
+ self.mod_read_all_dxt_records("DXT_POSIX")
+ self.mod_read_all_dxt_records("DXT_MPIIO")
+
+
+
+ ctx = {'groups': [], 'items': []}
+
+
+ groups = ctx['groups']
+ items = ctx['items']
+
+
+ start_time = datetime.datetime.fromtimestamp( self.data['metadata']['job']['start_time'] )
+
+
+
+ def groupify(rec, mod):
+ for seg in rec['write_segments']:
+ seg.update( {'type': 'w'} )
+
+ for seg in rec['read_segments']:
+ seg.update( {'type': 'r'} )
+
+
+ segments = rec['write_segments'] + rec['read_segments']
+ segments = sorted(segments, key=lambda k: k['start_time'])
+
+
+ start = float('inf')
+ end = float('-inf')
+
+
+ trace = []
+ minsize = 0
+ for seg in segments:
+ trace += [ seg['type'], seg['offset'], seg['length'], seg['start_time'], seg['end_time'] ]
+
+ seg_minsize = seg['offset'] + seg['length']
+ if minsize < seg_minsize:
+ minsize = seg_minsize
+
+ if start > seg['start_time']:
+ start = seg['start_time']
+
+ if end < seg['end_time']:
+ end = seg['end_time']
+
+ # reconstruct timestamps
+ start = start_time + datetime.timedelta(seconds=start)
+ end = start_time + datetime.timedelta(seconds=end)
+
+ rid = "%s:%d:%d" % (mod, rec['id'], rec['rank'])
+
+ item = {
+ "id": rid,
+ "rank": rec['rank'],
+ "hostname": rec['hostname'],
+ #"filename": rec['filename'],
+ "filename": "FIXME: NEED FILENAME",
+
+ "group": rid,
+ "start": start.isoformat(),
+ "end": end.isoformat(),
+ "limitSize": False, # required to prevent rendering glitches
+ "data": {
+ "duration": (end-start).total_seconds(),
+ "start": segments[0]['start_time'],
+ "size": minsize, # minimal estimated filesize
+ "trace": trace,
+ }
+ }
+
+ items.append(item)
+
+
+ group = {
+ "id": rid,
+ #"content": "[%s] " % (mod) + rec['filename'][-84:],
+ "content": "[%s] " % (mod) + "NEED FILENAME",
+ "order": seg['start_time']
+ }
+ groups.append(group)
+
+
+
+ supported = ['DXT_POSIX', 'DXT_MPIIO']
+ for mod in supported:
+ if mod in self.data['records']:
+ for rec in self.data['records'][mod]:
+ groupify(rec, mod)
+
+
+
+
+
+
+ # overwrite existing summary entry
+ if mode == "append":
+ self.summary['timeline'] = ctx
+
+
+ return ctx
+
+
=====================================
darshan-util/pydarshan/darshan/experimental/aggregators/name_records_summary.py
=====================================
@@ -8,7 +8,7 @@ def name_records_summary(self):
mod_name (str):
Return:
- None
+ dict with counts nested <nrec-id>/<mod>
"""
counts = {}
=====================================
darshan-util/pydarshan/darshan/experimental/aggregators/filter.py → darshan-util/pydarshan/darshan/experimental/operations/filter.py
=====================================
@@ -90,9 +90,9 @@ def filter(self, mods=None, name_records=None, pattern=None, regex=None):
if nrec in name_records:
if mod not in ctx:
- ctx[mod] = []
+ ctx[mod] = DarshanRecordCollection(mod=mod, report=r)
- ctx[mod].append(rec)
+ ctx[mod].append(rec._records[0])
r.records = ctx
=====================================
darshan-util/pydarshan/darshan/experimental/aggregators/merge.py → darshan-util/pydarshan/darshan/experimental/operations/merge.py
=====================================
@@ -16,7 +16,6 @@ def merge(self, other, reduce_first=False):
None
"""
-
# new report
nr = DarshanReport()
@@ -59,7 +58,7 @@ def merge(self, other, reduce_first=False):
if key not in nr.records:
nr.records[key] = copy.copy(records)
else:
- nr.records[key] += copy.copy(records)
+ nr.records[key]._records = nr.records[key]._records + copy.copy(records._records)
for key, mod in report.modules.items():
if key not in nr.modules:
@@ -69,8 +68,6 @@ def merge(self, other, reduce_first=False):
for key, counter in report.counters.items():
if key not in nr.counters:
nr.counters[key] = copy.copy(counter)
- # TODO: invalidate len/counters
-
for key, nrec in report.name_records.items():
if key not in nr.counters:
=====================================
darshan-util/pydarshan/darshan/experimental/operations/potential_functions.py.txt
=====================================
@@ -0,0 +1,27 @@
+ #def np: read_csv -> vs open?
+
+
+ #def copy() -> np: create deep copy
+ #def view() -> np: creates a view?! wiht the same data..
+
+ #def info()
+ #def size()
+ #def astype(typestring/object, numpy/pandas?)
+
+ # comparisons: ==, <
+ # comparison: np: array_equal (array_wise comparison)
+
+ #add
+ #multiply
+ #exp
+
+ #sum (np: array wise)
+ #cumsum (np: all elems)
+ #min
+ #max
+ #median
+ #mean
+ #std
+ #corrcoef ?? interpretation? seems to make more sense for two different reports? that can work i think
+ # special treatmet dxt? -> corr over time?
+
=====================================
darshan-util/pydarshan/darshan/experimental/aggregators/reduce.py → darshan-util/pydarshan/darshan/experimental/operations/reduce.py
=====================================
@@ -78,7 +78,7 @@ def reduce(self, operation="sum", mods=None, name_records=None, mode='append', d
if mod not in mods:
continue
- for rec in recs:
+ for i, rec in enumerate(recs):
nrec = rec['id']
if nrec in name_records:
@@ -95,7 +95,7 @@ def reduce(self, operation="sum", mods=None, name_records=None, mode='append', d
if nrec_pattern not in ctx[mod]:
ctx[mod][nrec_pattern] = {}
- if counters not in rec:
+ if counters not in rec._records[0]:
continue
if counters not in ctx[mod][nrec_pattern]:
@@ -108,7 +108,7 @@ def reduce(self, operation="sum", mods=None, name_records=None, mode='append', d
result = {}
for mod, name_records in ctx.items():
if mod not in result:
- result[mod] = []
+ result[mod] = DarshanRecordCollection(mod=mod, report=r)
for name_record, val in name_records.items():
rec = {"id": name_record, "rank": -1}
=====================================
darshan-util/pydarshan/darshan/experimental/plots/matplotlib.py
=====================================
@@ -24,7 +24,7 @@ def plot_access_histogram(self, mod, filter=None, data=None):
print("Summarizing... iohist", mod)
self.mod_agg_iohist(mod)
else:
- print("Can not create summary, mod_agg_iohist aggregator is not registered with the report clase.")
+ print("Can not create summary, mod_agg_iohist aggregator is not registered with the report class.")
=====================================
darshan-util/pydarshan/darshan/report.py
=====================================
@@ -18,6 +18,8 @@ import sys
import numpy as np
import pandas as pd
+import collections.abc
+
import logging
logger = logging.getLogger(__name__)
@@ -39,6 +41,248 @@ class DarshanReportJSONEncoder(json.JSONEncoder):
+class DarshanRecordCollection(collections.abc.MutableSequence):
+ """
+ Darshan log records may nest various properties (e.g., DXT, Lustre).
+ As such they can not faithfully represented using only a single
+ Numpy array or a Pandas dataframe.
+
+ The DarshanRecordCollection is used as a wrapper to offer
+ users a stable API to DarshanReports and contained records
+ in various popular formats while allowing to optimize
+ memory and internal representations as necessary.
+ """
+
+ def __init__(self, mod=None, report=None):
+ super(DarshanRecordCollection, self).__init__()
+ self.mod = mod # collections should be homogenous in module type
+ self.report = report # reference the report offering lookup for, e.g., counter names
+
+ self.rank = None # if all records in collection share rank, save memory
+ self.id = None # if all records in collection share id/nrec, save memory
+
+ self.timebase = None # allow fast time rebase without touching every record
+ self.start_time = None
+ self.end_time = None
+
+ self._type = "collection" # collection => list(), single => [record], nested => [[], ... ,[]]
+ self._records = list() # internal format before user conversion
+ pass
+
+ def __len__(self):
+ return len(self._records)
+
+ def __setitem__(self, key, val):
+ self._records[key] = val
+
+ def __getitem__(self, key):
+ if self._type == "record":
+ if isinstance(key, collections.abc.Hashable):
+ #TODO: might extend this style access to collection/nested type as well
+ # but do not want to offer an access which might not be feasible to maintain
+ return self._records[0][key]
+ else:
+ return self._records[0]
+
+ # Wrap single record in RecordCollection to attach conversions: to_json, to_dict, to_df, ...
+ # This way conversion logic can be shared.
+ record = DarshanRecordCollection(mod=self.mod, report=self.report)
+
+ if isinstance(key, slice):
+ record._type = "collection"
+ record._records = self._records[key]
+ else:
+ record._type = "record"
+ record.append(self._records[key])
+ return record
+
+ def __delitem__(self, key):
+ del self._list[ii]
+
+ def insert(self, key, val):
+ self._records.insert(key, val)
+
+ def append(self, val):
+ self.insert(len(self._records), val)
+
+
+ def __repr__(self):
+ if self._type == "record":
+ return self._records[0].__repr__()
+
+ return object.__repr__(self)
+
+ #def __repr__(self):
+ # print("DarshanRecordCollection.__repr__")
+ # repr = ""
+ # for rec in self._records:
+ # repr += f"{rec}\n"
+ # return repr
+
+ def info(self, describe=False, plot=False):
+ """
+ Print information about the record for inspection.
+
+ Args:
+ describe (bool): show detailed summary and statistics (default: False)
+ plot (bool): show plots for quick value overview for counters and fcounters (default: False)
+
+ Return:
+ None
+ """
+ mod = self.mod
+ records = self._records
+
+ print("Module: ", mod, sep="")
+ print("Records: ", len(self), sep="")
+ print("Coll. Type: ", self._type, sep="")
+
+ if mod in ['LUSTRE']:
+ for i, rec in enumerate(records):
+ pass
+ elif mod in ['DXT_POSIX', 'DXT_MPIIO']:
+ ids = set()
+ ranks = set()
+ hostnames = set()
+ reads = 0
+ writes = 0
+ for i, rec in enumerate(records):
+ ids.add(rec['id'])
+ ranks.add(rec['rank'])
+ hostnames.add(rec['hostname'])
+ reads += rec['read_count']
+ writes += rec['write_count']
+ print("Ranks: ", str(ranks), sep="")
+ print("Name Records: ", str(ids), sep="")
+ print("Hostnames: ", str(hostnames), sep="")
+ print("Read Events: ", str(reads), sep="")
+ print("Write Events: ", str(writes), sep="")
+
+
+ if describe or plot:
+ logger.warn("No plots/descriptions defined for DXT records info.")
+
+ else:
+ ids = set()
+ ranks = set()
+ for i, rec in enumerate(records):
+ ids.add(rec['id'])
+ ranks.add(rec['rank'])
+ print("Ranks: ", str(ranks), sep="")
+ print("Name Records: ", str(ids), sep="")
+
+
+ if describe or plot:
+ df = self.to_df(attach=None)
+ pd_max_rows = pd.get_option('display.max_rows')
+ pd_max_columns = pd.get_option('display.max_columns')
+ pd.set_option('display.max_rows', None)
+
+ if plot:
+ figw = 7
+ lh = 0.3 # lineheight
+ # get number of counters for plot height adjustment
+ nc = self[0]['counters'].size
+ nfc = self[0]['fcounters'].size
+
+ display(df['counters'].plot.box(vert=False, figsize=(figw, nc*lh)))
+ display(df['fcounters'].plot.box(vert=False, figsize=(figw, nfc*lh)))
+
+ if describe:
+ display(df['counters'].describe().transpose())
+ display(df['fcounters'].describe().transpose())
+
+ pd.set_option('display.max_rows', pd_max_rows)
+
+
+
+ ###########################################################################
+ # Export Conversions (following the pandas naming conventions)
+ ###########################################################################
+ def to_numpy(self):
+ records = copy.deepcopy(self._records)
+ return records
+
+ def to_list(self):
+ mod = self.mod
+ records = copy.deepcopy(self._records)
+
+ if mod in ['LUSTRE']:
+ raise NotImplementedError
+ elif mod in ['DXT_POSIX', 'DXT_MPIIO']:
+ raise NotImplementedError
+ else:
+ for i, rec in enumerate(records):
+ rec['counters'] = rec['counters'].tolist()
+ rec['fcounters'] = rec['fcounters'].tolist()
+ return records
+
+ def to_dict(self):
+ mod = self.mod
+ records = copy.deepcopy(self._records)
+ counters = self.report.counters[self.mod]
+ if mod in ['LUSTRE']:
+ raise NotImplementedError
+ elif mod in ['DXT_POSIX', 'DXT_MPIIO']:
+ # format already in a dict format, but may offer switches for expansion
+ logger.warn("WARNING: The output of DarshanRecordCollection.to_dict() may change in the future.")
+ pass
+ else:
+ for i, rec in enumerate(records):
+ rec['counters'] = dict(zip(counters['counters'], rec['counters']))
+ rec['fcounters'] = dict(zip(counters['fcounters'], rec['fcounters']))
+ return records
+
+ def to_json(self):
+ records = self.to_list()
+ return json.dumps(records, cls=DarshanReportJSONEncoder)
+
+ def to_df(self, attach="default"):
+ if attach == "default":
+ attach = ['id', 'rank']
+
+ mod = self.mod
+ records = copy.deepcopy(self._records)
+
+ if mod in ['LUSTRE']:
+ for i, rec in enumerate(records):
+ rec = rec
+ elif mod in ['DXT_POSIX', 'DXT_MPIIO']:
+ for i, rec in enumerate(records):
+ rec['read_segments'] = pd.DataFrame(rec['read_segments'])
+ rec['write_segments'] = pd.DataFrame(rec['write_segments'])
+ else:
+ counters = []
+ fcounters = []
+ ids = []
+ ranks = []
+
+ for i, rec in enumerate(records):
+ counters.append(rec['counters'])
+ fcounters.append(rec['fcounters'])
+ ids.append(rec['id'])
+ ranks.append(rec['rank'])
+
+ records = {"counters": None, "fcounters": None}
+ records['counters'] = pd.DataFrame(counters, columns=self.report.counters[mod]['counters'])
+ records['fcounters'] = pd.DataFrame(fcounters, columns=self.report.counters[mod]['fcounters'])
+
+ def flip_column_order(df):
+ return df[df.columns[::-1]]
+
+ # attach ids and ranks
+ if attach is not None:
+ for counter_type in ['counters', 'fcounters']:
+ records[counter_type] = flip_column_order(records[counter_type])
+ if 'id' in attach:
+ records[counter_type]['id'] = ids
+ if 'rank' in attach:
+ records[counter_type]['rank'] = ranks
+ records[counter_type] = flip_column_order(records[counter_type])
+
+ return records
+
+
class DarshanReport(object):
"""
The DarshanReport class provides a convienient wrapper to access darshan
@@ -46,12 +290,12 @@ class DarshanReport(object):
a number of common aggregations can be performed.
"""
- # a way to conser memory?
+ # a way to conserve memory?
#__slots__ = ['attr1', 'attr2']
def __init__(self,
- filename=None, dtype='pandas',
+ filename=None, dtype='numpy',
start_time=None, end_time=None,
automatic_summary=False,
read_all=True, lookup_name_records=True):
@@ -70,9 +314,7 @@ class DarshanReport(object):
self.filename = filename
# Behavioral Options
- self.dtype = dtype # Experimental: preferred internal representation: pandas/numpy useful for aggregations, dict good for export/REST
- # might require alternative granularity: e.g., records, vs summaries?
- # vs dict/pandas? dict/native?
+ self.dtype = dtype # default dtype to return when viewing records
self.automatic_summary = automatic_summary
self.lookup_name_records = lookup_name_records
@@ -81,16 +323,18 @@ class DarshanReport(object):
# Report Metadata
+ #
+ # Start/End + Timebase are
self.start_time = start_time if start_time else float('inf')
self.end_time = end_time if end_time else float('-inf')
self.timebase = self.start_time
# Initialize data namespaces
- self.metadata = {}
- self.modules = {}
- self.counters = {}
+ self._metadata = {}
+ self._modules = {}
+ self._counters = {}
self.records = {}
- self.mounts = {}
+ self._mounts = {}
self.name_records = {}
# initialize report/summary namespace
@@ -101,10 +345,10 @@ class DarshanReport(object):
# legacy references (deprecate before 1.0?)
self.data_revision = 0 # counter for consistency checks
self.data = {'version': 1}
- self.data['metadata'] = self.metadata
+ self.data['metadata'] = self._metadata
self.data['records'] = self.records
self.data['summary'] = self.summary
- self.data['modules'] = self.modules
+ self.data['modules'] = self._modules
self.data['counters'] = self.counters
self.data['name_records'] = self.name_records
@@ -121,6 +365,33 @@ class DarshanReport(object):
self.open(filename, read_all=read_all)
+ @property
+ def metadata(self):
+ return self._metadata
+
+ @property
+ def modules(self):
+ return self._modules
+
+ @property
+ def counters(self):
+ return self._counters
+
+# @property
+# def counters(self):
+# return self._counters
+#
+# @property
+# def name_records(self):
+# return self._name_records
+#
+#
+# @property
+# def summary(self):
+# return self._summary
+#
+
+
def open(self, filename, read_all=False):
"""
Open log file via CFFI backend.
@@ -147,7 +418,6 @@ class DarshanReport(object):
self.read_all()
-
def __add__(self, other):
"""
Allow reports to be merged using the addition operator.
@@ -201,7 +471,7 @@ class DarshanReport(object):
self.mounts = self.data['mounts']
self.data['modules'] = backend.log_get_modules(self.log)
- self.modules = self.data['modules']
+ self._modules = self.data['modules']
if read_all == True:
self.data["name_records"] = backend.log_get_name_records(self.log)
@@ -323,12 +593,12 @@ class DarshanReport(object):
dtype = dtype if dtype else self.dtype
- self.data['records'][mod] = []
+ self.records[mod] = DarshanRecordCollection(mod=mod, report=self)
cn = backend.counter_names(mod)
fcn = backend.fcounter_names(mod)
# update module metadata
- self.modules[mod]['num_records'] = 0
+ self._modules[mod]['num_records'] = 0
if mod not in self.counters:
self.counters[mod] = {}
self.counters[mod]['counters'] = cn
@@ -339,7 +609,7 @@ class DarshanReport(object):
rec = backend.log_get_generic_record(self.log, mod, dtype=dtype)
while rec != None:
self.records[mod].append(rec)
- self.modules[mod]['num_records'] += 1
+ self._modules[mod]['num_records'] += 1
# fetch next
rec = backend.log_get_generic_record(self.log, mod, dtype=dtype)
@@ -409,10 +679,10 @@ class DarshanReport(object):
dtype = dtype if dtype else self.dtype
- self.records[mod] = []
+ self.records[mod] = DarshanRecordCollection(mod=mod, report=self)
# update module metadata
- self.modules[mod]['num_records'] = 0
+ self._modules[mod]['num_records'] = 0
if mod not in self.counters:
self.counters[mod] = {}
@@ -465,11 +735,11 @@ class DarshanReport(object):
dtype = dtype if dtype else self.dtype
- self.records[mod] = []
+ self.records[mod] = DarshanRecordCollection(mod=mod, report=self)
cn = backend.counter_names(mod)
# update module metadata
- self.modules[mod]['num_records'] = 0
+ self._modules[mod]['num_records'] = 0
if mod not in self.counters:
self.counters[mod] = {}
self.counters[mod]['counters'] = cn
@@ -523,7 +793,7 @@ class DarshanReport(object):
.. warning::
Can't be used for now when alternating between different modules.
A temporary workaround can be to open the same log multiple times,
- as this ways buffers are not shared between get_record invocations
+ as this way buffers are not shared between get_record invocations
in the lower level library.
@@ -573,7 +843,7 @@ class DarshanReport(object):
print("Processes: ", self.metadata['job']['nprocs'], sep="")
print("JobID: ", self.metadata['job']['jobid'], sep="")
print("UID: ", self.metadata['job']['uid'], sep="")
- print("Modules in Log: ", list(self.modules.keys()), sep="")
+ print("Modules in Log: ", list(self._modules.keys()), sep="")
loaded = {}
for mod in self.records:
@@ -643,7 +913,7 @@ class DarshanReport(object):
return rebased_records
###########################################################################
- # Conversion
+ # Export Conversions
###########################################################################
def to_dict(self):
"""
@@ -659,18 +929,13 @@ class DarshanReport(object):
recs = data['records']
for mod in recs:
- for i, rec in enumerate(data['records'][mod]):
- try:
- recs[mod][i]['counters'] = rec['counters'].tolist()
- except KeyError:
- logger.debug(f" to_json: mod={mod} does not include counters")
- pass
-
- try:
- recs[mod][i]['fcounters'] = rec['fcounters'].tolist()
- except KeyError:
- logger.debug(f" to_json: mod={mod} does not include fcounters")
- pass
+ try:
+ #recs[mod] = recs[mod].to_dict()
+ recs[mod] = recs[mod].to_list()
+ except:
+ recs[mod] = "Not implemented."
+
+
return data
@@ -689,24 +954,14 @@ class DarshanReport(object):
recs = data['records']
for mod in recs:
- for i, rec in enumerate(data['records'][mod]):
- try:
- recs[mod][i]['counters'] = rec['counters'].tolist()
- except KeyError:
- logger.debug(f" to_json: mod={mod} does not include counters")
- pass
-
- try:
- recs[mod][i]['fcounters'] = rec['fcounters'].tolist()
- except KeyError:
- logger.debug(f" to_json: mod={mod} does not include fcounters")
- pass
+ try:
+ recs[mod] = recs[mod].to_list()
+ except:
+ recs[mod] = "Not implemented."
return json.dumps(data, cls=DarshanReportJSONEncoder)
- @staticmethod
- def from_string(string):
- return DarshanReport()
+
=====================================
darshan-util/pydarshan/devel/build-all.sh
=====================================
@@ -13,10 +13,11 @@ DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64
PLAT=manylinux1_x86_64
PRE_CMD=
+# On systems with SELinux it may be necessary to set the Z option for volumes.
docker pull $DOCKER_IMAGE
docker run --rm -e PLAT=$PLAT \
- -v $MNT_PYDARSHAN:/io \
- -v $MNT_DARSHAN:/darshan \
+ -v $MNT_PYDARSHAN:/io:Z \
+ -v $MNT_DARSHAN:/darshan:Z \
$DOCKER_IMAGE $PRE_CMD /io/devel/build-wheels.sh
@@ -27,8 +28,8 @@ PRE_CMD=linux32
docker pull $DOCKER_IMAGE
docker run --rm -e PLAT=$PLAT \
- -v $MNT_PYDARSHAN:/io \
- -v $MNT_DARSHAN:/darshan \
+ -v $MNT_PYDARSHAN:/io:Z \
+ -v $MNT_DARSHAN:/darshan:Z \
$DOCKER_IMAGE $PRE_CMD /io/devel/build-wheels.sh
@@ -39,7 +40,7 @@ PRE_CMD=
docker pull $DOCKER_IMAGE
docker run --rm -e PLAT=$PLAT \
- -v $MNT_PYDARSHAN:/io \
- -v $MNT_DARSHAN:/darshan \
+ -v $MNT_PYDARSHAN:/io:Z \
+ -v $MNT_DARSHAN:/darshan:Z \
$DOCKER_IMAGE $PRE_CMD /io/devel/build-wheels.sh
=====================================
darshan-util/pydarshan/examples/01_darshan-introduction.ipynb
=====================================
The diff for this file was not included because it is too large.
=====================================
darshan-util/pydarshan/examples/02_darshan-plotting-matplotlib.ipynb
=====================================
The diff for this file was not included because it is too large.
=====================================
darshan-util/pydarshan/examples/04_darshan-low-level-cffi-interface.ipynb
=====================================
@@ -18,7 +18,7 @@
},
{
"cell_type": "code",
- "execution_count": 43,
+ "execution_count": 18,
"metadata": {},
"outputs": [
{
@@ -232,7 +232,7 @@
},
{
"cell_type": "code",
- "execution_count": 51,
+ "execution_count": 19,
"metadata": {},
"outputs": [
{
@@ -276,7 +276,7 @@
},
{
"cell_type": "code",
- "execution_count": 52,
+ "execution_count": 20,
"metadata": {},
"outputs": [
{
@@ -306,7 +306,7 @@
},
{
"cell_type": "code",
- "execution_count": 45,
+ "execution_count": 21,
"metadata": {},
"outputs": [
{
@@ -529,7 +529,7 @@
},
{
"cell_type": "code",
- "execution_count": 53,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
@@ -756,7 +756,7 @@
},
{
"cell_type": "code",
- "execution_count": 55,
+ "execution_count": 23,
"metadata": {},
"outputs": [
{
@@ -779,7 +779,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
@@ -1046,24 +1046,7 @@
},
{
"cell_type": "code",
- "execution_count": 44,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "{'handle': <cdata 'void *' 0x5648c258ed90>, 'modules': {'POSIX': {'len': 186, 'ver': 3, 'idx': 1}, 'MPI-IO': {'len': 154, 'ver': 2, 'idx': 2}, 'LUSTRE': {'len': 87, 'ver': 1, 'idx': 7}, 'STDIO': {'len': 3234, 'ver': 1, 'idx': 8}}, 'name_records': None} {'handle': <cdata 'void *' 0x5648c26553f0>, 'modules': None, 'name_records': None}\n"
- ]
- }
- ],
- "source": [
- "print(log1, log2)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 45,
+ "execution_count": 25,
"metadata": {},
"outputs": [
{
@@ -1077,7 +1060,7 @@
" 'metadata': {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}}"
]
},
- "execution_count": 45,
+ "execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
@@ -1095,7 +1078,7 @@
},
{
"cell_type": "code",
- "execution_count": 46,
+ "execution_count": 26,
"metadata": {
"scrolled": true
},
@@ -1127,7 +1110,7 @@
" ('/', 'dvs')]"
]
},
- "execution_count": 46,
+ "execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
@@ -1138,7 +1121,7 @@
},
{
"cell_type": "code",
- "execution_count": 47,
+ "execution_count": 27,
"metadata": {
"scrolled": true
},
@@ -1152,7 +1135,7 @@
" 'STDIO': {'len': 3234, 'ver': 1, 'idx': 8}}"
]
},
- "execution_count": 47,
+ "execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
@@ -1179,7 +1162,7 @@
},
{
"cell_type": "code",
- "execution_count": 48,
+ "execution_count": 28,
"metadata": {
"scrolled": true
},
@@ -1258,7 +1241,7 @@
" 'POSIX_SLOWEST_RANK_BYTES']"
]
},
- "execution_count": 48,
+ "execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
@@ -1270,7 +1253,7 @@
},
{
"cell_type": "code",
- "execution_count": 49,
+ "execution_count": 29,
"metadata": {
"scrolled": true
},
@@ -1331,7 +1314,7 @@
" 'MPIIO_SLOWEST_RANK_BYTES']"
]
},
- "execution_count": 49,
+ "execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
@@ -1343,7 +1326,7 @@
},
{
"cell_type": "code",
- "execution_count": 51,
+ "execution_count": 30,
"metadata": {
"scrolled": true
},
@@ -1367,7 +1350,7 @@
" 'STDIO_SLOWEST_RANK_BYTES']"
]
},
- "execution_count": 51,
+ "execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
@@ -1379,7 +1362,7 @@
},
{
"cell_type": "code",
- "execution_count": 52,
+ "execution_count": 31,
"metadata": {
"scrolled": true
},
@@ -1414,7 +1397,7 @@
" 0.00000000e+00])}"
]
},
- "execution_count": 52,
+ "execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
@@ -1426,7 +1409,7 @@
},
{
"cell_type": "code",
- "execution_count": 53,
+ "execution_count": 32,
"metadata": {},
"outputs": [
{
@@ -1435,7 +1418,7 @@
"16402"
]
},
- "execution_count": 53,
+ "execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
@@ -1447,7 +1430,7 @@
},
{
"cell_type": "code",
- "execution_count": 54,
+ "execution_count": 33,
"metadata": {
"scrolled": true
},
@@ -1526,7 +1509,7 @@
" 'POSIX_SLOWEST_RANK_BYTES': 1073741824}"
]
},
- "execution_count": 54,
+ "execution_count": 33,
"metadata": {},
"output_type": "execute_result"
}
@@ -1563,12 +1546,19 @@
},
{
"cell_type": "code",
- "execution_count": 56,
+ "execution_count": 34,
"metadata": {},
"outputs": [],
"source": [
"backend.log_close(log)"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -1587,7 +1577,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.10"
+ "version": "3.9.2"
}
},
"nbformat": 4,
=====================================
darshan-util/pydarshan/examples/99-Debugging-PyDarshan-via-Logging-Interface.ipynb
=====================================
@@ -17,14 +17,14 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "ERROR:root:The 'log_level' trait of an IPKernelApp instance must be any of (0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'), but a value of 'this-workaround-ensures-stderr-is-in-handlers' <class 'str'> was specified.\n"
+ "ERROR:root:The 'log_level' trait of an IPKernelApp instance expected any of [0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL'], not the str 'this-workaround-ensures-stderr-is-in-handlers'.\n"
]
}
],
@@ -33,7 +33,8 @@
"# https://stackoverflow.com/questions/35326814/change-level-logged-to-ipython-jupyter-notebook\n",
"%config Application.log_level=\"this-workaround-ensures-stderr-is-in-handlers\"\n",
"import logging\n",
- "logging.getLogger().setLevel(logging.DEBUG)"
+ "logging.getLogger().setLevel(logging.DEBUG)\n",
+ "# Expect an error complaining about an unknown \"log_level\" trait. The error can be ignored, and logging should work now."
]
},
{
@@ -45,14 +46,13 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 29,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "DEBUG:darshan.discover_darshan: Found lib_version=3.2.1 via ffi.\n",
"DEBUG:darshan.report: Refreshing name_records for mod=POSIX\n",
"DEBUG:darshan.report: Refreshing name_records for mod=POSIX\n",
"DEBUG:darshan.report: Refreshing name_records for mod=MPI-IO\n",
@@ -66,32 +66,33 @@
]
},
{
- "data": {
- "text/plain": [
- "[{'rank': -1,\n",
- " 'id': -1,\n",
- " 'counters': rank id LUSTRE_OSTS LUSTRE_MDTS LUSTRE_STRIPE_OFFSET \\\n",
- " 0 -1 6301063301082038805 24 1 0 \n",
- " \n",
- " LUSTRE_STRIPE_SIZE LUSTRE_STRIPE_WIDTH \n",
- " 0 1048576 24 }]"
- ]
- },
- "execution_count": 2,
- "metadata": {},
- "output_type": "execute_result"
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Filename: example-logs/example.darshan\n",
+ "Times: 2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
+ "Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
+ "Processes: 2048\n",
+ "JobID: 4478544\n",
+ "UID: 69615\n",
+ "Modules in Log: ['POSIX', 'MPI-IO', 'LUSTRE', 'STDIO']\n",
+ "Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129, 'LUSTRE': 1}\n",
+ "Name Records: 4\n",
+ "Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
+ "DarshanReport: id(140532799874288) (tmp)\n"
+ ]
}
],
"source": [
"import darshan\n",
"report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True) # Default behavior\n",
"\n",
- "report.records['LUSTRE']"
+ "report.info()"
]
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
@@ -100,7 +101,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 30,
"metadata": {},
"outputs": [
{
@@ -118,7 +119,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 31,
"metadata": {},
"outputs": [
{
@@ -137,7 +138,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 21,
"metadata": {},
"outputs": [
{
@@ -155,7 +156,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
@@ -164,7 +165,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 25,
"metadata": {
"scrolled": true
},
@@ -182,25 +183,24 @@
"DEBUG:darshan.report: Refreshing name_records for mod=POSIX\n",
"DEBUG:darshan.report: Refreshing name_records for mod=MPI-IO\n",
"DEBUG:darshan.report: Refreshing name_records for mod=STDIO\n",
- "DEBUG:darshan.report: Refreshing name_records for mod=LUSTRE\n",
- "DEBUG:darshan.report: to_json: mod=LUSTRE does not include fcounters\n"
+ "DEBUG:darshan.report: Refreshing name_records for mod=LUSTRE\n"
]
},
{
"data": {
"text/plain": [
- "'{\"version\": 1, \"metadata\": {\"job\": {\"uid\": 69615, \"start_time\": 1490000867, \"end_time\": 1490000983, \"nprocs\": 2048, \"jobid\": 4478544, \"metadata\": {\"lib_ver\": \"3.1.3\", \"h\": \"romio_no_indep_rw=true;cb_nodes=4\"}}, \"exe\": \"/global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\"}, \"records\": {\"POSIX\": [{\"id\": 6301063301082038805, \"rank\": -1, \"counters\": [2049, -1, -1, 0, 16402, 16404, 0, 0, 0, 0, -1, -1, 0, 0, 0, 2199023259968, 0, 2199023261831, 0, 0, 0, 16384, 0, 0, 8, 16401, 1048576, 0, 134217728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, 0, 0, 16384, 0, 274743689216, 274743691264, 0, 0, 10240, 4096, 0, 0, 134217728, 272, 544, 328, 16384, 8, 2, 2, 597, 1073741824, 1312, 1073741824], \"fcounters\": [3.9191410541534424, 0.0, 3.940063953399658, 3.927093982696533, 3.936579942703247, 0.0, 115.0781660079956, 115.77035808563232, 0.0, 100397.60042190552, 11.300841808319092, 0.0, 17.940945863723755, 20.436099529266357, 85.47495031356812, 0.0, 0.0]}], \"MPI-IO\": [{\"id\": 6301063301082038805, \"rank\": -1, \"counters\": [0, 2048, 0, 18, 0, 16384, 0, 0, 0, 0, 0, 0, 32768, 9, 0, 2199023259968, 0, 0, 134217728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, 0, 0, 16384, 0, 134217728, 272, 544, 328, 16384, 8, 2, 2, 597, 1073741824, 1312, 1073741824], \"fcounters\": [3.912783145904541, 0.0, 3.9400370121002197, -1.0, -1.0, 0.0, 115.07816886901855, 115.7732138633728, 0.0, 100397.89705848694, 49.021146297454834, 0.0, 17.940969944000244, 20.454491138458252, 85.49222207069397, 95.43228094835244, 701.6630859375]}], \"STDIO\": [{\"id\": 15920181672442173319, \"rank\": 0, \"counters\": [1, -1, 0, 6, 0, 0, 280, 0, 0, 279, 0, 0, 0, 0], \"fcounters\": [0.0, 6.794929504394531e-05, 0.0, 0.0, 0.0, 0.07752799987792969, 0.0, 0.0, 0.0, 116.28358292579651, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 0, \"counters\": [1, -1, 0, 68, 0, 0, 3029, 0, 0, 3028, 0, 0, 0, 0], \"fcounters\": [0.0, -2662.7466337680817, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 16, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 32, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 48, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 64, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 80, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 96, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 112, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 128, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 144, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 160, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 176, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 192, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 208, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 224, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 240, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 256, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 272, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 288, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 304, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 320, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 336, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 352, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 368, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 384, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 400, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 416, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 432, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 448, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 464, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 480, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 496, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 512, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 528, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 544, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 560, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 576, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 592, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 608, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 624, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 640, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 656, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 672, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 688, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 704, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 720, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 736, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 752, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 768, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 784, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 800, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 816, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 832, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 848, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 864, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 880, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 896, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 912, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 928, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 944, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 960, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 976, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 992, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1008, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1024, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1040, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1056, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1072, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1088, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1104, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1120, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1136, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1152, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1168, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1184, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1200, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1216, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1232, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1248, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1264, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1280, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1296, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1312, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1328, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1344, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1360, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1376, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1392, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1408, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1424, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1440, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1456, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1472, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1488, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1504, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1520, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1536, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1552, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1568, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1584, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1600, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1616, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1632, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1648, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1664, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1680, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1696, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1712, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1728, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1744, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1760, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1776, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1792, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1808, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1824, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1840, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1856, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1872, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1888, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1904, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1920, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1936, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1952, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1968, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 1984, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 2000, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 2016, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}, {\"id\": 7238257241479193519, \"rank\": 2032, \"counters\": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], \"fcounters\": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}], \"LUSTRE\": [{\"id\": 6301063301082038805, \"rank\": -1, \"counters\": [24, 1, 0, 1048576, 24], \"ost_ids\": [7, 9, 23, 21, 1, 19, 20, 8, 18, 12, 6, 2, 10, 16, 4, 0, 22, 14, 13, 17, 5, 15, 11, 3]}]}, \"summary\": {}, \"modules\": {\"POSIX\": {\"len\": 186, \"ver\": 3, \"idx\": 1, \"num_records\": 1}, \"MPI-IO\": {\"len\": 154, \"ver\": 2, \"idx\": 2, \"num_records\": 1}, \"LUSTRE\": {\"len\": 87, \"ver\": 1, \"idx\": 7, \"num_records\": 1}, \"STDIO\": {\"len\": 3234, \"ver\": 1, \"idx\": 8, \"num_records\": 129}}, \"counters\": {\"POSIX\": {\"counters\": [\"POSIX_OPENS\", \"POSIX_FILENOS\", \"POSIX_DUPS\", \"POSIX_READS\", \"POSIX_WRITES\", \"POSIX_SEEKS\", \"POSIX_STATS\", \"POSIX_MMAPS\", \"POSIX_FSYNCS\", \"POSIX_FDSYNCS\", \"POSIX_RENAME_SOURCES\", \"POSIX_RENAME_TARGETS\", \"POSIX_RENAMED_FROM\", \"POSIX_MODE\", \"POSIX_BYTES_READ\", \"POSIX_BYTES_WRITTEN\", \"POSIX_MAX_BYTE_READ\", \"POSIX_MAX_BYTE_WRITTEN\", \"POSIX_CONSEC_READS\", \"POSIX_CONSEC_WRITES\", \"POSIX_SEQ_READS\", \"POSIX_SEQ_WRITES\", \"POSIX_RW_SWITCHES\", \"POSIX_MEM_NOT_ALIGNED\", \"POSIX_MEM_ALIGNMENT\", \"POSIX_FILE_NOT_ALIGNED\", \"POSIX_FILE_ALIGNMENT\", \"POSIX_MAX_READ_TIME_SIZE\", \"POSIX_MAX_WRITE_TIME_SIZE\", \"POSIX_SIZE_READ_0_100\", \"POSIX_SIZE_READ_100_1K\", \"POSIX_SIZE_READ_1K_10K\", \"POSIX_SIZE_READ_10K_100K\", \"POSIX_SIZE_READ_100K_1M\", \"POSIX_SIZE_READ_1M_4M\", \"POSIX_SIZE_READ_4M_10M\", \"POSIX_SIZE_READ_10M_100M\", \"POSIX_SIZE_READ_100M_1G\", \"POSIX_SIZE_READ_1G_PLUS\", \"POSIX_SIZE_WRITE_0_100\", \"POSIX_SIZE_WRITE_100_1K\", \"POSIX_SIZE_WRITE_1K_10K\", \"POSIX_SIZE_WRITE_10K_100K\", \"POSIX_SIZE_WRITE_100K_1M\", \"POSIX_SIZE_WRITE_1M_4M\", \"POSIX_SIZE_WRITE_4M_10M\", \"POSIX_SIZE_WRITE_10M_100M\", \"POSIX_SIZE_WRITE_100M_1G\", \"POSIX_SIZE_WRITE_1G_PLUS\", \"POSIX_STRIDE1_STRIDE\", \"POSIX_STRIDE2_STRIDE\", \"POSIX_STRIDE3_STRIDE\", \"POSIX_STRIDE4_STRIDE\", \"POSIX_STRIDE1_COUNT\", \"POSIX_STRIDE2_COUNT\", \"POSIX_STRIDE3_COUNT\", \"POSIX_STRIDE4_COUNT\", \"POSIX_ACCESS1_ACCESS\", \"POSIX_ACCESS2_ACCESS\", \"POSIX_ACCESS3_ACCESS\", \"POSIX_ACCESS4_ACCESS\", \"POSIX_ACCESS1_COUNT\", \"POSIX_ACCESS2_COUNT\", \"POSIX_ACCESS3_COUNT\", \"POSIX_ACCESS4_COUNT\", \"POSIX_FASTEST_RANK\", \"POSIX_FASTEST_RANK_BYTES\", \"POSIX_SLOWEST_RANK\", \"POSIX_SLOWEST_RANK_BYTES\"], \"fcounters\": [\"POSIX_F_OPEN_START_TIMESTAMP\", \"POSIX_F_READ_START_TIMESTAMP\", \"POSIX_F_WRITE_START_TIMESTAMP\", \"POSIX_F_CLOSE_START_TIMESTAMP\", \"POSIX_F_OPEN_END_TIMESTAMP\", \"POSIX_F_READ_END_TIMESTAMP\", \"POSIX_F_WRITE_END_TIMESTAMP\", \"POSIX_F_CLOSE_END_TIMESTAMP\", \"POSIX_F_READ_TIME\", \"POSIX_F_WRITE_TIME\", \"POSIX_F_META_TIME\", \"POSIX_F_MAX_READ_TIME\", \"POSIX_F_MAX_WRITE_TIME\", \"POSIX_F_FASTEST_RANK_TIME\", \"POSIX_F_SLOWEST_RANK_TIME\", \"POSIX_F_VARIANCE_RANK_TIME\", \"POSIX_F_VARIANCE_RANK_BYTES\"]}, \"MPI-IO\": {\"counters\": [\"MPIIO_INDEP_OPENS\", \"MPIIO_COLL_OPENS\", \"MPIIO_INDEP_READS\", \"MPIIO_INDEP_WRITES\", \"MPIIO_COLL_READS\", \"MPIIO_COLL_WRITES\", \"MPIIO_SPLIT_READS\", \"MPIIO_SPLIT_WRITES\", \"MPIIO_NB_READS\", \"MPIIO_NB_WRITES\", \"MPIIO_SYNCS\", \"MPIIO_HINTS\", \"MPIIO_VIEWS\", \"MPIIO_MODE\", \"MPIIO_BYTES_READ\", \"MPIIO_BYTES_WRITTEN\", \"MPIIO_RW_SWITCHES\", \"MPIIO_MAX_READ_TIME_SIZE\", \"MPIIO_MAX_WRITE_TIME_SIZE\", \"MPIIO_SIZE_READ_AGG_0_100\", \"MPIIO_SIZE_READ_AGG_100_1K\", \"MPIIO_SIZE_READ_AGG_1K_10K\", \"MPIIO_SIZE_READ_AGG_10K_100K\", \"MPIIO_SIZE_READ_AGG_100K_1M\", \"MPIIO_SIZE_READ_AGG_1M_4M\", \"MPIIO_SIZE_READ_AGG_4M_10M\", \"MPIIO_SIZE_READ_AGG_10M_100M\", \"MPIIO_SIZE_READ_AGG_100M_1G\", \"MPIIO_SIZE_READ_AGG_1G_PLUS\", \"MPIIO_SIZE_WRITE_AGG_0_100\", \"MPIIO_SIZE_WRITE_AGG_100_1K\", \"MPIIO_SIZE_WRITE_AGG_1K_10K\", \"MPIIO_SIZE_WRITE_AGG_10K_100K\", \"MPIIO_SIZE_WRITE_AGG_100K_1M\", \"MPIIO_SIZE_WRITE_AGG_1M_4M\", \"MPIIO_SIZE_WRITE_AGG_4M_10M\", \"MPIIO_SIZE_WRITE_AGG_10M_100M\", \"MPIIO_SIZE_WRITE_AGG_100M_1G\", \"MPIIO_SIZE_WRITE_AGG_1G_PLUS\", \"MPIIO_ACCESS1_ACCESS\", \"MPIIO_ACCESS2_ACCESS\", \"MPIIO_ACCESS3_ACCESS\", \"MPIIO_ACCESS4_ACCESS\", \"MPIIO_ACCESS1_COUNT\", \"MPIIO_ACCESS2_COUNT\", \"MPIIO_ACCESS3_COUNT\", \"MPIIO_ACCESS4_COUNT\", \"MPIIO_FASTEST_RANK\", \"MPIIO_FASTEST_RANK_BYTES\", \"MPIIO_SLOWEST_RANK\", \"MPIIO_SLOWEST_RANK_BYTES\"], \"fcounters\": [\"MPIIO_F_OPEN_START_TIMESTAMP\", \"MPIIO_F_READ_START_TIMESTAMP\", \"MPIIO_F_WRITE_START_TIMESTAMP\", \"MPIIO_F_CLOSE_START_TIMESTAMP\", \"MPIIO_F_OPEN_END_TIMESTAMP\", \"MPIIO_F_READ_END_TIMESTAMP\", \"MPIIO_F_WRITE_END_TIMESTAMP\", \"MPIIO_F_CLOSE_END_TIMESTAMP\", \"MPIIO_F_READ_TIME\", \"MPIIO_F_WRITE_TIME\", \"MPIIO_F_META_TIME\", \"MPIIO_F_MAX_READ_TIME\", \"MPIIO_F_MAX_WRITE_TIME\", \"MPIIO_F_FASTEST_RANK_TIME\", \"MPIIO_F_SLOWEST_RANK_TIME\", \"MPIIO_F_VARIANCE_RANK_TIME\", \"MPIIO_F_VARIANCE_RANK_BYTES\"]}, \"STDIO\": {\"counters\": [\"STDIO_OPENS\", \"STDIO_FDOPENS\", \"STDIO_READS\", \"STDIO_WRITES\", \"STDIO_SEEKS\", \"STDIO_FLUSHES\", \"STDIO_BYTES_WRITTEN\", \"STDIO_BYTES_READ\", \"STDIO_MAX_BYTE_READ\", \"STDIO_MAX_BYTE_WRITTEN\", \"STDIO_FASTEST_RANK\", \"STDIO_FASTEST_RANK_BYTES\", \"STDIO_SLOWEST_RANK\", \"STDIO_SLOWEST_RANK_BYTES\"], \"fcounters\": [\"STDIO_F_META_TIME\", \"STDIO_F_WRITE_TIME\", \"STDIO_F_READ_TIME\", \"STDIO_F_OPEN_START_TIMESTAMP\", \"STDIO_F_CLOSE_START_TIMESTAMP\", \"STDIO_F_WRITE_START_TIMESTAMP\", \"STDIO_F_READ_START_TIMESTAMP\", \"STDIO_F_OPEN_END_TIMESTAMP\", \"STDIO_F_CLOSE_END_TIMESTAMP\", \"STDIO_F_WRITE_END_TIMESTAMP\", \"STDIO_F_READ_END_TIMESTAMP\", \"STDIO_F_FASTEST_RANK_TIME\", \"STDIO_F_SLOWEST_RANK_TIME\", \"STDIO_F_VARIANCE_RANK_TIME\", \"STDIO_F_VARIANCE_RANK_BYTES\"]}, \"LUSTRE\": {\"counters\": [\"LUSTRE_OSTS\", \"LUSTRE_MDTS\", \"LUSTRE_STRIPE_OFFSET\", \"LUSTRE_STRIPE_SIZE\", \"LUSTRE_STRIPE_WIDTH\"]}}, \"name_records\": {\"14734109647742566553\": \"<STDIN>\", \"15920181672442173319\": \"<STDOUT>\", \"7238257241479193519\": \"<STDERR>\", \"6301063301082038805\": \"/scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5\"}, \"mounts\": [[\"/.shared/base/default/etc/dat.conf\", \"dvs\"], [\"/usr/lib64/libibverbs.so.1.0.0\", \"dvs\"], [\"/usr/lib64/libibumad.so.3.0.2\", \"dvs\"], [\"/usr/lib64/librdmacm.so.1.0.0\", \"dvs\"], [\"/usr/lib64/libibgni.so.1.0.0\", \"dvs\"], [\"/global/cscratch1\", \"lustre\"], [\"/global/projectb\", \"dvs\"], [\"/global/projecta\", \"dvs\"], [\"/usr/sbin/ibstat\", \"dvs\"], [\"/global/project\", \"dvs\"], [\"/global/common\", \"dvs\"], [\"/global/syscom\", \"dvs\"], [\"/global/dna\", \"dvs\"], [\"/opt/slurm\", \"dvs\"], [\"/global/u1\", \"dvs\"], [\"/global/u2\", \"dvs\"], [\"/scratch1\", \"lustre\"], [\"/scratch2\", \"lustre\"], [\"/scratch3\", \"lustre\"], [\"/etc\", \"dvs\"], [\"/\", \"rootfs\"], [\"/\", \"dvs\"]]}'"
+ "'[{\"id\": 6301063301082038805, \"rank\": -1, \"counters\": [2049, -1, -1, 0, 16402, 16404, 0, 0, 0, 0, -1, -1, 0, 0, 0, 2199023259968, 0, 2199023261831, 0, 0, 0, 16384, 0, 0, 8, 16401, 1048576, 0, 134217728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, 0, 0, 16384, 0, 274743689216, 274743691264, 0, 0, 10240, 4096, 0, 0, 134217728, 272, 544, 328, 16384, 8, 2, 2, 597, 1073741824, 1312, 1073741824], \"fcounters\": [3.9191410541534424, 0.0, 3.940063953399658, 3.927093982696533, 3.936579942703247, 0.0, 115.0781660079956, 115.77035808563232, 0.0, 100397.60042190552, 11.300841808319092, 0.0, 17.940945863723755, 20.436099529266357, 85.47495031356812, 0.0, 0.0]}]'"
]
},
- "execution_count": 9,
+ "execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import darshan\n",
- "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True, dtype='numpy')\n",
- "report.to_json()"
+ "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
+ "report.records['POSIX'].to_json()"
]
},
{
@@ -227,7 +227,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.5"
+ "version": "3.9.2"
}
},
"nbformat": 4,
=====================================
darshan-util/pydarshan/examples/99_darshan-experimental.ipynb
=====================================
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
@@ -21,25 +21,26 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
+ "Added method agg_ioops to DarshanReport.\n",
+ "Added method create_dxttimeline to DarshanReport.\n",
+ "Added method create_sankey to DarshanReport.\n",
+ "Added method create_time_summary to DarshanReport.\n",
"Added method mod_agg_iohist to DarshanReport.\n",
"Added method print_module_records to DarshanReport.\n",
+ "Added method records_as_dict to DarshanReport.\n",
"Added method summarize to DarshanReport.\n",
- "Added method merge to DarshanReport.\n",
- "Added method filter to DarshanReport.\n",
"Added method create_timeline to DarshanReport.\n",
- "Added method name_records_summary to DarshanReport.\n",
- "Added method create_time_summary to DarshanReport.\n",
+ "Added method filter to DarshanReport.\n",
"Added method reduce to DarshanReport.\n",
- "Added method create_sankey to DarshanReport.\n",
- "Added method records_as_dict to DarshanReport.\n",
- "Added method agg_ioops to DarshanReport.\n"
+ "Added method name_records_summary to DarshanReport.\n",
+ "Added method merge to DarshanReport.\n"
]
}
],
@@ -50,7 +51,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 25,
"metadata": {
"scrolled": true
},
@@ -60,7 +61,7 @@
"output_type": "stream",
"text": [
"Filename: example-logs/example.darshan\n",
- "Times: 2017-03-20 09:07:47 to 2017-03-20 09:09:43 (Duration 0:01:56)\n",
+ "Times: 2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
"Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
"Processes: 2048\n",
"JobID: 4478544\n",
@@ -69,90 +70,134 @@
"Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129, 'LUSTRE': 1}\n",
"Name Records: 4\n",
"Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
- "DarshanReport: id(140626964480304) (tmp)\n"
+ "DarshanReport: id(140233011845200) (tmp)\n"
]
}
],
"source": [
- "r1 = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True, dtype='numpy') # Default behavior\n",
+ "r1 = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True) # Default behavior\n",
"r1.info()"
]
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "{'POSIX': [{'id': 6301063301082038805,\n",
- " 'rank': -1,\n",
- " 'counters': array([ 2049, -1, -1, 0,\n",
- " 16402, 16404, 0, 0,\n",
- " 0, 0, -1, -1,\n",
- " 0, 0, 0, 2199023259968,\n",
- " 0, 2199023261831, 0, 0,\n",
- " 0, 16384, 0, 0,\n",
- " 8, 16401, 1048576, 0,\n",
- " 134217728, 0, 0, 0,\n",
- " 0, 0, 0, 0,\n",
- " 0, 0, 0, 4,\n",
- " 14, 0, 0, 0,\n",
- " 0, 0, 0, 16384,\n",
- " 0, 274743689216, 274743691264, 0,\n",
- " 0, 10240, 4096, 0,\n",
- " 0, 134217728, 272, 544,\n",
- " 328, 16384, 8, 2,\n",
- " 2, 597, 1073741824, 1312,\n",
- " 1073741824]),\n",
- " 'fcounters': array([3.91914105e+00, 0.00000000e+00, 3.94006395e+00, 3.92709398e+00,\n",
- " 3.93657994e+00, 0.00000000e+00, 1.15078166e+02, 1.15770358e+02,\n",
- " 0.00000000e+00, 1.00397600e+05, 1.13008418e+01, 0.00000000e+00,\n",
- " 1.79409459e+01, 2.04360995e+01, 8.54749503e+01, 0.00000000e+00,\n",
- " 0.00000000e+00])}],\n",
- " 'MPI-IO': [{'id': 6301063301082038805,\n",
- " 'rank': -1,\n",
- " 'counters': array([ 0, 2048, 0, 18,\n",
- " 0, 16384, 0, 0,\n",
- " 0, 0, 0, 0,\n",
- " 32768, 9, 0, 2199023259968,\n",
- " 0, 0, 134217728, 0,\n",
- " 0, 0, 0, 0,\n",
- " 0, 0, 0, 0,\n",
- " 0, 4, 14, 0,\n",
- " 0, 0, 0, 0,\n",
- " 0, 16384, 0, 134217728,\n",
- " 272, 544, 328, 16384,\n",
- " 8, 2, 2, 597,\n",
- " 1073741824, 1312, 1073741824]),\n",
- " 'fcounters': array([ 3.91278315e+00, 0.00000000e+00, 3.94003701e+00, -1.00000000e+00,\n",
- " -1.00000000e+00, 0.00000000e+00, 1.15078169e+02, 1.15773214e+02,\n",
- " 0.00000000e+00, 1.00397897e+05, 4.90211463e+01, 0.00000000e+00,\n",
- " 1.79409699e+01, 2.04544911e+01, 8.54922221e+01, 9.54322809e+01,\n",
- " 7.01663086e+02])}],\n",
- " 'STDIO': [{'id': 15920181672442173319,\n",
- " 'rank': 0,\n",
- " 'counters': array([ 1, -1, 0, 6, 0, 0, 280, 0, 0, 279, 0, 0, 0,\n",
- " 0]),\n",
- " 'fcounters': array([0.00000000e+00, 6.79492950e-05, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 7.75279999e-02, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 1.16283583e+02, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00])}],\n",
- " 'LUSTRE': [{'id': 6301063301082038805,\n",
- " 'rank': -1,\n",
- " 'counters': array([ 24, 1, 0, 1048576, 24]),\n",
- " 'ost_ids': array([ 7, 9, 23, 21, 1, 19, 20, 8, 18, 12, 6, 2, 10, 16, 4, 0, 22,\n",
- " 14, 13, 17, 5, 15, 11, 3])}]}"
+ "{14734109647742566553: '<STDIN>',\n",
+ " 15920181672442173319: '<STDOUT>',\n",
+ " 7238257241479193519: '<STDERR>',\n",
+ " 6301063301082038805: '/scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5'}"
]
},
- "execution_count": 19,
+ "execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "r1.filter(name_records=[6301063301082038805, 15920181672442173319]).records"
+ "r1.name_records"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Module: POSIX\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "Ranks: {-1}\n",
+ "Name Records: {6301063301082038805}\n",
+ "\n",
+ "Module: MPI-IO\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "Ranks: {-1}\n",
+ "Name Records: {6301063301082038805}\n",
+ "\n",
+ "Module: STDIO\n",
+ "Records: 129\n",
+ "Coll. Type: collection\n",
+ "Ranks: {0, 512, 1024, 1536, 16, 528, 1040, 1552, 32, 544, 1056, 1568, 48, 560, 1072, 1584, 64, 576, 1088, 1600, 80, 592, 1104, 1616, 96, 608, 1120, 1632, 112, 624, 1136, 1648, 128, 640, 1152, 1664, 144, 656, 1168, 1680, 160, 672, 1184, 1696, 176, 688, 1200, 1712, 192, 704, 1216, 1728, 208, 720, 1232, 1744, 224, 736, 1248, 1760, 240, 752, 1264, 1776, 256, 768, 1280, 1792, 272, 784, 1296, 1808, 288, 800, 1312, 1824, 304, 816, 1328, 1840, 320, 832, 1344, 1856, 336, 848, 1360, 1872, 352, 864, 1376, 1888, 368, 880, 1392, 1904, 384, 896, 1408, 1920, 400, 912, 1424, 1936, 416, 928, 1440, 1952, 432, 944, 1456, 1968, 448, 960, 1472, 1984, 464, 976, 1488, 2000, 480, 992, 1504, 2016, 496, 1008, 1520, 2032}\n",
+ "Name Records: {7238257241479193519, 15920181672442173319}\n",
+ "\n",
+ "Module: LUSTRE\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "for key, recs in r1.records.items():\n",
+ " recs.info()\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Module: POSIX\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "Ranks: {-1}\n",
+ "Name Records: {6301063301082038805}\n",
+ "\n",
+ "Module: MPI-IO\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "Ranks: {-1}\n",
+ "Name Records: {6301063301082038805}\n",
+ "\n",
+ "Module: LUSTRE\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "for key, recs in r1.filter(name_records=['/scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5']).records.items():\n",
+ " recs.info()\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Module: STDIO\n",
+ "Records: 128\n",
+ "Coll. Type: collection\n",
+ "Ranks: {0, 512, 1024, 1536, 16, 528, 1040, 1552, 32, 544, 1056, 1568, 48, 560, 1072, 1584, 64, 576, 1088, 1600, 80, 592, 1104, 1616, 96, 608, 1120, 1632, 112, 624, 1136, 1648, 128, 640, 1152, 1664, 144, 656, 1168, 1680, 160, 672, 1184, 1696, 176, 688, 1200, 1712, 192, 704, 1216, 1728, 208, 720, 1232, 1744, 224, 736, 1248, 1760, 240, 752, 1264, 1776, 256, 768, 1280, 1792, 272, 784, 1296, 1808, 288, 800, 1312, 1824, 304, 816, 1328, 1840, 320, 832, 1344, 1856, 336, 848, 1360, 1872, 352, 864, 1376, 1888, 368, 880, 1392, 1904, 384, 896, 1408, 1920, 400, 912, 1424, 1936, 416, 928, 1440, 1952, 432, 944, 1456, 1968, 448, 960, 1472, 1984, 464, 976, 1488, 2000, 480, 992, 1504, 2016, 496, 1008, 1520, 2032}\n",
+ "Name Records: {7238257241479193519}\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "for key, recs in r1.filter(name_records=['<STDERR>']).records.items():\n",
+ " recs.info()\n",
+ " print()"
]
},
{
@@ -179,7 +224,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.5"
+ "version": "3.9.2"
}
},
"nbformat": 4,
=====================================
darshan-util/pydarshan/examples/99_darshan-experimental_report-algebra.ipynb
=====================================
The diff for this file was not included because it is too large.
=====================================
darshan-util/pydarshan/examples/99_darshan-low-level-direct-libdarshanutils-interaction.ipynb
=====================================
@@ -11,9 +11,16 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "**Note:** In general be advised to use this interface with care until the API stablelized. Many parts of the functionality exposed here used to be for internal use by the command-line utilities only. For very performance sensitive analysis and for troubleshooting this notebook documents how to directly interface with libdarshan-util.so none the less."
+ "**Note:** In general be advised to use this interface with care until the API stablelized. Many parts of the functionality exposed here used to be for internal use by the darshan-util command-line utilities only. None the less, for very performance sensitive analysis and for troubleshooting this notebook documents how to directly interface with `libdarshan-util.so`."
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -42,9 +49,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "CFFI can used in multiple different configruations allowing for different performance trade offs.\n",
- "For PyDarshan we assume the library is proved as an already compiled binary (with CFFI in ABI mode), this way only valid header information for data types and function signatures have to be provided.\n",
- "A valid set of headers is provided as part of `darshan.api_def_c`:"
+ "CFFI can be used in multiple different configruations allowing for different performance trade offs.\n",
+ "For PyDarshan we assume the library is proved as an already compiled binary (thus we are using CFFI in ABI mode), this way only valid header information for data types and function signatures have to be provided.\n",
+ "A valid set of headers is provided as part of `darshan.backend.api_def_c`:"
]
},
{
@@ -141,12 +148,12 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "A darshan log contains besides some metadata describing the instrumented execution primarily log records collected by the different modules in compressed form. To access these a number of library interactions have to be performed to capture individual records.\n",
+ "A darshan log contains, besides some metadata describing the instrumented execution, primarily log records collected by the different modules in compressed form. To access these a number of library interactions have to be performed to capture individual records.\n",
"\n",
" 1) Open the log and optain a log handle\n",
- " 2) Request individual log records by requesting records for a particular index\n",
+ " 2) Request individual log records by requesting records for a particular module identifer\n",
" \n",
- "Module indexes are defined in ´darshan-log-format.h´:\n",
+ "Module identifiers are defined in ´darshan-log-format.h´:\n",
"\n",
" #define DARSHAN_MODULE_IDS \\\n",
" X(DARSHAN_NULL_MOD, \"NULL\", DARSHAN_NULL_VER, NULL) \\\n",
@@ -284,7 +291,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.5"
+ "version": "3.9.2"
}
},
"nbformat": 4,
=====================================
darshan-util/pydarshan/setup.py
=====================================
@@ -58,6 +58,6 @@ setup(
test_suite='tests',
tests_require=test_requirements,
url='https://www.mcs.anl.gov/research/projects/darshan/',
- version='0.0.7',
+ version='0.0.8.4',
zip_safe=False,
)
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/compare/755e68535b3129179bdf312c8351dd4b9b6a3e40...054de6708ac8ff0ea3b6f994dae6b6c0fe89a5ce
--
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/compare/755e68535b3129179bdf312c8351dd4b9b6a3e40...054de6708ac8ff0ea3b6f994dae6b6c0fe89a5ce
You're receiving this email because of your account on xgitlab.cels.anl.gov.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20210325/cda15bcd/attachment-0001.html>
More information about the Darshan-commits
mailing list