[Darshan-commits] [Git][darshan/darshan][autoperf-mod-update] overwrite changes to existing jupyter notebooks
Shane Snyder
xgitlab at cels.anl.gov
Mon Mar 29 18:58:19 CDT 2021
Shane Snyder pushed to branch autoperf-mod-update at darshan / darshan
Commits:
d0219299 by Shane Snyder at 2021-03-29T18:57:18-05:00
overwrite changes to existing jupyter notebooks
- - - - -
4 changed files:
- darshan-util/pydarshan/examples/01_darshan-introduction.ipynb
- darshan-util/pydarshan/examples/02_darshan-plotting-matplotlib.ipynb
- darshan-util/pydarshan/examples/99-graph-visualization.ipynb
- darshan-util/pydarshan/examples/99_darshan-experimental.ipynb
Changes:
=====================================
darshan-util/pydarshan/examples/01_darshan-introduction.ipynb
=====================================
@@ -16,163 +16,23 @@
"By default all records, metadata, available modules and the name records are loaded when opening a Darshan log:"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "%env LD_LIBRARY_PATH=/projects/Performance/chunduri/Software/Temp/build/darshan-util/install/lib/"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "%env PE_PKGCONFIG_LIBS=darshan-runtime \n",
- "%env PKG_CONFIG_PATH=/projects/Performance/chunduri/Software/Temp/build/darshan-runtime/lib/pkgconfig \n",
- "%env PATH=/projects/Performance/chunduri/Software/Temp/build/darshan-utils/bin:/opt/anaconda3x/bin:/opt/anaconda3x/condabin:/projects/Performance/chunduri/Work_backup_June252017/software/install/autotools/bin:/bin:/sbin:/opt/anaconda3x/bin:/usr/bin/usr/sbin:/usr/local/sbin:/usr/sbin:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc:/dbhome/db2cat/sqllib/gskit/bin:/opt/ibutils/bin:/home/chunduri/bin:/projects/Performance/chunduri/Software/Temp/build/darshan-util/install/bin"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "%env darshan_prefix=/projects/Performance/chunduri/Software/Temp/build/darshan-runtime/install\n",
- "%env darshan_share=/projects/Performance/chunduri/Software/Temp/build/darshan-runtime/install/share\n",
- "%env darshan_libdir= -L${darshan_prefix}/lib"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "!echo $LD_LIBRARY_PATH\n",
- "!echo $PATH\n",
- "!echo $PKG_CONFIG_PATH"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "/lus/theta-fs0/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples\n",
- "/lus/theta-fs0/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan\n",
- "/lus/theta-fs0/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan\n"
- ]
- }
- ],
- "source": [
- "!pwd\n",
- "%cd ..\n",
- "!pwd"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "import cffi\n",
- "import numpy\n",
- "import pandas\n",
- "import matplotlib\n",
- "# ffi = cffi.FFI()\n",
- "# libdutil = ffi.dlopen(\"/projects/Performance/chunduri/Software/Temp/build/darshan-util/install/lib/libdarshan-util.so\")"
- ]
- },
{
"cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [],
- "source": [
- "from darshan.backend.cffi_backend import ffi"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [],
- "source": [
- "import logging\n",
- "logger = logging.getLogger(__name__)\n",
- "logger\n",
- "from darshan.report import DarshanReport"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [],
- "source": [
- "import darshan.backend.cffi_backend as backend"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "import darshan"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Filename: /projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example.darshan\n",
- "Times: 2017-03-20 09:07:47 to 2017-03-20 09:09:43 (Duration 0:01:56)\n",
- "Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
- "Processes: 2048\n",
- "JobID: 4478544\n",
- "UID: 69615\n",
- "Modules in Log: ['POSIX', 'MPI-IO', 'LUSTRE', 'STDIO']\n",
- "Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 1, 'LUSTRE': 1}\n",
- "Name Records: 4\n",
- "Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
- "DarshanReport: id(139846982675088) (tmp)\n"
- ]
- }
- ],
- "source": [
- "#import darshan\n",
+ "import darshan\n",
"\n",
- "report = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example.darshan\", read_all=True) # Default behavior\n",
+ "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True) # Default behavior\n",
"report.info()"
]
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": null,
"metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "{'POSIX': {'len': 186, 'ver': 3, 'idx': 1, 'num_records': 1},\n",
- " 'MPI-IO': {'len': 154, 'ver': 2, 'idx': 2, 'num_records': 1},\n",
- " 'LUSTRE': {'len': 87, 'ver': 1, 'idx': 7, 'num_records': 1},\n",
- " 'STDIO': {'len': 3234, 'ver': 1, 'idx': 8, 'num_records': 129}}"
- ]
- },
- "execution_count": 17,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"report.modules"
]
@@ -271,7 +131,7 @@
"outputs": [],
"source": [
"import darshan\n",
- "report = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example.darshan\", read_all=False, lookup_name_records=True) # Loads no records!"
+ "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=False, lookup_name_records=True) # Loads no records!"
]
},
{
@@ -294,6 +154,13 @@
"Additional records then can be loaded selectively, for example, on a per module basis:"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Exporting Data"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -307,7 +174,17 @@
"metadata": {},
"outputs": [],
"source": [
- "report.mod_read_all_records(\"STDIO\", dtype=\"pandas\")"
+ "report.mod_read_all_records(\"STDIO\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(report.records['STDIO'].to_df()['counters'])\n",
+ "display(report.records['STDIO'].to_df()['fcounters'])"
]
},
{
@@ -316,10 +193,18 @@
"metadata": {},
"outputs": [],
"source": [
- "print('id', report.records['STDIO'][0]['id'])\n",
- "print('rank', report.records['STDIO'][0]['rank'])\n",
- "display(report.records['STDIO'][0]['counters'])\n",
- "display(report.records['STDIO'][0]['fcounters'])"
+ "# by default, export to pandas dataframes using .to_df() attaches id and rank information \n",
+ "# for aggregations, this can be supressed by providing attach=None, allowing you to get plots with sensible ranges directly using pandas plotting\n",
+ "report.records['STDIO'].to_df(attach=['rank'])['fcounters'].plot.box(vert=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "report.records['STDIO'].to_df(attach=['rank'])['counters'].plot.box(vert=False)"
]
},
{
@@ -335,8 +220,7 @@
"metadata": {},
"outputs": [],
"source": [
- "report.mod_read_all_records(\"STDIO\", dtype='dict')\n",
- "report.records['STDIO'][0]"
+ "report.mod_read_all_records(\"STDIO\")"
]
},
{
@@ -344,7 +228,9 @@
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": []
+ "source": [
+ "report.records['STDIO'][0].to_dict()"
+ ]
},
{
"cell_type": "markdown",
@@ -360,14 +246,24 @@
"outputs": [],
"source": [
"report.mod_read_all_records(\"STDIO\")\n",
- "report.records['STDIO'][0]"
+ "report.records['STDIO'][0].to_numpy()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(type(report.records['STDIO'][0].to_numpy()[0]['counters']))\n",
+ "print(type(report.records['STDIO'][0].to_numpy()[0]['fcounters']))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "#### The Log in Memory\n",
+ "#### The Darshan Log in Memory\n",
"\n",
"Let's have a look at how calling `report.mod_read_all_records(\"STDIO\")` changed the state of the log in memory."
]
@@ -399,7 +295,8 @@
"\n",
"# show first 10 records\n",
"for rec in report.records['STDIO'][0:10]:\n",
- " print(rec)"
+ " print(rec)\n",
+ " # do something with the record"
]
},
{
@@ -415,7 +312,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Report aggregations and summarization remains**experimental** for now, mostly to allow interfaces to stabilize. But experimental features can be switched on easily by invoking `darshan.enable_experimental()`:"
+ "Report aggregations and summarization remains **experimental** for now, mostly to allow interfaces to stabilize. But experimental features can be switched on easily by invoking `darshan.enable_experimental()`:"
]
},
{
@@ -459,31 +356,54 @@
"import darshan\n",
"darshan.enable_experimental()\n",
"\n",
- "report = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example.darshan\", read_all=True)\n",
+ "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
"report.name_records"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# The original report for reference. Take note of the \"Loaded Records\" section\n",
+ "report.info()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# name_records maybe filenames (or ids)\n",
+ "# Note how only records of the STDIO module remain\n",
+ "report.filter(name_records=['<STDIN>', '<STDOUT>', '<STDERR>']).info()"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "scrolled": true
+ "tags": []
},
"outputs": [],
"source": [
- "report.filter(name_records=[6301063301082038805, 15920181672442173319]).records"
+ "# name_records using a id\n",
+ "# Note how only one POSIX, one MPI-IO and one LUSTRE record remain\n",
+ "report.filter(name_records=[6301063301082038805]).info()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "scrolled": true
+ "tags": []
},
"outputs": [],
"source": [
"# reduce all after filtering\n",
- "report.filter(pattern=\"*.hdf5\").reduce().records"
+ "report.filter(pattern=\"*.hdf5\").reduce().info()"
]
},
{
@@ -591,7 +511,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "scrolled": true
+ "tags": []
},
"outputs": [],
"source": [
@@ -599,8 +519,8 @@
"from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
"from darshan.experimental.plots.matplotlib import plot_opcounts\n",
"\n",
- "r1 = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example.darshan\", read_all=True)\n",
- "r2 = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example2.darshan\", read_all=True)\n",
+ "r1 = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True, dtype='numpy')\n",
+ "r2 = darshan.DarshanReport(\"example-logs/example2.darshan\", read_all=True, dtype='numpy')\n",
"rx = r1 + r2\n",
"\n",
"for r in [r1, r2, rx]:\n",
@@ -662,7 +582,7 @@
"import darshan\n",
"darshan.enable_experimental(verbose=False)\n",
"\n",
- "r3 = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example.darshan\")\n",
+ "r3 = darshan.DarshanReport(\"example-logs/example.darshan\", dtype='numpy')\n",
"r3.mod_read_all_records('POSIX')\n",
"\n",
"from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
@@ -678,7 +598,7 @@
"import darshan\n",
"darshan.enable_experimental(verbose=False)\n",
"\n",
- "r3 = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example.darshan\")\n",
+ "r3 = darshan.DarshanReport(\"example-logs/example.darshan\", dtype='numpy')\n",
"r3.mod_read_all_records('MPI-IO')\n",
"\n",
"from darshan.experimental.plots.matplotlib import plot_access_histogram\n",
@@ -689,14 +609,14 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "scrolled": true
+ "tags": []
},
"outputs": [],
"source": [
"import darshan\n",
"darshan.enable_experimental(verbose=False)\n",
"\n",
- "r3 = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/example.darshan\")\n",
+ "r3 = darshan.DarshanReport(\"example-logs/example.darshan\", dtype='numpy')\n",
"r3.read_all()\n",
"\n",
"from darshan.experimental.plots.matplotlib import plot_opcounts\n",
@@ -712,13 +632,6 @@
"DXT records are also supported, and can be loaded individually on a per module basis as follows:\n"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
{
"cell_type": "code",
"execution_count": null,
@@ -727,7 +640,7 @@
"source": [
"import darshan\n",
"\n",
- "report2 = darshan.DarshanReport(\"/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples/example-logs/dxt.darshan\")\n",
+ "report2 = darshan.DarshanReport(\"example-logs/dxt.darshan\")\n",
"report2.info()"
]
},
@@ -737,7 +650,7 @@
"metadata": {},
"outputs": [],
"source": [
- "report2.records['DXT_POSIX'][0].keys()"
+ "report2.records['DXT_POSIX'][0]._records[0].keys()"
]
},
{
@@ -747,13 +660,6 @@
"Sometimes it is easier to visualize or transform data to get an overview:"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
{
"cell_type": "code",
"execution_count": null,
@@ -863,7 +769,7 @@
"outputs": [],
"source": [
"import darshan\n",
- "report = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True)\n",
+ "report = darshan.DarshanReport(\"example-logs/ior_hdf5_example.darshan\", read_all=True)\n",
"report.to_json()"
]
},
@@ -875,7 +781,7 @@
"\n",
"Currently, playing with two modes, both have their pros and cons.\n",
"\n",
- "Generally, should expose errors and let users handle them. At the same time, just skipping invalid load requests does little harm but greatly improves convienince.\n",
+ "Generally, should expose errors and let users handle them. At the same time, just skipping invalid load requests does little harm but greatly improves convenience.\n",
"\n",
"Could add a switch to enable disable these guards :/"
]
@@ -897,7 +803,7 @@
"metadata": {},
"outputs": [],
"source": [
- "report.mod_read_all_records(\"ABC\") # Expect KeyError"
+ "report.mod_read_all_records(\"MOD_ABC\") # Expect KeyError"
]
},
{
@@ -908,6 +814,13 @@
"source": [
"report.mod_read_all_dxt_records(\"ABC\") # Expect warning, but not exception"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -926,7 +839,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.4"
+ "version": "3.8.5"
}
},
"nbformat": 4,
=====================================
darshan-util/pydarshan/examples/02_darshan-plotting-matplotlib.ipynb
=====================================
The diff for this file was not included because it is too large.
=====================================
darshan-util/pydarshan/examples/99-graph-visualization.ipynb
=====================================
@@ -15,37 +15,29 @@
"cell_type": "code",
"execution_count": 1,
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "/lus/theta-fs0/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan/examples\n",
- "/lus/theta-fs0/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan\n",
- "/lus/theta-fs0/projects/Performance/chunduri/Software/Temp/darshan/darshan-util/pydarshan\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "!pwd\n",
- "%cd ..\n",
- "!pwd\n",
"import darshan\n",
"darshan.enable_experimental()"
]
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "[]"
+ "['darshan-graph/pq_app_read_id71317_7-31-5657-2037904274838284930_55623.darshan',\n",
+ " 'darshan-graph/pq_app_read_id71344_7-31-5658-2037904274838284930_55623.darshan',\n",
+ " 'darshan-graph/pq_app_write_id71303_7-31-5657-2037904274838284930_55623.darshan',\n",
+ " 'darshan-graph/pq_app_write_id71310_7-31-5657-2037904274838284930_55623.darshan',\n",
+ " 'darshan-graph/pq_app_write_id71296_7-31-5657-2037904274838284930_55623.darshan',\n",
+ " 'darshan-graph/pq_app_readAB_writeC_id71326_7-31-5658-2037904274838284930_55623.darshan']"
]
},
- "execution_count": 3,
+ "execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
@@ -53,7 +45,7 @@
"source": [
"# find relevant darshan logs\n",
"import glob\n",
- "darshan_logs = glob.glob(\"examples/sdarshan-graph/*.darshan\")\n",
+ "darshan_logs = glob.glob(\"darshan-graph/*.darshan\")\n",
"darshan_logs"
]
},
@@ -66,7 +58,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -452,7 +444,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.4"
+ "version": "3.6.10"
}
},
"nbformat": 4,
=====================================
darshan-util/pydarshan/examples/99_darshan-experimental.ipynb
=====================================
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
@@ -21,18 +21,26 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
- "ename": "ModuleNotFoundError",
- "evalue": "No module named 'darshan'",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
- "\u001b[0;32m<ipython-input-2-7f8bd811d6e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mdarshan\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mdarshan\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_experimental\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'darshan'"
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Added method agg_ioops to DarshanReport.\n",
+ "Added method create_dxttimeline to DarshanReport.\n",
+ "Added method create_sankey to DarshanReport.\n",
+ "Added method create_time_summary to DarshanReport.\n",
+ "Added method mod_agg_iohist to DarshanReport.\n",
+ "Added method print_module_records to DarshanReport.\n",
+ "Added method records_as_dict to DarshanReport.\n",
+ "Added method summarize to DarshanReport.\n",
+ "Added method create_timeline to DarshanReport.\n",
+ "Added method filter to DarshanReport.\n",
+ "Added method reduce to DarshanReport.\n",
+ "Added method name_records_summary to DarshanReport.\n",
+ "Added method merge to DarshanReport.\n"
]
}
],
@@ -43,7 +51,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 25,
"metadata": {
"scrolled": true
},
@@ -52,120 +60,152 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Filename: example.darshan\n",
+ "Filename: example-logs/example.darshan\n",
"Times: 2017-03-20 10:07:47 to 2017-03-20 10:09:43 (Duration 0:01:56)\n",
"Executeable: /global/project/projectdirs/m888/glock/tokio-abc-results/bin.edison/vpicio_uni /scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5 32\n",
"Processes: 2048\n",
"JobID: 4478544\n",
"UID: 69615\n",
"Modules in Log: ['POSIX', 'MPI-IO', 'LUSTRE', 'STDIO']\n",
- "Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129}\n",
+ "Loaded Records: {'POSIX': 1, 'MPI-IO': 1, 'STDIO': 129, 'LUSTRE': 1}\n",
"Name Records: 4\n",
"Darshan/Hints: {'lib_ver': '3.1.3', 'h': 'romio_no_indep_rw=true;cb_nodes=4'}\n",
- "DarshanReport: id(140456393426032) (tmp)\n"
+ "DarshanReport: id(140233011845200) (tmp)\n"
]
}
],
"source": [
- "r1 = darshan.DarshanReport(\"example.darshan\", read_all=True) # Default behavior\n",
+ "r1 = darshan.DarshanReport(\"example-logs/example.darshan\", read_all=True) # Default behavior\n",
"r1.info()"
]
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 26,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
- "{'POSIX': [{'id': 6301063301082038805,\n",
- " 'rank': -1,\n",
- " 'counters': array([ 2049, 18446744073709551615, 18446744073709551615,\n",
- " 0, 16402, 16404,\n",
- " 0, 0, 0,\n",
- " 0, 18446744073709551615, 18446744073709551615,\n",
- " 0, 0, 0,\n",
- " 2199023259968, 0, 2199023261831,\n",
- " 0, 0, 0,\n",
- " 16384, 0, 0,\n",
- " 8, 16401, 1048576,\n",
- " 0, 134217728, 0,\n",
- " 0, 0, 0,\n",
- " 0, 0, 0,\n",
- " 0, 0, 0,\n",
- " 4, 14, 0,\n",
- " 0, 0, 0,\n",
- " 0, 0, 16384,\n",
- " 0, 274743689216, 274743691264,\n",
- " 0, 0, 10240,\n",
- " 4096, 0, 0,\n",
- " 134217728, 272, 544,\n",
- " 328, 16384, 8,\n",
- " 2], dtype=uint64),\n",
- " 'fcounters': array([ 2.04900000e+03, -1.00000000e+00, -1.00000000e+00, 0.00000000e+00,\n",
- " 1.64020000e+04, 1.64040000e+04, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, -1.00000000e+00, -1.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 2.19902326e+12,\n",
- " 0.00000000e+00, 2.19902326e+12, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 1.63840000e+04, 0.00000000e+00, 0.00000000e+00,\n",
- " 8.00000000e+00, 1.64010000e+04, 1.04857600e+06, 0.00000000e+00,\n",
- " 1.34217728e+08, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 4.00000000e+00,\n",
- " 1.40000000e+01, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.63840000e+04,\n",
- " 0.00000000e+00, 2.74743689e+11, 2.74743691e+11, 0.00000000e+00,\n",
- " 0.00000000e+00, 1.02400000e+04, 4.09600000e+03, 0.00000000e+00,\n",
- " 0.00000000e+00, 1.34217728e+08, 2.72000000e+02, 5.44000000e+02,\n",
- " 3.28000000e+02, 1.63840000e+04, 8.00000000e+00, 2.00000000e+00])}],\n",
- " 'MPI-IO': [{'id': 6301063301082038805,\n",
- " 'rank': -1,\n",
- " 'counters': array([ 0, 2048, 0, 18,\n",
- " 0, 16384, 0, 0,\n",
- " 0, 0, 0, 0,\n",
- " 32768, 9, 0, 2199023259968,\n",
- " 0, 0, 134217728, 0,\n",
- " 0, 0, 0, 0,\n",
- " 0, 0, 0, 0,\n",
- " 0, 4, 14, 0,\n",
- " 0, 0, 0, 0,\n",
- " 0, 16384, 0, 134217728,\n",
- " 272, 544, 328, 16384,\n",
- " 8, 2, 2, 597,\n",
- " 1073741824, 1312, 1073741824], dtype=uint64),\n",
- " 'fcounters': array([0.00000000e+00, 2.04800000e+03, 0.00000000e+00, 1.80000000e+01,\n",
- " 0.00000000e+00, 1.63840000e+04, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
- " 3.27680000e+04, 9.00000000e+00, 0.00000000e+00, 2.19902326e+12,\n",
- " 0.00000000e+00, 0.00000000e+00, 1.34217728e+08, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 4.00000000e+00, 1.40000000e+01, 0.00000000e+00,\n",
- " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00,\n",
- " 0.00000000e+00, 1.63840000e+04, 0.00000000e+00, 1.34217728e+08,\n",
- " 2.72000000e+02, 5.44000000e+02, 3.28000000e+02, 1.63840000e+04,\n",
- " 8.00000000e+00, 2.00000000e+00, 2.00000000e+00, 5.97000000e+02,\n",
- " 1.07374182e+09, 1.31200000e+03, 1.07374182e+09])}],\n",
- " 'STDIO': [{'id': 15920181672442173319,\n",
- " 'rank': 0,\n",
- " 'counters': array([ 1, 18446744073709551615, 0,\n",
- " 6, 0, 0,\n",
- " 280, 0, 0,\n",
- " 279, 0, 0,\n",
- " 0], dtype=uint64),\n",
- " 'fcounters': array([ 1., -1., 0., 6., 0., 0., 280., 0., 0., 279., 0.,\n",
- " 0., 0.])}]}"
+ "{14734109647742566553: '<STDIN>',\n",
+ " 15920181672442173319: '<STDOUT>',\n",
+ " 7238257241479193519: '<STDERR>',\n",
+ " 6301063301082038805: '/scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5'}"
]
},
- "execution_count": 8,
+ "execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
- "r1.filter(name_records=[6301063301082038805, 15920181672442173319]).records"
+ "r1.name_records"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Module: POSIX\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "Ranks: {-1}\n",
+ "Name Records: {6301063301082038805}\n",
+ "\n",
+ "Module: MPI-IO\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "Ranks: {-1}\n",
+ "Name Records: {6301063301082038805}\n",
+ "\n",
+ "Module: STDIO\n",
+ "Records: 129\n",
+ "Coll. Type: collection\n",
+ "Ranks: {0, 512, 1024, 1536, 16, 528, 1040, 1552, 32, 544, 1056, 1568, 48, 560, 1072, 1584, 64, 576, 1088, 1600, 80, 592, 1104, 1616, 96, 608, 1120, 1632, 112, 624, 1136, 1648, 128, 640, 1152, 1664, 144, 656, 1168, 1680, 160, 672, 1184, 1696, 176, 688, 1200, 1712, 192, 704, 1216, 1728, 208, 720, 1232, 1744, 224, 736, 1248, 1760, 240, 752, 1264, 1776, 256, 768, 1280, 1792, 272, 784, 1296, 1808, 288, 800, 1312, 1824, 304, 816, 1328, 1840, 320, 832, 1344, 1856, 336, 848, 1360, 1872, 352, 864, 1376, 1888, 368, 880, 1392, 1904, 384, 896, 1408, 1920, 400, 912, 1424, 1936, 416, 928, 1440, 1952, 432, 944, 1456, 1968, 448, 960, 1472, 1984, 464, 976, 1488, 2000, 480, 992, 1504, 2016, 496, 1008, 1520, 2032}\n",
+ "Name Records: {7238257241479193519, 15920181672442173319}\n",
+ "\n",
+ "Module: LUSTRE\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "for key, recs in r1.records.items():\n",
+ " recs.info()\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Module: POSIX\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "Ranks: {-1}\n",
+ "Name Records: {6301063301082038805}\n",
+ "\n",
+ "Module: MPI-IO\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "Ranks: {-1}\n",
+ "Name Records: {6301063301082038805}\n",
+ "\n",
+ "Module: LUSTRE\n",
+ "Records: 1\n",
+ "Coll. Type: collection\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "for key, recs in r1.filter(name_records=['/scratch2/scratchdirs/glock/tokioabc-s.4478544/vpicio/vpicio.hdf5']).records.items():\n",
+ " recs.info()\n",
+ " print()"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 29,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Module: STDIO\n",
+ "Records: 128\n",
+ "Coll. Type: collection\n",
+ "Ranks: {0, 512, 1024, 1536, 16, 528, 1040, 1552, 32, 544, 1056, 1568, 48, 560, 1072, 1584, 64, 576, 1088, 1600, 80, 592, 1104, 1616, 96, 608, 1120, 1632, 112, 624, 1136, 1648, 128, 640, 1152, 1664, 144, 656, 1168, 1680, 160, 672, 1184, 1696, 176, 688, 1200, 1712, 192, 704, 1216, 1728, 208, 720, 1232, 1744, 224, 736, 1248, 1760, 240, 752, 1264, 1776, 256, 768, 1280, 1792, 272, 784, 1296, 1808, 288, 800, 1312, 1824, 304, 816, 1328, 1840, 320, 832, 1344, 1856, 336, 848, 1360, 1872, 352, 864, 1376, 1888, 368, 880, 1392, 1904, 384, 896, 1408, 1920, 400, 912, 1424, 1936, 416, 928, 1440, 1952, 432, 944, 1456, 1968, 448, 960, 1472, 1984, 464, 976, 1488, 2000, 480, 992, 1504, 2016, 496, 1008, 1520, 2032}\n",
+ "Name Records: {7238257241479193519}\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "for key, recs in r1.filter(name_records=['<STDERR>']).records.items():\n",
+ " recs.info()\n",
+ " print()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {
@@ -184,7 +224,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.7.4"
+ "version": "3.9.2"
}
},
"nbformat": 4,
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/commit/d0219299181b4fcdc480d2d654f65f310c950ee8
--
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/commit/d0219299181b4fcdc480d2d654f65f310c950ee8
You're receiving this email because of your account on xgitlab.cels.anl.gov.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20210329/900105d3/attachment-0001.html>
More information about the Darshan-commits
mailing list