[Darshan-commits] [Git][darshan/darshan][master] 2 commits: Add some troubleshooting notes about potentially missing dependencies and this...
Shane Snyder
xgitlab at cels.anl.gov
Tue Mar 16 13:49:41 CDT 2021
Shane Snyder pushed to branch master at darshan / darshan
Commits:
ae2b411e by Jakob Luettgau at 2021-03-16T13:49:38-05:00
Add some troubleshooting notes about potentially missing dependencies and this being tested only in bash (although all source are replaced by . now which enjoys better support.)
- - - - -
80c88c16 by Shane Snyder at 2021-03-16T13:49:38-05:00
Merge branch 'luettgau-python-cleanup' into 'master'
More Python Cleanup & Restore Wheel Building
See merge request darshan/darshan!85
- - - - -
12 changed files:
- darshan-util/pydarshan/.gitignore
- darshan-util/pydarshan/Makefile
- darshan-util/pydarshan/README.rst
- darshan-util/pydarshan/darshan/discover_darshan.py
- + darshan-util/pydarshan/darshan/extension.c
- + darshan-util/pydarshan/devel/activate
- + darshan-util/pydarshan/devel/build-libdarshanutil.sh
- darshan-util/pydarshan/docs/index.rst
- darshan-util/pydarshan/docs/usage.rst
- darshan-util/pydarshan/examples/tutorial/README
- darshan-util/pydarshan/setup.cfg
- darshan-util/pydarshan/setup.py
Changes:
=====================================
darshan-util/pydarshan/.gitignore
=====================================
@@ -1,4 +1,11 @@
playground/*
+examples/example-logs-private/
+devenv/*
+
+
+
+
+
# Byte-compiled / optimized / DLL files
__pycache__/
@@ -77,15 +84,6 @@ target/
# pyenv
.python-version
-# celery beat schedule file
-celerybeat-schedule
-
-# SageMath parsed files
-*.sage.py
-
-# dotenv
-.env
-
# virtualenv
.venv
venv/
@@ -101,5 +99,11 @@ ENV/
# mkdocs documentation
/site
-# mypy
-.mypy_cache/
+
+# Vim/Swap
+[._]*.s[a-v][a-z]
+!*.svg # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
=====================================
darshan-util/pydarshan/Makefile
=====================================
@@ -32,6 +32,17 @@ clean-test: # remove test and coverage artifacts
rm -rf .pytest_cache
rm -rf pkgtest
+clean-devenv:
+ rm -rf devenv/venv
+ rm -rf devenv/libdarshanutil
+ rm -rf devenv
+
+
+devenv:
+ ./devel/build-libdarshanutil.sh
+ python3 -m venv devenv/venv
+ source devenv/venv/bin/activate && pip install -r requirements_dev.txt
+
lint: # check style with flake8
flake8 darshan tests
@@ -48,7 +59,7 @@ coverage: # check code coverage quickly with the default Python
docs: clean-docs # generate Sphinx HTML documentation, including API docs
- sphinx-apidoc -M -H PyDarshan -o docs/ darshan
+ sphinx-apidoc -M -H PyDarshan -o docs/api/pydarshan darshan
$(MAKE) -C docs clean
$(MAKE) -C docs html
=====================================
darshan-util/pydarshan/README.rst
=====================================
@@ -1,17 +1,17 @@
-========
-Overview
-========
+=======================
+PyDarshan Documentation
+=======================
Python utilities to interact with Darshan log records of HPC applications.
-pydarshan requires darshan-utils version 3.3 or higher to be installed.
+PyDarshan requires darshan-utils version 3.3 or higher to be installed.
Features
--------
-* Darshan Report Object Wrapper
-* CFFI bindings to access darshan log files
+* Darshan Report Object for common interactive analysis tasks
+* Low-level CFFI bindings for efficient access to darshan log files
* Plots typically found in the darshan reports (matplotlib)
-* Auto-discover darshan-util.so (via darshan-parser in $PATH)
+* Bundled with darshan-utils while allowing site's darshan-utils to take precedence
Usage
@@ -25,7 +25,7 @@ A brief examples showing some of the basic functionality is the following::
import darshan
# Open darshan log
- report = darshan.DarshanReport('example.darshan')
+ report = darshan.DarshanReport('example.darshan', read_all=False)
# Load some report data
report.mod_read_all_records('POSIX')
=====================================
darshan-util/pydarshan/darshan/discover_darshan.py
=====================================
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
-"""Auxiliary to discover darshan-util install directory."""
+"""Auxiliary to discover a darshan-util installation."""
import os
=====================================
darshan-util/pydarshan/darshan/extension.c
=====================================
@@ -0,0 +1,67 @@
+/*
+ * NOTE: The pydarshan C extension is used to allow for an automatic
+ * build process to distribute binary wheels with a copy of pydarshan
+ * with darshan-utils included. It currently provides no functionality
+ * beyond this, but darshan-util functionality currently exposed using
+ * CFFI could be also integrated using the extension.
+ */
+
+
+#include "Python.h"
+
+
+static PyObject *
+py_verify_magic(PyObject *self, PyObject *args)
+{
+ /* kept for debugging */
+ double magic = 305419896; /* 0x12345678 */
+ return PyFloat_FromDouble(magic);
+}
+
+static PyMethodDef module_functions[] = {
+ {"verify_magic", py_verify_magic, METH_VARARGS, NULL},
+ {NULL, NULL} /* sentinel */
+};
+
+
+#if PY_MAJOR_VERSION >= 3
+ static struct PyModuleDef moduledef = {
+ PyModuleDef_HEAD_INIT,
+ "extension", /* m_name */
+ "", /* m_doc */
+ -1, /* m_size */
+ module_functions, /* m_methods */
+ NULL, /* m_reload */
+ NULL, /* m_traverse */
+ NULL, /* m_clear */
+ NULL, /* m_free */
+ };
+#endif
+
+
+static PyObject *
+moduleinit(void)
+{
+ PyObject *m;
+
+#if PY_MAJOR_VERSION >= 3
+ m = PyModule_Create(&moduledef);
+#else
+ m = Py_InitModule3("extension", module_functions, "Dependency Helper.");
+#endif
+ return m;
+}
+
+#if PY_MAJOR_VERSION < 3
+ PyMODINIT_FUNC
+ initextension(void)
+ {
+ moduleinit();
+ }
+#else
+ PyMODINIT_FUNC
+ PyInit_extension(void)
+ {
+ return moduleinit();
+ }
+#endif
=====================================
darshan-util/pydarshan/devel/activate
=====================================
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# make sourceable from everywhere
+THIS=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+>&2 echo $THIS
+
+
+# activate python environment containing dev tools
+. $THIS/../devenv/venv/bin/activate
+
+# expose darshan-util libs and binaries
+export PATH=$THIS/../devenv/libdarshanutil/bin:$PATH
+export LD_LIBRARY_PATH=$THIS/../devenv/libdarshanutil/lib:$LD_LIBRARY_PATH
+
+
+# ensure pydarshan is found
+export PYTHONPATH=$THIS/../:$PYTHONPATH
+
=====================================
darshan-util/pydarshan/devel/build-libdarshanutil.sh
=====================================
@@ -0,0 +1,14 @@
+# Build and install darshan-util
+
+PREFIX=$PWD/devenv/libdarshanutil
+
+cd ../
+./configure --prefix=${PREFIX} --enable-shared
+make install
+make distclean
+
+
+echo
+echo
+echo export PATH=$PREFIX/bin:\$PATH
+echo export LD_LIBRARY_PATH=$PREFIX/lib:\$LD_LIBRARY_PATH
=====================================
darshan-util/pydarshan/docs/index.rst
=====================================
@@ -18,7 +18,7 @@
:maxdepth: 2
:caption: API Reference:
- modules
+ api/pydarshan/modules
Indices and tables
==================
=====================================
darshan-util/pydarshan/docs/usage.rst
=====================================
@@ -6,24 +6,23 @@ Usage
Darshan Report Object Interface
-------------------------------
-To use pydarshan in a project::
+To get started with PyDarshan you can simply use::
- import darshan
+ import darshan
- report = darshan.DarshanReport(filename)
+ report = darshan.DarshanReport(filename)
- # read metadata, log records and name records
- report.read_all_generic_records()
+ # read metadata, log records and name records
+ report.read_all_generic_records()
# Python aggregations are still experimental and have to be activated:
# calculate or update aggregate statistics for currently loaded records
darshan.enable_experimental()
- report.summarize()
+ report.summarize()
- print(report.report)
-
+ print(report.report)
@@ -31,7 +30,7 @@ Directly interfacing through the CFFI Interface Wrappers
--------------------------------------------------------
Generally, it is more convienient to access a darshan log from Python using the default report object interface which also caches already fetched information such as log records on a per module basis.
-If this seems like an unwanted overhead the CFFI interface can be used which allows fine grained control on which information are loaded.
+If this seems like an unwanted overhead the CFFI interface can be used to gain fine-grained control about which information are being loaded.
To use pydarshan.cffi_parser in a project::
=====================================
darshan-util/pydarshan/examples/tutorial/README
=====================================
@@ -1,23 +1,23 @@
# Getting started with PyDarshan
-On a Linux distribution with python3 available, the following steps get you
-started. Step 1 could be ommitted, but generally it is recommended not to mess
-with the Python provided by the distribution.
+On a Linux distribution, using a Bash Shell and python3 available,
+the following steps get you started. Step 1 could be ommitted, but
+generally it is recommended not to mess with the Python provided
+by the distribution.
+1) Setup a "virtual environment", activate it and install pydarshan.
+We assume python-venv is installed on your system but some distributions do not include it by default.
-1) Setup a "virtual environment", activate it and install pydarshan:
-
-python3 -m venv ./venv # create a new virtual environment and store everything it requires into ./venv
-source ./venv/bin/activate # activate the environment
-pip install darshan # install darshan (and it's dependencies) to the virtual environment
-
+python3 -m venv ./venv # create a new virtual environment and store everything it requires into ./venv
+. ./venv/bin/activate # activate the environment
+pip install darshan # install darshan (and it's dependencies) to the virtual environment
2) Run as a prepared script:
$ python hello.py
-$ python plot.py
+$ python plot.py # This uses the interactive mode and may require you to install python3-tk
$ python tojson.py
$ python tojson.py --help
@@ -27,6 +27,7 @@ $ ./tojson.py ior_hdf5_example.darshan
+
3) Or interactively using the python intepreter shell:
$ python
=====================================
darshan-util/pydarshan/setup.cfg
=====================================
@@ -1,5 +1,5 @@
[bumpversion]
-current_version = 0.0.6
+current_version = 0.0.7
commit = False
tag = False
=====================================
darshan-util/pydarshan/setup.py
=====================================
@@ -14,7 +14,22 @@ setup_requirements = ['pytest-runner', ]
test_requirements = ['pytest']
+# NOTE: The Python C extension is currently only used to automate
+# the build process of binary wheels for distribution via PyPi.
+#
+# If you are building darshan yourself and make libdarshan-util.so
+# discoverable in the environment by means of LD_LIBRARY_PATH or
+# pkg-config there is no need to build the extension.
ext_modules = []
+if '--with-extension' in sys.argv:
+ ext_modules.append(Extension(
+ 'darshan.extension',
+ #optional=True,
+ sources=['darshan/extension.c'],
+ include_dirs=['/usr/include'],
+ libraries=['darshan-util']
+ ))
+ sys.argv.remove('--with-extension')
setup(
@@ -43,6 +58,6 @@ setup(
test_suite='tests',
tests_require=test_requirements,
url='https://www.mcs.anl.gov/research/projects/darshan/',
- version='0.0.6',
+ version='0.0.7',
zip_safe=False,
)
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/compare/286dda223cf9a90ad127fa2141e3d2f0c087faf1...80c88c161ecbf3447f06f28007323d070b88f64d
--
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/compare/286dda223cf9a90ad127fa2141e3d2f0c087faf1...80c88c161ecbf3447f06f28007323d070b88f64d
You're receiving this email because of your account on xgitlab.cels.anl.gov.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20210316/d8cb75ff/attachment-0001.html>
More information about the Darshan-commits
mailing list