[Darshan-commits] [Git][darshan/darshan][autoperf-mod-update] APMPI module analysis python script - optimizations

[Sudheer Chunduri]

Sudheer Chunduri pushed to branch autoperf-mod-update at darshan / darshan


Commits:
e279af8e by Sudheer Chunduri at 2021-02-17T23:36:39+00:00
APMPI module analysis python script - optimizations

- - - - -


1 changed file:

- darshan-util/pydarshan/examples/01_darshan-apmpi.py


Changes:

=====================================
darshan-util/pydarshan/examples/01_darshan-apmpi.py
=====================================
@@ -7,26 +7,6 @@
 
 # By default all AMMPI module records, metadata, and the name records are loaded when opening a Darshan log:
 
-
-"""
-get_ipython().run_line_magic('env', 'LD_LIBRARY_PATH=/projects/Performance/chunduri/Software/Temp/build/darshan-util/install/lib/')
-
-get_ipython().run_line_magic('env', 'PE_PKGCONFIG_LIBS=darshan-runtime')
-get_ipython().run_line_magic('env', 'PKG_CONFIG_PATH=/projects/Performance/chunduri/Software/Temp/build/darshan-runtime/lib/pkgconfig')
-get_ipython().run_line_magic('env', 'PATH=/projects/Performance/chunduri/Software/Temp/build/darshan-utils/bin:/opt/anaconda3x/bin:/opt/anaconda3x/condabin:/projects/Performance/chunduri/Work_backup_June252017/software/install/autotools/bin:/bin:/sbin:/opt/anaconda3x/bin:/usr/bin/usr/sbin:/usr/local/sbin:/usr/sbin:/dbhome/db2cat/sqllib/bin:/dbhome/db2cat/sqllib/adm:/dbhome/db2cat/sqllib/misc:/dbhome/db2cat/sqllib/gskit/bin:/opt/ibutils/bin:/home/chunduri/bin:/projects/Performance/chunduri/Software/Temp/build/darshan-util/install/bin')
-
-get_ipython().run_line_magic('env', 'darshan_prefix=/projects/Performance/chunduri/Software/Temp/build/darshan-runtime/install')
-get_ipython().run_line_magic('env', 'darshan_share=/projects/Performance/chunduri/Software/Temp/build/darshan-runtime/install/share')
-get_ipython().run_line_magic('env', 'darshan_libdir= -L${darshan_prefix}/lib')
-
-get_ipython().system('echo $LD_LIBRARY_PATH')
-get_ipython().system('echo $PATH')
-get_ipython().system('echo $PKG_CONFIG_PATH')
-"""
-
-#get_ipython().system("pwd")
-#get_ipython().run_line_magic("cd", "..")
-#get_ipython().system("pwd")
 import argparse
 import darshan
 
@@ -41,7 +21,6 @@ import logging
 from darshan.backend.cffi_backend import ffi
 
 logger = logging.getLogger(__name__)
-# logger
 from darshan.report import DarshanReport
 import darshan.backend.cffi_backend as backend
 import darshan
@@ -71,9 +50,8 @@ def main():
     if "APMPI" not in report.modules:
         print("This log does not contain AutoPerf MPI data")
         return
-
     r = report.mod_read_all_apmpi_records("APMPI")
-    # report.data.keys()
+
     report.update_name_records()
     report.info()
 
@@ -87,6 +65,7 @@ def main():
         )
 
     df_apmpi = pd.DataFrame()
+    list_mpiop = []
     for rec in report.records["APMPI"][1:]:  # skip the first record which is header record
         mpi_nonzero_callcount = []
         for k, v in rec["all_counters"].items():
@@ -128,28 +107,12 @@ def main():
             if sync_flag:
                 mpiopstat["Total_SYNC_Time"] = rec["all_counters"][totalsync]
 
-            df_mpiop = pd.DataFrame([mpiopstat], columns=mpiopstat.keys())
-            df_rank = pd.concat([df_rank, df_mpiop], axis=0).reset_index(drop=True)
-        df_rank = df_rank.sort_values(by=["Total_Time"], ascending=False)
-        df_apmpi = pd.concat([df_apmpi, df_rank], axis=0).reset_index(drop=True)
-    print(df_apmpi)
-
+            list_mpiop.append(mpiopstat)
+    df_apmpi = pd.DataFrame(list_mpiop)
+    df_apmpi = df_apmpi.sort_values(by=["Rank", "Total_Time"], ascending=[True, False])
+    #print(df_apmpi)
+    #df_apmpi.to_csv('apmpi.csv', index=False)
     return
 
 if __name__ == '__main__':
     main()
-# print(time.time(), time.clock())
-
-
-# A few of the internal data structures explained:
-
-# In[ ]:
-
-
-# report.metadata         # dictionary with raw metadata from darshan log
-# report.modules          # dictionary with raw module info from darshan log (need: technical, module idx)
-# report.name_records     # dictionary for resovling name records: id -> path/name
-# report.records          # per module "dataframes"/dictionaries holding loaded records
-
-
-# The darshan report holds a variety of namespaces for report related data. All of them are also referenced in `report.data` at the moment, but reliance on this internal organization of the report object is discouraged once the API stabilized. Currently, `report.data` references the following information:



View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/commit/e279af8eef51ff9addd2a7359c07d3c3da952a59

-- 
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/commit/e279af8eef51ff9addd2a7359c07d3c3da952a59
You're receiving this email because of your account on xgitlab.cels.anl.gov.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20210217/d9adf4c0/attachment-0001.html>