[Darshan-commits] [Git][darshan/darshan][autoperf-mod-update] prints MPI stats from max, mean and mean MPI time ranks

[Sudheer Chunduri]

Sudheer Chunduri pushed to branch autoperf-mod-update at darshan / darshan


Commits:
25e7e0da by Sudheer Chunduri at 2021-02-18T18:45:02+00:00
prints MPI stats from max, mean and mean MPI time ranks

- - - - -


1 changed file:

- darshan-util/pydarshan/examples/01_darshan-apmpi.py


Changes:

=====================================
darshan-util/pydarshan/examples/01_darshan-apmpi.py
=====================================
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # coding: utf-8
 
-# # DarshanUtils for Python
+# # DarshanUtils for Python for processing APMPI records
 #
 # This notebook gives an overwiew of features provided by the Python bindings for DarshanUtils.
 
@@ -9,7 +9,6 @@
 
 import argparse
 import darshan
-
 import cffi
 import numpy
 import pandas
@@ -26,8 +25,19 @@ import darshan.backend.cffi_backend as backend
 import darshan
 import pandas as pd
 import time
+from rich import print  as rprint
+from rich import pretty
+from rich.panel import Panel
+from rich import inspect
+from rich.color import Color
+from rich.console import Console
+console = Console()
 
 pp = pprint.PrettyPrinter()
+pretty.install()
+#color = Color.parse("blue")
+
+#inspect(color, methods=True)
 
 
 def main():
@@ -58,15 +68,21 @@ def main():
     header_rec = report.records["APMPI"][0]
     print("# darshan log version: ", header_rec["version"])
     sync_flag = header_rec["sync_flag"]
-    print("APMPI Variance in total mpi time: ", header_rec["variance_total_mpitime"], "\n")
+    print(
+        "APMPI Variance in total mpi time: ", header_rec["variance_total_mpitime"], "\n"
+    )
     if sync_flag:
         print(
-            "APMPI Variance in total mpi sync time: ", header_rec["variance_total_mpisynctime"]
+            "APMPI Variance in total mpi sync time: ",
+            header_rec["variance_total_mpisynctime"],
         )
 
     df_apmpi = pd.DataFrame()
     list_mpiop = []
-    for rec in report.records["APMPI"][1:]:  # skip the first record which is header record
+    list_rank = []
+    for rec in report.records["APMPI"][
+        1:
+    ]:  # skip the first record which is header record
         mpi_nonzero_callcount = []
         for k, v in rec["all_counters"].items():
             if k.endswith("_CALL_COUNT") and v > 0:
@@ -108,11 +124,37 @@ def main():
                 mpiopstat["Total_SYNC_Time"] = rec["all_counters"][totalsync]
 
             list_mpiop.append(mpiopstat)
-    df_apmpi = pd.DataFrame(list_mpiop)
+        rankstat = {}
+        rankstat["Rank"] = rec["rank"]
+        rankstat["Node_ID"] = rec["node_name"]
+        rankstat["Call"] = "Total_MPI_time"
+        rankstat["Total_Time"] = rec["all_counters"]["RANK_TOTAL_MPITIME"]
+        list_rank.append(rankstat)
+    df_rank = pd.DataFrame(list_rank)
+    avg_total_time = df_rank["Total_Time"].mean()
+    max_total_time = df_rank["Total_Time"].max()
+    min_total_time = df_rank["Total_Time"].min()
+    max_rank = df_rank.loc[df_rank["Total_Time"].idxmax()]["Rank"]
+    min_rank = df_rank.loc[df_rank["Total_Time"].idxmin()]["Rank"]
+    # assumption: row index and rank id are same in df_rank 
+    # .. need to check if that is an incorrect assumption
+    mean_rank = (
+        (df_rank["Total_Time"] - df_rank["Total_Time"].mean()).abs().argsort()[:1][0]
+    )
+
+    list_combined = list_mpiop + list_rank
+    df_apmpi = pd.DataFrame(list_combined)
     df_apmpi = df_apmpi.sort_values(by=["Rank", "Total_Time"], ascending=[True, False])
-    #print(df_apmpi)
-    #df_apmpi.to_csv('apmpi.csv', index=False)
+    Panel.fit("[bold green] MPI stats for rank with maximum MPI time", border_style="blue")
+    console.print("[bold green] MPI stats for rank with maximum MPI time\n", df_apmpi.loc[df_apmpi["Rank"] == max_rank])
+    Panel.fit("[bold green] MPI stats for rank with minimum MPI time", border_style="blue")
+    print(df_apmpi.loc[df_apmpi["Rank"] == min_rank])
+    Panel.fit("[bold green] MPI stats for rank with mean MPI time", border_style="blue")
+    print(df_apmpi.loc[df_apmpi["Rank"] == mean_rank])
+    # print(df_apmpi)
+    # df_apmpi.to_csv('apmpi.csv', index=False)
     return
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     main()



View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/commit/25e7e0dacd3df7fa38d0a40c082c87564f07f6d7

-- 
View it on GitLab: https://xgitlab.cels.anl.gov/darshan/darshan/-/commit/25e7e0dacd3df7fa38d0a40c082c87564f07f6d7
You're receiving this email because of your account on xgitlab.cels.anl.gov.


-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.mcs.anl.gov/pipermail/darshan-commits/attachments/20210218/a9eba6fc/attachment-0001.html>