[Swift-commit] r7074 - trunk/src/org/griphyn/vdl/karajan

[hategan at ci.uchicago.edu]

Author: hategan
Date: 2013-09-16 21:05:09 -0500 (Mon, 16 Sep 2013)
New Revision: 7074

Modified:
   trunk/src/org/griphyn/vdl/karajan/HangChecker.java
Log:
if no swift deadlocks are found, use JMX to find JVM level deadlocks; also exit if such a deadlock is found

Modified: trunk/src/org/griphyn/vdl/karajan/HangChecker.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/HangChecker.java	2013-09-16 21:52:05 UTC (rev 7073)
+++ trunk/src/org/griphyn/vdl/karajan/HangChecker.java	2013-09-17 02:05:09 UTC (rev 7074)
@@ -20,6 +20,11 @@
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.PrintStream;
+import java.lang.management.LockInfo;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -82,6 +87,7 @@
                 int running = s.getRunning();
                 boolean allOverloaded = s.allOverloaded();
                 if (running == 0 && !Scheduler.getScheduler().isAnythingRunning() && !allOverloaded) {
+                    boolean found = false;
                     lastHandledSequenceNumber = crtSequenceNumber;
                     logger.warn("No events in " + (CHECK_INTERVAL / 1000) + "s.");
                     ByteArrayOutputStream os = new ByteArrayOutputStream();
@@ -89,15 +95,25 @@
                     dumpThreads(ps);
                     try {
                         Graph g = buildGraph();
-                        if (!findCycles(ps, g)) {
-                            findThreadsToBlame(ps, g);
+                        if (!found) {
+                            found = findCycles(ps, g);
                         }
+                        if (!found) {
+                            found = findThreadsToBlame(ps, g);
+                        }
                     }
                     catch (Exception e) {
                         logger.warn("Failed to build dependency graph", e);
                     }
+                    if (!found) {
+                        found = findJVMDeadlocks(ps);
+                    }
                     logger.warn(os.toString());
                     ps.close();
+                    if (found) {
+                        System.err.println("Irrecoverable error found. Exiting.");
+                        System.exit(99);
+                    }
                 }
             }
         }
@@ -106,6 +122,65 @@
         }
     }
     
+    private boolean findJVMDeadlocks(PrintStream pw) {;
+        try {
+            ThreadMXBean b = ManagementFactory.getThreadMXBean();
+            long[] ids = b.findDeadlockedThreads();
+            if (ids != null && ids.length != 0) {
+                ThreadInfo[] tis = b.getThreadInfo(ids, true, true); 
+                pw.println("\nDeadlocked Java threads found:");
+                for (ThreadInfo ti : tis) {
+                    printThreadInfo(pw, ti);
+                }
+                return true;
+            }
+        }
+        catch (Exception e) {
+            logger.warn("Exception caught trying to find JVM deadlocks", e);
+        }
+        return false;
+    }
+
+    private void printThreadInfo(PrintStream pw, ThreadInfo ti) {
+        pw.println("\tThread \"" + ti.getThreadName() + "\" (" + hex(ti.getThreadId()) + ")");
+        LockInfo l = ti.getLockInfo();
+        pw.println("\t\twaiting for " + format(l) + " held by " + ti.getLockOwnerName() + " (" + hex(ti.getLockOwnerId()) + ")");
+        Map<StackTraceElement, MonitorInfo> mlocs = new HashMap<StackTraceElement, MonitorInfo>();
+        MonitorInfo[] mis = ti.getLockedMonitors();
+        if (mis.length > 0) {
+            pw.println("\tMonitors held:");
+            for (MonitorInfo mi : mis) {
+                mlocs.put(mi.getLockedStackFrame(), mi);
+                pw.println("\t\t" + format(mi));
+            }
+        }
+        LockInfo[] lis = ti.getLockedSynchronizers();
+        if (lis.length > 0) {
+            pw.println("\tSynchronizers held:");
+            for (LockInfo li : lis) {
+                pw.println("\t\t" + format(li));
+            }
+        }
+        pw.println("\tStack trace:");
+        StackTraceElement[] stes = ti.getStackTrace();
+        for (StackTraceElement ste : stes) {
+            pw.print("\t\t" + ste.getClassName() + "." + ste.getMethodName() + ":" + ste.getLineNumber());
+            if (mlocs.containsKey(ste)) {
+                pw.print(" -> locked " + format(mlocs.get(ste)));
+            }
+            pw.println();
+        }
+        pw.println();
+    }
+
+    private String format(LockInfo l) {
+        return l.getClassName() + " (" + hex(l.getIdentityHashCode()) + ")";
+    }
+
+    private String hex(long x) {
+        return String.format("0x%08x", x);
+    }
+
     public void dumpThreads() {
         dumpThreads(System.out);
     }
@@ -193,7 +268,7 @@
     }
 
     
-    private void findThreadsToBlame(PrintStream ps, Graph g) {
+    private boolean findThreadsToBlame(PrintStream ps, Graph g) {
         Map<LWThread, DSHandle> wt = WaitingThreadsMonitor.getAllThreads();
         Set<LWThread> sl = g.nodeSet();
         Set<LWThread> loners = new HashSet<LWThread>(wt.keySet());
@@ -211,6 +286,7 @@
             }
             ps.println("----");
         }
+        return !loners.isEmpty();
     }
 
     private Graph buildGraph() throws VariableNotFoundException {