[Swift-user] deadlock on workflow
Allan Espinosa
aespinosa at cs.uchicago.edu
Fri Oct 15 11:01:08 CDT 2010
When I changed my provider from coaster-persistent to plain caster, i
encountered this deadlock in my workflow:
2010-10-15 10:57:40
Full thread dump Java HotSpot(TM) 64-Bit Server VM (17.0-b16 mixed mode):
"Attach Listener" daemon prio=10 tid=0x0000000050d19000 nid=0x2a0d waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Queued Task Replication Sweeper" daemon prio=10 tid=0x00002aab385f1000 nid=0x2356 waiting on condition [0x00000000436e9000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.griphyn.vdl.karajan.lib.replication.Sweeper.run(Sweeper.java:44)
"NBS7" daemon prio=10 tid=0x0000000050d13800 nid=0x2355 waiting on condition [0x00000000435e8000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00002aab0cc945f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"NBS6" daemon prio=10 tid=0x0000000050d24000 nid=0x2354 waiting on condition [0x00000000434e7000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00002aab0cc945f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"NBS5" daemon prio=10 tid=0x0000000050d23000 nid=0x2352 waiting on condition [0x00000000433e6000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00002aab0cc945f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"NBS4" daemon prio=10 tid=0x00000000509ed800 nid=0x2350 waiting on condition [0x00000000432e5000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00002aab0cc945f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"NBS3" daemon prio=10 tid=0x00000000509ea800 nid=0x234f waiting on condition [0x00000000431e4000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00002aab0cc945f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"NBS2" daemon prio=10 tid=0x0000000050ce8800 nid=0x234e waiting on condition [0x00000000430e3000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00002aab0cc945f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"NBS1" daemon prio=10 tid=0x00000000509e3800 nid=0x234d waiting on condition [0x0000000042fe2000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00002aab0cc945f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"Scheduler" prio=10 tid=0x00002aab385e9800 nid=0x234c in Object.wait() [0x0000000042ee1000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00002aab0cc95200> (a org.griphyn.vdl.karajan.VDSAdaptiveScheduler)
at org.globus.cog.karajan.scheduler.LateBindingScheduler.sleep(LateBindingScheduler.java:305)
at org.globus.cog.karajan.scheduler.LateBindingScheduler.run(LateBindingScheduler.java:258)
- locked <0x00002aab0cc95200> (a org.griphyn.vdl.karajan.VDSAdaptiveScheduler)
"Progress ticker" daemon prio=10 tid=0x00000000509f4000 nid=0x234b waiting on condition [0x0000000042de0000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.griphyn.vdl.karajan.lib.RuntimeStats$ProgressTicker.run(RuntimeStats.java:137)
"Restart Log Sync" daemon prio=10 tid=0x00002aab385e8800 nid=0x234a in Object.wait() [0x0000000042cdf000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00002aab0cc7f350> (a org.globus.cog.karajan.workflow.nodes.restartLog.SyncThread)
at java.lang.Object.wait(Object.java:485)
at org.globus.cog.karajan.workflow.nodes.restartLog.SyncThread.run(SyncThread.java:45)
- locked <0x00002aab0cc7f350> (a org.globus.cog.karajan.workflow.nodes.restartLog.SyncThread)
"Swift console debugger" daemon prio=10 tid=0x0000000050d2c000 nid=0x2349 runnable [0x0000000042bde000]
java.lang.Thread.State: RUNNABLE
at java.io.FileInputStream.readBytes(Native Method)
at java.io.FileInputStream.read(FileInputStream.java:199)
at java.io.BufferedInputStream.fill(BufferedInputStream.java:218)
at java.io.BufferedInputStream.read(BufferedInputStream.java:237)
- locked <0x00002aaab43ce4a8> (a java.io.BufferedInputStream)
at org.griphyn.vdl.karajan.InHook.run(InHook.java:46)
at java.lang.Thread.run(Thread.java:619)
"network debugger" prio=10 tid=0x0000000050ce4800 nid=0x2348 runnable [0x0000000042add000]
java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept(Native Method)
at java.net.PlainSocketImpl.accept(PlainSocketImpl.java:390)
- locked <0x00002aab0cd99c60> (a java.net.SocksSocketImpl)
at java.net.ServerSocket.implAccept(ServerSocket.java:453)
at java.net.ServerSocket.accept(ServerSocket.java:421)
at org.griphyn.vdl.karajan.Monitor$Service.run(Monitor.java:428)
at java.lang.Thread.run(Thread.java:619)
"Overloaded Host Monitor" daemon prio=10 tid=0x0000000050b8a000 nid=0x2347 waiting on condition [0x00000000429dc000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.globus.cog.karajan.scheduler.OverloadedHostMonitor.run(OverloadedHostMonitor.java:47)
"Timer-0" daemon prio=10 tid=0x0000000050cac800 nid=0x2346 in Object.wait() [0x00000000428db000]
java.lang.Thread.State: TIMED_WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00002aab0cc2bf30> (a java.util.TaskQueue)
at java.util.TimerThread.mainLoop(Timer.java:509)
- locked <0x00002aab0cc2bf30> (a java.util.TaskQueue)
at java.util.TimerThread.run(Timer.java:462)
"NBS0" daemon prio=10 tid=0x00002aab38392800 nid=0x2345 waiting on condition [0x00000000427da000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00002aab0cc945f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:158)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1987)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:399)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:947)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
"pool-1-thread-4" prio=10 tid=0x0000000050b2a800 nid=0x2344 waiting for monitor entry [0x00000000426d8000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.griphyn.vdl.mapping.AbstractDataNode.addListener(AbstractDataNode.java:563)
- waiting to lock <0x00002aab08c9f3c8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.mapping.RootDataNode.innerInit(RootDataNode.java:50)
- locked <0x00002aab098f02c0> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.mapping.RootDataNode.handleClosed(RootDataNode.java:90)
at org.griphyn.vdl.mapping.AbstractDataNode.notifyListeners(AbstractDataNode.java:583)
- locked <0x00002aab0898bac8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.mapping.AbstractDataNode.closeShallow(AbstractDataNode.java:396)
- locked <0x00002aab0898bac8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.mapping.AbstractDataNode.setValue(AbstractDataNode.java:346)
at org.griphyn.vdl.mapping.RootDataNode.setValue(RootDataNode.java:218)
at org.griphyn.vdl.karajan.lib.SetFieldValue.deepCopy(SetFieldValue.java:83)
at org.griphyn.vdl.karajan.lib.SetFieldValue.function(SetFieldValue.java:46)
- locked <0x00002aab0cc8b0c0> (a org.griphyn.vdl.mapping.RootArrayDataNode)
- locked <0x00002aab0898bac8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:68)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:72)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.Sequential.startNext(Sequential.java:50)
at org.globus.cog.karajan.workflow.nodes.Sequential.executeChildren(Sequential.java:26)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.execute(FlowContainer.java:63)
at org.globus.cog.karajan.workflow.nodes.FlowNode.restart(FlowNode.java:238)
at org.globus.cog.karajan.workflow.nodes.FlowNode.start(FlowNode.java:289)
at org.globus.cog.karajan.workflow.nodes.FlowNode.controlEvent(FlowNode.java:402)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:343)
at org.globus.cog.karajan.workflow.FlowElementWrapper.event(FlowElementWrapper.java:230)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.events.EventTargetPair.run(EventTargetPair.java:44)
at edu.emory.mathcs.backport.java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:431)
at edu.emory.mathcs.backport.java.util.concurrent.FutureTask.run(FutureTask.java:166)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:643)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:668)
at java.lang.Thread.run(Thread.java:619)
"pool-1-thread-3" prio=10 tid=0x0000000050c7e000 nid=0x2343 waiting for monitor entry [0x00000000425d7000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.griphyn.vdl.karajan.lib.SetFieldValue.function(SetFieldValue.java:43)
- waiting to lock <0x00002aab0cc8b0c0> (a org.griphyn.vdl.mapping.RootArrayDataNode)
- locked <0x00002aab08ca6c98> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:68)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:72)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.Sequential.startNext(Sequential.java:50)
at org.globus.cog.karajan.workflow.nodes.Sequential.executeChildren(Sequential.java:26)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.execute(FlowContainer.java:63)
at org.globus.cog.karajan.workflow.nodes.FlowNode.restart(FlowNode.java:238)
at org.globus.cog.karajan.workflow.nodes.FlowNode.start(FlowNode.java:289)
at org.globus.cog.karajan.workflow.nodes.FlowNode.controlEvent(FlowNode.java:402)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:343)
at org.globus.cog.karajan.workflow.FlowElementWrapper.event(FlowElementWrapper.java:230)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.events.EventTargetPair.run(EventTargetPair.java:44)
at edu.emory.mathcs.backport.java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:431)
at edu.emory.mathcs.backport.java.util.concurrent.FutureTask.run(FutureTask.java:166)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:643)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:668)
at java.lang.Thread.run(Thread.java:619)
"pool-1-thread-2" prio=10 tid=0x00000000508d6800 nid=0x2342 waiting for monitor entry [0x00000000424d6000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.griphyn.vdl.karajan.lib.SetFieldValue.function(SetFieldValue.java:43)
- waiting to lock <0x00002aab0cc8b0c0> (a org.griphyn.vdl.mapping.RootArrayDataNode)
- locked <0x00002aab08994c90> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:68)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:72)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.Sequential.startNext(Sequential.java:50)
at org.globus.cog.karajan.workflow.nodes.Sequential.executeChildren(Sequential.java:26)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.execute(FlowContainer.java:63)
at org.globus.cog.karajan.workflow.nodes.FlowNode.restart(FlowNode.java:238)
at org.globus.cog.karajan.workflow.nodes.FlowNode.start(FlowNode.java:289)
at org.globus.cog.karajan.workflow.nodes.FlowNode.controlEvent(FlowNode.java:402)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:343)
at org.globus.cog.karajan.workflow.FlowElementWrapper.event(FlowElementWrapper.java:230)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.events.EventTargetPair.run(EventTargetPair.java:44)
at edu.emory.mathcs.backport.java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:431)
at edu.emory.mathcs.backport.java.util.concurrent.FutureTask.run(FutureTask.java:166)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:643)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:668)
at java.lang.Thread.run(Thread.java:619)
"pool-1-thread-1" prio=10 tid=0x0000000050955800 nid=0x2341 waiting for monitor entry [0x0000000041161000]
java.lang.Thread.State: BLOCKED (on object monitor)
at org.griphyn.vdl.karajan.lib.SetFieldValue.function(SetFieldValue.java:43)
- waiting to lock <0x00002aab0cc8b0c0> (a org.griphyn.vdl.mapping.RootArrayDataNode)
- locked <0x00002aab08c9f3c8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:68)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:72)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.Sequential.startNext(Sequential.java:50)
at org.globus.cog.karajan.workflow.nodes.Sequential.executeChildren(Sequential.java:26)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.execute(FlowContainer.java:63)
at org.globus.cog.karajan.workflow.nodes.FlowNode.restart(FlowNode.java:238)
at org.globus.cog.karajan.workflow.nodes.FlowNode.start(FlowNode.java:289)
at org.globus.cog.karajan.workflow.nodes.FlowNode.controlEvent(FlowNode.java:402)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:343)
at org.globus.cog.karajan.workflow.FlowElementWrapper.event(FlowElementWrapper.java:230)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.events.EventTargetPair.run(EventTargetPair.java:44)
at edu.emory.mathcs.backport.java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:431)
at edu.emory.mathcs.backport.java.util.concurrent.FutureTask.run(FutureTask.java:166)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:643)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:668)
at java.lang.Thread.run(Thread.java:619)
"Low Memory Detector" daemon prio=10 tid=0x00000000506e2000 nid=0x233f runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"CompilerThread1" daemon prio=10 tid=0x00000000506e0000 nid=0x233e waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"CompilerThread0" daemon prio=10 tid=0x00000000506da800 nid=0x233d waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" daemon prio=10 tid=0x00000000506d8000 nid=0x233c runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" daemon prio=10 tid=0x00000000506b3800 nid=0x233b in Object.wait() [0x0000000041eb4000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00002aab0cc2c2f8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:118)
- locked <0x00002aab0cc2c2f8> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:134)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:159)
"Reference Handler" daemon prio=10 tid=0x00000000506b1800 nid=0x233a in Object.wait() [0x0000000040eb5000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00002aaab43ce540> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:485)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)
- locked <0x00002aaab43ce540> (a java.lang.ref.Reference$Lock)
"main" prio=10 tid=0x0000000050650000 nid=0x2334 in Object.wait() [0x000000004051e000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00002aab0ca5e798> (a org.griphyn.vdl.karajan.VDL2ExecutionContext)
at java.lang.Object.wait(Object.java:485)
at org.globus.cog.karajan.workflow.ExecutionContext.waitFor(ExecutionContext.java:261)
- locked <0x00002aab0ca5e798> (a org.griphyn.vdl.karajan.VDL2ExecutionContext)
at org.griphyn.vdl.karajan.Loader.main(Loader.java:192)
"VM Thread" prio=10 tid=0x00000000506ad000 nid=0x2339 runnable
"GC task thread#0 (ParallelGC)" prio=10 tid=0x0000000050663000 nid=0x2335 runnable
"GC task thread#1 (ParallelGC)" prio=10 tid=0x0000000050665000 nid=0x2336 runnable
"GC task thread#2 (ParallelGC)" prio=10 tid=0x0000000050666800 nid=0x2337 runnable
"GC task thread#3 (ParallelGC)" prio=10 tid=0x0000000050668800 nid=0x2338 runnable
"VM Periodic Task Thread" prio=10 tid=0x00000000506ed000 nid=0x2340 waiting on condition
JNI global references: 1311
Found one Java-level deadlock:
=============================
"pool-1-thread-4":
waiting to lock monitor 0x0000000050ceb2e0 (object 0x00002aab08c9f3c8, a org.griphyn.vdl.mapping.RootDataNode),
which is held by "pool-1-thread-1"
"pool-1-thread-1":
waiting to lock monitor 0x00002aab387b99d8 (object 0x00002aab0cc8b0c0, a org.griphyn.vdl.mapping.RootArrayDataNode),
which is held by "pool-1-thread-4"
Java stack information for the threads listed above:
===================================================
"pool-1-thread-4":
at org.griphyn.vdl.mapping.AbstractDataNode.addListener(AbstractDataNode.java:563)
- waiting to lock <0x00002aab08c9f3c8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.mapping.RootDataNode.innerInit(RootDataNode.java:50)
- locked <0x00002aab098f02c0> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.mapping.RootDataNode.handleClosed(RootDataNode.java:90)
at org.griphyn.vdl.mapping.AbstractDataNode.notifyListeners(AbstractDataNode.java:583)
- locked <0x00002aab0898bac8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.mapping.AbstractDataNode.closeShallow(AbstractDataNode.java:396)
- locked <0x00002aab0898bac8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.mapping.AbstractDataNode.setValue(AbstractDataNode.java:346)
at org.griphyn.vdl.mapping.RootDataNode.setValue(RootDataNode.java:218)
at org.griphyn.vdl.karajan.lib.SetFieldValue.deepCopy(SetFieldValue.java:83)
at org.griphyn.vdl.karajan.lib.SetFieldValue.function(SetFieldValue.java:46)
- locked <0x00002aab0cc8b0c0> (a org.griphyn.vdl.mapping.RootArrayDataNode)
- locked <0x00002aab0898bac8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:68)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:72)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.Sequential.startNext(Sequential.java:50)
at org.globus.cog.karajan.workflow.nodes.Sequential.executeChildren(Sequential.java:26)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.execute(FlowContainer.java:63)
at org.globus.cog.karajan.workflow.nodes.FlowNode.restart(FlowNode.java:238)
at org.globus.cog.karajan.workflow.nodes.FlowNode.start(FlowNode.java:289)
at org.globus.cog.karajan.workflow.nodes.FlowNode.controlEvent(FlowNode.java:402)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:343)
at org.globus.cog.karajan.workflow.FlowElementWrapper.event(FlowElementWrapper.java:230)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.events.EventTargetPair.run(EventTargetPair.java:44)
at edu.emory.mathcs.backport.java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:431)
at edu.emory.mathcs.backport.java.util.concurrent.FutureTask.run(FutureTask.java:166)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:643)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:668)
at java.lang.Thread.run(Thread.java:619)
"pool-1-thread-1":
at org.griphyn.vdl.karajan.lib.SetFieldValue.function(SetFieldValue.java:43)
- waiting to lock <0x00002aab0cc8b0c0> (a org.griphyn.vdl.mapping.RootArrayDataNode)
- locked <0x00002aab08c9f3c8> (a org.griphyn.vdl.mapping.RootDataNode)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:68)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.griphyn.vdl.karajan.lib.VDLFunction.post(VDLFunction.java:72)
at org.globus.cog.karajan.workflow.nodes.AbstractSequentialWithArguments.childCompleted(AbstractSequentialWithArguments.java:192)
at org.globus.cog.karajan.workflow.nodes.Sequential.notificationEvent(Sequential.java:32)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:340)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.nodes.FlowNode.fireNotificationEvent(FlowNode.java:181)
at org.globus.cog.karajan.workflow.nodes.FlowNode.complete(FlowNode.java:309)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.post(FlowContainer.java:58)
at org.globus.cog.karajan.workflow.nodes.functions.Argument.post(Argument.java:45)
at org.globus.cog.karajan.workflow.nodes.Sequential.startNext(Sequential.java:50)
at org.globus.cog.karajan.workflow.nodes.Sequential.executeChildren(Sequential.java:26)
at org.globus.cog.karajan.workflow.nodes.FlowContainer.execute(FlowContainer.java:63)
at org.globus.cog.karajan.workflow.nodes.FlowNode.restart(FlowNode.java:238)
at org.globus.cog.karajan.workflow.nodes.FlowNode.start(FlowNode.java:289)
at org.globus.cog.karajan.workflow.nodes.FlowNode.controlEvent(FlowNode.java:402)
at org.globus.cog.karajan.workflow.nodes.FlowNode.event(FlowNode.java:343)
at org.globus.cog.karajan.workflow.FlowElementWrapper.event(FlowElementWrapper.java:230)
at org.globus.cog.karajan.workflow.events.EventBus.send(EventBus.java:173)
at org.globus.cog.karajan.workflow.events.EventTargetPair.run(EventTargetPair.java:44)
at edu.emory.mathcs.backport.java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:431)
at edu.emory.mathcs.backport.java.util.concurrent.FutureTask.run(FutureTask.java:166)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:643)
at edu.emory.mathcs.backport.java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:668)
at java.lang.Thread.run(Thread.java:619)
Found 1 deadlock.
This happened after 2 jobs (out of 400k) here's the main part of the workflow
int run_id = 664;
string datadir =
"/gpfs/pads/swift/aespinosa/science/cybershake/Results";
Station site = get_site(run_id);
Sgt sgt_var <ext; exec="getsgtvar.rb", r=run_id, s=site.name,
l="/gpfs/pads/swift/aespinosa/science/cybershake/SgtFiles">;
Rupture rups[] = get_ruptures(run_id);
foreach rup in rups {
string loc_sub = @strcat(datadir, "/", site.name, "/", rup.source, "/", rup.index);
Sgt sub <ext; exec="getsub.rb", l=loc_sub, n=site.name, s=rup.source,
r=rup.index>;
string var_str[] = get_variations( site, rup,
"/gpfs/pads/swift/aespinosa/science/cybershaku/RuptureVariations" );
Variation vars[] <array_mapper; files=var_str>;
sub = extract(sgt_var, site, vars[rup.size-1]);
Seismogram seis[] <simple_mapper; location=loc_sub,
prefix=@strcat("Seismogram_", site.name, "_", rup.source, "_", rup.index, "_"),
suffix=".grm">;
PeakValue peak[] <simple_mapper; location=loc_sub,
prefix=@strcat("PeakVals_", site.name, "_", rup.source, "_", rup.index, "_"),
suffix=".bsa">;
foreach var,i in vars {
seis[i] = seismogram(sub, var, site);
peak[i] = peak_calc(seis[i], var);
}
}
--
Allan M. Espinosa <http://amespinosa.wordpress.com>
PhD student, Computer Science
University of Chicago <http://people.cs.uchicago.edu/~aespinosa>
More information about the Swift-user
mailing list