harness: Kill console lockprocess if wait() hangs for more than 5 seconds

Signed-off-by: Simon Gerber <simon.gerber@inf.ethz.ch>

diff --git a/tools/harness/machines/eth.py b/tools/harness/machines/eth.py
index fbd04a4..b289480 100644 (file)
--- a/tools/harness/machines/eth.py
+++ b/tools/harness/machines/eth.py
@@ -11,6 +11,8 @@ import os, getpass, subprocess, socket, pty
 import debug, machines, eth_machinedata
 from machines import Machine, MachineLockedError
 
+from subprocess_timeout import wait_or_terminate
+
 TFTP_PATH='/home/netos/tftpboot'
 TOOLS_PATH='/home/netos/tools/bin'
 RACKBOOT=os.path.join(TOOLS_PATH, 'rackboot.sh')
@@ -93,7 +95,7 @@ class ETHBaseMachine(Machine):
         debug.verbose('quitting console process (%d)' % self.lockprocess.pid)
         # os.kill(self.lockprocess.pid, signal.SIGTERM)
         os.write(self.masterfd, "\x05c.")
-        self.lockprocess.wait()
+        wait_or_terminate(self.lockprocess)
         self.lockprocess = None
         self.masterfd = None
 

diff --git a/tools/harness/subprocess_timeout.py b/tools/harness/subprocess_timeout.py
new file mode 100644 (file)
index 0000000..7e5a0c4
--- /dev/null
+++ b/tools/harness/subprocess_timeout.py
@@ -0,0 +1,34 @@
+##########################################################################
+# Copyright (c) 2009-2016 ETH Zurich.
+# All rights reserved.
+#
+# This file is distributed under the terms in the attached LICENSE file.
+# If you do not find this file, copies can be found by writing to:
+# ETH Zurich D-INFK, Universitaetstr 6, CH-8092 Zurich. Attn: Systems Group.
+##########################################################################
+
+from threading import Timer
+
+# Wait for Popen instance p for timeout seconds and terminate/kill it after
+# the timeout expires
+# Adapted from
+# http://stackoverflow.com/questions/1191374/using-module-subprocess-with-timeout
+def wait_or_terminate(p, timeout=5):
+
+    # Kill process if terminate doesn't exit in 1 second
+    def cleanup(x):
+        k = lambda y: y.kill()
+        timer2 = Timer(1, k, [x])
+        try:
+            timer2.start()
+            x.terminate()
+        finally:
+            timer2.cancel()
+
+    # Termiate process if it doesn't voluntarily exit in `timeout` seconds
+    timer = Timer(timeout, cleanup, [p])
+    try:
+        timer.start()
+        p.wait()
+    finally:
+        timer.cancel()