From cbd1bce45a605df97aa00a42b7568d08419253f9 Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 15 Mar 2016 13:24:10 -0700 Subject: [PATCH] Fix two ways tests can hang Both have to do with the test runner's handling of the tests. With one it's the read thread somehow outliving the other threads (e.g. with ctrl-C). The other is due to a filled OS-level pipe's buffer causing a block while code is still holding the GIL in some gRPC core function. We can't empty the buffer from Python because the GIL is held, and the OS can't unblock because it's waiting for the buffer to get cleared: deadlock. --- src/python/grpcio/tests/_runner.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/python/grpcio/tests/_runner.py b/src/python/grpcio/tests/_runner.py index b0dbd92a495..32a31ce00e1 100644 --- a/src/python/grpcio/tests/_runner.py +++ b/src/python/grpcio/tests/_runner.py @@ -43,6 +43,13 @@ import uuid from tests import _loader from tests import _result +# This number needs to be large enough to outpace output on stdout and stderr +# from the gRPC core, otherwise we could end up in a potential deadlock. This +# stems from the OS waiting on someone to clear a filled pipe buffer while the +# GIL is held from a write to stderr from gRPC core, but said someone is in +# Python code thus necessitating GIL acquisition. +_READ_BYTES = 2**20 + class CapturePipe(object): """A context-manager pipe to redirect output to a byte array. @@ -76,6 +83,10 @@ class CapturePipe(object): flags = fcntl.fcntl(self._read_fd, fcntl.F_GETFL) fcntl.fcntl(self._read_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) self._read_thread = threading.Thread(target=self._read) + # If the user wants to exit from the Python program and hits ctrl-C and the + # read thread is somehow deadlocked with something else, the Python code may + # refuse to exit. This prevents that by making the read thread second-class. + self._read_thread.daemon = True self._read_thread.start() def stop(self): @@ -93,7 +104,7 @@ class CapturePipe(object): self.output = bytearray() while True: select.select([self._read_fd], [], []) - read_bytes = os.read(self._read_fd, 1024) + read_bytes = os.read(self._read_fd, _READ_BYTES) if read_bytes: self.output.extend(read_bytes) else: