Create Python stress test

src/python/grpcio/tests/stress/__init__.py

@@ -0,0 +1,28 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

src/python/grpcio/tests/stress/client.py

@@ -0,0 +1,132 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Entry point for running stress tests."""
+
+import argparse
+import Queue
+import threading
+
+from grpc.beta import implementations
+from src.proto.grpc.testing import metrics_pb2
+from src.proto.grpc.testing import test_pb2
+
+from tests.interop import methods
+from tests.qps import histogram
+from tests.stress import metrics_server
+from tests.stress import test_runner
+
+
+def _args():
+  parser = argparse.ArgumentParser(description='gRPC Python stress test client')
+  parser.add_argument(
+      '--server_addresses',
+      help='comma seperated list of hostname:port to run servers on',
+      default='localhost:8080', type=str)
+  parser.add_argument(
+      '--test_cases',
+      help='comma seperated list of testcase:weighting of tests to run',
+      default='large_unary:100',
+      type=str)
+  parser.add_argument(
+      '--test_duration_secs',
+      help='number of seconds to run the stress test',
+      default=-1, type=int)
+  parser.add_argument(
+      '--num_channels_per_server',
+      help='number of channels per server',
+      default=1, type=int)
+  parser.add_argument(
+      '--num_stubs_per_channel',
+      help='number of stubs to create per channel',
+      default=1, type=int)
+  parser.add_argument(
+      '--metrics_port',
+      help='the port to listen for metrics requests on',
+      default=8081, type=int)
+  return parser.parse_args()
+
+
+def _test_case_from_arg(test_case_arg):
+  for test_case in methods.TestCase:
+    if test_case_arg == test_case.value:
+      return test_case
+  else:
+    raise ValueError('No test case {}!'.format(test_case_arg))
+
+
+def _parse_weighted_test_cases(test_case_args):
+  weighted_test_cases = {}
+  for test_case_arg in test_case_args.split(','):
+    name, weight = test_case_arg.split(':', 1)
+    test_case = _test_case_from_arg(name)
+    weighted_test_cases[test_case] = int(weight)
+  return weighted_test_cases
+
+
+def run_test(args):
+  test_cases = _parse_weighted_test_cases(args.test_cases)
+  test_servers = args.server_addresses.split(',')
+  # Propagate any client exceptions with a queue
+  exception_queue = Queue.Queue()
+  stop_event = threading.Event()
+  hist = histogram.Histogram(1, 1)
+  runners = []
+
+  server = metrics_pb2.beta_create_MetricsService_server(
+      metrics_server.MetricsServer(hist))
+  server.add_insecure_port('[::]:{}'.format(args.metrics_port))
+  server.start()
+
+  for test_server in test_servers:
+    host, port = test_server.split(':', 1)
+    for _ in xrange(args.num_channels_per_server):
+      channel = implementations.insecure_channel(host, int(port))
+      for _ in xrange(args.num_stubs_per_channel):
+        stub = test_pb2.beta_create_TestService_stub(channel)
+        runner = test_runner.TestRunner(stub, test_cases, hist,
+                                        exception_queue, stop_event)
+        runners.append(runner)
+
+  for runner in runners:
+    runner.start()
+  try:
+    raise exception_queue.get(block=True, timeout=args.test_duration_secs)
+  except Queue.Empty:
+    # No exceptions thrown, success
+    pass
+  finally:
+    stop_event.set()
+    for runner in runners:
+      runner.join()
+      runner = None
+    server.stop(0)
+
+if __name__ == '__main__':
+  run_test(_args())

src/python/grpcio/tests/stress/metrics_server.py

@@ -0,0 +1,60 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""MetricsService for publishing stress test qps data."""
+
+import time
+
+from src.proto.grpc.testing import metrics_pb2
+
+GAUGE_NAME = 'python_overall_qps'
+
+
+class MetricsServer(metrics_pb2.BetaMetricsServiceServicer):
+
+  def __init__(self, histogram):
+    self._start_time = time.time()
+    self._histogram = histogram
+
+  def _get_qps(self):
+    count = self._histogram.get_data().count
+    delta = time.time() - self._start_time
+    self._histogram.reset()
+    self._start_time = time.time()
+    return int(count/delta)
+
+  def GetAllGauges(self, request, context):
+    qps = self._get_qps()
+    return [metrics_pb2.GaugeResponse(name=GAUGE_NAME, long_value=qps)]
+
+  def GetGauge(self, request, context):
+    if request.name != GAUGE_NAME:
+      raise Exception('Gauge {} does not exist'.format(request.name))
+    qps = self._get_qps()
+    return metrics_pb2.GaugeResponse(name=GAUGE_NAME, long_value=qps)

src/python/grpcio/tests/stress/test_runner.py

@@ -0,0 +1,73 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Thread that sends random weighted requests on a TestService stub."""
+
+import random
+import threading
+import time
+import traceback
+
+
+def _weighted_test_case_generator(weighted_cases):
+  weight_sum = sum(weighted_cases.itervalues())
+
+  while True:
+    val = random.uniform(0, weight_sum)
+    partial_sum = 0
+    for case in weighted_cases:
+      partial_sum += weighted_cases[case]
+      if val <= partial_sum:
+        yield case
+        break
+
+
+class TestRunner(threading.Thread):
+
+  def __init__(self, stub, test_cases, hist, exception_queue, stop_event):
+    super(TestRunner, self).__init__()
+    self._exception_queue = exception_queue
+    self._stop_event = stop_event
+    self._stub = stub
+    self._test_cases = _weighted_test_case_generator(test_cases)
+    self._histogram = hist
+
+  def run(self):
+    while not self._stop_event.is_set():
+      try:
+        test_case = next(self._test_cases)
+        start_time = time.time()
+        test_case.test_interoperability(self._stub, None)
+        end_time = time.time()
+        self._histogram.add((end_time - start_time)*1e9)
+      except Exception as e:
+        traceback.print_exc()
+        self._exception_queue.put(
+            Exception("An exception occured during test {}"
+                      .format(test_case), e))