Teach run_tests about individual Python tests

Allows:
- running python tests in parallel
- clearer Travis output
- subjects each python test to the five minute run_tests timeout,
  instead of ALL the tests to the five minute timeout
- easier benchmarking of which tests are slow

tools/run_tests/python_tests.json

@@ -0,0 +1,18 @@
+[
+  "grpc._adapter._blocking_invocation_inline_service_test",
+  "grpc._adapter._c_test",
+  "grpc._adapter._event_invocation_synchronous_event_service_test",
+  "grpc._adapter._future_invocation_asynchronous_event_service_test",
+  "grpc._adapter._links_test",
+  "grpc._adapter._lonely_rear_link_test",
+  "grpc._adapter._low_test",
+  "grpc.early_adopter.implementations_test",
+  "grpc.framework.assembly.implementations_test",
+  "grpc.framework.base.packets.implementations_test",
+  "grpc.framework.face.blocking_invocation_inline_service_test",
+  "grpc.framework.face.event_invocation_synchronous_event_service_test",
+  "grpc.framework.face.future_invocation_asynchronous_event_service_test",
+  "grpc.framework.foundation._later_test",
+  "grpc.framework.foundation._logging_pool_test"
+]
+

tools/run_tests/run_python.sh

@@ -36,24 +36,4 @@ cd $(dirname $0)/../..
 root=`pwd`
 export LD_LIBRARY_PATH=$root/libs/opt
 source python2.7_virtual_environment/bin/activate
-# TODO(issue 215): Properly itemize these in run_tests.py so that they can be parallelized.
+python2.7 -B -m $*
-# TODO(atash): Enable dynamic unused port discovery for this test.
-# TODO(mlumish): Re-enable this test when we can install protoc
-# python2.7 -B test/compiler/python_plugin_test.py --build_mode=opt
-python2.7 -B -m grpc._adapter._blocking_invocation_inline_service_test
-python2.7 -B -m grpc._adapter._c_test
-python2.7 -B -m grpc._adapter._event_invocation_synchronous_event_service_test
-python2.7 -B -m grpc._adapter._future_invocation_asynchronous_event_service_test
-python2.7 -B -m grpc._adapter._links_test
-python2.7 -B -m grpc._adapter._lonely_rear_link_test
-python2.7 -B -m grpc._adapter._low_test
-python2.7 -B -m grpc.early_adopter.implementations_test
-python2.7 -B -m grpc.framework.assembly.implementations_test
-python2.7 -B -m grpc.framework.base.packets.implementations_test
-python2.7 -B -m grpc.framework.face.blocking_invocation_inline_service_test
-python2.7 -B -m grpc.framework.face.event_invocation_synchronous_event_service_test
-python2.7 -B -m grpc.framework.face.future_invocation_asynchronous_event_service_test
-python2.7 -B -m grpc.framework.foundation._later_test
-python2.7 -B -m grpc.framework.foundation._logging_pool_test
-# TODO(nathaniel): Get tests working under 3.4 (requires 3.X-friendly protobuf)
-# python3.4 -B -m unittest discover -s src/python -p '*.py'

tools/run_tests/run_tests.py

@@ -57,8 +57,19 @@ class SimpleConfig(object):
     self.allow_hashing = (config != 'gcov')
     self.environ = environ
 
-  def job_spec(self, binary, hash_targets):
+  def job_spec(self, cmdline, hash_targets):
-    return jobset.JobSpec(cmdline=[binary],
+    """Construct a jobset.JobSpec for a test under this config
+
+       Args:
+         cmdline:      a list of strings specifying the command line the test
+                       would like to run
+         hash_targets: either None (don't do caching of test results), or
+                       a list of strings specifying files to include in a
+                       binary hash to check if a test has changed
+                       -- if used, all artifacts needed to run the test must
+                          be listed
+    """
+    return jobset.JobSpec(cmdline=cmdline,
                           environ=self.environ,
                           hash_targets=hash_targets
                               if self.allow_hashing else None)
@@ -74,9 +85,9 @@ class ValgrindConfig(object):
     self.maxjobs = 2 * multiprocessing.cpu_count()
     self.allow_hashing = False
 
-  def job_spec(self, binary, hash_targets):
+  def job_spec(self, cmdline, hash_targets):
     return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] +
-                          self.args + [binary],
+                          self.args + cmdline,
                           shortname='valgrind %s' % binary,
                           hash_targets=None)
 
@@ -95,7 +106,7 @@ class CLanguage(object):
       if travis and target['flaky']:
         continue
       binary = 'bins/%s/%s' % (config.build_config, target['name'])
-      out.append(config.job_spec(binary, [binary]))
+      out.append(config.job_spec([binary], [binary]))
     return out
 
   def make_targets(self):
@@ -108,7 +119,7 @@ class CLanguage(object):
 class NodeLanguage(object):
 
   def test_specs(self, config, travis):
-    return [config.job_spec('tools/run_tests/run_node.sh', None)]
+    return [config.job_spec(['tools/run_tests/run_node.sh'], None)]
 
   def make_targets(self):
     return ['static_c']
@@ -120,7 +131,7 @@ class NodeLanguage(object):
 class PhpLanguage(object):
 
   def test_specs(self, config, travis):
-    return [config.job_spec('src/php/bin/run_tests.sh', None)]
+    return [config.job_spec(['src/php/bin/run_tests.sh'], None)]
 
   def make_targets(self):
     return ['static_c']
@@ -131,8 +142,13 @@ class PhpLanguage(object):
 
 class PythonLanguage(object):
 
+  def __init__(self):
+    with open('tools/run_tests/python_tests.json') as f:
+      self._tests = json.load(f)
+
   def test_specs(self, config, travis):
-    return [config.job_spec('tools/run_tests/run_python.sh', None)]
+    return [config.job_spec(['tools/run_tests/run_python.sh', test], None)
+            for test in self._tests]
 
   def make_targets(self):
     return ['static_c']
@@ -143,7 +159,7 @@ class PythonLanguage(object):
 class RubyLanguage(object):
 
   def test_specs(self, config, travis):
-    return [config.job_spec('tools/run_tests/run_ruby.sh', None)]
+    return [config.job_spec(['tools/run_tests/run_ruby.sh'], None)]
 
   def make_targets(self):
     return ['static_c']